Open up link previews to work with all sites.

app/src/main/java/org/thoughtcrime/securesms/giph/net/GiphyLoader.java

@@ -15,7 +15,7 @@ import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.giph.model.GiphyImage;
 import org.thoughtcrime.securesms.giph.model.GiphyResponse;
 import org.thoughtcrime.securesms.net.ContentProxySelector;
-import org.thoughtcrime.securesms.net.UserAgentInterceptor;
+import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
 import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
 import org.thoughtcrime.securesms.util.AsyncLoader;
 import org.thoughtcrime.securesms.util.JsonUtils;
@@ -43,7 +43,7 @@ public abstract class GiphyLoader extends AsyncLoader<List<GiphyImage>> {
     this.searchString = searchString;
     this.client       = new OkHttpClient.Builder()
                                         .proxySelector(new ContentProxySelector())
-                                        .addInterceptor(new UserAgentInterceptor())
+                                        .addInterceptor(new StandardUserAgentInterceptor())
                                         .dns(SignalServiceNetworkAccess.DNS)
                                         .build();
   }

app/src/main/java/org/thoughtcrime/securesms/glide/ChunkedImageUrlLoader.java

@@ -11,8 +11,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory;
 import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl;
 import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor;
 import org.thoughtcrime.securesms.net.ContentProxySelector;
-import org.thoughtcrime.securesms.net.CustomDns;
+import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
-import org.thoughtcrime.securesms.net.UserAgentInterceptor;
 import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
 
 import java.io.InputStream;
@@ -45,7 +44,7 @@ public class ChunkedImageUrlLoader implements ModelLoader<ChunkedImageUrl, Input
       this.client  = new OkHttpClient.Builder()
                                      .proxySelector(new ContentProxySelector())
                                      .cache(null)
-                                     .addInterceptor(new UserAgentInterceptor())
+                                     .addInterceptor(new StandardUserAgentInterceptor())
                                      .addNetworkInterceptor(new ContentProxySafetyInterceptor())
                                      .addNetworkInterceptor(new PaddedHeadersInterceptor())
                                      .dns(SignalServiceNetworkAccess.DNS)

app/src/main/java/org/thoughtcrime/securesms/glide/OkHttpUrlLoader.java

@@ -10,7 +10,7 @@ import com.bumptech.glide.load.model.ModelLoaderFactory;
 import com.bumptech.glide.load.model.MultiModelLoaderFactory;
 
 import org.thoughtcrime.securesms.net.ContentProxySelector;
-import org.thoughtcrime.securesms.net.UserAgentInterceptor;
+import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
 import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
 
 import java.io.InputStream;
@@ -48,7 +48,7 @@ public class OkHttpUrlLoader implements ModelLoader<GlideUrl, InputStream> {
           if (internalClient == null) {
             internalClient = new OkHttpClient.Builder()
                                              .proxySelector(new ContentProxySelector())
-                                             .addInterceptor(new UserAgentInterceptor())
+                                             .addInterceptor(new StandardUserAgentInterceptor())
                                              .dns(SignalServiceNetworkAccess.DNS)
                                              .build();
           }

app/src/main/java/org/thoughtcrime/securesms/jobs/PushProcessMessageJob.java

@@ -124,7 +124,6 @@ import org.whispersystems.signalservice.api.messages.multidevice.ViewOnceOpenMes
 import org.whispersystems.signalservice.api.messages.shared.SharedContact;
 import org.whispersystems.signalservice.api.push.SignalServiceAddress;
 import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException;
-import org.whispersystems.signalservice.api.util.UuidUtil;
 
 import java.io.IOException;
 import java.security.SecureRandom;
@@ -135,7 +134,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 public final class PushProcessMessageJob extends BaseJob {
@@ -1695,7 +1693,7 @@ public final class PushProcessMessageJob extends BaseJob {
       Optional<String>     title         = Optional.fromNullable(preview.getTitle());
       boolean              hasContent    = !TextUtils.isEmpty(title.or("")) || thumbnail.isPresent();
       boolean              presentInBody = url.isPresent() && Stream.of(LinkPreviewUtil.findWhitelistedUrls(message)).map(Link::getUrl).collect(Collectors.toSet()).contains(url.get());
-      boolean              validDomain   = url.isPresent() && LinkPreviewUtil.isWhitelistedLinkUrl(url.get());
+      boolean              validDomain   = url.isPresent() && LinkPreviewUtil.isValidPreviewUrl(url.get());
 
       if (hasContent && presentInBody && validDomain) {
         LinkPreview linkPreview = new LinkPreview(url.get(), title.or(""), thumbnail);

app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java

@@ -1,38 +0,0 @@
-package org.thoughtcrime.securesms.linkpreview;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-
-public class LinkPreviewDomains {
-  public static final String STICKERS = "signal.org";
-
-  public static final Set<String> LINKS = new HashSet<>(Arrays.asList(
-      "youtube.com",
-      "www.youtube.com",
-      "m.youtube.com",
-      "youtu.be",
-      "reddit.com",
-      "www.reddit.com",
-      "m.reddit.com",
-      "imgur.com",
-      "www.imgur.com",
-      "m.imgur.com",
-      "instagram.com",
-      "www.instagram.com",
-      "m.instagram.com",
-      "pinterest.com",
-      "www.pinterest.com",
-      "pin.it"
-  ));
-
-  public static final Set<String> IMAGES = new HashSet<>(Arrays.asList(
-      "ytimg.com",
-      "cdninstagram.com",
-      "fbcdn.net",
-      "redd.it",
-      "imgur.com",
-      "pinimg.com",
-      "giphy.com"
-  ));
-}

app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java

@@ -2,31 +2,33 @@ package org.thoughtcrime.securesms.linkpreview;
 
 import android.content.Context;
 import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.net.Uri;
 import androidx.annotation.NonNull;
 import android.text.Html;
 import android.text.TextUtils;
 
 import com.bumptech.glide.load.engine.DiskCacheStrategy;
-import com.bumptech.glide.request.FutureTarget;
 
 import org.thoughtcrime.securesms.attachments.Attachment;
 import org.thoughtcrime.securesms.attachments.UriAttachment;
 import org.thoughtcrime.securesms.database.AttachmentDatabase;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
-import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl;
+import org.thoughtcrime.securesms.linkpreview.LinkPreviewUtil.OpenGraph;
 import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.mms.GlideApp;
 import org.thoughtcrime.securesms.net.CallRequestController;
 import org.thoughtcrime.securesms.net.CompositeRequestController;
 import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor;
-import org.thoughtcrime.securesms.net.ContentProxySelector;
 import org.thoughtcrime.securesms.net.RequestController;
+import org.thoughtcrime.securesms.net.UserAgentInterceptor;
 import org.thoughtcrime.securesms.providers.BlobProvider;
 import org.thoughtcrime.securesms.stickers.StickerRemoteUri;
 import org.thoughtcrime.securesms.stickers.StickerUrl;
+import org.thoughtcrime.securesms.util.ByteUnit;
 import org.thoughtcrime.securesms.util.Hex;
 import org.thoughtcrime.securesms.util.MediaUtil;
+import org.thoughtcrime.securesms.util.OkHttpUtil;
 import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
 import org.whispersystems.libsignal.InvalidMessageException;
 import org.whispersystems.libsignal.util.Pair;
@@ -34,10 +36,11 @@ import org.whispersystems.libsignal.util.guava.Optional;
 import org.whispersystems.signalservice.api.SignalServiceMessageReceiver;
 import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest;
 import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest.StickerInfo;
+import org.whispersystems.signalservice.api.util.OptionalUtil;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.util.concurrent.CancellationException;
+import java.io.InputStream;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -54,20 +57,22 @@ public class LinkPreviewRepository {
 
   private static final CacheControl NO_CACHE = new CacheControl.Builder().noCache().build();
 
+  private static final long FAILSAFE_MAX_TEXT_SIZE  = ByteUnit.MEGABYTES.toBytes(2);
+  private static final long FAILSAFE_MAX_IMAGE_SIZE = ByteUnit.MEGABYTES.toBytes(2);
+
   private final OkHttpClient client;
 
   public LinkPreviewRepository() {
     this.client = new OkHttpClient.Builder()
-                                  .proxySelector(new ContentProxySelector())
-                                  .addNetworkInterceptor(new ContentProxySafetyInterceptor())
                                   .cache(null)
+                                  .addInterceptor(new UserAgentInterceptor("WhatsApp"))
                                   .build();
   }
 
   RequestController getLinkPreview(@NonNull Context context, @NonNull String url, @NonNull Callback<Optional<LinkPreview>> callback) {
     CompositeRequestController compositeController = new CompositeRequestController();
 
-    if (!LinkPreviewUtil.isWhitelistedLinkUrl(url)) {
+    if (!LinkPreviewUtil.isValidPreviewUrl(url)) {
       Log.w(TAG, "Tried to get a link preview for a non-whitelisted domain.");
       callback.onComplete(Optional.absent());
       return compositeController;
@@ -89,7 +94,7 @@ public class LinkPreviewRepository {
           return;
         }
 
-        RequestController imageController = fetchThumbnail(context, metadata.getImageUrl().get(), attachment -> {
+        RequestController imageController = fetchThumbnail(metadata.getImageUrl().get(), attachment -> {
           if (!metadata.getTitle().isPresent() && !attachment.isPresent()) {
             callback.onComplete(Optional.absent());
           } else {
@@ -127,11 +132,12 @@ public class LinkPreviewRepository {
           return;
         }
 
-        String            body     = response.body().string();
+        String           body      = OkHttpUtil.readAsString(response.body(), FAILSAFE_MAX_TEXT_SIZE);
-        Optional<String>  title    = getProperty(body, "title");
+        OpenGraph        openGraph = LinkPreviewUtil.parseOpenGraphFields(body);
-        Optional<String>  imageUrl = getProperty(body, "image");
+        Optional<String> title     = openGraph.getTitle();
+        Optional<String> imageUrl  = openGraph.getImageUrl();
 
-        if (imageUrl.isPresent() && !LinkPreviewUtil.isWhitelistedMediaUrl(imageUrl.get())) {
+        if (imageUrl.isPresent() && !LinkPreviewUtil.isValidPreviewUrl(imageUrl.get())) {
           Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping.");
           imageUrl = Optional.absent();
         }
@@ -143,20 +149,23 @@ public class LinkPreviewRepository {
     return new CallRequestController(call);
   }
 
-  private @NonNull RequestController fetchThumbnail(@NonNull Context context, @NonNull String imageUrl, @NonNull Callback<Optional<Attachment>> callback) {
+  private @NonNull RequestController fetchThumbnail(@NonNull String imageUrl, @NonNull Callback<Optional<Attachment>> callback) {
-    FutureTarget<Bitmap> bitmapFuture = GlideApp.with(context).asBitmap()
+    Call                  call       = client.newCall(new Request.Builder().url(imageUrl).build());
-                                                              .load(new ChunkedImageUrl(imageUrl))
+    CallRequestController controller = new CallRequestController(call);
-                                                              .skipMemoryCache(true)
-                                                              .diskCacheStrategy(DiskCacheStrategy.NONE)
-                                                              .centerInside()
-                                                              .submit(1024, 1024);
-
-    RequestController controller = () -> bitmapFuture.cancel(false);
 
     SignalExecutors.UNBOUNDED.execute(() -> {
       try {
-        Bitmap                bitmap = bitmapFuture.get();
+        Response response = call.execute();
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        if (!response.isSuccessful() || response.body() == null) {
+          return;
+        }
+
+        InputStream bodyStream = response.body().byteStream();
+        controller.setStream(bodyStream);
+
+        byte[]                data   = OkHttpUtil.readAsBytes(bodyStream, FAILSAFE_MAX_IMAGE_SIZE);
+        Bitmap                bitmap = BitmapFactory.decodeByteArray(data, 0, data.length);
+        ByteArrayOutputStream baos   = new ByteArrayOutputStream();
 
         bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos);
 
@@ -181,27 +190,14 @@ public class LinkPreviewRepository {
                                                                        null));
 
         callback.onComplete(thumbnail);
-      } catch (CancellationException | ExecutionException | InterruptedException e) {
+      } catch (IOException e) {
+        Log.w(TAG, "Exception during link preview image retrieval.", e);
         controller.cancel();
         callback.onComplete(Optional.absent());
-      } finally {
-        bitmapFuture.cancel(false);
       }
     });
 
-    return () -> bitmapFuture.cancel(true);
+    return controller;
-  }
-
-  private @NonNull Optional<String> getProperty(@NonNull String searchText, @NonNull String property) {
-    Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
-    Matcher matcher = pattern.matcher(searchText);
-
-    if (matcher.find()) {
-      String text = Html.fromHtml(matcher.group(1)).toString();
-      return TextUtils.isEmpty(text) ? Optional.absent() : Optional.of(text);
-    }
-
-    return Optional.absent();
   }
 
   private RequestController fetchStickerPackLinkPreview(@NonNull Context context,

app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java

@@ -2,6 +2,9 @@ package org.thoughtcrime.securesms.linkpreview;
 
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
+import androidx.annotation.VisibleForTesting;
+
+import android.text.Html;
 import android.text.SpannableString;
 import android.text.TextUtils;
 import android.text.style.URLSpan;
@@ -10,9 +13,14 @@ import android.text.util.Linkify;
 import com.annimon.stream.Stream;
 
 import org.thoughtcrime.securesms.stickers.StickerUrl;
+import org.thoughtcrime.securesms.util.Util;
+import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.util.OptionalUtil;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -20,9 +28,14 @@ import okhttp3.HttpUrl;
 
 public final class LinkPreviewUtil {
 
-  private static final Pattern DOMAIN_PATTERN        = Pattern.compile("^(https?://)?([^/]+).*$");
+  private static final Pattern DOMAIN_PATTERN             = Pattern.compile("^(https?://)?([^/]+).*$");
-  private static final Pattern ALL_ASCII_PATTERN     = Pattern.compile("^[\\x00-\\x7F]*$");
+  private static final Pattern ALL_ASCII_PATTERN          = Pattern.compile("^[\\x00-\\x7F]*$");
-  private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$");
+  private static final Pattern ALL_NON_ASCII_PATTERN      = Pattern.compile("^[^\\x00-\\x7F]*$");
+  private static final Pattern OPEN_GRAPH_TAG_PATTERN     = Pattern.compile("<\\s*meta[^>]*property\\s*=\\s*\"\\s*og:([^\"]+)\"[^>]*/?\\s*>");
+  private static final Pattern OPEN_GRAPH_CONTENT_PATTERN = Pattern.compile("content\\s*=\\s*\"([^\"]*)\"");
+  private static final Pattern TITLE_PATTERN              = Pattern.compile("<\\s*title[^>]*>(.*)<\\s*/title[^>]*>");
+  private static final Pattern FAVICON_PATTERN            = Pattern.compile("<\\s*link[^>]*rel\\s*=\\s*\".*icon.*\"[^>]*>");
+  private static final Pattern FAVICON_HREF_PATTERN       = Pattern.compile("href\\s*=\\s*\"([^\"]*)\"");
 
   /**
    * @return All whitelisted URLs in the source text.
@@ -37,14 +50,14 @@ public final class LinkPreviewUtil {
 
     return Stream.of(spannable.getSpans(0, spannable.length(), URLSpan.class))
                  .map(span -> new Link(span.getURL(), spannable.getSpanStart(span)))
-                 .filter(link -> isWhitelistedLinkUrl(link.getUrl()))
+                 .filter(link -> isValidPreviewUrl(link.getUrl()))
                  .toList();
   }
 
   /**
    * @return True if the host is present in the link whitelist.
    */
-  public static boolean isWhitelistedLinkUrl(@Nullable String linkUrl) {
+  public static boolean isValidPreviewUrl(@Nullable String linkUrl) {
     if (linkUrl == null)                      return false;
     if (StickerUrl.isValidShareLink(linkUrl)) return true;
 
@@ -52,24 +65,9 @@ public final class LinkPreviewUtil {
     return url != null                                   &&
            !TextUtils.isEmpty(url.scheme())              &&
            "https".equals(url.scheme())                  &&
-           LinkPreviewDomains.LINKS.contains(url.host()) &&
            isLegalUrl(linkUrl);
   }
 
-  /**
-   * @return True if the top-level domain is present in the media whitelist.
-   */
-  public static boolean isWhitelistedMediaUrl(@Nullable String mediaUrl) {
-    if (mediaUrl == null) return false;
-
-    HttpUrl url = HttpUrl.parse(mediaUrl);
-    return url != null                                                &&
-           !TextUtils.isEmpty(url.scheme())                           &&
-           "https".equals(url.scheme())                               &&
-           LinkPreviewDomains.IMAGES.contains(url.topPrivateDomain()) &&
-           isLegalUrl(mediaUrl);
-  }
-
   public static boolean isLegalUrl(@NonNull String url) {
     Matcher matcher = DOMAIN_PATTERN.matcher(url);
 
@@ -83,4 +81,78 @@ public final class LinkPreviewUtil {
       return false;
     }
   }
+
+  public static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html) {
+    return parseOpenGraphFields(html, text -> Html.fromHtml(text).toString());
+  }
+
+  @VisibleForTesting
+  static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html, @NonNull HtmlDecoder htmlDecoder) {
+    if (html == null) {
+      return new OpenGraph(Collections.emptyMap(), null, null);
+    }
+
+    Map<String, String> openGraphTags    = new HashMap<>();
+    Matcher             openGraphMatcher = OPEN_GRAPH_TAG_PATTERN.matcher(html);
+
+    while (openGraphMatcher.find()) {
+      String tag      = openGraphMatcher.group();
+      String property = openGraphMatcher.groupCount() > 0 ? openGraphMatcher.group(1) : null;
+
+      if (property != null) {
+        Matcher contentMatcher = OPEN_GRAPH_CONTENT_PATTERN.matcher(tag);
+        if (contentMatcher.find() && contentMatcher.groupCount() > 0) {
+          String content = htmlDecoder.fromEncoded(contentMatcher.group(1));
+          openGraphTags.put(property, content);
+        }
+      }
+    }
+
+    String htmlTitle  = "";
+    String faviconUrl = "";
+
+    Matcher titleMatcher = TITLE_PATTERN.matcher(html);
+    if (titleMatcher.find() && titleMatcher.groupCount() > 0) {
+      htmlTitle = titleMatcher.group(1);
+    }
+
+    Matcher faviconMatcher = FAVICON_PATTERN.matcher(html);
+    if (faviconMatcher.find()) {
+      Matcher faviconHrefMatcher = FAVICON_HREF_PATTERN.matcher(faviconMatcher.group());
+      if (faviconHrefMatcher.find() && faviconHrefMatcher.groupCount() > 0) {
+        faviconUrl = faviconHrefMatcher.group(1);
+      }
+    }
+
+    return new OpenGraph(openGraphTags, htmlTitle, faviconUrl);
+  }
+
+  public static final class OpenGraph {
+
+    private final Map<String, String> values;
+
+    private final @Nullable String htmlTitle;
+    private final @Nullable String faviconUrl;
+
+    private static final String KEY_TITLE     = "title";
+    private static final String KEY_IMAGE_URL = "image";
+
+    public OpenGraph(@NonNull Map<String, String> values, @Nullable String htmlTitle, @Nullable String faviconUrl) {
+      this.values     = values;
+      this.htmlTitle  = htmlTitle;
+      this.faviconUrl = faviconUrl;
+    }
+
+    public @NonNull Optional<String> getTitle() {
+      return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_TITLE), htmlTitle));
+    }
+
+    public @NonNull Optional<String> getImageUrl() {
+      return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_IMAGE_URL), faviconUrl));
+    }
+  }
+
+  public interface HtmlDecoder {
+    @NonNull String fromEncoded(@NonNull String html);
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java

@@ -13,7 +13,7 @@ import org.json.JSONObject;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.logsubmit.util.Scrubber;
-import org.thoughtcrime.securesms.net.UserAgentInterceptor;
+import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
 import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
 import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
 import org.whispersystems.libsignal.util.guava.Optional;
@@ -91,7 +91,7 @@ public class SubmitDebugLogRepository {
     }
 
     try {
-      OkHttpClient client   = new OkHttpClient.Builder().addInterceptor(new UserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build();
+      OkHttpClient client   = new OkHttpClient.Builder().addInterceptor(new StandardUserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build();
       Response     response = client.newCall(new Request.Builder().url(API_ENDPOINT).get().build()).execute();
       ResponseBody body     = response.body();
 

app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java

@@ -1,5 +1,7 @@
 package org.thoughtcrime.securesms.net;
 
+import android.text.TextUtils;
+
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 
@@ -52,7 +54,13 @@ public class ContentProxySafetyInterceptor implements Interceptor {
     return isWhitelisted(url.toString());
   }
 
-  private static boolean isWhitelisted(@Nullable String url) {
+  private static boolean isWhitelisted(@Nullable String rawUrl) {
-    return LinkPreviewUtil.isWhitelistedLinkUrl(url) || LinkPreviewUtil.isWhitelistedMediaUrl(url);
+    if (rawUrl == null) return false;
+
+    HttpUrl url = HttpUrl.parse(rawUrl);
+
+    return url != null                      &&
+           "https".equals(url.scheme())     &&
+           ContentProxySelector.WHITELISTED_DOMAINS.contains(url.topPrivateDomain());
   }
 }

app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySelector.java

@@ -1,13 +1,9 @@
 package org.thoughtcrime.securesms.net;
 
 
-import android.os.AsyncTask;
-
-import org.thoughtcrime.securesms.linkpreview.LinkPreviewDomains;
 import org.thoughtcrime.securesms.logging.Log;
 
 import org.thoughtcrime.securesms.BuildConfig;
-import org.thoughtcrime.securesms.util.Util;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -25,10 +21,9 @@ public class ContentProxySelector extends ProxySelector {
 
   private static final String TAG = ContentProxySelector.class.getSimpleName();
 
-  private static final Set<String> WHITELISTED_DOMAINS = new HashSet<>();
+  public static final Set<String> WHITELISTED_DOMAINS = new HashSet<>();
   static {
-    WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.LINKS);
+    WHITELISTED_DOMAINS.add("giphy.com");
-    WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.IMAGES);
   }
 
   private final List<Proxy> CONTENT = new ArrayList<Proxy>(1) {{

app/src/main/java/org/thoughtcrime/securesms/net/StandardUserAgentInterceptor.java

@@ -0,0 +1,15 @@
+package org.thoughtcrime.securesms.net;
+
+import android.os.Build;
+
+import org.thoughtcrime.securesms.BuildConfig;
+
+/**
+ * The user agent that should be used by default -- includes app name, version, etc.
+ */
+public class StandardUserAgentInterceptor extends UserAgentInterceptor {
+
+  public StandardUserAgentInterceptor() {
+    super("Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")");
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/net/UserAgentInterceptor.java

@@ -1,11 +1,7 @@
 package org.thoughtcrime.securesms.net;
 
-import android.os.Build;
-
 import androidx.annotation.NonNull;
 
-import org.thoughtcrime.securesms.BuildConfig;
-
 import java.io.IOException;
 
 import okhttp3.Interceptor;
@@ -13,12 +9,16 @@ import okhttp3.Response;
 
 public class UserAgentInterceptor implements Interceptor {
 
-  private static final String USER_AGENT = "Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")";
+  private final String userAgent;
+
+  public UserAgentInterceptor(@NonNull String userAgent) {
+    this.userAgent = userAgent;
+  }
 
   @Override
   public Response intercept(@NonNull Chain chain) throws IOException {
     return chain.proceed(chain.request().newBuilder()
-                                        .header("User-Agent", USER_AGENT)
+                                        .header("User-Agent", userAgent)
                                         .build());
   }
 }

app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java

@@ -8,7 +8,7 @@ import androidx.annotation.Nullable;
 import org.thoughtcrime.securesms.BuildConfig;
 import org.thoughtcrime.securesms.net.CustomDns;
 import org.thoughtcrime.securesms.net.SequentialDns;
-import org.thoughtcrime.securesms.net.UserAgentInterceptor;
+import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
 import org.thoughtcrime.securesms.util.Base64;
 import org.thoughtcrime.securesms.util.TextSecurePreferences;
 import org.whispersystems.libsignal.util.guava.Optional;
@@ -161,7 +161,7 @@ public class SignalServiceNetworkAccess {
     final SignalStorageUrl omanGoogleStorage     = new SignalStorageUrl("https://www.google.com.om/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC);
     final SignalStorageUrl qatarGoogleStorage    = new SignalStorageUrl("https://www.google.com.qa/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC);
 
-    final List<Interceptor> interceptors = Collections.singletonList(new UserAgentInterceptor());
+    final List<Interceptor> interceptors = Collections.singletonList(new StandardUserAgentInterceptor());
     final Optional<Dns>     dns          = Optional.of(DNS);
 
     final byte[] zkGroupServerPublicParams;

app/src/main/java/org/thoughtcrime/securesms/util/OkHttpUtil.java

@@ -0,0 +1,49 @@
+package org.thoughtcrime.securesms.util;
+
+import androidx.annotation.NonNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Objects;
+
+import okhttp3.MediaType;
+import okhttp3.ResponseBody;
+
+import static okhttp3.internal.Util.UTF_8;
+
+public final class OkHttpUtil {
+
+  private OkHttpUtil() {}
+
+  public static byte[] readAsBytes(@NonNull InputStream bodyStream, long sizeLimit) throws IOException {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+
+    byte[] buffer      = new byte[(int) ByteUnit.KILOBYTES.toBytes(32)];
+    int    readLength  = 0;
+    int    totalLength = 0;
+
+    while ((readLength = bodyStream.read(buffer)) >= 0) {
+      if (totalLength + readLength > sizeLimit) {
+        throw new IOException("Exceeded maximum size during read!");
+      }
+
+      outputStream.write(buffer, 0, readLength);
+      totalLength += readLength;
+    }
+
+    return outputStream.toByteArray();
+  }
+  public static String readAsString(@NonNull ResponseBody body, long sizeLimit) throws IOException {
+    if (body.contentLength() > sizeLimit) {
+      throw new IOException("Content-Length exceeded maximum size!");
+    }
+
+    byte[]    data        = readAsBytes(body.byteStream(), sizeLimit);
+    MediaType contentType = body.contentType();
+    Charset   charset     = contentType != null ? contentType.charset(UTF_8) : UTF_8;
+
+    return new String(data, Objects.requireNonNull(charset));
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/util/Util.java

@@ -159,6 +159,10 @@ public class Util {
     return collection == null || collection.isEmpty();
   }
 
+  public static boolean isEmpty(@Nullable String value) {
+    return value == null || value.length() == 0;
+  }
+
   public static boolean hasItems(@Nullable Collection<?> collection) {
     return collection != null && !collection.isEmpty();
   }
@@ -169,7 +173,7 @@ public class Util {
 
   public static String getFirstNonEmpty(String... values) {
     for (String value : values) {
-      if (!TextUtils.isEmpty(value)) {
+      if (!Util.isEmpty(value)) {
         return value;
       }
     }

app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest.java

@@ -1,73 +0,0 @@
-package org.thoughtcrime.securesms.linkpreview;
-
-import org.junit.Test;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
-
-public class LinkPreviewUtilTest {
-
-  @Test
-  public void isLegal_allAscii_noProtocol() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("google.com"));
-  }
-
-  @Test
-  public void isLegal_allAscii_noProtocol_subdomain() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("foo.google.com"));
-  }
-
-  @Test
-  public void isLegal_allAscii_subdomain() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com"));
-  }
-
-  @Test
-  public void isLegal_allAscii_subdomain_path() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com/some/path.html"));
-  }
-
-  @Test
-  public void isLegal_cyrillicHostAsciiTld() {
-    assertFalse(LinkPreviewUtil.isLegalUrl("http://кц.com"));
-  }
-
-  @Test
-  public void isLegal_cyrillicHostAsciiTld_noProtocol() {
-    assertFalse(LinkPreviewUtil.isLegalUrl("кц.com"));
-  }
-
-  @Test
-  public void isLegal_mixedHost_noProtocol() {
-    assertFalse(LinkPreviewUtil.isLegalUrl("http://asĸ.com"));
-  }
-
-  @Test
-  public void isLegal_cyrillicHostAndTld_noProtocol() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф"));
-  }
-
-  @Test
-  public void isLegal_cyrillicHostAndTld_asciiPath_noProtocol() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф/some/path"));
-  }
-
-  @Test
-  public void isLegal_cyrillicHostAndTld_asciiPath() {
-    assertTrue(LinkPreviewUtil.isLegalUrl("https://кц.рф/some/path"));
-  }
-
-  @Test
-  public void isLegal_asciiSubdomain_cyrillicHostAndTld() {
-    assertFalse(LinkPreviewUtil.isLegalUrl("http://foo.кц.рф"));
-  }
-
-  @Test
-  public void isLegal_emptyUrl() {
-    assertFalse(LinkPreviewUtil.isLegalUrl(""));
-  }
-}

app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_isLegal.java

@@ -0,0 +1,44 @@
+package org.thoughtcrime.securesms.linkpreview;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static junit.framework.TestCase.assertEquals;
+
+@RunWith(Parameterized.class)
+public class LinkPreviewUtilTest_isLegal {
+
+  private final String  input;
+  private final boolean output;
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][]{
+        { "google.com",                            true  },
+        { "foo.google.com",                        true  },
+        { "https://foo.google.com",                true  },
+        { "https://foo.google.com/some/path.html", true  },
+        { "кц.рф",                                 true  },
+        { "https://кц.рф/some/path",               true  },
+        { "http://кц.com",                         false },
+        { "кц.com",                                false },
+        { "http://asĸ.com",                        false },
+        { "http://foo.кц.рф",                      false },
+        { "",                                      false }
+    });
+  }
+
+  public LinkPreviewUtilTest_isLegal(String input, boolean output) {
+    this.input  = input;
+    this.output = output;
+  }
+
+  @Test
+  public void isLegal() {
+    assertEquals(output, LinkPreviewUtil.isLegalUrl(input));
+  }
+}

app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_parseOpenGraphFields.java

@@ -0,0 +1,108 @@
+package org.thoughtcrime.securesms.linkpreview;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.whispersystems.libsignal.util.guava.Optional;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static junit.framework.TestCase.assertEquals;
+
+@RunWith(Parameterized.class)
+public class LinkPreviewUtilTest_parseOpenGraphFields {
+
+  private final String html;
+  private final String title;
+  private final String imageUrl;
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][]{
+        // Normal
+        { "<meta content=\"Daily Bugle\" property=\"og:title\">\n" +
+          "<meta content=\"https://images.com/my-image.jpg\" property=\"og:image\">",
+            "Daily Bugle",
+            "https://images.com/my-image.jpg"},
+
+        // Swap property orders
+        { "<meta property=\"og:title\" content=\"Daily Bugle\">\n" +
+          "<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">",
+            "Daily Bugle",
+            "https://images.com/my-image.jpg"},
+
+        // Funny spacing
+        { "< meta   property   = \"og:title\" content =  \"Daily Bugle\"  >\n\n" +
+          "<  meta  property = \"og:image\" content   =\"https://images.com/my-image.jpg\" >",
+            "Daily Bugle",
+            "https://images.com/my-image.jpg"},
+
+        // Garbage in various places
+        { "<meta property=\"og:title\" content=\"Daily Bugle\">\n" +
+          "asdfjkl\n" +
+          "<body>idk</body>\n" +
+          "<script type=\"text/javascript\">var a = </script>\n" +
+          "<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">",
+            "Daily Bugle",
+            "https://images.com/my-image.jpg"},
+
+        // Missing image
+        { "<meta content=\"Daily Bugle\" property=\"og:title\">",
+            "Daily Bugle",
+            null},
+
+        // Missing title
+        { "<meta content=\"https://images.com/my-image.jpg\" property=\"og:image\">",
+            null,
+            "https://images.com/my-image.jpg"},
+
+        // Has everything
+        { "<meta property=\"og:title\" content =  \"Daily Bugle\">\n" +
+          "<title>Daily Bugle HTML</title>\n" +
+          "<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">\n" +
+          "<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
+            "Daily Bugle",
+            "https://images.com/my-image.jpg"},
+
+        // Fallback to HTML title
+        { "<title>Daily Bugle HTML</title>\n" +
+          "<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">\n" +
+          "<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
+            "Daily Bugle HTML",
+            "https://images.com/my-image.jpg"},
+
+        // Fallback to favicon
+        { "<meta property=\"og:title\" content =  \"Daily Bugle\">\n" +
+          "<title>Daily Bugle HTML</title>\n" +
+          "<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
+            "Daily Bugle",
+            "https://images.com/favicon.png"},
+
+        // Fallback to HTML title and favicon
+        { "<title>Daily Bugle HTML</title>\n" +
+          "<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
+            "Daily Bugle HTML",
+            "https://images.com/favicon.png"},
+
+        // Different favicon formatting
+        { "<title>Daily Bugle HTML</title>\n" +
+          "<link rel=\"shortcut icon\" href=\"https://images.com/favicon.png\" />",
+            "Daily Bugle HTML",
+            "https://images.com/favicon.png"},
+    });
+  }
+
+  public LinkPreviewUtilTest_parseOpenGraphFields(String html, String title, String imageUrl) {
+    this.html     = html;
+    this.title    = title;
+    this.imageUrl = imageUrl;
+  }
+
+  @Test
+  public void parseOpenGraphFields() {
+    LinkPreviewUtil.OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(html, html -> html);
+    assertEquals(Optional.fromNullable(title), openGraph.getTitle());
+    assertEquals(Optional.fromNullable(imageUrl), openGraph.getImageUrl());
+  }
+}