diff --git a/app/src/main/java/org/thoughtcrime/securesms/giph/net/GiphyLoader.java b/app/src/main/java/org/thoughtcrime/securesms/giph/net/GiphyLoader.java index ea6a2e3056..55309b9459 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/giph/net/GiphyLoader.java +++ b/app/src/main/java/org/thoughtcrime/securesms/giph/net/GiphyLoader.java @@ -15,7 +15,7 @@ import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.giph.model.GiphyImage; import org.thoughtcrime.securesms.giph.model.GiphyResponse; import org.thoughtcrime.securesms.net.ContentProxySelector; -import org.thoughtcrime.securesms.net.UserAgentInterceptor; +import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor; import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess; import org.thoughtcrime.securesms.util.AsyncLoader; import org.thoughtcrime.securesms.util.JsonUtils; @@ -43,7 +43,7 @@ public abstract class GiphyLoader extends AsyncLoader> { this.searchString = searchString; this.client = new OkHttpClient.Builder() .proxySelector(new ContentProxySelector()) - .addInterceptor(new UserAgentInterceptor()) + .addInterceptor(new StandardUserAgentInterceptor()) .dns(SignalServiceNetworkAccess.DNS) .build(); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/glide/ChunkedImageUrlLoader.java b/app/src/main/java/org/thoughtcrime/securesms/glide/ChunkedImageUrlLoader.java index deb9c0d6cf..7b9455e913 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/glide/ChunkedImageUrlLoader.java +++ b/app/src/main/java/org/thoughtcrime/securesms/glide/ChunkedImageUrlLoader.java @@ -11,8 +11,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory; import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl; import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor; import org.thoughtcrime.securesms.net.ContentProxySelector; -import org.thoughtcrime.securesms.net.CustomDns; -import org.thoughtcrime.securesms.net.UserAgentInterceptor; +import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor; import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess; import java.io.InputStream; @@ -45,7 +44,7 @@ public class ChunkedImageUrlLoader implements ModelLoader { if (internalClient == null) { internalClient = new OkHttpClient.Builder() .proxySelector(new ContentProxySelector()) - .addInterceptor(new UserAgentInterceptor()) + .addInterceptor(new StandardUserAgentInterceptor()) .dns(SignalServiceNetworkAccess.DNS) .build(); } diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/PushProcessMessageJob.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/PushProcessMessageJob.java index f415a49552..fa9d8e21c6 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/PushProcessMessageJob.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/PushProcessMessageJob.java @@ -124,7 +124,6 @@ import org.whispersystems.signalservice.api.messages.multidevice.ViewOnceOpenMes import org.whispersystems.signalservice.api.messages.shared.SharedContact; import org.whispersystems.signalservice.api.push.SignalServiceAddress; import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException; -import org.whispersystems.signalservice.api.util.UuidUtil; import java.io.IOException; import java.security.SecureRandom; @@ -135,7 +134,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.concurrent.TimeUnit; public final class PushProcessMessageJob extends BaseJob { @@ -1695,7 +1693,7 @@ public final class PushProcessMessageJob extends BaseJob { Optional title = Optional.fromNullable(preview.getTitle()); boolean hasContent = !TextUtils.isEmpty(title.or("")) || thumbnail.isPresent(); boolean presentInBody = url.isPresent() && Stream.of(LinkPreviewUtil.findWhitelistedUrls(message)).map(Link::getUrl).collect(Collectors.toSet()).contains(url.get()); - boolean validDomain = url.isPresent() && LinkPreviewUtil.isWhitelistedLinkUrl(url.get()); + boolean validDomain = url.isPresent() && LinkPreviewUtil.isValidPreviewUrl(url.get()); if (hasContent && presentInBody && validDomain) { LinkPreview linkPreview = new LinkPreview(url.get(), title.or(""), thumbnail); diff --git a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java b/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java deleted file mode 100644 index 6e99f84ac9..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewDomains.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.thoughtcrime.securesms.linkpreview; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class LinkPreviewDomains { - public static final String STICKERS = "signal.org"; - - public static final Set LINKS = new HashSet<>(Arrays.asList( - "youtube.com", - "www.youtube.com", - "m.youtube.com", - "youtu.be", - "reddit.com", - "www.reddit.com", - "m.reddit.com", - "imgur.com", - "www.imgur.com", - "m.imgur.com", - "instagram.com", - "www.instagram.com", - "m.instagram.com", - "pinterest.com", - "www.pinterest.com", - "pin.it" - )); - - public static final Set IMAGES = new HashSet<>(Arrays.asList( - "ytimg.com", - "cdninstagram.com", - "fbcdn.net", - "redd.it", - "imgur.com", - "pinimg.com", - "giphy.com" - )); -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java b/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java index b306864b23..09bd4319bf 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java +++ b/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewRepository.java @@ -2,31 +2,33 @@ package org.thoughtcrime.securesms.linkpreview; import android.content.Context; import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.net.Uri; import androidx.annotation.NonNull; import android.text.Html; import android.text.TextUtils; import com.bumptech.glide.load.engine.DiskCacheStrategy; -import com.bumptech.glide.request.FutureTarget; import org.thoughtcrime.securesms.attachments.Attachment; import org.thoughtcrime.securesms.attachments.UriAttachment; import org.thoughtcrime.securesms.database.AttachmentDatabase; import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; -import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl; +import org.thoughtcrime.securesms.linkpreview.LinkPreviewUtil.OpenGraph; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.mms.GlideApp; import org.thoughtcrime.securesms.net.CallRequestController; import org.thoughtcrime.securesms.net.CompositeRequestController; import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor; -import org.thoughtcrime.securesms.net.ContentProxySelector; import org.thoughtcrime.securesms.net.RequestController; +import org.thoughtcrime.securesms.net.UserAgentInterceptor; import org.thoughtcrime.securesms.providers.BlobProvider; import org.thoughtcrime.securesms.stickers.StickerRemoteUri; import org.thoughtcrime.securesms.stickers.StickerUrl; +import org.thoughtcrime.securesms.util.ByteUnit; import org.thoughtcrime.securesms.util.Hex; import org.thoughtcrime.securesms.util.MediaUtil; +import org.thoughtcrime.securesms.util.OkHttpUtil; import org.thoughtcrime.securesms.util.concurrent.SignalExecutors; import org.whispersystems.libsignal.InvalidMessageException; import org.whispersystems.libsignal.util.Pair; @@ -34,10 +36,11 @@ import org.whispersystems.libsignal.util.guava.Optional; import org.whispersystems.signalservice.api.SignalServiceMessageReceiver; import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest; import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest.StickerInfo; +import org.whispersystems.signalservice.api.util.OptionalUtil; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.concurrent.CancellationException; +import java.io.InputStream; import java.util.concurrent.ExecutionException; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -54,20 +57,22 @@ public class LinkPreviewRepository { private static final CacheControl NO_CACHE = new CacheControl.Builder().noCache().build(); + private static final long FAILSAFE_MAX_TEXT_SIZE = ByteUnit.MEGABYTES.toBytes(2); + private static final long FAILSAFE_MAX_IMAGE_SIZE = ByteUnit.MEGABYTES.toBytes(2); + private final OkHttpClient client; public LinkPreviewRepository() { this.client = new OkHttpClient.Builder() - .proxySelector(new ContentProxySelector()) - .addNetworkInterceptor(new ContentProxySafetyInterceptor()) .cache(null) + .addInterceptor(new UserAgentInterceptor("WhatsApp")) .build(); } RequestController getLinkPreview(@NonNull Context context, @NonNull String url, @NonNull Callback> callback) { CompositeRequestController compositeController = new CompositeRequestController(); - if (!LinkPreviewUtil.isWhitelistedLinkUrl(url)) { + if (!LinkPreviewUtil.isValidPreviewUrl(url)) { Log.w(TAG, "Tried to get a link preview for a non-whitelisted domain."); callback.onComplete(Optional.absent()); return compositeController; @@ -89,7 +94,7 @@ public class LinkPreviewRepository { return; } - RequestController imageController = fetchThumbnail(context, metadata.getImageUrl().get(), attachment -> { + RequestController imageController = fetchThumbnail(metadata.getImageUrl().get(), attachment -> { if (!metadata.getTitle().isPresent() && !attachment.isPresent()) { callback.onComplete(Optional.absent()); } else { @@ -127,11 +132,12 @@ public class LinkPreviewRepository { return; } - String body = response.body().string(); - Optional title = getProperty(body, "title"); - Optional imageUrl = getProperty(body, "image"); + String body = OkHttpUtil.readAsString(response.body(), FAILSAFE_MAX_TEXT_SIZE); + OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(body); + Optional title = openGraph.getTitle(); + Optional imageUrl = openGraph.getImageUrl(); - if (imageUrl.isPresent() && !LinkPreviewUtil.isWhitelistedMediaUrl(imageUrl.get())) { + if (imageUrl.isPresent() && !LinkPreviewUtil.isValidPreviewUrl(imageUrl.get())) { Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping."); imageUrl = Optional.absent(); } @@ -143,20 +149,23 @@ public class LinkPreviewRepository { return new CallRequestController(call); } - private @NonNull RequestController fetchThumbnail(@NonNull Context context, @NonNull String imageUrl, @NonNull Callback> callback) { - FutureTarget bitmapFuture = GlideApp.with(context).asBitmap() - .load(new ChunkedImageUrl(imageUrl)) - .skipMemoryCache(true) - .diskCacheStrategy(DiskCacheStrategy.NONE) - .centerInside() - .submit(1024, 1024); - - RequestController controller = () -> bitmapFuture.cancel(false); + private @NonNull RequestController fetchThumbnail(@NonNull String imageUrl, @NonNull Callback> callback) { + Call call = client.newCall(new Request.Builder().url(imageUrl).build()); + CallRequestController controller = new CallRequestController(call); SignalExecutors.UNBOUNDED.execute(() -> { try { - Bitmap bitmap = bitmapFuture.get(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Response response = call.execute(); + if (!response.isSuccessful() || response.body() == null) { + return; + } + + InputStream bodyStream = response.body().byteStream(); + controller.setStream(bodyStream); + + byte[] data = OkHttpUtil.readAsBytes(bodyStream, FAILSAFE_MAX_IMAGE_SIZE); + Bitmap bitmap = BitmapFactory.decodeByteArray(data, 0, data.length); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos); @@ -181,27 +190,14 @@ public class LinkPreviewRepository { null)); callback.onComplete(thumbnail); - } catch (CancellationException | ExecutionException | InterruptedException e) { + } catch (IOException e) { + Log.w(TAG, "Exception during link preview image retrieval.", e); controller.cancel(); callback.onComplete(Optional.absent()); - } finally { - bitmapFuture.cancel(false); } }); - return () -> bitmapFuture.cancel(true); - } - - private @NonNull Optional getProperty(@NonNull String searchText, @NonNull String property) { - Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - Matcher matcher = pattern.matcher(searchText); - - if (matcher.find()) { - String text = Html.fromHtml(matcher.group(1)).toString(); - return TextUtils.isEmpty(text) ? Optional.absent() : Optional.of(text); - } - - return Optional.absent(); + return controller; } private RequestController fetchStickerPackLinkPreview(@NonNull Context context, diff --git a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java b/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java index 9c4bb64640..ff2cef061f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java +++ b/app/src/main/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtil.java @@ -2,6 +2,9 @@ package org.thoughtcrime.securesms.linkpreview; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; + +import android.text.Html; import android.text.SpannableString; import android.text.TextUtils; import android.text.style.URLSpan; @@ -10,9 +13,14 @@ import android.text.util.Linkify; import com.annimon.stream.Stream; import org.thoughtcrime.securesms.stickers.StickerUrl; +import org.thoughtcrime.securesms.util.Util; +import org.whispersystems.libsignal.util.guava.Optional; +import org.whispersystems.signalservice.api.util.OptionalUtil; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -20,9 +28,14 @@ import okhttp3.HttpUrl; public final class LinkPreviewUtil { - private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$"); - private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$"); - private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$"); + private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$"); + private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$"); + private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$"); + private static final Pattern OPEN_GRAPH_TAG_PATTERN = Pattern.compile("<\\s*meta[^>]*property\\s*=\\s*\"\\s*og:([^\"]+)\"[^>]*/?\\s*>"); + private static final Pattern OPEN_GRAPH_CONTENT_PATTERN = Pattern.compile("content\\s*=\\s*\"([^\"]*)\""); + private static final Pattern TITLE_PATTERN = Pattern.compile("<\\s*title[^>]*>(.*)<\\s*/title[^>]*>"); + private static final Pattern FAVICON_PATTERN = Pattern.compile("<\\s*link[^>]*rel\\s*=\\s*\".*icon.*\"[^>]*>"); + private static final Pattern FAVICON_HREF_PATTERN = Pattern.compile("href\\s*=\\s*\"([^\"]*)\""); /** * @return All whitelisted URLs in the source text. @@ -37,14 +50,14 @@ public final class LinkPreviewUtil { return Stream.of(spannable.getSpans(0, spannable.length(), URLSpan.class)) .map(span -> new Link(span.getURL(), spannable.getSpanStart(span))) - .filter(link -> isWhitelistedLinkUrl(link.getUrl())) + .filter(link -> isValidPreviewUrl(link.getUrl())) .toList(); } /** * @return True if the host is present in the link whitelist. */ - public static boolean isWhitelistedLinkUrl(@Nullable String linkUrl) { + public static boolean isValidPreviewUrl(@Nullable String linkUrl) { if (linkUrl == null) return false; if (StickerUrl.isValidShareLink(linkUrl)) return true; @@ -52,24 +65,9 @@ public final class LinkPreviewUtil { return url != null && !TextUtils.isEmpty(url.scheme()) && "https".equals(url.scheme()) && - LinkPreviewDomains.LINKS.contains(url.host()) && isLegalUrl(linkUrl); } - /** - * @return True if the top-level domain is present in the media whitelist. - */ - public static boolean isWhitelistedMediaUrl(@Nullable String mediaUrl) { - if (mediaUrl == null) return false; - - HttpUrl url = HttpUrl.parse(mediaUrl); - return url != null && - !TextUtils.isEmpty(url.scheme()) && - "https".equals(url.scheme()) && - LinkPreviewDomains.IMAGES.contains(url.topPrivateDomain()) && - isLegalUrl(mediaUrl); - } - public static boolean isLegalUrl(@NonNull String url) { Matcher matcher = DOMAIN_PATTERN.matcher(url); @@ -83,4 +81,78 @@ public final class LinkPreviewUtil { return false; } } + + public static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html) { + return parseOpenGraphFields(html, text -> Html.fromHtml(text).toString()); + } + + @VisibleForTesting + static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html, @NonNull HtmlDecoder htmlDecoder) { + if (html == null) { + return new OpenGraph(Collections.emptyMap(), null, null); + } + + Map openGraphTags = new HashMap<>(); + Matcher openGraphMatcher = OPEN_GRAPH_TAG_PATTERN.matcher(html); + + while (openGraphMatcher.find()) { + String tag = openGraphMatcher.group(); + String property = openGraphMatcher.groupCount() > 0 ? openGraphMatcher.group(1) : null; + + if (property != null) { + Matcher contentMatcher = OPEN_GRAPH_CONTENT_PATTERN.matcher(tag); + if (contentMatcher.find() && contentMatcher.groupCount() > 0) { + String content = htmlDecoder.fromEncoded(contentMatcher.group(1)); + openGraphTags.put(property, content); + } + } + } + + String htmlTitle = ""; + String faviconUrl = ""; + + Matcher titleMatcher = TITLE_PATTERN.matcher(html); + if (titleMatcher.find() && titleMatcher.groupCount() > 0) { + htmlTitle = titleMatcher.group(1); + } + + Matcher faviconMatcher = FAVICON_PATTERN.matcher(html); + if (faviconMatcher.find()) { + Matcher faviconHrefMatcher = FAVICON_HREF_PATTERN.matcher(faviconMatcher.group()); + if (faviconHrefMatcher.find() && faviconHrefMatcher.groupCount() > 0) { + faviconUrl = faviconHrefMatcher.group(1); + } + } + + return new OpenGraph(openGraphTags, htmlTitle, faviconUrl); + } + + public static final class OpenGraph { + + private final Map values; + + private final @Nullable String htmlTitle; + private final @Nullable String faviconUrl; + + private static final String KEY_TITLE = "title"; + private static final String KEY_IMAGE_URL = "image"; + + public OpenGraph(@NonNull Map values, @Nullable String htmlTitle, @Nullable String faviconUrl) { + this.values = values; + this.htmlTitle = htmlTitle; + this.faviconUrl = faviconUrl; + } + + public @NonNull Optional getTitle() { + return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_TITLE), htmlTitle)); + } + + public @NonNull Optional getImageUrl() { + return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_IMAGE_URL), faviconUrl)); + } + } + + public interface HtmlDecoder { + @NonNull String fromEncoded(@NonNull String html); + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java index 19da66aca8..9f57d2e99c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java +++ b/app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java @@ -13,7 +13,7 @@ import org.json.JSONObject; import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.logsubmit.util.Scrubber; -import org.thoughtcrime.securesms.net.UserAgentInterceptor; +import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor; import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess; import org.thoughtcrime.securesms.util.concurrent.SignalExecutors; import org.whispersystems.libsignal.util.guava.Optional; @@ -91,7 +91,7 @@ public class SubmitDebugLogRepository { } try { - OkHttpClient client = new OkHttpClient.Builder().addInterceptor(new UserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build(); + OkHttpClient client = new OkHttpClient.Builder().addInterceptor(new StandardUserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build(); Response response = client.newCall(new Request.Builder().url(API_ENDPOINT).get().build()).execute(); ResponseBody body = response.body(); diff --git a/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java b/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java index 992500b8c1..c88994e942 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySafetyInterceptor.java @@ -1,5 +1,7 @@ package org.thoughtcrime.securesms.net; +import android.text.TextUtils; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -52,7 +54,13 @@ public class ContentProxySafetyInterceptor implements Interceptor { return isWhitelisted(url.toString()); } - private static boolean isWhitelisted(@Nullable String url) { - return LinkPreviewUtil.isWhitelistedLinkUrl(url) || LinkPreviewUtil.isWhitelistedMediaUrl(url); + private static boolean isWhitelisted(@Nullable String rawUrl) { + if (rawUrl == null) return false; + + HttpUrl url = HttpUrl.parse(rawUrl); + + return url != null && + "https".equals(url.scheme()) && + ContentProxySelector.WHITELISTED_DOMAINS.contains(url.topPrivateDomain()); } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySelector.java b/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySelector.java index 7d08595d4e..cd48658e3c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySelector.java +++ b/app/src/main/java/org/thoughtcrime/securesms/net/ContentProxySelector.java @@ -1,13 +1,9 @@ package org.thoughtcrime.securesms.net; -import android.os.AsyncTask; - -import org.thoughtcrime.securesms.linkpreview.LinkPreviewDomains; import org.thoughtcrime.securesms.logging.Log; import org.thoughtcrime.securesms.BuildConfig; -import org.thoughtcrime.securesms.util.Util; import java.io.IOException; import java.net.InetSocketAddress; @@ -25,10 +21,9 @@ public class ContentProxySelector extends ProxySelector { private static final String TAG = ContentProxySelector.class.getSimpleName(); - private static final Set WHITELISTED_DOMAINS = new HashSet<>(); + public static final Set WHITELISTED_DOMAINS = new HashSet<>(); static { - WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.LINKS); - WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.IMAGES); + WHITELISTED_DOMAINS.add("giphy.com"); } private final List CONTENT = new ArrayList(1) {{ diff --git a/app/src/main/java/org/thoughtcrime/securesms/net/StandardUserAgentInterceptor.java b/app/src/main/java/org/thoughtcrime/securesms/net/StandardUserAgentInterceptor.java new file mode 100644 index 0000000000..0150b46a97 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/net/StandardUserAgentInterceptor.java @@ -0,0 +1,15 @@ +package org.thoughtcrime.securesms.net; + +import android.os.Build; + +import org.thoughtcrime.securesms.BuildConfig; + +/** + * The user agent that should be used by default -- includes app name, version, etc. + */ +public class StandardUserAgentInterceptor extends UserAgentInterceptor { + + public StandardUserAgentInterceptor() { + super("Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")"); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/net/UserAgentInterceptor.java b/app/src/main/java/org/thoughtcrime/securesms/net/UserAgentInterceptor.java index c9ddef85f3..0cde9ee12f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/net/UserAgentInterceptor.java +++ b/app/src/main/java/org/thoughtcrime/securesms/net/UserAgentInterceptor.java @@ -1,11 +1,7 @@ package org.thoughtcrime.securesms.net; -import android.os.Build; - import androidx.annotation.NonNull; -import org.thoughtcrime.securesms.BuildConfig; - import java.io.IOException; import okhttp3.Interceptor; @@ -13,12 +9,16 @@ import okhttp3.Response; public class UserAgentInterceptor implements Interceptor { - private static final String USER_AGENT = "Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")"; + private final String userAgent; + + public UserAgentInterceptor(@NonNull String userAgent) { + this.userAgent = userAgent; + } @Override public Response intercept(@NonNull Chain chain) throws IOException { return chain.proceed(chain.request().newBuilder() - .header("User-Agent", USER_AGENT) + .header("User-Agent", userAgent) .build()); } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java b/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java index 705ddfc7ba..d9d4e21462 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java +++ b/app/src/main/java/org/thoughtcrime/securesms/push/SignalServiceNetworkAccess.java @@ -8,7 +8,7 @@ import androidx.annotation.Nullable; import org.thoughtcrime.securesms.BuildConfig; import org.thoughtcrime.securesms.net.CustomDns; import org.thoughtcrime.securesms.net.SequentialDns; -import org.thoughtcrime.securesms.net.UserAgentInterceptor; +import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor; import org.thoughtcrime.securesms.util.Base64; import org.thoughtcrime.securesms.util.TextSecurePreferences; import org.whispersystems.libsignal.util.guava.Optional; @@ -161,7 +161,7 @@ public class SignalServiceNetworkAccess { final SignalStorageUrl omanGoogleStorage = new SignalStorageUrl("https://www.google.com.om/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); final SignalStorageUrl qatarGoogleStorage = new SignalStorageUrl("https://www.google.com.qa/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC); - final List interceptors = Collections.singletonList(new UserAgentInterceptor()); + final List interceptors = Collections.singletonList(new StandardUserAgentInterceptor()); final Optional dns = Optional.of(DNS); final byte[] zkGroupServerPublicParams; diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/OkHttpUtil.java b/app/src/main/java/org/thoughtcrime/securesms/util/OkHttpUtil.java new file mode 100644 index 0000000000..631aaed100 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/util/OkHttpUtil.java @@ -0,0 +1,49 @@ +package org.thoughtcrime.securesms.util; + +import androidx.annotation.NonNull; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Objects; + +import okhttp3.MediaType; +import okhttp3.ResponseBody; + +import static okhttp3.internal.Util.UTF_8; + +public final class OkHttpUtil { + + private OkHttpUtil() {} + + public static byte[] readAsBytes(@NonNull InputStream bodyStream, long sizeLimit) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + byte[] buffer = new byte[(int) ByteUnit.KILOBYTES.toBytes(32)]; + int readLength = 0; + int totalLength = 0; + + while ((readLength = bodyStream.read(buffer)) >= 0) { + if (totalLength + readLength > sizeLimit) { + throw new IOException("Exceeded maximum size during read!"); + } + + outputStream.write(buffer, 0, readLength); + totalLength += readLength; + } + + return outputStream.toByteArray(); + } + public static String readAsString(@NonNull ResponseBody body, long sizeLimit) throws IOException { + if (body.contentLength() > sizeLimit) { + throw new IOException("Content-Length exceeded maximum size!"); + } + + byte[] data = readAsBytes(body.byteStream(), sizeLimit); + MediaType contentType = body.contentType(); + Charset charset = contentType != null ? contentType.charset(UTF_8) : UTF_8; + + return new String(data, Objects.requireNonNull(charset)); + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/util/Util.java b/app/src/main/java/org/thoughtcrime/securesms/util/Util.java index 9e89157297..72bffd0166 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/util/Util.java +++ b/app/src/main/java/org/thoughtcrime/securesms/util/Util.java @@ -159,6 +159,10 @@ public class Util { return collection == null || collection.isEmpty(); } + public static boolean isEmpty(@Nullable String value) { + return value == null || value.length() == 0; + } + public static boolean hasItems(@Nullable Collection collection) { return collection != null && !collection.isEmpty(); } @@ -169,7 +173,7 @@ public class Util { public static String getFirstNonEmpty(String... values) { for (String value : values) { - if (!TextUtils.isEmpty(value)) { + if (!Util.isEmpty(value)) { return value; } } diff --git a/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest.java b/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest.java deleted file mode 100644 index e45d5b3897..0000000000 --- a/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.thoughtcrime.securesms.linkpreview; - -import org.junit.Test; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertFalse; -import static junit.framework.TestCase.assertTrue; - -public class LinkPreviewUtilTest { - - @Test - public void isLegal_allAscii_noProtocol() { - assertTrue(LinkPreviewUtil.isLegalUrl("google.com")); - } - - @Test - public void isLegal_allAscii_noProtocol_subdomain() { - assertTrue(LinkPreviewUtil.isLegalUrl("foo.google.com")); - } - - @Test - public void isLegal_allAscii_subdomain() { - assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com")); - } - - @Test - public void isLegal_allAscii_subdomain_path() { - assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com/some/path.html")); - } - - @Test - public void isLegal_cyrillicHostAsciiTld() { - assertFalse(LinkPreviewUtil.isLegalUrl("http://кц.com")); - } - - @Test - public void isLegal_cyrillicHostAsciiTld_noProtocol() { - assertFalse(LinkPreviewUtil.isLegalUrl("кц.com")); - } - - @Test - public void isLegal_mixedHost_noProtocol() { - assertFalse(LinkPreviewUtil.isLegalUrl("http://asĸ.com")); - } - - @Test - public void isLegal_cyrillicHostAndTld_noProtocol() { - assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф")); - } - - @Test - public void isLegal_cyrillicHostAndTld_asciiPath_noProtocol() { - assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф/some/path")); - } - - @Test - public void isLegal_cyrillicHostAndTld_asciiPath() { - assertTrue(LinkPreviewUtil.isLegalUrl("https://кц.рф/some/path")); - } - - @Test - public void isLegal_asciiSubdomain_cyrillicHostAndTld() { - assertFalse(LinkPreviewUtil.isLegalUrl("http://foo.кц.рф")); - } - - @Test - public void isLegal_emptyUrl() { - assertFalse(LinkPreviewUtil.isLegalUrl("")); - } -} diff --git a/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_isLegal.java b/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_isLegal.java new file mode 100644 index 0000000000..905337f5f1 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_isLegal.java @@ -0,0 +1,44 @@ +package org.thoughtcrime.securesms.linkpreview; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +import static junit.framework.TestCase.assertEquals; + +@RunWith(Parameterized.class) +public class LinkPreviewUtilTest_isLegal { + + private final String input; + private final boolean output; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + { "google.com", true }, + { "foo.google.com", true }, + { "https://foo.google.com", true }, + { "https://foo.google.com/some/path.html", true }, + { "кц.рф", true }, + { "https://кц.рф/some/path", true }, + { "http://кц.com", false }, + { "кц.com", false }, + { "http://asĸ.com", false }, + { "http://foo.кц.рф", false }, + { "", false } + }); + } + + public LinkPreviewUtilTest_isLegal(String input, boolean output) { + this.input = input; + this.output = output; + } + + @Test + public void isLegal() { + assertEquals(output, LinkPreviewUtil.isLegalUrl(input)); + } +} diff --git a/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_parseOpenGraphFields.java b/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_parseOpenGraphFields.java new file mode 100644 index 0000000000..502fafc4c5 --- /dev/null +++ b/app/src/test/java/org/thoughtcrime/securesms/linkpreview/LinkPreviewUtilTest_parseOpenGraphFields.java @@ -0,0 +1,108 @@ +package org.thoughtcrime.securesms.linkpreview; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.whispersystems.libsignal.util.guava.Optional; + +import java.util.Arrays; +import java.util.Collection; + +import static junit.framework.TestCase.assertEquals; + +@RunWith(Parameterized.class) +public class LinkPreviewUtilTest_parseOpenGraphFields { + + private final String html; + private final String title; + private final String imageUrl; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + // Normal + { "\n" + + "", + "Daily Bugle", + "https://images.com/my-image.jpg"}, + + // Swap property orders + { "\n" + + "", + "Daily Bugle", + "https://images.com/my-image.jpg"}, + + // Funny spacing + { "< meta property = \"og:title\" content = \"Daily Bugle\" >\n\n" + + "< meta property = \"og:image\" content =\"https://images.com/my-image.jpg\" >", + "Daily Bugle", + "https://images.com/my-image.jpg"}, + + // Garbage in various places + { "\n" + + "asdfjkl\n" + + "idk\n" + + "\n" + + "", + "Daily Bugle", + "https://images.com/my-image.jpg"}, + + // Missing image + { "", + "Daily Bugle", + null}, + + // Missing title + { "", + null, + "https://images.com/my-image.jpg"}, + + // Has everything + { "\n" + + "Daily Bugle HTML\n" + + "\n" + + "", + "Daily Bugle", + "https://images.com/my-image.jpg"}, + + // Fallback to HTML title + { "Daily Bugle HTML\n" + + "\n" + + "", + "Daily Bugle HTML", + "https://images.com/my-image.jpg"}, + + // Fallback to favicon + { "\n" + + "Daily Bugle HTML\n" + + "", + "Daily Bugle", + "https://images.com/favicon.png"}, + + // Fallback to HTML title and favicon + { "Daily Bugle HTML\n" + + "", + "Daily Bugle HTML", + "https://images.com/favicon.png"}, + + // Different favicon formatting + { "Daily Bugle HTML\n" + + "", + "Daily Bugle HTML", + "https://images.com/favicon.png"}, + }); + } + + public LinkPreviewUtilTest_parseOpenGraphFields(String html, String title, String imageUrl) { + this.html = html; + this.title = title; + this.imageUrl = imageUrl; + } + + @Test + public void parseOpenGraphFields() { + LinkPreviewUtil.OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(html, html -> html); + assertEquals(Optional.fromNullable(title), openGraph.getTitle()); + assertEquals(Optional.fromNullable(imageUrl), openGraph.getImageUrl()); + } +}