Open up link previews to work with all sites.
This commit is contained in:
Greyson Parrelli
parent
d569419e13
commit
6e6105af05
18 changed files with 377 additions and 200 deletions
|
|
@ -15,7 +15,7 @@ import org.thoughtcrime.securesms.logging.Log;
|
|||
import org.thoughtcrime.securesms.giph.model.GiphyImage;
|
||||
import org.thoughtcrime.securesms.giph.model.GiphyResponse;
|
||||
import org.thoughtcrime.securesms.net.ContentProxySelector;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
|
||||
import org.thoughtcrime.securesms.util.AsyncLoader;
|
||||
import org.thoughtcrime.securesms.util.JsonUtils;
|
||||
|
|
@ -43,7 +43,7 @@ public abstract class GiphyLoader extends AsyncLoader<List<GiphyImage>> {
|
|||
this.searchString = searchString;
|
||||
this.client = new OkHttpClient.Builder()
|
||||
.proxySelector(new ContentProxySelector())
|
||||
.addInterceptor(new UserAgentInterceptor())
|
||||
.addInterceptor(new StandardUserAgentInterceptor())
|
||||
.dns(SignalServiceNetworkAccess.DNS)
|
||||
.build();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,7 @@ import com.bumptech.glide.load.model.MultiModelLoaderFactory;
|
|||
import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl;
|
||||
import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor;
|
||||
import org.thoughtcrime.securesms.net.ContentProxySelector;
|
||||
import org.thoughtcrime.securesms.net.CustomDns;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
|
@ -45,7 +44,7 @@ public class ChunkedImageUrlLoader implements ModelLoader<ChunkedImageUrl, Input
|
|||
this.client = new OkHttpClient.Builder()
|
||||
.proxySelector(new ContentProxySelector())
|
||||
.cache(null)
|
||||
.addInterceptor(new UserAgentInterceptor())
|
||||
.addInterceptor(new StandardUserAgentInterceptor())
|
||||
.addNetworkInterceptor(new ContentProxySafetyInterceptor())
|
||||
.addNetworkInterceptor(new PaddedHeadersInterceptor())
|
||||
.dns(SignalServiceNetworkAccess.DNS)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import com.bumptech.glide.load.model.ModelLoaderFactory;
|
|||
import com.bumptech.glide.load.model.MultiModelLoaderFactory;
|
||||
|
||||
import org.thoughtcrime.securesms.net.ContentProxySelector;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
|
||||
|
||||
import java.io.InputStream;
|
||||
|
|
@ -48,7 +48,7 @@ public class OkHttpUrlLoader implements ModelLoader<GlideUrl, InputStream> {
|
|||
if (internalClient == null) {
|
||||
internalClient = new OkHttpClient.Builder()
|
||||
.proxySelector(new ContentProxySelector())
|
||||
.addInterceptor(new UserAgentInterceptor())
|
||||
.addInterceptor(new StandardUserAgentInterceptor())
|
||||
.dns(SignalServiceNetworkAccess.DNS)
|
||||
.build();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,7 +124,6 @@ import org.whispersystems.signalservice.api.messages.multidevice.ViewOnceOpenMes
|
|||
import org.whispersystems.signalservice.api.messages.shared.SharedContact;
|
||||
import org.whispersystems.signalservice.api.push.SignalServiceAddress;
|
||||
import org.whispersystems.signalservice.api.push.exceptions.PushNetworkException;
|
||||
import org.whispersystems.signalservice.api.util.UuidUtil;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.security.SecureRandom;
|
||||
|
|
@ -135,7 +134,6 @@ import java.util.List;
|
|||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public final class PushProcessMessageJob extends BaseJob {
|
||||
|
|
@ -1695,7 +1693,7 @@ public final class PushProcessMessageJob extends BaseJob {
|
|||
Optional<String> title = Optional.fromNullable(preview.getTitle());
|
||||
boolean hasContent = !TextUtils.isEmpty(title.or("")) || thumbnail.isPresent();
|
||||
boolean presentInBody = url.isPresent() && Stream.of(LinkPreviewUtil.findWhitelistedUrls(message)).map(Link::getUrl).collect(Collectors.toSet()).contains(url.get());
|
||||
boolean validDomain = url.isPresent() && LinkPreviewUtil.isWhitelistedLinkUrl(url.get());
|
||||
boolean validDomain = url.isPresent() && LinkPreviewUtil.isValidPreviewUrl(url.get());
|
||||
|
||||
if (hasContent && presentInBody && validDomain) {
|
||||
LinkPreview linkPreview = new LinkPreview(url.get(), title.or(""), thumbnail);
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
package org.thoughtcrime.securesms.linkpreview;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
public class LinkPreviewDomains {
|
||||
public static final String STICKERS = "signal.org";
|
||||
|
||||
public static final Set<String> LINKS = new HashSet<>(Arrays.asList(
|
||||
"youtube.com",
|
||||
"www.youtube.com",
|
||||
"m.youtube.com",
|
||||
"youtu.be",
|
||||
"reddit.com",
|
||||
"www.reddit.com",
|
||||
"m.reddit.com",
|
||||
"imgur.com",
|
||||
"www.imgur.com",
|
||||
"m.imgur.com",
|
||||
"instagram.com",
|
||||
"www.instagram.com",
|
||||
"m.instagram.com",
|
||||
"pinterest.com",
|
||||
"www.pinterest.com",
|
||||
"pin.it"
|
||||
));
|
||||
|
||||
public static final Set<String> IMAGES = new HashSet<>(Arrays.asList(
|
||||
"ytimg.com",
|
||||
"cdninstagram.com",
|
||||
"fbcdn.net",
|
||||
"redd.it",
|
||||
"imgur.com",
|
||||
"pinimg.com",
|
||||
"giphy.com"
|
||||
));
|
||||
}
|
||||
|
|
@ -2,31 +2,33 @@ package org.thoughtcrime.securesms.linkpreview;
|
|||
|
||||
import android.content.Context;
|
||||
import android.graphics.Bitmap;
|
||||
import android.graphics.BitmapFactory;
|
||||
import android.net.Uri;
|
||||
import androidx.annotation.NonNull;
|
||||
import android.text.Html;
|
||||
import android.text.TextUtils;
|
||||
|
||||
import com.bumptech.glide.load.engine.DiskCacheStrategy;
|
||||
import com.bumptech.glide.request.FutureTarget;
|
||||
|
||||
import org.thoughtcrime.securesms.attachments.Attachment;
|
||||
import org.thoughtcrime.securesms.attachments.UriAttachment;
|
||||
import org.thoughtcrime.securesms.database.AttachmentDatabase;
|
||||
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
|
||||
import org.thoughtcrime.securesms.giph.model.ChunkedImageUrl;
|
||||
import org.thoughtcrime.securesms.linkpreview.LinkPreviewUtil.OpenGraph;
|
||||
import org.thoughtcrime.securesms.logging.Log;
|
||||
import org.thoughtcrime.securesms.mms.GlideApp;
|
||||
import org.thoughtcrime.securesms.net.CallRequestController;
|
||||
import org.thoughtcrime.securesms.net.CompositeRequestController;
|
||||
import org.thoughtcrime.securesms.net.ContentProxySafetyInterceptor;
|
||||
import org.thoughtcrime.securesms.net.ContentProxySelector;
|
||||
import org.thoughtcrime.securesms.net.RequestController;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.providers.BlobProvider;
|
||||
import org.thoughtcrime.securesms.stickers.StickerRemoteUri;
|
||||
import org.thoughtcrime.securesms.stickers.StickerUrl;
|
||||
import org.thoughtcrime.securesms.util.ByteUnit;
|
||||
import org.thoughtcrime.securesms.util.Hex;
|
||||
import org.thoughtcrime.securesms.util.MediaUtil;
|
||||
import org.thoughtcrime.securesms.util.OkHttpUtil;
|
||||
import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
|
||||
import org.whispersystems.libsignal.InvalidMessageException;
|
||||
import org.whispersystems.libsignal.util.Pair;
|
||||
|
|
@ -34,10 +36,11 @@ import org.whispersystems.libsignal.util.guava.Optional;
|
|||
import org.whispersystems.signalservice.api.SignalServiceMessageReceiver;
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest;
|
||||
import org.whispersystems.signalservice.api.messages.SignalServiceStickerManifest.StickerInfo;
|
||||
import org.whispersystems.signalservice.api.util.OptionalUtil;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.io.InputStream;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
|
@ -54,20 +57,22 @@ public class LinkPreviewRepository {
|
|||
|
||||
private static final CacheControl NO_CACHE = new CacheControl.Builder().noCache().build();
|
||||
|
||||
private static final long FAILSAFE_MAX_TEXT_SIZE = ByteUnit.MEGABYTES.toBytes(2);
|
||||
private static final long FAILSAFE_MAX_IMAGE_SIZE = ByteUnit.MEGABYTES.toBytes(2);
|
||||
|
||||
private final OkHttpClient client;
|
||||
|
||||
public LinkPreviewRepository() {
|
||||
this.client = new OkHttpClient.Builder()
|
||||
.proxySelector(new ContentProxySelector())
|
||||
.addNetworkInterceptor(new ContentProxySafetyInterceptor())
|
||||
.cache(null)
|
||||
.addInterceptor(new UserAgentInterceptor("WhatsApp"))
|
||||
.build();
|
||||
}
|
||||
|
||||
RequestController getLinkPreview(@NonNull Context context, @NonNull String url, @NonNull Callback<Optional<LinkPreview>> callback) {
|
||||
CompositeRequestController compositeController = new CompositeRequestController();
|
||||
|
||||
if (!LinkPreviewUtil.isWhitelistedLinkUrl(url)) {
|
||||
if (!LinkPreviewUtil.isValidPreviewUrl(url)) {
|
||||
Log.w(TAG, "Tried to get a link preview for a non-whitelisted domain.");
|
||||
callback.onComplete(Optional.absent());
|
||||
return compositeController;
|
||||
|
|
@ -89,7 +94,7 @@ public class LinkPreviewRepository {
|
|||
return;
|
||||
}
|
||||
|
||||
RequestController imageController = fetchThumbnail(context, metadata.getImageUrl().get(), attachment -> {
|
||||
RequestController imageController = fetchThumbnail(metadata.getImageUrl().get(), attachment -> {
|
||||
if (!metadata.getTitle().isPresent() && !attachment.isPresent()) {
|
||||
callback.onComplete(Optional.absent());
|
||||
} else {
|
||||
|
|
@ -127,11 +132,12 @@ public class LinkPreviewRepository {
|
|||
return;
|
||||
}
|
||||
|
||||
String body = response.body().string();
|
||||
Optional<String> title = getProperty(body, "title");
|
||||
Optional<String> imageUrl = getProperty(body, "image");
|
||||
String body = OkHttpUtil.readAsString(response.body(), FAILSAFE_MAX_TEXT_SIZE);
|
||||
OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(body);
|
||||
Optional<String> title = openGraph.getTitle();
|
||||
Optional<String> imageUrl = openGraph.getImageUrl();
|
||||
|
||||
if (imageUrl.isPresent() && !LinkPreviewUtil.isWhitelistedMediaUrl(imageUrl.get())) {
|
||||
if (imageUrl.isPresent() && !LinkPreviewUtil.isValidPreviewUrl(imageUrl.get())) {
|
||||
Log.i(TAG, "Image URL was invalid or for a non-whitelisted domain. Skipping.");
|
||||
imageUrl = Optional.absent();
|
||||
}
|
||||
|
|
@ -143,20 +149,23 @@ public class LinkPreviewRepository {
|
|||
return new CallRequestController(call);
|
||||
}
|
||||
|
||||
private @NonNull RequestController fetchThumbnail(@NonNull Context context, @NonNull String imageUrl, @NonNull Callback<Optional<Attachment>> callback) {
|
||||
FutureTarget<Bitmap> bitmapFuture = GlideApp.with(context).asBitmap()
|
||||
.load(new ChunkedImageUrl(imageUrl))
|
||||
.skipMemoryCache(true)
|
||||
.diskCacheStrategy(DiskCacheStrategy.NONE)
|
||||
.centerInside()
|
||||
.submit(1024, 1024);
|
||||
|
||||
RequestController controller = () -> bitmapFuture.cancel(false);
|
||||
private @NonNull RequestController fetchThumbnail(@NonNull String imageUrl, @NonNull Callback<Optional<Attachment>> callback) {
|
||||
Call call = client.newCall(new Request.Builder().url(imageUrl).build());
|
||||
CallRequestController controller = new CallRequestController(call);
|
||||
|
||||
SignalExecutors.UNBOUNDED.execute(() -> {
|
||||
try {
|
||||
Bitmap bitmap = bitmapFuture.get();
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
Response response = call.execute();
|
||||
if (!response.isSuccessful() || response.body() == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
InputStream bodyStream = response.body().byteStream();
|
||||
controller.setStream(bodyStream);
|
||||
|
||||
byte[] data = OkHttpUtil.readAsBytes(bodyStream, FAILSAFE_MAX_IMAGE_SIZE);
|
||||
Bitmap bitmap = BitmapFactory.decodeByteArray(data, 0, data.length);
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos);
|
||||
|
||||
|
|
@ -181,27 +190,14 @@ public class LinkPreviewRepository {
|
|||
null));
|
||||
|
||||
callback.onComplete(thumbnail);
|
||||
} catch (CancellationException | ExecutionException | InterruptedException e) {
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "Exception during link preview image retrieval.", e);
|
||||
controller.cancel();
|
||||
callback.onComplete(Optional.absent());
|
||||
} finally {
|
||||
bitmapFuture.cancel(false);
|
||||
}
|
||||
});
|
||||
|
||||
return () -> bitmapFuture.cancel(true);
|
||||
}
|
||||
|
||||
private @NonNull Optional<String> getProperty(@NonNull String searchText, @NonNull String property) {
|
||||
Pattern pattern = Pattern.compile("<\\s*meta\\s+property\\s*=\\s*\"\\s*og:" + property + "\\s*\"\\s+[^>]*content\\s*=\\s*\"(.*?)\"[^>]*/?\\s*>", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
|
||||
Matcher matcher = pattern.matcher(searchText);
|
||||
|
||||
if (matcher.find()) {
|
||||
String text = Html.fromHtml(matcher.group(1)).toString();
|
||||
return TextUtils.isEmpty(text) ? Optional.absent() : Optional.of(text);
|
||||
}
|
||||
|
||||
return Optional.absent();
|
||||
return controller;
|
||||
}
|
||||
|
||||
private RequestController fetchStickerPackLinkPreview(@NonNull Context context,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@ package org.thoughtcrime.securesms.linkpreview;
|
|||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.annotation.VisibleForTesting;
|
||||
|
||||
import android.text.Html;
|
||||
import android.text.SpannableString;
|
||||
import android.text.TextUtils;
|
||||
import android.text.style.URLSpan;
|
||||
|
|
@ -10,9 +13,14 @@ import android.text.util.Linkify;
|
|||
import com.annimon.stream.Stream;
|
||||
|
||||
import org.thoughtcrime.securesms.stickers.StickerUrl;
|
||||
import org.thoughtcrime.securesms.util.Util;
|
||||
import org.whispersystems.libsignal.util.guava.Optional;
|
||||
import org.whispersystems.signalservice.api.util.OptionalUtil;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -20,9 +28,14 @@ import okhttp3.HttpUrl;
|
|||
|
||||
public final class LinkPreviewUtil {
|
||||
|
||||
private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$");
|
||||
private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$");
|
||||
private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$");
|
||||
private static final Pattern DOMAIN_PATTERN = Pattern.compile("^(https?://)?([^/]+).*$");
|
||||
private static final Pattern ALL_ASCII_PATTERN = Pattern.compile("^[\\x00-\\x7F]*$");
|
||||
private static final Pattern ALL_NON_ASCII_PATTERN = Pattern.compile("^[^\\x00-\\x7F]*$");
|
||||
private static final Pattern OPEN_GRAPH_TAG_PATTERN = Pattern.compile("<\\s*meta[^>]*property\\s*=\\s*\"\\s*og:([^\"]+)\"[^>]*/?\\s*>");
|
||||
private static final Pattern OPEN_GRAPH_CONTENT_PATTERN = Pattern.compile("content\\s*=\\s*\"([^\"]*)\"");
|
||||
private static final Pattern TITLE_PATTERN = Pattern.compile("<\\s*title[^>]*>(.*)<\\s*/title[^>]*>");
|
||||
private static final Pattern FAVICON_PATTERN = Pattern.compile("<\\s*link[^>]*rel\\s*=\\s*\".*icon.*\"[^>]*>");
|
||||
private static final Pattern FAVICON_HREF_PATTERN = Pattern.compile("href\\s*=\\s*\"([^\"]*)\"");
|
||||
|
||||
/**
|
||||
* @return All whitelisted URLs in the source text.
|
||||
|
|
@ -37,14 +50,14 @@ public final class LinkPreviewUtil {
|
|||
|
||||
return Stream.of(spannable.getSpans(0, spannable.length(), URLSpan.class))
|
||||
.map(span -> new Link(span.getURL(), spannable.getSpanStart(span)))
|
||||
.filter(link -> isWhitelistedLinkUrl(link.getUrl()))
|
||||
.filter(link -> isValidPreviewUrl(link.getUrl()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return True if the host is present in the link whitelist.
|
||||
*/
|
||||
public static boolean isWhitelistedLinkUrl(@Nullable String linkUrl) {
|
||||
public static boolean isValidPreviewUrl(@Nullable String linkUrl) {
|
||||
if (linkUrl == null) return false;
|
||||
if (StickerUrl.isValidShareLink(linkUrl)) return true;
|
||||
|
||||
|
|
@ -52,24 +65,9 @@ public final class LinkPreviewUtil {
|
|||
return url != null &&
|
||||
!TextUtils.isEmpty(url.scheme()) &&
|
||||
"https".equals(url.scheme()) &&
|
||||
LinkPreviewDomains.LINKS.contains(url.host()) &&
|
||||
isLegalUrl(linkUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return True if the top-level domain is present in the media whitelist.
|
||||
*/
|
||||
public static boolean isWhitelistedMediaUrl(@Nullable String mediaUrl) {
|
||||
if (mediaUrl == null) return false;
|
||||
|
||||
HttpUrl url = HttpUrl.parse(mediaUrl);
|
||||
return url != null &&
|
||||
!TextUtils.isEmpty(url.scheme()) &&
|
||||
"https".equals(url.scheme()) &&
|
||||
LinkPreviewDomains.IMAGES.contains(url.topPrivateDomain()) &&
|
||||
isLegalUrl(mediaUrl);
|
||||
}
|
||||
|
||||
public static boolean isLegalUrl(@NonNull String url) {
|
||||
Matcher matcher = DOMAIN_PATTERN.matcher(url);
|
||||
|
||||
|
|
@ -83,4 +81,78 @@ public final class LinkPreviewUtil {
|
|||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html) {
|
||||
return parseOpenGraphFields(html, text -> Html.fromHtml(text).toString());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static @NonNull OpenGraph parseOpenGraphFields(@Nullable String html, @NonNull HtmlDecoder htmlDecoder) {
|
||||
if (html == null) {
|
||||
return new OpenGraph(Collections.emptyMap(), null, null);
|
||||
}
|
||||
|
||||
Map<String, String> openGraphTags = new HashMap<>();
|
||||
Matcher openGraphMatcher = OPEN_GRAPH_TAG_PATTERN.matcher(html);
|
||||
|
||||
while (openGraphMatcher.find()) {
|
||||
String tag = openGraphMatcher.group();
|
||||
String property = openGraphMatcher.groupCount() > 0 ? openGraphMatcher.group(1) : null;
|
||||
|
||||
if (property != null) {
|
||||
Matcher contentMatcher = OPEN_GRAPH_CONTENT_PATTERN.matcher(tag);
|
||||
if (contentMatcher.find() && contentMatcher.groupCount() > 0) {
|
||||
String content = htmlDecoder.fromEncoded(contentMatcher.group(1));
|
||||
openGraphTags.put(property, content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String htmlTitle = "";
|
||||
String faviconUrl = "";
|
||||
|
||||
Matcher titleMatcher = TITLE_PATTERN.matcher(html);
|
||||
if (titleMatcher.find() && titleMatcher.groupCount() > 0) {
|
||||
htmlTitle = titleMatcher.group(1);
|
||||
}
|
||||
|
||||
Matcher faviconMatcher = FAVICON_PATTERN.matcher(html);
|
||||
if (faviconMatcher.find()) {
|
||||
Matcher faviconHrefMatcher = FAVICON_HREF_PATTERN.matcher(faviconMatcher.group());
|
||||
if (faviconHrefMatcher.find() && faviconHrefMatcher.groupCount() > 0) {
|
||||
faviconUrl = faviconHrefMatcher.group(1);
|
||||
}
|
||||
}
|
||||
|
||||
return new OpenGraph(openGraphTags, htmlTitle, faviconUrl);
|
||||
}
|
||||
|
||||
public static final class OpenGraph {
|
||||
|
||||
private final Map<String, String> values;
|
||||
|
||||
private final @Nullable String htmlTitle;
|
||||
private final @Nullable String faviconUrl;
|
||||
|
||||
private static final String KEY_TITLE = "title";
|
||||
private static final String KEY_IMAGE_URL = "image";
|
||||
|
||||
public OpenGraph(@NonNull Map<String, String> values, @Nullable String htmlTitle, @Nullable String faviconUrl) {
|
||||
this.values = values;
|
||||
this.htmlTitle = htmlTitle;
|
||||
this.faviconUrl = faviconUrl;
|
||||
}
|
||||
|
||||
public @NonNull Optional<String> getTitle() {
|
||||
return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_TITLE), htmlTitle));
|
||||
}
|
||||
|
||||
public @NonNull Optional<String> getImageUrl() {
|
||||
return OptionalUtil.absentIfEmpty(Util.getFirstNonEmpty(values.get(KEY_IMAGE_URL), faviconUrl));
|
||||
}
|
||||
}
|
||||
|
||||
public interface HtmlDecoder {
|
||||
@NonNull String fromEncoded(@NonNull String html);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import org.json.JSONObject;
|
|||
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
|
||||
import org.thoughtcrime.securesms.logging.Log;
|
||||
import org.thoughtcrime.securesms.logsubmit.util.Scrubber;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
|
||||
import org.thoughtcrime.securesms.util.concurrent.SignalExecutors;
|
||||
import org.whispersystems.libsignal.util.guava.Optional;
|
||||
|
|
@ -91,7 +91,7 @@ public class SubmitDebugLogRepository {
|
|||
}
|
||||
|
||||
try {
|
||||
OkHttpClient client = new OkHttpClient.Builder().addInterceptor(new UserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build();
|
||||
OkHttpClient client = new OkHttpClient.Builder().addInterceptor(new StandardUserAgentInterceptor()).dns(SignalServiceNetworkAccess.DNS).build();
|
||||
Response response = client.newCall(new Request.Builder().url(API_ENDPOINT).get().build()).execute();
|
||||
ResponseBody body = response.body();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
package org.thoughtcrime.securesms.net;
|
||||
|
||||
import android.text.TextUtils;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
|
|
@ -52,7 +54,13 @@ public class ContentProxySafetyInterceptor implements Interceptor {
|
|||
return isWhitelisted(url.toString());
|
||||
}
|
||||
|
||||
private static boolean isWhitelisted(@Nullable String url) {
|
||||
return LinkPreviewUtil.isWhitelistedLinkUrl(url) || LinkPreviewUtil.isWhitelistedMediaUrl(url);
|
||||
private static boolean isWhitelisted(@Nullable String rawUrl) {
|
||||
if (rawUrl == null) return false;
|
||||
|
||||
HttpUrl url = HttpUrl.parse(rawUrl);
|
||||
|
||||
return url != null &&
|
||||
"https".equals(url.scheme()) &&
|
||||
ContentProxySelector.WHITELISTED_DOMAINS.contains(url.topPrivateDomain());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
package org.thoughtcrime.securesms.net;
|
||||
|
||||
|
||||
import android.os.AsyncTask;
|
||||
|
||||
import org.thoughtcrime.securesms.linkpreview.LinkPreviewDomains;
|
||||
import org.thoughtcrime.securesms.logging.Log;
|
||||
|
||||
import org.thoughtcrime.securesms.BuildConfig;
|
||||
import org.thoughtcrime.securesms.util.Util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
|
|
@ -25,10 +21,9 @@ public class ContentProxySelector extends ProxySelector {
|
|||
|
||||
private static final String TAG = ContentProxySelector.class.getSimpleName();
|
||||
|
||||
private static final Set<String> WHITELISTED_DOMAINS = new HashSet<>();
|
||||
public static final Set<String> WHITELISTED_DOMAINS = new HashSet<>();
|
||||
static {
|
||||
WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.LINKS);
|
||||
WHITELISTED_DOMAINS.addAll(LinkPreviewDomains.IMAGES);
|
||||
WHITELISTED_DOMAINS.add("giphy.com");
|
||||
}
|
||||
|
||||
private final List<Proxy> CONTENT = new ArrayList<Proxy>(1) {{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
package org.thoughtcrime.securesms.net;
|
||||
|
||||
import android.os.Build;
|
||||
|
||||
import org.thoughtcrime.securesms.BuildConfig;
|
||||
|
||||
/**
|
||||
* The user agent that should be used by default -- includes app name, version, etc.
|
||||
*/
|
||||
public class StandardUserAgentInterceptor extends UserAgentInterceptor {
|
||||
|
||||
public StandardUserAgentInterceptor() {
|
||||
super("Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +1,7 @@
|
|||
package org.thoughtcrime.securesms.net;
|
||||
|
||||
import android.os.Build;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
|
||||
import org.thoughtcrime.securesms.BuildConfig;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import okhttp3.Interceptor;
|
||||
|
|
@ -13,12 +9,16 @@ import okhttp3.Response;
|
|||
|
||||
public class UserAgentInterceptor implements Interceptor {
|
||||
|
||||
private static final String USER_AGENT = "Signal-Android " + BuildConfig.VERSION_NAME + " (API " + Build.VERSION.SDK_INT + ")";
|
||||
private final String userAgent;
|
||||
|
||||
public UserAgentInterceptor(@NonNull String userAgent) {
|
||||
this.userAgent = userAgent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Response intercept(@NonNull Chain chain) throws IOException {
|
||||
return chain.proceed(chain.request().newBuilder()
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("User-Agent", userAgent)
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import androidx.annotation.Nullable;
|
|||
import org.thoughtcrime.securesms.BuildConfig;
|
||||
import org.thoughtcrime.securesms.net.CustomDns;
|
||||
import org.thoughtcrime.securesms.net.SequentialDns;
|
||||
import org.thoughtcrime.securesms.net.UserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
|
||||
import org.thoughtcrime.securesms.util.Base64;
|
||||
import org.thoughtcrime.securesms.util.TextSecurePreferences;
|
||||
import org.whispersystems.libsignal.util.guava.Optional;
|
||||
|
|
@ -161,7 +161,7 @@ public class SignalServiceNetworkAccess {
|
|||
final SignalStorageUrl omanGoogleStorage = new SignalStorageUrl("https://www.google.com.om/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC);
|
||||
final SignalStorageUrl qatarGoogleStorage = new SignalStorageUrl("https://www.google.com.qa/storage", SERVICE_REFLECTOR_HOST, trustStore, GMAIL_CONNECTION_SPEC);
|
||||
|
||||
final List<Interceptor> interceptors = Collections.singletonList(new UserAgentInterceptor());
|
||||
final List<Interceptor> interceptors = Collections.singletonList(new StandardUserAgentInterceptor());
|
||||
final Optional<Dns> dns = Optional.of(DNS);
|
||||
|
||||
final byte[] zkGroupServerPublicParams;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
package org.thoughtcrime.securesms.util;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Objects;
|
||||
|
||||
import okhttp3.MediaType;
|
||||
import okhttp3.ResponseBody;
|
||||
|
||||
import static okhttp3.internal.Util.UTF_8;
|
||||
|
||||
public final class OkHttpUtil {
|
||||
|
||||
private OkHttpUtil() {}
|
||||
|
||||
public static byte[] readAsBytes(@NonNull InputStream bodyStream, long sizeLimit) throws IOException {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
|
||||
byte[] buffer = new byte[(int) ByteUnit.KILOBYTES.toBytes(32)];
|
||||
int readLength = 0;
|
||||
int totalLength = 0;
|
||||
|
||||
while ((readLength = bodyStream.read(buffer)) >= 0) {
|
||||
if (totalLength + readLength > sizeLimit) {
|
||||
throw new IOException("Exceeded maximum size during read!");
|
||||
}
|
||||
|
||||
outputStream.write(buffer, 0, readLength);
|
||||
totalLength += readLength;
|
||||
}
|
||||
|
||||
return outputStream.toByteArray();
|
||||
}
|
||||
public static String readAsString(@NonNull ResponseBody body, long sizeLimit) throws IOException {
|
||||
if (body.contentLength() > sizeLimit) {
|
||||
throw new IOException("Content-Length exceeded maximum size!");
|
||||
}
|
||||
|
||||
byte[] data = readAsBytes(body.byteStream(), sizeLimit);
|
||||
MediaType contentType = body.contentType();
|
||||
Charset charset = contentType != null ? contentType.charset(UTF_8) : UTF_8;
|
||||
|
||||
return new String(data, Objects.requireNonNull(charset));
|
||||
}
|
||||
}
|
||||
|
|
@ -159,6 +159,10 @@ public class Util {
|
|||
return collection == null || collection.isEmpty();
|
||||
}
|
||||
|
||||
public static boolean isEmpty(@Nullable String value) {
|
||||
return value == null || value.length() == 0;
|
||||
}
|
||||
|
||||
public static boolean hasItems(@Nullable Collection<?> collection) {
|
||||
return collection != null && !collection.isEmpty();
|
||||
}
|
||||
|
|
@ -169,7 +173,7 @@ public class Util {
|
|||
|
||||
public static String getFirstNonEmpty(String... values) {
|
||||
for (String value : values) {
|
||||
if (!TextUtils.isEmpty(value)) {
|
||||
if (!Util.isEmpty(value)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
package org.thoughtcrime.securesms.linkpreview;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static junit.framework.TestCase.assertEquals;
|
||||
import static junit.framework.TestCase.assertFalse;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
|
||||
public class LinkPreviewUtilTest {
|
||||
|
||||
@Test
|
||||
public void isLegal_allAscii_noProtocol() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("google.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_allAscii_noProtocol_subdomain() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("foo.google.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_allAscii_subdomain() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_allAscii_subdomain_path() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("https://foo.google.com/some/path.html"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_cyrillicHostAsciiTld() {
|
||||
assertFalse(LinkPreviewUtil.isLegalUrl("http://кц.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_cyrillicHostAsciiTld_noProtocol() {
|
||||
assertFalse(LinkPreviewUtil.isLegalUrl("кц.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_mixedHost_noProtocol() {
|
||||
assertFalse(LinkPreviewUtil.isLegalUrl("http://asĸ.com"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_cyrillicHostAndTld_noProtocol() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_cyrillicHostAndTld_asciiPath_noProtocol() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("кц.рф/some/path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_cyrillicHostAndTld_asciiPath() {
|
||||
assertTrue(LinkPreviewUtil.isLegalUrl("https://кц.рф/some/path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_asciiSubdomain_cyrillicHostAndTld() {
|
||||
assertFalse(LinkPreviewUtil.isLegalUrl("http://foo.кц.рф"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal_emptyUrl() {
|
||||
assertFalse(LinkPreviewUtil.isLegalUrl(""));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
package org.thoughtcrime.securesms.linkpreview;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
import static junit.framework.TestCase.assertEquals;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class LinkPreviewUtilTest_isLegal {
|
||||
|
||||
private final String input;
|
||||
private final boolean output;
|
||||
|
||||
@Parameterized.Parameters
|
||||
public static Collection<Object[]> data() {
|
||||
return Arrays.asList(new Object[][]{
|
||||
{ "google.com", true },
|
||||
{ "foo.google.com", true },
|
||||
{ "https://foo.google.com", true },
|
||||
{ "https://foo.google.com/some/path.html", true },
|
||||
{ "кц.рф", true },
|
||||
{ "https://кц.рф/some/path", true },
|
||||
{ "http://кц.com", false },
|
||||
{ "кц.com", false },
|
||||
{ "http://asĸ.com", false },
|
||||
{ "http://foo.кц.рф", false },
|
||||
{ "", false }
|
||||
});
|
||||
}
|
||||
|
||||
public LinkPreviewUtilTest_isLegal(String input, boolean output) {
|
||||
this.input = input;
|
||||
this.output = output;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void isLegal() {
|
||||
assertEquals(output, LinkPreviewUtil.isLegalUrl(input));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
package org.thoughtcrime.securesms.linkpreview;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.whispersystems.libsignal.util.guava.Optional;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
import static junit.framework.TestCase.assertEquals;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class LinkPreviewUtilTest_parseOpenGraphFields {
|
||||
|
||||
private final String html;
|
||||
private final String title;
|
||||
private final String imageUrl;
|
||||
|
||||
@Parameterized.Parameters
|
||||
public static Collection<Object[]> data() {
|
||||
return Arrays.asList(new Object[][]{
|
||||
// Normal
|
||||
{ "<meta content=\"Daily Bugle\" property=\"og:title\">\n" +
|
||||
"<meta content=\"https://images.com/my-image.jpg\" property=\"og:image\">",
|
||||
"Daily Bugle",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Swap property orders
|
||||
{ "<meta property=\"og:title\" content=\"Daily Bugle\">\n" +
|
||||
"<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">",
|
||||
"Daily Bugle",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Funny spacing
|
||||
{ "< meta property = \"og:title\" content = \"Daily Bugle\" >\n\n" +
|
||||
"< meta property = \"og:image\" content =\"https://images.com/my-image.jpg\" >",
|
||||
"Daily Bugle",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Garbage in various places
|
||||
{ "<meta property=\"og:title\" content=\"Daily Bugle\">\n" +
|
||||
"asdfjkl\n" +
|
||||
"<body>idk</body>\n" +
|
||||
"<script type=\"text/javascript\">var a = </script>\n" +
|
||||
"<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">",
|
||||
"Daily Bugle",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Missing image
|
||||
{ "<meta content=\"Daily Bugle\" property=\"og:title\">",
|
||||
"Daily Bugle",
|
||||
null},
|
||||
|
||||
// Missing title
|
||||
{ "<meta content=\"https://images.com/my-image.jpg\" property=\"og:image\">",
|
||||
null,
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Has everything
|
||||
{ "<meta property=\"og:title\" content = \"Daily Bugle\">\n" +
|
||||
"<title>Daily Bugle HTML</title>\n" +
|
||||
"<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">\n" +
|
||||
"<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
|
||||
"Daily Bugle",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Fallback to HTML title
|
||||
{ "<title>Daily Bugle HTML</title>\n" +
|
||||
"<meta property=\"og:image\" content=\"https://images.com/my-image.jpg\">\n" +
|
||||
"<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
|
||||
"Daily Bugle HTML",
|
||||
"https://images.com/my-image.jpg"},
|
||||
|
||||
// Fallback to favicon
|
||||
{ "<meta property=\"og:title\" content = \"Daily Bugle\">\n" +
|
||||
"<title>Daily Bugle HTML</title>\n" +
|
||||
"<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
|
||||
"Daily Bugle",
|
||||
"https://images.com/favicon.png"},
|
||||
|
||||
// Fallback to HTML title and favicon
|
||||
{ "<title>Daily Bugle HTML</title>\n" +
|
||||
"<link rel=\"icon\" href=\"https://images.com/favicon.png\" />",
|
||||
"Daily Bugle HTML",
|
||||
"https://images.com/favicon.png"},
|
||||
|
||||
// Different favicon formatting
|
||||
{ "<title>Daily Bugle HTML</title>\n" +
|
||||
"<link rel=\"shortcut icon\" href=\"https://images.com/favicon.png\" />",
|
||||
"Daily Bugle HTML",
|
||||
"https://images.com/favicon.png"},
|
||||
});
|
||||
}
|
||||
|
||||
public LinkPreviewUtilTest_parseOpenGraphFields(String html, String title, String imageUrl) {
|
||||
this.html = html;
|
||||
this.title = title;
|
||||
this.imageUrl = imageUrl;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseOpenGraphFields() {
|
||||
LinkPreviewUtil.OpenGraph openGraph = LinkPreviewUtil.parseOpenGraphFields(html, html -> html);
|
||||
assertEquals(Optional.fromNullable(title), openGraph.getTitle());
|
||||
assertEquals(Optional.fromNullable(imageUrl), openGraph.getImageUrl());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue