empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Transcode video during attachment upload.

Browse source
This commit is contained in:
Alan Evans 2019-07-25 15:29:31 -04:00
parent f9946083dd
commit e8e80e5d05
29 changed files with 853 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
jni/utils/org_thoughtcrime_securesms_util_FileUtils.cpp
View file

@ -3,6 +3,8 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <linux/memfd.h>
#include <syscall.h>
jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner
(JNIEnv *env, jclass clazz, jobject fileDescriptor)
@ -28,4 +30,16 @@ jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwn
}
return stat_struct.st_uid;
}
}
JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor
(JNIEnv *env, jclass clazz, jstring jname)
{
const char *name = env->GetStringUTFChars(jname, NULL);
int fd = syscall(SYS_memfd_create, name, MFD_CLOEXEC);
env->ReleaseStringUTFChars(jname, name);
return fd;
}

8
jni/utils/org_thoughtcrime_securesms_util_FileUtils.h
View file

@ -15,6 +15,14 @@ extern "C" {
JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_getFileDescriptorOwner
(JNIEnv *, jclass, jobject);
/*
* Class: org_thoughtcrime_securesms_util_FileUtils
* Method: createMemoryFileDescriptor
* Signature: (Ljava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_org_thoughtcrime_securesms_util_FileUtils_createMemoryFileDescriptor
(JNIEnv *, jclass, jstring);
#ifdef __cplusplus
}
#endif

BIN
libs/arm64-v8a/libnative-utils.so
View file

Binary file not shown.

BIN
libs/armeabi-v7a/libnative-utils.so
View file

Binary file not shown.

BIN
libs/x86/libnative-utils.so
View file

Binary file not shown.

BIN
libs/x86_64/libnative-utils.so
View file

Binary file not shown.

1
res/values/strings.xml
View file

@ -72,6 +72,7 @@
<!-- AttachmentUploadJob -->
<string name="AttachmentUploadJob_uploading_media">Uploading media...</string>
<string name="AttachmentUploadJob_compressing_video_start">Compressing video...</string>
<!-- AudioSlidePlayer -->
<string name="AudioSlidePlayer_error_playing_audio">Error playing audio!</string>

83
src/org/thoughtcrime/securesms/database/AttachmentDatabase.java
View file

@ -16,20 +16,22 @@
*/
package org.thoughtcrime.securesms.database;
import android.annotation.SuppressLint;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.graphics.Bitmap;
import android.media.MediaDataSource;
import android.media.MediaMetadataRetriever;
import android.net.Uri;
import android.os.Build;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import android.text.TextUtils;
import android.util.Pair;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.VisibleForTesting;
import com.bumptech.glide.Glide;
import net.sqlcipher.database.SQLiteDatabase;
@ -495,13 +497,12 @@ public class AttachmentDatabase extends Database {
return insertedAttachments;
}
public @NonNull Attachment updateAttachmentData(@NonNull Attachment attachment,
@NonNull MediaStream mediaStream)
public @NonNull DatabaseAttachment updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment,
@NonNull MediaStream mediaStream)
throws MmsException
{
SQLiteDatabase database = databaseHelper.getWritableDatabase();
DatabaseAttachment databaseAttachment = (DatabaseAttachment) attachment;
DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA);
SQLiteDatabase database = databaseHelper.getWritableDatabase();
DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA);
if (dataInfo == null) {
throw new MmsException("No attachment data found!");
@ -839,8 +840,9 @@ public class AttachmentDatabase extends Database {
Bitmap bitmap = MediaUtil.getVideoThumbnail(context, attachment.getDataUri());
if (bitmap != null) {
ThumbnailData thumbnailData = new ThumbnailData(bitmap);
updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio());
try (ThumbnailData thumbnailData = new ThumbnailData(bitmap)) {
updateAttachmentThumbnail(attachmentId, thumbnailData.toDataStream(), thumbnailData.getAspectRatio());
}
} else {
Log.w(TAG, "Retrieving video thumbnail failed, submitting thumbnail generation job...");
thumbnailExecutor.submit(new ThumbnailFetchCallable(attachmentId));
@ -912,46 +914,53 @@ public class AttachmentDatabase extends Database {
return null;
}
ThumbnailData data = null;
if (MediaUtil.isVideoType(attachment.getContentType())) {
data = generateVideoThumbnail(attachmentId);
try (ThumbnailData data = generateVideoThumbnail(attachmentId)) {
if (data != null) {
updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio());
return getDataStream(attachmentId, THUMBNAIL, 0);
}
}
}
if (data == null) {
return null;
}
updateAttachmentThumbnail(attachmentId, data.toDataStream(), data.getAspectRatio());
return getDataStream(attachmentId, THUMBNAIL, 0);
return null;
}
@SuppressLint("NewApi")
private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) {
private ThumbnailData generateVideoThumbnail(AttachmentId attachmentId) throws IOException {
if (Build.VERSION.SDK_INT < 23) {
Log.w(TAG, "Video thumbnails not supported...");
return null;
}
DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA);
try (MediaDataSource dataSource = mediaDataSourceFor(attachmentId)) {
if (dataSource == null) return null;
if (dataInfo == null) {
Log.w(TAG, "No data file found for video thumbnail...");
return null;
MediaMetadataRetriever retriever = new MediaMetadataRetriever();
retriever.setDataSource(dataSource);
Bitmap bitmap = retriever.getFrameAtTime(1000);
Log.i(TAG, "Generated video thumbnail...");
return bitmap != null ? new ThumbnailData(bitmap) : null;
}
EncryptedMediaDataSource dataSource = new EncryptedMediaDataSource(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length);
MediaMetadataRetriever retriever = new MediaMetadataRetriever();
retriever.setDataSource(dataSource);
Bitmap bitmap = retriever.getFrameAtTime(1000);
Log.i(TAG, "Generated video thumbnail...");
return new ThumbnailData(bitmap);
}
}
@RequiresApi(23)
public @Nullable MediaDataSource mediaDataSourceFor(@NonNull AttachmentId attachmentId) {
DataInfo dataInfo = getAttachmentDataFileInfo(attachmentId, DATA);
if (dataInfo == null) {
Log.w(TAG, "No data file found for video attachment...");
return null;
}
return EncryptedMediaDataSource.createFor(attachmentSecret, dataInfo.file, dataInfo.random, dataInfo.length);
}
private static class DataInfo {
private final File file;
private final long length;

68
src/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.java
View file

@ -18,8 +18,6 @@ import org.thoughtcrime.securesms.jobmanager.Job;
import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.mms.MediaConstraints;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.mms.PartAuthority;
import org.thoughtcrime.securesms.service.GenericForegroundService;
import org.thoughtcrime.securesms.service.NotificationController;
@ -48,13 +46,26 @@ public class AttachmentUploadJob extends BaseJob {
*/
private static final int FOREGROUND_LIMIT = 10 * 1024 * 1024;
private AttachmentId attachmentId;
/**
* The {@link PartProgressEvent} on the {@link EventBus} is shared between transcoding and uploading.
* <p>
* This number is the ratio that represents the transcoding effort, after which it will hand
* over to the to complete the progress.
*/
private static final double ENCODING_PROGRESS_RATIO = 0.75;
public AttachmentUploadJob(AttachmentId attachmentId) {
private final AttachmentId attachmentId;
public static AttachmentUploadJob fromAttachment(DatabaseAttachment databaseAttachment) {
return new AttachmentUploadJob(databaseAttachment.getAttachmentId(), MediaUtil.isVideo(databaseAttachment) && MediaConstraints.isVideoTranscodeAvailable());
}
private AttachmentUploadJob(AttachmentId attachmentId, boolean isVideoTranscode) {
this(new Job.Parameters.Builder()
.addConstraint(NetworkConstraint.KEY)
.setLifespan(TimeUnit.DAYS.toMillis(1))
.setMaxAttempts(Parameters.UNLIMITED)
.setQueue(isVideoTranscode ? "VIDEO_TRANSCODE" : null)
.build(),
attachmentId);
}
@ -86,11 +97,13 @@ public class AttachmentUploadJob extends BaseJob {
throw new IllegalStateException("Cannot find the specified attachment.");
}
MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints();
Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment);
MediaConstraints mediaConstraints = MediaConstraints.getPushMediaConstraints();
Attachment scaledAttachment = scaleAndStripExif(database, mediaConstraints, databaseAttachment);
boolean videoTranscodeOccurred = databaseAttachment != scaledAttachment && MediaUtil.isVideo(scaledAttachment);
double progressStartPoint = videoTranscodeOccurred ? ENCODING_PROGRESS_RATIO : 0;
try (NotificationController notification = getNotificationForAttachment(scaledAttachment)) {
SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification);
SignalServiceAttachment localAttachment = getAttachmentFor(scaledAttachment, notification, progressStartPoint);
SignalServiceAttachmentPointer remoteAttachment = messageSender.uploadAttachment(localAttachment.asStream(), databaseAttachment.isSticker());
Attachment attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.getFastPreflightId()).get();
@ -114,7 +127,12 @@ public class AttachmentUploadJob extends BaseJob {
return exception instanceof IOException;
}
private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification) {
/**
* @param progressStartPoint A value from 0..1 that represents any progress already shown.
* The {@link PartProgressEvent} of this task will fit in the remaining
* 1 - progressStartPoint.
*/
private SignalServiceAttachment getAttachmentFor(Attachment attachment, @Nullable NotificationController notification, double progressStartPoint) {
try {
if (attachment.getDataUri() == null || attachment.getSize() == 0) throw new IOException("Assertion failed, outgoing attachment has no data!");
InputStream is = PartAuthority.getAttachmentStream(context, attachment.getDataUri());
@ -128,7 +146,8 @@ public class AttachmentUploadJob extends BaseJob {
.withHeight(attachment.getHeight())
.withCaption(attachment.getCaption())
.withListener((total, progress) -> {
EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, progress));
long cumulativeProgress = (long) ((1.0 - progressStartPoint) * progress + total * progressStartPoint);
EventBus.getDefault().postSticky(new PartProgressEvent(attachment, total, cumulativeProgress));
if (notification != null) {
notification.setProgress(total, progress);
}
@ -142,26 +161,21 @@ public class AttachmentUploadJob extends BaseJob {
private Attachment scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase,
@NonNull MediaConstraints constraints,
@NonNull Attachment attachment)
@NonNull DatabaseAttachment attachment)
throws UndeliverableMessageException
{
try {
if (constraints.isSatisfied(context, attachment)) {
if (MediaUtil.isJpeg(attachment)) {
MediaStream stripped = constraints.getResizedMedia(context, attachment);
return attachmentDatabase.updateAttachmentData(attachment, stripped);
} else {
return attachment;
}
} else if (constraints.canResize(attachment)) {
MediaStream resized = constraints.getResizedMedia(context, attachment);
return attachmentDatabase.updateAttachmentData(attachment, resized);
} else {
throw new UndeliverableMessageException("Size constraints could not be met!");
}
} catch (IOException | MmsException e) {
throw new UndeliverableMessageException(e);
}
MediaResizer mediaResizer = new MediaResizer(context, constraints);
MediaResizer.ProgressListener progressListener = (progress, total) -> {
PartProgressEvent event = new PartProgressEvent(attachment,
total,
(long) (progress * ENCODING_PROGRESS_RATIO));
EventBus.getDefault().postSticky(event);
};
return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase,
attachment,
progressListener);
}
public static final class Factory implements Job.Factory<AttachmentUploadJob> {

17
src/org/thoughtcrime/securesms/jobs/BaseJob.java
View file

@ -1,6 +1,7 @@
package org.thoughtcrime.securesms.jobs;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.jobmanager.Job;
import org.thoughtcrime.securesms.jobmanager.JobLogger;
@ -33,4 +34,20 @@ public abstract class BaseJob extends Job {
protected abstract void onRun() throws Exception;
protected abstract boolean onShouldRetry(@NonNull Exception e);
protected void log(@NonNull String tag, @NonNull String message) {
Log.i(tag, JobLogger.format(this, message));
}
protected void warn(@NonNull String tag, @NonNull String message) {
warn(tag, message, null);
}
protected void warn(@NonNull String tag, @Nullable Throwable t) {
warn(tag, "", t);
}
protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) {
Log.w(tag, JobLogger.format(this, message), t);
}
}

142
src/org/thoughtcrime/securesms/jobs/MediaResizer.java Normal file
View file

@ -0,0 +1,142 @@
package org.thoughtcrime.securesms.jobs;
import android.content.Context;
import android.media.MediaDataSource;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader;
import org.thoughtcrime.securesms.mms.MediaConstraints;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.service.GenericForegroundService;
import org.thoughtcrime.securesms.service.NotificationController;
import org.thoughtcrime.securesms.transport.UndeliverableMessageException;
import org.thoughtcrime.securesms.util.BitmapDecodingException;
import org.thoughtcrime.securesms.util.BitmapUtil;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.video.InMemoryTranscoder;
import org.thoughtcrime.securesms.video.videoconverter.BadVideoException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
final class MediaResizer {
@NonNull private final Context context;
@NonNull private final MediaConstraints constraints;
MediaResizer(@NonNull Context context,
@NonNull MediaConstraints constraints)
{
this.context = context;
this.constraints = constraints;
}
List<Attachment> scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase,
@NonNull List<Attachment> attachments)
throws UndeliverableMessageException
{
List<Attachment> results = new ArrayList<>(attachments.size());
for (Attachment attachment : attachments) {
results.add(scaleAndStripExifToDatabase(attachmentDatabase, (DatabaseAttachment) attachment, null));
}
return results;
}
DatabaseAttachment scaleAndStripExifToDatabase(@NonNull AttachmentDatabase attachmentDatabase,
@NonNull DatabaseAttachment attachment,
@Nullable ProgressListener transcodeProgressListener)
throws UndeliverableMessageException
{
try {
if (MediaUtil.isVideo(attachment) && MediaConstraints.isVideoTranscodeAvailable()) {
return transcodeVideoIfNeededToDatabase(attachmentDatabase, attachment, transcodeProgressListener);
} else if (constraints.isSatisfied(context, attachment)) {
if (MediaUtil.isJpeg(attachment)) {
MediaStream stripped = getResizedMedia(context, attachment);
return attachmentDatabase.updateAttachmentData(attachment, stripped);
} else {
return attachment;
}
} else if (constraints.canResize(attachment)) {
MediaStream resized = getResizedMedia(context, attachment);
return attachmentDatabase.updateAttachmentData(attachment, resized);
} else {
throw new UndeliverableMessageException("Size constraints could not be met!");
}
} catch (IOException | MmsException e) {
throw new UndeliverableMessageException(e);
}
}
@RequiresApi(26)
private @NonNull DatabaseAttachment transcodeVideoIfNeededToDatabase(@NonNull AttachmentDatabase attachmentDatabase,
@NonNull DatabaseAttachment attachment,
@Nullable ProgressListener progressListener)
throws UndeliverableMessageException
{
try (NotificationController notification = GenericForegroundService.startForegroundTask(context, context.getString(R.string.AttachmentUploadJob_compressing_video_start))) {
notification.setIndeterminateProgress();
try (MediaDataSource dataSource = attachmentDatabase.mediaDataSourceFor(attachment.getAttachmentId())) {
if (dataSource == null) {
throw new UndeliverableMessageException("Cannot get media data source for attachment.");
}
try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, constraints.getCompressedVideoMaxSize(context))) {
if (transcoder.isTranscodeRequired()) {
MediaStream mediaStream = transcoder.transcode(percent -> {
notification.setProgress(100, percent);
if (progressListener != null) {
progressListener.onProgress(percent, 100);
}
});
return attachmentDatabase.updateAttachmentData(attachment, mediaStream);
} else {
return attachment;
}
}
}
} catch (IOException | MmsException | BadVideoException e) {
throw new UndeliverableMessageException("Failed to transcode", e);
}
}
private MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment)
throws IOException
{
if (!constraints.canResize(attachment)) {
throw new UnsupportedOperationException("Cannot resize this content type");
}
try {
// XXX - This is loading everything into memory! We want the send path to be stream-like.
BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableStreamUriLoader.DecryptableUri(attachment.getDataUri()), constraints);
return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight());
} catch (BitmapDecodingException e) {
throw new IOException(e);
}
}
public interface ProgressListener {
void onProgress(long progress, long total);
}
}

2
src/org/thoughtcrime/securesms/jobs/PushGroupSendJob.java
View file

@ -94,7 +94,7 @@ public class PushGroupSendJob extends PushSendJob {
attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList());
attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList());
List<AttachmentUploadJob> attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList();
List<AttachmentUploadJob> attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList();
if (attachmentJobs.isEmpty()) {
jobManager.add(new PushGroupSendJob(messageId, destination, filterAddress));

2
src/org/thoughtcrime/securesms/jobs/PushMediaSendJob.java
View file

@ -81,7 +81,7 @@ public class PushMediaSendJob extends PushSendJob {
attachments.addAll(Stream.of(message.getLinkPreviews()).filter(p -> p.getThumbnail().isPresent()).map(p -> p.getThumbnail().get()).toList());
attachments.addAll(Stream.of(message.getSharedContacts()).filter(c -> c.getAvatar() != null).map(c -> c.getAvatar().getAttachment()).withoutNulls().toList());
List<AttachmentUploadJob> attachmentJobs = Stream.of(attachments).map(a -> new AttachmentUploadJob(((DatabaseAttachment) a).getAttachmentId())).toList();
List<AttachmentUploadJob> attachmentJobs = Stream.of(attachments).map(a -> AttachmentUploadJob.fromAttachment((DatabaseAttachment) a)).toList();
if (attachmentJobs.isEmpty()) {
jobManager.add(new PushMediaSendJob(messageId, destination));

47
src/org/thoughtcrime/securesms/jobs/SendJob.java
View file

@ -1,7 +1,6 @@
package org.thoughtcrime.securesms.jobs;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.BuildConfig;
import org.thoughtcrime.securesms.TextSecureExpiredException;
@ -9,17 +8,11 @@ import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.database.AttachmentDatabase;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.jobmanager.Job;
import org.thoughtcrime.securesms.jobmanager.JobLogger;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.mms.MediaConstraints;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.transport.UndeliverableMessageException;
import org.thoughtcrime.securesms.util.MediaUtil;
import org.thoughtcrime.securesms.util.Util;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
public abstract class SendJob extends BaseJob {
@ -58,45 +51,9 @@ public abstract class SendJob extends BaseJob {
@NonNull List<Attachment> attachments)
throws UndeliverableMessageException
{
MediaResizer mediaResizer = new MediaResizer(context, constraints);
AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context);
List<Attachment> results = new LinkedList<>();
for (Attachment attachment : attachments) {
try {
if (constraints.isSatisfied(context, attachment)) {
if (MediaUtil.isJpeg(attachment)) {
MediaStream stripped = constraints.getResizedMedia(context, attachment);
results.add(attachmentDatabase.updateAttachmentData(attachment, stripped));
} else {
results.add(attachment);
}
} else if (constraints.canResize(attachment)) {
MediaStream resized = constraints.getResizedMedia(context, attachment);
results.add(attachmentDatabase.updateAttachmentData(attachment, resized));
} else {
throw new UndeliverableMessageException("Size constraints could not be met!");
}
} catch (IOException | MmsException e) {
throw new UndeliverableMessageException(e);
}
}
return results;
}
protected void log(@NonNull String tag, @NonNull String message) {
Log.i(tag, JobLogger.format(this, message));
}
protected void warn(@NonNull String tag, @NonNull String message) {
warn(tag, message, null);
}
protected void warn(@NonNull String tag, @Nullable Throwable t) {
warn(tag, "", t);
}
protected void warn(@NonNull String tag, @NonNull String message, @Nullable Throwable t) {
Log.w(tag, JobLogger.format(this, message), t);
return mediaResizer.scaleAndStripExifToDatabase(attachmentDatabase, attachments);
}
}

2
src/org/thoughtcrime/securesms/mediasend/MediaSendViewModel.java
View file

@ -466,7 +466,7 @@ class MediaSendViewModel extends ViewModel {
.filter(m -> {
return (MediaUtil.isImageType(m.getMimeType()) && !MediaUtil.isGif(m.getMimeType())) ||
(MediaUtil.isGif(m.getMimeType()) && m.getSize() < mediaConstraints.getGifMaxSize(context)) ||
(MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getVideoMaxSize(context));
(MediaUtil.isVideoType(m.getMimeType()) && m.getSize() < mediaConstraints.getUncompressedVideoMaxSize(context));
}).toList();
}

40
src/org/thoughtcrime/securesms/mms/MediaConstraints.java
View file

@ -2,18 +2,18 @@ package org.thoughtcrime.securesms.mms;
import android.content.Context;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.logging.Log;
import android.os.Build;
import android.util.Pair;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.mms.DecryptableStreamUriLoader.DecryptableUri;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.util.BitmapDecodingException;
import org.thoughtcrime.securesms.util.BitmapUtil;
import org.thoughtcrime.securesms.util.MediaUtil;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -34,6 +34,15 @@ public abstract class MediaConstraints {
public abstract int getGifMaxSize(Context context);
public abstract int getVideoMaxSize(Context context);
public int getUncompressedVideoMaxSize(Context context) {
return getVideoMaxSize(context);
}
public int getCompressedVideoMaxSize(Context context) {
return getVideoMaxSize(context);
}
public abstract int getAudioMaxSize(Context context);
public abstract int getDocumentMaxSize(Context context);
@ -61,23 +70,12 @@ public abstract class MediaConstraints {
}
}
public boolean canResize(@Nullable Attachment attachment) {
return attachment != null && MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment);
public boolean canResize(@NonNull Attachment attachment) {
return MediaUtil.isImage(attachment) && !MediaUtil.isGif(attachment) ||
MediaUtil.isVideo(attachment) && isVideoTranscodeAvailable();
}
public MediaStream getResizedMedia(@NonNull Context context, @NonNull Attachment attachment)
throws IOException
{
if (!canResize(attachment)) {
throw new UnsupportedOperationException("Cannot resize this content type");
}
try {
// XXX - This is loading everything into memory! We want the send path to be stream-like.
BitmapUtil.ScaleResult scaleResult = BitmapUtil.createScaledBytes(context, new DecryptableUri(attachment.getDataUri()), this);
return new MediaStream(new ByteArrayInputStream(scaleResult.getBitmap()), MediaUtil.IMAGE_JPEG, scaleResult.getWidth(), scaleResult.getHeight());
} catch (BitmapDecodingException e) {
throw new IOException(e);
}
public static boolean isVideoTranscodeAvailable() {
return Build.VERSION.SDK_INT >= 26;
}
}

5
src/org/thoughtcrime/securesms/mms/MmsMediaConstraints.java
View file

@ -39,6 +39,11 @@ final class MmsMediaConstraints extends MediaConstraints {
return getMaxMessageSize(context);
}
@Override
public int getUncompressedVideoMaxSize(Context context) {
return Math.max(getVideoMaxSize(context), 15 * 1024 * 1024);
}
@Override
public int getAudioMaxSize(Context context) {
return getMaxMessageSize(context);

12
src/org/thoughtcrime/securesms/mms/PushMediaConstraints.java
View file

@ -36,6 +36,18 @@ public class PushMediaConstraints extends MediaConstraints {
return 100 * MB;
}
@Override
public int getUncompressedVideoMaxSize(Context context) {
return isVideoTranscodeAvailable() ? 200 * MB
: getVideoMaxSize(context);
}
@Override
public int getCompressedVideoMaxSize(Context context) {
return Util.isLowMemory(context) ? 30 * MB
: 50 * MB;
}
@Override
public int getAudioMaxSize(Context context) {
return 100 * MB;

2
src/org/thoughtcrime/securesms/mms/Slide.java
View file

@ -105,7 +105,7 @@ public abstract class Slide {
public @NonNull String getContentDescription() { return ""; }
public Attachment asAttachment() {
public @NonNull Attachment asAttachment() {
return attachment;
}

40
src/org/thoughtcrime/securesms/sms/MessageSender.java
View file

@ -20,6 +20,8 @@ import android.app.Application;
import android.content.Context;
import androidx.annotation.NonNull;
import com.annimon.stream.Stream;
import org.thoughtcrime.securesms.attachments.AttachmentId;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.database.MessagingDatabase.SyncMessageId;
@ -133,14 +135,14 @@ public class MessageSender {
return;
}
ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context);
MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context);
AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context);
List<List<AttachmentId>> attachmentIds = new ArrayList<>(messages.get(0).getAttachments().size());
List<Long> messageIds = new ArrayList<>(messages.size());
ThreadDatabase threadDatabase = DatabaseFactory.getThreadDatabase(context);
MmsDatabase mmsDatabase = DatabaseFactory.getMmsDatabase(context);
AttachmentDatabase attachmentDatabase = DatabaseFactory.getAttachmentDatabase(context);
List<List<DatabaseAttachment>> databaseAttachments = new ArrayList<>(messages.get(0).getAttachments().size());
List<Long> messageIds = new ArrayList<>(messages.size());
for (int i = 0; i < messages.get(0).getAttachments().size(); i++) {
attachmentIds.add(new ArrayList<>(messages.size()));
databaseAttachments.add(new ArrayList<>(messages.size()));
}
try {
@ -152,13 +154,13 @@ public class MessageSender {
long messageId = mmsDatabase.insertMessageOutbox(message, allocatedThreadId, false, null);
List<DatabaseAttachment> attachments = attachmentDatabase.getAttachmentsForMessage(messageId);
if (attachments.size() != attachmentIds.size()) {
Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + attachmentIds.size() + " Actual: "+ attachments.size());
if (attachments.size() != databaseAttachments.size()) {
Log.w(TAG, "Got back an attachment list that was a different size than expected. Expected: " + databaseAttachments.size() + " Actual: "+ attachments.size());
return;
}
for (int i = 0; i < attachments.size(); i++) {
attachmentIds.get(i).add(attachments.get(i).getAttachmentId());
databaseAttachments.get(i).add(attachments.get(i));
}
messageIds.add(messageId);
@ -169,16 +171,20 @@ public class MessageSender {
mmsDatabase.endTransaction();
}
List<AttachmentUploadJob> uploadJobs = new ArrayList<>(attachmentIds.size());
List<AttachmentCopyJob> copyJobs = new ArrayList<>(attachmentIds.size());
List<Job> messageJobs = new ArrayList<>(attachmentIds.get(0).size());
List<AttachmentUploadJob> uploadJobs = new ArrayList<>(databaseAttachments.size());
List<AttachmentCopyJob> copyJobs = new ArrayList<>(databaseAttachments.size());
List<Job> messageJobs = new ArrayList<>(databaseAttachments.get(0).size());
for (List<AttachmentId> idList : attachmentIds) {
uploadJobs.add(new AttachmentUploadJob(idList.get(0)));
for (List<DatabaseAttachment> attachmentList : databaseAttachments) {
DatabaseAttachment source = attachmentList.get(0);
if (idList.size() > 1) {
AttachmentId sourceId = idList.get(0);
List<AttachmentId> destinationIds = idList.subList(1, idList.size());
uploadJobs.add(AttachmentUploadJob.fromAttachment(source));
if (attachmentList.size() > 1) {
AttachmentId sourceId = source.getAttachmentId();
List<AttachmentId> destinationIds = Stream.of(attachmentList.subList(1, attachmentList.size()))
.map(DatabaseAttachment::getAttachmentId)
.toList();
copyJobs.add(new AttachmentCopyJob(sourceId, destinationIds));
}

4
src/org/thoughtcrime/securesms/util/FileUtils.java
View file

@ -7,7 +7,7 @@ import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class FileUtils {
public final class FileUtils {
static {
System.loadLibrary("native-utils");
@ -15,6 +15,8 @@ public class FileUtils {
public static native int getFileDescriptorOwner(FileDescriptor fileDescriptor);
static native int createMemoryFileDescriptor(String name);
public static byte[] getFileDigest(FileInputStream fin) throws IOException {
try {
MessageDigest digest = MessageDigest.getInstance("SHA256");

16
src/org/thoughtcrime/securesms/util/MediaUtil.java
View file

@ -280,16 +280,17 @@ public class MediaUtil {
return sections.length > 1 ? sections[0] : null;
}
public static class ThumbnailData {
Bitmap bitmap;
float aspectRatio;
public static class ThumbnailData implements AutoCloseable {
public ThumbnailData(Bitmap bitmap) {
@NonNull private final Bitmap bitmap;
private final float aspectRatio;
public ThumbnailData(@NonNull Bitmap bitmap) {
this.bitmap = bitmap;
this.aspectRatio = (float) bitmap.getWidth() / (float) bitmap.getHeight();
}
public Bitmap getBitmap() {
public @NonNull Bitmap getBitmap() {
return bitmap;
}
@ -300,5 +301,10 @@ public class MediaUtil {
public InputStream toDataStream() {
return BitmapUtil.toCompressedJpeg(bitmap);
}
@Override
public void close() {
bitmap.recycle();
}
}
}

158
src/org/thoughtcrime/securesms/util/MemoryFileDescriptor.java Normal file
View file

@ -0,0 +1,158 @@
package org.thoughtcrime.securesms.util;
import android.app.ActivityManager;
import android.content.Context;
import android.os.ParcelFileDescriptor;
import androidx.annotation.NonNull;
import org.thoughtcrime.securesms.logging.Log;
import java.io.Closeable;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.text.NumberFormat;
import java.util.Locale;
import java.util.concurrent.atomic.AtomicLong;
public final class MemoryFileDescriptor implements Closeable {
private static final String TAG = Log.tag(MemoryFileDescriptor.class);
private final ParcelFileDescriptor parcelFileDescriptor;
private final AtomicLong sizeEstimate;
/**
* memfd files do not show on the available RAM, so we must track our allocations in addition.
*/
private static long sizeOfAllMemoryFileDescriptors;
private MemoryFileDescriptor(@NonNull ParcelFileDescriptor parcelFileDescriptor, long sizeEstimate) {
this.parcelFileDescriptor = parcelFileDescriptor;
this.sizeEstimate = new AtomicLong(sizeEstimate);
}
/**
* @param debugName The name supplied in name is used as a filename and will be displayed
* as the target of the corresponding symbolic link in the directory
* /proc/self/fd/. The displayed name is always prefixed with memfd:
* and serves only for debugging purposes. Names do not affect the
* behavior of the file descriptor, and as such multiple files can have
* the same name without any side effects.
* @param sizeEstimate An estimated upper bound on this file. This is used to check there will be
* enough RAM available and to register with a global counter of reservations.
* Use zero to avoid RAM check.
* @return MemoryFileDescriptor
* @throws MemoryLimitException If there is not enough available RAM to comfortably fit this file.
* @throws IOException If fails to create a memory file descriptor.
*/
public static MemoryFileDescriptor newMemoryFileDescriptor(@NonNull Context context,
@NonNull String debugName,
long sizeEstimate)
throws MemoryLimitException, IOException
{
if (sizeEstimate < 0) throw new IllegalArgumentException();
if (sizeEstimate > 0) {
ActivityManager activityManager = ServiceUtil.getActivityManager(context);
ActivityManager.MemoryInfo memoryInfo = new ActivityManager.MemoryInfo();
synchronized (MemoryFileDescriptor.class) {
activityManager.getMemoryInfo(memoryInfo);
long remainingRam = memoryInfo.availMem - memoryInfo.threshold - sizeEstimate - sizeOfAllMemoryFileDescriptors;
if (remainingRam <= 0) {
NumberFormat numberFormat = NumberFormat.getInstance(Locale.US);
Log.w(TAG, String.format("Not enough RAM available without taking the system into a low memory state.%n" +
"Available: %s%n" +
"Low memory threshold: %s%n" +
"Requested: %s%n" +
"Total MemoryFileDescriptor limit: %s%n" +
"Shortfall: %s",
numberFormat.format(memoryInfo.availMem),
numberFormat.format(memoryInfo.threshold),
numberFormat.format(sizeEstimate),
numberFormat.format(sizeOfAllMemoryFileDescriptors),
numberFormat.format(remainingRam)
));
throw new MemoryLimitException();
}
sizeOfAllMemoryFileDescriptors += sizeEstimate;
}
}
int fileDescriptor = FileUtils.createMemoryFileDescriptor(debugName);
if (fileDescriptor < 0) {
throw new IOException("Failed to create a memory file descriptor " + fileDescriptor);
}
return new MemoryFileDescriptor(ParcelFileDescriptor.adoptFd(fileDescriptor), sizeEstimate);
}
@Override
public void close() throws IOException {
try {
clearAndRemoveAllocation();
} catch (Exception e) {
Log.w(TAG, "Failed to clear data in MemoryFileDescriptor", e);
} finally {
parcelFileDescriptor.close();
}
}
private void clearAndRemoveAllocation() throws IOException {
clear();
long oldEstimate = sizeEstimate.getAndSet(0);
synchronized (MemoryFileDescriptor.class) {
sizeOfAllMemoryFileDescriptors -= oldEstimate;
}
}
/** Rewinds and clears all bytes. */
private void clear() throws IOException {
long size;
try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) {
FileChannel channel = fileInputStream.getChannel();
size = channel.size();
if (size == 0) return;
channel.position(0);
}
byte[] zeros = new byte[16 * 1024];
try (FileOutputStream output = new FileOutputStream(getFileDescriptor())) {
while (size > 0) {
int limit = (int) Math.min(size, zeros.length);
output.write(zeros, 0, limit);
size -= limit;
}
}
}
public FileDescriptor getFileDescriptor() {
return parcelFileDescriptor.getFileDescriptor();
}
public void seek(long position) throws IOException {
try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) {
fileInputStream.getChannel().position(position);
}
}
public long size() throws IOException {
try (FileInputStream fileInputStream = new FileInputStream(getFileDescriptor())) {
return fileInputStream.getChannel().size();
}
}
}

6
src/org/thoughtcrime/securesms/util/MemoryLimitException.java Normal file
View file

@ -0,0 +1,6 @@
package org.thoughtcrime.securesms.util;
import java.io.IOException;
public final class MemoryLimitException extends IOException {
}

5
src/org/thoughtcrime/securesms/util/ServiceUtil.java
View file

@ -1,6 +1,7 @@
package org.thoughtcrime.securesms.util;
import android.app.Activity;
import android.app.ActivityManager;
import android.app.AlarmManager;
import android.app.NotificationManager;
import android.app.job.JobScheduler;
@ -69,4 +70,8 @@ public class ServiceUtil {
public static @Nullable SubscriptionManager getSubscriptionManager(@NonNull Context context) {
return (SubscriptionManager) context.getSystemService(Context.TELEPHONY_SUBSCRIPTION_SERVICE);
}
public static ActivityManager getActivityManager(@NonNull Context context) {
return (ActivityManager) context.getSystemService(Context.ACTIVITY_SERVICE);
}
}

54
src/org/thoughtcrime/securesms/video/ClassicEncryptedMediaDataSource.java Normal file
View file

@ -0,0 +1,54 @@
package org.thoughtcrime.securesms.video;
import android.media.MediaDataSource;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
import org.thoughtcrime.securesms.util.Util;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@RequiresApi(23)
final class ClassicEncryptedMediaDataSource extends MediaDataSource {
private final AttachmentSecret attachmentSecret;
private final File mediaFile;
private final long length;
ClassicEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, long length) {
this.attachmentSecret = attachmentSecret;
this.mediaFile = mediaFile;
this.length = length;
}
@Override
public int readAt(long position, byte[] bytes, int offset, int length) throws IOException {
try (InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile)) {
byte[] buffer = new byte[4096];
long headerRemaining = position;
while (headerRemaining > 0) {
int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining)));
if (read == -1) return -1;
headerRemaining -= read;
}
return inputStream.read(bytes, offset, length);
}
}
@Override
public long getSize() {
return length;
}
@Override
public void close() {}
}

73
src/org/thoughtcrime/securesms/video/EncryptedMediaDataSource.java
View file

@ -1,78 +1,23 @@
package org.thoughtcrime.securesms.video;
import android.annotation.TargetApi;
import android.media.MediaDataSource;
import android.os.Build;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
import org.thoughtcrime.securesms.util.Util;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@TargetApi(Build.VERSION_CODES.M)
public class EncryptedMediaDataSource extends MediaDataSource {
@RequiresApi(23)
public final class EncryptedMediaDataSource {
private final AttachmentSecret attachmentSecret;
private final File mediaFile;
private final byte[] random;
private final long length;
public EncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) {
this.attachmentSecret = attachmentSecret;
this.mediaFile = mediaFile;
this.random = random;
this.length = length;
}
@Override
public int readAt(long position, byte[] bytes, int offset, int length) throws IOException {
if (random == null) return readAtClassic(position, bytes, offset, length);
else return readAtModern(position, bytes, offset, length);
}
private int readAtClassic(long position, byte[] bytes, int offset, int length) throws IOException {
InputStream inputStream = ClassicDecryptingPartInputStream.createFor(attachmentSecret, mediaFile);
byte[] buffer = new byte[4096];
long headerRemaining = position;
while (headerRemaining > 0) {
int read = inputStream.read(buffer, 0, Util.toIntExact(Math.min((long)buffer.length, headerRemaining)));
if (read == -1) return -1;
headerRemaining -= read;
public static MediaDataSource createFor(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @Nullable byte[] random, long length) {
if (random == null) {
return new ClassicEncryptedMediaDataSource(attachmentSecret, mediaFile, length);
} else {
return new ModernEncryptedMediaDataSource(attachmentSecret, mediaFile, random, length);
}
int returnValue = inputStream.read(bytes, offset, length);
inputStream.close();
return returnValue;
}
private int readAtModern(long position, byte[] bytes, int offset, int length) throws IOException {
assert(random != null);
InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position);
int returnValue = inputStream.read(bytes, offset, length);
inputStream.close();
return returnValue;
}
@Override
public long getSize() throws IOException {
return length;
}
@Override
public void close() throws IOException {
}
}

201
src/org/thoughtcrime/securesms/video/InMemoryTranscoder.java Normal file
View file

@ -0,0 +1,201 @@
package org.thoughtcrime.securesms.video;
import android.content.Context;
import android.media.MediaDataSource;
import android.media.MediaMetadataRetriever;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.util.MimeTypes;
import org.thoughtcrime.securesms.logging.Log;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.transport.UndeliverableMessageException;
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
import org.thoughtcrime.securesms.video.videoconverter.BadVideoException;
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter;
import java.io.Closeable;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.IOException;
import java.text.NumberFormat;
import java.util.Locale;
@RequiresApi(26)
public final class InMemoryTranscoder implements Closeable {
private static final String TAG = Log.tag(InMemoryTranscoder.class);
private static final int MAXIMUM_TARGET_VIDEO_BITRATE = 2_000_000;
private static final int LOW_RES_TARGET_VIDEO_BITRATE = 1_750_000;
private static final int MINIMUM_TARGET_VIDEO_BITRATE = 500_000;
private static final int AUDIO_BITRATE = 192_000;
private static final int OUTPUT_FORMAT = 720;
private static final int LOW_RES_OUTPUT_FORMAT = 480;
private final Context context;
private final MediaDataSource dataSource;
private final long upperSizeLimit;
private final long inSize;
private final long duration;
private final int inputBitRate;
private final int targetVideoBitRate;
private final long memoryFileEstimate;
private final boolean transcodeRequired;
private final long fileSizeEstimate;
private final int outputFormat;
private @Nullable MemoryFileDescriptor memoryFile;
/**
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
*/
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, long upperSizeLimit) throws IOException {
this.context = context;
this.dataSource = dataSource;
final MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
mediaMetadataRetriever.setDataSource(dataSource);
long upperSizeLimitWithMargin = (long) (upperSizeLimit / 1.1);
this.inSize = dataSource.getSize();
this.duration = getDuration(mediaMetadataRetriever);
this.inputBitRate = bitRate(inSize, duration);
this.targetVideoBitRate = getTargetVideoBitRate(upperSizeLimitWithMargin, duration);
this.upperSizeLimit = upperSizeLimit;
this.transcodeRequired = inputBitRate >= targetVideoBitRate * 1.2 || inSize > upperSizeLimit || containsLocation(mediaMetadataRetriever);
if (!transcodeRequired) {
Log.i(TAG, "Video is within 20% of target bitrate, below the size limit and contained no location metadata.");
}
this.fileSizeEstimate = (targetVideoBitRate + AUDIO_BITRATE) * duration / 8000;
this.memoryFileEstimate = (long) (fileSizeEstimate * 1.1);
this.outputFormat = targetVideoBitRate < LOW_RES_TARGET_VIDEO_BITRATE
? LOW_RES_OUTPUT_FORMAT
: OUTPUT_FORMAT;
}
public @NonNull MediaStream transcode(@NonNull Progress progress) throws IOException, UndeliverableMessageException, BadVideoException {
if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder");
float durationSec = duration / 1000f;
NumberFormat numberFormat = NumberFormat.getInstance(Locale.US);
Log.i(TAG, String.format(Locale.US,
"Transcoding:\n" +
"Target bitrate : %s + %s = %s\n" +
"Target format : %dp\n" +
"Video duration : %.1fs\n" +
"Size limit : %s kB\n" +
"Estimate : %s kB\n" +
"Input size : %s kB\n" +
"Input bitrate : %s bps",
numberFormat.format(targetVideoBitRate),
numberFormat.format(AUDIO_BITRATE),
numberFormat.format(targetVideoBitRate + AUDIO_BITRATE),
outputFormat,
durationSec,
numberFormat.format(upperSizeLimit / 1024),
numberFormat.format(fileSizeEstimate / 1024),
numberFormat.format(inSize / 1024),
numberFormat.format(inputBitRate)));
if (fileSizeEstimate > upperSizeLimit) {
throw new UndeliverableMessageException("Size constraints could not be met!");
}
memoryFile = MemoryFileDescriptor.newMemoryFileDescriptor(context,
"TRANSCODE",
memoryFileEstimate);
final long startTime = System.currentTimeMillis();
final FileDescriptor memoryFileFileDescriptor = memoryFile.getFileDescriptor();
final MediaConverter converter = new MediaConverter();
converter.setInput(dataSource);
converter.setOutput(memoryFileFileDescriptor);
converter.setVideoResolution(outputFormat);
converter.setVideoBitrate(targetVideoBitRate);
converter.setAudioBitrate(AUDIO_BITRATE);
converter.setListener(percent -> {
progress.onProgress(percent);
return false;
});
converter.convert();
// output details of the transcoding
long outSize = memoryFile.size();
float encodeDurationSec = (System.currentTimeMillis() - startTime) / 1000f;
Log.i(TAG, String.format(Locale.US,
"Transcoding complete:\n" +
"Transcode time : %.1fs (%.1fx)\n" +
"Output size : %s kB\n" +
" of Original : %.1f%%\n" +
" of Estimate : %.1f%%\n" +
" of Memory : %.1f%%\n" +
"Output bitrate : %s bps",
encodeDurationSec,
durationSec / encodeDurationSec,
numberFormat.format(outSize / 1024),
(outSize * 100d) / inSize,
(outSize * 100d) / fileSizeEstimate,
(outSize * 100d) / memoryFileEstimate,
numberFormat.format(bitRate(outSize, duration))));
if (outSize > upperSizeLimit) {
throw new UndeliverableMessageException("Size constraints could not be met!");
}
memoryFile.seek(0);
return new MediaStream(new FileInputStream(memoryFileFileDescriptor), MimeTypes.VIDEO_MP4, 0, 0);
}
public boolean isTranscodeRequired() {
return transcodeRequired;
}
@Override
public void close() throws IOException {
if (memoryFile != null) {
memoryFile.close();
}
}
private static int bitRate(long bytes, long duration) {
return (int) (bytes * 8 / (duration / 1000f));
}
private static int getTargetVideoBitRate(long sizeGuideBytes, long duration) {
sizeGuideBytes -= (duration / 1000d) * AUDIO_BITRATE / 8;
double targetAttachmentSizeBits = sizeGuideBytes * 8L;
double bitRateToFixTarget = targetAttachmentSizeBits / (duration / 1000d);
return Math.max(MINIMUM_TARGET_VIDEO_BITRATE, Math.min(MAXIMUM_TARGET_VIDEO_BITRATE, (int) bitRateToFixTarget));
}
private static long getDuration(MediaMetadataRetriever mediaMetadataRetriever) {
return Long.parseLong(mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION));
}
private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetriever) {
String locationString = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_LOCATION);
return locationString != null;
}
public interface Progress {
void onProgress(int percent);
}
}

71
src/org/thoughtcrime/securesms/video/ModernEncryptedMediaDataSource.java Normal file
View file

@ -0,0 +1,71 @@
package org.thoughtcrime.securesms.video;
import android.media.MediaDataSource;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
/**
* Create via {@link EncryptedMediaDataSource}.
* <p>
* A {@link MediaDataSource} that points to an encrypted file.
* <p>
* It is "modern" compared to the {@link ClassicEncryptedMediaDataSource}. And "modern" refers to
* the presence of a random part of the key supplied in the constructor.
*/
@RequiresApi(23)
final class ModernEncryptedMediaDataSource extends MediaDataSource {
private final AttachmentSecret attachmentSecret;
private final File mediaFile;
private final byte[] random;
private final long length;
ModernEncryptedMediaDataSource(@NonNull AttachmentSecret attachmentSecret, @NonNull File mediaFile, @NonNull byte[] random, long length) {
this.attachmentSecret = attachmentSecret;
this.mediaFile = mediaFile;
this.random = random;
this.length = length;
}
@Override
public int readAt(long position, byte[] bytes, int offset, int length) throws IOException {
try (InputStream inputStream = ModernDecryptingPartInputStream.createFor(attachmentSecret, random, mediaFile, position)) {
int totalRead = 0;
while (length > 0) {
int read = inputStream.read(bytes, offset, length);
if (read == -1) {
if (totalRead == 0) {
return -1;
} else {
return totalRead;
}
}
length -= read;
offset += read;
totalRead += read;
}
return totalRead;
}
}
@Override
public long getSize() {
return length;
}
@Override
public void close() {
}
}