+ * The constructor takes a Surface obtained from MediaCodec.createInputSurface(), and uses that + * to create an EGL window surface. Calls to eglSwapBuffers() cause a frame of data to be sent + * to the video encoder. + */ +final class InputSurface { + private static final String TAG = "InputSurface"; + private static final boolean VERBOSE = false; + + private static final int EGL_RECORDABLE_ANDROID = 0x3142; + private static final int EGL_OPENGL_ES2_BIT = 4; + + private EGLDisplay mEGLDisplay; + private EGLContext mEGLContext; + private EGLSurface mEGLSurface; + + private Surface mSurface; + + /** + * Creates an InputSurface from a Surface. + */ + public InputSurface(Surface surface) { + if (surface == null) { + throw new NullPointerException(); + } + mSurface = surface; + + eglSetup(); + } + + /** + * Prepares EGL. We want a GLES 2.0 context and a surface that supports recording. + */ + private void eglSetup() { + mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); + if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) { + throw new RuntimeException("unable to get EGL14 display"); + } + int[] version = new int[2]; + if (!EGL14.eglInitialize(mEGLDisplay, version, 0, version, 1)) { + mEGLDisplay = null; + throw new RuntimeException("unable to initialize EGL14"); + } + + // Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits + // to be able to tell if the frame is reasonable. + int[] attribList = { + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RECORDABLE_ANDROID, 1, + EGL14.EGL_NONE + }; + EGLConfig[] configs = new EGLConfig[1]; + int[] numConfigs = new int[1]; + if (!EGL14.eglChooseConfig(mEGLDisplay, attribList, 0, configs, 0, configs.length, + numConfigs, 0)) { + throw new RuntimeException("unable to find RGB888+recordable ES2 EGL config"); + } + + // Configure context for OpenGL ES 2.0. + int[] attrib_list = { + EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, + EGL14.EGL_NONE + }; + mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT, + attrib_list, 0); + checkEglError("eglCreateContext"); + if (mEGLContext == null) { + throw new RuntimeException("null context"); + } + + // Create a window surface, and attach it to the Surface we received. + int[] surfaceAttribs = { + EGL14.EGL_NONE + }; + mEGLSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, configs[0], mSurface, + surfaceAttribs, 0); + checkEglError("eglCreateWindowSurface"); + if (mEGLSurface == null) { + throw new RuntimeException("surface was null"); + } + } + + /** + * Discard all resources held by this class, notably the EGL context. Also releases the + * Surface that was passed to our constructor. + */ + public void release() { + if (EGL14.eglGetCurrentContext().equals(mEGLContext)) { + // Clear the current context and surface to ensure they are discarded immediately. + EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_CONTEXT); + } + EGL14.eglDestroySurface(mEGLDisplay, mEGLSurface); + EGL14.eglDestroyContext(mEGLDisplay, mEGLContext); + //EGL14.eglTerminate(mEGLDisplay); + + mSurface.release(); + + // null everything out so future attempts to use this object will cause an NPE + mEGLDisplay = null; + mEGLContext = null; + mEGLSurface = null; + + mSurface = null; + } + + /** + * Makes our EGL context and surface current. + */ + public void makeCurrent() { + if (!EGL14.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) { + throw new RuntimeException("eglMakeCurrent failed"); + } + } + + /** + * Calls eglSwapBuffers. Use this to "publish" the current frame. + */ + public boolean swapBuffers() { + return EGL14.eglSwapBuffers(mEGLDisplay, mEGLSurface); + } + + /** + * Returns the Surface that the MediaCodec receives buffers from. + */ + public Surface getSurface() { + return mSurface; + } + + /** + * Sends the presentation time stamp to EGL. Time is expressed in nanoseconds. + */ + public void setPresentationTime(long nsecs) { + EGLExt.eglPresentationTimeANDROID(mEGLDisplay, mEGLSurface, nsecs); + } + + /** + * Checks for EGL errors. + */ + private void checkEglError(String msg) { + boolean failed = false; + int error; + while ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) { + Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error)); + failed = true; + } + if (failed) { + throw new RuntimeException("EGL error encountered (see log)"); + } + } +} diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java b/src/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java new file mode 100644 index 0000000000..89b344d07b --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/videoconverter/MediaConverter.java @@ -0,0 +1,409 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file has been modified by Signal. + */ + +package org.thoughtcrime.securesms.video.videoconverter; + +import android.content.Context; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaDataSource; +import android.media.MediaExtractor; +import android.media.MediaFormat; +import android.net.Uri; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.annotation.StringDef; +import androidx.annotation.WorkerThread; + +import org.thoughtcrime.securesms.logging.Log; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@SuppressWarnings("WeakerAccess") +public final class MediaConverter { + private static final String TAG = "media-converter"; + private static final boolean VERBOSE = false; // lots of logging + + // Describes when the annotation will be discarded + @Retention(RetentionPolicy.SOURCE) + @StringDef({VIDEO_CODEC_H264, VIDEO_CODEC_H265}) + public @interface VideoCodec {} + public static final String VIDEO_CODEC_H264 = "video/avc"; + public static final String VIDEO_CODEC_H265 = "video/hevc"; + + private Input mInput; + private Output mOutput; + + private long mTimeFrom; + private long mTimeTo; + private int mVideoResolution; + private int mVideoBitrate = 2000000; // 2Mbps + private @VideoCodec String mVideoCodec = VIDEO_CODEC_H264; + private int mAudioBitrate = 128000; // 128Kbps + + private Listener mListener; + private boolean mCancelled; + + public interface Listener { + boolean onProgress(int percent); + } + + public MediaConverter() { + } + + @SuppressWarnings("unused") + public void setInput(final @NonNull File file) { + mInput = new FileInput(file); + } + + @SuppressWarnings("unused") + public void setInput(final @NonNull Context context, final @NonNull Uri uri) { + mInput = new UriInput(context, uri); + } + + @RequiresApi(23) + @SuppressWarnings("unused") + public void setInput(final @NonNull MediaDataSource mediaDataSource) { + mInput = new MediaDataSourceInput(mediaDataSource); + } + + @SuppressWarnings("unused") + public void setOutput(final @NonNull File file) { + mOutput = new FileOutput(file); + } + + @SuppressWarnings("unused") + @RequiresApi(26) + public void setOutput(final @NonNull FileDescriptor fileDescriptor) { + mOutput = new FileDescriptorOutput(fileDescriptor); + } + + @SuppressWarnings("unused") + public void setTimeRange(long timeFrom, long timeTo) { + mTimeFrom = timeFrom; + mTimeTo = timeTo; + + if (timeTo > 0 && timeFrom >= timeTo) { + throw new IllegalArgumentException("timeFrom:" + timeFrom + " timeTo:" + timeTo); + } + } + + @SuppressWarnings("unused") + public void setVideoResolution(int videoResolution) { + mVideoResolution = videoResolution; + } + + @SuppressWarnings("unused") + public void setVideoCodec(final @VideoCodec String videoCodec) throws FileNotFoundException { + if (selectCodec(videoCodec) == null) { + throw new FileNotFoundException(); + } + mVideoCodec = videoCodec; + } + + @SuppressWarnings("unused") + public void setVideoBitrate(final int videoBitrate) { + mVideoBitrate = videoBitrate; + } + + @SuppressWarnings("unused") + public void setAudioBitrate(final int audioBitrate) { + mAudioBitrate = audioBitrate; + } + + @SuppressWarnings("unused") + public void setListener(final Listener listener) { + mListener = listener; + } + + @WorkerThread + public void convert() throws BadVideoException, IOException { + // Exception that may be thrown during release. + Exception exception = null; + Muxer muxer = null; + VideoTrackConverter videoTrackConverter = null; + AudioTrackConverter audioTrackConverter = null; + + try { + videoTrackConverter = VideoTrackConverter.create(mInput, mTimeFrom, mTimeTo, mVideoResolution, mVideoBitrate, mVideoCodec); + audioTrackConverter = AudioTrackConverter.create(mInput, mTimeFrom, mTimeTo, mAudioBitrate); + + if (videoTrackConverter == null && audioTrackConverter == null) { + Log.e(TAG, "no video and audio tracks"); + throw new BadVideoException(); + } + + muxer = mOutput.createMuxer(); + + doExtractDecodeEditEncodeMux( + videoTrackConverter, + audioTrackConverter, + muxer); + + } catch (BadVideoException | IOException e) { + Log.e(TAG, "error converting", e); + exception = e; + throw e; + } catch (Exception e) { + Log.e(TAG, "error converting", e); + exception = e; + } finally { + if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); + // Try to release everything we acquired, even if one of the releases fails, in which + // case we save the first exception we got and re-throw at the end (unless something + // other exception has already been thrown). This guarantees the first exception thrown + // is reported as the cause of the error, everything is (attempted) to be released, and + // all other exceptions appear in the logs. + try { + if (videoTrackConverter != null) { + videoTrackConverter.release(); + } + } catch (Exception e) { + if (exception == null) { + exception = e; + } + } + try { + if (audioTrackConverter != null) { + audioTrackConverter.release(); + } + } catch (Exception e) { + if (exception == null) { + exception = e; + } + } + try { + if (muxer != null) { + muxer.stop(); + muxer.release(); + } + } catch (Exception e) { + Log.e(TAG, "error while releasing muxer", e); + if (exception == null) { + exception = e; + } + } + } + if (exception != null) { + throw new RuntimeException(exception); + } + } + + /** + * Does the actual work for extracting, decoding, encoding and muxing. + */ + private void doExtractDecodeEditEncodeMux( + final @Nullable VideoTrackConverter videoTrackConverter, + final @Nullable AudioTrackConverter audioTrackConverter, + final @NonNull Muxer muxer) throws IOException { + + boolean muxing = false; + int percentProcessed = 0; + long inputDuration = Math.max( + videoTrackConverter == null ? 0 : videoTrackConverter.mInputDuration, + audioTrackConverter == null ? 0 : audioTrackConverter.mInputDuration); + + while (!mCancelled && + ((videoTrackConverter != null && !videoTrackConverter.mVideoEncoderDone) || + (audioTrackConverter != null &&!audioTrackConverter.mAudioEncoderDone))) { + + if (VERBOSE) { + Log.d(TAG, "loop: " + + (videoTrackConverter == null ? "" : videoTrackConverter.dumpState()) + + (audioTrackConverter == null ? "" : audioTrackConverter.dumpState()) + + " muxing:" + muxing); + } + + if (videoTrackConverter != null && (audioTrackConverter == null || audioTrackConverter.mAudioExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime <= audioTrackConverter.mMuxingAudioPresentationTime)) { + videoTrackConverter.step(); + } + + if (audioTrackConverter != null && (videoTrackConverter == null || videoTrackConverter.mVideoExtractorDone || videoTrackConverter.mMuxingVideoPresentationTime >= audioTrackConverter.mMuxingAudioPresentationTime)) { + audioTrackConverter.step(); + } + + if (inputDuration != 0 && mListener != null) { + final long timeFromUs = mTimeFrom <= 0 ? 0 : mTimeFrom * 1000; + final long timeToUs = mTimeTo <= 0 ? inputDuration : mTimeTo * 1000; + final int curPercentProcessed = (int) (100 * + (Math.max( + videoTrackConverter == null ? 0 : videoTrackConverter.mMuxingVideoPresentationTime, + audioTrackConverter == null ? 0 : audioTrackConverter.mMuxingAudioPresentationTime) + - timeFromUs) / (timeToUs - timeFromUs)); + + if (curPercentProcessed != percentProcessed) { + percentProcessed = curPercentProcessed; + mCancelled = mCancelled || mListener.onProgress(percentProcessed); + } + } + + if (!muxing + && (videoTrackConverter == null || videoTrackConverter.mEncoderOutputVideoFormat != null) + && (audioTrackConverter == null || audioTrackConverter.mEncoderOutputAudioFormat != null)) { + if (videoTrackConverter != null) { + videoTrackConverter.setMuxer(muxer); + } + if (audioTrackConverter != null) { + audioTrackConverter.setMuxer(muxer); + } + Log.d(TAG, "muxer: starting"); + muxer.start(); + muxing = true; + } + } + + // Basic sanity checks. + if (videoTrackConverter != null) { + videoTrackConverter.verifyEndState(); + } + if (audioTrackConverter != null) { + audioTrackConverter.verifyEndState(); + } + + // TODO: Check the generated output file. + } + + static String getMimeTypeFor(MediaFormat format) { + return format.getString(MediaFormat.KEY_MIME); + } + + /** + * Returns the first codec capable of encoding the specified MIME type, or null if no match was + * found. + */ + static MediaCodecInfo selectCodec(final String mimeType) { + final int numCodecs = MediaCodecList.getCodecCount(); + for (int i = 0; i < numCodecs; i++) { + final MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); + + if (!codecInfo.isEncoder()) { + continue; + } + + final String[] types = codecInfo.getSupportedTypes(); + for (String type : types) { + if (type.equalsIgnoreCase(mimeType)) { + return codecInfo; + } + } + } + return null; + } + + interface Input { + @NonNull + MediaExtractor createExtractor() throws IOException; + } + + private static class FileInput implements Input { + + final File file; + + FileInput(final @NonNull File file) { + this.file = file; + } + + @Override + public @NonNull + MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(file.getAbsolutePath()); + return extractor; + } + } + + private static class UriInput implements Input { + + final Uri uri; + final Context context; + + UriInput(final @NonNull Context context, final @NonNull Uri uri) { + this.uri = uri; + this.context = context; + } + + @Override + public @NonNull + MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(context, uri, null); + return extractor; + } + } + + @RequiresApi(23) + private static class MediaDataSourceInput implements Input { + + private final MediaDataSource mediaDataSource; + + MediaDataSourceInput(final @NonNull MediaDataSource mediaDataSource) { + this.mediaDataSource = mediaDataSource; + } + + @Override + public @NonNull + MediaExtractor createExtractor() throws IOException { + final MediaExtractor extractor = new MediaExtractor(); + extractor.setDataSource(mediaDataSource); + return extractor; + } + } + + interface Output { + @NonNull + Muxer createMuxer() throws IOException; + } + + private static class FileOutput implements Output { + + final File file; + + FileOutput(final @NonNull File file) { + this.file = file; + } + + @Override + public @NonNull + Muxer createMuxer() throws IOException { + return new AndroidMuxer(file); + } + } + + @RequiresApi(26) + private static class FileDescriptorOutput implements Output { + + final FileDescriptor fileDescriptor; + + FileDescriptorOutput(final @NonNull FileDescriptor fileDescriptor) { + this.fileDescriptor = fileDescriptor; + } + + @Override + public @NonNull + Muxer createMuxer() throws IOException { + return new AndroidMuxer(fileDescriptor); + } + } +} diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/Muxer.java b/src/org/thoughtcrime/securesms/video/videoconverter/Muxer.java new file mode 100644 index 0000000000..28c05f338f --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/videoconverter/Muxer.java @@ -0,0 +1,22 @@ +package org.thoughtcrime.securesms.video.videoconverter; + +import android.media.MediaCodec; +import android.media.MediaFormat; + +import androidx.annotation.NonNull; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public interface Muxer { + + void start() throws IOException; + + void stop() throws IOException; + + int addTrack(@NonNull MediaFormat format) throws IOException; + + void writeSampleData(int trackIndex, @NonNull ByteBuffer byteBuf, @NonNull MediaCodec.BufferInfo bufferInfo) throws IOException; + + void release(); +} diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java b/src/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java new file mode 100644 index 0000000000..5c279e6dbb --- /dev/null +++ b/src/org/thoughtcrime/securesms/video/videoconverter/OutputSurface.java @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file has been modified by Signal. + */ + +package org.thoughtcrime.securesms.video.videoconverter; + +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.view.Surface; + +import org.thoughtcrime.securesms.logging.Log; + +import javax.microedition.khronos.egl.EGL10; +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.egl.EGLContext; +import javax.microedition.khronos.egl.EGLDisplay; +import javax.microedition.khronos.egl.EGLSurface; + +/** + * Holds state associated with a Surface used for MediaCodec decoder output. + *
+ * The (width,height) constructor for this class will prepare GL, create a SurfaceTexture, + * and then create a Surface for that SurfaceTexture. The Surface can be passed to + * MediaCodec.configure() to receive decoder output. When a frame arrives, we latch the + * texture with updateTexImage, then render the texture with GL to a pbuffer. + *
+ * The no-arg constructor skips the GL preparation step and doesn't allocate a pbuffer. + * Instead, it just creates the Surface and SurfaceTexture, and when a frame arrives + * we just draw it on whatever surface is current. + *
+ * By default, the Surface will be using a BufferQueue in asynchronous mode, so we
+ * can potentially drop frames.
+ */
+final class OutputSurface implements SurfaceTexture.OnFrameAvailableListener {
+ private static final String TAG = "OutputSurface";
+ private static final boolean VERBOSE = false;
+
+ private static final int EGL_OPENGL_ES2_BIT = 4;
+
+ private EGL10 mEGL;
+ private EGLDisplay mEGLDisplay;
+ private EGLContext mEGLContext;
+ private EGLSurface mEGLSurface;
+
+ private SurfaceTexture mSurfaceTexture;
+ private Surface mSurface;
+
+ private final Object mFrameSyncObject = new Object(); // guards mFrameAvailable
+ private boolean mFrameAvailable;
+
+ private TextureRender mTextureRender;
+
+ /**
+ * Creates an OutputSurface backed by a pbuffer with the specifed dimensions. The new
+ * EGL context and surface will be made current. Creates a Surface that can be passed
+ * to MediaCodec.configure().
+ */
+ public OutputSurface(int width, int height) {
+ if (width <= 0 || height <= 0) {
+ throw new IllegalArgumentException();
+ }
+
+ eglSetup(width, height);
+ makeCurrent();
+
+ setup();
+ }
+
+ /**
+ * Creates an OutputSurface using the current EGL context. Creates a Surface that can be
+ * passed to MediaCodec.configure().
+ */
+ public OutputSurface() {
+ setup();
+ }
+
+ /**
+ * Creates instances of TextureRender and SurfaceTexture, and a Surface associated
+ * with the SurfaceTexture.
+ */
+ private void setup() {
+ mTextureRender = new TextureRender();
+ mTextureRender.surfaceCreated();
+
+ // Even if we don't access the SurfaceTexture after the constructor returns, we
+ // still need to keep a reference to it. The Surface doesn't retain a reference
+ // at the Java level, so if we don't either then the object can get GCed, which
+ // causes the native finalizer to run.
+ if (VERBOSE) Log.d(TAG, "textureID=" + mTextureRender.getTextureId());
+ mSurfaceTexture = new SurfaceTexture(mTextureRender.getTextureId());
+
+ // This doesn't work if OutputSurface is created on the thread that CTS started for
+ // these test cases.
+ //
+ // The CTS-created thread has a Looper, and the SurfaceTexture constructor will
+ // create a Handler that uses it. The "frame available" message is delivered
+ // there, but since we're not a Looper-based thread we'll never see it. For
+ // this to do anything useful, OutputSurface must be created on a thread without
+ // a Looper, so that SurfaceTexture uses the main application Looper instead.
+ //
+ // Java language note: passing "this" out of a constructor is generally unwise,
+ // but we should be able to get away with it here.
+ mSurfaceTexture.setOnFrameAvailableListener(this);
+
+ mSurface = new Surface(mSurfaceTexture);
+ }
+
+ /**
+ * Prepares EGL. We want a GLES 2.0 context and a surface that supports pbuffer.
+ */
+ private void eglSetup(int width, int height) {
+ mEGL = (EGL10)EGLContext.getEGL();
+ mEGLDisplay = mEGL.eglGetDisplay(EGL10.EGL_DEFAULT_DISPLAY);
+ if (!mEGL.eglInitialize(mEGLDisplay, null)) {
+ throw new RuntimeException("unable to initialize EGL10");
+ }
+
+ // Configure EGL for pbuffer and OpenGL ES 2.0. We want enough RGB bits
+ // to be able to tell if the frame is reasonable.
+ int[] attribList = {
+ EGL10.EGL_RED_SIZE, 8,
+ EGL10.EGL_GREEN_SIZE, 8,
+ EGL10.EGL_BLUE_SIZE, 8,
+ EGL10.EGL_SURFACE_TYPE, EGL10.EGL_PBUFFER_BIT,
+ EGL10.EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+ EGL10.EGL_NONE
+ };
+ EGLConfig[] configs = new EGLConfig[1];
+ int[] numConfigs = new int[1];
+ if (!mEGL.eglChooseConfig(mEGLDisplay, attribList, configs, 1, numConfigs)) {
+ throw new RuntimeException("unable to find RGB888+pbuffer EGL config");
+ }
+
+ // Configure context for OpenGL ES 2.0.
+ int[] attrib_list = {
+ EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
+ EGL10.EGL_NONE
+ };
+ mEGLContext = mEGL.eglCreateContext(mEGLDisplay, configs[0], EGL10.EGL_NO_CONTEXT,
+ attrib_list);
+ checkEglError("eglCreateContext");
+ if (mEGLContext == null) {
+ throw new RuntimeException("null context");
+ }
+
+ // Create a pbuffer surface. By using this for output, we can use glReadPixels
+ // to test values in the output.
+ int[] surfaceAttribs = {
+ EGL10.EGL_WIDTH, width,
+ EGL10.EGL_HEIGHT, height,
+ EGL10.EGL_NONE
+ };
+ mEGLSurface = mEGL.eglCreatePbufferSurface(mEGLDisplay, configs[0], surfaceAttribs);
+ checkEglError("eglCreatePbufferSurface");
+ if (mEGLSurface == null) {
+ throw new RuntimeException("surface was null");
+ }
+ }
+
+ /**
+ * Discard all resources held by this class, notably the EGL context.
+ */
+ public void release() {
+ if (mEGL != null) {
+ if (mEGL.eglGetCurrentContext().equals(mEGLContext)) {
+ // Clear the current context and surface to ensure they are discarded immediately.
+ mEGL.eglMakeCurrent(mEGLDisplay, EGL10.EGL_NO_SURFACE, EGL10.EGL_NO_SURFACE,
+ EGL10.EGL_NO_CONTEXT);
+ }
+ mEGL.eglDestroySurface(mEGLDisplay, mEGLSurface);
+ mEGL.eglDestroyContext(mEGLDisplay, mEGLContext);
+ //mEGL.eglTerminate(mEGLDisplay);
+ }
+
+ mSurface.release();
+
+ // this causes a bunch of warnings that appear harmless but might confuse someone:
+ // W BufferQueue: [unnamed-3997-2] cancelBuffer: BufferQueue has been abandoned!
+ //mSurfaceTexture.release();
+
+ // null everything out so future attempts to use this object will cause an NPE
+ mEGLDisplay = null;
+ mEGLContext = null;
+ mEGLSurface = null;
+ mEGL = null;
+
+ mTextureRender = null;
+ mSurface = null;
+ mSurfaceTexture = null;
+ }
+
+ /**
+ * Makes our EGL context and surface current.
+ */
+ public void makeCurrent() {
+ if (mEGL == null) {
+ throw new RuntimeException("not configured for makeCurrent");
+ }
+ checkEglError("before makeCurrent");
+ if (!mEGL.eglMakeCurrent(mEGLDisplay, mEGLSurface, mEGLSurface, mEGLContext)) {
+ throw new RuntimeException("eglMakeCurrent failed");
+ }
+ }
+
+ /**
+ * Returns the Surface that we draw onto.
+ */
+ public Surface getSurface() {
+ return mSurface;
+ }
+
+ /**
+ * Replaces the fragment shader.
+ */
+ public void changeFragmentShader(String fragmentShader) {
+ mTextureRender.changeFragmentShader(fragmentShader);
+ }
+
+ /**
+ * Latches the next buffer into the texture. Must be called from the thread that created
+ * the OutputSurface object, after the onFrameAvailable callback has signaled that new
+ * data is available.
+ */
+ public void awaitNewImage() {
+ final int TIMEOUT_MS = 500;
+
+ synchronized (mFrameSyncObject) {
+ while (!mFrameAvailable) {
+ try {
+ // Wait for onFrameAvailable() to signal us. Use a timeout to avoid
+ // stalling the test if it doesn't arrive.
+ mFrameSyncObject.wait(TIMEOUT_MS);
+ if (!mFrameAvailable) {
+ // TODO: if "spurious wakeup", continue while loop
+ throw new RuntimeException("Surface frame wait timed out");
+ }
+ } catch (InterruptedException ie) {
+ // shouldn't happen
+ throw new RuntimeException(ie);
+ }
+ }
+ mFrameAvailable = false;
+ }
+
+ // Latch the data.
+ mTextureRender.checkGlError("before updateTexImage");
+ mSurfaceTexture.updateTexImage();
+ }
+
+ /**
+ * Draws the data from SurfaceTexture onto the current EGL surface.
+ */
+ public void drawImage() {
+ mTextureRender.drawFrame(mSurfaceTexture);
+ }
+
+ @Override
+ public void onFrameAvailable(SurfaceTexture st) {
+ if (VERBOSE) Log.d(TAG, "new frame available");
+ synchronized (mFrameSyncObject) {
+ if (mFrameAvailable) {
+ throw new RuntimeException("mFrameAvailable already set, frame could be dropped");
+ }
+ mFrameAvailable = true;
+ mFrameSyncObject.notifyAll();
+ }
+ }
+
+ /**
+ * Checks for EGL errors.
+ */
+ private void checkEglError(String msg) {
+ boolean failed = false;
+ int error;
+ while ((error = mEGL.eglGetError()) != EGL10.EGL_SUCCESS) {
+ Log.e(TAG, msg + ": EGL error: 0x" + Integer.toHexString(error));
+ failed = true;
+ }
+ if (failed) {
+ throw new RuntimeException("EGL error encountered (see log)");
+ }
+ }
+}
diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/Preconditions.java b/src/org/thoughtcrime/securesms/video/videoconverter/Preconditions.java
new file mode 100644
index 0000000000..389effd399
--- /dev/null
+++ b/src/org/thoughtcrime/securesms/video/videoconverter/Preconditions.java
@@ -0,0 +1,10 @@
+package org.thoughtcrime.securesms.video.videoconverter;
+
+final class Preconditions {
+
+ static void checkState(final Object errorMessage, final boolean expression) {
+ if (!expression) {
+ throw new IllegalStateException(String.valueOf(errorMessage));
+ }
+ }
+}
diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java b/src/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java
new file mode 100644
index 0000000000..5950f71a83
--- /dev/null
+++ b/src/org/thoughtcrime/securesms/video/videoconverter/TextureRender.java
@@ -0,0 +1,245 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * This file has been modified by Signal.
+ */
+
+package org.thoughtcrime.securesms.video.videoconverter;
+
+import android.graphics.SurfaceTexture;
+import android.opengl.GLES11Ext;
+import android.opengl.GLES20;
+import android.opengl.Matrix;
+
+import org.thoughtcrime.securesms.logging.Log;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
+
+/**
+ * Code for rendering a texture onto a surface using OpenGL ES 2.0.
+ */
+final class TextureRender {
+ private static final String TAG = "TextureRender";
+
+ private static final int FLOAT_SIZE_BYTES = 4;
+ private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
+ private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
+ private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
+ private final float[] mTriangleVerticesData = {
+ // X, Y, Z, U, V
+ -1.0f, -1.0f, 0, 0.f, 0.f,
+ 1.0f, -1.0f, 0, 1.f, 0.f,
+ -1.0f, 1.0f, 0, 0.f, 1.f,
+ 1.0f, 1.0f, 0, 1.f, 1.f,
+ };
+
+ private final FloatBuffer mTriangleVertices;
+
+ private static final String VERTEX_SHADER =
+ "uniform mat4 uMVPMatrix;\n" +
+ "uniform mat4 uSTMatrix;\n" +
+ "attribute vec4 aPosition;\n" +
+ "attribute vec4 aTextureCoord;\n" +
+ "varying vec2 vTextureCoord;\n" +
+ "void main() {\n" +
+ " gl_Position = uMVPMatrix * aPosition;\n" +
+ " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
+ "}\n";
+
+ private static final String FRAGMENT_SHADER =
+ "#extension GL_OES_EGL_image_external : require\n" +
+ "precision mediump float;\n" + // highp here doesn't seem to matter
+ "varying vec2 vTextureCoord;\n" +
+ "uniform samplerExternalOES sTexture;\n" +
+ "void main() {\n" +
+ " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
+ "}\n";
+
+ private final float[] mMVPMatrix = new float[16];
+ private final float[] mSTMatrix = new float[16];
+
+ private int mProgram;
+ private int mTextureID = -12345;
+ private int muMVPMatrixHandle;
+ private int muSTMatrixHandle;
+ private int maPositionHandle;
+ private int maTextureHandle;
+
+ public TextureRender() {
+ mTriangleVertices = ByteBuffer.allocateDirect(
+ mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
+ .order(ByteOrder.nativeOrder()).asFloatBuffer();
+ mTriangleVertices.put(mTriangleVerticesData).position(0);
+
+ Matrix.setIdentityM(mSTMatrix, 0);
+ }
+
+ public int getTextureId() {
+ return mTextureID;
+ }
+
+ public void drawFrame(SurfaceTexture st) {
+ checkGlError("onDrawFrame start");
+ st.getTransformMatrix(mSTMatrix);
+
+ GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
+ GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
+
+ GLES20.glUseProgram(mProgram);
+ checkGlError("glUseProgram");
+
+ GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
+ GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
+
+ mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
+ GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
+ TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
+ checkGlError("glVertexAttribPointer maPosition");
+ GLES20.glEnableVertexAttribArray(maPositionHandle);
+ checkGlError("glEnableVertexAttribArray maPositionHandle");
+
+ mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
+ GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false,
+ TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
+ checkGlError("glVertexAttribPointer maTextureHandle");
+ GLES20.glEnableVertexAttribArray(maTextureHandle);
+ checkGlError("glEnableVertexAttribArray maTextureHandle");
+
+ Matrix.setIdentityM(mMVPMatrix, 0);
+ GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
+ GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
+
+ GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
+ checkGlError("glDrawArrays");
+ GLES20.glFinish();
+ }
+
+ /**
+ * Initializes GL state. Call this after the EGL surface has been created and made current.
+ */
+ public void surfaceCreated() {
+ mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER);
+ if (mProgram == 0) {
+ throw new RuntimeException("failed creating program");
+ }
+ maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
+ checkGlError("glGetAttribLocation aPosition");
+ if (maPositionHandle == -1) {
+ throw new RuntimeException("Could not get attrib location for aPosition");
+ }
+ maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
+ checkGlError("glGetAttribLocation aTextureCoord");
+ if (maTextureHandle == -1) {
+ throw new RuntimeException("Could not get attrib location for aTextureCoord");
+ }
+
+ muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
+ checkGlError("glGetUniformLocation uMVPMatrix");
+ if (muMVPMatrixHandle == -1) {
+ throw new RuntimeException("Could not get attrib location for uMVPMatrix");
+ }
+
+ muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
+ checkGlError("glGetUniformLocation uSTMatrix");
+ if (muSTMatrixHandle == -1) {
+ throw new RuntimeException("Could not get attrib location for uSTMatrix");
+ }
+
+ int[] textures = new int[1];
+ GLES20.glGenTextures(1, textures, 0);
+
+ mTextureID = textures[0];
+ GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID);
+ checkGlError("glBindTexture mTextureID");
+
+ GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
+ GLES20.GL_LINEAR);
+ GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
+ GLES20.GL_LINEAR);
+ GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S,
+ GLES20.GL_CLAMP_TO_EDGE);
+ GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T,
+ GLES20.GL_CLAMP_TO_EDGE);
+ checkGlError("glTexParameter");
+ }
+
+ /**
+ * Replaces the fragment shader.
+ */
+ public void changeFragmentShader(String fragmentShader) {
+ GLES20.glDeleteProgram(mProgram);
+ mProgram = createProgram(VERTEX_SHADER, fragmentShader);
+ if (mProgram == 0) {
+ throw new RuntimeException("failed creating program");
+ }
+ }
+
+ private int loadShader(int shaderType, String source) {
+ int shader = GLES20.glCreateShader(shaderType);
+ checkGlError("glCreateShader type=" + shaderType);
+ GLES20.glShaderSource(shader, source);
+ GLES20.glCompileShader(shader);
+ int[] compiled = new int[1];
+ GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
+ if (compiled[0] == 0) {
+ Log.e(TAG, "Could not compile shader " + shaderType + ":");
+ Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader));
+ GLES20.glDeleteShader(shader);
+ shader = 0;
+ }
+ return shader;
+ }
+
+ private int createProgram(String vertexSource, String fragmentSource) {
+ int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
+ if (vertexShader == 0) {
+ return 0;
+ }
+ int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
+ if (pixelShader == 0) {
+ return 0;
+ }
+
+ int program = GLES20.glCreateProgram();
+ checkGlError("glCreateProgram");
+ if (program == 0) {
+ Log.e(TAG, "Could not create program");
+ }
+ GLES20.glAttachShader(program, vertexShader);
+ checkGlError("glAttachShader");
+ GLES20.glAttachShader(program, pixelShader);
+ checkGlError("glAttachShader");
+ GLES20.glLinkProgram(program);
+ int[] linkStatus = new int[1];
+ GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
+ if (linkStatus[0] != GLES20.GL_TRUE) {
+ Log.e(TAG, "Could not link program: ");
+ Log.e(TAG, GLES20.glGetProgramInfoLog(program));
+ GLES20.glDeleteProgram(program);
+ program = 0;
+ }
+ return program;
+ }
+
+ public void checkGlError(String op) {
+ int error;
+ while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
+ Log.e(TAG, op + ": glError " + error);
+ throw new RuntimeException(op + ": glError " + error);
+ }
+ }
+}
diff --git a/src/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java b/src/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
new file mode 100644
index 0000000000..99ab762aef
--- /dev/null
+++ b/src/org/thoughtcrime/securesms/video/videoconverter/VideoTrackConverter.java
@@ -0,0 +1,501 @@
+package org.thoughtcrime.securesms.video.videoconverter;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaExtractor;
+import android.media.MediaFormat;
+import android.view.Surface;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+
+import org.thoughtcrime.securesms.logging.Log;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicReference;
+
+final class VideoTrackConverter {
+
+ private static final String TAG = "media-converter";
+ private static final boolean VERBOSE = false; // lots of logging
+
+ private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 1; // 1 second between I-frames
+ private static final int OUTPUT_VIDEO_FRAME_RATE = 30; // needed only for MediaFormat.KEY_I_FRAME_INTERVAL to work; the actual frame rate matches the source
+
+ private static final int TIMEOUT_USEC = 10000;
+
+ private final long mTimeFrom;
+ private final long mTimeTo;
+
+ final long mInputDuration;
+
+ private final MediaExtractor mVideoExtractor;
+ private final MediaCodec mVideoDecoder;
+ private final MediaCodec mVideoEncoder;
+
+ private final InputSurface mInputSurface;
+ private final OutputSurface mOutputSurface;
+
+ private final ByteBuffer[] mVideoDecoderInputBuffers;
+ private ByteBuffer[] mVideoEncoderOutputBuffers;
+ private final MediaCodec.BufferInfo mVideoDecoderOutputBufferInfo;
+ private final MediaCodec.BufferInfo mVideoEncoderOutputBufferInfo;
+
+ MediaFormat mEncoderOutputVideoFormat;
+
+ boolean mVideoExtractorDone;
+ private boolean mVideoDecoderDone;
+ boolean mVideoEncoderDone;
+
+ private int mOutputVideoTrack = -1;
+
+ long mMuxingVideoPresentationTime;
+
+ private int mVideoExtractedFrameCount;
+ private int mVideoDecodedFrameCount;
+ private int mVideoEncodedFrameCount;
+
+ private Muxer mMuxer;
+
+ static @Nullable
+ VideoTrackConverter create(
+ final @NonNull MediaConverter.Input input,
+ final long timeFrom,
+ final long timeTo,
+ final int videoResolution,
+ final int videoBitrate,
+ final @NonNull String videoCodec) throws IOException {
+
+ final MediaExtractor videoExtractor = input.createExtractor();
+ final int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor);
+ if (videoInputTrack == -1) {
+ videoExtractor.release();
+ return null;
+ }
+ return new VideoTrackConverter(videoExtractor, videoInputTrack, timeFrom, timeTo, videoResolution, videoBitrate, videoCodec);
+ }
+
+ private VideoTrackConverter(
+ final @NonNull MediaExtractor videoExtractor,
+ final int videoInputTrack,
+ final long timeFrom,
+ final long timeTo,
+ final int videoResolution,
+ final int videoBitrate,
+ final @NonNull String videoCodec) throws IOException {
+
+ mTimeFrom = timeFrom;
+ mTimeTo = timeTo;
+ mVideoExtractor = videoExtractor;
+
+ final MediaCodecInfo videoCodecInfo = MediaConverter.selectCodec(videoCodec);
+ if (videoCodecInfo == null) {
+ // Don't fail CTS if they don't have an AVC codec (not here, anyway).
+ Log.e(TAG, "Unable to find an appropriate codec for " + videoCodec);
+ throw new FileNotFoundException();
+ }
+ if (VERBOSE) Log.d(TAG, "video found codec: " + videoCodecInfo.getName());
+
+ final MediaFormat inputVideoFormat = mVideoExtractor.getTrackFormat(videoInputTrack);
+
+ mInputDuration = inputVideoFormat.containsKey(MediaFormat.KEY_DURATION) ? inputVideoFormat.getLong(MediaFormat.KEY_DURATION) : 0;
+
+ final int rotation = inputVideoFormat.containsKey(MediaFormat.KEY_ROTATION) ? inputVideoFormat.getInteger(MediaFormat.KEY_ROTATION) : 0;
+ final int width = inputVideoFormat.getInteger(MediaFormat.KEY_WIDTH);
+ final int height = inputVideoFormat.getInteger(MediaFormat.KEY_HEIGHT);
+ int outputWidth = width;
+ int outputHeight = height;
+ if (outputWidth < outputHeight) {
+ outputWidth = videoResolution;
+ outputHeight = height * outputWidth / width;
+ } else {
+ outputHeight = videoResolution;
+ outputWidth = width * outputHeight / height;
+ }
+ // many encoders do not work when height and width are not multiple of 16 (also, some iPhones do not play some heights)
+ outputHeight = (outputHeight + 7) & ~0xF;
+ outputWidth = (outputWidth + 7) & ~0xF;
+
+ final int outputWidthRotated;
+ final int outputHeightRotated;
+ if ((rotation % 180 == 90)) {
+ //noinspection SuspiciousNameCombination
+ outputWidthRotated = outputHeight;
+ //noinspection SuspiciousNameCombination
+ outputHeightRotated = outputWidth;
+ } else {
+ outputWidthRotated = outputWidth;
+ outputHeightRotated = outputHeight;
+ }
+
+ final MediaFormat outputVideoFormat = MediaFormat.createVideoFormat(videoCodec, outputWidthRotated, outputHeightRotated);
+
+ // Set some properties. Failing to specify some of these can cause the MediaCodec
+ // configure() call to throw an unhelpful exception.
+ outputVideoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
+ outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, videoBitrate);
+ outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE);
+ outputVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL);
+ if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat);
+
+ // Create a MediaCodec for the desired codec, then configure it as an encoder with
+ // our desired properties. Request a Surface to use for input.
+ final AtomicReference