diff --git a/app/build.gradle b/app/build.gradle index 79b81a1b47..c7a26458c4 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -302,7 +302,7 @@ dependencies { implementation 'org.signal:argon2:13.1@aar' - implementation 'org.signal:ringrtc-android:2.0.1' + implementation 'org.signal:ringrtc-android:2.0.3' implementation "me.leolin:ShortcutBadger:1.1.16" implementation 'se.emilsjolander:stickylistheaders:2.7.0' diff --git a/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java b/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java index 77b6a5082c..5705e754ec 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java +++ b/app/src/main/java/org/thoughtcrime/securesms/WebRtcCallActivity.java @@ -35,7 +35,6 @@ import android.view.Window; import android.view.WindowManager; import androidx.annotation.NonNull; -import androidx.annotation.StringRes; import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; import androidx.appcompat.widget.AppCompatTextView; @@ -61,7 +60,6 @@ import org.thoughtcrime.securesms.util.FeatureFlags; import org.thoughtcrime.securesms.util.TextSecurePreferences; import org.thoughtcrime.securesms.util.VerifySpan; import org.thoughtcrime.securesms.util.ViewUtil; -import org.webrtc.SurfaceViewRenderer; import org.whispersystems.libsignal.IdentityKey; import org.whispersystems.libsignal.SignalProtocolAddress; import org.whispersystems.signalservice.api.messages.calls.HangupMessage; diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/SurfaceTextureEglRenderer.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/SurfaceTextureEglRenderer.java new file mode 100644 index 0000000000..d87112a96d --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/SurfaceTextureEglRenderer.java @@ -0,0 +1,142 @@ +package org.thoughtcrime.securesms.components.webrtc; + +import android.graphics.SurfaceTexture; +import android.view.TextureView; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.thoughtcrime.securesms.logging.Log; +import org.webrtc.EglBase; +import org.webrtc.EglRenderer; +import org.webrtc.RendererCommon; +import org.webrtc.ThreadUtils; +import org.webrtc.VideoFrame; + +import java.util.concurrent.CountDownLatch; + +/** + * This class is a modified copy of {@link org.webrtc.SurfaceViewRenderer} designed to work with a + * {@link SurfaceTexture} to facilitate easier animation, rounding, elevation, etc. + */ +public class SurfaceTextureEglRenderer extends EglRenderer implements TextureView.SurfaceTextureListener { + + private static final String TAG = Log.tag(SurfaceTextureEglRenderer.class); + + private final Object layoutLock = new Object(); + + private RendererCommon.RendererEvents rendererEvents; + private boolean isFirstFrameRendered; + private boolean isRenderingPaused; + private int rotatedFrameWidth; + private int rotatedFrameHeight; + private int frameRotation; + + public SurfaceTextureEglRenderer(@NonNull String name) { + super(name); + } + + public void init(@Nullable EglBase.Context sharedContext, @Nullable RendererCommon.RendererEvents rendererEvents, @NonNull int[] configAttributes, @NonNull RendererCommon.GlDrawer drawer) { + ThreadUtils.checkIsOnMainThread(); + this.rendererEvents = rendererEvents; + synchronized (this.layoutLock) { + this.isFirstFrameRendered = false; + this.rotatedFrameWidth = 0; + this.rotatedFrameHeight = 0; + this.frameRotation = 0; + } + + super.init(sharedContext, configAttributes, drawer); + } + + @Override + public void init(@Nullable EglBase.Context sharedContext, @NonNull int[] configAttributes, @NonNull RendererCommon.GlDrawer drawer) { + this.init(sharedContext, null, configAttributes, drawer); + } + + @Override + public void setFpsReduction(float fps) { + synchronized(this.layoutLock) { + this.isRenderingPaused = fps == 0.0F; + } + + super.setFpsReduction(fps); + } + + @Override + public void disableFpsReduction() { + synchronized(this.layoutLock) { + this.isRenderingPaused = false; + } + + super.disableFpsReduction(); + } + + @Override + public void pauseVideo() { + synchronized(this.layoutLock) { + this.isRenderingPaused = true; + } + + super.pauseVideo(); + } + + @Override + public void onFrame(@NonNull VideoFrame frame) { + this.updateFrameDimensionsAndReportEvents(frame); + super.onFrame(frame); + } + + @Override + public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) { + ThreadUtils.checkIsOnMainThread(); + createEglSurface(surface); + } + + @Override + public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) { + ThreadUtils.checkIsOnMainThread(); + Log.d(TAG, "onSurfaceTextureSizeChanged: size: " + width + "x" + height); + } + + @Override + public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) { + ThreadUtils.checkIsOnMainThread(); + + CountDownLatch completionLatch = new CountDownLatch(1); + + releaseEglSurface(completionLatch::countDown); + ThreadUtils.awaitUninterruptibly(completionLatch); + + return true; + } + + @Override + public void onSurfaceTextureUpdated(SurfaceTexture surface) { + } + + private void updateFrameDimensionsAndReportEvents(VideoFrame frame) { + synchronized(this.layoutLock) { + if (!this.isRenderingPaused) { + if (!this.isFirstFrameRendered) { + this.isFirstFrameRendered = true; + Log.d(TAG, "Reporting first rendered frame."); + if (this.rendererEvents != null) { + this.rendererEvents.onFirstFrameRendered(); + } + } + + if (this.rotatedFrameWidth != frame.getRotatedWidth() || this.rotatedFrameHeight != frame.getRotatedHeight() || this.frameRotation != frame.getRotation()) { + Log.d(TAG, "Reporting frame resolution changed to " + frame.getBuffer().getWidth() + "x" + frame.getBuffer().getHeight() + " with rotation " + frame.getRotation()); + if (this.rendererEvents != null) { + this.rendererEvents.onFrameResolutionChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight(), frame.getRotation()); + } + + this.rotatedFrameWidth = frame.getRotatedWidth(); + this.rotatedFrameHeight = frame.getRotatedHeight(); + this.frameRotation = frame.getRotation(); + } + } + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java new file mode 100644 index 0000000000..b017a246a2 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/TextureViewRenderer.java @@ -0,0 +1,257 @@ +package org.thoughtcrime.securesms.components.webrtc; + +import android.content.Context; +import android.content.res.Resources; +import android.graphics.Point; +import android.graphics.SurfaceTexture; +import android.os.Looper; +import android.util.AttributeSet; +import android.view.TextureView; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import org.thoughtcrime.securesms.logging.Log; +import org.webrtc.EglBase; +import org.webrtc.EglRenderer; +import org.webrtc.GlRectDrawer; +import org.webrtc.RendererCommon; +import org.webrtc.ThreadUtils; +import org.webrtc.VideoFrame; +import org.webrtc.VideoSink; + +/** + * This class is a modified version of {@link org.webrtc.SurfaceViewRenderer} which is based on {@link TextureView} + */ +public class TextureViewRenderer extends TextureView implements TextureView.SurfaceTextureListener, VideoSink, RendererCommon.RendererEvents { + + private static final String TAG = Log.tag(TextureViewRenderer.class); + + private final SurfaceTextureEglRenderer eglRenderer; + private final RendererCommon.VideoLayoutMeasure videoLayoutMeasure = new RendererCommon.VideoLayoutMeasure(); + + private RendererCommon.RendererEvents rendererEvents; + private int rotatedFrameWidth; + private int rotatedFrameHeight; + private boolean enableFixedSize; + private int surfaceWidth; + private int surfaceHeight; + + public TextureViewRenderer(@NonNull Context context) { + super(context); + this.eglRenderer = new SurfaceTextureEglRenderer(getResourceName()); + this.setSurfaceTextureListener(this); + } + + public TextureViewRenderer(@NonNull Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + this.eglRenderer = new SurfaceTextureEglRenderer(getResourceName()); + this.setSurfaceTextureListener(this); + } + + public void init(@NonNull EglBase.Context sharedContext, @NonNull RendererCommon.RendererEvents rendererEvents) { + this.init(sharedContext, rendererEvents, EglBase.CONFIG_PLAIN, new GlRectDrawer()); + } + + public void init(@NonNull EglBase.Context sharedContext, @NonNull RendererCommon.RendererEvents rendererEvents, @NonNull int[] configAttributes, @NonNull RendererCommon.GlDrawer drawer) { + ThreadUtils.checkIsOnMainThread(); + + this.rendererEvents = rendererEvents; + this.rotatedFrameWidth = 0; + this.rotatedFrameHeight = 0; + + this.eglRenderer.init(sharedContext, this, configAttributes, drawer); + } + + public void release() { + eglRenderer.release(); + } + + public void addFrameListener(@NonNull EglRenderer.FrameListener listener, float scale, @NonNull RendererCommon.GlDrawer drawerParam) { + eglRenderer.addFrameListener(listener, scale, drawerParam); + } + + public void addFrameListener(@NonNull EglRenderer.FrameListener listener, float scale) { + eglRenderer.addFrameListener(listener, scale); + } + + public void removeFrameListener(@NonNull EglRenderer.FrameListener listener) { + eglRenderer.removeFrameListener(listener); + } + + public void setEnableHardwareScaler(boolean enabled) { + ThreadUtils.checkIsOnMainThread(); + + enableFixedSize = enabled; + + updateSurfaceSize(); + } + + public void setMirror(boolean mirror) { + eglRenderer.setMirror(mirror); + } + + public void setScalingType(@NonNull RendererCommon.ScalingType scalingType) { + ThreadUtils.checkIsOnMainThread(); + + videoLayoutMeasure.setScalingType(scalingType); + + requestLayout(); + } + + public void setScalingType(@NonNull RendererCommon.ScalingType scalingTypeMatchOrientation, + @NonNull RendererCommon.ScalingType scalingTypeMismatchOrientation) + { + ThreadUtils.checkIsOnMainThread(); + + videoLayoutMeasure.setScalingType(scalingTypeMatchOrientation, scalingTypeMismatchOrientation); + + requestLayout(); + } + + public void setFpsReduction(float fps) { + eglRenderer.setFpsReduction(fps); + } + + public void disableFpsReduction() { + eglRenderer.disableFpsReduction(); + } + + public void pauseVideo() { + eglRenderer.pauseVideo(); + } + + @Override + protected void onMeasure(int widthSpec, int heightSpec) { + ThreadUtils.checkIsOnMainThread(); + + Point size = videoLayoutMeasure.measure(widthSpec, heightSpec, this.rotatedFrameWidth, this.rotatedFrameHeight); + + setMeasuredDimension(size.x, size.y); + + Log.d(TAG, "onMeasure(). New size: " + size.x + "x" + size.y); + } + + @Override + protected void onLayout(boolean changed, int left, int top, int right, int bottom) { + ThreadUtils.checkIsOnMainThread(); + + eglRenderer.setLayoutAspectRatio((float)(right - left) / (float)(bottom - top)); + + updateSurfaceSize(); + } + + private void updateSurfaceSize() { + ThreadUtils.checkIsOnMainThread(); + + if (!isAvailable()) { + return; + } + + if (this.enableFixedSize && this.rotatedFrameWidth != 0 && this.rotatedFrameHeight != 0 && this.getWidth() != 0 && this.getHeight() != 0) { + + float layoutAspectRatio = (float)this.getWidth() / (float)this.getHeight(); + float frameAspectRatio = (float)this.rotatedFrameWidth / (float)this.rotatedFrameHeight; + + int drawnFrameWidth; + int drawnFrameHeight; + + if (frameAspectRatio > layoutAspectRatio) { + drawnFrameWidth = (int)((float)this.rotatedFrameHeight * layoutAspectRatio); + drawnFrameHeight = this.rotatedFrameHeight; + } else { + drawnFrameWidth = this.rotatedFrameWidth; + drawnFrameHeight = (int)((float)this.rotatedFrameWidth / layoutAspectRatio); + } + + int width = Math.min(this.getWidth(), drawnFrameWidth); + int height = Math.min(this.getHeight(), drawnFrameHeight); + + Log.d(TAG, "updateSurfaceSize. Layout size: " + this.getWidth() + "x" + this.getHeight() + ", frame size: " + this.rotatedFrameWidth + "x" + this.rotatedFrameHeight + ", requested surface size: " + width + "x" + height + ", old surface size: " + this.surfaceWidth + "x" + this.surfaceHeight); + + if (width != this.surfaceWidth || height != this.surfaceHeight) { + this.surfaceWidth = width; + this.surfaceHeight = height; + getSurfaceTexture().setDefaultBufferSize(width, height); + } + } else { + this.surfaceWidth = this.surfaceHeight = 0; + this.getSurfaceTexture().setDefaultBufferSize(getMeasuredWidth(), getMeasuredHeight()); + } + } + + @Override + public void onFirstFrameRendered() { + if (this.rendererEvents != null) { + this.rendererEvents.onFirstFrameRendered(); + } + } + + @Override + public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation) { + if (this.rendererEvents != null) { + this.rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation); + } + + int rotatedWidth = rotation != 0 && rotation != 180 ? videoHeight : videoWidth; + int rotatedHeight = rotation != 0 && rotation != 180 ? videoWidth : videoHeight; + this.postOrRun(() -> { + this.rotatedFrameWidth = rotatedWidth; + this.rotatedFrameHeight = rotatedHeight; + this.updateSurfaceSize(); + this.requestLayout(); + }); + } + + @Override + public void onFrame(VideoFrame videoFrame) { + eglRenderer.onFrame(videoFrame); + } + + @Override + public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) { + ThreadUtils.checkIsOnMainThread(); + + surfaceWidth = 0; + surfaceHeight = 0; + + updateSurfaceSize(); + + eglRenderer.onSurfaceTextureAvailable(surface, width, height); + } + + @Override + public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) { + eglRenderer.onSurfaceTextureSizeChanged(surface, width, height); + } + + @Override + public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) { + return eglRenderer.onSurfaceTextureDestroyed(surface); + } + + @Override + public void onSurfaceTextureUpdated(SurfaceTexture surface) { + } + + private String getResourceName() { + try { + return this.getResources().getResourceEntryName(this.getId()); + } catch (Resources.NotFoundException var2) { + return ""; + } + } + + public void clearImage() { + this.eglRenderer.clearImage(); + } + + private void postOrRun(Runnable r) { + if (Thread.currentThread() == Looper.getMainLooper().getThread()) { + r.run(); + } else { + this.post(r); + } + + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java index a02a7405f1..98804c190d 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java +++ b/app/src/main/java/org/thoughtcrime/securesms/components/webrtc/WebRtcCallView.java @@ -36,7 +36,6 @@ import org.thoughtcrime.securesms.ringrtc.CameraState; import org.thoughtcrime.securesms.util.AvatarUtil; import org.thoughtcrime.securesms.util.ViewUtil; import org.webrtc.RendererCommon; -import org.webrtc.SurfaceViewRenderer; import org.whispersystems.signalservice.api.messages.calls.HangupMessage; public class WebRtcCallView extends FrameLayout { @@ -46,7 +45,7 @@ public class WebRtcCallView extends FrameLayout { public static final int FADE_OUT_DELAY = 5000; - private SurfaceViewRenderer localRenderer; + private TextureViewRenderer localRenderer; private Group ongoingCallButtons; private Group incomingCallButtons; private Group answerWithVoiceGroup; @@ -202,7 +201,7 @@ public class WebRtcCallView extends FrameLayout { } } - public void setLocalRenderer(@Nullable SurfaceViewRenderer surfaceViewRenderer) { + public void setLocalRenderer(@Nullable TextureViewRenderer surfaceViewRenderer) { if (localRenderer == surfaceViewRenderer) { return; } @@ -218,12 +217,11 @@ public class WebRtcCallView extends FrameLayout { } } - public void setRemoteRenderer(@Nullable SurfaceViewRenderer surfaceViewRenderer) { - setRenderer(remoteRenderContainer, surfaceViewRenderer); + public void setRemoteRenderer(@Nullable TextureViewRenderer remoteRenderer) { + setRenderer(remoteRenderContainer, remoteRenderer); } public void setLocalRenderState(WebRtcLocalRenderState localRenderState) { - boolean enableZOverlay = localRenderState == WebRtcLocalRenderState.SMALL; videoToggle.setChecked(localRenderState != WebRtcLocalRenderState.GONE, false); @@ -254,10 +252,6 @@ public class WebRtcCallView extends FrameLayout { setRenderer(smallLocalRenderContainer, localRenderer); } } - - if (localRenderer != null) { - localRenderer.setZOrderMediaOverlay(enableZOverlay); - } } public void setCameraDirection(@NonNull CameraState.Direction cameraDirection) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java b/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java index bf70ce11a8..ef99346822 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java +++ b/app/src/main/java/org/thoughtcrime/securesms/events/WebRtcViewModel.java @@ -3,6 +3,7 @@ package org.thoughtcrime.securesms.events; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import org.thoughtcrime.securesms.components.webrtc.TextureViewRenderer; import org.thoughtcrime.securesms.recipients.Recipient; import org.thoughtcrime.securesms.ringrtc.CameraState; import org.webrtc.SurfaceViewRenderer; @@ -43,16 +44,16 @@ public class WebRtcViewModel { private final boolean isRemoteVideoOffer; private final CameraState localCameraState; - private final SurfaceViewRenderer localRenderer; - private final SurfaceViewRenderer remoteRenderer; + private final TextureViewRenderer localRenderer; + private final TextureViewRenderer remoteRenderer; private final long callConnectedTime; public WebRtcViewModel(@NonNull State state, @NonNull Recipient recipient, @NonNull CameraState localCameraState, - @NonNull SurfaceViewRenderer localRenderer, - @NonNull SurfaceViewRenderer remoteRenderer, + @NonNull TextureViewRenderer localRenderer, + @NonNull TextureViewRenderer remoteRenderer, boolean remoteVideoEnabled, boolean isBluetoothAvailable, boolean isMicrophoneEnabled, @@ -76,8 +77,8 @@ public class WebRtcViewModel { @NonNull Recipient recipient, @Nullable IdentityKey identityKey, @NonNull CameraState localCameraState, - @NonNull SurfaceViewRenderer localRenderer, - @NonNull SurfaceViewRenderer remoteRenderer, + @NonNull TextureViewRenderer localRenderer, + @NonNull TextureViewRenderer remoteRenderer, boolean remoteVideoEnabled, boolean isBluetoothAvailable, boolean isMicrophoneEnabled, @@ -129,11 +130,11 @@ public class WebRtcViewModel { return isRemoteVideoOffer; } - public SurfaceViewRenderer getLocalRenderer() { + public TextureViewRenderer getLocalRenderer() { return localRenderer; } - public SurfaceViewRenderer getRemoteRenderer() { + public TextureViewRenderer getRemoteRenderer() { return remoteRenderer; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java index d5dc158e93..2c1b4d2f65 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java +++ b/app/src/main/java/org/thoughtcrime/securesms/service/WebRtcCallService.java @@ -5,6 +5,7 @@ import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; import android.content.IntentFilter; +import android.graphics.Bitmap; import android.media.AudioManager; import android.net.Uri; import android.os.Build; @@ -25,6 +26,7 @@ import org.signal.ringrtc.CallManager.CallEvent; import org.signal.ringrtc.Remote; import org.thoughtcrime.securesms.ApplicationContext; import org.thoughtcrime.securesms.WebRtcCallActivity; +import org.thoughtcrime.securesms.components.webrtc.TextureViewRenderer; import org.thoughtcrime.securesms.crypto.UnidentifiedAccessUtil; import org.thoughtcrime.securesms.database.DatabaseFactory; import org.thoughtcrime.securesms.database.RecipientDatabase.VibrateState; @@ -56,6 +58,7 @@ import org.thoughtcrime.securesms.webrtc.audio.OutgoingRinger; import org.thoughtcrime.securesms.webrtc.audio.SignalAudioManager; import org.thoughtcrime.securesms.webrtc.locks.LockManager; import org.webrtc.EglBase; +import org.webrtc.EglRenderer; import org.webrtc.IceCandidate; import org.webrtc.PeerConnection; import org.webrtc.SurfaceViewRenderer; @@ -181,8 +184,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, @Nullable private CallManager callManager; @Nullable private RemotePeer activePeer; - @Nullable private SurfaceViewRenderer localRenderer; - @Nullable private SurfaceViewRenderer remoteRenderer; + @Nullable private TextureViewRenderer localRenderer; + @Nullable private TextureViewRenderer remoteRenderer; @Nullable private EglBase eglBase; @Nullable private Camera camera; @@ -1207,8 +1210,8 @@ public class WebRtcCallService extends Service implements CallManager.Observer, Util.runOnMainSync(() -> { eglBase = EglBase.create(); - localRenderer = new SurfaceViewRenderer(WebRtcCallService.this); - remoteRenderer = new SurfaceViewRenderer(WebRtcCallService.this); + localRenderer = new TextureViewRenderer(WebRtcCallService.this); + remoteRenderer = new TextureViewRenderer(WebRtcCallService.this); localRenderer.init(eglBase.getEglBaseContext(), null); remoteRenderer.init(eglBase.getEglBaseContext(), null); diff --git a/app/src/main/res/layout/webrtc_call_view.xml b/app/src/main/res/layout/webrtc_call_view.xml index 78a8a32919..e781dc423e 100644 --- a/app/src/main/res/layout/webrtc_call_view.xml +++ b/app/src/main/res/layout/webrtc_call_view.xml @@ -65,14 +65,16 @@ app:layout_constraintTop_toTopOf="parent" app:layout_constraintVertical_bias="0"> - @@ -90,7 +92,7 @@ android:paddingEnd="9dp" android:paddingBottom="10dp" android:src="@drawable/ic_switch_camera_32" /> - +