Experimental HEVC encoding support for videos.

This commit is contained in:
Nicholas Tinsley 2024-08-23 10:09:22 -04:00
parent 5f66e2eb15
commit 0f7f866562
14 changed files with 124 additions and 34 deletions

View file

@ -383,6 +383,19 @@ class InternalSettingsFragment : DSLSettingsFragment(R.string.preferences__inter
dividerPref()
sectionHeaderPref(DSLSettingsText.from("Media"))
switchPref(
title = DSLSettingsText.from("Enable HEVC Encoding for HD Videos"),
summary = DSLSettingsText.from("Videos sent in \"HD\" quality will be encoded in HEVC on compatible devices."),
isChecked = state.hevcEncoding,
onClick = {
viewModel.setHevcEncoding(!state.hevcEncoding)
}
)
dividerPref()
sectionHeaderPref(DSLSettingsText.from("Conversations and Shortcuts"))
clickPref(

View file

@ -23,5 +23,6 @@ data class InternalSettingsState(
val canClearOnboardingState: Boolean,
val pnpInitialized: Boolean,
val useConversationItemV2ForMedia: Boolean,
val hasPendingOneTimeDonation: Boolean
val hasPendingOneTimeDonation: Boolean,
val hevcEncoding: Boolean
)

View file

@ -119,6 +119,11 @@ class InternalSettingsViewModel(private val repository: InternalSettingsReposito
refresh()
}
fun setHevcEncoding(enabled: Boolean) {
SignalStore.internal.hevcEncoding = enabled
refresh()
}
fun addSampleReleaseNote() {
repository.addSampleReleaseNote()
}
@ -159,7 +164,8 @@ class InternalSettingsViewModel(private val repository: InternalSettingsReposito
canClearOnboardingState = SignalStore.story.hasDownloadedOnboardingStory && Stories.isFeatureEnabled(),
pnpInitialized = SignalStore.misc.hasPniInitializedDevices,
useConversationItemV2ForMedia = SignalStore.internal.useConversationItemV2Media(),
hasPendingOneTimeDonation = SignalStore.inAppPayments.getPendingOneTimeDonation() != null
hasPendingOneTimeDonation = SignalStore.inAppPayments.getPendingOneTimeDonation() != null,
hevcEncoding = SignalStore.internal.hevcEncoding
)
fun onClearOnboardingState() {

View file

@ -32,6 +32,7 @@ public final class InternalValues extends SignalStoreValues {
public static final String CONVERSATION_ITEM_V2_MEDIA = "internal.conversation_item_v2_media";
public static final String FORCE_ENTER_RESTORE_V2_FLOW = "internal.force_enter_restore_v2_flow";
public static final String WEB_SOCKET_SHADOWING_STATS = "internal.web_socket_shadowing_stats";
public static final String ENCODE_HEVC = "internal.hevc_encoding";
InternalValues(KeyValueStore store) {
super(store);
@ -183,6 +184,14 @@ public final class InternalValues extends SignalStoreValues {
}
}
public void setHevcEncoding(boolean enabled) {
putBoolean(ENCODE_HEVC, enabled);
}
public boolean getHevcEncoding() {
return getBoolean(ENCODE_HEVC, false);
}
public void setLastScrollPosition(int position) {
putInteger(LAST_SCROLL_POSITION, position);
}

View file

@ -41,10 +41,6 @@ public abstract class MediaConstraints {
return TranscodingPreset.LEVEL_1;
}
public boolean isHighQuality() {
return false;
}
/**
* Provide a list of dimensions that should be attempted during compression. We will keep moving
* down the list until the image can be scaled to fit under {@link #getImageMaxSize(Context)}.

View file

@ -7,10 +7,12 @@ import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import org.thoughtcrime.securesms.dependencies.AppDependencies;
import org.thoughtcrime.securesms.keyvalue.SignalStore;
import org.thoughtcrime.securesms.util.RemoteConfig;
import org.thoughtcrime.securesms.util.LocaleRemoteConfig;
import org.thoughtcrime.securesms.util.Util;
import org.thoughtcrime.securesms.video.TranscodingPreset;
import org.thoughtcrime.securesms.video.videoconverter.utils.DeviceCapabilities;
import java.util.Arrays;
@ -25,11 +27,6 @@ public class PushMediaConstraints extends MediaConstraints {
currentConfig = getCurrentConfig(AppDependencies.getApplication(), sentMediaQuality);
}
@Override
public boolean isHighQuality() {
return currentConfig == MediaConfig.LEVEL_3;
}
@Override
public int getImageMaxWidth(Context context) {
return currentConfig.imageSizeTargets[0];
@ -102,7 +99,11 @@ public class PushMediaConstraints extends MediaConstraints {
}
if (sentMediaQuality == SentMediaQuality.HIGH) {
return MediaConfig.LEVEL_3;
if (DeviceCapabilities.canEncodeHevc() && (RemoteConfig.useHevcEncoder() || SignalStore.internal().getHevcEncoding())) {
return MediaConfig.LEVEL_4;
} else {
return MediaConfig.LEVEL_3;
}
}
return LocaleRemoteConfig.getMediaQualityLevel().orElse(MediaConfig.getDefault(context));
}
@ -112,7 +113,8 @@ public class PushMediaConstraints extends MediaConstraints {
LEVEL_1(false, 1, MB, new int[] { 1600, 1024, 768, 512 }, 70, TranscodingPreset.LEVEL_1),
LEVEL_2(false, 2, (int) (1.5 * MB), new int[] { 2048, 1600, 1024, 768, 512 }, 75, TranscodingPreset.LEVEL_2),
LEVEL_3(false, 3, (int) (3 * MB), new int[] { 4096, 3072, 2048, 1600, 1024, 768, 512 }, 75, TranscodingPreset.LEVEL_3);
LEVEL_3(false, 3, (int) (3 * MB), new int[] { 4096, 3072, 2048, 1600, 1024, 768, 512 }, 75, TranscodingPreset.LEVEL_3),
LEVEL_4(false, 4, 3 * MB, new int[] { 4096, 3072, 2048, 1600, 1024, 768, 512 }, 75, TranscodingPreset.LEVEL_4);
private final boolean isLowMemory;
private final int level;

View file

@ -1110,5 +1110,13 @@ object RemoteConfig {
hotSwappable = false
)
@JvmStatic
@get:JvmName("useHevcEncoder")
val useHevcEncoder: Boolean by remoteBoolean(
key = "android.useHevcEncoder",
defaultValue = false,
hotSwappable = false
)
// endregion
}

View file

@ -50,6 +50,7 @@ class TranscodeTestRepository(context: Context) {
if (transcodingPreset != null) {
inputData.putString(TranscodeWorker.KEY_TRANSCODING_PRESET_NAME, transcodingPreset.name)
} else if (customTranscodingOptions != null) {
inputData.putString(TranscodeWorker.KEY_VIDEO_CODEC, customTranscodingOptions.videoCodec)
inputData.putInt(TranscodeWorker.KEY_LONG_EDGE, customTranscodingOptions.videoResolution.longEdge)
inputData.putInt(TranscodeWorker.KEY_SHORT_EDGE, customTranscodingOptions.videoResolution.shortEdge)
inputData.putInt(TranscodeWorker.KEY_VIDEO_BIT_RATE, customTranscodingOptions.videoBitrate)
@ -104,7 +105,7 @@ class TranscodeTestRepository(context: Context) {
}
}
data class CustomTranscodingOptions(val videoResolution: VideoResolution, val videoBitrate: Int, val audioBitrate: Int, val enableFastStart: Boolean, val enableAudioRemux: Boolean)
data class CustomTranscodingOptions(val videoCodec: String, val videoResolution: VideoResolution, val videoBitrate: Int, val audioBitrate: Int, val enableFastStart: Boolean, val enableAudioRemux: Boolean)
companion object {
private const val TAG = "TranscodingTestRepository"

View file

@ -18,6 +18,7 @@ import androidx.work.WorkInfo
import kotlinx.coroutines.flow.Flow
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.TranscodingQuality
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter
import java.util.UUID
import kotlin.math.roundToInt
@ -39,6 +40,7 @@ class TranscodeTestViewModel : ViewModel() {
var videoMegaBitrate by mutableFloatStateOf(calculateVideoMegaBitrateFromPreset(transcodingPreset))
var videoResolution by mutableStateOf(convertPresetToVideoResolution(transcodingPreset))
var audioKiloBitrate by mutableIntStateOf(calculateAudioKiloBitrateFromPreset(transcodingPreset))
var useHevc by mutableStateOf(false)
var useAutoTranscodingSettings by mutableStateOf(true)
var enableFastStart by mutableStateOf(true)
var enableAudioRemux by mutableStateOf(true)
@ -64,6 +66,7 @@ class TranscodeTestViewModel : ViewModel() {
output,
forceSequentialQueueProcessing,
TranscodeTestRepository.CustomTranscodingOptions(
if (useHevc) MediaConverter.VIDEO_CODEC_H265 else MediaConverter.VIDEO_CODEC_H264,
videoResolution,
(videoMegaBitrate * MEGABIT).roundToInt(),
audioKiloBitrate * KILOBIT,

View file

@ -25,6 +25,7 @@ import org.signal.core.util.readLength
import org.thoughtcrime.securesms.video.StreamingTranscoder
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.postprocessing.Mp4FaststartPostProcessor
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter.VideoCodec
import org.thoughtcrime.securesms.video.videoconverter.mediadatasource.InputStreamMediaDataSource
import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants
import org.thoughtcrime.video.app.R
@ -61,7 +62,6 @@ class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(
val finalFilename = "$filenameBase$OUTPUT_FILE_EXTENSION"
setForeground(createForegroundInfo(-1, inputParams.notificationId))
applicationContext.openFileOutput(tempFilename, Context.MODE_PRIVATE).use { outputStream ->
if (outputStream == null) {
Log.w(TAG, "$logPrefix Could not open temp file for I/O!")
@ -80,8 +80,12 @@ class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(
val datasource = WorkerMediaDataSource(File(applicationContext.filesDir, inputFilename))
val transcoder = if (inputParams.resolution > 0 && inputParams.videoBitrate > 0) {
Log.d(TAG, "$logPrefix Initializing StreamingTranscoder with custom parameters: B:V=${inputParams.videoBitrate}, B:A=${inputParams.audioBitrate}, res=${inputParams.resolution}, audioRemux=${inputParams.audioRemux}")
StreamingTranscoder.createManuallyForTesting(datasource, null, inputParams.videoBitrate, inputParams.audioBitrate, inputParams.resolution, inputParams.audioRemux)
if (inputParams.videoCodec == null) {
Log.w(TAG, "$logPrefix Video codec was null!")
return Result.failure()
}
Log.d(TAG, "$logPrefix Initializing StreamingTranscoder with custom parameters: CODEC:${inputParams.videoCodec} B:V=${inputParams.videoBitrate}, B:A=${inputParams.audioBitrate}, res=${inputParams.resolution}, audioRemux=${inputParams.audioRemux}")
StreamingTranscoder.createManuallyForTesting(datasource, null, inputParams.videoCodec, inputParams.videoBitrate, inputParams.audioBitrate, inputParams.resolution, inputParams.audioRemux)
} else if (inputParams.transcodingPreset != null) {
StreamingTranscoder(datasource, null, inputParams.transcodingPreset, DEFAULT_FILE_SIZE_LIMIT, inputParams.audioRemux)
} else {
@ -224,6 +228,8 @@ class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(
val outputDirUri: Uri = Uri.parse(inputData.getString(KEY_OUTPUT_URI))
val postProcessForFastStart: Boolean = inputData.getBoolean(KEY_ENABLE_FASTSTART, true)
val transcodingPreset: TranscodingPreset? = inputData.getString(KEY_TRANSCODING_PRESET_NAME)?.let { TranscodingPreset.valueOf(it) }
@VideoCodec val videoCodec: String? = inputData.getString(KEY_VIDEO_CODEC)
val resolution: Int = inputData.getInt(KEY_SHORT_EDGE, -1)
val videoBitrate: Int = inputData.getInt(KEY_VIDEO_BIT_RATE, -1)
val audioBitrate: Int = inputData.getInt(KEY_AUDIO_BIT_RATE, -1)
@ -239,6 +245,7 @@ class TranscodeWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(
const val KEY_OUTPUT_URI = "output_uri"
const val KEY_TRANSCODING_PRESET_NAME = "transcoding_quality_preset"
const val KEY_PROGRESS = "progress"
const val KEY_VIDEO_CODEC = "video_codec"
const val KEY_LONG_EDGE = "resolution_long_edge"
const val KEY_SHORT_EDGE = "resolution_short_edge"
const val KEY_VIDEO_BIT_RATE = "video_bit_rate"

View file

@ -32,6 +32,7 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.lifecycle.viewmodel.compose.viewModel
import org.thoughtcrime.securesms.video.TranscodingPreset
import org.thoughtcrime.securesms.video.videoconverter.utils.DeviceCapabilities
import org.thoughtcrime.video.app.transcode.MAX_VIDEO_MEGABITRATE
import org.thoughtcrime.video.app.transcode.MIN_VIDEO_MEGABITRATE
import org.thoughtcrime.video.app.transcode.OPTIONS_AUDIO_KILOBITRATES
@ -45,6 +46,7 @@ import kotlin.math.roundToInt
*/
@Composable
fun ConfigureEncodingParameters(
hevcCapable: Boolean = DeviceCapabilities.canEncodeHevc(),
modifier: Modifier = Modifier,
viewModel: TranscodeTestViewModel = viewModel()
) {
@ -104,6 +106,8 @@ fun ConfigureEncodingParameters(
CustomSettings(
selectedResolution = viewModel.videoResolution,
onResolutionSelected = { viewModel.videoResolution = it },
useHevc = viewModel.useHevc,
onUseHevcSettingChanged = { viewModel.useHevc = it },
fastStartChecked = viewModel.enableFastStart,
onFastStartSettingCheckChanged = { viewModel.enableFastStart = it },
audioRemuxChecked = viewModel.enableAudioRemux,
@ -112,6 +116,7 @@ fun ConfigureEncodingParameters(
updateVideoSliderPosition = { viewModel.videoMegaBitrate = it },
audioSliderPosition = viewModel.audioKiloBitrate,
updateAudioSliderPosition = { viewModel.audioKiloBitrate = it.roundToInt() },
hevcCapable = hevcCapable,
modifier = Modifier.padding(vertical = 16.dp)
)
}
@ -169,6 +174,8 @@ private fun PresetPicker(
private fun CustomSettings(
selectedResolution: VideoResolution,
onResolutionSelected: (VideoResolution) -> Unit,
useHevc: Boolean,
onUseHevcSettingChanged: (Boolean) -> Unit,
fastStartChecked: Boolean,
onFastStartSettingCheckChanged: (Boolean) -> Unit,
audioRemuxChecked: Boolean,
@ -177,6 +184,7 @@ private fun CustomSettings(
updateVideoSliderPosition: (Float) -> Unit,
audioSliderPosition: Int,
updateAudioSliderPosition: (Float) -> Unit,
hevcCapable: Boolean,
modifier: Modifier = Modifier
) {
Row(
@ -210,6 +218,22 @@ private fun CustomSettings(
}
VideoBitrateSlider(videoSliderPosition, updateVideoSliderPosition)
AudioBitrateSlider(audioSliderPosition, updateAudioSliderPosition)
if (hevcCapable) {
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
.padding(vertical = 8.dp, horizontal = 8.dp)
.fillMaxWidth()
) {
Checkbox(
checked = useHevc,
onCheckedChange = { onUseHevcSettingChanged(it) }
)
Text(text = "Use HEVC encoder", style = MaterialTheme.typography.bodySmall)
}
}
Row(
verticalAlignment = Alignment.CenterVertically,
modifier = Modifier
@ -297,5 +321,5 @@ private fun ConfigurationScreenPreviewUnchecked() {
val vm: TranscodeTestViewModel = viewModel()
vm.selectedVideos = listOf(Uri.parse("content://1"), Uri.parse("content://2"))
vm.useAutoTranscodingSettings = false
ConfigureEncodingParameters()
ConfigureEncodingParameters(hevcCapable = true)
}

View file

@ -84,6 +84,7 @@ public final class StreamingTranscoder {
private StreamingTranscoder(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
String codec,
int videoBitrate,
int audioBitrate,
int shortEdge,
@ -105,7 +106,7 @@ public final class StreamingTranscoder {
this.inSize = dataSource.getSize();
this.duration = getDuration(mediaMetadataRetriever);
this.inputBitRate = TranscodingQuality.bitRate(inSize, duration);
this.targetQuality = TranscodingQuality.createManuallyForTesting(shortEdge, videoBitrate, audioBitrate, duration);
this.targetQuality = TranscodingQuality.createManuallyForTesting(codec, shortEdge, videoBitrate, audioBitrate, duration);
this.upperSizeLimit = 0L;
this.transcodeRequired = true;
@ -116,13 +117,14 @@ public final class StreamingTranscoder {
@VisibleForTesting
public static StreamingTranscoder createManuallyForTesting(@NonNull MediaDataSource dataSource,
@Nullable TranscoderOptions options,
@NonNull @MediaConverter.VideoCodec String codec,
int videoBitrate,
int audioBitrate,
int shortEdge,
boolean allowAudioRemux)
throws VideoSourceException, IOException
{
return new StreamingTranscoder(dataSource, options, videoBitrate, audioBitrate, shortEdge, allowAudioRemux);
return new StreamingTranscoder(dataSource, options, codec, videoBitrate, audioBitrate, shortEdge, allowAudioRemux);
}
public void transcode(@NonNull Progress progress,
@ -171,6 +173,7 @@ public final class StreamingTranscoder {
outStream = new CountingOutputStream(stream);
}
converter.setOutput(outStream);
converter.setVideoCodec(targetQuality.getCodec());
converter.setVideoResolution(targetQuality.getOutputResolution());
converter.setVideoBitrate(targetQuality.getTargetVideoBitRate());
converter.setAudioBitrate(targetQuality.getTargetAudioBitRate());

View file

@ -4,22 +4,24 @@
*/
package org.thoughtcrime.securesms.video
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter
import org.thoughtcrime.securesms.video.videoconverter.MediaConverter.VideoCodec
import org.thoughtcrime.securesms.video.videoconverter.utils.VideoConstants
/**
* A data class to hold various video transcoding parameters, such as bitrate.
*/
class TranscodingQuality private constructor(val outputResolution: Int, val targetVideoBitRate: Int, val targetAudioBitRate: Int, private val durationMs: Long) {
class TranscodingQuality private constructor(@VideoCodec val codec: String, val outputResolution: Int, val targetVideoBitRate: Int, val targetAudioBitRate: Int, private val durationMs: Long) {
companion object {
@JvmStatic
fun createFromPreset(preset: TranscodingPreset, durationMs: Long): TranscodingQuality {
return TranscodingQuality(preset.videoShortEdge, preset.videoBitRate, preset.audioBitRate, durationMs)
return TranscodingQuality(preset.videoCodec, preset.videoShortEdge, preset.videoBitRate, preset.audioBitRate, durationMs)
}
@JvmStatic
fun createManuallyForTesting(outputShortEdge: Int, videoBitrate: Int, audioBitrate: Int, durationMs: Long): TranscodingQuality {
return TranscodingQuality(outputShortEdge, videoBitrate, audioBitrate, durationMs)
fun createManuallyForTesting(codec: String, outputShortEdge: Int, videoBitrate: Int, audioBitrate: Int, durationMs: Long): TranscodingQuality {
return TranscodingQuality(codec, outputShortEdge, videoBitrate, audioBitrate, durationMs)
}
@JvmStatic
@ -32,19 +34,15 @@ class TranscodingQuality private constructor(val outputResolution: Int, val targ
val byteCountEstimate = (targetTotalBitRate / 8) * (durationMs / 1000)
override fun toString(): String {
return "Quality{" +
"targetVideoBitRate=" + targetVideoBitRate +
", targetAudioBitRate=" + targetAudioBitRate +
", duration=" + durationMs +
", filesize=" + byteCountEstimate +
'}'
return "Quality{codec=$codec, targetVideoBitRate=$targetVideoBitRate, targetAudioBitRate=$targetAudioBitRate, duration=$durationMs, filesize=$byteCountEstimate}"
}
}
enum class TranscodingPreset(val videoShortEdge: Int, val videoBitRate: Int, val audioBitRate: Int) {
LEVEL_1(VideoConstants.VIDEO_SHORT_EDGE_SD, VideoConstants.VIDEO_BITRATE_L1, VideoConstants.AUDIO_BITRATE),
LEVEL_2(VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L2, VideoConstants.AUDIO_BITRATE),
LEVEL_3(VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L3, VideoConstants.AUDIO_BITRATE);
enum class TranscodingPreset(@VideoCodec val videoCodec: String, val videoShortEdge: Int, val videoBitRate: Int, val audioBitRate: Int) {
LEVEL_1(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_SD, VideoConstants.VIDEO_BITRATE_L1, VideoConstants.AUDIO_BITRATE),
LEVEL_2(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L2, VideoConstants.AUDIO_BITRATE),
LEVEL_3(MediaConverter.VIDEO_CODEC_H264, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L3, VideoConstants.AUDIO_BITRATE),
LEVEL_4(MediaConverter.VIDEO_CODEC_H265, VideoConstants.VIDEO_SHORT_EDGE_HD, VideoConstants.VIDEO_BITRATE_L3, VideoConstants.AUDIO_BITRATE);
fun calculateMaxVideoUploadDurationInSeconds(upperFileSizeLimit: Long): Int {
val upperFileSizeLimitWithMargin = (upperFileSizeLimit / 1.1).toLong()

View file

@ -0,0 +1,19 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.video.videoconverter.utils
import android.media.MediaCodecList
import android.media.MediaFormat
import org.signal.core.util.isNotNullOrBlank
object DeviceCapabilities {
@JvmStatic
fun canEncodeHevc(): Boolean {
val mediaCodecList = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val encoder = mediaCodecList.findEncoderForFormat(MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_HEVC, VideoConstants.VIDEO_LONG_EDGE_HD, VideoConstants.VIDEO_SHORT_EDGE_HD))
return encoder.isNotNullOrBlank()
}
}