empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Improve UX around seeing audio wave forms.

Browse source 
- Attempts to generate the wave form on download instead on display
- Allows multi-threaded generation of wave forms instead of serial
  executor
This commit is contained in:
Cody Henthorne 2023-02-15 09:43:16 -05:00 • committed by GitHub
parent d4ce8458a4
commit 9610339f38
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 497 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.idea/codeStyles/Project.xml generated
View file

@ -41,6 +41,11 @@
</value>
</option>
</JavaCodeStyleSettings>
<JetCodeStyleSettings>
<option name="PACKAGES_TO_USE_STAR_IMPORTS">
<value />
</option>
</JetCodeStyleSettings>
<codeStyleSettings language="HTML">
<indentOptions>
<option name="INDENT_SIZE" value="2" />

45
app/src/main/java/org/thoughtcrime/securesms/audio/AudioFileInfo.java Normal file
View file

@ -0,0 +1,45 @@
package org.thoughtcrime.securesms.audio;
import androidx.annotation.NonNull;
import com.google.protobuf.ByteString;
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
import java.util.concurrent.TimeUnit;
public class AudioFileInfo {
private final long durationUs;
private final byte[] waveFormBytes;
private final float[] waveForm;
public static @NonNull AudioFileInfo fromDatabaseProtobuf(@NonNull AudioWaveFormData audioWaveForm) {
return new AudioFileInfo(audioWaveForm.getDurationUs(), audioWaveForm.getWaveForm().toByteArray());
}
AudioFileInfo(long durationUs, byte[] waveFormBytes) {
this.durationUs = durationUs;
this.waveFormBytes = waveFormBytes;
this.waveForm = new float[waveFormBytes.length];
for (int i = 0; i < waveFormBytes.length; i++) {
int unsigned = waveFormBytes[i] & 0xff;
this.waveForm[i] = unsigned / 255f;
}
}
public long getDuration(@NonNull TimeUnit timeUnit) {
return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS);
}
public float[] getWaveForm() {
return waveForm;
}
public @NonNull AudioWaveFormData toDatabaseProtobuf() {
return AudioWaveFormData.newBuilder()
.setDurationUs(durationUs)
.setWaveForm(ByteString.copyFrom(waveFormBytes))
.build();
}
}

325
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForm.java
View file

@ -1,325 +0,0 @@
package org.thoughtcrime.securesms.audio;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.net.Uri;
import android.os.Build;
import android.util.LruCache;
import androidx.annotation.AnyThread;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.annotation.WorkerThread;
import androidx.core.util.Consumer;
import com.google.protobuf.ByteString;
import org.signal.core.util.ThreadUtil;
import org.signal.core.util.concurrent.SignalExecutors;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.attachments.Attachment;
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
import org.thoughtcrime.securesms.database.AttachmentTable;
import org.thoughtcrime.securesms.database.SignalDatabase;
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.media.MediaInput;
import org.thoughtcrime.securesms.mms.AudioSlide;
import org.thoughtcrime.securesms.util.concurrent.SerialExecutor;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Locale;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
@RequiresApi(api = Build.VERSION_CODES.M)
public final class AudioWaveForm {
private static final String TAG = Log.tag(AudioWaveForm.class);
private static final int BAR_COUNT = 46;
private static final int SAMPLES_PER_BAR = 4;
private final Context context;
private final AudioSlide slide;
public AudioWaveForm(@NonNull Context context, @NonNull AudioSlide slide) {
this.context = context.getApplicationContext();
this.slide = slide;
}
private static final LruCache<String, AudioFileInfo> WAVE_FORM_CACHE = new LruCache<>(200);
private static final Executor AUDIO_DECODER_EXECUTOR = new SerialExecutor(SignalExecutors.BOUNDED);
@AnyThread
public void getWaveForm(@NonNull Consumer<AudioFileInfo> onSuccess, @NonNull Runnable onFailure) {
Uri uri = slide.getUri();
Attachment attachment = slide.asAttachment();
if (uri == null) {
Log.w(TAG, "No uri");
ThreadUtil.runOnMain(onFailure);
return;
}
String cacheKey = uri.toString();
AudioFileInfo cached = WAVE_FORM_CACHE.get(cacheKey);
if (cached != null) {
Log.i(TAG, "Loaded wave form from cache " + cacheKey);
ThreadUtil.runOnMain(() -> onSuccess.accept(cached));
return;
}
AUDIO_DECODER_EXECUTOR.execute(() -> {
AudioFileInfo cachedInExecutor = WAVE_FORM_CACHE.get(cacheKey);
if (cachedInExecutor != null) {
Log.i(TAG, "Loaded wave form from cache inside executor" + cacheKey);
ThreadUtil.runOnMain(() -> onSuccess.accept(cachedInExecutor));
return;
}
AudioHash audioHash = attachment.getAudioHash();
if (audioHash != null) {
AudioFileInfo audioFileInfo = AudioFileInfo.fromDatabaseProtobuf(audioHash.getAudioWaveForm());
if (audioFileInfo.waveForm.length == 0) {
Log.w(TAG, "Recovering from a wave form generation error " + cacheKey);
ThreadUtil.runOnMain(onFailure);
return;
} else if (audioFileInfo.waveForm.length != BAR_COUNT) {
Log.w(TAG, "Wave form from database does not match bar count, regenerating " + cacheKey);
} else {
WAVE_FORM_CACHE.put(cacheKey, audioFileInfo);
Log.i(TAG, "Loaded wave form from DB " + cacheKey);
ThreadUtil.runOnMain(() -> onSuccess.accept(audioFileInfo));
return;
}
}
if (attachment instanceof DatabaseAttachment) {
try {
AttachmentTable attachmentDatabase = SignalDatabase.attachments();
DatabaseAttachment dbAttachment = (DatabaseAttachment) attachment;
long startTime = System.currentTimeMillis();
attachmentDatabase.writeAudioHash(dbAttachment.getAttachmentId(), AudioWaveFormData.getDefaultInstance());
Log.i(TAG, String.format("Starting wave form generation (%s)", cacheKey));
AudioFileInfo fileInfo = generateWaveForm(uri);
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms (%s)", System.currentTimeMillis() - startTime, cacheKey));
attachmentDatabase.writeAudioHash(dbAttachment.getAttachmentId(), fileInfo.toDatabaseProtobuf());
WAVE_FORM_CACHE.put(cacheKey, fileInfo);
ThreadUtil.runOnMain(() -> onSuccess.accept(fileInfo));
} catch (Throwable e) {
Log.w(TAG, "Failed to create audio wave form for " + cacheKey, e);
ThreadUtil.runOnMain(onFailure);
}
} else {
try {
Log.i(TAG, "Not in database and not cached. Generating wave form on-the-fly.");
long startTime = System.currentTimeMillis();
Log.i(TAG, String.format("Starting wave form generation (%s)", cacheKey));
AudioFileInfo fileInfo = generateWaveForm(uri);
Log.i(TAG, String.format(Locale.US, "Audio wave form generation time %d ms (%s)", System.currentTimeMillis() - startTime, cacheKey));
WAVE_FORM_CACHE.put(cacheKey, fileInfo);
ThreadUtil.runOnMain(() -> onSuccess.accept(fileInfo));
} catch (IOException e) {
Log.w(TAG, "Failed to create audio wave form for " + cacheKey, e);
ThreadUtil.runOnMain(onFailure);
}
}
});
}
/**
* Based on decode sample from:
* <p>
* https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java
*/
@WorkerThread
@RequiresApi(api = 23)
private @NonNull AudioFileInfo generateWaveForm(@NonNull Uri uri) throws IOException {
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
long[] wave = new long[BAR_COUNT];
int[] waveSamples = new int[BAR_COUNT];
MediaExtractor extractor = dataSource.createExtractor();
if (extractor.getTrackCount() == 0) {
throw new IOException("No audio track");
}
MediaFormat format = extractor.getTrackFormat(0);
if (!format.containsKey(MediaFormat.KEY_DURATION)) {
throw new IOException("Unknown duration");
}
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
String mime = format.getString(MediaFormat.KEY_MIME);
if (!mime.startsWith("audio/")) {
throw new IOException("Mime not audio");
}
MediaCodec codec = MediaCodec.createDecoderByType(mime);
if (totalDurationUs == 0) {
throw new IOException("Zero duration");
}
codec.configure(format, null, null, 0);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
long kTimeOutUs = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while (!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if (!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0,
sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
sawInputEOS = !extractor.advance();
int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) {
sawInputEOS = !extractor.advance();
if (!sawInputEOS) {
nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
}
}
}
}
}
int outputBufferIndex;
do {
outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
if (outputBufferIndex >= 0) {
if (info.size > 0) {
noOutputCounter = 0;
}
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs);
long total = 0;
for (int i = 0; i < info.size; i += 2 * 4) {
short aShort = buf.getShort(i);
total += Math.abs(aShort);
}
if (barIndex >= 0 && barIndex < wave.length) {
wave[barIndex] += total;
waveSamples[barIndex] += info.size / 2;
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
Log.d(TAG, "output format has changed to " + codec.getOutputFormat());
}
} while (outputBufferIndex >= 0);
}
codec.stop();
codec.release();
extractor.release();
float[] floats = new float[BAR_COUNT];
byte[] bytes = new byte[BAR_COUNT];
float max = 0;
for (int i = 0; i < BAR_COUNT; i++) {
if (waveSamples[i] == 0) continue;
floats[i] = wave[i] / (float) waveSamples[i];
if (floats[i] > max) {
max = floats[i];
}
}
for (int i = 0; i < BAR_COUNT; i++) {
float normalized = floats[i] / max;
bytes[i] = (byte) (255 * normalized);
}
return new AudioFileInfo(totalDurationUs, bytes);
}
}
public static class AudioFileInfo {
private final long durationUs;
private final byte[] waveFormBytes;
private final float[] waveForm;
private static @NonNull AudioFileInfo fromDatabaseProtobuf(@NonNull AudioWaveFormData audioWaveForm) {
return new AudioFileInfo(audioWaveForm.getDurationUs(), audioWaveForm.getWaveForm().toByteArray());
}
private AudioFileInfo(long durationUs, byte[] waveFormBytes) {
this.durationUs = durationUs;
this.waveFormBytes = waveFormBytes;
this.waveForm = new float[waveFormBytes.length];
for (int i = 0; i < waveFormBytes.length; i++) {
int unsigned = waveFormBytes[i] & 0xff;
this.waveForm[i] = unsigned / 255f;
}
}
public long getDuration(@NonNull TimeUnit timeUnit) {
return timeUnit.convert(durationUs, TimeUnit.MICROSECONDS);
}
public float[] getWaveForm() {
return waveForm;
}
private @NonNull AudioWaveFormData toDatabaseProtobuf() {
return AudioWaveFormData.newBuilder()
.setDurationUs(durationUs)
.setWaveForm(ByteString.copyFrom(waveFormBytes))
.build();
}
}
}

173
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveFormGenerator.java Normal file
View file

@ -0,0 +1,173 @@
package org.thoughtcrime.securesms.audio;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.net.Uri;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.annotation.WorkerThread;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.media.DecryptableUriMediaInput;
import org.thoughtcrime.securesms.media.MediaInput;
import java.io.IOException;
import java.nio.ByteBuffer;
@RequiresApi(api = 23)
public final class AudioWaveFormGenerator {
private static final String TAG = Log.tag(AudioWaveFormGenerator.class);
public static final int BAR_COUNT = 46;
private static final int SAMPLES_PER_BAR = 4;
private AudioWaveFormGenerator() {}
/**
* Based on decode sample from:
* <p>
* https://android.googlesource.com/platform/cts/+/jb-mr2-release/tests/tests/media/src/android/media/cts/DecoderTest.java
*/
@WorkerThread
public static @NonNull AudioFileInfo generateWaveForm(@NonNull Context context, @NonNull Uri uri) throws IOException {
try (MediaInput dataSource = DecryptableUriMediaInput.createForUri(context, uri)) {
long[] wave = new long[BAR_COUNT];
int[] waveSamples = new int[BAR_COUNT];
MediaExtractor extractor = dataSource.createExtractor();
if (extractor.getTrackCount() == 0) {
throw new IOException("No audio track");
}
MediaFormat format = extractor.getTrackFormat(0);
if (!format.containsKey(MediaFormat.KEY_DURATION)) {
throw new IOException("Unknown duration");
}
long totalDurationUs = format.getLong(MediaFormat.KEY_DURATION);
String mime = format.getString(MediaFormat.KEY_MIME);
if (!mime.startsWith("audio/")) {
throw new IOException("Mime not audio");
}
MediaCodec codec = MediaCodec.createDecoderByType(mime);
if (totalDurationUs == 0) {
throw new IOException("Zero duration");
}
codec.configure(format, null, null, 0);
codec.start();
ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
extractor.selectTrack(0);
long kTimeOutUs = 5000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
boolean sawInputEOS = false;
boolean sawOutputEOS = false;
int noOutputCounter = 0;
while (!sawOutputEOS && noOutputCounter < 50) {
noOutputCounter++;
if (!sawInputEOS) {
int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
if (inputBufIndex >= 0) {
ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
int sampleSize = extractor.readSampleData(dstBuf, 0);
long presentationTimeUs = 0;
if (sampleSize < 0) {
sawInputEOS = true;
sampleSize = 0;
} else {
presentationTimeUs = extractor.getSampleTime();
}
codec.queueInputBuffer(
inputBufIndex,
0,
sampleSize,
presentationTimeUs,
sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
if (!sawInputEOS) {
int barSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
sawInputEOS = !extractor.advance();
int nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
while (!sawInputEOS && nextBarSampleIndex == barSampleIndex) {
sawInputEOS = !extractor.advance();
if (!sawInputEOS) {
nextBarSampleIndex = (int) (SAMPLES_PER_BAR * (wave.length * extractor.getSampleTime()) / totalDurationUs);
}
}
}
}
}
int outputBufferIndex;
do {
outputBufferIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
if (outputBufferIndex >= 0) {
if (info.size > 0) {
noOutputCounter = 0;
}
ByteBuffer buf = codecOutputBuffers[outputBufferIndex];
int barIndex = (int) ((wave.length * info.presentationTimeUs) / totalDurationUs);
long total = 0;
for (int i = 0; i < info.size; i += 2 * 4) {
short aShort = buf.getShort(i);
total += Math.abs(aShort);
}
if (barIndex >= 0 && barIndex < wave.length) {
wave[barIndex] += total;
waveSamples[barIndex] += info.size / 2;
}
codec.releaseOutputBuffer(outputBufferIndex, false);
if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
sawOutputEOS = true;
}
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
codecOutputBuffers = codec.getOutputBuffers();
} else if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
Log.d(TAG, "output format has changed to " + codec.getOutputFormat());
}
} while (outputBufferIndex >= 0);
}
codec.stop();
codec.release();
extractor.release();
float[] floats = new float[BAR_COUNT];
byte[] bytes = new byte[BAR_COUNT];
float max = 0;
for (int i = 0; i < BAR_COUNT; i++) {
if (waveSamples[i] == 0) continue;
floats[i] = wave[i] / (float) waveSamples[i];
if (floats[i] > max) {
max = floats[i];
}
}
for (int i = 0; i < BAR_COUNT; i++) {
float normalized = floats[i] / max;
bytes[i] = (byte) (255 * normalized);
}
return new AudioFileInfo(totalDurationUs, bytes);
}
}
}

152
app/src/main/java/org/thoughtcrime/securesms/audio/AudioWaveForms.kt Normal file
View file

@ -0,0 +1,152 @@
package org.thoughtcrime.securesms.audio
import android.content.Context
import android.net.Uri
import android.util.LruCache
import androidx.annotation.AnyThread
import androidx.annotation.RequiresApi
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.schedulers.Schedulers
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.attachments.Attachment
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData
import java.io.IOException
import java.util.concurrent.locks.ReentrantReadWriteLock
import kotlin.concurrent.read
import kotlin.concurrent.write
/**
* Uses [AudioWaveFormGenerator] to generate audio wave forms.
*
* Maintains an in-memory cache of recently requested wave forms.
*/
@RequiresApi(23)
object AudioWaveForms {
private val TAG = Log.tag(AudioWaveForms::class.java)
private val cache = ThreadSafeLruCache(200)
@AnyThread
@JvmStatic
fun getWaveForm(context: Context, attachment: Attachment): Single<AudioFileInfo> {
val uri = attachment.uri
if (uri == null) {
Log.i(TAG, "No uri")
return Single.error(IllegalArgumentException("No uri from attachment"))
}
val cacheKey = uri.toString()
val cachedInfo = cache.get(cacheKey)
if (cachedInfo != null) {
Log.i(TAG, "Loaded wave form from cache $cacheKey")
return Single.just(cachedInfo)
}
val databaseCache = Single.fromCallable {
val audioHash = attachment.audioHash
return@fromCallable if (audioHash != null) {
checkDatabaseCache(cacheKey, audioHash.audioWaveForm)
} else {
Miss
}
}.subscribeOn(Schedulers.io())
val generateWaveForm: Single<CacheCheckResult> = if (attachment is DatabaseAttachment) {
Single.fromCallable { generateWaveForm(context, uri, cacheKey, attachment.attachmentId) }
} else {
Single.fromCallable { generateWaveForm(context, uri, cacheKey) }
}.subscribeOn(Schedulers.io())
return databaseCache
.flatMap { r ->
if (r is Miss) {
generateWaveForm
} else {
Single.just(r)
}
}
.map { r ->
if (r is Success) {
r.audioFileInfo
} else {
throw IOException("Unable to generate wave form")
}
}
}
private fun checkDatabaseCache(cacheKey: String, audioWaveForm: AudioWaveFormData): CacheCheckResult {
val audioFileInfo = AudioFileInfo.fromDatabaseProtobuf(audioWaveForm)
if (audioFileInfo.waveForm.isEmpty()) {
Log.w(TAG, "Recovering from a wave form generation error $cacheKey")
return Failure
} else if (audioFileInfo.waveForm.size != AudioWaveFormGenerator.BAR_COUNT) {
Log.w(TAG, "Wave form from database does not match bar count, regenerating $cacheKey")
} else {
cache.put(cacheKey, audioFileInfo)
Log.i(TAG, "Loaded wave form from DB $cacheKey")
return Success(audioFileInfo)
}
return Miss
}
private fun generateWaveForm(context: Context, uri: Uri, cacheKey: String, attachmentId: AttachmentId): CacheCheckResult {
try {
val startTime = System.currentTimeMillis()
SignalDatabase.attachments.writeAudioHash(attachmentId, AudioWaveFormData.getDefaultInstance())
Log.i(TAG, "Starting wave form generation ($cacheKey)")
val fileInfo: AudioFileInfo = AudioWaveFormGenerator.generateWaveForm(context, uri)
Log.i(TAG, "Audio wave form generation time ${System.currentTimeMillis() - startTime} ms ($cacheKey)")
SignalDatabase.attachments.writeAudioHash(attachmentId, fileInfo.toDatabaseProtobuf())
cache.put(cacheKey, fileInfo)
return Success(fileInfo)
} catch (e: Throwable) {
Log.w(TAG, "Failed to create audio wave form for $cacheKey", e)
return Failure
}
}
private fun generateWaveForm(context: Context, uri: Uri, cacheKey: String): CacheCheckResult {
try {
Log.i(TAG, "Not in database and not cached. Generating wave form on-the-fly.")
val startTime = System.currentTimeMillis()
Log.i(TAG, "Starting wave form generation ($cacheKey)")
val fileInfo: AudioFileInfo = AudioWaveFormGenerator.generateWaveForm(context, uri)
Log.i(TAG, "Audio wave form generation time ${System.currentTimeMillis() - startTime} ms ($cacheKey)")
cache.put(cacheKey, fileInfo)
return Success(fileInfo)
} catch (e: Throwable) {
Log.w(TAG, "Failed to create audio wave form for $cacheKey", e)
return Failure
}
}
private class ThreadSafeLruCache(maxSize: Int) {
private val cache = LruCache<String, AudioFileInfo>(maxSize)
private val lock = ReentrantReadWriteLock()
fun put(key: String, info: AudioFileInfo) {
lock.write { cache.put(key, info) }
}
fun get(key: String): AudioFileInfo? {
return lock.read { cache.get(key) }
}
}
private sealed class CacheCheckResult
private class Success(val audioFileInfo: AudioFileInfo) : CacheCheckResult()
private object Failure : CacheCheckResult()
private object Miss : CacheCheckResult()
}

32
app/src/main/java/org/thoughtcrime/securesms/components/AudioView.java
View file

@ -33,7 +33,7 @@ import org.greenrobot.eventbus.Subscribe;
import org.greenrobot.eventbus.ThreadMode;
import org.signal.core.util.logging.Log;
import org.thoughtcrime.securesms.R;
import org.thoughtcrime.securesms.audio.AudioWaveForm;
import org.thoughtcrime.securesms.audio.AudioWaveForms;
import org.thoughtcrime.securesms.components.voice.VoiceNotePlaybackState;
import org.thoughtcrime.securesms.database.AttachmentTable;
import org.thoughtcrime.securesms.events.PartProgressEvent;
@ -43,6 +43,9 @@ import org.thoughtcrime.securesms.mms.SlideClickListener;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.disposables.Disposable;
public final class AudioView extends FrameLayout {
private static final String TAG = Log.tag(AudioView.class);
@ -77,6 +80,8 @@ public final class AudioView extends FrameLayout {
private AudioSlide audioSlide;
private Callbacks callbacks;
private Disposable disposable = Disposable.disposed();
private final Observer<VoiceNotePlaybackState> playbackStateObserver = this::onPlaybackState;
public AudioView(Context context) {
@ -155,6 +160,7 @@ public final class AudioView extends FrameLayout {
protected void onDetachedFromWindow() {
super.onDetachedFromWindow();
EventBus.getDefault().unregister(this);
disposable.dispose();
}
public void setProgressAndPlayBackgroundTint(@ColorInt int color) {
@ -170,6 +176,7 @@ public final class AudioView extends FrameLayout {
final boolean showControls,
final boolean forceHideDuration)
{
this.disposable.dispose();
this.callbacks = callbacks;
if (duration != null) {
@ -212,16 +219,19 @@ public final class AudioView extends FrameLayout {
WaveFormSeekBarView waveFormView = (WaveFormSeekBarView) seekBar;
waveFormView.setColors(waveFormPlayedBarsColor, waveFormUnplayedBarsColor, waveFormThumbTint);
if (android.os.Build.VERSION.SDK_INT >= 23) {
new AudioWaveForm(getContext(), audio).getWaveForm(
data -> {
durationMillis = data.getDuration(TimeUnit.MILLISECONDS);
updateProgress(0, 0);
if (!forceHideDuration && duration != null) {
duration.setVisibility(VISIBLE);
}
waveFormView.setWaveData(data.getWaveForm());
},
() -> waveFormView.setWaveMode(false));
disposable = AudioWaveForms.getWaveForm(getContext(), audioSlide.asAttachment())
.observeOn(AndroidSchedulers.mainThread())
.subscribe(
data -> {
durationMillis = data.getDuration(TimeUnit.MILLISECONDS);
updateProgress(0, 0);
if (!forceHideDuration && duration != null) {
duration.setVisibility(VISIBLE);
}
waveFormView.setWaveData(data.getWaveForm());
},
t -> waveFormView.setWaveMode(false)
);
} else {
waveFormView.setWaveMode(false);
if (duration != null) {

5
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.java
View file

@ -51,6 +51,7 @@ import org.thoughtcrime.securesms.crypto.ClassicDecryptingPartInputStream;
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
import org.thoughtcrime.securesms.database.model.databaseprotos.AudioWaveFormData;
import org.thoughtcrime.securesms.jobs.GenerateAudioWaveFormJob;
import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.mms.PartAuthority;
@ -656,6 +657,10 @@ public class AttachmentTable extends DatabaseTable {
//noinspection ResultOfMethodCallIgnored
transferFile.delete();
}
if (placeholder != null && MediaUtil.isAudio(placeholder)) {
GenerateAudioWaveFormJob.enqueue(placeholder.getAttachmentId());
}
}
private static @Nullable String getVisualHashStringOrNull(@Nullable Attachment attachment) {

94
app/src/main/java/org/thoughtcrime/securesms/jobs/GenerateAudioWaveFormJob.kt Normal file
View file

@ -0,0 +1,94 @@
package org.thoughtcrime.securesms.jobs
import android.os.Build
import androidx.annotation.RequiresApi
import org.signal.core.util.concurrent.safeBlockingGet
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.DatabaseAttachment
import org.thoughtcrime.securesms.audio.AudioWaveForms
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies
import org.thoughtcrime.securesms.jobmanager.Data
import org.thoughtcrime.securesms.jobmanager.Job
import org.thoughtcrime.securesms.util.MediaUtil
import kotlin.time.Duration.Companion.days
/**
* Generate and save wave form for an audio attachment.
*/
class GenerateAudioWaveFormJob private constructor(private val attachmentId: AttachmentId, parameters: Parameters) : BaseJob(parameters) {
companion object {
private val TAG = Log.tag(GenerateAudioWaveFormJob::class.java)
private const val KEY_PART_ROW_ID = "part_row_id"
private const val KEY_PAR_UNIQUE_ID = "part_unique_id"
const val KEY = "GenerateAudioWaveFormJob"
@JvmStatic
fun enqueue(attachmentId: AttachmentId) {
if (Build.VERSION.SDK_INT < 23) {
Log.i(TAG, "Unable to generate waveform on this version of Android")
} else {
ApplicationDependencies.getJobManager().add(GenerateAudioWaveFormJob(attachmentId))
}
}
}
private constructor(attachmentId: AttachmentId) : this(
attachmentId,
Parameters.Builder()
.setQueue("GenerateAudioWaveFormJob")
.setLifespan(1.days.inWholeMilliseconds)
.setMaxAttempts(1)
.build()
)
override fun serialize(): Data {
return Data.Builder()
.putLong(KEY_PART_ROW_ID, attachmentId.rowId)
.putLong(KEY_PAR_UNIQUE_ID, attachmentId.uniqueId)
.build()
}
override fun getFactoryKey(): String = KEY
@RequiresApi(23)
override fun onRun() {
val attachment: DatabaseAttachment? = SignalDatabase.attachments.getAttachment(attachmentId)
if (attachment == null) {
Log.i(TAG, "Unable to find attachment in database.")
return
}
if (!MediaUtil.isAudio(attachment)) {
Log.w(TAG, "Attempting to generate wave form for a non-audio attachment type: ${attachment.contentType}")
return
}
try {
AudioWaveForms
.getWaveForm(context, attachment)
.safeBlockingGet()
Log.i(TAG, "Generation successful")
} catch (e: Exception) {
Log.i(TAG, "Generation failed", e)
}
}
override fun onShouldRetry(e: Exception): Boolean {
return false
}
override fun onFailure() = Unit
class Factory : Job.Factory<GenerateAudioWaveFormJob> {
override fun create(parameters: Parameters, data: Data): GenerateAudioWaveFormJob {
return GenerateAudioWaveFormJob(AttachmentId(data.getLong(KEY_PART_ROW_ID), data.getLong(KEY_PAR_UNIQUE_ID)), parameters)
}
}
}

3
app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java
View file

@ -105,8 +105,8 @@ public final class JobManagerFactories {
put(FontDownloaderJob.KEY, new FontDownloaderJob.Factory());
put(ForceUpdateGroupV2Job.KEY, new ForceUpdateGroupV2Job.Factory());
put(ForceUpdateGroupV2WorkerJob.KEY, new ForceUpdateGroupV2WorkerJob.Factory());
put(GenerateAudioWaveFormJob.KEY, new GenerateAudioWaveFormJob.Factory());
put(GiftSendJob.KEY, new GiftSendJob.Factory());
put(SendPaymentsActivatedJob.KEY, new SendPaymentsActivatedJob.Factory());
put(GroupV1MigrationJob.KEY, new GroupV1MigrationJob.Factory());
put(GroupCallUpdateSendJob.KEY, new GroupCallUpdateSendJob.Factory());
put(GroupCallPeekJob.KEY, new GroupCallPeekJob.Factory());
@ -177,6 +177,7 @@ public final class JobManagerFactories {
put(RotateProfileKeyJob.KEY, new RotateProfileKeyJob.Factory());
put(SenderKeyDistributionSendJob.KEY, new SenderKeyDistributionSendJob.Factory());
put(SendDeliveryReceiptJob.KEY, new SendDeliveryReceiptJob.Factory());
put(SendPaymentsActivatedJob.KEY, new SendPaymentsActivatedJob.Factory());
put(SendReadReceiptJob.KEY, new SendReadReceiptJob.Factory(application));
put(SendRetryReceiptJob.KEY, new SendRetryReceiptJob.Factory());
put(SendViewedReceiptJob.KEY, new SendViewedReceiptJob.Factory(application));