empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Refactor and improve attachment deduping logic.

Browse source
This commit is contained in:
Greyson Parrelli 2024-03-12 13:38:16 -04:00 • committed by Cody Henthorne
parent b7ee6bfcb3
commit 6df1a68213
16 changed files with 1150 additions and 565 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt
View file

@ -51,18 +51,16 @@ class AttachmentTableTest {
SignalDatabase.attachments.updateAttachmentData(
attachment,
createMediaStream(byteArrayOf(1, 2, 3, 4, 5)),
false
createMediaStream(byteArrayOf(1, 2, 3, 4, 5))
)
SignalDatabase.attachments.updateAttachmentData(
attachment2,
createMediaStream(byteArrayOf(1, 2, 3)),
false
createMediaStream(byteArrayOf(1, 2, 3))
)
val attachment1Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment.attachmentId, AttachmentTable.DATA_FILE)
val attachment2Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment2.attachmentId, AttachmentTable.DATA_FILE)
val attachment1Info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
val attachment2Info = SignalDatabase.attachments.getDataFileInfo(attachment2.attachmentId)
assertNotEquals(attachment1Info, attachment2Info)
}
@ -79,18 +77,16 @@ class AttachmentTableTest {
SignalDatabase.attachments.updateAttachmentData(
attachment,
createMediaStream(byteArrayOf(1, 2, 3, 4, 5)),
true
createMediaStream(byteArrayOf(1, 2, 3, 4, 5))
)
SignalDatabase.attachments.updateAttachmentData(
attachment2,
createMediaStream(byteArrayOf(1, 2, 3, 4)),
true
createMediaStream(byteArrayOf(1, 2, 3, 4))
)
val attachment1Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment.attachmentId, AttachmentTable.DATA_FILE)
val attachment2Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment2.attachmentId, AttachmentTable.DATA_FILE)
val attachment1Info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
val attachment2Info = SignalDatabase.attachments.getDataFileInfo(attachment2.attachmentId)
assertNotEquals(attachment1Info, attachment2Info)
}
@ -121,15 +117,14 @@ class AttachmentTableTest {
val highDatabaseAttachment = SignalDatabase.attachments.insertAttachmentForPreUpload(highQualityPreUpload)
// WHEN
SignalDatabase.attachments.updateAttachmentData(standardDatabaseAttachment, createMediaStream(compressedData), false)
SignalDatabase.attachments.updateAttachmentData(standardDatabaseAttachment, createMediaStream(compressedData))
// THEN
val previousInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(previousDatabaseAttachmentId, AttachmentTable.DATA_FILE)!!
val standardInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(standardDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
val highInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(highDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
val previousInfo = SignalDatabase.attachments.getDataFileInfo(previousDatabaseAttachmentId)!!
val standardInfo = SignalDatabase.attachments.getDataFileInfo(standardDatabaseAttachment.attachmentId)!!
val highInfo = SignalDatabase.attachments.getDataFileInfo(highDatabaseAttachment.attachmentId)!!
assertNotEquals(standardInfo, highInfo)
standardInfo.file assertIs previousInfo.file
highInfo.file assertIsNot standardInfo.file
highInfo.file.exists() assertIs true
}
@ -158,9 +153,9 @@ class AttachmentTableTest {
val secondHighDatabaseAttachment = SignalDatabase.attachments.insertAttachmentForPreUpload(secondHighQualityPreUpload)
// THEN
val standardInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(standardDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
val highInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(highDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
val secondHighInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(secondHighDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
val standardInfo = SignalDatabase.attachments.getDataFileInfo(standardDatabaseAttachment.attachmentId)!!
val highInfo = SignalDatabase.attachments.getDataFileInfo(highDatabaseAttachment.attachmentId)!!
val secondHighInfo = SignalDatabase.attachments.getDataFileInfo(secondHighDatabaseAttachment.attachmentId)!!
highInfo.file assertIsNot standardInfo.file
secondHighInfo.file assertIs highInfo.file

596
app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt Normal file
View file

@ -0,0 +1,596 @@
package org.thoughtcrime.securesms.database
import androidx.test.ext.junit.runners.AndroidJUnit4
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertNotEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.thoughtcrime.securesms.attachments.AttachmentId
import org.thoughtcrime.securesms.attachments.PointerAttachment
import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties
import org.thoughtcrime.securesms.keyvalue.SignalStore
import org.thoughtcrime.securesms.mms.MediaStream
import org.thoughtcrime.securesms.mms.OutgoingMessage
import org.thoughtcrime.securesms.mms.QuoteModel
import org.thoughtcrime.securesms.mms.SentMediaQuality
import org.thoughtcrime.securesms.providers.BlobProvider
import org.thoughtcrime.securesms.recipients.Recipient
import org.thoughtcrime.securesms.util.MediaUtil
import org.whispersystems.signalservice.api.push.ServiceId
import java.io.File
import java.util.UUID
import kotlin.random.Random
import kotlin.time.Duration.Companion.days
/**
* Collection of [AttachmentTable] tests focused around deduping logic.
*/
@RunWith(AndroidJUnit4::class)
class AttachmentTableTest_deduping {
companion object {
val DATA_A = byteArrayOf(1, 2, 3)
val DATA_A_COMPRESSED = byteArrayOf(4, 5, 6)
val DATA_B = byteArrayOf(7, 8, 9)
}
@Before
fun setUp() {
SignalStore.account().setAci(ServiceId.ACI.from(UUID.randomUUID()))
SignalStore.account().setPni(ServiceId.PNI.from(UUID.randomUUID()))
SignalStore.account().setE164("+15558675309")
SignalDatabase.attachments.deleteAllAttachments()
}
/**
* Creates two different files with different data. Should not dedupe.
*/
@Test
fun differentFiles() {
test {
val id1 = insertWithData(DATA_A)
val id2 = insertWithData(DATA_B)
assertDataFilesAreDifferent(id1, id2)
}
}
/**
* Inserts files with identical data but with transform properties that make them incompatible. Should not dedupe.
*/
@Test
fun identicalFiles_incompatibleTransforms() {
// Non-matching qualities
test {
val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
assertDataFilesAreDifferent(id1, id2)
assertDataHashStartMatches(id1, id2)
}
// Non-matching video trim flag
test {
val id1 = insertWithData(DATA_A, TransformProperties())
val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true))
assertDataFilesAreDifferent(id1, id2)
assertDataHashStartMatches(id1, id2)
}
// Non-matching video trim start time
test {
val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 2))
assertDataFilesAreDifferent(id1, id2)
assertDataHashStartMatches(id1, id2)
}
// Non-matching video trim end time
test {
val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 1))
val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 2))
assertDataFilesAreDifferent(id1, id2)
assertDataHashStartMatches(id1, id2)
}
// Non-matching mp4 fast start
test {
val id1 = insertWithData(DATA_A, TransformProperties(mp4FastStart = true))
val id2 = insertWithData(DATA_A, TransformProperties(mp4FastStart = false))
assertDataFilesAreDifferent(id1, id2)
assertDataHashStartMatches(id1, id2)
}
}
/**
* Inserts files with identical data and compatible transform properties. Should dedupe.
*/
@Test
fun identicalFiles_compatibleTransforms() {
test {
val id1 = insertWithData(DATA_A)
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, false)
assertSkipTransform(id2, false)
}
test {
val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, false)
assertSkipTransform(id2, false)
}
test {
val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, false)
assertSkipTransform(id2, false)
}
test {
val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, false)
assertSkipTransform(id2, false)
}
}
/**
* Walks through various scenarios where files are compressed and uploaded.
*/
@Test
fun compressionAndUploads() {
// Matches after the first is compressed, skip transform properly set
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, true)
}
// Matches after the first is uploaded, skip transform and ending hash properly set
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
upload(id1)
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertDataHashEndMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, true)
}
// Mimics sending two files at once. Ensures all fields are kept in sync as we compress and upload.
test {
val id1 = insertWithData(DATA_A)
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, false)
assertSkipTransform(id2, false)
compress(id1, DATA_A_COMPRESSED)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, true)
upload(id1)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertDataHashEndMatches(id1, id2)
assertRemoteFieldsMatch(id1, id2)
}
// Re-use the upload when uploaded recently
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertDataHashEndMatches(id1, id2)
assertRemoteFieldsMatch(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, true)
}
// Do not re-use old uploads
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis() - 100.days.inWholeMilliseconds)
val id2 = insertWithData(DATA_A)
assertDataFilesAreTheSame(id1, id2)
assertDataHashStartMatches(id1, id2)
assertDataHashEndMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, true)
assertDoesNotHaveRemoteFields(id2)
}
// This isn't so much "desirable behavior" as it is documenting how things work.
// If an attachment is compressed but not uploaded yet, it will have a DATA_HASH_START that doesn't match the actual file content.
// This means that if we insert a new attachment with data that matches the compressed data, we won't find a match.
// This is ok because we don't allow forwarding unsent messages, so the chances of the user somehow sending a file that matches data we compressed are very low.
// What *is* more common is that the user may send DATA_A again, and in this case we will still catch the dedupe (which is already tested above).
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
val id2 = insertWithData(DATA_A_COMPRESSED)
assertDataFilesAreDifferent(id1, id2)
}
// This represents what would happen if you forward an already-send compressed attachment. We should match, skip transform, and skip upload.
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A_COMPRESSED)
assertDataFilesAreTheSame(id1, id2)
assertDataHashEndMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id1, true)
assertRemoteFieldsMatch(id1, id2)
}
// This represents what would happen if you edited a video, sent it, then forwarded it. We should match, skip transform, and skip upload.
test {
val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A_COMPRESSED)
assertDataFilesAreTheSame(id1, id2)
assertDataHashEndMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id1, true)
assertRemoteFieldsMatch(id1, id2)
}
// This represents what would happen if you edited a video, sent it, then forwarded it, but *edited the forwarded video*. We should not dedupe.
test {
val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
assertDataFilesAreDifferent(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, false)
assertDoesNotHaveRemoteFields(id2)
}
// This represents what would happen if you sent an image using standard quality, then forwarded it using high quality.
// Since you're forwarding, it doesn't matter if the new thing has a higher quality, we should still match and skip transform.
test {
val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
assertDataFilesAreTheSame(id1, id2)
assertDataHashEndMatches(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id1, true)
assertRemoteFieldsMatch(id1, id2)
}
// This represents what would happen if you sent an image using high quality, then forwarded it using standard quality.
// Lowering the quality would change the output, so we shouldn't dedupe.
test {
val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
compress(id1, DATA_A_COMPRESSED)
upload(id1, uploadTimestamp = System.currentTimeMillis())
val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
assertDataFilesAreDifferent(id1, id2)
assertSkipTransform(id1, true)
assertSkipTransform(id2, false)
assertDoesNotHaveRemoteFields(id2)
}
}
/**
* Various deletion scenarios to ensure that duped files don't deleted while there's still references.
*/
@Test
fun deletions() {
// Delete original then dupe
test {
val id1 = insertWithData(DATA_A)
val id2 = insertWithData(DATA_A)
val dataFile = dataFile(id1)
assertDataFilesAreTheSame(id1, id2)
delete(id1)
assertDeleted(id1)
assertRowAndFileExists(id2)
assertTrue(dataFile.exists())
delete(id2)
assertDeleted(id2)
assertFalse(dataFile.exists())
}
// Delete dupe then original
test {
val id1 = insertWithData(DATA_A)
val id2 = insertWithData(DATA_A)
val dataFile = dataFile(id1)
assertDataFilesAreTheSame(id1, id2)
delete(id2)
assertDeleted(id2)
assertRowAndFileExists(id1)
assertTrue(dataFile.exists())
delete(id1)
assertDeleted(id1)
assertFalse(dataFile.exists())
}
// Delete original after it was compressed
test {
val id1 = insertWithData(DATA_A)
compress(id1, DATA_A_COMPRESSED)
val id2 = insertWithData(DATA_A)
delete(id1)
assertDeleted(id1)
assertRowAndFileExists(id2)
assertSkipTransform(id2, true)
}
// Quotes are weak references and should not prevent us from deleting the file
test {
val id1 = insertWithData(DATA_A)
val id2 = insertQuote(id1)
val dataFile = dataFile(id1)
delete(id1)
assertDeleted(id1)
assertRowExists(id2)
assertFalse(dataFile.exists())
}
}
private class TestContext {
fun insertWithData(data: ByteArray, transformProperties: TransformProperties = TransformProperties.empty()): AttachmentId {
val uri = BlobProvider.getInstance().forData(data).createForSingleSessionInMemory()
val attachment = UriAttachmentBuilder.build(
id = Random.nextLong(),
uri = uri,
contentType = MediaUtil.IMAGE_JPEG,
transformProperties = transformProperties
)
return SignalDatabase.attachments.insertAttachmentForPreUpload(attachment).attachmentId
}
fun insertQuote(attachmentId: AttachmentId): AttachmentId {
val originalAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.self())
val messageId = SignalDatabase.messages.insertMessageOutbox(
message = OutgoingMessage(
threadRecipient = Recipient.self(),
sentTimeMillis = System.currentTimeMillis(),
body = "some text",
outgoingQuote = QuoteModel(
id = 123,
author = Recipient.self().id,
text = "Some quote text",
isOriginalMissing = false,
attachments = listOf(originalAttachment),
mentions = emptyList(),
type = QuoteModel.Type.NORMAL,
bodyRanges = null
)
),
threadId = threadId,
forceSms = false,
insertListener = null
)
val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId)
return attachments[0].attachmentId
}
fun compress(attachmentId: AttachmentId, newData: ByteArray, mp4FastStart: Boolean = false) {
val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
SignalDatabase.attachments.updateAttachmentData(databaseAttachment, newData.asMediaStream())
SignalDatabase.attachments.markAttachmentAsTransformed(attachmentId, withFastStart = mp4FastStart)
}
fun upload(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()) {
SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createPointerAttachment(attachmentId, uploadTimestamp), uploadTimestamp)
}
fun delete(attachmentId: AttachmentId) {
SignalDatabase.attachments.deleteAttachment(attachmentId)
}
fun dataFile(attachmentId: AttachmentId): File {
return SignalDatabase.attachments.getDataFileInfo(attachmentId)!!.file
}
fun assertDeleted(attachmentId: AttachmentId) {
assertNull("$attachmentId exists, but it shouldn't!", SignalDatabase.attachments.getAttachment(attachmentId))
}
fun assertRowAndFileExists(attachmentId: AttachmentId) {
val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)
assertNotNull("$attachmentId does not exist!", databaseAttachment)
val dataFileInfo = SignalDatabase.attachments.getDataFileInfo(attachmentId)
assertTrue("The file for $attachmentId does not exist!", dataFileInfo!!.file.exists())
}
fun assertRowExists(attachmentId: AttachmentId) {
val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)
assertNotNull("$attachmentId does not exist!", databaseAttachment)
}
fun assertDataFilesAreTheSame(lhs: AttachmentId, rhs: AttachmentId) {
val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
assert(lhsInfo.file.exists())
assert(rhsInfo.file.exists())
assertEquals(lhsInfo.file, rhsInfo.file)
assertEquals(lhsInfo.length, rhsInfo.length)
assertArrayEquals(lhsInfo.random, rhsInfo.random)
}
fun assertDataFilesAreDifferent(lhs: AttachmentId, rhs: AttachmentId) {
val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
assert(lhsInfo.file.exists())
assert(rhsInfo.file.exists())
assertNotEquals(lhsInfo.file, rhsInfo.file)
}
fun assertDataHashStartMatches(lhs: AttachmentId, rhs: AttachmentId) {
val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
assertEquals("DATA_HASH_START's did not match!", lhsInfo.hashStart, rhsInfo.hashStart)
}
fun assertDataHashEndMatches(lhs: AttachmentId, rhs: AttachmentId) {
val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
assertEquals("DATA_HASH_END's did not match!", lhsInfo.hashEnd, rhsInfo.hashEnd)
}
fun assertRemoteFieldsMatch(lhs: AttachmentId, rhs: AttachmentId) {
val lhsAttachment = SignalDatabase.attachments.getAttachment(lhs)!!
val rhsAttachment = SignalDatabase.attachments.getAttachment(rhs)!!
assertEquals(lhsAttachment.remoteLocation, rhsAttachment.remoteLocation)
assertEquals(lhsAttachment.remoteKey, rhsAttachment.remoteKey)
assertArrayEquals(lhsAttachment.remoteDigest, rhsAttachment.remoteDigest)
assertArrayEquals(lhsAttachment.incrementalDigest, rhsAttachment.incrementalDigest)
assertEquals(lhsAttachment.incrementalMacChunkSize, rhsAttachment.incrementalMacChunkSize)
assertEquals(lhsAttachment.cdnNumber, rhsAttachment.cdnNumber)
}
fun assertDoesNotHaveRemoteFields(attachmentId: AttachmentId) {
val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
assertEquals(0, databaseAttachment.uploadTimestamp)
assertNull(databaseAttachment.remoteLocation)
assertNull(databaseAttachment.remoteDigest)
assertNull(databaseAttachment.remoteKey)
assertEquals(0, databaseAttachment.cdnNumber)
}
fun assertSkipTransform(attachmentId: AttachmentId, state: Boolean) {
val transformProperties = SignalDatabase.attachments.getTransformProperties(attachmentId)!!
assertEquals("Incorrect skipTransform!", transformProperties.skipTransform, state)
}
private fun ByteArray.asMediaStream(): MediaStream {
return MediaStream(this.inputStream(), MediaUtil.IMAGE_JPEG, 2, 2)
}
private fun createPointerAttachment(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): PointerAttachment {
val location = "somewhere-${Random.nextLong()}"
val key = "somekey-${Random.nextLong()}"
val digest = Random.nextBytes(32)
val incrementalDigest = Random.nextBytes(16)
val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
return PointerAttachment(
"image/jpeg",
AttachmentTable.TRANSFER_PROGRESS_DONE,
databaseAttachment.size, // size
null,
3, // cdnNumber
location,
key,
digest,
incrementalDigest,
5, // incrementalMacChunkSize
null,
databaseAttachment.voiceNote,
databaseAttachment.borderless,
databaseAttachment.videoGif,
databaseAttachment.width,
databaseAttachment.height,
uploadTimestamp,
databaseAttachment.caption,
databaseAttachment.stickerLocator,
databaseAttachment.blurHash
)
}
}
private fun test(content: TestContext.() -> Unit) {
SignalDatabase.attachments.deleteAllAttachments()
val context = TestContext()
context.content()
}
}

7
app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentId.kt
View file

@ -3,13 +3,14 @@ package org.thoughtcrime.securesms.attachments
import android.os.Parcelable
import com.fasterxml.jackson.annotation.JsonProperty
import kotlinx.parcelize.Parcelize
import org.signal.core.util.DatabaseId
@Parcelize
data class AttachmentId(
@JsonProperty("rowId")
@JvmField
val id: Long
) : Parcelable {
) : Parcelable, DatabaseId {
val isValid: Boolean
get() = id >= 0
@ -17,4 +18,8 @@ data class AttachmentId(
override fun toString(): String {
return "AttachmentId::$id"
}
override fun serialize(): String {
return id.toString()
}
}

4
app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt
View file

@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.attachments
import android.net.Uri
import android.os.Parcel
import androidx.annotation.VisibleForTesting
import org.signal.core.util.Base64.encodeWithPadding
import org.thoughtcrime.securesms.blurhash.BlurHash
import org.thoughtcrime.securesms.database.AttachmentTable
@ -14,7 +15,8 @@ import org.whispersystems.signalservice.internal.push.DataMessage
import java.util.Optional
class PointerAttachment : Attachment {
private constructor(
@VisibleForTesting
constructor(
contentType: String,
transferState: Int,
size: Long,

2
app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupImporter.java
View file

@ -194,7 +194,7 @@ public class FullBackupImporter extends FullBackupBase {
private static void processAttachment(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret, @NonNull SQLiteDatabase db, @NonNull Attachment attachment, BackupRecordInputStream inputStream)
throws IOException
{
File dataFile = AttachmentTable.newFile(context);
File dataFile = AttachmentTable.newDataFile(context);
Pair<byte[], OutputStream> output = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false);
boolean isLegacyTable = SqlUtil.tableExists(db, "part");

984
app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt
View file

File diff suppressed because it is too large Load diff

10
app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt
View file

@ -49,7 +49,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH},
${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
${MessageTable.TABLE_NAME}.${MessageTable.TYPE},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT},
${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED},
@ -71,13 +71,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
${MessageTable.VIEW_ONCE} = 0 AND
${MessageTable.STORY_TYPE} = 0 AND
${MessageTable.LATEST_REVISION_ID} IS NULL AND
(
${AttachmentTable.QUOTE} = 0 OR
(
${AttachmentTable.QUOTE} = 1 AND
${AttachmentTable.DATA_HASH} IS NULL
)
) AND
${AttachmentTable.QUOTE} = 0 AND
${AttachmentTable.STICKER_PACK_ID} IS NULL AND
${MessageTable.TABLE_NAME}.${MessageTable.FROM_RECIPIENT_ID} > 0 AND
$THREAD_RECIPIENT_ID > 0

6
app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt
View file

@ -79,6 +79,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V218_RecipientPniSi
import org.thoughtcrime.securesms.database.helpers.migration.V219_PniPreKeyStores
import org.thoughtcrime.securesms.database.helpers.migration.V220_PreKeyConstraints
import org.thoughtcrime.securesms.database.helpers.migration.V221_AddReadColumnToCallEventsTable
import org.thoughtcrime.securesms.database.helpers.migration.V222_DataHashRefactor
/**
* Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@ -160,10 +161,11 @@ object SignalDatabaseMigrations {
218 to V218_RecipientPniSignatureVerified,
219 to V219_PniPreKeyStores,
220 to V220_PreKeyConstraints,
221 to V221_AddReadColumnToCallEventsTable
221 to V221_AddReadColumnToCallEventsTable,
222 to V222_DataHashRefactor
)
const val DATABASE_VERSION = 221
const val DATABASE_VERSION = 222
@JvmStatic
fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

25
app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V222_DataHashRefactor.kt Normal file
View file

@ -0,0 +1,25 @@
/*
* Copyright 2024 Signal Messenger, LLC
* SPDX-License-Identifier: AGPL-3.0-only
*/
package org.thoughtcrime.securesms.database.helpers.migration
import android.app.Application
import net.zetetic.database.sqlcipher.SQLiteDatabase
/**
* Adds the new data hash columns and indexes.
*/
@Suppress("ClassName")
object V222_DataHashRefactor : SignalDatabaseMigration {
override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
db.execSQL("DROP INDEX attachment_data_hash_index")
db.execSQL("ALTER TABLE attachment DROP COLUMN data_hash")
db.execSQL("ALTER TABLE attachment ADD COLUMN data_hash_start TEXT DEFAULT NULL")
db.execSQL("ALTER TABLE attachment ADD COLUMN data_hash_end TEXT DEFAULT NULL")
db.execSQL("CREATE INDEX attachment_data_hash_start_index ON attachment (data_hash_start)")
db.execSQL("CREATE INDEX attachment_data_hash_end_index ON attachment (data_hash_end)")
}
}

10
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java
View file

@ -205,7 +205,7 @@ public final class AttachmentCompressionJob extends BaseJob {
} else if (constraints.canResize(attachment)) {
Log.i(TAG, "Compressing image.");
try (MediaStream converted = compressImage(context, attachment, constraints)) {
attachmentDatabase.updateAttachmentData(attachment, converted, false);
attachmentDatabase.updateAttachmentData(attachment, converted);
}
attachmentDatabase.markAttachmentAsTransformed(attachmentId, false);
} else if (constraints.isSatisfied(context, attachment)) {
@ -263,7 +263,7 @@ public final class AttachmentCompressionJob extends BaseJob {
Log.i(TAG, "Compressing with streaming muxer");
AttachmentSecret attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret();
File file = AttachmentTable.newFile(context);
File file = AttachmentTable.newDataFile(context);
file.deleteOnExit();
boolean faststart = false;
@ -296,7 +296,7 @@ public final class AttachmentCompressionJob extends BaseJob {
final long plaintextLength = ModernEncryptingPartOutputStream.getPlaintextLength(file.length());
try (MediaStream mediaStream = new MediaStream(postProcessor.process(plaintextLength), MimeTypes.VIDEO_MP4, 0, 0, true)) {
attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
attachmentDatabase.updateAttachmentData(attachment, mediaStream);
faststart = true;
} catch (VideoPostProcessingException e) {
Log.w(TAG, "Exception thrown during post processing.", e);
@ -310,7 +310,7 @@ public final class AttachmentCompressionJob extends BaseJob {
if (!faststart) {
try (MediaStream mediaStream = new MediaStream(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0), MimeTypes.VIDEO_MP4, 0, 0, false)) {
attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
attachmentDatabase.updateAttachmentData(attachment, mediaStream);
}
}
} finally {
@ -339,7 +339,7 @@ public final class AttachmentCompressionJob extends BaseJob {
100,
percent));
}, cancelationSignal)) {
attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
attachmentDatabase.updateAttachmentData(attachment, mediaStream);
attachmentDatabase.markAttachmentAsTransformed(attachment.attachmentId, mediaStream.getFaststart());
}

12
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.java
View file

@ -168,7 +168,7 @@ public final class AttachmentDownloadJob extends BaseJob {
if (attachment.cdnNumber != ReleaseChannel.CDN_NUMBER) {
retrieveAttachment(messageId, attachmentId, attachment);
} else {
retrieveUrlAttachment(messageId, attachmentId, attachment);
retrieveAttachmentForReleaseChannel(messageId, attachmentId, attachment);
}
}
@ -216,7 +216,7 @@ public final class AttachmentDownloadJob extends BaseJob {
return isCanceled();
}
});
database.insertAttachmentsForPlaceholder(messageId, attachmentId, stream);
database.finalizeAttachmentAfterDownload(messageId, attachmentId, stream);
} catch (RangeException e) {
Log.w(TAG, "Range exception, file size " + attachmentFile.length(), e);
if (attachmentFile.delete()) {
@ -278,9 +278,9 @@ public final class AttachmentDownloadJob extends BaseJob {
}
}
private void retrieveUrlAttachment(long messageId,
final AttachmentId attachmentId,
final Attachment attachment)
private void retrieveAttachmentForReleaseChannel(long messageId,
final AttachmentId attachmentId,
final Attachment attachment)
throws IOException
{
try (Response response = S3.getObject(Objects.requireNonNull(attachment.fileName))) {
@ -289,7 +289,7 @@ public final class AttachmentDownloadJob extends BaseJob {
if (body.contentLength() > FeatureFlags.maxAttachmentReceiveSizeBytes()) {
throw new MmsException("Attachment too large, failing download");
}
SignalDatabase.attachments().insertAttachmentsForPlaceholder(messageId, attachmentId, Okio.buffer(body.source()).inputStream());
SignalDatabase.attachments().finalizeAttachmentAfterDownload(messageId, attachmentId, Okio.buffer(body.source()).inputStream());
}
} catch (MmsException e) {
Log.w(TAG, "Experienced exception while trying to download an attachment.", e);

5
app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt
View file

@ -42,6 +42,7 @@ import java.io.IOException
import java.util.Objects
import java.util.Optional
import java.util.concurrent.TimeUnit
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.milliseconds
/**
@ -60,7 +61,7 @@ class AttachmentUploadJob private constructor(
private val TAG = Log.tag(AttachmentUploadJob::class.java)
private val UPLOAD_REUSE_THRESHOLD = TimeUnit.DAYS.toMillis(3)
val UPLOAD_REUSE_THRESHOLD = 3.days.inWholeMilliseconds
/**
* Foreground notification shows while uploading attachments above this.
@ -162,7 +163,7 @@ class AttachmentUploadJob private constructor(
buildAttachmentStream(databaseAttachment, notification, uploadSpec!!).use { localAttachment ->
val remoteAttachment = messageSender.uploadAttachment(localAttachment)
val attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.fastPreflightId).get()
SignalDatabase.attachments.updateAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.uploadTimestamp)
SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.uploadTimestamp)
}
}
} catch (e: NonSuccessfulResumableUploadResponseCodeException) {

2
app/src/main/java/org/thoughtcrime/securesms/jobs/LegacyAttachmentUploadJob.java
View file

@ -139,7 +139,7 @@ public final class LegacyAttachmentUploadJob extends BaseJob {
SignalServiceAttachmentPointer remoteAttachment = messageSender.uploadAttachment(localAttachment);
Attachment attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.fastPreflightId).get();
database.updateAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.getUploadTimestamp());
database.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.getUploadTimestamp());
}
} catch (NonSuccessfulResumableUploadResponseCodeException e) {
if (e.getCode() == 400) {

5
app/src/main/java/org/thoughtcrime/securesms/stickers/StickerSearchRepository.java
View file

@ -64,11 +64,6 @@ public final class StickerSearchRepository {
return out;
}
public @NonNull Single<Boolean> getStickerFeatureAvailability() {
return Single.fromCallable(this::getStickerFeatureAvailabilitySync)
.observeOn(Schedulers.io());
}
public void getStickerFeatureAvailability(@NonNull Callback<Boolean> callback) {
SignalExecutors.BOUNDED.execute(() -> {
callback.onResult(getStickerFeatureAvailabilitySync());

9
core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt
View file

@ -7,7 +7,6 @@ package org.signal.core.util
import java.io.IOException
import java.io.InputStream
import kotlin.jvm.Throws
/**
* Reads a 32-bit variable-length integer from the stream.
@ -80,3 +79,11 @@ fun InputStream.readLength(): Long {
return count
}
/**
* Reads the contents of the stream and discards them.
*/
@Throws(IOException::class)
fun InputStream.drain() {
this.readLength()
}

3
libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageReceiver.java
View file

@ -142,7 +142,8 @@ public class SignalServiceMessageReceiver {
}
/**
* Retrieves a SignalServiceAttachment.
* Retrieves a SignalServiceAttachment. The encrypted data is written to @{code destination}, and then an {@link InputStream} is returned that decrypts the
* contents of the destination file, giving you access to the plaintext content.
*
* @param pointer The {@link SignalServiceAttachmentPointer}
* received in a {@link SignalServiceDataMessage}.