Refactor and improve attachment deduping logic.

app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest.kt

@@ -51,18 +51,16 @@ class AttachmentTableTest {
 
     SignalDatabase.attachments.updateAttachmentData(
       attachment,
-      createMediaStream(byteArrayOf(1, 2, 3, 4, 5)),
+      createMediaStream(byteArrayOf(1, 2, 3, 4, 5))
-      false
     )
 
     SignalDatabase.attachments.updateAttachmentData(
       attachment2,
-      createMediaStream(byteArrayOf(1, 2, 3)),
+      createMediaStream(byteArrayOf(1, 2, 3))
-      false
     )
 
-    val attachment1Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment.attachmentId, AttachmentTable.DATA_FILE)
+    val attachment1Info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
-    val attachment2Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment2.attachmentId, AttachmentTable.DATA_FILE)
+    val attachment2Info = SignalDatabase.attachments.getDataFileInfo(attachment2.attachmentId)
 
     assertNotEquals(attachment1Info, attachment2Info)
   }
@@ -79,18 +77,16 @@ class AttachmentTableTest {
 
     SignalDatabase.attachments.updateAttachmentData(
       attachment,
-      createMediaStream(byteArrayOf(1, 2, 3, 4, 5)),
+      createMediaStream(byteArrayOf(1, 2, 3, 4, 5))
-      true
     )
 
     SignalDatabase.attachments.updateAttachmentData(
       attachment2,
-      createMediaStream(byteArrayOf(1, 2, 3, 4)),
+      createMediaStream(byteArrayOf(1, 2, 3, 4))
-      true
     )
 
-    val attachment1Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment.attachmentId, AttachmentTable.DATA_FILE)
+    val attachment1Info = SignalDatabase.attachments.getDataFileInfo(attachment.attachmentId)
-    val attachment2Info = SignalDatabase.attachments.getAttachmentDataFileInfo(attachment2.attachmentId, AttachmentTable.DATA_FILE)
+    val attachment2Info = SignalDatabase.attachments.getDataFileInfo(attachment2.attachmentId)
 
     assertNotEquals(attachment1Info, attachment2Info)
   }
@@ -121,15 +117,14 @@ class AttachmentTableTest {
     val highDatabaseAttachment = SignalDatabase.attachments.insertAttachmentForPreUpload(highQualityPreUpload)
 
     // WHEN
-    SignalDatabase.attachments.updateAttachmentData(standardDatabaseAttachment, createMediaStream(compressedData), false)
+    SignalDatabase.attachments.updateAttachmentData(standardDatabaseAttachment, createMediaStream(compressedData))
 
     // THEN
-    val previousInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(previousDatabaseAttachmentId, AttachmentTable.DATA_FILE)!!
+    val previousInfo = SignalDatabase.attachments.getDataFileInfo(previousDatabaseAttachmentId)!!
-    val standardInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(standardDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
+    val standardInfo = SignalDatabase.attachments.getDataFileInfo(standardDatabaseAttachment.attachmentId)!!
-    val highInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(highDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
+    val highInfo = SignalDatabase.attachments.getDataFileInfo(highDatabaseAttachment.attachmentId)!!
 
     assertNotEquals(standardInfo, highInfo)
-    standardInfo.file assertIs previousInfo.file
     highInfo.file assertIsNot standardInfo.file
     highInfo.file.exists() assertIs true
   }
@@ -158,9 +153,9 @@ class AttachmentTableTest {
     val secondHighDatabaseAttachment = SignalDatabase.attachments.insertAttachmentForPreUpload(secondHighQualityPreUpload)
 
     // THEN
-    val standardInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(standardDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
+    val standardInfo = SignalDatabase.attachments.getDataFileInfo(standardDatabaseAttachment.attachmentId)!!
-    val highInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(highDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
+    val highInfo = SignalDatabase.attachments.getDataFileInfo(highDatabaseAttachment.attachmentId)!!
-    val secondHighInfo = SignalDatabase.attachments.getAttachmentDataFileInfo(secondHighDatabaseAttachment.attachmentId, AttachmentTable.DATA_FILE)!!
+    val secondHighInfo = SignalDatabase.attachments.getDataFileInfo(secondHighDatabaseAttachment.attachmentId)!!
 
     highInfo.file assertIsNot standardInfo.file
     secondHighInfo.file assertIs highInfo.file

app/src/androidTest/java/org/thoughtcrime/securesms/database/AttachmentTableTest_deduping.kt

@@ -0,0 +1,596 @@
+package org.thoughtcrime.securesms.database
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertNotEquals
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.thoughtcrime.securesms.attachments.AttachmentId
+import org.thoughtcrime.securesms.attachments.PointerAttachment
+import org.thoughtcrime.securesms.database.AttachmentTable.TransformProperties
+import org.thoughtcrime.securesms.keyvalue.SignalStore
+import org.thoughtcrime.securesms.mms.MediaStream
+import org.thoughtcrime.securesms.mms.OutgoingMessage
+import org.thoughtcrime.securesms.mms.QuoteModel
+import org.thoughtcrime.securesms.mms.SentMediaQuality
+import org.thoughtcrime.securesms.providers.BlobProvider
+import org.thoughtcrime.securesms.recipients.Recipient
+import org.thoughtcrime.securesms.util.MediaUtil
+import org.whispersystems.signalservice.api.push.ServiceId
+import java.io.File
+import java.util.UUID
+import kotlin.random.Random
+import kotlin.time.Duration.Companion.days
+
+/**
+ * Collection of [AttachmentTable] tests focused around deduping logic.
+ */
+@RunWith(AndroidJUnit4::class)
+class AttachmentTableTest_deduping {
+
+  companion object {
+    val DATA_A = byteArrayOf(1, 2, 3)
+    val DATA_A_COMPRESSED = byteArrayOf(4, 5, 6)
+
+    val DATA_B = byteArrayOf(7, 8, 9)
+  }
+
+  @Before
+  fun setUp() {
+    SignalStore.account().setAci(ServiceId.ACI.from(UUID.randomUUID()))
+    SignalStore.account().setPni(ServiceId.PNI.from(UUID.randomUUID()))
+    SignalStore.account().setE164("+15558675309")
+
+    SignalDatabase.attachments.deleteAllAttachments()
+  }
+
+  /**
+   * Creates two different files with different data. Should not dedupe.
+   */
+  @Test
+  fun differentFiles() {
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertWithData(DATA_B)
+
+      assertDataFilesAreDifferent(id1, id2)
+    }
+  }
+
+  /**
+   * Inserts files with identical data but with transform properties that make them incompatible. Should not dedupe.
+   */
+  @Test
+  fun identicalFiles_incompatibleTransforms() {
+    // Non-matching qualities
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
+      val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+    }
+
+    // Non-matching video trim flag
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties())
+      val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+    }
+
+    // Non-matching video trim start time
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+      val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 2))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+    }
+
+    // Non-matching video trim end time
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 1))
+      val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 0, videoTrimEndTimeUs = 2))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+    }
+
+    // Non-matching mp4 fast start
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(mp4FastStart = true))
+      val id2 = insertWithData(DATA_A, TransformProperties(mp4FastStart = false))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+    }
+  }
+
+  /**
+   * Inserts files with identical data and compatible transform properties. Should dedupe.
+   */
+  @Test
+  fun identicalFiles_compatibleTransforms() {
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, false)
+      assertSkipTransform(id2, false)
+    }
+
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
+      val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, false)
+      assertSkipTransform(id2, false)
+    }
+
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
+      val id2 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, false)
+      assertSkipTransform(id2, false)
+    }
+
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+      val id2 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, false)
+      assertSkipTransform(id2, false)
+    }
+  }
+
+  /**
+   * Walks through various scenarios where files are compressed and uploaded.
+   */
+  @Test
+  fun compressionAndUploads() {
+    // Matches after the first is compressed, skip transform properly set
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, true)
+    }
+
+    // Matches after the first is uploaded, skip transform and ending hash properly set
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1)
+
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, true)
+    }
+
+    // Mimics sending two files at once. Ensures all fields are kept in sync as we compress and upload.
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, false)
+      assertSkipTransform(id2, false)
+
+      compress(id1, DATA_A_COMPRESSED)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, true)
+
+      upload(id1)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertRemoteFieldsMatch(id1, id2)
+    }
+
+    // Re-use the upload when uploaded recently
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertRemoteFieldsMatch(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, true)
+    }
+
+    // Do not re-use old uploads
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis() - 100.days.inWholeMilliseconds)
+
+      val id2 = insertWithData(DATA_A)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashStartMatches(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, true)
+
+      assertDoesNotHaveRemoteFields(id2)
+    }
+
+    // This isn't so much "desirable behavior" as it is documenting how things work.
+    // If an attachment is compressed but not uploaded yet, it will have a DATA_HASH_START that doesn't match the actual file content.
+    // This means that if we insert a new attachment with data that matches the compressed data, we won't find a match.
+    // This is ok because we don't allow forwarding unsent messages, so the chances of the user somehow sending a file that matches data we compressed are very low.
+    // What *is* more common is that the user may send DATA_A again, and in this case we will still catch the dedupe (which is already tested above).
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+
+      val id2 = insertWithData(DATA_A_COMPRESSED)
+
+      assertDataFilesAreDifferent(id1, id2)
+    }
+
+    // This represents what would happen if you forward an already-send compressed attachment. We should match, skip transform, and skip upload.
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A_COMPRESSED)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id1, true)
+      assertRemoteFieldsMatch(id1, id2)
+    }
+
+    // This represents what would happen if you edited a video, sent it, then forwarded it. We should match, skip transform, and skip upload.
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A_COMPRESSED)
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id1, true)
+      assertRemoteFieldsMatch(id1, id2)
+    }
+
+    // This represents what would happen if you edited a video, sent it, then forwarded it, but *edited the forwarded video*. We should not dedupe.
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(videoTrim = true, videoTrimStartTimeUs = 1, videoTrimEndTimeUs = 2))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, false)
+      assertDoesNotHaveRemoteFields(id2)
+    }
+
+    // This represents what would happen if you sent an image using standard quality, then forwarded it using high quality.
+    // Since you're forwarding, it doesn't matter if the new thing has a higher quality, we should still match and skip transform.
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
+
+      assertDataFilesAreTheSame(id1, id2)
+      assertDataHashEndMatches(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id1, true)
+      assertRemoteFieldsMatch(id1, id2)
+    }
+
+    // This represents what would happen if you sent an image using high quality, then forwarded it using standard quality.
+    // Lowering the quality would change the output, so we shouldn't dedupe.
+    test {
+      val id1 = insertWithData(DATA_A, TransformProperties(sentMediaQuality = SentMediaQuality.HIGH.code))
+      compress(id1, DATA_A_COMPRESSED)
+      upload(id1, uploadTimestamp = System.currentTimeMillis())
+
+      val id2 = insertWithData(DATA_A_COMPRESSED, TransformProperties(sentMediaQuality = SentMediaQuality.STANDARD.code))
+
+      assertDataFilesAreDifferent(id1, id2)
+      assertSkipTransform(id1, true)
+      assertSkipTransform(id2, false)
+      assertDoesNotHaveRemoteFields(id2)
+    }
+  }
+
+  /**
+   * Various deletion scenarios to ensure that duped files don't deleted while there's still references.
+   */
+  @Test
+  fun deletions() {
+    // Delete original then dupe
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertWithData(DATA_A)
+      val dataFile = dataFile(id1)
+
+      assertDataFilesAreTheSame(id1, id2)
+
+      delete(id1)
+
+      assertDeleted(id1)
+      assertRowAndFileExists(id2)
+      assertTrue(dataFile.exists())
+
+      delete(id2)
+
+      assertDeleted(id2)
+      assertFalse(dataFile.exists())
+    }
+
+    // Delete dupe then original
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertWithData(DATA_A)
+      val dataFile = dataFile(id1)
+
+      assertDataFilesAreTheSame(id1, id2)
+
+      delete(id2)
+      assertDeleted(id2)
+      assertRowAndFileExists(id1)
+      assertTrue(dataFile.exists())
+
+      delete(id1)
+      assertDeleted(id1)
+      assertFalse(dataFile.exists())
+    }
+
+    // Delete original after it was compressed
+    test {
+      val id1 = insertWithData(DATA_A)
+      compress(id1, DATA_A_COMPRESSED)
+
+      val id2 = insertWithData(DATA_A)
+
+      delete(id1)
+
+      assertDeleted(id1)
+      assertRowAndFileExists(id2)
+      assertSkipTransform(id2, true)
+    }
+
+    // Quotes are weak references and should not prevent us from deleting the file
+    test {
+      val id1 = insertWithData(DATA_A)
+      val id2 = insertQuote(id1)
+
+      val dataFile = dataFile(id1)
+
+      delete(id1)
+      assertDeleted(id1)
+      assertRowExists(id2)
+      assertFalse(dataFile.exists())
+    }
+  }
+
+  private class TestContext {
+    fun insertWithData(data: ByteArray, transformProperties: TransformProperties = TransformProperties.empty()): AttachmentId {
+      val uri = BlobProvider.getInstance().forData(data).createForSingleSessionInMemory()
+
+      val attachment = UriAttachmentBuilder.build(
+        id = Random.nextLong(),
+        uri = uri,
+        contentType = MediaUtil.IMAGE_JPEG,
+        transformProperties = transformProperties
+      )
+
+      return SignalDatabase.attachments.insertAttachmentForPreUpload(attachment).attachmentId
+    }
+
+    fun insertQuote(attachmentId: AttachmentId): AttachmentId {
+      val originalAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
+      val threadId = SignalDatabase.threads.getOrCreateThreadIdFor(Recipient.self())
+      val messageId = SignalDatabase.messages.insertMessageOutbox(
+        message = OutgoingMessage(
+          threadRecipient = Recipient.self(),
+          sentTimeMillis = System.currentTimeMillis(),
+          body = "some text",
+          outgoingQuote = QuoteModel(
+            id = 123,
+            author = Recipient.self().id,
+            text = "Some quote text",
+            isOriginalMissing = false,
+            attachments = listOf(originalAttachment),
+            mentions = emptyList(),
+            type = QuoteModel.Type.NORMAL,
+            bodyRanges = null
+          )
+        ),
+        threadId = threadId,
+        forceSms = false,
+        insertListener = null
+      )
+
+      val attachments = SignalDatabase.attachments.getAttachmentsForMessage(messageId)
+      return attachments[0].attachmentId
+    }
+
+    fun compress(attachmentId: AttachmentId, newData: ByteArray, mp4FastStart: Boolean = false) {
+      val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
+      SignalDatabase.attachments.updateAttachmentData(databaseAttachment, newData.asMediaStream())
+      SignalDatabase.attachments.markAttachmentAsTransformed(attachmentId, withFastStart = mp4FastStart)
+    }
+
+    fun upload(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()) {
+      SignalDatabase.attachments.finalizeAttachmentAfterUpload(attachmentId, createPointerAttachment(attachmentId, uploadTimestamp), uploadTimestamp)
+    }
+
+    fun delete(attachmentId: AttachmentId) {
+      SignalDatabase.attachments.deleteAttachment(attachmentId)
+    }
+
+    fun dataFile(attachmentId: AttachmentId): File {
+      return SignalDatabase.attachments.getDataFileInfo(attachmentId)!!.file
+    }
+
+    fun assertDeleted(attachmentId: AttachmentId) {
+      assertNull("$attachmentId exists, but it shouldn't!", SignalDatabase.attachments.getAttachment(attachmentId))
+    }
+
+    fun assertRowAndFileExists(attachmentId: AttachmentId) {
+      val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)
+      assertNotNull("$attachmentId does not exist!", databaseAttachment)
+
+      val dataFileInfo = SignalDatabase.attachments.getDataFileInfo(attachmentId)
+      assertTrue("The file for $attachmentId does not exist!", dataFileInfo!!.file.exists())
+    }
+
+    fun assertRowExists(attachmentId: AttachmentId) {
+      val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)
+      assertNotNull("$attachmentId does not exist!", databaseAttachment)
+    }
+
+    fun assertDataFilesAreTheSame(lhs: AttachmentId, rhs: AttachmentId) {
+      val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
+      val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
+
+      assert(lhsInfo.file.exists())
+      assert(rhsInfo.file.exists())
+
+      assertEquals(lhsInfo.file, rhsInfo.file)
+      assertEquals(lhsInfo.length, rhsInfo.length)
+      assertArrayEquals(lhsInfo.random, rhsInfo.random)
+    }
+
+    fun assertDataFilesAreDifferent(lhs: AttachmentId, rhs: AttachmentId) {
+      val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
+      val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
+
+      assert(lhsInfo.file.exists())
+      assert(rhsInfo.file.exists())
+
+      assertNotEquals(lhsInfo.file, rhsInfo.file)
+    }
+
+    fun assertDataHashStartMatches(lhs: AttachmentId, rhs: AttachmentId) {
+      val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
+      val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
+
+      assertEquals("DATA_HASH_START's did not match!", lhsInfo.hashStart, rhsInfo.hashStart)
+    }
+
+    fun assertDataHashEndMatches(lhs: AttachmentId, rhs: AttachmentId) {
+      val lhsInfo = SignalDatabase.attachments.getDataFileInfo(lhs)!!
+      val rhsInfo = SignalDatabase.attachments.getDataFileInfo(rhs)!!
+
+      assertEquals("DATA_HASH_END's did not match!", lhsInfo.hashEnd, rhsInfo.hashEnd)
+    }
+
+    fun assertRemoteFieldsMatch(lhs: AttachmentId, rhs: AttachmentId) {
+      val lhsAttachment = SignalDatabase.attachments.getAttachment(lhs)!!
+      val rhsAttachment = SignalDatabase.attachments.getAttachment(rhs)!!
+
+      assertEquals(lhsAttachment.remoteLocation, rhsAttachment.remoteLocation)
+      assertEquals(lhsAttachment.remoteKey, rhsAttachment.remoteKey)
+      assertArrayEquals(lhsAttachment.remoteDigest, rhsAttachment.remoteDigest)
+      assertArrayEquals(lhsAttachment.incrementalDigest, rhsAttachment.incrementalDigest)
+      assertEquals(lhsAttachment.incrementalMacChunkSize, rhsAttachment.incrementalMacChunkSize)
+      assertEquals(lhsAttachment.cdnNumber, rhsAttachment.cdnNumber)
+    }
+
+    fun assertDoesNotHaveRemoteFields(attachmentId: AttachmentId) {
+      val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
+      assertEquals(0, databaseAttachment.uploadTimestamp)
+      assertNull(databaseAttachment.remoteLocation)
+      assertNull(databaseAttachment.remoteDigest)
+      assertNull(databaseAttachment.remoteKey)
+      assertEquals(0, databaseAttachment.cdnNumber)
+    }
+
+    fun assertSkipTransform(attachmentId: AttachmentId, state: Boolean) {
+      val transformProperties = SignalDatabase.attachments.getTransformProperties(attachmentId)!!
+      assertEquals("Incorrect skipTransform!", transformProperties.skipTransform, state)
+    }
+
+    private fun ByteArray.asMediaStream(): MediaStream {
+      return MediaStream(this.inputStream(), MediaUtil.IMAGE_JPEG, 2, 2)
+    }
+
+    private fun createPointerAttachment(attachmentId: AttachmentId, uploadTimestamp: Long = System.currentTimeMillis()): PointerAttachment {
+      val location = "somewhere-${Random.nextLong()}"
+      val key = "somekey-${Random.nextLong()}"
+      val digest = Random.nextBytes(32)
+      val incrementalDigest = Random.nextBytes(16)
+
+      val databaseAttachment = SignalDatabase.attachments.getAttachment(attachmentId)!!
+
+      return PointerAttachment(
+        "image/jpeg",
+        AttachmentTable.TRANSFER_PROGRESS_DONE,
+        databaseAttachment.size, // size
+        null,
+        3, // cdnNumber
+        location,
+        key,
+        digest,
+        incrementalDigest,
+        5, // incrementalMacChunkSize
+        null,
+        databaseAttachment.voiceNote,
+        databaseAttachment.borderless,
+        databaseAttachment.videoGif,
+        databaseAttachment.width,
+        databaseAttachment.height,
+        uploadTimestamp,
+        databaseAttachment.caption,
+        databaseAttachment.stickerLocator,
+        databaseAttachment.blurHash
+      )
+    }
+  }
+
+  private fun test(content: TestContext.() -> Unit) {
+    SignalDatabase.attachments.deleteAllAttachments()
+    val context = TestContext()
+    context.content()
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/attachments/AttachmentId.kt

@@ -3,13 +3,14 @@ package org.thoughtcrime.securesms.attachments
 import android.os.Parcelable
 import com.fasterxml.jackson.annotation.JsonProperty
 import kotlinx.parcelize.Parcelize
+import org.signal.core.util.DatabaseId
 
 @Parcelize
 data class AttachmentId(
   @JsonProperty("rowId")
   @JvmField
   val id: Long
-) : Parcelable {
+) : Parcelable, DatabaseId {
 
   val isValid: Boolean
     get() = id >= 0
@@ -17,4 +18,8 @@ data class AttachmentId(
   override fun toString(): String {
     return "AttachmentId::$id"
   }
+
+  override fun serialize(): String {
+    return id.toString()
+  }
 }

app/src/main/java/org/thoughtcrime/securesms/attachments/PointerAttachment.kt

@@ -2,6 +2,7 @@ package org.thoughtcrime.securesms.attachments
 
 import android.net.Uri
 import android.os.Parcel
+import androidx.annotation.VisibleForTesting
 import org.signal.core.util.Base64.encodeWithPadding
 import org.thoughtcrime.securesms.blurhash.BlurHash
 import org.thoughtcrime.securesms.database.AttachmentTable
@@ -14,7 +15,8 @@ import org.whispersystems.signalservice.internal.push.DataMessage
 import java.util.Optional
 
 class PointerAttachment : Attachment {
-  private constructor(
+  @VisibleForTesting
+  constructor(
     contentType: String,
     transferState: Int,
     size: Long,

app/src/main/java/org/thoughtcrime/securesms/backup/FullBackupImporter.java

@@ -194,7 +194,7 @@ public class FullBackupImporter extends FullBackupBase {
   private static void processAttachment(@NonNull Context context, @NonNull AttachmentSecret attachmentSecret, @NonNull SQLiteDatabase db, @NonNull Attachment attachment, BackupRecordInputStream inputStream)
       throws IOException
   {
-    File                       dataFile      = AttachmentTable.newFile(context);
+    File                       dataFile      = AttachmentTable.newDataFile(context);
     Pair<byte[], OutputStream> output        = ModernEncryptingPartOutputStream.createFor(attachmentSecret, dataFile, false);
     boolean                    isLegacyTable = SqlUtil.tableExists(db, "part");
 

app/src/main/java/org/thoughtcrime/securesms/database/MediaTable.kt

@@ -49,7 +49,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
         ${AttachmentTable.TABLE_NAME}.${AttachmentTable.UPLOAD_TIMESTAMP}, 
         ${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST}, 
         ${AttachmentTable.TABLE_NAME}.${AttachmentTable.REMOTE_INCREMENTAL_DIGEST_CHUNK_SIZE},
-        ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH},
+        ${AttachmentTable.TABLE_NAME}.${AttachmentTable.DATA_HASH_END},
         ${MessageTable.TABLE_NAME}.${MessageTable.TYPE}, 
         ${MessageTable.TABLE_NAME}.${MessageTable.DATE_SENT}, 
         ${MessageTable.TABLE_NAME}.${MessageTable.DATE_RECEIVED}, 
@@ -71,13 +71,7 @@ class MediaTable internal constructor(context: Context?, databaseHelper: SignalD
         ${MessageTable.VIEW_ONCE} = 0 AND 
         ${MessageTable.STORY_TYPE} = 0 AND
         ${MessageTable.LATEST_REVISION_ID} IS NULL AND 
-        (
+        ${AttachmentTable.QUOTE} = 0 AND 
-          ${AttachmentTable.QUOTE} = 0 OR 
-          (
-            ${AttachmentTable.QUOTE} = 1 AND 
-            ${AttachmentTable.DATA_HASH} IS NULL
-          )
-        ) AND 
         ${AttachmentTable.STICKER_PACK_ID} IS NULL AND 
         ${MessageTable.TABLE_NAME}.${MessageTable.FROM_RECIPIENT_ID} > 0 AND 
         $THREAD_RECIPIENT_ID > 0

app/src/main/java/org/thoughtcrime/securesms/database/helpers/SignalDatabaseMigrations.kt

@@ -79,6 +79,7 @@ import org.thoughtcrime.securesms.database.helpers.migration.V218_RecipientPniSi
 import org.thoughtcrime.securesms.database.helpers.migration.V219_PniPreKeyStores
 import org.thoughtcrime.securesms.database.helpers.migration.V220_PreKeyConstraints
 import org.thoughtcrime.securesms.database.helpers.migration.V221_AddReadColumnToCallEventsTable
+import org.thoughtcrime.securesms.database.helpers.migration.V222_DataHashRefactor
 
 /**
  * Contains all of the database migrations for [SignalDatabase]. Broken into a separate file for cleanliness.
@@ -160,10 +161,11 @@ object SignalDatabaseMigrations {
     218 to V218_RecipientPniSignatureVerified,
     219 to V219_PniPreKeyStores,
     220 to V220_PreKeyConstraints,
-    221 to V221_AddReadColumnToCallEventsTable
+    221 to V221_AddReadColumnToCallEventsTable,
+    222 to V222_DataHashRefactor
   )
 
-  const val DATABASE_VERSION = 221
+  const val DATABASE_VERSION = 222
 
   @JvmStatic
   fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {

app/src/main/java/org/thoughtcrime/securesms/database/helpers/migration/V222_DataHashRefactor.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.thoughtcrime.securesms.database.helpers.migration
+
+import android.app.Application
+import net.zetetic.database.sqlcipher.SQLiteDatabase
+
+/**
+ * Adds the new data hash columns and indexes.
+ */
+@Suppress("ClassName")
+object V222_DataHashRefactor : SignalDatabaseMigration {
+  override fun migrate(context: Application, db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
+    db.execSQL("DROP INDEX attachment_data_hash_index")
+    db.execSQL("ALTER TABLE attachment DROP COLUMN data_hash")
+
+    db.execSQL("ALTER TABLE attachment ADD COLUMN data_hash_start TEXT DEFAULT NULL")
+    db.execSQL("ALTER TABLE attachment ADD COLUMN data_hash_end TEXT DEFAULT NULL")
+    db.execSQL("CREATE INDEX attachment_data_hash_start_index ON attachment (data_hash_start)")
+    db.execSQL("CREATE INDEX attachment_data_hash_end_index ON attachment (data_hash_end)")
+  }
+}

app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java

@@ -205,7 +205,7 @@ public final class AttachmentCompressionJob extends BaseJob {
       } else if (constraints.canResize(attachment)) {
         Log.i(TAG, "Compressing image.");
         try (MediaStream converted = compressImage(context, attachment, constraints)) {
-          attachmentDatabase.updateAttachmentData(attachment, converted, false);
+          attachmentDatabase.updateAttachmentData(attachment, converted);
         }
         attachmentDatabase.markAttachmentAsTransformed(attachmentId, false);
       } else if (constraints.isSatisfied(context, attachment)) {
@@ -263,7 +263,7 @@ public final class AttachmentCompressionJob extends BaseJob {
             Log.i(TAG, "Compressing with streaming muxer");
             AttachmentSecret attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret();
 
-            File file = AttachmentTable.newFile(context);
+            File file = AttachmentTable.newDataFile(context);
             file.deleteOnExit();
 
             boolean faststart = false;
@@ -296,7 +296,7 @@ public final class AttachmentCompressionJob extends BaseJob {
 
               final long plaintextLength = ModernEncryptingPartOutputStream.getPlaintextLength(file.length());
               try (MediaStream mediaStream = new MediaStream(postProcessor.process(plaintextLength), MimeTypes.VIDEO_MP4, 0, 0, true)) {
-                attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
+                attachmentDatabase.updateAttachmentData(attachment, mediaStream);
                 faststart = true;
               } catch (VideoPostProcessingException e) {
                 Log.w(TAG, "Exception thrown during post processing.", e);
@@ -310,7 +310,7 @@ public final class AttachmentCompressionJob extends BaseJob {
 
               if (!faststart) {
                 try (MediaStream mediaStream = new MediaStream(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0), MimeTypes.VIDEO_MP4, 0, 0, false)) {
-                  attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
+                  attachmentDatabase.updateAttachmentData(attachment, mediaStream);
                 }
               }
             } finally {
@@ -339,7 +339,7 @@ public final class AttachmentCompressionJob extends BaseJob {
                                                           100,
                                                           percent));
               }, cancelationSignal)) {
-                attachmentDatabase.updateAttachmentData(attachment, mediaStream, true);
+                attachmentDatabase.updateAttachmentData(attachment, mediaStream);
                 attachmentDatabase.markAttachmentAsTransformed(attachment.attachmentId, mediaStream.getFaststart());
               }
 

app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentDownloadJob.java

@@ -168,7 +168,7 @@ public final class AttachmentDownloadJob extends BaseJob {
     if (attachment.cdnNumber != ReleaseChannel.CDN_NUMBER) {
       retrieveAttachment(messageId, attachmentId, attachment);
     } else {
-      retrieveUrlAttachment(messageId, attachmentId, attachment);
+      retrieveAttachmentForReleaseChannel(messageId, attachmentId, attachment);
     }
   }
 
@@ -216,7 +216,7 @@ public final class AttachmentDownloadJob extends BaseJob {
                                                                   return isCanceled();
                                                                 }
                                                               });
-      database.insertAttachmentsForPlaceholder(messageId, attachmentId, stream);
+      database.finalizeAttachmentAfterDownload(messageId, attachmentId, stream);
     } catch (RangeException e) {
       Log.w(TAG, "Range exception, file size " + attachmentFile.length(), e);
       if (attachmentFile.delete()) {
@@ -278,9 +278,9 @@ public final class AttachmentDownloadJob extends BaseJob {
     }
   }
 
-  private void retrieveUrlAttachment(long messageId,
+  private void retrieveAttachmentForReleaseChannel(long messageId,
-                                     final AttachmentId attachmentId,
+                                                   final AttachmentId attachmentId,
-                                     final Attachment attachment)
+                                                   final Attachment attachment)
       throws IOException
   {
     try (Response response = S3.getObject(Objects.requireNonNull(attachment.fileName))) {
@@ -289,7 +289,7 @@ public final class AttachmentDownloadJob extends BaseJob {
         if (body.contentLength() > FeatureFlags.maxAttachmentReceiveSizeBytes()) {
           throw new MmsException("Attachment too large, failing download");
         }
-        SignalDatabase.attachments().insertAttachmentsForPlaceholder(messageId, attachmentId, Okio.buffer(body.source()).inputStream());
+        SignalDatabase.attachments().finalizeAttachmentAfterDownload(messageId, attachmentId, Okio.buffer(body.source()).inputStream());
       }
     } catch (MmsException e) {
       Log.w(TAG, "Experienced exception while trying to download an attachment.", e);

app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentUploadJob.kt

@@ -42,6 +42,7 @@ import java.io.IOException
 import java.util.Objects
 import java.util.Optional
 import java.util.concurrent.TimeUnit
+import kotlin.time.Duration.Companion.days
 import kotlin.time.Duration.Companion.milliseconds
 
 /**
@@ -60,7 +61,7 @@ class AttachmentUploadJob private constructor(
 
     private val TAG = Log.tag(AttachmentUploadJob::class.java)
 
-    private val UPLOAD_REUSE_THRESHOLD = TimeUnit.DAYS.toMillis(3)
+    val UPLOAD_REUSE_THRESHOLD = 3.days.inWholeMilliseconds
 
     /**
      * Foreground notification shows while uploading attachments above this.
@@ -162,7 +163,7 @@ class AttachmentUploadJob private constructor(
         buildAttachmentStream(databaseAttachment, notification, uploadSpec!!).use { localAttachment ->
           val remoteAttachment = messageSender.uploadAttachment(localAttachment)
           val attachment = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.fastPreflightId).get()
-          SignalDatabase.attachments.updateAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.uploadTimestamp)
+          SignalDatabase.attachments.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.uploadTimestamp)
         }
       }
     } catch (e: NonSuccessfulResumableUploadResponseCodeException) {

app/src/main/java/org/thoughtcrime/securesms/jobs/LegacyAttachmentUploadJob.java

@@ -139,7 +139,7 @@ public final class LegacyAttachmentUploadJob extends BaseJob {
         SignalServiceAttachmentPointer remoteAttachment = messageSender.uploadAttachment(localAttachment);
         Attachment                     attachment       = PointerAttachment.forPointer(Optional.of(remoteAttachment), null, databaseAttachment.fastPreflightId).get();
 
-        database.updateAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.getUploadTimestamp());
+        database.finalizeAttachmentAfterUpload(databaseAttachment.attachmentId, attachment, remoteAttachment.getUploadTimestamp());
       }
     } catch (NonSuccessfulResumableUploadResponseCodeException e) {
       if (e.getCode() == 400) {

app/src/main/java/org/thoughtcrime/securesms/stickers/StickerSearchRepository.java

@@ -64,11 +64,6 @@ public final class StickerSearchRepository {
     return out;
   }
 
-  public @NonNull Single<Boolean> getStickerFeatureAvailability() {
-    return Single.fromCallable(this::getStickerFeatureAvailabilitySync)
-                 .observeOn(Schedulers.io());
-  }
-
   public void getStickerFeatureAvailability(@NonNull Callback<Boolean> callback) {
     SignalExecutors.BOUNDED.execute(() -> {
       callback.onResult(getStickerFeatureAvailabilitySync());

core-util-jvm/src/main/java/org/signal/core/util/InputStreamExtensions.kt

@@ -7,7 +7,6 @@ package org.signal.core.util
 
 import java.io.IOException
 import java.io.InputStream
-import kotlin.jvm.Throws
 
 /**
  * Reads a 32-bit variable-length integer from the stream.
@@ -80,3 +79,11 @@ fun InputStream.readLength(): Long {
 
   return count
 }
+
+/**
+ * Reads the contents of the stream and discards them.
+ */
+@Throws(IOException::class)
+fun InputStream.drain() {
+  this.readLength()
+}

libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageReceiver.java

@@ -142,7 +142,8 @@ public class SignalServiceMessageReceiver {
   }
 
   /**
-   * Retrieves a SignalServiceAttachment.
+   * Retrieves a SignalServiceAttachment. The encrypted data is written to @{code destination}, and then an {@link InputStream} is returned that decrypts the
+   * contents of the destination file, giving you access to the plaintext content.
    *
    * @param pointer The {@link SignalServiceAttachmentPointer}
    *                received in a {@link SignalServiceDataMessage}.