From 6fa833705872c8d884b62b1517c704ba1b33074e Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Tue, 18 Jun 2024 16:28:46 -0400 Subject: [PATCH] Update to the latest backup v2 spec. Removes some dead protos, removes some sticker details, adds in gift badges. --- .../securesms/backup/v2/ImportExportTest.kt | 86 ++++++++++++++++++- .../v2/database/ChatItemExportIterator.kt | 36 ++++++-- .../v2/database/ChatItemImportInserter.kt | 25 +++++- .../v2/processor/StickerBackupProcessor.kt | 30 +------ app/src/main/protowire/Backup.proto | 41 ++++----- 5 files changed, 153 insertions(+), 65 deletions(-) diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ImportExportTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ImportExportTest.kt index 00c5ee7a3b..75e8302a89 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ImportExportTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/backup/v2/ImportExportTest.kt @@ -31,6 +31,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.DistributionListItem import org.thoughtcrime.securesms.backup.v2.proto.ExpirationTimerChatUpdate import org.thoughtcrime.securesms.backup.v2.proto.FilePointer import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge import org.thoughtcrime.securesms.backup.v2.proto.Group import org.thoughtcrime.securesms.backup.v2.proto.IndividualCall import org.thoughtcrime.securesms.backup.v2.proto.MessageAttachment @@ -50,6 +51,7 @@ import org.thoughtcrime.securesms.backup.v2.proto.ThreadMergeChatUpdate import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupWriter import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.util.Util import org.whispersystems.signalservice.api.kbs.MasterKey import org.whispersystems.signalservice.api.push.DistributionId import org.whispersystems.signalservice.api.push.ServiceId @@ -1236,6 +1238,76 @@ class ImportExportTest { ) } + @Test + fun giftBadgeMessage() { + var dateSentStart = 100L + importExport( + *standardFrames, + alice, + buildChat(alice, 1), + ChatItem( + chatId = 1, + authorId = alice.id, + dateSent = dateSentStart++, + incoming = ChatItem.IncomingMessageDetails( + dateReceived = dateSentStart, + dateServerSent = dateSentStart, + read = true, + sealedSender = true + ), + giftBadge = GiftBadge( + receiptCredentialPresentation = Util.getSecretBytes(32).toByteString(), + state = GiftBadge.State.OPENED + ) + ), + ChatItem( + chatId = 1, + authorId = alice.id, + dateSent = dateSentStart++, + incoming = ChatItem.IncomingMessageDetails( + dateReceived = dateSentStart, + dateServerSent = dateSentStart, + read = true, + sealedSender = true + ), + giftBadge = GiftBadge( + receiptCredentialPresentation = Util.getSecretBytes(32).toByteString(), + state = GiftBadge.State.FAILED + ) + ), + ChatItem( + chatId = 1, + authorId = alice.id, + dateSent = dateSentStart++, + incoming = ChatItem.IncomingMessageDetails( + dateReceived = dateSentStart, + dateServerSent = dateSentStart, + read = true, + sealedSender = true + ), + giftBadge = GiftBadge( + receiptCredentialPresentation = Util.getSecretBytes(32).toByteString(), + state = GiftBadge.State.REDEEMED + ) + ), + ChatItem( + chatId = 1, + authorId = alice.id, + dateSent = dateSentStart++, + incoming = ChatItem.IncomingMessageDetails( + dateReceived = dateSentStart, + dateServerSent = dateSentStart, + read = true, + sealedSender = true + ), + giftBadge = GiftBadge( + receiptCredentialPresentation = Util.getSecretBytes(32).toByteString(), + state = GiftBadge.State.UNOPENED + ) + ) + ) + } + fun enumerateIncomingMessageDetails(dateSent: Long): List { val details = mutableListOf() details.add( @@ -1508,17 +1580,29 @@ class ImportExportTest { private inline fun > prettyAssertEquals(import: List, export: List, crossinline selector: (T) -> R?) { if (import.size != export.size) { - var msg = StringBuilder() + val msg = StringBuilder() + msg.append("There's a different number of items in the lists!\n\n") + + msg.append("Imported:\n") for (i in import) { msg.append(i) msg.append("\n") } + if (import.isEmpty()) { + msg.append("") + } + msg.append("\n") + msg.append("Exported:\n") for (i in export) { msg.append(i) msg.append("\n") } + if (export.isEmpty()) { + msg.append("") + } Assert.fail(msg.toString()) } + Assert.assertEquals(import.size, export.size) val sortedImport = import.sortedBy(selector) val sortedExport = export.sortedBy(selector) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt index d009b8ae99..d440244d72 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemExportIterator.kt @@ -8,8 +8,6 @@ package org.thoughtcrime.securesms.backup.v2.database import android.database.Cursor import okio.ByteString.Companion.toByteString import org.signal.core.util.Base64 -import org.signal.core.util.Base64.decode -import org.signal.core.util.Base64.decodeOrThrow import org.signal.core.util.Hex import org.signal.core.util.logging.Log import org.signal.core.util.requireBlob @@ -59,6 +57,7 @@ import org.thoughtcrime.securesms.database.model.Mention import org.thoughtcrime.securesms.database.model.ReactionRecord import org.thoughtcrime.securesms.database.model.databaseprotos.BodyRangeList import org.thoughtcrime.securesms.database.model.databaseprotos.DecryptedGroupV2Context +import org.thoughtcrime.securesms.database.model.databaseprotos.GiftBadge import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExtras import org.thoughtcrime.securesms.database.model.databaseprotos.ProfileChangeDetails import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchoverEvent @@ -79,6 +78,7 @@ import java.util.LinkedList import java.util.Queue import kotlin.jvm.optionals.getOrNull import org.thoughtcrime.securesms.backup.v2.proto.BodyRange as BackupBodyRange +import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge /** * An iterator for chat items with a clever performance twist: rather than do the extra queries one at a time (for reactions, @@ -173,7 +173,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: MessageTypes.isSessionSwitchoverType(record.type) -> { builder.updateMessage = ChatUpdateMessage( sessionSwitchover = try { - val event = SessionSwitchoverEvent.ADAPTER.decode(decodeOrThrow(record.body!!)) + val event = SessionSwitchoverEvent.ADAPTER.decode(Base64.decodeOrThrow(record.body!!)) SessionSwitchoverChatUpdate(event.e164.e164ToLong()!!) } catch (e: Exception) { SessionSwitchoverChatUpdate() @@ -183,7 +183,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: MessageTypes.isThreadMergeType(record.type) -> { builder.updateMessage = ChatUpdateMessage( threadMerge = try { - val event = ThreadMergeEvent.ADAPTER.decode(decodeOrThrow(record.body!!)) + val event = ThreadMergeEvent.ADAPTER.decode(Base64.decodeOrThrow(record.body!!)) ThreadMergeChatUpdate(event.previousE164.e164ToLong()!!) } catch (e: Exception) { ThreadMergeChatUpdate() @@ -198,7 +198,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: ) } else if (record.body != null) { try { - val decoded: ByteArray = decode(record.body) + val decoded: ByteArray = Base64.decode(record.body) val context = DecryptedGroupV2Context.ADAPTER.decode(decoded) builder.updateMessage = ChatUpdateMessage( groupChange = GroupsV2UpdateMessageConverter.translateDecryptedChange(selfIds = SignalStore.account().getServiceIds(), context) @@ -347,6 +347,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: ) } } + MessageTypes.isGiftBadge(record.type) -> { builder.giftBadge = record.toBackupGiftBadge() } else -> { if (record.body == null && !attachmentsById.containsKey(record.id)) { Log.w(TAG, "Record missing a body and doesnt have attachments, skipping") @@ -479,6 +480,25 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: } } + private fun BackupMessageRecord.toBackupGiftBadge(): BackupGiftBadge { + val giftBadge = try { + GiftBadge.ADAPTER.decode(Base64.decode(this.body ?: "")) + } catch (e: IOException) { + Log.w(TAG, "Failed to decode GiftBadge!") + return BackupGiftBadge() + } + + return BackupGiftBadge( + receiptCredentialPresentation = giftBadge.redemptionToken, + state = when (giftBadge.redemptionState) { + GiftBadge.RedemptionState.REDEEMED -> BackupGiftBadge.State.REDEEMED + GiftBadge.RedemptionState.FAILED -> BackupGiftBadge.State.FAILED + GiftBadge.RedemptionState.PENDING -> BackupGiftBadge.State.UNOPENED + GiftBadge.RedemptionState.STARTED -> BackupGiftBadge.State.OPENED + } + ) + } + private fun List.toBackupQuoteAttachments(): List { return this.map { attachment -> Quote.QuotedAttachment( @@ -507,7 +527,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: builder.backupLocator = FilePointer.BackupLocator( mediaName = archiveMediaName ?: this.getMediaName().toString(), cdnNumber = if (archiveMediaName != null) archiveCdn else Cdn.CDN_3.cdnNumber, // TODO (clark): Update when new proto with optional cdn is landed - key = decode(remoteKey).toByteString(), + key = Base64.decode(remoteKey).toByteString(), size = this.size.toInt(), digest = remoteDigest.toByteString() ) @@ -519,7 +539,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: cdnKey = this.remoteLocation, cdnNumber = this.cdn.cdnNumber, uploadTimestamp = this.uploadTimestamp, - key = decode(remoteKey).toByteString(), + key = Base64.decode(remoteKey).toByteString(), size = this.size.toInt(), digest = remoteDigest.toByteString() ) @@ -538,7 +558,7 @@ class ChatItemExportIterator(private val cursor: Cursor, private val batchSize: } else { MessageAttachment.Flag.NONE }, - uuid = uuid?.let { UuidUtil.toByteString(uuid) } + clientUuid = uuid?.let { UuidUtil.toByteString(uuid) } ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt index 13de09ab78..1324a462b7 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/database/ChatItemImportInserter.kt @@ -53,6 +53,7 @@ import org.thoughtcrime.securesms.database.model.Mention import org.thoughtcrime.securesms.database.model.databaseprotos.BodyRangeList import org.thoughtcrime.securesms.database.model.databaseprotos.CryptoValue import org.thoughtcrime.securesms.database.model.databaseprotos.GV2UpdateDescription +import org.thoughtcrime.securesms.database.model.databaseprotos.GiftBadge import org.thoughtcrime.securesms.database.model.databaseprotos.MessageExtras import org.thoughtcrime.securesms.database.model.databaseprotos.PaymentTombstone import org.thoughtcrime.securesms.database.model.databaseprotos.ProfileChangeDetails @@ -77,6 +78,7 @@ import org.whispersystems.signalservice.internal.push.DataMessage import java.math.BigInteger import java.util.Optional import java.util.UUID +import org.thoughtcrime.securesms.backup.v2.proto.GiftBadge as BackupGiftBadge /** * An object that will ingest all fo the [ChatItem]s you want to write, buffer them until hitting a specified batch size, and then batch insert them @@ -410,6 +412,7 @@ class ChatItemImportInserter( this.remoteDeletedMessage != null -> contentValues.put(MessageTable.REMOTE_DELETED, 1) this.updateMessage != null -> contentValues.addUpdateMessage(this.updateMessage) this.paymentNotification != null -> contentValues.addPaymentNotification(this, chatRecipientId) + this.giftBadge != null -> contentValues.addGiftBadge(this.giftBadge) } return contentValues @@ -517,6 +520,10 @@ class ChatItemImportInserter( type = type or MessageTypes.SECURE_MESSAGE_BIT or MessageTypes.PUSH_MESSAGE_BIT } + if (this.giftBadge != null) { + type = type or MessageTypes.SPECIAL_TYPE_GIFT_BADGE + } + return type } @@ -695,6 +702,20 @@ class ChatItemImportInserter( ) } + private fun ContentValues.addGiftBadge(giftBadge: BackupGiftBadge) { + val dbGiftBadge = GiftBadge( + redemptionToken = giftBadge.receiptCredentialPresentation, + redemptionState = when (giftBadge.state) { + BackupGiftBadge.State.UNOPENED -> GiftBadge.RedemptionState.PENDING + BackupGiftBadge.State.OPENED -> GiftBadge.RedemptionState.STARTED + BackupGiftBadge.State.REDEEMED -> GiftBadge.RedemptionState.REDEEMED + BackupGiftBadge.State.FAILED -> GiftBadge.RedemptionState.FAILED + } + ) + + put(MessageTable.BODY, Base64.encodeWithPadding(GiftBadge.ADAPTER.encode(dbGiftBadge))) + } + private fun String?.tryParseMoney(): Money? { if (this.isNullOrEmpty()) { return null @@ -915,7 +936,7 @@ class ChatItemImportInserter( gif = flag == MessageAttachment.Flag.GIF, borderless = flag == MessageAttachment.Flag.BORDERLESS, wasDownloaded = wasDownloaded, - uuid = uuid + uuid = clientUuid ) } @@ -927,7 +948,7 @@ class ChatItemImportInserter( wasDownloaded = wasDownloaded, contentType = contentType, fileName = fileName, - uuid = uuid + uuid = clientUuid ) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt index 61380e24ac..bc5099cf54 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt @@ -5,18 +5,14 @@ package org.thoughtcrime.securesms.backup.v2.processor -import androidx.annotation.WorkerThread import okio.ByteString.Companion.toByteString import org.signal.core.util.Hex import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.proto.StickerPack -import org.thoughtcrime.securesms.backup.v2.proto.StickerPackSticker import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.database.StickerTable.StickerPackRecordReader -import org.thoughtcrime.securesms.database.StickerTable.StickerRecordReader import org.thoughtcrime.securesms.database.model.StickerPackRecord -import org.thoughtcrime.securesms.database.model.StickerRecord import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.StickerPackDownloadJob @@ -41,36 +37,12 @@ object StickerBackupProcessor { } } -@WorkerThread -private fun getStickersFromDatabase(packId: String): List { - val stickers: MutableList = java.util.ArrayList() - - SignalDatabase.stickers.getStickersForPack(packId).use { cursor -> - val reader = StickerRecordReader(cursor) - var record: StickerRecord? = reader.next - while (record != null) { - stickers.add( - StickerPackSticker( - emoji = record.emoji, - id = record.stickerId - ) - ) - record = reader.next - } - } - return stickers -} - private fun StickerPackRecord.toBackupFrame(): Frame { val packIdBytes = Hex.fromStringCondensed(packId) val packKey = Hex.fromStringCondensed(packKey) - val stickers = getStickersFromDatabase(packId) val pack = StickerPack( packId = packIdBytes.toByteString(), - packKey = packKey.toByteString(), - title = title.orElse(""), - author = author.orElse(""), - stickers = stickers + packKey = packKey.toByteString() ) return Frame(stickerPack = pack) } diff --git a/app/src/main/protowire/Backup.proto b/app/src/main/protowire/Backup.proto index 9631f20a2c..b516ce25a0 100644 --- a/app/src/main/protowire/Backup.proto +++ b/app/src/main/protowire/Backup.proto @@ -301,15 +301,6 @@ message DistributionList { repeated uint64 memberRecipientIds = 4; // generated recipient id } -message Identity { - bytes serviceId = 1; - bytes identityKey = 2; - uint64 timestamp = 3; - bool firstUse = 4; - bool verified = 5; - bool nonblockingApproval = 6; -} - message ChatItem { message IncomingMessageDetails { uint64 dateReceived = 1; @@ -346,6 +337,7 @@ message ChatItem { RemoteDeletedMessage remoteDeletedMessage = 14; ChatUpdateMessage updateMessage = 15; PaymentNotification paymentNotification = 16; + GiftBadge giftBadge = 17; } } @@ -436,6 +428,18 @@ message PaymentNotification { } +message GiftBadge { + enum State { + UNOPENED = 0; + OPENED = 1; + REDEEMED = 2; + FAILED = 3; + } + + bytes receiptCredentialPresentation = 1; + State state = 2; +} + message ContactAttachment { message Name { optional string givenName = 1; @@ -501,12 +505,6 @@ message ContactAttachment { optional string organization = 7; } -message DocumentMessage { - Text text = 1; - FilePointer document = 2; - repeated Reaction reactions = 3; -} - message StickerMessage { Sticker sticker = 1; repeated Reaction reactions = 2; @@ -551,7 +549,9 @@ message MessageAttachment { FilePointer pointer = 1; Flag flag = 2; bool wasDownloaded = 3; - optional bytes uuid = 4; + // Cross-client identifier for this attachment among all attachments on the + // owning message. See: SignalService.AttachmentPointer.clientUuid. + optional bytes clientUuid = 4; } message FilePointer { @@ -1028,17 +1028,8 @@ message GroupExpirationTimerUpdate { message StickerPack { bytes packId = 1; bytes packKey = 2; - string title = 3; - string author = 4; - repeated StickerPackSticker stickers = 5; // First one should be cover sticker. } -message StickerPackSticker { - string emoji = 1; - uint32 id = 2; -} - - message ChatStyle { message Gradient { uint32 angle = 1; // degrees