empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Create backups from copies of the database file.

Browse source 
Still more work here to do with regards to certain tables, like
SignalStore and Recipient.
This commit is contained in:
Greyson Parrelli 2024-06-19 11:51:09 -04:00
parent 890facc6f6
commit cbb3c0911c
9 changed files with 141 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt
View file

@ -12,6 +12,7 @@ import kotlinx.coroutines.withContext
import org.greenrobot.eventbus.EventBus
import org.signal.core.util.Base64
import org.signal.core.util.EventTimer
import org.signal.core.util.fullWalCheckpoint
import org.signal.core.util.logging.Log
import org.signal.core.util.money.FiatMoney
import org.signal.core.util.withinTransaction
@ -43,6 +44,8 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter
import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType
import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsTypeFeature
import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider
import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
import org.thoughtcrime.securesms.database.DistributionListTables
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
@ -71,6 +74,7 @@ import org.whispersystems.signalservice.internal.push.SubscriptionsConfiguration
import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
import java.io.ByteArrayOutputStream
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.math.BigDecimal
@ -83,6 +87,7 @@ object BackupRepository {
private val TAG = Log.tag(BackupRepository::class.java)
private const val VERSION = 1L
private const val DB_SNAPSHOT_NAME = "signal-snapshot.db"
private val resetInitializedStateErrorAction: StatusCodeErrorAction = { error ->
when (error.code) {
@ -105,64 +110,106 @@ object BackupRepository {
SignalStore.backup().backupTier = null
}
private fun createSignalDatabaseSnapshot(): SignalDatabase {
// Need to do a WAL checkpoint to ensure that the database file we're copying has all pending writes
if (!SignalDatabase.rawDatabase.fullWalCheckpoint()) {
Log.w(TAG, "Failed to checkpoint WAL! Not guaranteed to be using the most recent data.")
}
// We make a copy of the database within a transaction to ensure that no writes occur while we're copying the file
return SignalDatabase.rawDatabase.withinTransaction {
val context = AppDependencies.application
val existingDbFile = context.getDatabasePath(SignalDatabase.DATABASE_NAME)
val targetFile = File(existingDbFile.parentFile, DB_SNAPSHOT_NAME)
try {
existingDbFile.copyTo(targetFile, overwrite = true)
} catch (e: IOException) {
// TODO [backup] Gracefully handle this error
throw IllegalStateException("Failed to copy database file!", e)
}
SignalDatabase(
context = context,
databaseSecret = DatabaseSecretProvider.getOrCreateDatabaseSecret(context),
attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret(),
name = DB_SNAPSHOT_NAME
)
}
}
private fun deleteDatabaseSnapshot() {
val targetFile = AppDependencies.application.getDatabasePath(DB_SNAPSHOT_NAME)
if (!targetFile.delete()) {
Log.w(TAG, "Failed to delete database snapshot!")
}
}
fun export(outputStream: OutputStream, append: (ByteArray) -> Unit, plaintext: Boolean = false) {
val eventTimer = EventTimer()
val writer: BackupExportWriter = if (plaintext) {
PlainTextBackupWriter(outputStream)
} else {
EncryptedBackupWriter(
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
aci = SignalStore.account().aci!!,
outputStream = outputStream,
append = append
)
}
val dbSnapshot: SignalDatabase = createSignalDatabaseSnapshot()
val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia)
writer.use {
writer.write(
BackupInfo(
version = VERSION,
backupTimeMs = exportState.backupTime
try {
val writer: BackupExportWriter = if (plaintext) {
PlainTextBackupWriter(outputStream)
} else {
EncryptedBackupWriter(
key = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey(),
aci = SignalStore.account().aci!!,
outputStream = outputStream,
append = append
)
)
// Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
// writes from other threads are blocked. This is something to think more about.
SignalDatabase.rawDatabase.withinTransaction {
AccountDataProcessor.export {
writer.write(it)
eventTimer.emit("account")
}
}
RecipientBackupProcessor.export(exportState) {
writer.write(it)
eventTimer.emit("recipient")
}
val exportState = ExportState(backupTime = System.currentTimeMillis(), allowMediaBackup = SignalStore.backup().backsUpMedia)
ChatBackupProcessor.export(exportState) { frame ->
writer.write(frame)
eventTimer.emit("thread")
}
writer.use {
writer.write(
BackupInfo(
version = VERSION,
backupTimeMs = exportState.backupTime
)
)
// Note: Without a transaction, we may export inconsistent state. But because we have a transaction,
// writes from other threads are blocked. This is something to think more about.
dbSnapshot.rawWritableDatabase.withinTransaction {
AccountDataProcessor.export(dbSnapshot) {
writer.write(it)
eventTimer.emit("account")
}
AdHocCallBackupProcessor.export { frame ->
writer.write(frame)
eventTimer.emit("call")
}
RecipientBackupProcessor.export(dbSnapshot, exportState) {
writer.write(it)
eventTimer.emit("recipient")
}
StickerBackupProcessor.export { frame ->
writer.write(frame)
eventTimer.emit("sticker-pack")
}
ChatBackupProcessor.export(dbSnapshot, exportState) { frame ->
writer.write(frame)
eventTimer.emit("thread")
}
ChatItemBackupProcessor.export(exportState) { frame ->
writer.write(frame)
eventTimer.emit("message")
AdHocCallBackupProcessor.export(dbSnapshot) { frame ->
writer.write(frame)
eventTimer.emit("call")
}
StickerBackupProcessor.export(dbSnapshot) { frame ->
writer.write(frame)
eventTimer.emit("sticker-pack")
}
ChatItemBackupProcessor.export(dbSnapshot, exportState) { frame ->
writer.write(frame)
eventTimer.emit("message")
}
}
}
}
Log.d(TAG, "export() ${eventTimer.stop().summary}")
Log.d(TAG, "export() ${eventTimer.stop().summary}")
} finally {
deleteDatabaseSnapshot()
}
}
fun export(plaintext: Boolean = false): ByteArray {

9
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AccountDataProcessor.kt
View file

@ -14,7 +14,6 @@ import org.thoughtcrime.securesms.backup.v2.stream.BackupFrameEmitter
import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository
import org.thoughtcrime.securesms.components.settings.app.usernamelinks.UsernameQrCodeColorScheme
import org.thoughtcrime.securesms.database.SignalDatabase
import org.thoughtcrime.securesms.database.SignalDatabase.Companion.recipients
import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord
import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob
@ -34,12 +33,14 @@ import kotlin.jvm.optionals.getOrNull
object AccountDataProcessor {
fun export(emitter: BackupFrameEmitter) {
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
val context = AppDependencies.application
// TODO [backup] Need to get it from the db snapshot
val self = Recipient.self().fresh()
val record = recipients.getRecordForSync(self.id)
val record = db.recipientTable.getRecordForSync(self.id)
// TODO [backup] Need to get it from the db snapshot
val subscriber: InAppPaymentSubscriberRecord? = InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION)
emitter.emit(
@ -80,7 +81,7 @@ object AccountDataProcessor {
}
fun import(accountData: AccountData, selfId: RecipientId) {
recipients.restoreSelfFromBackup(accountData, selfId)
SignalDatabase.recipients.restoreSelfFromBackup(accountData, selfId)
SignalStore.account().setRegistered(true)

4
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/AdHocCallBackupProcessor.kt
View file

@ -18,8 +18,8 @@ object AdHocCallBackupProcessor {
val TAG = Log.tag(AdHocCallBackupProcessor::class.java)
fun export(emitter: BackupFrameEmitter) {
SignalDatabase.calls.getAdhocCallsForBackup().use { reader ->
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
db.callTable.getAdhocCallsForBackup().use { reader ->
for (callLog in reader) {
if (callLog != null) {
emitter.emit(Frame(adHocCall = callLog))

4
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatBackupProcessor.kt
View file

@ -19,8 +19,8 @@ import org.thoughtcrime.securesms.recipients.RecipientId
object ChatBackupProcessor {
val TAG = Log.tag(ChatBackupProcessor::class.java)
fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
SignalDatabase.threads.getThreadsForBackup().use { reader ->
fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
db.threadTable.getThreadsForBackup().use { reader ->
for (chat in reader) {
if (exportState.recipientIds.contains(chat.recipientId)) {
exportState.threadIds.add(chat.id)

4
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/ChatItemBackupProcessor.kt
View file

@ -18,8 +18,8 @@ import org.thoughtcrime.securesms.database.SignalDatabase
object ChatItemBackupProcessor {
val TAG = Log.tag(ChatItemBackupProcessor::class.java)
fun export(exportState: ExportState, emitter: BackupFrameEmitter) {
SignalDatabase.messages.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
fun export(db: SignalDatabase, exportState: ExportState, emitter: BackupFrameEmitter) {
db.messageTable.getMessagesForBackup(exportState.backupTime, exportState.allowMediaBackup).use { chatItems ->
while (chatItems.hasNext()) {
val chatItem = chatItems.next()
if (chatItem != null) {

11
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/RecipientBackupProcessor.kt
View file

@ -30,7 +30,8 @@ object RecipientBackupProcessor {
val TAG = Log.tag(RecipientBackupProcessor::class.java)
fun export(state: ExportState, emitter: BackupFrameEmitter) {
fun export(db: SignalDatabase, state: ExportState, emitter: BackupFrameEmitter) {
// TODO [backup] Need to get it from the db snapshot
val selfId = Recipient.self().id.toLong()
val releaseChannelId = SignalStore.releaseChannelValues().releaseChannelRecipientId
if (releaseChannelId != null) {
@ -44,7 +45,7 @@ object RecipientBackupProcessor {
)
}
SignalDatabase.recipients.getContactsForBackup(selfId).use { reader ->
db.recipientTable.getContactsForBackup(selfId).use { reader ->
for (backupRecipient in reader) {
if (backupRecipient != null) {
state.recipientIds.add(backupRecipient.id)
@ -53,19 +54,19 @@ object RecipientBackupProcessor {
}
}
SignalDatabase.recipients.getGroupsForBackup().use { reader ->
db.recipientTable.getGroupsForBackup().use { reader ->
for (backupRecipient in reader) {
state.recipientIds.add(backupRecipient.id)
emitter.emit(Frame(recipient = backupRecipient))
}
}
SignalDatabase.distributionLists.getAllForBackup().forEach {
db.distributionListTables.getAllForBackup().forEach {
state.recipientIds.add(it.id)
emitter.emit(Frame(recipient = it))
}
SignalDatabase.callLinks.getCallLinksForBackup().forEach {
db.callLinkTable.getCallLinksForBackup().forEach {
state.recipientIds.add(it.id)
emitter.emit(Frame(recipient = it))
}

4
app/src/main/java/org/thoughtcrime/securesms/backup/v2/processor/StickerBackupProcessor.kt
View file

@ -17,8 +17,8 @@ import org.thoughtcrime.securesms.dependencies.AppDependencies
import org.thoughtcrime.securesms.jobs.StickerPackDownloadJob
object StickerBackupProcessor {
fun export(emitter: BackupFrameEmitter) {
StickerPackRecordReader(SignalDatabase.stickers.allStickerPacks).use { reader ->
fun export(db: SignalDatabase, emitter: BackupFrameEmitter) {
StickerPackRecordReader(db.stickerTable.allStickerPacks).use { reader ->
var record: StickerPackRecord? = reader.next
while (record != null) {
if (record.isInstalled) {

4
app/src/main/java/org/thoughtcrime/securesms/database/SignalDatabase.kt
View file

@ -23,7 +23,7 @@ import org.thoughtcrime.securesms.service.KeyCachingService
import org.thoughtcrime.securesms.util.TextSecurePreferences
import java.io.File
open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret) :
open class SignalDatabase(private val context: Application, databaseSecret: DatabaseSecret, attachmentSecret: AttachmentSecret, private val name: String = DATABASE_NAME) :
SQLiteOpenHelper(
context,
DATABASE_NAME,
@ -219,7 +219,7 @@ open class SignalDatabase(private val context: Application, databaseSecret: Data
companion object {
private val TAG = Log.tag(SignalDatabase::class.java)
private const val DATABASE_NAME = "signal.db"
const val DATABASE_NAME = "signal.db"
@JvmStatic
@Volatile

27
core-util/src/main/java/org/signal/core/util/SQLiteDatabaseExtensions.kt
View file

@ -89,6 +89,33 @@ fun SupportSQLiteDatabase.areForeignKeyConstraintsEnabled(): Boolean {
}
}
/**
* Does a full WAL checkpoint (TRUNCATE mode, where the log is for sure flushed and the log is zero'd out).
* Will try up to [maxAttempts] times. Can technically fail if the database is too active and the checkpoint
* can't complete in a reasonable amount of time.
*
* See: https://www.sqlite.org/pragma.html#pragma_wal_checkpoint
*/
fun SupportSQLiteDatabase.fullWalCheckpoint(maxAttempts: Int = 3): Boolean {
var attempts = 0
while (attempts < maxAttempts) {
if (this.walCheckpoint()) {
return true
}
attempts++
}
return false
}
private fun SupportSQLiteDatabase.walCheckpoint(): Boolean {
return this.query("PRAGMA wal_checkpoint(TRUNCATE)").use { cursor ->
cursor.moveToFirst() && cursor.getInt(0) == 0
}
}
fun SupportSQLiteDatabase.getIndexes(): List<Index> {
return this.query("SELECT name, tbl_name FROM sqlite_master WHERE type='index' ORDER BY name ASC").readToList { cursor ->
val indexName = cursor.requireNonNullString("name")