diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.java b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.java index 754196cb94..55ddfa6115 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.java +++ b/app/src/main/java/org/thoughtcrime/securesms/database/MessageTable.java @@ -79,6 +79,7 @@ import org.thoughtcrime.securesms.database.model.databaseprotos.SessionSwitchove import org.thoughtcrime.securesms.dependencies.ApplicationDependencies; import org.thoughtcrime.securesms.groups.GroupMigrationMembershipChange; import org.thoughtcrime.securesms.insights.InsightsConstants; +import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob; import org.thoughtcrime.securesms.jobs.TrimThreadJob; import org.thoughtcrime.securesms.keyvalue.SignalStore; import org.thoughtcrime.securesms.linkpreview.LinkPreview; @@ -1534,6 +1535,7 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie String[] args = SqlUtil.buildArgs(parentStoryId); db.delete(TABLE_NAME, PARENT_STORY_ID + " = ?", args); + OptimizeMessageSearchIndexJob.enqueue(); } public int deleteStoriesOlderThan(long timestamp, boolean hasSeenReleaseChannelStories) { @@ -1578,6 +1580,10 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie } } + if (deletedStoryCount > 0) { + OptimizeMessageSearchIndexJob.enqueue(); + } + db.setTransactionSuccessful(); return deletedStoryCount; } finally { @@ -3041,7 +3047,16 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie return deleteMessage(messageId, threadId); } + public boolean deleteMessage(long messageId, boolean notify) { + long threadId = getThreadIdForMessage(messageId); + return deleteMessage(messageId, threadId, notify); + } + public boolean deleteMessage(long messageId, long threadId) { + return deleteMessage(messageId, threadId, true); + } + + private boolean deleteMessage(long messageId, long threadId, boolean notify) { Log.d(TAG, "deleteMessage(" + messageId + ")"); AttachmentTable attachmentDatabase = SignalDatabase.attachments(); @@ -3058,9 +3073,14 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie SignalDatabase.threads().setLastScrolled(threadId, 0); boolean threadDeleted = SignalDatabase.threads().update(threadId, false); - notifyConversationListeners(threadId); - notifyStickerListeners(); - notifyStickerPackListeners(); + + if (notify) { + notifyConversationListeners(threadId); + notifyStickerListeners(); + notifyStickerPackListeners(); + OptimizeMessageSearchIndexJob.enqueue(); + } + return threadDeleted; } @@ -3269,6 +3289,7 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie setTransactionSuccessful(); } finally { endTransaction(); + OptimizeMessageSearchIndexJob.enqueue(); } } @@ -3286,16 +3307,27 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie try (Cursor cursor = db.query(TABLE_NAME, new String[] {ID}, where, null, null, null, null)) { while (cursor != null && cursor.moveToNext()) { - deleteMessage(cursor.getLong(0)); + deleteMessage(cursor.getLong(0), false); } } + + notifyConversationListeners(threadIds); + notifyStickerListeners(); + notifyStickerPackListeners(); + OptimizeMessageSearchIndexJob.enqueue(); } int deleteMessagesInThreadBeforeDate(long threadId, long date) { SQLiteDatabase db = databaseHelper.getSignalWritableDatabase(); String where = THREAD_ID + " = ? AND " + DATE_RECEIVED + " < " + date; - return db.delete(TABLE_NAME, where, SqlUtil.buildArgs(threadId)); + int count = db.delete(TABLE_NAME, where, SqlUtil.buildArgs(threadId)); + + if (count > 0) { + OptimizeMessageSearchIndexJob.enqueue(); + } + + return count; } void deleteAbandonedMessages() { @@ -3305,6 +3337,7 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie int deletes = db.delete(TABLE_NAME, where, null); if (deletes > 0) { Log.i(TAG, "Deleted " + deletes + " abandoned messages"); + OptimizeMessageSearchIndexJob.enqueue(); } } @@ -3342,6 +3375,7 @@ public class MessageTable extends DatabaseTable implements MessageTypes, Recipie SQLiteDatabase database = databaseHelper.getSignalWritableDatabase(); database.delete(TABLE_NAME, null, null); + OptimizeMessageSearchIndexJob.enqueue(); } public @Nullable ViewOnceExpirationInfo getNearestExpiringViewOnceMessage() { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt index f06566c761..21c8cb6bb1 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/SearchTable.kt @@ -6,7 +6,9 @@ import android.database.Cursor import android.text.TextUtils import org.intellij.lang.annotations.Language import org.signal.core.util.SqlUtil +import org.signal.core.util.ThreadUtil import org.signal.core.util.logging.Log +import org.signal.core.util.withinTransaction /** * Contains all databases necessary for full-text search (FTS). @@ -148,6 +150,73 @@ class SearchTable(context: Context, databaseHelper: SignalDatabase) : DatabaseTa } } + /** + * This performs the same thing as the `optimize` command in SQLite, but broken into iterative stages to avoid locking up the database for too long. + * If what's going on in this method seems weird, that's because it is, but please read the sqlite docs -- we're following their algorithm: + * https://www.sqlite.org/fts5.html#the_optimize_command + * + * Note that in order for the [SqlUtil.getTotalChanges] call to work, we have to be within a transaction, or else the connection pool screws everything up + * (the stats are on a per-connection basis). + * + * There's this double-batching mechanism happening here to strike a balance between making individual transactions short while also not hammering the + * database with a ton of independent transactions. + * + * To give you some ballpark numbers, on a large database (~400k messages), it takes ~75 iterations to fully optimize everything. + */ + fun optimizeIndex(timeout: Long): Boolean { + val pageSize = 64 // chosen through experimentation + val batchSize = 10 // chosen through experimentation + val noChangeThreshold = 2 // if less changes occurred than this, operation is considered no-op (see sqlite docs ref'd in kdoc) + + val startTime = System.currentTimeMillis() + var totalIterations = 0 + var totalBatches = 0 + var actualWorkTime = 0L + var finished = false + + while (!finished) { + var batchIterations = 0 + val batchStartTime = System.currentTimeMillis() + + writableDatabase.withinTransaction { db -> + // Note the negative page size -- see sqlite docs ref'd in kdoc + db.execSQL("INSERT INTO $MMS_FTS_TABLE_NAME ($MMS_FTS_TABLE_NAME, rank) values ('merge', -$pageSize)") + var previousCount = SqlUtil.getTotalChanges(db) + + val iterativeStatement = db.compileStatement("INSERT INTO $MMS_FTS_TABLE_NAME ($MMS_FTS_TABLE_NAME, rank) values ('merge', $pageSize)") + iterativeStatement.execute() + var count = SqlUtil.getTotalChanges(db) + + while (batchIterations < batchSize && count - previousCount >= noChangeThreshold) { + previousCount = count + iterativeStatement.execute() + + count = SqlUtil.getTotalChanges(db) + batchIterations++ + } + + if (count - previousCount < noChangeThreshold) { + finished = true + } + } + + totalIterations += batchIterations + totalBatches++ + actualWorkTime += System.currentTimeMillis() - batchStartTime + + if (actualWorkTime >= timeout) { + Log.w(TAG, "Timed out during optimization! We did $totalIterations iterations across $totalBatches batches, taking ${System.currentTimeMillis() - startTime} ms. Bailed out to avoid database lockup.") + return false + } + + // We want to sleep in between batches to give other db operations a chance to run + ThreadUtil.sleep(50) + } + + Log.d(TAG, "Took ${System.currentTimeMillis() - startTime} ms and $totalIterations iterations across $totalBatches batches to optimize. Of that time, $actualWorkTime ms were spent actually working (~${actualWorkTime / totalBatches} ms/batch). The rest was spent sleeping.") + return true + } + private fun createFullTextSearchQuery(query: String): String { return query .split(" ") diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java index a04b2fdbbd..6a4b6c09bf 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/JobManagerFactories.java @@ -46,6 +46,7 @@ import org.thoughtcrime.securesms.migrations.EmojiDownloadMigrationJob; import org.thoughtcrime.securesms.migrations.KbsEnclaveMigrationJob; import org.thoughtcrime.securesms.migrations.LegacyMigrationJob; import org.thoughtcrime.securesms.migrations.MigrationCompleteJob; +import org.thoughtcrime.securesms.migrations.OptimizeMessageSearchIndexMigrationJob; import org.thoughtcrime.securesms.migrations.PassingMigrationJob; import org.thoughtcrime.securesms.migrations.PinOptOutMigration; import org.thoughtcrime.securesms.migrations.PinReminderMigrationJob; @@ -140,6 +141,7 @@ public final class JobManagerFactories { put(MultiDeviceViewOnceOpenJob.KEY, new MultiDeviceViewOnceOpenJob.Factory()); put(MultiDeviceViewedUpdateJob.KEY, new MultiDeviceViewedUpdateJob.Factory()); put(NullMessageSendJob.KEY, new NullMessageSendJob.Factory()); + put(OptimizeMessageSearchIndexJob.KEY, new OptimizeMessageSearchIndexJob.Factory()); put(PaymentLedgerUpdateJob.KEY, new PaymentLedgerUpdateJob.Factory()); put(PaymentNotificationSendJob.KEY, new PaymentNotificationSendJob.Factory()); put(PaymentNotificationSendJobV2.KEY, new PaymentNotificationSendJobV2.Factory()); @@ -214,7 +216,7 @@ public final class JobManagerFactories { put(KbsEnclaveMigrationJob.KEY, new KbsEnclaveMigrationJob.Factory()); put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory()); put(MigrationCompleteJob.KEY, new MigrationCompleteJob.Factory()); - put(SyncDistributionListsMigrationJob.KEY, new SyncDistributionListsMigrationJob.Factory()); + put(OptimizeMessageSearchIndexMigrationJob.KEY,new OptimizeMessageSearchIndexMigrationJob.Factory()); put(PinOptOutMigration.KEY, new PinOptOutMigration.Factory()); put(PinReminderMigrationJob.KEY, new PinReminderMigrationJob.Factory()); put(PniAccountInitializationMigrationJob.KEY, new PniAccountInitializationMigrationJob.Factory()); @@ -233,6 +235,7 @@ public final class JobManagerFactories { put(StorageServiceSystemNameMigrationJob.KEY, new StorageServiceSystemNameMigrationJob.Factory()); put(StoryReadStateMigrationJob.KEY, new StoryReadStateMigrationJob.Factory()); put(StoryViewedReceiptsStateMigrationJob.KEY, new StoryViewedReceiptsStateMigrationJob.Factory()); + put(SyncDistributionListsMigrationJob.KEY, new SyncDistributionListsMigrationJob.Factory()); put(TrimByLengthSettingsMigrationJob.KEY, new TrimByLengthSettingsMigrationJob.Factory()); put(UpdateSmsJobsMigrationJob.KEY, new UpdateSmsJobsMigrationJob.Factory()); put(UserNotificationMigrationJob.KEY, new UserNotificationMigrationJob.Factory()); diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt new file mode 100644 index 0000000000..6fabc870c5 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/OptimizeMessageSearchIndexJob.kt @@ -0,0 +1,51 @@ +package org.thoughtcrime.securesms.jobs + +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.dependencies.ApplicationDependencies +import org.thoughtcrime.securesms.jobmanager.Data +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.transport.RetryLaterException +import java.lang.Exception +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds + +/** + * Optimizes the message search index incrementally. + */ +class OptimizeMessageSearchIndexJob private constructor(parameters: Parameters) : BaseJob(parameters) { + + companion object { + const val KEY = "OptimizeMessageSearchIndexJob" + + @JvmStatic + fun enqueue() { + ApplicationDependencies.getJobManager().add(OptimizeMessageSearchIndexJob()) + } + } + + constructor() : this( + Parameters.Builder() + .setQueue("OptimizeMessageSearchIndexJob") + .setMaxAttempts(5) + .setMaxInstancesForQueue(2) + .build() + ) + + override fun serialize(): Data = Data.EMPTY + override fun getFactoryKey() = KEY + override fun onFailure() = Unit + override fun onShouldRetry(e: Exception) = e is RetryLaterException + override fun getNextRunAttemptBackoff(pastAttemptCount: Int, exception: Exception): Long = 1.minutes.inWholeMilliseconds + + override fun onRun() { + val success = SignalDatabase.messageSearch.optimizeIndex(10.seconds.inWholeMilliseconds) + + if (!success) { + throw RetryLaterException() + } + } + + class Factory : Job.Factory { + override fun create(parameters: Parameters, data: Data) = OptimizeMessageSearchIndexJob(parameters) + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java index 950f8071c5..e39d1e3798 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/ApplicationMigrations.java @@ -115,9 +115,10 @@ public class ApplicationMigrations { static final int SMS_MMS_MERGE = 71; static final int REBUILD_MESSAGE_FTS_INDEX = 72; static final int UPDATE_SMS_JOBS = 73; + static final int OPTIMIZE_MESSAGE_FTS_INDEX = 74; } - public static final int CURRENT_VERSION = 73; + public static final int CURRENT_VERSION = 74; /** * This *must* be called after the {@link JobManager} has been instantiated, but *before* the call @@ -511,6 +512,10 @@ public class ApplicationMigrations { jobs.put(Version.UPDATE_SMS_JOBS, new UpdateSmsJobsMigrationJob()); } + if (lastSeenVersion < Version.OPTIMIZE_MESSAGE_FTS_INDEX) { + jobs.put(Version.OPTIMIZE_MESSAGE_FTS_INDEX, new OptimizeMessageSearchIndexMigrationJob()); + } + return jobs; } diff --git a/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt b/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt new file mode 100644 index 0000000000..087ac7764a --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/migrations/OptimizeMessageSearchIndexMigrationJob.kt @@ -0,0 +1,35 @@ +package org.thoughtcrime.securesms.migrations + +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.jobmanager.Data +import org.thoughtcrime.securesms.jobmanager.Job +import org.thoughtcrime.securesms.jobs.OptimizeMessageSearchIndexJob + +/** + * Kicks off a job to optimize the message search index. + */ +internal class OptimizeMessageSearchIndexMigrationJob( + parameters: Parameters = Parameters.Builder().build() +) : MigrationJob(parameters) { + + companion object { + val TAG = Log.tag(OptimizeMessageSearchIndexMigrationJob::class.java) + const val KEY = "OptimizeMessageSearchIndexMigrationJob" + } + + override fun getFactoryKey(): String = KEY + + override fun isUiBlocking(): Boolean = false + + override fun performMigration() { + OptimizeMessageSearchIndexJob.enqueue() + } + + override fun shouldRetry(e: Exception): Boolean = false + + class Factory : Job.Factory { + override fun create(parameters: Parameters, data: Data): OptimizeMessageSearchIndexMigrationJob { + return OptimizeMessageSearchIndexMigrationJob(parameters) + } + } +} diff --git a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt index 345097bbaa..757d45d4aa 100644 --- a/core-util/src/main/java/org/signal/core/util/SqlUtil.kt +++ b/core-util/src/main/java/org/signal/core/util/SqlUtil.kt @@ -36,6 +36,15 @@ object SqlUtil { return tables } + /** + * Returns the total number of changes that have been made since the creation of this database connection. + * + * IMPORTANT: Due to how connection pooling is handled in the app, the only way to have this return useful numbers is to call it within a transaction. + */ + fun getTotalChanges(db: SupportSQLiteDatabase): Long { + return db.query("SELECT total_changes()", null).readToSingleLong() + } + @JvmStatic fun getAllTriggers(db: SupportSQLiteDatabase): List { val tables: MutableList = LinkedList()