empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Update logging to be size-limited and more performant.

Browse source
This commit is contained in:
Greyson Parrelli 2021-07-23 07:39:16 -04:00
parent 3c748b2df6
commit 15a5f5966d
18 changed files with 253 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java
View file

@ -18,6 +18,7 @@ package org.thoughtcrime.securesms;
import android.content.Context;
import android.os.Build;
import android.os.SystemClock;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@ -37,6 +38,7 @@ import org.signal.glide.SignalGlideCodecs;
import org.signal.ringrtc.CallManager;
import org.thoughtcrime.securesms.avatar.AvatarPickerStorage;
import org.thoughtcrime.securesms.database.DatabaseFactory;
import org.thoughtcrime.securesms.database.LogDatabase;
import org.thoughtcrime.securesms.database.SqlCipherLibraryLoader;
import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
@ -246,10 +248,12 @@ public class ApplicationContext extends MultiDexApplication implements AppForegr
}
private void initializeLogging() {
persistentLogger = new PersistentLogger(this, TimeUnit.DAYS.toMillis(2));
persistentLogger = new PersistentLogger(this);
org.signal.core.util.logging.Log.initialize(FeatureFlags::internalUser, new AndroidLogger(), persistentLogger);
SignalProtocolLoggerProvider.setProvider(new CustomSignalProtocolLogger());
SignalExecutors.UNBOUNDED.execute(() -> LogDatabase.getInstance(this).trimToSize());
}
private void initializeCrashHandling() {

100
app/src/main/java/org/thoughtcrime/securesms/database/LogDatabase.kt
View file

@ -10,14 +10,18 @@ import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.crypto.DatabaseSecret
import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
import org.thoughtcrime.securesms.database.model.LogEntry
import org.thoughtcrime.securesms.util.ByteUnit
import org.thoughtcrime.securesms.util.CursorUtil
import org.thoughtcrime.securesms.util.SqlUtil
import org.thoughtcrime.securesms.util.Stopwatch
import java.io.Closeable
import java.util.concurrent.TimeUnit
/**
* Stores logs.
*
* Logs are very performance-critical, particularly inserts and deleting old entries.
* Logs are very performance critical. Even though this database is written to on a low-priority background thread, we want to keep throughput high and ensure
* that we aren't creating excess garbage.
*
* This is it's own separate physical database, so it cannot do joins or queries with any other
* tables.
@ -38,13 +42,17 @@ class LogDatabase private constructor(
companion object {
private val TAG = Log.tag(LogDatabase::class.java)
private const val DATABASE_VERSION = 1
private val MAX_FILE_SIZE = ByteUnit.MEGABYTES.toBytes(10)
private val DEFAULT_LIFESPAN = TimeUnit.DAYS.toMillis(2)
private val LONGER_LIFESPAN = TimeUnit.DAYS.toMillis(7)
private const val DATABASE_VERSION = 2
private const val DATABASE_NAME = "signal-logs.db"
private const val TABLE_NAME = "log"
private const val ID = "_id"
private const val CREATED_AT = "created_at"
private const val EXPIRES_AT = "expires_at"
private const val KEEP_LONGER = "keep_longer"
private const val BODY = "body"
private const val SIZE = "size"
@ -52,14 +60,15 @@ class LogDatabase private constructor(
CREATE TABLE $TABLE_NAME (
$ID INTEGER PRIMARY KEY,
$CREATED_AT INTEGER,
$EXPIRES_AT INTEGER,
$BODY TEXT,
$KEEP_LONGER INTEGER DEFAULT 0,
$BODY TEXT,
$SIZE INTEGER
)
""".trimIndent()
private val CREATE_INDEXES = arrayOf(
"CREATE INDEX log_expires_at_index ON $TABLE_NAME ($EXPIRES_AT)"
"CREATE INDEX keep_longer_index ON $TABLE_NAME ($KEEP_LONGER)",
"CREATE INDEX log_created_at_keep_longer_index ON $TABLE_NAME ($CREATED_AT, $KEEP_LONGER)"
)
@SuppressLint("StaticFieldLeak") // We hold an Application context, not a view context
@ -88,6 +97,13 @@ class LogDatabase private constructor(
override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
Log.i(TAG, "onUpgrade($oldVersion, $newVersion)")
if (oldVersion < 2) {
db.execSQL("DROP TABLE log")
db.execSQL("CREATE TABLE log (_id INTEGER PRIMARY KEY, created_at INTEGER, keep_longer INTEGER DEFAULT 0, body TEXT, size INTEGER)")
db.execSQL("CREATE INDEX keep_longer_index ON $TABLE_NAME ($KEEP_LONGER)")
db.execSQL("CREATE INDEX log_created_at_keep_longer_index ON $TABLE_NAME ($CREATED_AT, $KEEP_LONGER)")
}
}
override fun getSqlCipherDatabase(): SQLiteDatabase {
@ -102,7 +118,13 @@ class LogDatabase private constructor(
logs.forEach { log ->
db.insert(TABLE_NAME, null, buildValues(log))
}
db.delete(TABLE_NAME, "$EXPIRES_AT < ?", SqlUtil.buildArgs(currentTime))
db.delete(
TABLE_NAME,
"($CREATED_AT < ? AND $KEEP_LONGER = ?) OR ($CREATED_AT < ? AND $KEEP_LONGER = ?)",
SqlUtil.buildArgs(currentTime - DEFAULT_LIFESPAN, 0, currentTime - LONGER_LIFESPAN, 1)
)
db.setTransactionSuccessful()
} finally {
db.endTransaction()
@ -110,7 +132,7 @@ class LogDatabase private constructor(
}
fun getAllBeforeTime(time: Long): Reader {
return Reader(readableDatabase.query(TABLE_NAME, arrayOf(BODY), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null))
return CursorReader(readableDatabase.query(TABLE_NAME, arrayOf(BODY), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null))
}
fun getRangeBeforeTime(start: Int, length: Int, time: Long): List<String> {
@ -125,6 +147,52 @@ class LogDatabase private constructor(
return lines
}
fun trimToSize() {
val currentTime = System.currentTimeMillis()
val stopwatch = Stopwatch("trim")
val sizeOfSpecialLogs: Long = getSize("$KEEP_LONGER = ?", arrayOf("1"))
val remainingSize = MAX_FILE_SIZE - sizeOfSpecialLogs
stopwatch.split("keepers-size")
if (remainingSize <= 0) {
writableDatabase.delete(TABLE_NAME, "$KEEP_LONGER = ?", arrayOf("0"))
return
}
val sizeDiffThreshold = MAX_FILE_SIZE * 0.01
var lhs: Long = currentTime - DEFAULT_LIFESPAN
var rhs: Long = currentTime
var mid: Long = 0
var sizeOfChunk: Long
while (lhs < rhs - 2) {
mid = (lhs + rhs) / 2
sizeOfChunk = getSize("$CREATED_AT > ? AND $CREATED_AT < ? AND $KEEP_LONGER = ?", SqlUtil.buildArgs(mid, currentTime, 0))
if (sizeOfChunk > remainingSize) {
lhs = mid
} else if (sizeOfChunk < remainingSize) {
if (remainingSize - sizeOfChunk < sizeDiffThreshold) {
break
} else {
rhs = mid
}
} else {
break
}
}
stopwatch.split("binary-search")
writableDatabase.delete(TABLE_NAME, "$CREATED_AT < ? AND $KEEP_LONGER = ?", SqlUtil.buildArgs(mid, 0))
stopwatch.split("delete")
stopwatch.stop(TAG)
}
fun getLogCountBeforeTime(time: Long): Int {
readableDatabase.query(TABLE_NAME, arrayOf("COUNT(*)"), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null).use { cursor ->
return if (cursor.moveToFirst()) {
@ -138,19 +206,31 @@ class LogDatabase private constructor(
private fun buildValues(log: LogEntry): ContentValues {
return ContentValues().apply {
put(CREATED_AT, log.createdAt)
put(EXPIRES_AT, log.createdAt + log.lifespan)
put(KEEP_LONGER, if (log.keepLonger) 1 else 0)
put(BODY, log.body)
put(SIZE, log.body.length)
}
}
private fun getSize(query: String?, args: Array<String>?): Long {
readableDatabase.query(TABLE_NAME, arrayOf("SUM($SIZE)"), query, args, null, null, null).use { cursor ->
return if (cursor.moveToFirst()) {
cursor.getLong(0)
} else {
0
}
}
}
private val readableDatabase: SQLiteDatabase
get() = getReadableDatabase(databaseSecret.asString())
private val writableDatabase: SQLiteDatabase
get() = getWritableDatabase(databaseSecret.asString())
class Reader(private val cursor: Cursor) : Iterator<String>, Closeable {
interface Reader : Iterator<String>, Closeable
class CursorReader(private val cursor: Cursor) : Reader {
override fun hasNext(): Boolean {
return !cursor.isLast && cursor.count > 0
}

36
app/src/main/java/org/thoughtcrime/securesms/database/RecipientDatabase.java
View file

@ -156,8 +156,6 @@ public class RecipientDatabase extends Database {
private static final String IDENTITY_STATUS = "identity_status";
private static final String IDENTITY_KEY = "identity_key";
static final long IMPORTANT_LOG_DURATION = TimeUnit.DAYS.toMillis(7);
/**
* Values that represent the index in the capabilities bitmask. Each index can store a 2-bit
* value, which in this case is the value of {@link Recipient.Capability}.
@ -440,7 +438,7 @@ public class RecipientDatabase extends Database {
RecipientId finalId;
if (!byE164.isPresent() && !byUuid.isPresent()) {
Log.i(TAG, "Discovered a completely new user. Inserting.", IMPORTANT_LOG_DURATION);
Log.i(TAG, "Discovered a completely new user. Inserting.", true);
if (highTrust) {
long id = db.insert(TABLE_NAME, null, buildContentValuesForNewUser(e164, uuid));
finalId = RecipientId.from(id);
@ -453,7 +451,7 @@ public class RecipientDatabase extends Database {
RecipientSettings e164Settings = getRecipientSettings(byE164.get());
if (e164Settings.uuid != null) {
if (highTrust) {
Log.w(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s), but that user already has a UUID (%s). Likely a case of re-registration. High-trust, so stripping the E164 from the existing account and assigning it to a new entry.", uuid, byE164.get(), e164Settings.uuid), IMPORTANT_LOG_DURATION);
Log.w(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s), but that user already has a UUID (%s). Likely a case of re-registration. High-trust, so stripping the E164 from the existing account and assigning it to a new entry.", uuid, byE164.get(), e164Settings.uuid), true);
removePhoneNumber(byE164.get(), db);
recipientNeedingRefresh = byE164.get();
@ -464,18 +462,18 @@ public class RecipientDatabase extends Database {
long id = db.insert(TABLE_NAME, null, insertValues);
finalId = RecipientId.from(id);
} else {
Log.w(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s), but that user already has a UUID (%s). Likely a case of re-registration. Low-trust, so making a new user for the UUID.", uuid, byE164.get(), e164Settings.uuid), IMPORTANT_LOG_DURATION);
Log.w(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s), but that user already has a UUID (%s). Likely a case of re-registration. Low-trust, so making a new user for the UUID.", uuid, byE164.get(), e164Settings.uuid), true);
long id = db.insert(TABLE_NAME, null, buildContentValuesForNewUser(null, uuid));
finalId = RecipientId.from(id);
}
} else {
if (highTrust) {
Log.i(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s). High-trust, so updating.", uuid, byE164.get()), IMPORTANT_LOG_DURATION);
Log.i(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s). High-trust, so updating.", uuid, byE164.get()), true);
markRegisteredOrThrow(byE164.get(), uuid);
finalId = byE164.get();
} else {
Log.i(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s). Low-trust, so making a new user for the UUID.", uuid, byE164.get()), IMPORTANT_LOG_DURATION);
Log.i(TAG, String.format(Locale.US, "Found out about a UUID (%s) for a known E164 user (%s). Low-trust, so making a new user for the UUID.", uuid, byE164.get()), true);
long id = db.insert(TABLE_NAME, null, buildContentValuesForNewUser(null, uuid));
finalId = RecipientId.from(id);
}
@ -486,11 +484,11 @@ public class RecipientDatabase extends Database {
} else if (!byE164.isPresent() && byUuid.isPresent()) {
if (e164 != null) {
if (highTrust) {
Log.i(TAG, String.format(Locale.US, "Found out about an E164 (%s) for a known UUID user (%s). High-trust, so updating.", e164, byUuid.get()), IMPORTANT_LOG_DURATION);
Log.i(TAG, String.format(Locale.US, "Found out about an E164 (%s) for a known UUID user (%s). High-trust, so updating.", e164, byUuid.get()), true);
setPhoneNumberOrThrow(byUuid.get(), e164);
finalId = byUuid.get();
} else {
Log.i(TAG, String.format(Locale.US, "Found out about an E164 (%s) for a known UUID user (%s). Low-trust, so doing nothing.", e164, byUuid.get()), IMPORTANT_LOG_DURATION);
Log.i(TAG, String.format(Locale.US, "Found out about an E164 (%s) for a known UUID user (%s). Low-trust, so doing nothing.", e164, byUuid.get()), true);
finalId = byUuid.get();
}
} else {
@ -500,13 +498,13 @@ public class RecipientDatabase extends Database {
if (byE164.equals(byUuid)) {
finalId = byUuid.get();
} else {
Log.w(TAG, String.format(Locale.US, "Hit a conflict between %s (E164 of %s) and %s (UUID %s). They map to different recipients.", byE164.get(), e164, byUuid.get(), uuid), new Throwable(), IMPORTANT_LOG_DURATION);
Log.w(TAG, String.format(Locale.US, "Hit a conflict between %s (E164 of %s) and %s (UUID %s). They map to different recipients.", byE164.get(), e164, byUuid.get(), uuid), new Throwable(), true);
RecipientSettings e164Settings = getRecipientSettings(byE164.get());
if (e164Settings.getUuid() != null) {
if (highTrust) {
Log.w(TAG, "The E164 contact has a different UUID. Likely a case of re-registration. High-trust, so stripping the E164 from the existing account and assigning it to the UUID entry.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "The E164 contact has a different UUID. Likely a case of re-registration. High-trust, so stripping the E164 from the existing account and assigning it to the UUID entry.", true);
removePhoneNumber(byE164.get(), db);
recipientNeedingRefresh = byE164.get();
@ -515,17 +513,17 @@ public class RecipientDatabase extends Database {
finalId = byUuid.get();
} else {
Log.w(TAG, "The E164 contact has a different UUID. Likely a case of re-registration. Low-trust, so doing nothing.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "The E164 contact has a different UUID. Likely a case of re-registration. Low-trust, so doing nothing.", true);
finalId = byUuid.get();
}
} else {
if (highTrust) {
Log.w(TAG, "We have one contact with just an E164, and another with UUID. High-trust, so merging the two rows together.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "We have one contact with just an E164, and another with UUID. High-trust, so merging the two rows together.", true);
finalId = merge(byUuid.get(), byE164.get());
recipientNeedingRefresh = byUuid.get();
remapped = new Pair<>(byE164.get(), byUuid.get());
} else {
Log.w(TAG, "We have one contact with just an E164, and another with UUID. Low-trust, so doing nothing.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "We have one contact with just an E164, and another with UUID. Low-trust, so doing nothing.", true);
finalId = byUuid.get();
}
}
@ -2881,7 +2879,7 @@ public class RecipientDatabase extends Database {
RecipientSettings e164Settings = getRecipientSettings(byE164);
// Recipient
Log.w(TAG, "Deleting recipient " + byE164, IMPORTANT_LOG_DURATION);
Log.w(TAG, "Deleting recipient " + byE164, true);
db.delete(TABLE_NAME, ID_WHERE, SqlUtil.buildArgs(byE164));
RemappedRecords.getInstance().addRecipient(context, byE164, byUuid);
@ -2966,17 +2964,17 @@ public class RecipientDatabase extends Database {
boolean hasUuidSession = DatabaseFactory.getSessionDatabase(context).getAllFor(byUuid).size() > 0;
if (hasE164Session && hasUuidSession) {
Log.w(TAG, "Had a session for both users. Deleting the E164.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "Had a session for both users. Deleting the E164.", true);
db.delete(SessionDatabase.TABLE_NAME, SessionDatabase.RECIPIENT_ID + " = ?", SqlUtil.buildArgs(byE164));
} else if (hasE164Session && !hasUuidSession) {
Log.w(TAG, "Had a session for E164, but not UUID. Re-assigning to the UUID.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "Had a session for E164, but not UUID. Re-assigning to the UUID.", true);
ContentValues values = new ContentValues();
values.put(SessionDatabase.RECIPIENT_ID, byUuid.serialize());
db.update(SessionDatabase.TABLE_NAME, values, SessionDatabase.RECIPIENT_ID + " = ?", SqlUtil.buildArgs(byE164));
} else if (!hasE164Session && hasUuidSession) {
Log.w(TAG, "Had a session for UUID, but not E164. No action necessary.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "Had a session for UUID, but not E164. No action necessary.", true);
} else {
Log.w(TAG, "Had no sessions. No action necessary.", IMPORTANT_LOG_DURATION);
Log.w(TAG, "Had no sessions. No action necessary.", true);
}
// Mentions

8
app/src/main/java/org/thoughtcrime/securesms/database/ThreadDatabase.java
View file

@ -1276,16 +1276,16 @@ public class ThreadDatabase extends Database {
throw new IllegalStateException("Must be in a transaction!");
}
Log.w(TAG, "Merging threads. Primary: " + primaryRecipientId + ", Secondary: " + secondaryRecipientId, RecipientDatabase.IMPORTANT_LOG_DURATION);
Log.w(TAG, "Merging threads. Primary: " + primaryRecipientId + ", Secondary: " + secondaryRecipientId, true);
ThreadRecord primary = getThreadRecord(getThreadIdFor(primaryRecipientId));
ThreadRecord secondary = getThreadRecord(getThreadIdFor(secondaryRecipientId));
if (primary != null && secondary == null) {
Log.w(TAG, "[merge] Only had a thread for primary. Returning that.", RecipientDatabase.IMPORTANT_LOG_DURATION);
Log.w(TAG, "[merge] Only had a thread for primary. Returning that.", true);
return new MergeResult(primary.getThreadId(), -1, false);
} else if (primary == null && secondary != null) {
Log.w(TAG, "[merge] Only had a thread for secondary. Updating it to have the recipientId of the primary.", RecipientDatabase.IMPORTANT_LOG_DURATION);
Log.w(TAG, "[merge] Only had a thread for secondary. Updating it to have the recipientId of the primary.", true);
ContentValues values = new ContentValues();
values.put(RECIPIENT_ID, primaryRecipientId.serialize());
@ -1296,7 +1296,7 @@ public class ThreadDatabase extends Database {
Log.w(TAG, "[merge] No thread for either.");
return new MergeResult(-1, -1, false);
} else {
Log.w(TAG, "[merge] Had a thread for both. Deleting the secondary and merging the attributes together.", RecipientDatabase.IMPORTANT_LOG_DURATION);
Log.w(TAG, "[merge] Had a thread for both. Deleting the secondary and merging the attributes together.", true);
SQLiteDatabase db = databaseHelper.getWritableDatabase();

2
app/src/main/java/org/thoughtcrime/securesms/database/model/LogEntry.kt
View file

@ -2,6 +2,6 @@ package org.thoughtcrime.securesms.database.model
data class LogEntry(
val createdAt: Long,
val lifespan: Long,
val keepLonger: Boolean,
val body: String
)

39
app/src/main/java/org/thoughtcrime/securesms/logging/PersistentLogger.kt
View file

@ -6,6 +6,7 @@ import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.BuildConfig
import org.thoughtcrime.securesms.database.LogDatabase
import org.thoughtcrime.securesms.database.model.LogEntry
import org.thoughtcrime.securesms.logsubmit.util.Scrubber
import java.io.ByteArrayOutputStream
import java.io.PrintStream
import java.text.SimpleDateFormat
@ -23,9 +24,8 @@ import java.util.Locale
* - The [WriteThread] constantly pulls from that queue, formats the logs, and writes them to the database.
*/
class PersistentLogger(
application: Application,
defaultLifespan: Long
) : Log.Logger(defaultLifespan) {
application: Application
) : Log.Logger() {
companion object {
private const val LOG_V = "V"
@ -33,7 +33,6 @@ class PersistentLogger(
private const val LOG_I = "I"
private const val LOG_W = "W"
private const val LOG_E = "E"
private const val LOG_WTF = "A"
}
private val logEntries = LogRequests()
@ -46,32 +45,32 @@ class PersistentLogger(
}.start()
}
override fun v(tag: String?, message: String?, t: Throwable?, duration: Long) {
write(LOG_V, tag, message, t, duration)
override fun v(tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
write(LOG_V, tag, message, t, keepLonger)
}
override fun d(tag: String?, message: String?, t: Throwable?, duration: Long) {
write(LOG_D, tag, message, t, duration)
override fun d(tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
write(LOG_D, tag, message, t, keepLonger)
}
override fun i(tag: String?, message: String?, t: Throwable?, duration: Long) {
write(LOG_I, tag, message, t, duration)
override fun i(tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
write(LOG_I, tag, message, t, keepLonger)
}
override fun w(tag: String?, message: String?, t: Throwable?, duration: Long) {
write(LOG_W, tag, message, t, duration)
override fun w(tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
write(LOG_W, tag, message, t, keepLonger)
}
override fun e(tag: String?, message: String?, t: Throwable?, duration: Long) {
write(LOG_E, tag, message, t, duration)
override fun e(tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
write(LOG_E, tag, message, t, keepLonger)
}
override fun flush() {
logEntries.blockForFlushed()
}
private fun write(level: String, tag: String?, message: String?, t: Throwable?, lifespan: Long) {
logEntries.add(LogRequest(level, tag ?: "null", message, Date(), getThreadString(), t, lifespan))
private fun write(level: String, tag: String?, message: String?, t: Throwable?, keepLonger: Boolean) {
logEntries.add(LogRequest(level, tag ?: "null", message, Date(), getThreadString(), t, keepLonger))
}
private fun getThreadString(): String {
@ -97,7 +96,7 @@ class PersistentLogger(
val date: Date,
val threadString: String,
val throwable: Throwable?,
val lifespan: Long
val keepLonger: Boolean
)
private class WriteThread(
@ -123,7 +122,7 @@ class PersistentLogger(
out.add(
LogEntry(
createdAt = request.date.time,
lifespan = request.lifespan,
keepLonger = request.keepLonger,
body = formatBody(request.threadString, request.date, request.level, request.tag, request.message)
)
)
@ -138,7 +137,7 @@ class PersistentLogger(
val entries = lines.map { line ->
LogEntry(
createdAt = request.date.time,
lifespan = request.lifespan,
keepLonger = request.keepLonger,
body = formatBody(request.threadString, request.date, request.level, request.tag, line)
)
}
@ -150,7 +149,7 @@ class PersistentLogger(
}
fun formatBody(threadString: String, date: Date, level: String, tag: String, message: String?): String {
return "[${BuildConfig.VERSION_NAME}] [$threadString] ${dateFormat.format(date)} $level $tag: $message"
return "[${BuildConfig.VERSION_NAME}] [$threadString] ${dateFormat.format(date)} $level $tag: ${Scrubber.scrub(message ?: "")}"
}
}

7
app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSection.java
View file

@ -18,4 +18,11 @@ interface LogSection {
* one line at a time.
*/
@NonNull CharSequence getContent(@NonNull Context context);
/**
* Whether or not this section has content.
*/
default boolean hasContent() {
return true;
}
}

5
app/src/main/java/org/thoughtcrime/securesms/logsubmit/LogSectionLoggerHeader.java
View file

@ -18,4 +18,9 @@ public class LogSectionLoggerHeader implements LogSection {
public @NonNull CharSequence getContent(@NonNull Context context) {
return "";
}
@Override
public boolean hasContent() {
return false;
}
}

31
app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogRepository.java
View file

@ -5,6 +5,7 @@ import android.content.Context;
import android.net.Uri;
import android.os.Build;
import android.os.ParcelFileDescriptor;
import android.os.SystemClock;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
@ -24,14 +25,19 @@ import org.thoughtcrime.securesms.logsubmit.util.Scrubber;
import org.thoughtcrime.securesms.net.StandardUserAgentInterceptor;
import org.thoughtcrime.securesms.providers.BlobProvider;
import org.thoughtcrime.securesms.push.SignalServiceNetworkAccess;
import org.thoughtcrime.securesms.util.ByteUnit;
import org.thoughtcrime.securesms.util.Stopwatch;
import org.thoughtcrime.securesms.util.Util;
import org.whispersystems.libsignal.util.guava.Optional;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
@ -135,6 +141,8 @@ public class SubmitDebugLogRepository {
}
try {
Stopwatch stopwatch = new Stopwatch("log-upload");
ParcelFileDescriptor[] fds = ParcelFileDescriptor.createPipe();
Uri gzipUri = BlobProvider.getInstance()
.forData(new ParcelFileDescriptor.AutoCloseInputStream(fds[0]), 0)
@ -145,15 +153,19 @@ public class SubmitDebugLogRepository {
gzipOutput.write(prefixStringBuilder.toString().getBytes());
stopwatch.split("front-matter");
try (LogDatabase.Reader reader = LogDatabase.getInstance(context).getAllBeforeTime(untilTime)) {
while (reader.hasNext()) {
gzipOutput.write(Scrubber.scrub(reader.next()).toString().getBytes());
gzipOutput.write(reader.next().getBytes());
gzipOutput.write("\n".getBytes());
}
}
StreamUtil.close(gzipOutput);
stopwatch.split("body");
String logUrl = uploadContent("application/gzip", new RequestBody() {
@Override
public @NonNull MediaType contentType() {
@ -171,6 +183,9 @@ public class SubmitDebugLogRepository {
}
});
stopwatch.split("upload");
stopwatch.stop(TAG);
BlobProvider.getInstance().delete(context, gzipUri);
return Optional.of(logUrl);
@ -258,14 +273,16 @@ public class SubmitDebugLogRepository {
List<LogLine> out = new ArrayList<>();
out.add(new SimpleLogLine(formatTitle(section.getTitle(), maxTitleLength), LogLine.Style.NONE, LogLine.Placeholder.NONE));
CharSequence content = Scrubber.scrub(section.getContent(context));
if (section.hasContent()) {
CharSequence content = Scrubber.scrub(section.getContent(context));
List<LogLine> lines = Stream.of(Pattern.compile("\\n").split(content))
.map(s -> new SimpleLogLine(s, LogStyleParser.parseStyle(s), LogStyleParser.parsePlaceholderType(s)))
.map(line -> (LogLine) line)
.toList();
List<LogLine> lines = Stream.of(Pattern.compile("\\n").split(content))
.map(s -> new SimpleLogLine(s, LogStyleParser.parseStyle(s), LogStyleParser.parsePlaceholderType(s)))
.map(line -> (LogLine) line)
.toList();
out.addAll(lines);
out.addAll(lines);
}
Log.d(TAG, "[" + section.getTitle() + "] Took " + (System.currentTimeMillis() - startTime) + " ms");

3
app/src/main/java/org/thoughtcrime/securesms/logsubmit/SubmitDebugLogViewModel.java
View file

@ -13,6 +13,7 @@ import org.signal.paging.PagedData;
import org.signal.paging.PagingConfig;
import org.signal.paging.PagingController;
import org.signal.paging.ProxyPagingController;
import org.thoughtcrime.securesms.database.LogDatabase;
import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
import org.whispersystems.libsignal.util.guava.Optional;
@ -42,6 +43,8 @@ public class SubmitDebugLogViewModel extends ViewModel {
repo.getPrefixLogLines(staticLines -> {
this.staticLines.addAll(staticLines);
LogDatabase.getInstance(ApplicationDependencies.getApplication()).trimToSize();
LogDataSource dataSource = new LogDataSource(ApplicationDependencies.getApplication(), staticLines, firstViewTime);
PagingConfig config = new PagingConfig.Builder().setPageSize(100)
.setBufferPages(3)

2
app/src/main/java/org/thoughtcrime/securesms/util/SignalUncaughtExceptionHandler.java
View file

@ -20,7 +20,7 @@ public class SignalUncaughtExceptionHandler implements Thread.UncaughtExceptionH
@Override
public void uncaughtException(@NonNull Thread t, @NonNull Throwable e) {
Log.e(TAG, "", e, TimeUnit.DAYS.toMillis(7));
Log.e(TAG, "", e, true);
SignalStore.blockUntilAllWritesFinished();
Log.blockUntilAllWritesFinished();
ApplicationDependencies.getJobManager().flush();

16
app/src/test/java/org/thoughtcrime/securesms/testutil/EmptyLogger.java
View file

@ -3,24 +3,20 @@ package org.thoughtcrime.securesms.testutil;
import org.signal.core.util.logging.Log;
public class EmptyLogger extends Log.Logger {
public EmptyLogger() {
super(0);
}
@Override
public void v(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void v(String tag, String message, Throwable t, long duration) { }
public void d(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void d(String tag, String message, Throwable t, long duration) { }
public void i(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void i(String tag, String message, Throwable t, long duration) { }
public void w(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void w(String tag, String message, Throwable t, long duration) { }
@Override
public void e(String tag, String message, Throwable t, long duration) { }
public void e(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void flush() { }

14
app/src/test/java/org/thoughtcrime/securesms/testutil/LogRecorder.java
View file

@ -17,32 +17,28 @@ public final class LogRecorder extends Log.Logger {
private final List<Entry> errors = new ArrayList<>();
private final List<Entry> wtf = new ArrayList<>();
public LogRecorder() {
super(0);
}
@Override
public void v(String tag, String message, Throwable t, long duration) {
public void v(String tag, String message, Throwable t, boolean keepLonger) {
verbose.add(new Entry(tag, message, t));
}
@Override
public void d(String tag, String message, Throwable t, long duration) {
public void d(String tag, String message, Throwable t, boolean keepLonger) {
debug.add(new Entry(tag, message, t));
}
@Override
public void i(String tag, String message, Throwable t, long duration) {
public void i(String tag, String message, Throwable t, boolean keepLonger) {
information.add(new Entry(tag, message, t));
}
@Override
public void w(String tag, String message, Throwable t, long duration) {
public void w(String tag, String message, Throwable t, boolean keepLonger) {
warnings.add(new Entry(tag, message, t));
}
@Override
public void e(String tag, String message, Throwable t, long duration) {
public void e(String tag, String message, Throwable t, boolean keepLonger) {
errors.add(new Entry(tag, message, t));
}

14
app/src/test/java/org/thoughtcrime/securesms/testutil/SystemOutLogger.java
View file

@ -3,32 +3,28 @@ package org.thoughtcrime.securesms.testutil;
import org.signal.core.util.logging.Log;
public final class SystemOutLogger extends Log.Logger {
public SystemOutLogger() {
super(0);
}
@Override
public void v(String tag, String message, Throwable t, long duration) {
public void v(String tag, String message, Throwable t, boolean keepLonger) {
printlnFormatted('v', tag, message, t);
}
@Override
public void d(String tag, String message, Throwable t, long duration) {
public void d(String tag, String message, Throwable t, boolean keepLonger) {
printlnFormatted('d', tag, message, t);
}
@Override
public void i(String tag, String message, Throwable t, long duration) {
public void i(String tag, String message, Throwable t, boolean keepLonger) {
printlnFormatted('i', tag, message, t);
}
@Override
public void w(String tag, String message, Throwable t, long duration) {
public void w(String tag, String message, Throwable t, boolean keepLonger) {
printlnFormatted('w', tag, message, t);
}
@Override
public void e(String tag, String message, Throwable t, long duration) {
public void e(String tag, String message, Throwable t, boolean keepLonger) {
printlnFormatted('e', tag, message, t);
}

14
core-util/src/main/java/org/signal/core/util/logging/AndroidLogger.java
View file

@ -5,32 +5,28 @@ import android.annotation.SuppressLint;
@SuppressLint("LogNotSignal")
public final class AndroidLogger extends Log.Logger {
public AndroidLogger() {
super(0);
}
@Override
public void v(String tag, String message, Throwable t, long lifespan) {
public void v(String tag, String message, Throwable t, boolean keepLonger) {
android.util.Log.v(tag, message, t);
}
@Override
public void d(String tag, String message, Throwable t, long lifespan) {
public void d(String tag, String message, Throwable t, boolean keepLonger) {
android.util.Log.d(tag, message, t);
}
@Override
public void i(String tag, String message, Throwable t, long lifespan) {
public void i(String tag, String message, Throwable t, boolean keepLonger) {
android.util.Log.i(tag, message, t);
}
@Override
public void w(String tag, String message, Throwable t, long lifespan) {
public void w(String tag, String message, Throwable t, boolean keepLonger) {
android.util.Log.w(tag, message, t);
}
@Override
public void e(String tag, String message, Throwable t, long lifespan) {
public void e(String tag, String message, Throwable t, boolean keepLonger) {
android.util.Log.e(tag, message, t);
}

21
core-util/src/main/java/org/signal/core/util/logging/CompoundLogger.java
View file

@ -11,42 +11,41 @@ class CompoundLogger extends Log.Logger {
private final Log.Logger[] loggers;
CompoundLogger(@NonNull Log.Logger... loggers) {
super(0);
this.loggers = loggers;
}
@Override
public void v(String tag, String message, Throwable t, long duration) {
public void v(String tag, String message, Throwable t, boolean keepLonger) {
for (Log.Logger logger : loggers) {
logger.v(tag, message, t, duration);
logger.v(tag, message, t, keepLonger);
}
}
@Override
public void d(String tag, String message, Throwable t, long duration) {
public void d(String tag, String message, Throwable t, boolean keepLonger) {
for (Log.Logger logger : loggers) {
logger.d(tag, message, t, duration);
logger.d(tag, message, t, keepLonger);
}
}
@Override
public void i(String tag, String message, Throwable t, long duration) {
public void i(String tag, String message, Throwable t, boolean keepLonger) {
for (Log.Logger logger : loggers) {
logger.i(tag, message, t, duration);
logger.i(tag, message, t, keepLonger);
}
}
@Override
public void w(String tag, String message, Throwable t, long duration) {
public void w(String tag, String message, Throwable t, boolean keepLonger) {
for (Log.Logger logger : loggers) {
logger.w(tag, message, t, duration);
logger.w(tag, message, t, keepLonger);
}
}
@Override
public void e(String tag, String message, Throwable t, long duration) {
public void e(String tag, String message, Throwable t, boolean keepLonger) {
for (Log.Logger logger : loggers) {
logger.e(tag, message, t, duration);
logger.e(tag, message, t, keepLonger);
}
}

86
core-util/src/main/java/org/signal/core/util/logging/Log.java
View file

@ -87,44 +87,44 @@ public final class Log {
logger.e(tag, message, t);
}
public static void v(String tag, String message, long duration) {
logger.v(tag, message, duration);
public static void v(String tag, String message, boolean keepLonger) {
logger.v(tag, message, keepLonger);
}
public static void d(String tag, String message, long duration) {
logger.d(tag, message, duration);
public static void d(String tag, String message, boolean keepLonger) {
logger.d(tag, message, keepLonger);
}
public static void i(String tag, String message, long duration) {
logger.i(tag, message, duration);
public static void i(String tag, String message, boolean keepLonger) {
logger.i(tag, message, keepLonger);
}
public static void w(String tag, String message, long duration) {
logger.w(tag, message, duration);
public static void w(String tag, String message, boolean keepLonger) {
logger.w(tag, message, keepLonger);
}
public static void e(String tag, String message, long duration) {
logger.e(tag, message, duration);
public static void e(String tag, String message, boolean keepLonger) {
logger.e(tag, message, keepLonger);
}
public static void v(String tag, String message, Throwable t, long duration) {
logger.v(tag, message, t, duration);
public static void v(String tag, String message, Throwable t, boolean keepLonger) {
logger.v(tag, message, t, keepLonger);
}
public static void d(String tag, String message, Throwable t, long duration) {
logger.d(tag, message, t, duration);
public static void d(String tag, String message, Throwable t, boolean keepLonger) {
logger.d(tag, message, t, keepLonger);
}
public static void i(String tag, String message, Throwable t, long duration) {
logger.i(tag, message, t, duration);
public static void i(String tag, String message, Throwable t, boolean keepLonger) {
logger.i(tag, message, t, keepLonger);
}
public static void w(String tag, String message, Throwable t, long duration) {
logger.w(tag, message, t, duration);
public static void w(String tag, String message, Throwable t, boolean keepLonger) {
logger.w(tag, message, t, keepLonger);
}
public static void e(String tag, String message, Throwable t, long duration) {
logger.e(tag, message, t, duration);
public static void e(String tag, String message, Throwable t, boolean keepLonger) {
logger.e(tag, message, t, keepLonger);
}
public static String tag(Class<?> clazz) {
@ -155,57 +155,51 @@ public final class Log {
public static abstract class Logger {
private final long defaultLifespan;
protected Logger(long defaultLifespan) {
this.defaultLifespan = defaultLifespan;
}
public abstract void v(String tag, String message, Throwable t, long lifespan);
public abstract void d(String tag, String message, Throwable t, long lifespan);
public abstract void i(String tag, String message, Throwable t, long lifespan);
public abstract void w(String tag, String message, Throwable t, long lifespan);
public abstract void e(String tag, String message, Throwable t, long lifespan);
public abstract void v(String tag, String message, Throwable t, boolean keepLonger);
public abstract void d(String tag, String message, Throwable t, boolean keepLonger);
public abstract void i(String tag, String message, Throwable t, boolean keepLonger);
public abstract void w(String tag, String message, Throwable t, boolean keepLonger);
public abstract void e(String tag, String message, Throwable t, boolean keepLonger);
public abstract void flush();
public void v(String tag, String message, long lifespan) {
v(tag, message, null, lifespan);
public void v(String tag, String message, boolean keepLonger) {
v(tag, message, null, keepLonger);
}
public void d(String tag, String message, long lifespan) {
d(tag, message, null, lifespan);
public void d(String tag, String message, boolean keepLonger) {
d(tag, message, null, keepLonger);
}
public void i(String tag, String message, long lifespan) {
i(tag, message, null, lifespan);
public void i(String tag, String message, boolean keepLonger) {
i(tag, message, null, keepLonger);
}
public void w(String tag, String message, long lifespan) {
w(tag, message, null, lifespan);
public void w(String tag, String message, boolean keepLonger) {
w(tag, message, null, keepLonger);
}
public void e(String tag, String message, long lifespan) {
e(tag, message, null, lifespan);
public void e(String tag, String message, boolean keepLonger) {
e(tag, message, null, keepLonger);
}
public void v(String tag, String message, Throwable t) {
v(tag, message, t, defaultLifespan);
v(tag, message, t, false);
}
public void d(String tag, String message, Throwable t) {
d(tag, message, t, defaultLifespan);
d(tag, message, t, false);
}
public void i(String tag, String message, Throwable t) {
i(tag, message, t, defaultLifespan);
i(tag, message, t, false);
}
public void w(String tag, String message, Throwable t) {
w(tag, message, t, defaultLifespan);
w(tag, message, t, false);
}
public void e(String tag, String message, Throwable t) {
e(tag, message, t, defaultLifespan);
e(tag, message, t, false);
}
public void v(String tag, String message) {

16
core-util/src/main/java/org/signal/core/util/logging/NoopLogger.java
View file

@ -4,24 +4,20 @@ package org.signal.core.util.logging;
* A logger that does nothing.
*/
class NoopLogger extends Log.Logger {
NoopLogger() {
super(0);
}
@Override
public void v(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void v(String tag, String message, Throwable t, long duration) { }
public void d(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void d(String tag, String message, Throwable t, long duration) { }
public void i(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void i(String tag, String message, Throwable t, long duration) { }
public void w(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void w(String tag, String message, Throwable t, long duration) { }
@Override
public void e(String tag, String message, Throwable t, long duration) { }
public void e(String tag, String message, Throwable t, boolean keepLonger) { }
@Override
public void flush() { }