empathicqubit/Signal-Android
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Store file hash to avoid data duplication.

Browse source
This commit is contained in:
alex-signal 2019-09-26 17:29:38 -03:00 • committed by Greyson Parrelli
parent 8449d75684
commit 7d02bb8487
2 changed files with 193 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

227
src/org/thoughtcrime/securesms/database/AttachmentDatabase.java
View file

@ -34,6 +34,7 @@ import androidx.annotation.VisibleForTesting;
import com.bumptech.glide.Glide;
import net.sqlcipher.DatabaseUtils;
import net.sqlcipher.database.SQLiteDatabase;
import org.json.JSONArray;
@ -51,6 +52,7 @@ import org.thoughtcrime.securesms.mms.MediaStream;
import org.thoughtcrime.securesms.mms.MmsException;
import org.thoughtcrime.securesms.mms.PartAuthority;
import org.thoughtcrime.securesms.stickers.StickerLocator;
import org.thoughtcrime.securesms.util.Base64;
import org.thoughtcrime.securesms.util.BitmapDecodingException;
import org.thoughtcrime.securesms.util.BitmapUtil;
import org.thoughtcrime.securesms.util.JsonUtils;
@ -59,12 +61,16 @@ import org.thoughtcrime.securesms.util.MediaUtil.ThumbnailData;
import org.thoughtcrime.securesms.util.StorageUtil;
import org.thoughtcrime.securesms.util.Util;
import org.thoughtcrime.securesms.video.EncryptedMediaDataSource;
import org.whispersystems.libsignal.util.guava.Optional;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@ -105,6 +111,7 @@ public class AttachmentDatabase extends Database {
static final String WIDTH = "width";
static final String HEIGHT = "height";
static final String CAPTION = "caption";
static final String DATA_HASH = "data_hash";
public static final String DIRECTORY = "parts";
@ -113,7 +120,8 @@ public class AttachmentDatabase extends Database {
public static final int TRANSFER_PROGRESS_PENDING = 2;
public static final int TRANSFER_PROGRESS_FAILED = 3;
private static final String PART_ID_WHERE = ROW_ID + " = ? AND " + UNIQUE_ID + " = ?";
private static final String PART_ID_WHERE = ROW_ID + " = ? AND " + UNIQUE_ID + " = ?";
private static final String PART_ID_WHERE_NOT = ROW_ID + " != ? AND " + UNIQUE_ID + " != ?";
private static final String[] PROJECTION = new String[] {ROW_ID,
MMS_ID, CONTENT_TYPE, NAME, CONTENT_DISPOSITION,
@ -121,7 +129,8 @@ public class AttachmentDatabase extends Database {
SIZE, FILE_NAME, THUMBNAIL, THUMBNAIL_ASPECT_RATIO,
UNIQUE_ID, DIGEST, FAST_PREFLIGHT_ID, VOICE_NOTE,
QUOTE, DATA_RANDOM, THUMBNAIL_RANDOM, WIDTH, HEIGHT,
CAPTION, STICKER_PACK_ID, STICKER_PACK_KEY, STICKER_ID};
CAPTION, STICKER_PACK_ID, STICKER_PACK_KEY, STICKER_ID,
DATA_HASH};
public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + " (" + ROW_ID + " INTEGER PRIMARY KEY, " +
MMS_ID + " INTEGER, " + "seq" + " INTEGER DEFAULT 0, " +
@ -135,12 +144,14 @@ public class AttachmentDatabase extends Database {
VOICE_NOTE + " INTEGER DEFAULT 0, " + DATA_RANDOM + " BLOB, " + THUMBNAIL_RANDOM + " BLOB, " +
QUOTE + " INTEGER DEFAULT 0, " + WIDTH + " INTEGER DEFAULT 0, " + HEIGHT + " INTEGER DEFAULT 0, " +
CAPTION + " TEXT DEFAULT NULL, " + STICKER_PACK_ID + " TEXT DEFAULT NULL, " +
STICKER_PACK_KEY + " DEFAULT NULL, " + STICKER_ID + " INTEGER DEFAULT -1);";
STICKER_PACK_KEY + " DEFAULT NULL, " + STICKER_ID + " INTEGER DEFAULT -1, " +
DATA_HASH + " TEXT DEFAULT NULL);";
public static final String[] CREATE_INDEXS = {
"CREATE INDEX IF NOT EXISTS part_mms_id_index ON " + TABLE_NAME + " (" + MMS_ID + ");",
"CREATE INDEX IF NOT EXISTS pending_push_index ON " + TABLE_NAME + " (" + TRANSFER_STATE + ");",
"CREATE INDEX IF NOT EXISTS part_sticker_pack_id_index ON " + TABLE_NAME + " (" + STICKER_PACK_ID + ");",
"CREATE INDEX IF NOT EXISTS part_data_hash_index ON " + TABLE_NAME + " (" + DATA_HASH + ");"
};
private final ExecutorService thumbnailExecutor = Util.newSingleThreadedLifoExecutor();
@ -279,11 +290,16 @@ public class AttachmentDatabase extends Database {
Cursor cursor = null;
try {
cursor = database.query(TABLE_NAME, new String[] {DATA, THUMBNAIL, CONTENT_TYPE}, MMS_ID + " = ?",
cursor = database.query(TABLE_NAME, new String[] {DATA, THUMBNAIL, CONTENT_TYPE, DATA_HASH, ROW_ID, UNIQUE_ID}, MMS_ID + " = ?",
new String[] {mmsId+""}, null, null, null);
while (cursor != null && cursor.moveToNext()) {
deleteAttachmentOnDisk(cursor.getString(0), cursor.getString(1), cursor.getString(2));
deleteAttachmentOnDisk(cursor.getString(cursor.getColumnIndex(DATA)),
cursor.getString(cursor.getColumnIndex(THUMBNAIL)),
cursor.getString(cursor.getColumnIndex(CONTENT_TYPE)),
cursor.getString(cursor.getColumnIndex(DATA_HASH)),
new AttachmentId(cursor.getLong(cursor.getColumnIndex(ROW_ID)),
cursor.getLong(cursor.getColumnIndex(UNIQUE_ID))));
}
} finally {
if (cursor != null)
@ -299,11 +315,16 @@ public class AttachmentDatabase extends Database {
Cursor cursor = null;
try {
cursor = database.query(TABLE_NAME, new String[] {DATA, THUMBNAIL, CONTENT_TYPE}, MMS_ID + " = ?",
cursor = database.query(TABLE_NAME, new String[] {DATA, THUMBNAIL, CONTENT_TYPE, DATA_HASH, ROW_ID, UNIQUE_ID}, MMS_ID + " = ?",
new String[] {mmsId+""}, null, null, null);
while (cursor != null && cursor.moveToNext()) {
deleteAttachmentOnDisk(cursor.getString(0), cursor.getString(1), cursor.getString(2));
deleteAttachmentOnDisk(cursor.getString(cursor.getColumnIndex(DATA)),
cursor.getString(cursor.getColumnIndex(THUMBNAIL)),
cursor.getString(cursor.getColumnIndex(CONTENT_TYPE)),
cursor.getString(cursor.getColumnIndex(DATA_HASH)),
new AttachmentId(cursor.getLong(cursor.getColumnIndex(ROW_ID)),
cursor.getLong(cursor.getColumnIndex(UNIQUE_ID))));
}
} finally {
if (cursor != null)
@ -336,7 +357,7 @@ public class AttachmentDatabase extends Database {
SQLiteDatabase database = databaseHelper.getWritableDatabase();
try (Cursor cursor = database.query(TABLE_NAME,
new String[]{DATA, THUMBNAIL, CONTENT_TYPE},
new String[]{DATA, THUMBNAIL, CONTENT_TYPE, DATA_HASH},
PART_ID_WHERE,
id.toStrings(),
null,
@ -347,12 +368,13 @@ public class AttachmentDatabase extends Database {
Log.w(TAG, "Tried to delete an attachment, but it didn't exist.");
return;
}
String data = cursor.getString(0);
String thumbnail = cursor.getString(1);
String contentType = cursor.getString(2);
String data = cursor.getString(cursor.getColumnIndex(DATA));
String thumbnail = cursor.getString(cursor.getColumnIndex(THUMBNAIL));
String contentType = cursor.getString(cursor.getColumnIndex(CONTENT_TYPE));
String dataHash = cursor.getString(cursor.getColumnIndex(DATA_HASH));
database.delete(TABLE_NAME, PART_ID_WHERE, id.toStrings());
deleteAttachmentOnDisk(data, thumbnail, contentType);
deleteAttachmentOnDisk(data, thumbnail, contentType, dataHash, id);
notifyAttachmentListeners();
}
}
@ -373,8 +395,19 @@ public class AttachmentDatabase extends Database {
}
@SuppressWarnings("ResultOfMethodCallIgnored")
private void deleteAttachmentOnDisk(@Nullable String data, @Nullable String thumbnail, @Nullable String contentType) {
if (!TextUtils.isEmpty(data)) {
private void deleteAttachmentOnDisk(@Nullable String data,
@Nullable String thumbnail,
@Nullable String contentType,
@Nullable String attachmentHash,
@NonNull AttachmentId attachmentId)
{
boolean dataInUse = isDataUsedByAnotherAttachment(attachmentHash, attachmentId);
if (dataInUse) {
Log.i(TAG, "Data is used by another attachment, skipping deletion");
}
if (!TextUtils.isEmpty(data) && !dataInUse) {
new File(data).delete();
}
@ -387,13 +420,31 @@ public class AttachmentDatabase extends Database {
}
}
private boolean isDataUsedByAnotherAttachment(@Nullable String attachmentHash, @NonNull AttachmentId attachmentId) {
if (attachmentHash == null) return false;
SQLiteDatabase database = databaseHelper.getReadableDatabase();
long matches = DatabaseUtils.longForQuery(database,
"SELECT count(*) FROM " + TABLE_NAME + " WHERE " + DATA_HASH + " = ? AND " + UNIQUE_ID + " != ? AND " + ROW_ID + " != ?;",
new String[]{attachmentHash,
Long.toString(attachmentId.getUniqueId()),
Long.toString(attachmentId.getRowId())});
return matches != 0;
}
public void insertAttachmentsForPlaceholder(long mmsId, @NonNull AttachmentId attachmentId, @NonNull InputStream inputStream)
throws MmsException
{
DatabaseAttachment placeholder = getAttachment(attachmentId);
SQLiteDatabase database = databaseHelper.getWritableDatabase();
ContentValues values = new ContentValues();
DataInfo dataInfo = setAttachmentData(inputStream);
DataInfo oldInfo = getAttachmentDataFileInfo(attachmentId, DATA);
DataInfo dataInfo = setAttachmentData(inputStream, false, attachmentId);
if (oldInfo != null) {
updateAttachmentDataHash(database, dataInfo.hash, oldInfo.hash);
}
if (placeholder != null && placeholder.isQuote() && !placeholder.getContentType().startsWith("image")) {
values.put(THUMBNAIL, dataInfo.file.getAbsolutePath());
@ -402,6 +453,7 @@ public class AttachmentDatabase extends Database {
values.put(DATA, dataInfo.file.getAbsolutePath());
values.put(SIZE, dataInfo.length);
values.put(DATA_RANDOM, dataInfo.random);
values.put(DATA_HASH, dataInfo.hash);
}
values.put(TRANSFER_STATE, TRANSFER_PROGRESS_DONE);
@ -423,7 +475,7 @@ public class AttachmentDatabase extends Database {
}
public void copyAttachmentData(@NonNull AttachmentId sourceId, @NonNull AttachmentId destinationId)
throws MmsException, IOException
throws MmsException
{
DatabaseAttachment sourceAttachment = getAttachment(sourceId);
@ -432,18 +484,18 @@ public class AttachmentDatabase extends Database {
}
SQLiteDatabase database = databaseHelper.getWritableDatabase();
DataInfo copyToDataInfo = getAttachmentDataFileInfo(destinationId, DATA);
DataInfo sourceDataInfo = getAttachmentDataFileInfo(sourceId, DATA);
if (copyToDataInfo == null) {
throw new MmsException("No attachment data found for destination!");
if (sourceDataInfo == null) {
throw new MmsException("No attachment data found for source!");
}
copyToDataInfo = setAttachmentData(copyToDataInfo.file, getAttachmentStream(sourceId, 0));
ContentValues contentValues = new ContentValues();
contentValues.put(SIZE, copyToDataInfo.length);
contentValues.put(DATA_RANDOM, copyToDataInfo.random);
contentValues.put(DATA, sourceDataInfo.file.getAbsolutePath());
contentValues.put(DATA_HASH, sourceDataInfo.hash);
contentValues.put(SIZE, sourceDataInfo.length);
contentValues.put(DATA_RANDOM, sourceDataInfo.random);
contentValues.put(TRANSFER_STATE, sourceAttachment.getTransferState());
contentValues.put(CONTENT_LOCATION, sourceAttachment.getLocation());
@ -501,14 +553,18 @@ public class AttachmentDatabase extends Database {
@NonNull MediaStream mediaStream)
throws MmsException
{
SQLiteDatabase database = databaseHelper.getWritableDatabase();
DataInfo dataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA);
SQLiteDatabase database = databaseHelper.getWritableDatabase();
DataInfo oldDataInfo = getAttachmentDataFileInfo(databaseAttachment.getAttachmentId(), DATA);
if (dataInfo == null) {
if (oldDataInfo == null) {
throw new MmsException("No attachment data found!");
}
dataInfo = setAttachmentData(dataInfo.file, mediaStream.getStream());
DataInfo dataInfo = setAttachmentData(oldDataInfo.file,
mediaStream.getStream(),
false,
databaseAttachment.getAttachmentId());
updateAttachmentDataHash(database, dataInfo.hash, oldDataInfo.hash);
ContentValues contentValues = new ContentValues();
contentValues.put(SIZE, dataInfo.length);
@ -516,10 +572,25 @@ public class AttachmentDatabase extends Database {
contentValues.put(WIDTH, mediaStream.getWidth());
contentValues.put(HEIGHT, mediaStream.getHeight());
contentValues.put(DATA_RANDOM, dataInfo.random);
contentValues.put(DATA_HASH, dataInfo.hash);
database.update(TABLE_NAME, contentValues, PART_ID_WHERE, databaseAttachment.getAttachmentId().toStrings());
}
private static void updateAttachmentDataHash(@NonNull SQLiteDatabase database,
@NonNull String newHash,
@Nullable String oldHash)
{
if (oldHash == null) return;
ContentValues contentValues = new ContentValues();
contentValues.put(DATA_HASH, newHash);
database.update(TABLE_NAME,
contentValues,
DATA_HASH + " = ?",
new String[]{oldHash});
}
public void updateAttachmentFileName(@NonNull AttachmentId attachmentId,
@Nullable String fileName)
{
@ -627,7 +698,7 @@ public class AttachmentDatabase extends Database {
}
try {
cursor = database.query(TABLE_NAME, new String[]{dataType, SIZE, randomColumn}, PART_ID_WHERE, attachmentId.toStrings(),
cursor = database.query(TABLE_NAME, new String[]{dataType, SIZE, randomColumn, DATA_HASH}, PART_ID_WHERE, attachmentId.toStrings(),
null, null, null);
if (cursor != null && cursor.moveToFirst()) {
@ -637,7 +708,8 @@ public class AttachmentDatabase extends Database {
return new DataInfo(new File(cursor.getString(0)),
cursor.getLong(1),
cursor.getBlob(2));
cursor.getBlob(2),
cursor.getString(3));
} else {
return null;
}
@ -648,42 +720,112 @@ public class AttachmentDatabase extends Database {
}
private @NonNull DataInfo setAttachmentData(@NonNull Uri uri)
private @NonNull DataInfo setAttachmentData(@NonNull Uri uri,
boolean isThumbnail,
@Nullable AttachmentId attachmentId)
throws MmsException
{
try {
InputStream inputStream = PartAuthority.getAttachmentStream(context, uri);
return setAttachmentData(inputStream);
return setAttachmentData(inputStream, isThumbnail, attachmentId);
} catch (IOException e) {
throw new MmsException(e);
}
}
private @NonNull DataInfo setAttachmentData(@NonNull InputStream in)
private @NonNull DataInfo setAttachmentData(@NonNull InputStream in,
boolean isThumbnail,
@Nullable AttachmentId attachmentId)
throws MmsException
{
try {
File partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE);
File dataFile = File.createTempFile("part", ".mms", partsDirectory);
return setAttachmentData(dataFile, in);
return setAttachmentData(dataFile, in, isThumbnail, attachmentId);
} catch (IOException e) {
throw new MmsException(e);
}
}
private @NonNull DataInfo setAttachmentData(@NonNull File destination, @NonNull InputStream in)
private @NonNull DataInfo setAttachmentData(@NonNull File destination,
@NonNull InputStream in,
boolean isThumbnail,
@Nullable AttachmentId attachmentId)
throws MmsException
{
try {
Pair<byte[], OutputStream> out = ModernEncryptingPartOutputStream.createFor(attachmentSecret, destination, false);
long length = Util.copy(in, out.second);
MessageDigest messageDigest = MessageDigest.getInstance("SHA-256");
DigestInputStream digestInputStream = new DigestInputStream(in, messageDigest);
Pair<byte[], OutputStream> out = ModernEncryptingPartOutputStream.createFor(attachmentSecret, destination, false);
long length = Util.copy(digestInputStream, out.second);
String hash = Base64.encodeBytes(digestInputStream.getMessageDigest().digest());
return new DataInfo(destination, length, out.first);
} catch (IOException e) {
if (!isThumbnail) {
Log.i(TAG, "setAttachmentData: " + destination.getAbsolutePath());
SQLiteDatabase database = databaseHelper.getWritableDatabase();
Optional<DataInfo> sharedDataInfo = findDuplicateDataFileInfo(database, hash, attachmentId);
if (sharedDataInfo.isPresent()) {
if (sharedDataInfo.get().file != destination && !destination.delete()) {
Log.w(TAG, "setAttachmentData: Tried to delete original destination file but failed :(");
}
return sharedDataInfo.get();
}
}
return new DataInfo(destination, length, out.first, hash);
} catch (IOException | NoSuchAlgorithmException e) {
throw new MmsException(e);
}
}
private static @NonNull Optional<DataInfo> findDuplicateDataFileInfo(@NonNull SQLiteDatabase database,
@NonNull String hash,
@Nullable AttachmentId excludedAttachmentId)
{
Pair<String, String[]> selectorArgs = buildSharedFileSelectorArgs(hash, excludedAttachmentId);
try (Cursor cursor = database.query(TABLE_NAME,
new String[]{DATA, DATA_RANDOM, SIZE},
selectorArgs.first,
selectorArgs.second,
null,
null,
null,
"1"))
{
if (cursor == null || !cursor.moveToFirst()) return Optional.absent();
if (cursor.getCount() > 0) {
DataInfo dataInfo = new DataInfo(new File(cursor.getString(cursor.getColumnIndex(DATA))),
cursor.getLong(cursor.getColumnIndex(SIZE)),
cursor.getBlob(cursor.getColumnIndex(DATA_RANDOM)),
hash);
return Optional.of(dataInfo);
} else {
return Optional.absent();
}
}
}
private static Pair<String, String[]> buildSharedFileSelectorArgs(@NonNull String newHash,
@Nullable AttachmentId attachmentId)
{
final String selector;
final String[] selection;
if (attachmentId == null) {
selector = DATA_HASH + " = ?";
selection = new String[]{newHash};
} else {
selector = PART_ID_WHERE_NOT + " AND " + DATA_HASH + " = ?";
selection = new String[]{Long.toString(attachmentId.getRowId()),
Long.toString(attachmentId.getUniqueId()),
newHash};
}
return Pair.create(selector, selection);
}
public List<DatabaseAttachment> getAttachment(@NonNull Cursor cursor) {
try {
if (cursor.getColumnIndex(AttachmentDatabase.ATTACHMENT_JSON_ALIAS) != -1) {
@ -767,7 +909,7 @@ public class AttachmentDatabase extends Database {
long uniqueId = System.currentTimeMillis();
if (attachment.getDataUri() != null) {
dataInfo = setAttachmentData(attachment.getDataUri());
dataInfo = setAttachmentData(attachment.getDataUri(), false, null);
Log.d(TAG, "Wrote part to file: " + dataInfo.file.getAbsolutePath());
}
@ -799,6 +941,7 @@ public class AttachmentDatabase extends Database {
contentValues.put(DATA, dataInfo.file.getAbsolutePath());
contentValues.put(SIZE, dataInfo.length);
contentValues.put(DATA_RANDOM, dataInfo.random);
contentValues.put(DATA_HASH, dataInfo.hash);
}
boolean notifyPacks = attachment.isSticker() && !hasStickerAttachments();
@ -851,7 +994,7 @@ public class AttachmentDatabase extends Database {
{
Log.i(TAG, "updating part thumbnail for #" + attachmentId);
DataInfo thumbnailFile = setAttachmentData(in);
DataInfo thumbnailFile = setAttachmentData(in, true, attachmentId);
SQLiteDatabase database = databaseHelper.getWritableDatabase();
ContentValues values = new ContentValues(2);
@ -949,11 +1092,13 @@ public class AttachmentDatabase extends Database {
private final File file;
private final long length;
private final byte[] random;
private final String hash;
private DataInfo(File file, long length, byte[] random) {
private DataInfo(File file, long length, byte[] random, String hash) {
this.file = file;
this.length = length;
this.random = random;
this.hash = hash;
}
}
}

8
src/org/thoughtcrime/securesms/database/helpers/SQLCipherOpenHelper.java
View file

@ -76,8 +76,9 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
private static final int RECIPIENT_SEARCH = 25;
private static final int RECIPIENT_CLEANUP = 26;
private static final int MMS_RECIPIENT_CLEANUP = 27;
private static final int ATTACHMENT_HASHING = 28;
private static final int DATABASE_VERSION = 27;
private static final int DATABASE_VERSION = 28;
private static final String DATABASE_NAME = "signal.db";
private final Context context;
@ -522,6 +523,11 @@ public class SQLCipherOpenHelper extends SQLiteOpenHelper {
Log.i(TAG, "MMS recipient cleanup updated " + count + " rows.");
}
if (oldVersion < ATTACHMENT_HASHING) {
db.execSQL("ALTER TABLE part ADD COLUMN data_hash TEXT DEFAULT NULL");
db.execSQL("CREATE INDEX IF NOT EXISTS part_data_hash_index ON part (data_hash)");
}
db.setTransactionSuccessful();
} finally {
db.endTransaction();