Fallback to profile fetches for unlisted contacts.

app/src/main/java/org/thoughtcrime/securesms/NewConversationActivity.java

@@ -102,7 +102,7 @@ public class NewConversationActivity extends ContactSelectionActivity
     intent.putExtra(ConversationActivity.TEXT_EXTRA, getIntent().getStringExtra(ConversationActivity.TEXT_EXTRA));
     intent.setDataAndType(getIntent().getData(), getIntent().getType());
 
-    long existingThread = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient);
+    long existingThread = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient.getId());
 
     intent.putExtra(ConversationActivity.THREAD_ID_EXTRA, existingThread);
     intent.putExtra(ConversationActivity.DISTRIBUTION_TYPE_EXTRA, ThreadDatabase.DistributionTypes.DEFAULT);

app/src/main/java/org/thoughtcrime/securesms/SmsSendtoActivity.java

@@ -48,7 +48,7 @@ public class SmsSendtoActivity extends Activity {
       Toast.makeText(this, R.string.ConversationActivity_specify_recipient, Toast.LENGTH_LONG).show();
     } else {
       Recipient recipient = Recipient.external(this, destination.getDestination());
-      long      threadId  = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient);
+      long      threadId  = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient.getId());
 
       nextIntent = new Intent(this, ConversationActivity.class);
       nextIntent.putExtra(ConversationActivity.TEXT_EXTRA, destination.getBody());

app/src/main/java/org/thoughtcrime/securesms/contacts/sync/DirectoryHelper.java

@@ -25,20 +25,21 @@ import org.thoughtcrime.securesms.contacts.ContactAccessor;
 import org.thoughtcrime.securesms.contacts.ContactsDatabase;
 import org.thoughtcrime.securesms.crypto.SessionUtil;
 import org.thoughtcrime.securesms.database.DatabaseFactory;
-import org.thoughtcrime.securesms.database.MessageDatabase;
 import org.thoughtcrime.securesms.database.MessageDatabase.InsertResult;
 import org.thoughtcrime.securesms.database.RecipientDatabase;
 import org.thoughtcrime.securesms.database.RecipientDatabase.BulkOperationsHandle;
 import org.thoughtcrime.securesms.database.RecipientDatabase.RegisteredState;
+import org.thoughtcrime.securesms.database.SessionDatabase;
+import org.thoughtcrime.securesms.database.ThreadDatabase;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.jobs.MultiDeviceContactUpdateJob;
+import org.thoughtcrime.securesms.jobs.RetrieveProfileJob;
 import org.thoughtcrime.securesms.jobs.StorageSyncJob;
 import org.thoughtcrime.securesms.keyvalue.SignalStore;
 import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.notifications.NotificationChannels;
 import org.thoughtcrime.securesms.permissions.Permissions;
 import org.thoughtcrime.securesms.phonenumbers.PhoneNumberFormatter;
-import org.thoughtcrime.securesms.recipients.RecipientDetails;
 import org.thoughtcrime.securesms.registration.RegistrationUtil;
 import org.thoughtcrime.securesms.storage.StorageSyncHelper;
 import org.thoughtcrime.securesms.recipients.Recipient;
@@ -50,10 +51,13 @@ import org.thoughtcrime.securesms.util.SetUtil;
 import org.thoughtcrime.securesms.util.Stopwatch;
 import org.thoughtcrime.securesms.util.TextSecurePreferences;
 import org.thoughtcrime.securesms.util.Util;
+import org.whispersystems.libsignal.util.Pair;
 import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.profiles.ProfileAndCredential;
 import org.whispersystems.signalservice.api.profiles.SignalServiceProfile;
 import org.whispersystems.signalservice.api.push.exceptions.NotFoundException;
 import org.whispersystems.signalservice.api.util.UuidUtil;
+import org.whispersystems.signalservice.internal.util.concurrent.ListenableFuture;
 
 import java.io.IOException;
 import java.util.Calendar;
@@ -179,6 +183,13 @@ public class DirectoryHelper {
       } else {
         recipientDatabase.markRegistered(recipient.getId());
       }
+    } else if (recipient.hasUuid() && recipient.isRegistered() && hasCommunicatedWith(context, recipient)) {
+      if (isUuidRegistered(context, recipient)) {
+        recipientDatabase.markRegistered(recipient.getId(), recipient.requireUuid());
+      } else {
+        recipientDatabase.markUnregistered(recipient.getId());
+      }
+      stopwatch.split("e164-unlisted-network");
     } else {
       recipientDatabase.markUnregistered(recipient.getId());
     }
@@ -244,6 +255,17 @@ public class DirectoryHelper {
 
     stopwatch.split("process-cds");
 
+    UnlistedResult unlistedResult = filterForUnlistedUsers(context, inactiveIds);
+
+    inactiveIds.removeAll(unlistedResult.getPossiblyActive());
+
+    if (unlistedResult.getRetries().size() > 0) {
+      Log.i(TAG, "Some profile fetches failed to resolve. Assuming not-inactive for now and scheduling a retry.");
+      RetrieveProfileJob.enqueue(unlistedResult.getRetries());
+    }
+
+    stopwatch.split("handle-unlisted");
+
     recipientDatabase.bulkUpdatedRegisteredStatus(uuidMap, inactiveIds);
 
     stopwatch.split("update-registered");
@@ -275,16 +297,10 @@ public class DirectoryHelper {
 
   private static boolean isUuidRegistered(@NonNull Context context, @NonNull Recipient recipient) throws IOException {
     try {
-      ProfileUtil.retrieveProfile(context, recipient, SignalServiceProfile.RequestType.PROFILE).get(10, TimeUnit.SECONDS);
+      ProfileUtil.retrieveProfileSync(context, recipient, SignalServiceProfile.RequestType.PROFILE);
       return true;
-    } catch (ExecutionException e) {
+    } catch (NotFoundException e) {
-      if (e.getCause() instanceof NotFoundException) {
+      return false;
-        return false;
-      } else {
-        throw new IOException(e);
-      }
-    } catch (InterruptedException | TimeoutException e) {
-      throw new IOException(e);
     }
   }
 
@@ -420,6 +436,50 @@ public class DirectoryHelper {
     }).collect(Collectors.toSet());
   }
 
+  /**
+   * Users can mark themselves as 'unlisted' in CDS, meaning that even if CDS says they're
+   * unregistered, they might actually be registered. We need to double-check users who we already
+   * have UUIDs for. Also, we only want to bother doing this for users we have conversations for,
+   * so we will also only check for users that have a thread.
+   */
+  private static UnlistedResult filterForUnlistedUsers(@NonNull Context context, @NonNull Set<RecipientId> inactiveIds) {
+    List<Recipient> possiblyUnlisted = Stream.of(inactiveIds)
+                                             .map(Recipient::resolved)
+                                             .filter(Recipient::isRegistered)
+                                             .filter(Recipient::hasUuid)
+                                             .filter(r -> hasCommunicatedWith(context, r))
+                                             .toList();
+
+    List<Pair<Recipient, ListenableFuture<ProfileAndCredential>>> futures = Stream.of(possiblyUnlisted)
+                                                                                  .map(r -> new Pair<>(r, ProfileUtil.retrieveProfile(context, r, SignalServiceProfile.RequestType.PROFILE)))
+                                                                                  .toList();
+    Set<RecipientId> potentiallyActiveIds = new HashSet<>();
+    Set<RecipientId> retries              = new HashSet<>();
+
+    Stream.of(futures)
+          .forEach(pair -> {
+            try {
+              pair.second().get(5, TimeUnit.SECONDS);
+              potentiallyActiveIds.add(pair.first().getId());
+            } catch (InterruptedException | TimeoutException e) {
+              retries.add(pair.first().getId());
+              potentiallyActiveIds.add(pair.first().getId());
+            } catch (ExecutionException e) {
+              if (!(e.getCause() instanceof NotFoundException)) {
+                retries.add(pair.first().getId());
+                potentiallyActiveIds.add(pair.first().getId());
+              }
+            }
+          });
+
+    return new UnlistedResult(potentiallyActiveIds, retries);
+  }
+
+  private static boolean hasCommunicatedWith(@NonNull Context context, @NonNull Recipient recipient) {
+    return DatabaseFactory.getThreadDatabase(context).hasThread(recipient.getId()) ||
+           DatabaseFactory.getSessionDatabase(context).hasSessionFor(recipient.getId());
+  }
+
   static class DirectoryResult {
     private final Map<String, UUID>   registeredNumbers;
     private final Map<String, String> numberRewrites;
@@ -441,6 +501,24 @@ public class DirectoryHelper {
     }
   }
 
+  private static class UnlistedResult {
+    private final Set<RecipientId> possiblyActive;
+    private final Set<RecipientId> retries;
+
+    private UnlistedResult(@NonNull Set<RecipientId> possiblyActive, @NonNull Set<RecipientId> retries) {
+      this.possiblyActive = possiblyActive;
+      this.retries        = retries;
+    }
+
+    @NonNull Set<RecipientId> getPossiblyActive() {
+      return possiblyActive;
+    }
+
+    @NonNull Set<RecipientId> getRetries() {
+      return retries;
+    }
+  }
+
   private static class AccountHolder {
     private final boolean fresh;
     private final Account account;

app/src/main/java/org/thoughtcrime/securesms/conversationlist/ConversationListFragment.java

@@ -368,7 +368,7 @@ public class ConversationListFragment extends MainFragment implements ActionMode
   @Override
   public void onContactClicked(@NonNull Recipient contact) {
     SimpleTask.run(getViewLifecycleOwner().getLifecycle(), () -> {
-      return DatabaseFactory.getThreadDatabase(getContext()).getThreadIdIfExistsFor(contact);
+      return DatabaseFactory.getThreadDatabase(getContext()).getThreadIdIfExistsFor(contact.getId());
     }, threadId -> {
       hideKeyboard();
       getNavigator().goToConversation(contact.getId(),

app/src/main/java/org/thoughtcrime/securesms/database/RecipientDatabase.java

@@ -1801,6 +1801,12 @@ public class RecipientDatabase extends Database {
     }
   }
 
+  /**
+   * Handles inserts the (e164, UUID) pairs, which could result in merges. Does not mark users as
+   * registered.
+   *
+   * @return A mapping of (RecipientId, UUID)
+   */
   public @NonNull Map<RecipientId, String> bulkProcessCdsResult(@NonNull Map<String, UUID> mapping) {
     SQLiteDatabase               db      = databaseHelper.getWritableDatabase();
     HashMap<RecipientId, String> uuidMap = new HashMap<>();

app/src/main/java/org/thoughtcrime/securesms/database/SessionDatabase.java

@@ -12,6 +12,7 @@ import net.sqlcipher.database.SQLiteDatabase;
 import org.thoughtcrime.securesms.database.helpers.SQLCipherOpenHelper;
 import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.recipients.RecipientId;
+import org.thoughtcrime.securesms.util.SqlUtil;
 import org.whispersystems.libsignal.state.SessionRecord;
 import org.whispersystems.signalservice.api.push.SignalServiceAddress;
 
@@ -145,6 +146,16 @@ public class SessionDatabase extends Database {
     database.delete(TABLE_NAME, RECIPIENT_ID + " = ?", new String[] {recipientId.serialize()});
   }
 
+  public boolean hasSessionFor(@NonNull RecipientId recipientId) {
+    SQLiteDatabase database = databaseHelper.getReadableDatabase();
+    String         query    = RECIPIENT_ID + " = ?";
+    String[]       args     = SqlUtil.buildArgs(recipientId);
+
+    try (Cursor cursor = database.query(TABLE_NAME, new String[] { ID }, query, args, null, null, null, "1")) {
+      return cursor != null && cursor.moveToFirst();
+    }
+  }
+
   public static final class SessionRow {
     private final RecipientId   recipientId;
     private final int           deviceId;

app/src/main/java/org/thoughtcrime/securesms/database/ThreadDatabase.java

@@ -872,22 +872,17 @@ public class ThreadDatabase extends Database {
     deleteAllThreads();
   }
 
-  public long getThreadIdIfExistsFor(Recipient recipient) {
+  public long getThreadIdIfExistsFor(@NonNull RecipientId recipientId) {
     SQLiteDatabase db            = databaseHelper.getReadableDatabase();
     String         where         = RECIPIENT_ID + " = ?";
-    String[]       recipientsArg = new String[] {recipient.getId().serialize()};
+    String[]       recipientsArg = new String[] {recipientId.serialize()};
-    Cursor         cursor        = null;
 
-    try {
+    try (Cursor cursor = db.query(TABLE_NAME, new String[]{ ID }, where, recipientsArg, null, null, null, "1")) {
-      cursor = db.query(TABLE_NAME, new String[]{ID}, where, recipientsArg, null, null, null);
+      if (cursor != null && cursor.moveToFirst()) {
-
+        return CursorUtil.requireLong(cursor, ID);
-      if (cursor != null && cursor.moveToFirst())
+      } else {
-        return cursor.getLong(cursor.getColumnIndexOrThrow(ID));
+        return -1;
-      else
+      }
-        return -1L;
-    } finally {
-      if (cursor != null)
-        cursor.close();
     }
   }
 
@@ -950,6 +945,10 @@ public class ThreadDatabase extends Database {
     return Recipient.resolved(id);
   }
 
+  public boolean hasThread(@NonNull RecipientId recipientId) {
+    return getThreadIdIfExistsFor(recipientId) > -1;
+  }
+
   public void setHasSent(long threadId, boolean hasSent) {
     ContentValues contentValues = new ContentValues(1);
     contentValues.put(HAS_SENT, hasSent ? 1 : 0);

app/src/main/java/org/thoughtcrime/securesms/jobs/RetrieveProfileJob.java

@@ -28,6 +28,7 @@ import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.profiles.ProfileName;
 import org.thoughtcrime.securesms.recipients.Recipient;
 import org.thoughtcrime.securesms.recipients.RecipientId;
+import org.thoughtcrime.securesms.recipients.RecipientUtil;
 import org.thoughtcrime.securesms.transport.RetryLaterException;
 import org.thoughtcrime.securesms.util.Base64;
 import org.thoughtcrime.securesms.util.IdentityUtil;
@@ -54,7 +55,9 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -218,11 +221,17 @@ public class RetrieveProfileJob extends BaseJob {
 
   @Override
   public void onRun() throws IOException, RetryLaterException {
-    Stopwatch        stopwatch = new Stopwatch("RetrieveProfile");
+    Stopwatch         stopwatch         = new Stopwatch("RetrieveProfile");
-    Set<RecipientId> retries   = new HashSet<>();
+    RecipientDatabase recipientDatabase = DatabaseFactory.getRecipientDatabase(context);
+    Set<RecipientId>  retries           = new HashSet<>();
+    Set<RecipientId>  unregistered      = new HashSet<>();
 
-    List<Recipient> recipients = Stream.of(recipientIds).map(Recipient::resolved).toList();
+    RecipientUtil.ensureUuidsAreAvailable(context, Stream.of(Recipient.resolvedList(recipientIds))
-    stopwatch.split("resolve");
+                                                         .filter(r -> r.getRegistered() != RecipientDatabase.RegisteredState.NOT_REGISTERED)
+                                                         .toList());
+
+    List<Recipient> recipients = Recipient.resolvedList(recipientIds);
+    stopwatch.split("resolve-ensure");
 
     List<Pair<Recipient, ListenableFuture<ProfileAndCredential>>> futures = Stream.of(recipients)
                                                                                   .filter(Recipient::hasServiceIdentifier)
@@ -244,6 +253,9 @@ public class RetrieveProfileJob extends BaseJob {
                                                                        retries.add(recipient.getId());
                                                                      } else if (e.getCause() instanceof NotFoundException) {
                                                                        Log.w(TAG, "Failed to find a profile for " + recipient.getId());
+                                                                       if (recipient.isRegistered()) {
+                                                                         unregistered.add(recipient.getId());
+                                                                       }
                                                                      } else {
                                                                        Log.w(TAG, "Failed to retrieve profile for " + recipient.getId());
                                                                      }
@@ -259,7 +271,18 @@ public class RetrieveProfileJob extends BaseJob {
     }
 
     Set<RecipientId> success = SetUtil.difference(recipientIds, retries);
-    DatabaseFactory.getRecipientDatabase(context).markProfilesFetched(success, System.currentTimeMillis());
+    recipientDatabase.markProfilesFetched(success, System.currentTimeMillis());
+
+    Map<RecipientId, String> newlyRegistered = Stream.of(profiles)
+                                                     .map(Pair::first)
+                                                     .filterNot(Recipient::isRegistered)
+                                                     .collect(Collectors.toMap(Recipient::getId,
+                                                              r -> r.getUuid().transform(UUID::toString).orNull()));
+
+    if (unregistered.size() > 0 || newlyRegistered.size() > 0) {
+      Log.i(TAG, "Marking " + newlyRegistered.size() + " users as registered and " + unregistered.size() + " users as unregistered.");
+      recipientDatabase.bulkUpdatedRegisteredStatus(newlyRegistered, unregistered);
+    }
 
     stopwatch.split("process");
 

app/src/main/java/org/thoughtcrime/securesms/recipients/RecipientUtil.java

@@ -7,7 +7,6 @@ import androidx.annotation.Nullable;
 import androidx.annotation.WorkerThread;
 
 import com.annimon.stream.Stream;
-import com.google.android.gms.common.Feature;
 
 import org.thoughtcrime.securesms.contacts.sync.DirectoryHelper;
 import org.thoughtcrime.securesms.database.DatabaseFactory;
@@ -29,7 +28,6 @@ import org.whispersystems.libsignal.util.guava.Optional;
 import org.whispersystems.signalservice.api.push.SignalServiceAddress;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
@@ -87,6 +85,17 @@ public class RecipientUtil {
 
   public static @NonNull List<SignalServiceAddress> toSignalServiceAddressesFromResolved(@NonNull Context context, @NonNull List<Recipient> recipients)
       throws IOException
+  {
+    ensureUuidsAreAvailable(context, recipients);
+
+    return Stream.of(recipients)
+                 .map(Recipient::resolve)
+                 .map(r -> new SignalServiceAddress(r.getUuid().orNull(), r.getE164().orNull()))
+                 .toList();
+  }
+
+  public static void ensureUuidsAreAvailable(@NonNull Context context, @NonNull Collection<Recipient> recipients)
+      throws IOException
   {
     if (FeatureFlags.cds()) {
       List<Recipient> recipientsWithoutUuids = Stream.of(recipients)
@@ -98,11 +107,6 @@ public class RecipientUtil {
         DirectoryHelper.refreshDirectoryFor(context, recipientsWithoutUuids, false);
       }
     }
-
-    return Stream.of(recipients)
-                 .map(Recipient::resolve)
-                 .map(r -> new SignalServiceAddress(r.getUuid().orNull(), r.getE164().orNull()))
-                 .toList();
   }
 
   public static boolean isBlockable(@NonNull Recipient recipient) {
@@ -241,7 +245,7 @@ public class RecipientUtil {
 
   @WorkerThread
   public static void shareProfileIfFirstSecureMessage(@NonNull Context context, @NonNull Recipient recipient) {
-    long threadId = DatabaseFactory.getThreadDatabase(context).getThreadIdIfExistsFor(recipient);
+    long threadId = DatabaseFactory.getThreadDatabase(context).getThreadIdIfExistsFor(recipient.getId());
 
     if (isPreMessageRequestThread(context, threadId)) {
       return;

app/src/main/java/org/thoughtcrime/securesms/sharing/ShareActivity.java

@@ -160,7 +160,7 @@ public class ShareActivity extends PassphraseRequiredActivity
         recipient = Recipient.external(this, number);
       }
 
-      long existingThread = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient);
+      long existingThread = DatabaseFactory.getThreadDatabase(this).getThreadIdIfExistsFor(recipient.getId());
       return new Pair<>(existingThread, recipient);
     }, result -> onDestinationChosen(result.first(), result.second().getId()));
 

app/src/main/java/org/thoughtcrime/securesms/util/ProfileUtil.java

@@ -5,11 +5,13 @@ import android.content.Context;
 import androidx.annotation.NonNull;
 import androidx.annotation.Nullable;
 import androidx.annotation.WorkerThread;
+import androidx.navigation.ActionOnlyNavDirections;
 
 import org.signal.zkgroup.VerificationFailedException;
 import org.signal.zkgroup.profiles.ProfileKey;
 import org.thoughtcrime.securesms.crypto.ProfileKeyUtil;
 import org.thoughtcrime.securesms.crypto.UnidentifiedAccessUtil;
+import org.thoughtcrime.securesms.database.RecipientDatabase;
 import org.thoughtcrime.securesms.dependencies.ApplicationDependencies;
 import org.thoughtcrime.securesms.logging.Log;
 import org.thoughtcrime.securesms.recipients.Recipient;
@@ -56,6 +58,8 @@ public final class ProfileUtil {
     } catch (ExecutionException e) {
       if (e.getCause() instanceof PushNetworkException) {
         throw (PushNetworkException) e.getCause();
+      } else if (e.getCause() instanceof NotFoundException) {
+        throw (NotFoundException) e.getCause();
       } else {
         throw new IOException(e);
       }
@@ -68,7 +72,7 @@ public final class ProfileUtil {
                                                                                 @NonNull Recipient recipient,
                                                                                 @NonNull SignalServiceProfile.RequestType requestType)
   {
-    SignalServiceAddress         address            = RecipientUtil.toSignalServiceAddressBestEffort(context, recipient);
+    SignalServiceAddress         address            = toSignalServiceAddress(context, recipient);
     Optional<UnidentifiedAccess> unidentifiedAccess = getUnidentifiedAccess(context, recipient);
     Optional<ProfileKey>         profileKey         = ProfileKeyUtil.profileKeyOptional(recipient.getProfileKey());
 
@@ -131,4 +135,12 @@ public final class ProfileUtil {
 
     return Optional.absent();
   }
+
+  private static @NonNull SignalServiceAddress toSignalServiceAddress(@NonNull Context context, @NonNull Recipient recipient) {
+    if (recipient.getRegistered() == RecipientDatabase.RegisteredState.NOT_REGISTERED) {
+      return new SignalServiceAddress(recipient.getUuid().orNull(), recipient.getE164().orNull());
+    } else {
+      return RecipientUtil.toSignalServiceAddressBestEffort(context, recipient);
+    }
+  }
 }