Improve Sqlite namespace loader implementation (#6488)

changelog/@unreleased/pr-6488.v2.yml

@@ -0,0 +1,5 @@
+type: improvement
+improvement:
+  description: Improve Sqlite namespace loader implementation
+  links:
+  - https://github.com/palantir/atlasdb/pull/6488

leader-election-impl/src/main/java/com/palantir/paxos/SqlitePaxosStateLog.java

@@ -17,8 +17,8 @@
 package com.palantir.paxos;
 
 import com.palantir.common.persist.Persistable;
+import java.util.Optional;
 import java.util.OptionalLong;
-import java.util.Set;
 import java.util.function.Function;
 import javax.sql.DataSource;
 import org.jdbi.v3.core.Jdbi;
@@ -138,7 +138,9 @@ <V extends Persistable & Versionable> boolean[] writeBatchOfRounds(
                 @Bind("useCase") String useCase,
                 @BindPojo("round") Iterable<PaxosRound<V>> rounds);
 
-        @SqlQuery("SELECT DISTINCT(namespace) FROM paxosLog")
-        Set<String> getAllNamespaces();
+        // This is performant as long as the query plan is SEARCH.
+        @SqlQuery("SELECT MIN(namespace) FROM paxosLog WHERE namespace > :maybeLastReadNamespace")
+        Optional<String> getNextLexicographicallySmallestNamespace(
+                @Bind("maybeLastReadNamespace") String maybeLastReadNamespace);
     }
 }

timelock-api/src/main/conjure/timelock-management-api.yml

@@ -10,7 +10,8 @@ services:
         returns: set<string>
         docs: |
           The endpoint loads all persisted namespaces. ``leaderPaxos`` is filtered out from the set
-          as it is not a namespace.
+          as it is not a namespace. No transactionality guarantees are given: namespace additions and
+          deletions while the request is running may or may not be reflected in the output.
 
       getActiveNamespaces:
         http: POST /getActiveNamespaces

timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/PersistentNamespaceLoader.java

@@ -22,6 +22,8 @@
 public interface PersistentNamespaceLoader {
     /**
      * Gets all namespaces that have been persisted (via the persistence method under question).
+     * No transactionality guarantees are given: namespace additions and deletions while the
+     * request is running may or may not be reflected in the output.
      */
     Set<Client> getAllPersistedNamespaces();
 }

timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/SqliteNamespaceLoader.java

@@ -18,13 +18,16 @@
 
 import com.palantir.paxos.Client;
 import com.palantir.paxos.SqlitePaxosStateLog;
+import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 import javax.sql.DataSource;
 import org.jdbi.v3.core.Jdbi;
 import org.jdbi.v3.sqlobject.SqlObjectPlugin;
 
 final class SqliteNamespaceLoader implements PersistentNamespaceLoader {
+    private static final String EMPTY_STRING = "";
+
     private final Jdbi jdbi;
 
     private SqliteNamespaceLoader(Jdbi jdbi) {
@@ -39,10 +42,23 @@ public static PersistentNamespaceLoader create(DataSource dataSource) {
 
     @Override
     public Set<Client> getAllPersistedNamespaces() {
-        return jdbi
-                .withExtension(SqlitePaxosStateLog.Queries.class, SqlitePaxosStateLog.Queries::getAllNamespaces)
-                .stream()
-                .map(Client::of)
-                .collect(Collectors.toSet());
+        Set<Client> clients = new HashSet<>();
+
+        // Namespaces contain at least one character implying the empty string is lexicographically strictly smaller
+        // than any namespace.
+        Optional<String> currentNamespace = getNextLexicographicallySmallestNamespace(EMPTY_STRING);
+        while (currentNamespace.isPresent()) {
+            String namespaceString = currentNamespace.get();
+            clients.add(Client.of(namespaceString));
+            currentNamespace = getNextLexicographicallySmallestNamespace(namespaceString);
+        }
+
+        return clients;
+    }
+
+    private Optional<String> getNextLexicographicallySmallestNamespace(String maybeLastReadNamespace) {
+        return jdbi.withExtension(
+                SqlitePaxosStateLog.Queries.class,
+                dao -> dao.getNextLexicographicallySmallestNamespace(maybeLastReadNamespace));
     }
 }