diff --git a/changelog/@unreleased/pr-6488.v2.yml b/changelog/@unreleased/pr-6488.v2.yml new file mode 100644 index 00000000000..d950412e346 --- /dev/null +++ b/changelog/@unreleased/pr-6488.v2.yml @@ -0,0 +1,5 @@ +type: improvement +improvement: + description: Improve Sqlite namespace loader implementation + links: + - https://github.com/palantir/atlasdb/pull/6488 diff --git a/leader-election-impl/src/main/java/com/palantir/paxos/SqlitePaxosStateLog.java b/leader-election-impl/src/main/java/com/palantir/paxos/SqlitePaxosStateLog.java index f1f2c3baec8..88bd214a582 100644 --- a/leader-election-impl/src/main/java/com/palantir/paxos/SqlitePaxosStateLog.java +++ b/leader-election-impl/src/main/java/com/palantir/paxos/SqlitePaxosStateLog.java @@ -17,8 +17,8 @@ package com.palantir.paxos; import com.palantir.common.persist.Persistable; +import java.util.Optional; import java.util.OptionalLong; -import java.util.Set; import java.util.function.Function; import javax.sql.DataSource; import org.jdbi.v3.core.Jdbi; @@ -138,7 +138,9 @@ boolean[] writeBatchOfRounds( @Bind("useCase") String useCase, @BindPojo("round") Iterable> rounds); - @SqlQuery("SELECT DISTINCT(namespace) FROM paxosLog") - Set getAllNamespaces(); + // This is performant as long as the query plan is SEARCH. + @SqlQuery("SELECT MIN(namespace) FROM paxosLog WHERE namespace > :maybeLastReadNamespace") + Optional getNextLexicographicallySmallestNamespace( + @Bind("maybeLastReadNamespace") String maybeLastReadNamespace); } } diff --git a/timelock-api/src/main/conjure/timelock-management-api.yml b/timelock-api/src/main/conjure/timelock-management-api.yml index 8fd2fe20d1f..81f2ab027cd 100644 --- a/timelock-api/src/main/conjure/timelock-management-api.yml +++ b/timelock-api/src/main/conjure/timelock-management-api.yml @@ -10,7 +10,8 @@ services: returns: set docs: | The endpoint loads all persisted namespaces. ``leaderPaxos`` is filtered out from the set - as it is not a namespace. + as it is not a namespace. No transactionality guarantees are given: namespace additions and + deletions while the request is running may or may not be reflected in the output. getActiveNamespaces: http: POST /getActiveNamespaces diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/PersistentNamespaceLoader.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/PersistentNamespaceLoader.java index 3eaf6e22839..f51d42df113 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/PersistentNamespaceLoader.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/PersistentNamespaceLoader.java @@ -22,6 +22,8 @@ public interface PersistentNamespaceLoader { /** * Gets all namespaces that have been persisted (via the persistence method under question). + * No transactionality guarantees are given: namespace additions and deletions while the + * request is running may or may not be reflected in the output. */ Set getAllPersistedNamespaces(); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/SqliteNamespaceLoader.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/SqliteNamespaceLoader.java index c0efceacafe..9fe3888160a 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/SqliteNamespaceLoader.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/SqliteNamespaceLoader.java @@ -18,13 +18,16 @@ import com.palantir.paxos.Client; import com.palantir.paxos.SqlitePaxosStateLog; +import java.util.HashSet; +import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import javax.sql.DataSource; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.sqlobject.SqlObjectPlugin; final class SqliteNamespaceLoader implements PersistentNamespaceLoader { + private static final String EMPTY_STRING = ""; + private final Jdbi jdbi; private SqliteNamespaceLoader(Jdbi jdbi) { @@ -39,10 +42,23 @@ public static PersistentNamespaceLoader create(DataSource dataSource) { @Override public Set getAllPersistedNamespaces() { - return jdbi - .withExtension(SqlitePaxosStateLog.Queries.class, SqlitePaxosStateLog.Queries::getAllNamespaces) - .stream() - .map(Client::of) - .collect(Collectors.toSet()); + Set clients = new HashSet<>(); + + // Namespaces contain at least one character implying the empty string is lexicographically strictly smaller + // than any namespace. + Optional currentNamespace = getNextLexicographicallySmallestNamespace(EMPTY_STRING); + while (currentNamespace.isPresent()) { + String namespaceString = currentNamespace.get(); + clients.add(Client.of(namespaceString)); + currentNamespace = getNextLexicographicallySmallestNamespace(namespaceString); + } + + return clients; + } + + private Optional getNextLexicographicallySmallestNamespace(String maybeLastReadNamespace) { + return jdbi.withExtension( + SqlitePaxosStateLog.Queries.class, + dao -> dao.getNextLexicographicallySmallestNamespace(maybeLastReadNamespace)); } }