Use integer ids for Sqlite bidirectional index (buildfarm#1599)

The cost in size for a single table bidirectional index is vast compared
to the use of 3nf integer keys. Experimental estimates offer a decrease
in file size of 90%.

src/main/java/build/buildfarm/cas/cfc/SqliteFileDirectoriesIndex.java

@@ -57,13 +57,19 @@ private void open() {
         throw new RuntimeException(e);
       }
 
+      String createDirectoriesSql =
+          "CREATE TABLE directories (id INTEGER PRIMARY KEY, name VARCHAR UNIQUE)";
+      String createFilesSql = "CREATE TABLE files (id INTEGER PRIMARY KEY, name VARCHAR UNIQUE)";
       String createEntriesSql =
           "CREATE TABLE entries (\n"
-              + "    path TEXT NOT NULL,\n"
-              + "    directory TEXT NOT NULL\n"
+              + "    file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,\n"
+              + "    directory_id INTEGER NOT NULL REFERENCES directories(id) ON DELETE CASCADE,\n"
+              + "    PRIMARY KEY (file_id, directory_id)\n"
               + ")";
 
       try (Statement stmt = conn.createStatement()) {
+        stmt.execute(createDirectoriesSql);
+        stmt.execute(createFilesSql);
         stmt.execute(createEntriesSql);
       } catch (SQLException e) {
         throw new RuntimeException(e);
@@ -77,11 +83,13 @@ private void open() {
   public synchronized void start() {
     open();
 
-    String createPathIndexSql = "CREATE INDEX path_idx ON entries (path)";
-    String createDirectoryIndexSql = "CREATE INDEX directory_idx ON entries (directory)";
+    String createPathIndexSql = "CREATE INDEX file_idx ON entries (file_id)";
+    String createDirectoryIndexSql = "CREATE INDEX directory_idx ON entries (directory_id)";
+    String enforceForeignKeys = "PRAGMA foreign_keys=ON";
     try (Statement stmt = conn.createStatement()) {
       stmt.execute(createPathIndexSql);
       stmt.execute(createDirectoryIndexSql);
+      stmt.execute(enforceForeignKeys);
     } catch (SQLException e) {
       throw new RuntimeException(e);
     }
@@ -101,7 +109,8 @@ public void close() {
   private Set<Digest> removeEntryDirectories(String entry) {
     open();
 
-    String selectSql = "SELECT directory FROM entries WHERE path = ?";
+    String selectSql =
+        "SELECT d.name as directory FROM files f INNER JOIN entries e ON f.id = e.file_id INNER JOIN directories d ON d.id = e.directory_id WHERE f.name = ?";
 
     ImmutableSet.Builder<Digest> directoriesBuilder = ImmutableSet.builder();
     try (PreparedStatement selectStatement = conn.prepareStatement(selectSql)) {
@@ -116,7 +125,7 @@ private Set<Digest> removeEntryDirectories(String entry) {
     }
     // all directories featuring this entry are now invalid
     ImmutableSet<Digest> directories = directoriesBuilder.build();
-    String deleteSql = "DELETE FROM entries where directory = ?";
+    String deleteSql = "DELETE FROM directories where name = ?";
     try (PreparedStatement deleteStatement = conn.prepareStatement(deleteSql)) {
       conn.setAutoCommit(false);
       for (Digest directory : directories) {
@@ -128,6 +137,14 @@ private Set<Digest> removeEntryDirectories(String entry) {
     } catch (SQLException e) {
       throw new RuntimeException(e);
     }
+    // clear out orphaned files
+    try (Statement orphanStatement = conn.createStatement()) {
+      String deleteOrphanSql =
+          "DELETE FROM files WHERE id NOT IN (SELECT DISTINCT file_id FROM entries)";
+      orphanStatement.execute(deleteOrphanSql);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
     return directories;
   }
 
@@ -138,16 +155,41 @@ public synchronized Set<Digest> removeEntry(String entry) throws IOException {
     return directories;
   }
 
+  // inserts here specifically avoids integer key maintenance in java
   private synchronized void addEntriesDirectory(Set<String> entries, Digest directory) {
     open();
 
-    String digest = DigestUtil.toString(directory);
-    String insertSql = "INSERT INTO entries (path, directory) VALUES (?,?)";
-    try (PreparedStatement insertStatement = conn.prepareStatement(insertSql)) {
+    String directoryName = DigestUtil.toString(directory);
+    String filesInsertSql = "INSERT OR IGNORE INTO files (name) VALUES (?)";
+    try (PreparedStatement filesInsertStatement = conn.prepareStatement(filesInsertSql)) {
+      conn.setAutoCommit(false);
+      for (String entry : entries) {
+        filesInsertStatement.setString(1, entry);
+        filesInsertStatement.addBatch();
+      }
+      filesInsertStatement.executeBatch();
+      conn.commit();
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+    // should be novel directory
+    String directoriesInsertSql = "INSERT INTO directories (name) VALUES (?)";
+    try (PreparedStatement directoriesInsertStatement =
+        conn.prepareStatement(directoriesInsertSql)) {
+      conn.setAutoCommit(false);
+      directoriesInsertStatement.setString(1, directoryName);
+      directoriesInsertStatement.executeUpdate();
+      conn.commit();
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+    String entriesInsertSql =
+        "INSERT INTO entries (file_id, directory_id) SELECT f.id, d.id FROM files f, directories d WHERE f.name = ? AND d.name = ?";
+    try (PreparedStatement insertStatement = conn.prepareStatement(entriesInsertSql)) {
       conn.setAutoCommit(false);
-      insertStatement.setString(2, digest);
       for (String entry : entries) {
         insertStatement.setString(1, entry);
+        insertStatement.setString(2, directoryName);
         insertStatement.addBatch();
       }
       insertStatement.executeBatch();
@@ -168,8 +210,9 @@ private void removeEntriesDirectory(Digest directory) {
     open();
 
     String digest = DigestUtil.toString(directory);
-    String deleteSql = "DELETE FROM entries WHERE directory = ?";
+    String deleteSql = "DELETE FROM directories WHERE name = ?";
     try (PreparedStatement deleteStatement = conn.prepareStatement(deleteSql)) {
+      conn.setAutoCommit(true);
       deleteStatement.setString(1, digest);
       deleteStatement.executeUpdate();
     } catch (SQLException e) {

src/test/java/build/buildfarm/cas/cfc/DirectoriesIndexTest.java

@@ -51,6 +51,7 @@ protected DirectoriesIndexTest(Path root, DirectoriesIndexType type) {
     } else {
       throw new IllegalArgumentException("DirectoriesIndex type is not supported.");
     }
+    directoriesIndex.start();
   }
 
   @Before