diff --git a/src/main/java/build/buildfarm/cas/cfc/SqliteFileDirectoriesIndex.java b/src/main/java/build/buildfarm/cas/cfc/SqliteFileDirectoriesIndex.java index 030c749037..e634616660 100644 --- a/src/main/java/build/buildfarm/cas/cfc/SqliteFileDirectoriesIndex.java +++ b/src/main/java/build/buildfarm/cas/cfc/SqliteFileDirectoriesIndex.java @@ -57,13 +57,19 @@ private void open() { throw new RuntimeException(e); } + String createDirectoriesSql = + "CREATE TABLE directories (id INTEGER PRIMARY KEY, name VARCHAR UNIQUE)"; + String createFilesSql = "CREATE TABLE files (id INTEGER PRIMARY KEY, name VARCHAR UNIQUE)"; String createEntriesSql = "CREATE TABLE entries (\n" - + " path TEXT NOT NULL,\n" - + " directory TEXT NOT NULL\n" + + " file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,\n" + + " directory_id INTEGER NOT NULL REFERENCES directories(id) ON DELETE CASCADE,\n" + + " PRIMARY KEY (file_id, directory_id)\n" + ")"; try (Statement stmt = conn.createStatement()) { + stmt.execute(createDirectoriesSql); + stmt.execute(createFilesSql); stmt.execute(createEntriesSql); } catch (SQLException e) { throw new RuntimeException(e); @@ -77,11 +83,13 @@ private void open() { public synchronized void start() { open(); - String createPathIndexSql = "CREATE INDEX path_idx ON entries (path)"; - String createDirectoryIndexSql = "CREATE INDEX directory_idx ON entries (directory)"; + String createPathIndexSql = "CREATE INDEX file_idx ON entries (file_id)"; + String createDirectoryIndexSql = "CREATE INDEX directory_idx ON entries (directory_id)"; + String enforceForeignKeys = "PRAGMA foreign_keys=ON"; try (Statement stmt = conn.createStatement()) { stmt.execute(createPathIndexSql); stmt.execute(createDirectoryIndexSql); + stmt.execute(enforceForeignKeys); } catch (SQLException e) { throw new RuntimeException(e); } @@ -101,7 +109,8 @@ public void close() { private Set removeEntryDirectories(String entry) { open(); - String selectSql = "SELECT directory FROM entries WHERE path = ?"; + String selectSql = + "SELECT d.name as directory FROM files f INNER JOIN entries e ON f.id = e.file_id INNER JOIN directories d ON d.id = e.directory_id WHERE f.name = ?"; ImmutableSet.Builder directoriesBuilder = ImmutableSet.builder(); try (PreparedStatement selectStatement = conn.prepareStatement(selectSql)) { @@ -116,7 +125,7 @@ private Set removeEntryDirectories(String entry) { } // all directories featuring this entry are now invalid ImmutableSet directories = directoriesBuilder.build(); - String deleteSql = "DELETE FROM entries where directory = ?"; + String deleteSql = "DELETE FROM directories where name = ?"; try (PreparedStatement deleteStatement = conn.prepareStatement(deleteSql)) { conn.setAutoCommit(false); for (Digest directory : directories) { @@ -128,6 +137,14 @@ private Set removeEntryDirectories(String entry) { } catch (SQLException e) { throw new RuntimeException(e); } + // clear out orphaned files + try (Statement orphanStatement = conn.createStatement()) { + String deleteOrphanSql = + "DELETE FROM files WHERE id NOT IN (SELECT DISTINCT file_id FROM entries)"; + orphanStatement.execute(deleteOrphanSql); + } catch (SQLException e) { + throw new RuntimeException(e); + } return directories; } @@ -138,16 +155,41 @@ public synchronized Set removeEntry(String entry) throws IOException { return directories; } + // inserts here specifically avoids integer key maintenance in java private synchronized void addEntriesDirectory(Set entries, Digest directory) { open(); - String digest = DigestUtil.toString(directory); - String insertSql = "INSERT INTO entries (path, directory) VALUES (?,?)"; - try (PreparedStatement insertStatement = conn.prepareStatement(insertSql)) { + String directoryName = DigestUtil.toString(directory); + String filesInsertSql = "INSERT OR IGNORE INTO files (name) VALUES (?)"; + try (PreparedStatement filesInsertStatement = conn.prepareStatement(filesInsertSql)) { + conn.setAutoCommit(false); + for (String entry : entries) { + filesInsertStatement.setString(1, entry); + filesInsertStatement.addBatch(); + } + filesInsertStatement.executeBatch(); + conn.commit(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + // should be novel directory + String directoriesInsertSql = "INSERT INTO directories (name) VALUES (?)"; + try (PreparedStatement directoriesInsertStatement = + conn.prepareStatement(directoriesInsertSql)) { + conn.setAutoCommit(false); + directoriesInsertStatement.setString(1, directoryName); + directoriesInsertStatement.executeUpdate(); + conn.commit(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + String entriesInsertSql = + "INSERT INTO entries (file_id, directory_id) SELECT f.id, d.id FROM files f, directories d WHERE f.name = ? AND d.name = ?"; + try (PreparedStatement insertStatement = conn.prepareStatement(entriesInsertSql)) { conn.setAutoCommit(false); - insertStatement.setString(2, digest); for (String entry : entries) { insertStatement.setString(1, entry); + insertStatement.setString(2, directoryName); insertStatement.addBatch(); } insertStatement.executeBatch(); @@ -168,8 +210,9 @@ private void removeEntriesDirectory(Digest directory) { open(); String digest = DigestUtil.toString(directory); - String deleteSql = "DELETE FROM entries WHERE directory = ?"; + String deleteSql = "DELETE FROM directories WHERE name = ?"; try (PreparedStatement deleteStatement = conn.prepareStatement(deleteSql)) { + conn.setAutoCommit(true); deleteStatement.setString(1, digest); deleteStatement.executeUpdate(); } catch (SQLException e) { diff --git a/src/test/java/build/buildfarm/cas/cfc/DirectoriesIndexTest.java b/src/test/java/build/buildfarm/cas/cfc/DirectoriesIndexTest.java index 3eec66def7..b4effa2bc8 100644 --- a/src/test/java/build/buildfarm/cas/cfc/DirectoriesIndexTest.java +++ b/src/test/java/build/buildfarm/cas/cfc/DirectoriesIndexTest.java @@ -51,6 +51,7 @@ protected DirectoriesIndexTest(Path root, DirectoriesIndexType type) { } else { throw new IllegalArgumentException("DirectoriesIndex type is not supported."); } + directoriesIndex.start(); } @Before