diff --git a/backend/daemon/storage/migrations.go b/backend/daemon/storage/migrations.go index c8da766cb..bd7dc6745 100644 --- a/backend/daemon/storage/migrations.go +++ b/backend/daemon/storage/migrations.go @@ -362,6 +362,12 @@ var migrations = []migration{ DELETE FROM kv WHERE key = 'last_reindex_time'; `)) }}, + {Version: "2024-03-01.03", Run: func(_ *Dir, conn *sqlite.Conn) error { + return sqlitex.ExecScript(conn, sqlfmt(` + CREATE INDEX blobs_metadata ON blobs (id, multihash, codec, size, insert_time); + CREATE INDEX blobs_metadata_by_hash ON blobs (multihash, codec, size, insert_time); + `)) + }}, } const ( diff --git a/backend/daemon/storage/schema.gensum b/backend/daemon/storage/schema.gensum index eb9e3d4ef..16ac5085b 100644 --- a/backend/daemon/storage/schema.gensum +++ b/backend/daemon/storage/schema.gensum @@ -1,2 +1,2 @@ -srcs: d768fdb89d3eb5d4bee55227ac36c42d +srcs: 31b832ee95aa3ac2dba0877815c96e4e outs: 6e8f9aaea92a324bcd6776afeaa0230c diff --git a/backend/daemon/storage/schema.sql b/backend/daemon/storage/schema.sql index c6a51681b..620562891 100644 --- a/backend/daemon/storage/schema.sql +++ b/backend/daemon/storage/schema.sql @@ -35,6 +35,12 @@ CREATE TABLE blobs ( insert_time INTEGER DEFAULT (strftime('%s', 'now')) NOT NULL ); +-- Index for better data locality when we need to iterate over blobs without their data. +-- Without the index loading the entire list of blobs into memory takes forever, +-- because SQLite has to read way too many pages skipping the actual blob data. +CREATE INDEX blobs_metadata ON blobs (id, multihash, codec, size, insert_time); +CREATE INDEX blobs_metadata_by_hash ON blobs (multihash, codec, size, insert_time); + -- Stores some relevant attributes for structural blobs, -- which are those blobs that we can understand more deeply than just an opaque blob. CREATE TABLE structural_blobs ( diff --git a/backend/hyper/blockstore_test.go b/backend/hyper/blockstore_test.go index 7118394fb..19fd66de0 100644 --- a/backend/hyper/blockstore_test.go +++ b/backend/hyper/blockstore_test.go @@ -16,6 +16,7 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" ) func TestGet(t *testing.T) { @@ -154,12 +155,12 @@ func TestAllKeysRespectsContext(t *testing.T) { // consume 2, then cancel context. v, ok := <-ch - require.Equal(t, keys[0], v) require.True(t, ok) + require.True(t, slices.Contains(keys, v)) v, ok = <-ch - require.Equal(t, keys[1], v) require.True(t, ok) + require.True(t, slices.Contains(keys, v)) cancel() diff --git a/backend/hyper/hypersql/queries.gen.go b/backend/hyper/hypersql/queries.gen.go index 429557520..2835a7975 100644 --- a/backend/hyper/hypersql/queries.gen.go +++ b/backend/hyper/hypersql/queries.gen.go @@ -196,7 +196,10 @@ type BlobsListKnownResult struct { func BlobsListKnown(conn *sqlite.Conn) ([]BlobsListKnownResult, error) { const query = `SELECT blobs.id, blobs.multihash, blobs.codec FROM blobs -WHERE blobs.size >= 0` +LEFT JOIN drafts ON drafts.blob = blobs.id +WHERE blobs.size >= 0 +AND drafts.blob IS NULL +ORDER BY blobs.id` var out []BlobsListKnownResult diff --git a/backend/hyper/hypersql/queries.gensum b/backend/hyper/hypersql/queries.gensum index eae7c003d..041a16c7d 100644 --- a/backend/hyper/hypersql/queries.gensum +++ b/backend/hyper/hypersql/queries.gensum @@ -1,2 +1,2 @@ -srcs: 710260753f9d777d05b98094175d347d -outs: 9427f9db7ecb4403869c9cd98100e1cc +srcs: 800af070fac742f44f90f3142c9ffcdd +outs: ed8e2acc2115513302d723aab1d251e1 diff --git a/backend/hyper/hypersql/queries.go b/backend/hyper/hypersql/queries.go index 48cb88d54..0d9f5b9d0 100644 --- a/backend/hyper/hypersql/queries.go +++ b/backend/hyper/hypersql/queries.go @@ -67,7 +67,10 @@ func generateQueries() error { s.BlobsCodec, ), '\n', "FROM", s.Blobs, '\n', - "WHERE", s.BlobsSize, ">=", "0", + "LEFT JOIN", s.Drafts, "ON", s.DraftsBlob, "=", s.BlobsID, '\n', + "WHERE", s.BlobsSize, ">=", "0", '\n', + "AND", s.DraftsBlob, "IS NULL", '\n', + "ORDER BY", s.BlobsID, ), qb.MakeQuery(s.Schema, "BlobLinksInsertOrIgnore", sgen.QueryKindExec,