From fdac551e40b773fab8b14f59bf0d886b920390ea Mon Sep 17 00:00:00 2001 From: Julio Garcia <11684004+juligasa@users.noreply.github.com> Date: Fri, 29 Mar 2024 22:46:38 +0100 Subject: [PATCH] List performance (#1679) * wip(daemon): recursive get publications * wip(daemon): add trusted only * wip(daemon): remove resource duplicates * wip(daemon): exclude drafts from listPublications * wip(daemon): remove children from tests * fix(daemon): Don't report drafts on listpublications * wip(daemon): list drafts * wip(daemon): include mentions * fix(daemon): account for untitled documents * fix(daemon): update migration to match schema * wip(daemon): page token on list drafts * wip(daemon): page token on list documents * implementing pagination in the frontend WIP * revert pagination in accounts model --------- Co-authored-by: Horacio Herrera --- .../daemon/api/activity/v1alpha/activity.go | 2 +- .../daemon/api/documents/v1alpha/documents.go | 413 ++++++++++++++++-- .../api/documents/v1alpha/documents_test.go | 6 +- .../daemon/api/entities/v1alpha/entities.go | 13 +- backend/daemon/daemon_e2e_test.go | 12 +- backend/daemon/storage/migrations.go | 39 ++ backend/daemon/storage/schema.gen.go | 19 + backend/daemon/storage/schema.gensum | 4 +- backend/daemon/storage/schema.sql | 37 ++ frontend/packages/app/models/contacts.ts | 14 +- frontend/packages/app/models/documents.ts | 65 ++- frontend/packages/app/models/groups.ts | 20 +- frontend/packages/app/pages/contacts-page.tsx | 6 + frontend/packages/app/pages/feed.tsx | 19 +- frontend/packages/app/pages/groups.tsx | 1 + .../app/pages/publication-list-page.tsx | 21 +- frontend/packages/shared/package.json | 3 +- .../shared/src/publication-content.tsx | 4 +- frontend/packages/ui/src/list.tsx | 73 +--- yarn.lock | 1 + 20 files changed, 612 insertions(+), 160 deletions(-) diff --git a/backend/daemon/api/activity/v1alpha/activity.go b/backend/daemon/api/activity/v1alpha/activity.go index d98fec7b1..05b55b0cb 100644 --- a/backend/daemon/api/activity/v1alpha/activity.go +++ b/backend/daemon/api/activity/v1alpha/activity.go @@ -133,7 +133,7 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque joinIDStr = "JOIN " + storage.Blobs.String() + " ON " + storage.BlobsID.String() + "=" + storage.StructuralBlobsID.String() joinpkStr = "JOIN " + storage.PublicKeys.String() + " ON " + storage.StructuralBlobsAuthor.String() + "=" + storage.PublicKeysID.String() leftjoinStr = "LEFT JOIN " + storage.Resources.String() + " ON " + storage.StructuralBlobsResource.String() + "=" + storage.ResourcesID.String() - pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_token" + pageTokenStr = storage.BlobsID.String() + " <= :idx AND (" + storage.ResourcesIRI.String() + " NOT IN (SELECT " + storage.DraftsViewResource.String() + " from " + storage.DraftsView.String() + ") OR " + storage.ResourcesIRI.String() + " IS NULL) ORDER BY " + storage.BlobsID.String() + " desc limit :page_size" ) var getEventsStr = fmt.Sprintf(` diff --git a/backend/daemon/api/documents/v1alpha/documents.go b/backend/daemon/api/documents/v1alpha/documents.go index eecaece61..56451473f 100644 --- a/backend/daemon/api/documents/v1alpha/documents.go +++ b/backend/daemon/api/documents/v1alpha/documents.go @@ -4,18 +4,25 @@ package documents import ( "bytes" "context" + "encoding/base64" + "encoding/hex" "fmt" + "math" "mintter/backend/core" "mintter/backend/daemon/api/documents/v1alpha/docmodel" groups "mintter/backend/daemon/api/groups/v1alpha" documents "mintter/backend/genproto/documents/v1alpha" groups_proto "mintter/backend/genproto/groups/v1alpha" "mintter/backend/mttnet" + "strconv" + "strings" + "time" "mintter/backend/hyper" "mintter/backend/hyper/hypersql" "mintter/backend/logging" "mintter/backend/pkg/colx" + "mintter/backend/pkg/dqb" "mintter/backend/pkg/future" "crawshaw.io/sqlite" @@ -25,6 +32,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" ) // Discoverer is a subset of the syncing service that @@ -251,28 +259,161 @@ func (api *Server) GetDraft(ctx context.Context, in *documents.GetDraftRequest) return mut.Hydrate(ctx, api.blobs) } +var qListAllDrafts = dqb.Str(` + WITH RECURSIVE resource_authors AS ( + SELECT + r.iri, + r.create_time, + r.owner, + mv.meta, + pk.principal AS author_raw, + sb.ts, + sb.id AS blob_id + FROM + resources r + JOIN structural_blobs sb ON r.id = sb.resource + JOIN public_keys pk ON sb.author = pk.id + JOIN meta_view mv ON r.iri = mv.iri + WHERE + sb.author IS NOT NULL + AND r.iri GLOB :pattern + AND sb.id in (SELECT distinct blob from drafts) + UNION ALL + SELECT + ra.iri, + ra.create_time, + ra.owner, + sb.meta, + pk.principal, + sb.ts, + sb.id + FROM + resource_authors ra + JOIN structural_blobs sb ON ra.iri = sb.resource + JOIN public_keys pk ON sb.author = pk.id + WHERE + sb.author IS NOT NULL + AND ra.iri GLOB :pattern + ), + owners_raw AS ( + SELECT + id, + principal AS owner_raw + FROM + public_keys + ), + latest_blobs AS ( + SELECT + ra.iri, + MAX(ra.ts) AS latest_ts, + b.multihash, + b.codec + FROM + resource_authors ra + JOIN blobs b ON ra.blob_id = b.id + GROUP BY ra.iri + ) + SELECT + ra.iri, + ra.create_time, + GROUP_CONCAT(DISTINCT HEX(ra.author_raw)) AS authors_hex, + ra.meta, + MAX(ra.ts) AS latest_ts, + HEX(oraw.owner_raw), + ra.blob_id + FROM + resource_authors ra + LEFT JOIN owners_raw oraw ON ra.owner = oraw.id + LEFT JOIN latest_blobs lb ON ra.iri = lb.iri + WHERE ra.blob_id <= :idx + GROUP BY + ra.iri, ra.create_time, ra.meta + ORDER BY ra.blob_id asc LIMIT :page_size; +`) + // ListDrafts implements the corresponding gRPC method. -func (api *Server) ListDrafts(ctx context.Context, _ *documents.ListDraftsRequest) (*documents.ListDraftsResponse, error) { - entities, err := api.blobs.ListEntities(ctx, "hm://d/*") +func (api *Server) ListDrafts(ctx context.Context, req *documents.ListDraftsRequest) (*documents.ListDraftsResponse, error) { + var ( + entities []hyper.EntityID + err error + ) + me, ok := api.me.Get() + if !ok { + return nil, fmt.Errorf("account is not initialized yet") + } + conn, cancel, err := api.db.Conn(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("Can't get a connection from the db: %w", err) } - + defer cancel() resp := &documents.ListDraftsResponse{ Documents: make([]*documents.Document, 0, len(entities)), } - - for _, e := range entities { - docid := string(e) - draft, err := api.GetDraft(ctx, &documents.GetDraftRequest{ - DocumentId: docid, - }) + var cursorBlobID int64 = math.MaxInt32 + if req.PageSize == 0 { + req.PageSize = 30 + } + if req.PageToken != "" { + pageTokenBytes, _ := base64.StdEncoding.DecodeString(req.PageToken) if err != nil { - continue + return nil, fmt.Errorf("Token encoding not valid: %w", err) + } + clearPageToken, err := me.DeviceKey().Decrypt(pageTokenBytes) + if err != nil { + return nil, fmt.Errorf("Token not valid: %w", err) + } + pageToken, err := strconv.ParseUint(string(clearPageToken), 10, 32) + if err != nil { + return nil, fmt.Errorf("Token not valid: %w", err) + } + cursorBlobID = int64(pageToken) + } + pattern := "hm://d/*" + var lastBlobID int64 + err = sqlitex.Exec(conn, qListAllDrafts(), func(stmt *sqlite.Stmt) error { + var ( + id = stmt.ColumnText(0) + createTime = stmt.ColumnInt64(1) + editorsStr = stmt.ColumnText(2) + title = stmt.ColumnText(3) + updatedTime = stmt.ColumnInt64(4) + ownerHex = stmt.ColumnText(5) + ) + lastBlobID = stmt.ColumnInt64(6) + editors := []string{} + for _, editorHex := range strings.Split(editorsStr, ",") { + editorBin, err := hex.DecodeString(editorHex) + if err != nil { + return err + } + editors = append(editors, core.Principal(editorBin).String()) } - resp.Documents = append(resp.Documents, draft) + ownerBin, err := hex.DecodeString(ownerHex) + if err != nil { + return err + } + pub := &documents.Document{ + Id: id, + Title: title, + Author: core.Principal(ownerBin).String(), + Editors: editors, + CreateTime: timestamppb.New(time.Unix(int64(createTime), 0)), + UpdateTime: timestamppb.New(time.Unix(int64(updatedTime/1000000), (updatedTime%1000000)*1000)), + } + resp.Documents = append(resp.Documents, pub) + return nil + }, pattern, cursorBlobID, req.PageSize) + if err != nil { + return nil, err } + pageToken, err := me.DeviceKey().Encrypt([]byte(strconv.Itoa(int(lastBlobID - 1)))) + if err != nil { + return nil, err + } + if lastBlobID != 0 && req.PageSize == int32(len(resp.Documents)) { + resp.NextPageToken = base64.StdEncoding.EncodeToString(pageToken) + } return resp, nil } @@ -528,42 +669,250 @@ func (api *Server) PushPublication(ctx context.Context, in *documents.PushPublic return &emptypb.Empty{}, nil } +var qListAllPublications = dqb.Str(` + WITH RECURSIVE resource_authors AS ( + SELECT + r.iri, + r.create_time, + r.owner, + mv.meta, + pk.principal AS author_raw, + sb.ts, + sb.id AS blob_id + FROM + resources r + JOIN structural_blobs sb ON r.id = sb.resource + JOIN public_keys pk ON sb.author = pk.id + JOIN meta_view mv ON r.iri = mv.iri + WHERE + sb.author IS NOT NULL + AND r.iri GLOB :pattern + AND sb.id not in (SELECT distinct blob from drafts) + UNION ALL + SELECT + ra.iri, + ra.create_time, + ra.owner, + sb.meta, + pk.principal, + sb.ts, + sb.id + FROM + resource_authors ra + JOIN structural_blobs sb ON ra.iri = sb.resource + JOIN public_keys pk ON sb.author = pk.id + WHERE + sb.author IS NOT NULL + AND ra.iri GLOB :pattern + ), + owners_raw AS ( + SELECT + id, + principal AS owner_raw + FROM + public_keys + ), + latest_blobs AS ( + SELECT + ra.iri, + MAX(ra.ts) AS latest_ts, + b.multihash, + b.codec + FROM + resource_authors ra + JOIN blobs b ON ra.blob_id = b.id + GROUP BY ra.iri + ) + SELECT + ra.iri, + ra.create_time, + GROUP_CONCAT(DISTINCT HEX(ra.author_raw)) AS authors_hex, + ra.meta, + MAX(ra.ts) AS latest_ts, + HEX(oraw.owner_raw), + lb.multihash AS latest_multihash, + lb.codec AS latest_codec, + ra.blob_id + FROM + resource_authors ra + LEFT JOIN owners_raw oraw ON ra.owner = oraw.id + LEFT JOIN latest_blobs lb ON ra.iri = lb.iri + WHERE ra.blob_id <= :idx + GROUP BY + ra.iri, ra.create_time, ra.meta + ORDER BY ra.blob_id asc LIMIT :page_size; +`) + +var qListTrustedPublications = dqb.Str(` + WITH RECURSIVE resource_authors AS ( + SELECT + r.iri, + r.create_time, + r.owner, + mv.meta, + pk.principal AS author_raw, + sb.ts, + sb.id AS blob_id + FROM + resources r + JOIN structural_blobs sb ON r.id = sb.resource + JOIN public_keys pk ON sb.author = pk.id + JOIN meta_view mv ON r.iri = mv.iri + JOIN trusted_accounts ON trusted_accounts.id = r.owner + WHERE + sb.author IS NOT NULL + AND r.iri GLOB :pattern + AND r.id not in (SELECT resource from drafts) + UNION ALL + SELECT + ra.iri, + ra.create_time, + ra.owner, + sb.meta, + pk.principal, + sb.ts, + sb.id + FROM + resource_authors ra + JOIN structural_blobs sb ON ra.iri = sb.resource + JOIN public_keys pk ON sb.author = pk.id + WHERE + sb.author IS NOT NULL + AND ra.iri GLOB :pattern + ), + owners_raw AS ( + SELECT + id, + principal AS owner_raw + FROM + public_keys + ), + latest_blobs AS ( + SELECT + ra.iri, + MAX(ra.ts) AS latest_ts, + b.multihash, + b.codec + FROM + resource_authors ra + JOIN blobs b ON ra.blob_id = b.id + GROUP BY ra.iri + ) + SELECT + ra.iri, + ra.create_time, + GROUP_CONCAT(DISTINCT HEX(ra.author_raw)) AS authors_hex, + ra.meta, + MAX(ra.ts) AS latest_ts, + HEX(oraw.owner_raw), + lb.multihash AS latest_multihash, + lb.codec AS latest_codec, + ra.blob_id + FROM + resource_authors ra + LEFT JOIN owners_raw oraw ON ra.owner = oraw.id + LEFT JOIN latest_blobs lb ON ra.iri = lb.iri + WHERE ra.blob_id <= :idx + GROUP BY + ra.iri, ra.create_time, ra.meta + ORDER BY ra.blob_id asc LIMIT :page_size; +`) + // ListPublications implements the corresponding gRPC method. func (api *Server) ListPublications(ctx context.Context, in *documents.ListPublicationsRequest) (*documents.ListPublicationsResponse, error) { var ( entities []hyper.EntityID err error ) - if in.TrustedOnly { - entities, err = api.blobs.ListTrustedEntities(ctx, "hm://d/*") + me, ok := api.me.Get() + if !ok { + return nil, fmt.Errorf("account is not initialized yet") + } + conn, cancel, err := api.db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("Can't get a connection from the db: %w", err) + } + defer cancel() + resp := &documents.ListPublicationsResponse{ + Publications: make([]*documents.Publication, 0, len(entities)), + } + var cursorBlobID int64 = math.MaxInt32 + if in.PageSize == 0 { + in.PageSize = 30 + } + if in.PageToken != "" { + pageTokenBytes, _ := base64.StdEncoding.DecodeString(in.PageToken) if err != nil { - return nil, err + return nil, fmt.Errorf("Token encoding not valid: %w", err) } - } else { - entities, err = api.blobs.ListEntities(ctx, "hm://d/*") + clearPageToken, err := me.DeviceKey().Decrypt(pageTokenBytes) if err != nil { - return nil, err + return nil, fmt.Errorf("Token not valid: %w", err) + } + pageToken, err := strconv.ParseUint(string(clearPageToken), 10, 32) + if err != nil { + return nil, fmt.Errorf("Token not valid: %w", err) } + cursorBlobID = int64(pageToken) } - - resp := &documents.ListPublicationsResponse{ - Publications: make([]*documents.Publication, 0, len(entities)), + pattern := "hm://d/*" + query := qListAllPublications + if in.TrustedOnly { + query = qListTrustedPublications } - - // TODO(burdiyan): this is very inefficient. Index the attributes necessary for listing, - // and use the database without loading the changes from disk all the time one by one. - for _, e := range entities { - docid := string(e) - pub, err := api.GetPublication(ctx, &documents.GetPublicationRequest{ - DocumentId: docid, - LocalOnly: true, - }) + var lastBlobID int64 + err = sqlitex.Exec(conn, query(), func(stmt *sqlite.Stmt) error { + var ( + id = stmt.ColumnText(0) + createTime = stmt.ColumnInt64(1) + editorsStr = stmt.ColumnText(2) + title = stmt.ColumnText(3) + updatedTime = stmt.ColumnInt64(4) + ownerHex = stmt.ColumnText(5) + mhash = stmt.ColumnBytes(6) + codec = stmt.ColumnInt64(7) + ) + lastBlobID = stmt.ColumnInt64(7) + editors := []string{} + for _, editorHex := range strings.Split(editorsStr, ",") { + editorBin, err := hex.DecodeString(editorHex) + if err != nil { + return err + } + editors = append(editors, core.Principal(editorBin).String()) + } + ownerBin, err := hex.DecodeString(ownerHex) if err != nil { - continue + return err + } + version := cid.NewCidV1(uint64(codec), mhash) + pub := &documents.Publication{ + Version: version.String(), + Document: &documents.Document{ + Id: id, + Title: title, + Author: core.Principal(ownerBin).String(), + Editors: editors, + Children: []*documents.BlockNode{}, + + CreateTime: timestamppb.New(time.Unix(int64(createTime), 0)), + UpdateTime: timestamppb.New(time.Unix(int64(updatedTime/1000000), (updatedTime%1000000)*1000)), + PublishTime: timestamppb.New(time.Unix(int64(updatedTime/1000000), (updatedTime%1000000)*1000)), + }, } resp.Publications = append(resp.Publications, pub) + return nil + }, pattern, cursorBlobID, in.PageSize) + if err != nil { + return nil, err + } + pageToken, err := me.DeviceKey().Encrypt([]byte(strconv.Itoa(int(lastBlobID - 1)))) + if err != nil { + return nil, err + } + if lastBlobID != 0 && in.PageSize == int32(len(resp.Publications)) { + resp.NextPageToken = base64.StdEncoding.EncodeToString(pageToken) } - return resp, nil } diff --git a/backend/daemon/api/documents/v1alpha/documents_test.go b/backend/daemon/api/documents/v1alpha/documents_test.go index eac8e20b4..ab89b432c 100644 --- a/backend/daemon/api/documents/v1alpha/documents_test.go +++ b/backend/daemon/api/documents/v1alpha/documents_test.go @@ -466,6 +466,7 @@ func TestListDrafts(t *testing.T) { Text: "Hello world!", }}}, }) + updated.Children = nil require.Equal(t, draft.CreateTime.AsTime().UnixMicro(), updated.CreateTime.AsTime().UnixMicro()) require.Greater(t, updated.UpdateTime.AsTime().UnixMicro(), draft.UpdateTime.AsTime().UnixMicro()) @@ -501,8 +502,9 @@ func TestAPIPublishDraft_E2E(t *testing.T) { published, err := api.PublishDraft(ctx, &documents.PublishDraftRequest{DocumentId: draft.Id}) require.NoError(t, err) + published.Document.Children = []*documents.BlockNode{} updated.PublishTime = published.Document.PublishTime // Drafts don't have publish time. - + updated.Children = published.Document.Children diff := cmp.Diff(updated, published.Document, testutil.ExportedFieldsFilter()) if diff != "" { t.Fatal(diff, "published document doesn't match") @@ -531,6 +533,7 @@ func TestAPIPublishDraft_E2E(t *testing.T) { // Must get publication after publishing. got, err := api.GetPublication(ctx, &documents.GetPublicationRequest{DocumentId: draft.Id}) + got.Document.Children = []*documents.BlockNode{} require.NoError(t, err, "must get document after publishing") testutil.ProtoEqual(t, published, got, "published document doesn't match") @@ -698,6 +701,7 @@ func TestCreateDraftFromPublication(t *testing.T) { }) pub2, err := api.PublishDraft(ctx, &documents.PublishDraftRequest{DocumentId: draft2.Id}) + pub2.Document.Children = []*documents.BlockNode{} require.NoError(t, err) require.NotNil(t, pub2) diff --git a/backend/daemon/api/entities/v1alpha/entities.go b/backend/daemon/api/entities/v1alpha/entities.go index eda39d649..eda0e66f7 100644 --- a/backend/daemon/api/entities/v1alpha/entities.go +++ b/backend/daemon/api/entities/v1alpha/entities.go @@ -401,17 +401,8 @@ func (api *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti } var qGetEntityTitles = dqb.Str(` - SELECT sb.meta, resources.iri, public_keys.principal - FROM structural_blobs sb - JOIN public_keys ON public_keys.id = sb.author - JOIN resources ON resources.id = sb.resource - JOIN ( - SELECT resource, MAX(ts) AS max_ts - FROM structural_blobs - WHERE type='Change' AND meta IS NOT NULL - GROUP BY resource - ) AS latest_blobs ON sb.resource = latest_blobs.resource AND sb.ts = latest_blobs.max_ts; -`) + SELECT meta, iri, principal + FROM meta_view;`) // ListEntityMentions implements listing mentions of an entity in other resources. func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEntityMentionsRequest) (*entities.ListEntityMentionsResponse, error) { diff --git a/backend/daemon/daemon_e2e_test.go b/backend/daemon/daemon_e2e_test.go index f2898a303..e593de91e 100644 --- a/backend/daemon/daemon_e2e_test.go +++ b/backend/daemon/daemon_e2e_test.go @@ -12,6 +12,7 @@ import ( "mintter/backend/mttnet" "mintter/backend/pkg/must" "mintter/backend/testutil" + "strconv" "testing" "time" @@ -251,13 +252,12 @@ func TestBug_PublicationsListInconsistent(t *testing.T) { return pub } - want := []*documents.Publication{ - publish(ctx, t, "Doc-1", "This is a doc-1"), - publish(ctx, t, "Doc-2", "This is a doc-2"), - publish(ctx, t, "Doc-3", "This is a doc-3"), - publish(ctx, t, "Doc-4", "This is a doc-4"), + want := []*documents.Publication{} + for i := 1; i <= 4; i++ { + doc := publish(ctx, t, "Doc-"+strconv.Itoa(i), "This is a doc-"+strconv.Itoa(i)) + doc.Document.Children = []*documents.BlockNode{} + want = append(want, doc) } - var g errgroup.Group // Trying this more than once and expecting it to return the same result. This is what bug was mostly about. diff --git a/backend/daemon/storage/migrations.go b/backend/daemon/storage/migrations.go index 94417e355..2185d2f74 100644 --- a/backend/daemon/storage/migrations.go +++ b/backend/daemon/storage/migrations.go @@ -414,6 +414,45 @@ var migrations = []migration{ JOIN blobs INDEXED BY blobs_metadata ON blobs.id = drafts.blob; `)) }}, + {Version: "2024-03-25.01", Run: func(_ *Dir, conn *sqlite.Conn) error { + return sqlitex.ExecScript(conn, sqlfmt(` + DROP VIEW IF EXISTS meta_view; + CREATE VIEW if not exists meta_view AS + WITH RankedBlobs AS ( + SELECT + sb.id, + sb.meta, + sb.author, + sb.resource, + sb.ts, + ROW_NUMBER() OVER ( + PARTITION BY sb.resource + ORDER BY + (CASE WHEN sb.meta IS NOT NULL THEN 0 ELSE 1 END), + sb.ts DESC + ) AS rank + FROM structural_blobs sb + WHERE sb.type = 'Change' + ), + LatestBlobs AS ( + SELECT + rb.id, + rb.meta, + rb.author, + rb.resource, + rb.ts + FROM RankedBlobs rb + WHERE rb.rank = 1 + ) + SELECT + lb.meta, + res.iri, + pk.principal + FROM LatestBlobs lb + JOIN resources res ON res.id = lb.resource + JOIN public_keys pk ON pk.id = lb.author; + `)) + }}, } const ( diff --git a/backend/daemon/storage/schema.gen.go b/backend/daemon/storage/schema.gen.go index 34e9ce11c..e441d70bd 100644 --- a/backend/daemon/storage/schema.gen.go +++ b/backend/daemon/storage/schema.gen.go @@ -168,6 +168,22 @@ const ( C_KVValue = "kv.value" ) +// Table meta_view. +const ( + MetaView sqlitegen.Table = "meta_view" + MetaViewIRI sqlitegen.Column = "meta_view.iri" + MetaViewMeta sqlitegen.Column = "meta_view.meta" + MetaViewPrincipal sqlitegen.Column = "meta_view.principal" +) + +// Table meta_view. Plain strings. +const ( + T_MetaView = "meta_view" + C_MetaViewIRI = "meta_view.iri" + C_MetaViewMeta = "meta_view.meta" + C_MetaViewPrincipal = "meta_view.principal" +) + // Table public_keys. const ( PublicKeys sqlitegen.Table = "public_keys" @@ -377,6 +393,9 @@ var Schema = sqlitegen.Schema{ KeyDelegationsViewIssuer: {Table: KeyDelegationsView, SQLType: "BLOB"}, KVKey: {Table: KV, SQLType: "TEXT"}, KVValue: {Table: KV, SQLType: "TEXT"}, + MetaViewIRI: {Table: MetaView, SQLType: "TEXT"}, + MetaViewMeta: {Table: MetaView, SQLType: "TEXT"}, + MetaViewPrincipal: {Table: MetaView, SQLType: "BLOB"}, PublicKeysID: {Table: PublicKeys, SQLType: "INTEGER"}, PublicKeysPrincipal: {Table: PublicKeys, SQLType: "BLOB"}, ResourceLinksID: {Table: ResourceLinks, SQLType: "INTEGER"}, diff --git a/backend/daemon/storage/schema.gensum b/backend/daemon/storage/schema.gensum index 6f124c386..1b01b9ecf 100644 --- a/backend/daemon/storage/schema.gensum +++ b/backend/daemon/storage/schema.gensum @@ -1,2 +1,2 @@ -srcs: c0bb41a9dd9db2130a02aa83b977269f -outs: 0efc08608cead13c5b8756e4beee2ecd +srcs: 8f7984b962c288b761a8346ebfcab040 +outs: 67a994f0ff45ef5e3e3e60482412dc96 diff --git a/backend/daemon/storage/schema.sql b/backend/daemon/storage/schema.sql index 4e2f4741d..7ff9ef08e 100644 --- a/backend/daemon/storage/schema.sql +++ b/backend/daemon/storage/schema.sql @@ -81,6 +81,43 @@ FROM structural_blobs JOIN blobs ON blobs.id = structural_blobs.id JOIN resources ON structural_blobs.resource = resources.id; +-- View blobs metadata It returns the latest non null title or the +-- latest blob in case of untitled meta. +CREATE VIEW meta_view AS +WITH RankedBlobs AS ( + SELECT + sb.id, + sb.meta, + sb.author, + sb.resource, + sb.ts, + ROW_NUMBER() OVER ( + PARTITION BY sb.resource + ORDER BY + (CASE WHEN sb.meta IS NOT NULL THEN 0 ELSE 1 END), + sb.ts DESC + ) AS rank + FROM structural_blobs sb + WHERE sb.type = 'Change' +), +LatestBlobs AS ( + SELECT + rb.id, + rb.meta, + rb.author, + rb.resource, + rb.ts + FROM RankedBlobs rb + WHERE rb.rank = 1 +) +SELECT + lb.meta, + res.iri, + pk.principal +FROM LatestBlobs lb +JOIN resources res ON res.id = lb.resource +JOIN public_keys pk ON pk.id = lb.author; + -- Stores extra information for key delegation blobs. CREATE TABLE key_delegations ( id INTEGER PRIMARY KEY REFERENCES blobs (id) ON UPDATE CASCADE NOT NULL, diff --git a/frontend/packages/app/models/contacts.ts b/frontend/packages/app/models/contacts.ts index b123a8f4e..8e08e3ab8 100644 --- a/frontend/packages/app/models/contacts.ts +++ b/frontend/packages/app/models/contacts.ts @@ -1,6 +1,6 @@ import {queryKeys} from '@mintter/app/models/query-keys' import {Device} from '@mintter/shared' -import {UseMutationOptions, useMutation, useQuery} from '@tanstack/react-query' +import {UseMutationOptions, useMutation} from '@tanstack/react-query' import {decompressFromEncodedURIComponent} from 'lz-string' import {useGRPCClient, useQueryInvalidator} from '../app-context' import appError from '../errors' @@ -8,18 +8,6 @@ import {useAccount} from './accounts' import {useConnectedPeers} from './networking' import {fullInvalidate} from './query-keys' -export function useContactsList() { - const grpcClient = useGRPCClient() - const contacts = useQuery({ - queryKey: [queryKeys.GET_ALL_ACCOUNTS], - queryFn: async () => { - return await grpcClient.accounts.listAccounts({}) - }, - refetchInterval: 20_000, - }) - return contacts -} - export function useConnectionSummary() { const peerInfo = useConnectedPeers({ refetchInterval: 15_000, diff --git a/frontend/packages/app/models/documents.ts b/frontend/packages/app/models/documents.ts index fcbc49685..13da6df22 100644 --- a/frontend/packages/app/models/documents.ts +++ b/frontend/packages/app/models/documents.ts @@ -36,8 +36,10 @@ import { import {UpdateDraftResponse} from '@mintter/shared/src/client/.generated/documents/v1alpha/documents_pb' import { FetchQueryOptions, + UseInfiniteQueryOptions, UseMutationOptions, UseQueryOptions, + useInfiniteQuery, useMutation, useQueries, useQuery, @@ -61,38 +63,58 @@ import {useGroupContent, useGroups} from './groups' import {queryKeys} from './query-keys' export function usePublicationList( - opts?: UseQueryOptions & {trustedOnly: boolean}, + opts?: UseInfiniteQueryOptions & { + trustedOnly: boolean + }, ) { const {trustedOnly, ...queryOpts} = opts || {} const grpcClient = useGRPCClient() - return useQuery({ + const pubListQuery = useInfiniteQuery({ ...queryOpts, queryKey: [ queryKeys.GET_PUBLICATION_LIST, trustedOnly ? 'trusted' : 'global', ], refetchOnMount: true, - queryFn: async () => { + queryFn: async (context) => { const result = await grpcClient.publications.listPublications({ trustedOnly: trustedOnly, + pageSize: 50, + pageToken: context.pageParam, }) let publications = result.publications.sort((a, b) => sortDocuments(a.document?.updateTime, b.document?.updateTime), ) || [] - publications = publications.filter((pub) => { - return pub.document?.title !== '(HIDDEN) Group Navigation' - }) + // publications = publications.filter((pub) => { + // return pub.document?.title !== '(HIDDEN) Group Navigation' + // }) return { ...result, publications, } }, + getNextPageParam: (lastPage) => { + return lastPage.nextPageToken + }, }) + + const allPublications = + pubListQuery.data?.pages.flatMap((page) => page.publications) || [] + console.log(`== ~ allPublications:`, allPublications) + return { + ...pubListQuery, + data: { + ...pubListQuery.data, + publications: allPublications, + }, + } } export function usePublicationFullList( - opts?: UseQueryOptions & {trustedOnly: boolean}, + opts?: UseInfiniteQueryOptions & { + trustedOnly: boolean + }, ) { const pubList = usePublicationList(opts) const accounts = useAllAccounts() @@ -114,14 +136,15 @@ export function usePublicationFullList( export function useDraftList() { const grpcClient = useGRPCClient() - return useQuery({ + const draftListQuery = useInfiniteQuery({ queryKey: [queryKeys.GET_DRAFT_LIST], refetchOnMount: true, - queryFn: async () => { + queryFn: async (context) => { const result = await grpcClient.drafts.listDrafts({ - pageSize: undefined, - pageToken: undefined, + pageToken: context.pageParam, }) + + console.log(`== ~ queryFn: ~ result:`, result) const documents = result.documents.sort((a, b) => sortDocuments(a.updateTime, b.updateTime), @@ -131,7 +154,27 @@ export function useDraftList() { documents, } }, + getNextPageParam: (lastPage) => { + return lastPage.nextPageToken || undefined + }, }) + + const allDrafts = + draftListQuery.data?.pages.flatMap((page) => page.documents) || [] + + console.log( + `== ~ useDraftList ~ draftListQuery:`, + draftListQuery.data, + allDrafts, + ) + + return { + ...draftListQuery, + data: { + ...draftListQuery.data, + documents: allDrafts, + }, + } } export function useDeleteDraft( diff --git a/frontend/packages/app/models/groups.ts b/frontend/packages/app/models/groups.ts index 3d126cf9c..f2aa3d404 100644 --- a/frontend/packages/app/models/groups.ts +++ b/frontend/packages/app/models/groups.ts @@ -13,8 +13,10 @@ import { } from '@mintter/shared' import {ListDocumentGroupsResponse_Item} from '@mintter/shared/src/client/.generated/groups/v1alpha/groups_pb' import { + UseInfiniteQueryOptions, UseMutationOptions, UseQueryOptions, + useInfiniteQuery, useMutation, useQueries, useQuery, @@ -35,23 +37,31 @@ import { } from '@mintter/shared' import {queryKeys} from './query-keys' -export function useAllGroups(opts?: UseQueryOptions) { +export function useAllGroups( + opts?: UseInfiniteQueryOptions, +) { const grpcClient = useGRPCClient() - const groupsQuery = useQuery({ + const groupsQuery = useInfiniteQuery({ ...opts, queryKey: [queryKeys.GET_GROUPS], - queryFn: async () => { - return await grpcClient.groups.listGroups({}) + queryFn: async (context) => { + return await grpcClient.groups.listGroups({ + pageSize: 50, + pageToken: context.pageParam, + }) }, + getNextPageParam: (lastPage) => lastPage?.nextPageToken ?? undefined, }) + const allGroups = groupsQuery.data?.pages.flatMap((page) => page.groups) || [] + return useMemo(() => { return { ...groupsQuery, data: { ...groupsQuery.data, groups: - groupsQuery.data?.groups?.sort((a, b) => + allGroups?.sort((a, b) => sortDocuments(a.updateTime, b.updateTime), ) || [], }, diff --git a/frontend/packages/app/pages/contacts-page.tsx b/frontend/packages/app/pages/contacts-page.tsx index 559e4275d..873ae80ae 100644 --- a/frontend/packages/app/pages/contacts-page.tsx +++ b/frontend/packages/app/pages/contacts-page.tsx @@ -105,6 +105,8 @@ function ErrorPage({}: {error: any}) { export default function ContactsPage() { const contacts = useAllAccounts(true) + + console.log(`== ~ ContactsPage ~ contacts:`, contacts) const myAccount = useMyAccount() const allAccounts = contacts.data?.accounts || [] const trustedAccounts = allAccounts.filter( @@ -147,6 +149,7 @@ export default function ContactsPage() { { return ( ) }} + onEndReached={() => { + contacts.fetchNextPage() + }} /> {copyDialogContent} diff --git a/frontend/packages/app/pages/feed.tsx b/frontend/packages/app/pages/feed.tsx index 2f8db16dd..8a61717c1 100644 --- a/frontend/packages/app/pages/feed.tsx +++ b/frontend/packages/app/pages/feed.tsx @@ -18,13 +18,13 @@ import { import { Button, ButtonText, - FeedList, - FeedListHandle, Globe, + List, PageContainer, RadioButtons, SizableText, Spinner, + TextProps, Theme, UIAvatar, View, @@ -33,7 +33,7 @@ import { toast, } from '@mintter/ui' import {ArrowRight, ChevronUp, Verified} from '@tamagui/lucide-icons' -import React, {PropsWithChildren, ReactNode, useRef} from 'react' +import React, {PropsWithChildren, ReactNode} from 'react' import Footer from '../components/footer' import {MainWrapperNoScroll} from '../components/main-wrapper' import {useAccount} from '../models/accounts' @@ -162,13 +162,15 @@ type CommentFeedItemProps = { function EntityLink({ id, children, -}: { + ...props +}: TextProps & { id: UnpackedHypermediaId children: ReactNode }) { const navigate = useNavigate('push') return ( { e.stopPropagation() @@ -181,6 +183,7 @@ function EntityLink({ }} numberOfLines={1} textOverflow="ellipsis" // not working. long titles don't look great + {...props} > {children} @@ -762,12 +765,10 @@ const Feed = React.memo(function Feed({tab}: {tab: 'trusted' | 'all'}) { const feed = useFeedWithLatest(tab === 'trusted') const route = useNavRoute() const replace = useNavigate('replace') - const scrollRef = useRef(null) if (route.key !== 'feed') throw new Error('invalid route') return ( - @@ -837,11 +838,9 @@ export const ResourceFeed = React.memo(function ResourceFeed({ id: string }) { const feed = useResourceFeed(id) - const scrollRef = useRef(null) return ( - } footer={ feed.data?.pages?.length && ( diff --git a/frontend/packages/app/pages/groups.tsx b/frontend/packages/app/pages/groups.tsx index 08ad75f94..6d68c9177 100644 --- a/frontend/packages/app/pages/groups.tsx +++ b/frontend/packages/app/pages/groups.tsx @@ -191,6 +191,7 @@ export default function GroupsPage() { ) : groups.length > 0 ? ( { if (!item.group) return null diff --git a/frontend/packages/app/pages/publication-list-page.tsx b/frontend/packages/app/pages/publication-list-page.tsx index bcee81aec..e642109a4 100644 --- a/frontend/packages/app/pages/publication-list-page.tsx +++ b/frontend/packages/app/pages/publication-list-page.tsx @@ -42,8 +42,9 @@ import { } from '../models/documents' import {useGatewayUrl} from '../models/gateway-settings' import {useWaitForPublication} from '../models/web-links' -import {DraftRoute, useNavRoute} from '../utils/navigation' +import {useNavRoute} from '../utils/navigation' import {useOpenDraft} from '../utils/open-draft' +import {DraftRoute} from '../utils/routes' import {useClickNavigate, useNavigate} from '../utils/useNavigate' export const PublicationListPage = memo(PublicationListPageUnmemo) @@ -253,6 +254,10 @@ export function PublicationsList({ key={trustedOnly ? 'trusted' : 'all'} items={items} header={header} + fixedItemHeight={52} + onEndReached={() => { + publications.fetchNextPage() + }} renderItem={({item}) => { const {publication, author, editors} = item if (!publication.document) return null @@ -326,6 +331,7 @@ function DraftsList() { if (drafts.data?.documents.length === 0) { return ( } items={['You have no current Drafts.']} renderItem={({item}) => { @@ -345,9 +351,14 @@ function DraftsList() { } items={drafts.data.documents} + fixedItemHeight={52} renderItem={({item}) => { return }} + onEndReached={() => { + console.log('== ~ DraftsList ~ onEndReached') + drafts.fetchNextPage() + }} /> ) } @@ -362,9 +373,13 @@ const DraftListItem = React.memo(function DraftListItem({ const navigate = useClickNavigate() const {queryClient, grpcClient} = useAppContext() if (!draft.id) throw new Error('DraftListItem requires an id') - const draftRoute: DraftRoute = {key: 'draft', draftId: draft.id} + const draftRoute: DraftRoute = { + key: 'draft', + draftId: draft.id, + variant: null, + } const goToItem = (e: any) => { - navigate(draftRoute, e) + navigate(draftRoute as DraftRoute, e) } return ( <> diff --git a/frontend/packages/shared/package.json b/frontend/packages/shared/package.json index 179a68948..cdafd2bd5 100644 --- a/frontend/packages/shared/package.json +++ b/frontend/packages/shared/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@bufbuild/protobuf": "1.4.1", - "@connectrpc/connect-web": "1.1.3" + "@connectrpc/connect-web": "1.1.3", + "react-tweet": "^3.2.0" } } diff --git a/frontend/packages/shared/src/publication-content.tsx b/frontend/packages/shared/src/publication-content.tsx index 43aaa0106..199d5bd03 100644 --- a/frontend/packages/shared/src/publication-content.tsx +++ b/frontend/packages/shared/src/publication-content.tsx @@ -640,7 +640,7 @@ function BlockContent(props: BlockContentProps) { } } - if (props.block.type == 'web-embed') { + if (props.block.type == 'web-embed' && props.block.ref) { return } @@ -1575,7 +1575,7 @@ export function BlockContentNostr({block, ...props}: BlockContentProps) { export function BlockContentTwitter({block, ...props}: BlockContentProps) { const {layoutUnit, onLinkClick} = usePublicationContentContext() - const urlArray = block.ref.split('/') + const urlArray = block.ref?.split('/') ?? [] const tweetId = urlArray[urlArray.length - 1].split('?')[0] const {data, error, isLoading} = useTweet(tweetId) diff --git a/frontend/packages/ui/src/list.tsx b/frontend/packages/ui/src/list.tsx index 73ec6f29d..96dc33918 100644 --- a/frontend/packages/ui/src/list.tsx +++ b/frontend/packages/ui/src/list.tsx @@ -1,80 +1,24 @@ -import {ReactNode, forwardRef, useRef, useState} from 'react' +import {ReactNode, forwardRef, useState} from 'react' import {Virtuoso, VirtuosoHandle} from 'react-virtuoso' import {View, XStack, YStack} from 'tamagui' -export function List({ - items, - renderItem, - header, - footer, -}: { - items: Item[] - renderItem: (row: {item: Item; containerWidth: number}) => ReactNode - header?: ReactNode | null - footer?: ReactNode | null -}) { - const virtuoso = useRef(null) - const [containerWidth, setContainerWidth] = useState(0) - const [containerHeight, setContainerHeight] = useState(0) - return ( - { - setContainerHeight(e.nativeEvent.layout.height) - setContainerWidth(e.nativeEvent.layout.width) - }} - > - header || null, - Footer: () => footer || , - }} - className="main-scroll-wrapper" - totalCount={items?.length || 0} - itemContent={(index) => { - const item = items?.[index] - if (!item) return null - return ( - - {renderItem({item, containerWidth})} - - ) - }} - /> - - ) -} - -export type FeedListHandle = VirtuosoHandle +export type ListHandle = VirtuosoHandle -export const FeedList = forwardRef(function FeedListComponent( +export const List = forwardRef(function ListComponent( { items, renderItem, header, footer, onEndReached, + fixedItemHeight, }: { items: Item[] renderItem: (row: {item: Item; containerWidth: number}) => ReactNode header?: ReactNode | null footer?: ReactNode | null onEndReached?: () => void + fixedItemHeight?: number }, ref: React.Ref, ) { @@ -92,6 +36,7 @@ export const FeedList = forwardRef(function FeedListComponent( > { onEndReached?.() }} @@ -115,7 +60,11 @@ export const FeedList = forwardRef(function FeedListComponent( const item = items?.[index] if (!item) return null return ( - + {renderItem({item, containerWidth})} ) diff --git a/yarn.lock b/yarn.lock index 5f89f8f9f..49629f7d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5156,6 +5156,7 @@ __metadata: "@bufbuild/protobuf": 1.4.1 "@connectrpc/connect-web": 1.1.3 "@mintter/prettier": "*" + react-tweet: ^3.2.0 typescript: 5.1.6 vitest: 0.34.2 languageName: unknown