consensus/metadata: fetch logo URL from keybase

.changelog/707.feature.md

@@ -0,0 +1,10 @@
+Validator media updates
+
+Adds `LogoUrl` field to the `ValidatorMedia` type.
+Also updates `ValidatorMedia` fields to match `RegistryMetadata`:
+
+- `website_link` -> `url`
+- `email_address` -> `email`
+- `twitter_acc` -> `twitter`
+- `tg_chat` removed
+- `keybase` added

analyzer/metadata_registry/metadata_registry.go

@@ -2,21 +2,27 @@ package metadata_registry
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/url"
 	"time"
 
 	registry "github.com/oasisprotocol/metadata-registry-tools"
 
-	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
-
 	"github.com/oasisprotocol/nexus/analyzer"
+	"github.com/oasisprotocol/nexus/analyzer/evmnfts/httpmisc"
+	"github.com/oasisprotocol/nexus/analyzer/evmnfts/pubclient"
 	"github.com/oasisprotocol/nexus/analyzer/item"
 	"github.com/oasisprotocol/nexus/analyzer/queries"
 	"github.com/oasisprotocol/nexus/config"
+	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
 	"github.com/oasisprotocol/nexus/log"
 	"github.com/oasisprotocol/nexus/storage"
 )
 
+const keybaseMaxResponseSize = 10 * 1024 * 1024 // 10 MiB.
+
 const MetadataRegistryAnalyzerName = "metadata_registry"
 
 type processor struct {
@@ -70,11 +76,21 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch,
 	}
 
 	for id, meta := range entities {
+		var logoUrl string
+		if meta.Keybase != "" {
+			logoUrl, _ = p.fetchKeybaseLogoUrl(ctx, meta.Keybase)
+			if ctx.Err() != nil {
+				// Exit early if the context is done.
+				return ctx.Err()
+			}
+		}
+
 		batch.Queue(
 			queries.ConsensusEntityMetaUpsert,
 			id.String(),
 			staking.NewAddress(id).String(),
 			meta,
+			logoUrl,
 		)
 	}
 
@@ -85,3 +101,45 @@ func (p *processor) QueueLength(ctx context.Context) (int, error) {
 	// The concept of a work queue does not apply to this analyzer.
 	return 0, nil
 }
+
+func (p *processor) fetchKeybaseLogoUrl(ctx context.Context, handle string) (string, error) {
+	url := fmt.Sprintf("https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames=%s", url.QueryEscape(handle))
+	rc, err := func(url string) (io.ReadCloser, error) {
+		resp, err := pubclient.GetWithContext(ctx, url)
+		if err != nil {
+			return nil, err
+		}
+		if err = httpmisc.ResponseOK(resp); err != nil {
+			return nil, err
+		}
+		return resp.Body, nil
+	}(url)
+	if err != nil {
+		p.logger.Warn("failed to fetch Keybase info", "err", err, "url", url)
+		return "", err
+	}
+	defer rc.Close()
+
+	return parseKeybaseLogoUrl(io.LimitReader(rc, keybaseMaxResponseSize))
+}
+
+func parseKeybaseLogoUrl(r io.Reader) (string, error) {
+	var response struct {
+		Them []struct {
+			Id       string `json:"id"`
+			Pictures *struct {
+				Primary struct {
+					Url string `json:"url"`
+				} `json:"primary"`
+			} `json:"pictures"`
+		} `json:"them"`
+	}
+	if err := json.NewDecoder(r).Decode(&response); err != nil {
+		return "", err
+	}
+
+	if len(response.Them) > 0 && response.Them[0].Pictures != nil {
+		return response.Them[0].Pictures.Primary.Url, nil
+	}
+	return "", nil
+}

analyzer/metadata_registry/metadata_registry_test.go

@@ -0,0 +1,51 @@
+package metadata_registry
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseKeybaseResponse(t *testing.T) {
+	tc := []struct {
+		json      string
+		shouldErr bool
+		resp      string
+	}{
+		// https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames=ptrs
+		{
+			json:      `{"status":{"code":0,"name":"OK"},"them":[{"id":"149815106745ab70d3e9cdc00fc96419","pictures":{"primary":{"url":"https://s3.amazonaws.com/keybase_processed_uploads/ef2c20c2c1e1a6584d91c49303fd8e05_360_360.jpg","source":null}}},{"id":"a19ba1453ec120ed157c6e85b245c119"}]}`,
+			shouldErr: false,
+			resp:      "https://s3.amazonaws.com/keybase_processed_uploads/ef2c20c2c1e1a6584d91c49303fd8e05_360_360.jpg",
+		},
+		// https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames=test
+		{
+			json:      `{"status":{"code":0,"name":"OK"},"them":[{"id":"a19ba1453ec120ed157c6e85b245c119"}]}`,
+			shouldErr: false,
+			resp:      "",
+		},
+		// Invalid JSON.
+		{
+			json:      `{"status":123, "code}`,
+			shouldErr: true,
+			resp:      "",
+		},
+		// Empty response.
+		{
+			json:      "",
+			shouldErr: true,
+			resp:      "",
+		},
+	}
+
+	for _, tt := range tc {
+		url, err := parseKeybaseLogoUrl(strings.NewReader(tt.json))
+		if tt.shouldErr {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, tt.resp, url)
+		}
+	}
+}

analyzer/queries/queries.go

@@ -313,8 +313,8 @@ var (
     DELETE FROM chain.nodes WHERE id = $1`
 
 	ConsensusEntityMetaUpsert = `
-    INSERT INTO chain.entities(id, address, meta)
-      VALUES ($1, $2, $3)
+    INSERT INTO chain.entities(id, address, meta, logo_url)
+      VALUES ($1, $2, $3, $4)
     ON CONFLICT (id) DO UPDATE SET
       meta = excluded.meta`
 
@@ -973,7 +973,7 @@ var (
       runtime = $1 AND verification_info_downloaded_at IS NULL`
 
 	RuntimeEVMVerifiedContracts = `
-    SELECT 
+    SELECT
       contracts.contract_address,
       contracts.verification_level
     FROM chain.evm_contracts AS contracts

api/spec/v1.yaml

@@ -1849,8 +1849,11 @@ components:
           description: Keybase handle.
         name:
           type: string
-          description: The human-readable name of this validator.
+          description: The human-readable name of this entity.
           example: WhaleStake
+        logoUrl:
+          type: string
+          description: URL to a logo image for the entity.
 
     Validator:
       type: object

storage/client/client.go

@@ -1188,6 +1188,7 @@ func (c *StorageClient) Validators(ctx context.Context, p apiTypes.GetConsensusV
 	for res.rows.Next() {
 		var v Validator
 		var schedule staking.CommissionSchedule
+		var logoUrl *string
 		if err := res.rows.Scan(
 			&v.EntityID,
 			&v.EntityAddress,
@@ -1197,10 +1198,17 @@ func (c *StorageClient) Validators(ctx context.Context, p apiTypes.GetConsensusV
 			&v.Active,
 			&v.Status,
 			&v.Media,
+			&logoUrl,
 		); err != nil {
 			return nil, wrapError(err)
 		}
 
+		if logoUrl != nil && *logoUrl != "" {
+			if v.Media == nil {
+				v.Media = &ValidatorMedia{}
+			}
+			v.Media.LogoUrl = logoUrl
+		}
 		currentRate := schedule.CurrentRate(beacon.EpochTime(epoch.ID))
 		if currentRate != nil {
 			v.CurrentRate = currentRate.ToBigInt().Uint64()

storage/client/queries/queries.go

@@ -300,7 +300,8 @@ const (
 				COALESCE(chain.commissions.schedule, '{}'::JSONB) AS commissions_schedule,
 				EXISTS(SELECT NULL FROM chain.nodes WHERE chain.entities.id = chain.nodes.entity_id AND voting_power > 0) AS active,
 				EXISTS(SELECT NULL FROM chain.nodes WHERE chain.entities.id = chain.nodes.entity_id AND chain.nodes.roles like '%validator%') AS status,
-				chain.entities.meta AS meta
+				chain.entities.meta AS meta,
+				chain.entities.logo_url as logo_url
 			FROM chain.entities
 			JOIN chain.accounts ON chain.entities.address = chain.accounts.address
 			LEFT JOIN chain.commissions ON chain.entities.address = chain.commissions.address

storage/migrations/00_consensus.up.sql

@@ -110,9 +110,10 @@ CREATE INDEX ix_epochs_id ON chain.epochs (id);
 -- Registry Backend Data
 CREATE TABLE chain.entities
 (
-  id      base64_ed25519_pubkey PRIMARY KEY,
-  address oasis_addr NOT NULL, -- Deterministically derived from the ID.
-  meta    JSONB  -- Signed statements about the entity from https://github.com/oasisprotocol/metadata-registry
+  id       base64_ed25519_pubkey PRIMARY KEY,
+  address  oasis_addr NOT NULL, -- Deterministically derived from the ID.
+  meta     JSONB  -- Signed statements about the entity from https://github.com/oasisprotocol/metadata-registry
+  -- logo_url TEXT -- Added in 16_entity_logo.up.sql.
 );
 
 CREATE TABLE chain.nodes

storage/migrations/16_entity_logo_url.up.sql

@@ -0,0 +1,6 @@
+BEGIN;
+
+ALTER TABLE chain.entities
+    ADD COLUMN logo_url TEXT;
+
+COMMIT;