Merge pull request #707 from oasisprotocol/ptrus/fix/validator-media

Validator media

.changelog/707.feature.md

@@ -0,0 +1,10 @@
+Validator media updates
+
+Adds `LogoUrl` field to the `ValidatorMedia` type.
+Also updates `ValidatorMedia` fields to match `RegistryMetadata`:
+
+- `website_link` -> `url`
+- `email_address` -> `email`
+- `twitter_acc` -> `twitter`
+- `tg_chat` removed
+- `keybase` added

analyzer/evmnfts/ipfsclient/gateway.go

@@ -6,7 +6,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/oasisprotocol/nexus/analyzer/evmnfts/httpmisc"
+	"github.com/oasisprotocol/nexus/analyzer/httpmisc"
 )
 
 // gatewayHTTPClient is an *http.Client to use for accessing gateways.

analyzer/evmnfts/multiproto/multiproto.go

@@ -9,9 +9,9 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/oasisprotocol/nexus/analyzer/evmnfts/httpmisc"
 	"github.com/oasisprotocol/nexus/analyzer/evmnfts/ipfsclient"
-	"github.com/oasisprotocol/nexus/analyzer/evmnfts/pubclient"
+	"github.com/oasisprotocol/nexus/analyzer/httpmisc"
+	"github.com/oasisprotocol/nexus/analyzer/pubclient"
 )
 
 // multiproto gets a file from a URL by looking at the URL's scheme and

analyzer/metadata_registry/metadata_registry.go

@@ -2,21 +2,31 @@ package metadata_registry
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/url"
 	"time"
 
 	registry "github.com/oasisprotocol/metadata-registry-tools"
 
 	staking "github.com/oasisprotocol/nexus/coreapi/v22.2.11/staking/api"
 
 	"github.com/oasisprotocol/nexus/analyzer"
+	"github.com/oasisprotocol/nexus/analyzer/httpmisc"
 	"github.com/oasisprotocol/nexus/analyzer/item"
+	"github.com/oasisprotocol/nexus/analyzer/pubclient"
 	"github.com/oasisprotocol/nexus/analyzer/queries"
 	"github.com/oasisprotocol/nexus/config"
 	"github.com/oasisprotocol/nexus/log"
 	"github.com/oasisprotocol/nexus/storage"
 )
 
+const (
+	keybaseMaxResponseSize = 10 * 1024 * 1024 // 10 MiB.
+	keybaseLookupUrl       = "https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames="
+)
+
 const MetadataRegistryAnalyzerName = "metadata_registry"
 
 type processor struct {
@@ -70,11 +80,24 @@ func (p *processor) ProcessItem(ctx context.Context, batch *storage.QueryBatch,
 	}
 
 	for id, meta := range entities {
+		var logoUrl string
+		if meta.Keybase != "" {
+			logoUrl, err = fetchKeybaseLogoUrl(ctx, meta.Keybase)
+			if err != nil {
+				p.logger.Warn("failed to fetch keybase url", "err", err, "handle", meta.Keybase)
+			}
+			if ctx.Err() != nil {
+				// Exit early if the context is done.
+				return ctx.Err()
+			}
+		}
+
 		batch.Queue(
 			queries.ConsensusEntityMetaUpsert,
 			id.String(),
 			staking.NewAddress(id).String(),
 			meta,
+			logoUrl,
 		)
 	}
 
@@ -85,3 +108,37 @@ func (p *processor) QueueLength(ctx context.Context) (int, error) {
 	// The concept of a work queue does not apply to this analyzer.
 	return 0, nil
 }
+
+func fetchKeybaseLogoUrl(ctx context.Context, handle string) (string, error) {
+	resp, err := pubclient.GetWithContext(ctx, keybaseLookupUrl+url.QueryEscape(handle))
+	if err != nil {
+		return "", err
+	}
+	if err = httpmisc.ResponseOK(resp); err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	return parseKeybaseLogoUrl(io.LimitReader(resp.Body, keybaseMaxResponseSize))
+}
+
+func parseKeybaseLogoUrl(r io.Reader) (string, error) {
+	var response struct {
+		Them []struct {
+			Id       string `json:"id"`
+			Pictures *struct {
+				Primary struct {
+					Url string `json:"url"`
+				} `json:"primary"`
+			} `json:"pictures"`
+		} `json:"them"`
+	}
+	if err := json.NewDecoder(r).Decode(&response); err != nil {
+		return "", err
+	}
+
+	if len(response.Them) > 0 && response.Them[0].Pictures != nil {
+		return response.Them[0].Pictures.Primary.Url, nil
+	}
+	return "", nil
+}

analyzer/metadata_registry/metadata_registry_test.go

@@ -0,0 +1,58 @@
+package metadata_registry
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseKeybaseResponse(t *testing.T) {
+	tc := []struct {
+		json      string
+		shouldErr bool
+		resp      string
+	}{
+		// https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames=ptrs
+		{
+			json:      `{"status":{"code":0,"name":"OK"},"them":[{"id":"149815106745ab70d3e9cdc00fc96419","pictures":{"primary":{"url":"https://s3.amazonaws.com/keybase_processed_uploads/ef2c20c2c1e1a6584d91c49303fd8e05_360_360.jpg","source":null}}},{"id":"a19ba1453ec120ed157c6e85b245c119"}]}`,
+			shouldErr: false,
+			resp:      "https://s3.amazonaws.com/keybase_processed_uploads/ef2c20c2c1e1a6584d91c49303fd8e05_360_360.jpg",
+		},
+		// https://keybase.io/_/api/1.0/user/lookup.json?fields=pictures&usernames=test
+		{
+			json:      `{"status":{"code":0,"name":"OK"},"them":[{"id":"a19ba1453ec120ed157c6e85b245c119"}]}`,
+			shouldErr: false,
+			resp:      "",
+		},
+		// Invalid JSON.
+		{
+			json:      `{"status":123, "code}`,
+			shouldErr: true,
+			resp:      "",
+		},
+		// Empty response.
+		{
+			json:      "",
+			shouldErr: true,
+			resp:      "",
+		},
+	}
+
+	for _, tt := range tc {
+		url, err := parseKeybaseLogoUrl(strings.NewReader(tt.json))
+		if tt.shouldErr {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, tt.resp, url)
+		}
+	}
+}
+
+func TestFetchKeybase(t *testing.T) {
+	logo, err := fetchKeybaseLogoUrl(context.Background(), "ptrs")
+	require.NoError(t, err)
+	require.NotEmpty(t, logo)
+}

analyzer/evmnfts/pubclient/pub_client.go → analyzer/pubclient/pub_client.go

@@ -11,7 +11,7 @@ import (
 
 	coreCommon "github.com/oasisprotocol/oasis-core/go/common"
 
-	"github.com/oasisprotocol/nexus/analyzer/evmnfts/httpmisc"
+	"github.com/oasisprotocol/nexus/analyzer/httpmisc"
 )
 
 // Use this package for connecting to untrusted URLs.

analyzer/evmnfts/pubclient/pub_client_test.go → analyzer/pubclient/pub_client_test.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/oasisprotocol/nexus/analyzer/evmnfts/httpmisc"
+	"github.com/oasisprotocol/nexus/analyzer/httpmisc"
 )
 
 func wasteResp(resp *http.Response) error {

analyzer/queries/queries.go

@@ -313,8 +313,8 @@ var (
     DELETE FROM chain.nodes WHERE id = $1`
 
 	ConsensusEntityMetaUpsert = `
-    INSERT INTO chain.entities(id, address, meta)
-      VALUES ($1, $2, $3)
+    INSERT INTO chain.entities(id, address, meta, logo_url)
+      VALUES ($1, $2, $3, $4)
     ON CONFLICT (id) DO UPDATE SET
       meta = excluded.meta`
 
@@ -973,7 +973,7 @@ var (
       runtime = $1 AND verification_info_downloaded_at IS NULL`
 
 	RuntimeEVMVerifiedContracts = `
-    SELECT 
+    SELECT
       contracts.contract_address,
       contracts.verification_level
     FROM chain.evm_contracts AS contracts

api/spec/v1.yaml

@@ -1128,7 +1128,7 @@ paths:
         - in: path
           name: address
           required: true
-          schema: 
+          schema:
             allOf: [$ref: '#/components/schemas/EthOrOasisAddress']
           description: The staking address of the account to return.
       responses:
@@ -1530,8 +1530,8 @@ components:
           type: object
           description: The method call body. This spec does not encode the many possible types;
             instead, see [the Go API](https://pkg.go.dev/github.com/oasisprotocol/oasis-core/go) of oasis-core.
-            This object will conform to one of the types passed to variable instantiations using `NewMethodName` 
-            two levels down the hierarchy, e.g. `MethodTransfer` from `oasis-core/go/staking/api` seen 
+            This object will conform to one of the types passed to variable instantiations using `NewMethodName`
+            two levels down the hierarchy, e.g. `MethodTransfer` from `oasis-core/go/staking/api` seen
             [here](https://pkg.go.dev/github.com/oasisprotocol/oasis-core/[email protected]/staking/api#pkg-variables).
           example: *tx_body_1
         success:
@@ -1561,9 +1561,9 @@ components:
             This field, like `code` and `module`, can represent an error that originated
             anywhere in the paratime, i.e. either inside or outside a smart contract.
 
-            A common special case worth calling out: When the paratime is 
-            EVM-compatible (e.g. Emerald or Sapphire) and the error originates 
-            inside a smart contract (using `revert` in solidity), the following 
+            A common special case worth calling out: When the paratime is
+            EVM-compatible (e.g. Emerald or Sapphire) and the error originates
+            inside a smart contract (using `revert` in solidity), the following
             will be true:
             - `module` will be "evm" and `code` will be 8; see [here](https://github.com/oasisprotocol/oasis-sdk/blob/runtime-sdk/v0.8.3/runtime-sdk/modules/evm/src/lib.rs#L128) for other possible errors in the `evm` module.
             - `message` will contain the best-effort human-readable revert reason.
@@ -1573,12 +1573,12 @@ components:
             allOf: [$ref: '#/components/schemas/EvmAbiParam']
           description: |
             The error parameters, as decoded using the contract abi. Present only when
-            - the error originated from within a smart contract (e.g. via `revert` in Solidity), and 
+            - the error originated from within a smart contract (e.g. via `revert` in Solidity), and
             - the contract is verified or the revert reason is a plain String.
             If this field is present, `message` will include the name of the error, e.g. 'InsufficentBalance'.
             Note that users should be cautious when evaluating error data since the
             data origin is not tracked and error information can be faked.
-            
+
     ConsensusEventType:
       type: string
       enum:
@@ -1835,25 +1835,25 @@ components:
       type: object
       # All of the fields are optional.
       properties:
-        website_link:
+        url:
           type: string
           description: An URL associated with the entity.
-        email_address:
+        email:
           type: string
-          description: An email address for the validator.
-        twitter_acc:
+          description: An email address associated with the entity.
+        twitter:
           type: string
-          description: A Twitter handle.
-        tg_chat:
+          description: Twitter handle.
+        keybase:
           type: string
-          description: An Telegram handle.
-        logotype:
-          type: string
-          description: A logo type.
+          description: Keybase handle.
         name:
           type: string
-          description: The human-readable name of this validator.
+          description: The human-readable name of this entity.
           example: WhaleStake
+        logoUrl:
+          type: string
+          description: URL to a logo image for the entity.
 
     Validator:
       type: object
@@ -2424,13 +2424,13 @@ components:
             This object will conform to one of the `*Event` types two levels down
             the hierarchy (e.g. `MintEvent` from `accounts > Event > MintEvent`),
             OR `evm > Event`. For object fields that specify an oasis-style address, Nexus
-            will add a field specifying the corresponding Ethereum address, if known. Currently, 
+            will add a field specifying the corresponding Ethereum address, if known. Currently,
             the only such possible fields are `from_eth`, `to_eth`, and `owner_eth`.
         evm_log_name:
           type: string
           description: |
-            If the event type is `evm.log`, this field describes the human-readable type of 
-            evm event, e.g. `Transfer`. 
+            If the event type is `evm.log`, this field describes the human-readable type of
+            evm event, e.g. `Transfer`.
             Absent if the event type is not `evm.log`.
           example: 'Transfer'
         evm_log_params:

storage/client/client.go

@@ -1188,6 +1188,7 @@ func (c *StorageClient) Validators(ctx context.Context, p apiTypes.GetConsensusV
 	for res.rows.Next() {
 		var v Validator
 		var schedule staking.CommissionSchedule
+		var logoUrl *string
 		if err := res.rows.Scan(
 			&v.EntityID,
 			&v.EntityAddress,
@@ -1197,10 +1198,17 @@ func (c *StorageClient) Validators(ctx context.Context, p apiTypes.GetConsensusV
 			&v.Active,
 			&v.Status,
 			&v.Media,
+			&logoUrl,
 		); err != nil {
 			return nil, wrapError(err)
 		}
 
+		if logoUrl != nil && *logoUrl != "" {
+			if v.Media == nil {
+				v.Media = &ValidatorMedia{}
+			}
+			v.Media.LogoUrl = logoUrl
+		}
 		currentRate := schedule.CurrentRate(beacon.EpochTime(epoch.ID))
 		if currentRate != nil {
 			v.CurrentRate = currentRate.ToBigInt().Uint64()

storage/client/queries/queries.go

@@ -300,7 +300,8 @@ const (
 				COALESCE(chain.commissions.schedule, '{}'::JSONB) AS commissions_schedule,
 				EXISTS(SELECT NULL FROM chain.nodes WHERE chain.entities.id = chain.nodes.entity_id AND voting_power > 0) AS active,
 				EXISTS(SELECT NULL FROM chain.nodes WHERE chain.entities.id = chain.nodes.entity_id AND chain.nodes.roles like '%validator%') AS status,
-				chain.entities.meta AS meta
+				chain.entities.meta AS meta,
+				chain.entities.logo_url as logo_url
 			FROM chain.entities
 			JOIN chain.accounts ON chain.entities.address = chain.accounts.address
 			LEFT JOIN chain.commissions ON chain.entities.address = chain.commissions.address

storage/migrations/00_consensus.up.sql

@@ -110,9 +110,10 @@ CREATE INDEX ix_epochs_id ON chain.epochs (id);
 -- Registry Backend Data
 CREATE TABLE chain.entities
 (
-  id      base64_ed25519_pubkey PRIMARY KEY,
-  address oasis_addr NOT NULL, -- Deterministically derived from the ID.
-  meta    JSONB  -- Signed statements about the entity from https://github.com/oasisprotocol/metadata-registry
+  id       base64_ed25519_pubkey PRIMARY KEY,
+  address  oasis_addr NOT NULL, -- Deterministically derived from the ID.
+  meta     JSONB  -- Signed statements about the entity from https://github.com/oasisprotocol/metadata-registry
+  -- logo_url TEXT -- Added in 16_entity_logo.up.sql.
 );
 
 CREATE TABLE chain.nodes

storage/migrations/16_entity_logo_url.up.sql

@@ -0,0 +1,6 @@
+BEGIN;
+
+ALTER TABLE chain.entities
+    ADD COLUMN logo_url TEXT;
+
+COMMIT;