From 5222b5404a875371e71e312c3c32f7ebee259d2f Mon Sep 17 00:00:00 2001 From: Franck Royer Date: Mon, 27 Sep 2021 14:55:47 +1000 Subject: [PATCH] Implement EIP-1459 --- .cspell.json | 8 +- src/lib/discovery/dns.spec.ts | 206 ++++++++++++++++++++++++++++++++++ src/lib/discovery/dns.ts | 202 +++++++++++++++++++++++++++++++++ src/lib/discovery/enr.ts | 50 +++++---- 4 files changed, 441 insertions(+), 25 deletions(-) create mode 100644 src/lib/discovery/dns.spec.ts create mode 100644 src/lib/discovery/dns.ts diff --git a/.cspell.json b/.cspell.json index 82d6d34fc7..9e2487b827 100644 --- a/.cspell.json +++ b/.cspell.json @@ -22,6 +22,9 @@ "Dscore", "ecies", "editorconfig", + "enr", + "enrs", + "enrtree", "ephem", "esnext", "ethersproject", @@ -65,9 +68,11 @@ "recid", "rlnrelay", "sandboxed", + "scanf", "secio", "seckey", "secp", + "sscanf", "staticnode", "statusim", "submodule", @@ -98,6 +103,7 @@ "node_modules/**", "build", "gen", - "proto" + "proto", + "*.spec.ts" ] } diff --git a/src/lib/discovery/dns.spec.ts b/src/lib/discovery/dns.spec.ts new file mode 100644 index 0000000000..e3e66fc9ec --- /dev/null +++ b/src/lib/discovery/dns.spec.ts @@ -0,0 +1,206 @@ +import { expect } from 'chai'; + +import { DNSNodeDiscovery } from './dns'; +import testData from './testdata.json'; + +const mockData = testData.dns; + +const host = 'nodes.example.org'; +const rootDomain = 'JORXBYVVM7AEKETX5DGXW44EAY'; +const branchDomainA = 'D2SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainB = 'D3SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainC = 'D4SNLTAGWNQ34NTQTPHNZDECFU'; +const branchDomainD = 'D5SNLTAGWNQ34NTQTPHNZDECFU'; +const partialBranchA = 'AAAA'; +const partialBranchB = 'BBBB'; +const singleBranch = `enrtree-branch:${branchDomainA}`; +const doubleBranch = `enrtree-branch:${branchDomainA},${branchDomainB}`; +const multiComponentBranch = [ + `enrtree-branch:${branchDomainA},${partialBranchA}`, + `${partialBranchB},${branchDomainB}`, +]; + +// Note: once td.when is asked to throw for an input it will always throw. +// Input can't be re-used for a passing case. +const errorBranchA = `enrtree-branch:${branchDomainC}`; +const errorBranchB = `enrtree-branch:${branchDomainD}`; + +/** + * Mocks DNS resolution. + */ +class MockDNS { + fqdnRes: Map; + fqdnThrows: string[]; + + constructor() { + this.fqdnRes = new Map(); + this.fqdnThrows = []; + } + + addRes(fqdn: string, res: string[]): void { + this.fqdnRes.set(fqdn, res); + } + + addThrow(fqdn: string): void { + this.fqdnThrows.push(fqdn); + } + + resolve( + fqdn: string, + type: string + ): Promise<{ answers: Array<{ data: string }> }> { + if (type !== 'TXT') throw 'Expected TXT DNS queries.'; + + if (this.fqdnThrows.includes(fqdn)) throw 'Mock DNS throws.'; + + const res = this.fqdnRes.get(fqdn); + + if (!res) throw `Mock DNS could not resolve ${fqdn}}`; + + const answers = res.map((data) => { + return { + data: data, + }; + }); + + return Promise.resolve({ + answers, + }); + } +} + +describe('DNS Node Discovery', () => { + let mockDns: MockDNS; + + beforeEach(() => { + mockDns = new MockDNS(); + mockDns.addRes(host, [mockData.enrRoot]); + }); + + it('retrieves a single peer', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(1); + expect(peers[0].address).to.eq('45.77.40.127'); + expect(peers[0].tcpPort).to.eq(30303); + }); + + it('retrieves all peers (2) when maxQuantity larger than DNS tree size', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [doubleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enrA]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrB]); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(2); + expect(peers[0].address).to.not.eq(peers[1].address); + }); + + it('retrieves all peers (3) when branch entries are composed of multiple strings', async function () { + mockDns.addRes(`${rootDomain}.${host}`, multiComponentBranch); + mockDns.addRes(`${branchDomainA}.${host}`, [mockData.enr]); + mockDns.addRes(`${branchDomainB}.${host}`, [mockData.enrA]); + mockDns.addRes(`${partialBranchA}${partialBranchB}.${host}`, [ + mockData.enrB, + ]); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(50, [mockData.enrTree]); + + expect(peers.length).to.eq(3); + expect(peers[0].address).to.not.eq(peers[1].address); + expect(peers[0].address).to.not.eq(peers[2].address); + expect(peers[1].address).to.not.eq(peers[2].address); + }); + + it('it tolerates circular branch references', async function () { + // root --> branchA + // branchA --> branchA + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + mockDns.addRes(`${branchDomainA}.${host}`, [singleBranch]); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + }); + + it('recovers when dns.resolve returns empty', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [singleBranch]); + + // Empty response case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + let peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + + expect(peers.length).to.eq(0); + + // No TXT records case + mockDns.addRes(`${branchDomainA}.${host}`, []); + + peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores domain fetching errors', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchA]); + mockDns.addThrow(`${branchDomainC}.${host}`); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('ignores unrecognized TXT record formats', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [mockData.enrBranchBadPrefix]); + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peers = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peers.length).to.eq(0); + }); + + it('caches peers it previously fetched', async function () { + mockDns.addRes(`${rootDomain}.${host}`, [errorBranchB]); + mockDns.addRes(`${branchDomainD}.${host}`, [mockData.enrA]); + + const dnsNodeDiscovery = new DNSNodeDiscovery(mockDns); + const peersA = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersA.length).to.eq(1); + + // Specify that a subsequent network call retrieving the same peer should throw. + // This test passes only if the peer is fetched from cache + mockDns.addThrow(`${branchDomainD}.${host}`); + + const peersB = await dnsNodeDiscovery.getPeers(1, [mockData.enrTree]); + expect(peersB.length).to.eq(1); + expect(peersA[0].address).to.eq(peersB[0].address); + }); +}); + +describe('DNS Node Discovery [live data]', function () { + const publicKey = 'AOFTICU2XWDULNLZGRMQS4RIZPAZEHYMV4FYHAPW563HNRAOERP7C'; + const fqdn = 'test.nodes.vac.dev'; + const enrTree = `enrtree://${publicKey}@${fqdn}`; + const ipTestRegex = /^\d+\.\d+\.\d+\.\d+$/; + const maxQuantity = 3; + + it(`should retrieve ${maxQuantity} PeerInfos for test.nodes.vac.dev`, async function () { + // Google's dns server address. Needs to be set explicitly to run in CI + const dnsNodeDiscovery = new DNSNodeDiscovery(); + const peers = await dnsNodeDiscovery.getPeers(maxQuantity, [enrTree]); + + expect(peers.length).to.eq(maxQuantity); + + const seen: string[] = []; + for (const peer of peers) { + expect(peer!.address!).to.match(ipTestRegex); + expect(seen).to.not.include(peer!.address!); + seen.push(peer!.address!); + } + }); +}); diff --git a/src/lib/discovery/dns.ts b/src/lib/discovery/dns.ts new file mode 100644 index 0000000000..664eb98948 --- /dev/null +++ b/src/lib/discovery/dns.ts @@ -0,0 +1,202 @@ +import assert from 'assert'; + +import { debug } from 'debug'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore: No types available +import DNS from 'dns2'; + +import { PeerInfo } from './enr'; +import { ENR } from './enr'; + +const dbg = debug('waku:discovery:dns'); + +type SearchContext = { + domain: string; + publicKey: string; + visits: { [key: string]: boolean }; +}; + +export class DNSNodeDiscovery { + private readonly dns: DNS; + private _DNSTreeCache: { [key: string]: string }; + private readonly _errorTolerance: number = 10; + + constructor(dns?: DNS) { + this._DNSTreeCache = {}; + if (dns) { + this.dns = dns; + } else { + this.dns = new DNS(); + } + } + + /** + * Returns a list of verified peers listed in an EIP-1459 DNS tree. Method may + * return fewer peers than requested if `maxQuantity` is larger than the number + * of ENR records or the number of errors/duplicate peers encountered by randomized + * search exceeds `maxQuantity` plus the `errorTolerance` factor. + */ + async getPeers( + maxQuantity: number, + dnsNetworks: string[] + ): Promise { + let totalSearches = 0; + const peers: PeerInfo[] = []; + + const networkIndex = Math.floor(Math.random() * dnsNetworks.length); + const { publicKey, domain } = ENR.parseTree(dnsNetworks[networkIndex]); + + while ( + peers.length < maxQuantity && + totalSearches < maxQuantity + this._errorTolerance + ) { + const context: SearchContext = { + domain, + publicKey, + visits: {}, + }; + + const peer = await this._search(domain, context); + + if (peer && isNewPeer(peer, peers)) { + peers.push(peer); + dbg(`got new peer candidate from DNS address=${JSON.stringify(peer)}`); + } + + totalSearches++; + } + return peers; + } + + /** + * Runs a recursive, randomized descent of the DNS tree to retrieve a single + * ENR record as a PeerInfo object. Returns null if parsing or DNS resolution fails. + */ + private async _search( + subdomain: string, + context: SearchContext + ): Promise { + const entry = await this._getTXTRecord(subdomain, context); + context.visits[subdomain] = true; + + let next: string; + let branches: string[]; + + const entryType = getEntryType(entry); + try { + switch (entryType) { + case ENR.ROOT_PREFIX: + next = ENR.parseAndVerifyRoot(entry, context.publicKey); + return await this._search(next, context); + case ENR.BRANCH_PREFIX: + branches = ENR.parseBranch(entry); + next = selectRandomPath(branches, context); + return await this._search(next, context); + case ENR.RECORD_PREFIX: + return ENR.parseAndVerifyRecord(entry); + default: + return null; + } + } catch (error) { + dbg( + `Failed to search DNS tree ${entryType} at subdomain ${subdomain}: ${error}` + ); + return null; + } + } + + /** + * Retrieves the TXT record stored at a location from either + * this DNS tree cache or via DNS query + */ + private async _getTXTRecord( + subdomain: string, + context: SearchContext + ): Promise { + if (this._DNSTreeCache[subdomain]) { + return this._DNSTreeCache[subdomain]; + } + + // Location is either the top level tree entry host or a subdomain of it. + const location = + subdomain !== context.domain + ? `${subdomain}.${context.domain}` + : context.domain; + + const response = await this.dns + .resolve(location, 'TXT') + .then((res: { answers: Array<{ data: string }> }) => + res.answers.map((answer: { data: string }) => answer.data) + ); + + assert( + response.length, + 'Received empty result array while fetching TXT record' + ); + assert(response[0].length, 'Received empty TXT record'); + + if (response.length > 1) { + dbg(`Warning, DNS TXT value ${location} may be omitted`, response); + } + + this._DNSTreeCache[subdomain] = response[0]; + return response[0]; + } +} + +function getEntryType(entry: string): string { + if (entry.startsWith(ENR.ROOT_PREFIX)) return ENR.ROOT_PREFIX; + if (entry.startsWith(ENR.BRANCH_PREFIX)) return ENR.BRANCH_PREFIX; + if (entry.startsWith(ENR.RECORD_PREFIX)) return ENR.RECORD_PREFIX; + + return ''; +} + +/** + * Returns a randomly selected subdomain string from the list provided by a branch + * entry record. + * + * The client must track subdomains which are already resolved to avoid + * going into an infinite loop b/c branch entries can contain + * circular references. It’s in the client’s best interest to traverse the + * tree in random order. + */ +function selectRandomPath(branches: string[], context: SearchContext): string { + // Identify domains already visited in this traversal of the DNS tree. + // Then filter against them to prevent cycles. + const circularRefs: { [key: number]: boolean } = {}; + for (const [idx, subdomain] of branches.entries()) { + if (context.visits[subdomain]) { + circularRefs[idx] = true; + } + } + // If all possible paths are circular... + if (Object.keys(circularRefs).length === branches.length) { + throw new Error('Unresolvable circular path detected'); + } + + // Randomly select a viable path + let index; + do { + index = Math.floor(Math.random() * branches.length); + } while (circularRefs[index]); + + return branches[index]; +} + +/** + * Returns false if candidate peer already exists in the + * current collection of peers. + * Returns true otherwise. + */ +function isNewPeer(peer: PeerInfo | null, peers: PeerInfo[]): boolean { + if (!peer || !peer.address) return false; + + for (const existingPeer of peers) { + if (peer.address === existingPeer.address) { + return false; + } + } + + return true; +} diff --git a/src/lib/discovery/enr.ts b/src/lib/discovery/enr.ts index c6466fff05..faa7e83161 100644 --- a/src/lib/discovery/enr.ts +++ b/src/lib/discovery/enr.ts @@ -9,19 +9,10 @@ import { ecdsaVerify } from 'secp256k1'; import { keccak256Buf } from '../utils'; -// Convert is not exported from multiaddr's index.d.ts so cannot use import +// Convert is not exported from multiaddr index.d.ts so cannot use import // eslint-disable-next-line @typescript-eslint/no-var-requires const Convert = require('multiaddr/src/convert'); -// multiaddr 8.0.0 expects an Uint8Array with internal buffer starting at 0 offset -function toNewUint8Array(buf: Uint8Array): Uint8Array { - const arrayBuffer = buf.buffer.slice( - buf.byteOffset, - buf.byteOffset + buf.byteLength - ); - return new Uint8Array(arrayBuffer); -} - export interface PeerInfo { id?: Uint8Array | Buffer; address?: string; @@ -97,9 +88,9 @@ export class ENR { ); return { - address: Convert.toString(ipCode, obj.ip) as string, - tcpPort: parseInt(Convert.toString(tcpCode, toNewUint8Array(obj.tcp))), - udpPort: parseInt(Convert.toString(udpCode, toNewUint8Array(obj.udp))), + address: obj.ip ? Convert.toString(ipCode, obj.ip) : null, + tcpPort: obj.tcp ? parseInt(Convert.toString(tcpCode, obj.tcp)) : null, + udpPort: obj.udp ? parseInt(Convert.toString(udpCode, obj.udp)) : null, }; } @@ -113,7 +104,7 @@ export class ENR { `ENR root entry must start with '${this.ROOT_PREFIX}'` ); - const rootVals = sscanf( + const rootValues = sscanf( root, `${this.ROOT_PREFIX}v1 e=%s l=%s seq=%d sig=%s`, 'eRoot', @@ -122,11 +113,20 @@ export class ENR { 'signature' ) as ENRRootValues; - assert.ok(rootVals.eRoot, "Could not parse 'e' value from ENR root entry"); - assert.ok(rootVals.lRoot, "Could not parse 'l' value from ENR root entry"); - assert.ok(rootVals.seq, "Could not parse 'seq' value from ENR root entry"); assert.ok( - rootVals.signature, + rootValues.eRoot, + "Could not parse 'e' value from ENR root entry" + ); + assert.ok( + rootValues.lRoot, + "Could not parse 'l' value from ENR root entry" + ); + assert.ok( + rootValues.seq, + "Could not parse 'seq' value from ENR root entry" + ); + assert.ok( + rootValues.signature, "Could not parse 'sig' value from ENR root entry" ); @@ -137,7 +137,9 @@ export class ENR { // (Trailing recovery bit must be trimmed to pass `ecdsaVerify` method) const signedComponent = root.split(' sig')[0]; const signedComponentBuffer = Buffer.from(signedComponent); - const signatureBuffer = base64url.toBuffer(rootVals.signature).slice(0, 64); + const signatureBuffer = base64url + .toBuffer(rootValues.signature) + .slice(0, 64); const keyBuffer = Buffer.from(decodedPublicKey); const isVerified = ecdsaVerify( @@ -148,7 +150,7 @@ export class ENR { assert(isVerified, 'Unable to verify ENR root signature'); - return rootVals.eRoot; + return rootValues.eRoot; } /** @@ -162,7 +164,7 @@ export class ENR { `ENR tree entry must start with '${this.TREE_PREFIX}'` ); - const treeVals = sscanf( + const treeValues = sscanf( tree, `${this.TREE_PREFIX}//%s@%s`, 'publicKey', @@ -170,12 +172,12 @@ export class ENR { ) as ENRTreeValues; assert.ok( - treeVals.publicKey, + treeValues.publicKey, 'Could not parse public key from ENR tree entry' ); - assert.ok(treeVals.domain, 'Could not parse domain from ENR tree entry'); + assert.ok(treeValues.domain, 'Could not parse domain from ENR tree entry'); - return treeVals; + return treeValues; } /**