Improve local test utils

local-test-configuration/README.md

@@ -7,7 +7,8 @@ the setup:
   different API. One of the APIs is delayed so that the beacons have different values.
 - Use 2 signed APIs and each Pusher pushes to a separate Signed API.
 - We run Airseeker only on Hardhat network for setup simplicity. Initially, I wanted to have a Polygon testnet as well,
-  but gave up on that idea for now.
+  but gave up on that idea for now. The setup can be easily extended to run on any chain, just by not starting Hardhat
+  network and using a different RPCs.
 
 All configurations are based on the example files, but have been slightly modified. I had to also choose more volatile
 assets from the Nodary API to see Airseeker updates.
@@ -25,13 +26,13 @@ assets from the Nodary API to see Airseeker updates.
 - Start Signed API 1 on port `4001` (in a separate terminal):
 
 ```sh
-docker run --publish 4001:8090 -it --init --volume $(pwd)/local-test-configuration/signed-api-1:/app/config --env-file ./local-test-configuration/signed-api-1/.env --rm --memory=256m api3/signed-api:latest
+docker run --publish 4001:80 -it --init --volume $(pwd)/local-test-configuration/signed-api-1:/app/config --env-file ./local-test-configuration/signed-api-1/.env --rm --memory=256m api3/signed-api:latest
 ```
 
 - Start Signed API 2 on port `4002` (in a separate terminal):
 
 ```sh
-docker run --publish 4002:8090 -it --init --volume $(pwd)/local-test-configuration/signed-api-2:/app/config --env-file ./local-test-configuration/signed-api-2/.env --rm --memory=256m api3/signed-api:latest
+docker run --publish 4002:80 -it --init --volume $(pwd)/local-test-configuration/signed-api-2:/app/config --env-file ./local-test-configuration/signed-api-2/.env --rm --memory=256m api3/signed-api:latest
 ```
 
 You can go to `http://localhost:4001/` and `http://localhost:4002/` to see the Signed API 1 and 2 respectively.
@@ -65,9 +66,12 @@ also required for the monitoring page.
 
 - Open the monitoring page located in `local-test-configuration/monitoring/index.html` in a browser with the following
   query parameters appended
-  `api3ServerV1Address=<DEPLOYED_API3_SERVER_V1_ADDRESS>&dapiDataRegistryAddress=<DEPLOYED_DAPI_DATA_REGISTRY_ADDRESS>`
+  `?api3ServerV1Address=<DEPLOYED_API3_SERVER_V1_ADDRESS>&dapiDataRegistryAddress=<DEPLOYED_DAPI_DATA_REGISTRY_ADDRESS>&rpcUrl=<RPC_URL>&airseekerMnemonic=<AIRSEEKER_MNEMONIC>`
   and open console.
 
+The `AIRSEEKER_MNEMONIC` needs to be URI encoded via `encodeURIComponent` in JS. For example,
+`test%20test%20test%20test%20test%20test%20test%20test%20test%20test%20test%20junk`.
+
 Initially, you should see errors because the beacons are not initialized. After you run Airseeker, it will do the
 updates and the errors should be gone. The page constantly polls the chain and respective signed APIs and compares the
 on-chain and off-chain values. If the deviation exceeds the treshold, the value is marked bold and should be updated by

local-test-configuration/monitoring/index.html

@@ -1449,9 +1449,10 @@ <h2>Active dAPIs</h2>
 
     // Configuration
     const urlParams = new URLSearchParams(window.location.search);
-    const rpcUrl = 'http://127.0.0.1:8545',
+    const rpcUrl = urlParams.get('rpcUrl'),
       api3ServerV1Address = urlParams.get('api3ServerV1Address'),
-      dapiDataRegistryAddress = urlParams.get('dapiDataRegistryAddress');
+      dapiDataRegistryAddress = urlParams.get('dapiDataRegistryAddress'),
+      airseekerMnemonic = decodeURIComponent(urlParams.get('airseekerMnemonic'));
 
     if (!api3ServerV1Address) throw new Error('api3ServerV1Address must be provided as URL parameter');
     if (!dapiDataRegistryAddress) throw new Error('dapiDataRegistryAddress must be provided as URL parameter');
@@ -1534,6 +1535,28 @@ <h2>Active dAPIs</h2>
       return updateInPercentage.gt(deviationThreshold);
     };
 
+    function deriveWalletPathFromSponsorAddress(sponsorAddress) {
+      const sponsorAddressBN = ethers.BigNumber.from(sponsorAddress);
+      const paths = [];
+      for (let i = 0; i < 6; i++) {
+        const shiftedSponsorAddressBN = sponsorAddressBN.shr(31 * i);
+        paths.push(shiftedSponsorAddressBN.mask(31).toString());
+      }
+      const AIRSEEKER_PROTOCOL_ID = '5'; // From: https://github.com/api3dao/airnode/blob/ef16c54f33d455a1794e7886242567fc47ee14ef/packages/airnode-protocol/src/index.ts#L46
+      return `${AIRSEEKER_PROTOCOL_ID}/${paths.join('/')}`;
+    }
+
+    const deriveSponsorWallet = (sponsorWalletMnemonic, dapiName) => {
+      // Take first 20 bytes of dapiName as sponsor address together with the "0x" prefix.
+      const sponsorAddress = ethers.utils.getAddress(dapiName.slice(0, 42));
+      const sponsorWallet = ethers.Wallet.fromMnemonic(
+        sponsorWalletMnemonic,
+        `m/44'/60'/0'/${deriveWalletPathFromSponsorAddress(sponsorAddress)}`
+      );
+
+      return sponsorWallet;
+    };
+
     setInterval(async () => {
       const provider = new ethers.providers.JsonRpcProvider(rpcUrl);
       const dapiDataRegistry = new ethers.Contract(dapiDataRegistryAddress, dapiDataRegistryAbi, provider);
@@ -1580,7 +1603,7 @@ <h2>Active dAPIs</h2>
 
         const deviationPercentage = calculateUpdateInPercentage(value, newBeaconSetValue).toNumber() / 1e6;
         const deviationThresholdPercentage = deviationThresholdInPercentage.toNumber() / 1e6;
-
+        const sponsorWallet = deriveSponsorWallet(airseekerMnemonic, dapiName);
         const dapiInfo = {
           dapiName: dapiName,
           decodedDapiName: ethers.utils.parseBytes32String(dapiName),
@@ -1592,11 +1615,13 @@ <h2>Active dAPIs</h2>
           deviationPercentage:
             deviationPercentage > deviationThresholdPercentage ? `<b>${deviationPercentage}</b>` : deviationPercentage,
           deviationThresholdPercentage: deviationThresholdPercentage,
+          sponsorWalletAddress: sponsorWallet.address,
+          sponsorWalletBalance: ethers.utils.formatEther(await provider.getBalance(sponsorWallet.address)),
         };
 
         newActiveDapisHtml += JSON.stringify(dapiInfo, null, 2) + '\n\n';
       }
       document.getElementById('activeDapis').innerHTML = newActiveDapisHtml;
-    }, 1000);
+    }, 3000);
   </script>
 </html>

local-test-configuration/scripts/initialize-chain.ts

@@ -8,7 +8,7 @@ import {
 } from '@api3/airnode-protocol-v1';
 import { StandardMerkleTree } from '@openzeppelin/merkle-tree';
 import dotenv from 'dotenv';
-import type { Signer } from 'ethers';
+import type { ContractTransaction, Signer } from 'ethers';
 import { ethers } from 'ethers';
 import { zip } from 'lodash';
 
@@ -50,6 +50,36 @@ export const deriveRole = (adminRole: string, roleDescription: string) => {
   );
 };
 
+export const refundFunder = async (funderWallet: ethers.Wallet) => {
+  const airseekerSecrets = dotenv.parse(readFileSync(join(__dirname, `/../airseeker`, 'secrets.env'), 'utf8'));
+  const airseekerWalletMnemonic = airseekerSecrets.SPONSOR_WALLET_MNEMONIC;
+  if (!airseekerWalletMnemonic) throw new Error('SPONSOR_WALLET_MNEMONIC not found in Airseeker secrets');
+
+  // Initialize sponsor wallets
+  for (const beaconSetName of getBeaconSetNames()) {
+    const dapiName = ethers.utils.formatBytes32String(beaconSetName);
+
+    const sponsorWallet = deriveSponsorWallet(airseekerWalletMnemonic, dapiName);
+    const sponsorWalletBalance = await funderWallet.provider.getBalance(sponsorWallet.address);
+    console.info('Sponsor wallet balance:', ethers.utils.formatEther(sponsorWalletBalance.toString()));
+
+    const tx = await sponsorWallet.sendTransaction({
+      to: funderWallet.address,
+      value: sponsorWalletBalance,
+    });
+    await tx.wait();
+
+    console.info(`Refunding funder wallet from sponsor wallet`, {
+      dapiName,
+      sponsorWalletAddress: sponsorWallet.address,
+    });
+  }
+};
+
+const joinUrl = (url: string, path: string) => {
+  return new URL(path, url).href;
+};
+
 const loadPusherConfig = (pusherDir: 'pusher-1' | 'pusher-2') => {
   const configPath = join(__dirname, `/../`, pusherDir);
   const rawConfig = JSON.parse(readFileSync(join(configPath, 'pusher.json'), 'utf8'));
@@ -59,27 +89,38 @@ const loadPusherConfig = (pusherDir: 'pusher-1' | 'pusher-2') => {
   return interpolateSecrets(rawConfig, secrets);
 };
 
-export const fundAirseekerSponsorWallet = async (
-  funderWallet: ethers.Wallet,
-  { beaconSetNames }: { beaconSetNames: string[] }
-) => {
+const getBeaconSetNames = () => {
+  const pusher = loadPusherConfig('pusher-1');
+  const pusherWallet = ethers.Wallet.fromMnemonic(pusher.nodeSettings.airnodeWalletMnemonic);
+  const pusherBeacons = Object.values(pusher.templates).map((template: any) => {
+    return deriveBeaconData({ ...template, airnodeAddress: pusherWallet.address });
+  });
+
+  return pusherBeacons.map((beacon) => beacon.parameters[0]!.value);
+};
+
+export const fundAirseekerSponsorWallet = async (funderWallet: ethers.Wallet) => {
   const airseekerSecrets = dotenv.parse(readFileSync(join(__dirname, `/../airseeker`, 'secrets.env'), 'utf8'));
   const airseekerWalletMnemonic = airseekerSecrets.SPONSOR_WALLET_MNEMONIC;
   if (!airseekerWalletMnemonic) throw new Error('SPONSOR_WALLET_MNEMONIC not found in Airseeker secrets');
 
   // Initialize sponsor wallets
-  for (const beaconSetName of beaconSetNames) {
+  for (const beaconSetName of getBeaconSetNames()) {
     const dapiName = ethers.utils.formatBytes32String(beaconSetName);
 
-    const sponsor1Wallet = deriveSponsorWallet(airseekerWalletMnemonic, dapiName);
-    await funderWallet.sendTransaction({
-      to: sponsor1Wallet.address,
+    const sponsorWallet = deriveSponsorWallet(airseekerWalletMnemonic, dapiName);
+    const sponsorWalletBalance = await funderWallet.provider.getBalance(sponsorWallet.address);
+    console.info('Sponsor wallet balance:', ethers.utils.formatEther(sponsorWalletBalance.toString()));
+
+    const tx = await funderWallet.sendTransaction({
+      to: sponsorWallet.address,
       value: ethers.utils.parseEther('1'),
     });
+    await tx.wait();
 
     console.info(`Funding sponsor wallets`, {
       dapiName,
-      sponsor1WalletAddress: sponsor1Wallet.address,
+      sponsorWalletAddress: sponsorWallet.address,
     });
   }
 };
@@ -91,22 +132,25 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
     registryOwner = funderWallet,
     api3MarketContract = funderWallet,
     rootSigner1 = funderWallet,
-    randomPerson = funderWallet,
-    walletFunder = funderWallet;
+    randomPerson = funderWallet;
 
   // Deploy contracts
   const accessControlRegistryFactory = new AccessControlRegistryFactory(deployer as Signer);
   const accessControlRegistry = await accessControlRegistryFactory.deploy();
+  await accessControlRegistry.deployTransaction.wait();
   const api3ServerV1Factory = new Api3ServerV1Factory(deployer as Signer);
   const api3ServerV1AdminRoleDescription = 'Api3ServerV1 admin';
   const api3ServerV1 = await api3ServerV1Factory.deploy(
     accessControlRegistry.address,
     api3ServerV1AdminRoleDescription,
     manager.address
   );
+  await api3ServerV1.deployTransaction.wait();
   const hashRegistryFactory = new HashRegistryFactory(deployer as Signer);
   const hashRegistry = await hashRegistryFactory.deploy();
-  await hashRegistry.connect(deployer).transferOwnership(registryOwner.address);
+  await hashRegistry.deployTransaction.wait();
+  const transferOwnershipTx = await hashRegistry.connect(deployer).transferOwnership(registryOwner.address);
+  await transferOwnershipTx.wait();
   const dapiDataRegistryFactory = new DapiDataRegistryFactory(deployer as Signer);
   const dapiDataRegistryAdminRoleDescription = 'DapiDataRegistry admin';
   const dapiDataRegistry = await dapiDataRegistryFactory.deploy(
@@ -116,42 +160,44 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
     hashRegistry.address,
     api3ServerV1.address
   );
+  await dapiDataRegistry.deployTransaction.wait();
 
   // Set up roles
   const rootRole = deriveRootRole(manager.address);
   const dapiDataRegistryAdminRole = deriveRole(rootRole, dapiDataRegistryAdminRoleDescription);
   const dapiAdderRoleDescription = await dapiDataRegistry.DAPI_ADDER_ROLE_DESCRIPTION();
   const dapiAdderRole = deriveRole(dapiDataRegistryAdminRole, dapiAdderRoleDescription);
   const dapiRemoverRoleDescription = await dapiDataRegistry.DAPI_REMOVER_ROLE_DESCRIPTION();
-  await accessControlRegistry
+  let tx: ContractTransaction;
+  tx = await accessControlRegistry
     .connect(manager)
     .initializeRoleAndGrantToSender(rootRole, dapiDataRegistryAdminRoleDescription);
-  await accessControlRegistry
+  await tx.wait();
+  tx = await accessControlRegistry
     .connect(manager)
     .initializeRoleAndGrantToSender(dapiDataRegistryAdminRole, dapiAdderRoleDescription);
-  await accessControlRegistry
+  await tx.wait();
+  tx = await accessControlRegistry
     .connect(manager)
     .initializeRoleAndGrantToSender(dapiDataRegistryAdminRole, dapiRemoverRoleDescription);
-  await accessControlRegistry.connect(manager).grantRole(dapiAdderRole, api3MarketContract.address);
-  await accessControlRegistry
+  await tx.wait();
+  tx = await accessControlRegistry.connect(manager).grantRole(dapiAdderRole, api3MarketContract.address);
+  await tx.wait();
+  tx = await accessControlRegistry
     .connect(manager)
     .initializeRoleAndGrantToSender(rootRole, api3ServerV1AdminRoleDescription);
-  await accessControlRegistry
+  await tx.wait();
+  tx = await accessControlRegistry
     .connect(manager)
     .initializeRoleAndGrantToSender(
       await api3ServerV1.adminRole(),
       await api3ServerV1.DAPI_NAME_SETTER_ROLE_DESCRIPTION()
     );
-  await accessControlRegistry
+  await tx.wait();
+  tx = await accessControlRegistry
     .connect(manager)
     .grantRole(await api3ServerV1.dapiNameSetterRole(), dapiDataRegistry.address);
-
-  // Initialize special wallet for contract initialization
-  const airseekerInitializationWallet = ethers.Wallet.createRandom().connect(provider);
-  await walletFunder.sendTransaction({
-    to: airseekerInitializationWallet.address,
-    value: ethers.utils.parseEther('1'),
-  });
+  await tx.wait();
 
   // Create templates
   const pusher1 = loadPusherConfig('pusher-1');
@@ -174,8 +220,8 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
   // Register merkle tree hashes
   const timestamp = Math.floor(Date.now() / 1000);
   const apiTreeValues = [
-    [pusher1Wallet.address, `${pusher1.signedApis[0].url}/default`], // NOTE: Pusher pushes to the "/" of the signed API, but we need to query it additional path.
-    [pusher2Wallet.address, `${pusher2.signedApis[0].url}/default`], // NOTE: Pusher pushes to the "/" of the signed API, but we need to query it additional path.
+    [pusher1Wallet.address, joinUrl(pusher1.signedApis[0].url, 'default')], // NOTE: Pusher pushes to the "/" of the signed API, but we need to query it additional path.
+    [pusher2Wallet.address, joinUrl(pusher2.signedApis[0].url, 'default')], // NOTE: Pusher pushes to the "/" of the signed API, but we need to query it additional path.
   ] as const;
   const apiTree = StandardMerkleTree.of(apiTreeValues as any, ['address', 'string']);
   const apiHashType = ethers.utils.solidityKeccak256(['string'], ['Signed API URL Merkle tree root']);
@@ -186,18 +232,25 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
   const apiTreeRootSignatures = await Promise.all(
     rootSigners.map(async (rootSigner) => rootSigner.signMessage(apiMessages))
   );
-  await hashRegistry.connect(registryOwner).setupSigners(
+  tx = await hashRegistry.connect(registryOwner).setupSigners(
     apiHashType,
     rootSigners.map((rootSigner) => rootSigner.address)
   );
-  await hashRegistry.registerHash(apiHashType, apiTree.root, timestamp, apiTreeRootSignatures);
+  await tx.wait();
+  tx = await hashRegistry.registerHash(apiHashType, apiTree.root, timestamp, apiTreeRootSignatures);
+  await tx.wait();
 
   // Add dAPIs hashes
-  const dapiNamesInfo = zip(beaconSetNames, beaconSetIds).map(
-    ([beaconSetName, beaconSetId]) => [beaconSetName!, beaconSetId!, airseekerInitializationWallet.address] as const
-  );
+  const airseekerSecrets = dotenv.parse(readFileSync(join(__dirname, `/../airseeker`, 'secrets.env'), 'utf8'));
+  const airseekerWalletMnemonic = airseekerSecrets.SPONSOR_WALLET_MNEMONIC;
+  if (!airseekerWalletMnemonic) throw new Error('SPONSOR_WALLET_MNEMONIC not found in Airseeker secrets');
+  const dapiNamesInfo = zip(beaconSetNames, beaconSetIds).map(([beaconSetName, beaconSetId]) => {
+    const dapiName = ethers.utils.formatBytes32String(beaconSetName!);
+    const sponsorWallet = deriveSponsorWallet(airseekerWalletMnemonic, dapiName);
+    return [dapiName, beaconSetId!, sponsorWallet.address] as const;
+  });
   const dapiTreeValues = dapiNamesInfo.map(([dapiName, beaconSetId, sponsorWalletAddress]) => {
-    return [ethers.utils.formatBytes32String(dapiName), beaconSetId, sponsorWalletAddress];
+    return [dapiName, beaconSetId, sponsorWalletAddress];
   });
   const dapiTree = StandardMerkleTree.of(dapiTreeValues, ['bytes32', 'bytes32', 'address']);
   const dapiTreeRoot = dapiTree.root;
@@ -208,19 +261,22 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
   const dapiTreeRootSignatures = await Promise.all(
     rootSigners.map(async (rootSigner) => rootSigner.signMessage(dapiMessages))
   );
-  await hashRegistry.connect(registryOwner).setupSigners(
+  tx = await hashRegistry.connect(registryOwner).setupSigners(
     dapiHashType,
     rootSigners.map((rootSigner) => rootSigner.address)
   );
-  await hashRegistry.registerHash(dapiHashType, dapiTreeRoot, timestamp, dapiTreeRootSignatures);
+  await tx.wait();
+  tx = await hashRegistry.registerHash(dapiHashType, dapiTreeRoot, timestamp, dapiTreeRootSignatures);
+  await tx.wait();
 
   // Set active dAPIs
   const apiTreeRoot = apiTree.root;
   for (const [airnode, url] of apiTreeValues) {
     const apiTreeProof = apiTree.getProof([airnode, url]);
-    await dapiDataRegistry
+    tx = await dapiDataRegistry
       .connect(api3MarketContract)
       .registerAirnodeSignedApiUrl(airnode, url, apiTreeRoot, apiTreeProof);
+    await tx.wait();
   }
   const dapiInfos = zip(pusher1Beacons, pusher2Beacons).map(([pusher1Beacon, pusher2Beacon], i) => {
     return {
@@ -236,13 +292,14 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
       ['address[]', 'bytes32[]'],
       [airnodes, templateIds]
     );
-    await dapiDataRegistry.connect(randomPerson).registerDataFeed(encodedBeaconSetData);
+    tx = await dapiDataRegistry.connect(randomPerson).registerDataFeed(encodedBeaconSetData);
+    await tx.wait();
     const HUNDRED_PERCENT = 1e8;
-    const deviationThresholdInPercentage = ethers.BigNumber.from(HUNDRED_PERCENT / 1000); // 0.1%
+    const deviationThresholdInPercentage = ethers.BigNumber.from(HUNDRED_PERCENT / 100); // 1%
     const deviationReference = ethers.constants.Zero; // Not used in Airseeker V1
     const heartbeatInterval = ethers.BigNumber.from(86_400); // 24 hrs
     const [dapiName, beaconSetId, sponsorWalletMnemonic] = dapiTreeValue;
-    await dapiDataRegistry
+    tx = await dapiDataRegistry
       .connect(api3MarketContract)
       .addDapi(
         dapiName!,
@@ -254,6 +311,7 @@ export const deploy = async (funderWallet: ethers.Wallet, provider: ethers.provi
         dapiTree.root,
         dapiTree.getProof(dapiTreeValue)
       );
+    await tx.wait();
   }
 
   return {
@@ -282,12 +340,12 @@ async function main() {
   console.info('Funder balance:', ethers.utils.formatEther(balance.toString()));
   console.info();
 
-  const { beaconSetNames, api3ServerV1, dapiDataRegistry } = await deploy(funderWallet, provider);
+  const { api3ServerV1, dapiDataRegistry } = await deploy(funderWallet, provider);
   console.info('Api3ServerV1 deployed at:', api3ServerV1.address);
   console.info('DapiDataRegistry deployed at:', dapiDataRegistry.address);
   console.info();
 
-  await fundAirseekerSponsorWallet(funderWallet, { beaconSetNames });
+  await fundAirseekerSponsorWallet(funderWallet);
 }
 
 void main();