Handle API errors (#10)

* max result length

* try_aggregate_external

* report error

* error callback

* deployment

* deploy consumer

README.md

@@ -4,9 +4,9 @@
 
 ### On Sepolia
 
-XAPI: `0x06e1aCbb0A1d23F57798ca36eb3e20D552AcfC62`
+XAPI: `0x844EB21C712914cC25Fa0Df8330f02b58495B250`
 
-XAPI Consumer Example: `0xcd3627925b1396104126Fa400715A01C09703D66`
+XAPI Consumer Example: `0x64A4d18E4e61b8a7274A9BF3D424250c5ff1c7F8`
 
 ### On Near testnet
 

aggregator/package.json

@@ -14,7 +14,7 @@
     "test": "$npm_execpath run build && ava -- ./build/ormp_aggregator.wasm",
     "publish-test": "near contract call-function as-transaction ormpaggregator.guantong.testnet publish json-args {} prepaid-gas '300.0 Tgas' attached-deposit '0 NEAR' sign-as ormpaggregator.guantong.testnet network-config testnet sign-with-legacy-keychain send",
     "publish-external": "near contract call-function as-transaction ormpaggregator.guantong.testnet publish_external json-args '{\"request_id\": \"70021766616531051842153016788507494922593962344450640499185811462\",\"mpc_options\":{\"nonce\":\"2\",\"gas_limit\":\"1000000\",\"max_fee_per_gas\":\"305777608991\",\"max_priority_fee_per_gas\":\"2500000000\"}}' prepaid-gas '300.0 Tgas' attached-deposit '0 NEAR' sign-as guantong.testnet network-config testnet sign-with-legacy-keychain send",
-    "report": "near contract call-function as-transaction ormpaggregator.guantong.testnet report json-args '{\"request_id\":\"70021766616531051842153016788507494922593962344450640499185811462\",\"answers\":[{\"data_source_name\":\"test-source\",\"result\":\"test-result\"}],\"reward_address\":\"0x9F33a4809aA708d7a399fedBa514e0A0d15EfA85\"}' prepaid-gas '300.0 Tgas' attached-deposit '0 NEAR' sign-as guantong.testnet network-config testnet sign-with-legacy-keychain send",
+    "report": "near contract call-function as-transaction ormpaggregator.guantong.testnet report json-args '{\"request_id\":\"70021766616531051842153016788507494922593962344450640499185811462\",\"answers\":[{\"data_source_name\":\"test-source\",\"result\":\"test-resu\"}],\"reward_address\":\"0x9F33a4809aA708d7a399fedBa514e0A0d15EfA85\"}' prepaid-gas '300.0 Tgas' attached-deposit '0 NEAR' sign-as guantong.testnet network-config testnet sign-with-legacy-keychain send",
     "set-publish-config-event": "near contract call-function as-transaction ormpaggregator.guantong.testnet set_publish_chain_config json-args '{\"chain_id\":\"11155111\",\"xapi_address\":\"0x6984ebE378F8cb815546Cb68a98807C1fA121A81\",\"reporters_fee\":\"300\",\"publish_fee\":\"400\",\"reward_address\":\"0x9F33a4809aA708d7a399fedBa514e0A0d15EfA85\"}' prepaid-gas '100.0 Tgas' attached-deposit '0 NEAR' sign-as ormpaggregator.guantong.testnet network-config testnet sign-with-legacy-keychain send",
     "sync-publish-config": "near contract call-function as-transaction ormpaggregator.guantong.testnet sync_publish_config_to_remote json-args '{\"chain_id\": \"11155111\",\"mpc_options\":{\"nonce\":\"1\",\"gas_limit\":\"200000\",\"max_fee_per_gas\":\"305777608991\",\"max_priority_fee_per_gas\":\"2500000000\"}}' prepaid-gas '300.0 Tgas' attached-deposit '0 NEAR' sign-as guantong.testnet network-config testnet sign-with-legacy-keychain send",
     "add-data-source": "near contract call-function as-transaction ormpaggregator.guantong.testnet add_data_source json-args '{\"name\":\"httpbin\",\"url\":\"https://httpbin.org/anything\",\"method\":\"post\",\"headers\":{\"hello\":\"xapi\"},\"body_json\":{\"hello\":\"httpbin\"},\"result_path\":\"headers.host\",\"auth\":{\"place_path\":\"headers.Authorization\",\"value_path\":\"abcdefauth\"}}' prepaid-gas '100.0 Tgas' attached-deposit '0 NEAR' sign-as ormpaggregator.guantong.testnet network-config testnet sign-with-legacy-keychain send",

aggregator/src/abstract/aggregator.abstract.ts

@@ -78,10 +78,13 @@ export class DataSource {
 export class Answer {
   data_source_name: string;
   result: string;
+  // Set the error message if there is an error, leave it null/blank if there's no error.
+  error: string;
 
-  constructor({ data_source_name, result }: { data_source_name: string, result: string }) {
+  constructor({ data_source_name, result, error }: { data_source_name: string, result: string, error: string }) {
     this.data_source_name = data_source_name;
     this.result = result;
+    this.error = error;
   }
 }
 
@@ -183,11 +186,14 @@ export class Response {
 
   // 👇 These values should be aggregated from reporter's answer
   result: string;
+  // Leave it 0 if there's no error. MAX: 65535(uint16)
+  error_code: number;
 
   constructor(request_id: RequestId) {
     this.request_id = request_id;
     this.started_at = near.blockTimestamp().toString();
     this.status = RequestStatus[RequestStatus.FETCHING];
+    this.error_code = 0;
   }
 }
 
@@ -237,6 +243,8 @@ const DERIVATION_PATH_PREFIX = "XAPI";
 export abstract class Aggregator extends ContractBase {
   description: string;
   latest_request_id: RequestId;
+  // The max length of the report answer. Preventing gas limit being exceeded when publishing.
+  max_result_length: number;
 
   mpc_config: MpcConfig;
   staking_contract: AccountId;
@@ -254,6 +262,7 @@ export abstract class Aggregator extends ContractBase {
   constructor({ description, mpc_config, reporter_required, staking_contract, contract_metadata, }: { description: string, mpc_config: MpcConfig, reporter_required: ReporterRequired, staking_contract: AccountId, contract_metadata: ContractSourceMetadata }) {
     super(contract_metadata);
     this.description = description;
+    this.max_result_length = 500;
     this.mpc_config = mpc_config;
     this.reporter_required = reporter_required;
     this.staking_contract = staking_contract;
@@ -271,6 +280,19 @@ export abstract class Aggregator extends ContractBase {
     return this.description;
   }
 
+  abstract set_max_result_length({ max_result_length }: { max_result_length: number }): void;
+  _set_max_result_length({ max_result_length }: { max_result_length: number }): void {
+    this._assert_operator();
+    max_result_length = Number(max_result_length);
+    assert(max_result_length > 0, "max_result_length should be greater than 0");
+    this.max_result_length = max_result_length;
+  }
+
+  abstract get_max_result_length(): number;
+  _get_max_result_length(): number {
+    return this.max_result_length;
+  }
+
   abstract set_mpc_config(mpc_config: MpcConfig): void;
   _set_mpc_config(mpc_config: MpcConfig): void {
     this._assert_operator();
@@ -439,6 +461,10 @@ export abstract class Aggregator extends ContractBase {
     assert(answers != null && answers.length > 0, "answers is empty");
     assert(reward_address != null, "reward_address is null");
 
+    for (let i = 0; i < answers.length; i++) {
+      assert(answers[i].result.length <= this.max_result_length, `answers[${i}].result.length > ${this.max_result_length}`);
+    }
+
     const __report = new Report({
       request_id,
       answers,
@@ -538,8 +564,9 @@ export abstract class Aggregator extends ContractBase {
   }
 
   abstract _can_aggregate({ request_id }: { request_id: RequestId }): boolean;
-  abstract _aggregate({ request_id, top_staked }: { request_id: RequestId, top_staked: Staked[] }): boolean;
+  abstract _aggregate({ request_id, top_staked }: { request_id: RequestId, top_staked: Staked[] }): void;
 
+  abstract try_aggregate_external({ request_id }: { request_id: RequestId }): NearPromise;
   _try_aggregate({ request_id }: { request_id: RequestId }): NearPromise {
     if (this._can_aggregate({ request_id })) {
       near.log("try_aggregate: ", request_id);
@@ -601,12 +628,13 @@ export abstract class Aggregator extends ContractBase {
 
     // Relay it https://sepolia.etherscan.io/tx/0xfe2e2e0018f609b5d10250a823f191942fc42d597ad1cccfb4842f43f1d9196e
     const function_call_data = encodePublishCall({
-      functionSignature: "fulfill(uint256,(address[],bytes))",
+      functionSignature: "fulfill(uint256,(address[],bytes,uint16))",
       params: [
         BigInt(request_id),
         [
           _response.reporter_reward_addresses,
-          stringToBytes(_response.result)
+          stringToBytes(_response.result),
+          _response.error_code
         ]
       ]
     })

aggregator/src/lib/ethereum.ts

@@ -146,18 +146,24 @@ export function encodePublishCall({ functionSignature, params }: { functionSigna
     const bytesParams = params[1][1]
     const bytesValue = encodeParameter('bytes', bytesParams);
 
+    // encode uint16
+    const uint16Value = encodeParameter('uint256', params[1][2]);
+
     // offset
     const offsetTuple = (64).toString(16).padStart(64, '0');
     const offsetAddresses = (64).toString(16).padStart(64, '0');
     const offsetBytes = ((64 + addressesData.length / 2).toString(16)).padStart(64, '0');
+    const offsetUint16 = ((64 + addressesData.length / 2 + bytesValue.length / 2).toString(16)).padStart(64, '0');
 
     const encodeParams = [
         encodeParameter("uint256", params[0]),
         offsetTuple,
         offsetAddresses,
         offsetBytes,
+        offsetUint16,
         addressesData,
-        bytesValue
+        bytesValue,
+        uint16Value
     ].join('');
     return selector + encodeParams;
 }
@@ -180,6 +186,9 @@ export function encodeSetConfigCall({ functionSignature, params }: { functionSig
 }
 
 export function stringToBytes(str: string): string {
+    if (!str) {
+        return "0x";
+    }
     const bytes: number[] = [];
     for (let i = 0; i < str.length; i++) {
         bytes.push(str.charCodeAt(i));

aggregator/src/ormp.aggregator.ts

@@ -30,7 +30,7 @@ class OrmpAggregator extends Aggregator {
     return this.report_lookup.get(request_id).length >= this.reporter_required.threshold;
   }
 
-  _aggregate({ request_id, top_staked }: { request_id: RequestId, top_staked: Staked[] }): boolean {
+  _aggregate({ request_id, top_staked }: { request_id: RequestId, top_staked: Staked[] }): void {
     // 1. Filter invalid reports via top staked reporters
     const _reporters = this.report_lookup.get(request_id).map(r => r.reporter);
     const _valid_reporters = _reporters.filter(reporter =>
@@ -43,37 +43,63 @@ class OrmpAggregator extends Aggregator {
     const _each_reporter_report = [];
     const _each_reporter_result = new Map<string, string>();
 
+    let _error_reporters = [];
+    let _error = "";
+
     // 2. Aggregate from multi datasources of one report.
     _valid_reports.forEach(report => {
+      // calc error reports
+      for (const a of report.answers) {
+        if (a.error) {
+          _error = a.error;
+          _error_reporters.push(report.reporter);
+          break;
+        }
+      }
       const _result = this._aggregate_answer(report.answers.map(r => r.result)).result;
       _each_reporter_result.set(report.reporter, _result);
       _each_reporter_report.push(_result);
     });
 
-    // 3. Aggregate from multi reports
-    const most_common_report = this._aggregate_answer(_each_reporter_report);
-    near.log("most_common_report: ", most_common_report);
-
-    const result = most_common_report.result;
-    assert(_valid_reporters.length >= this.reporter_required.quorum, `Quorum: required ${this.reporter_required.quorum}, but got ${_valid_reporters.length}`);
-    assert(most_common_report.count >= this.reporter_required.threshold, `Threshold: required ${this.reporter_required.threshold}, but got ${most_common_report.count}`)
-
-    const _response = this.response_lookup.get(request_id);
-
-    _response.valid_reporters = [];
-    _response.reporter_reward_addresses = []
-    // 4. Filter most common reporter, only most common report will be rewarded
-    for (const _report of _valid_reports) {
-      const key = _each_reporter_result.get(_report.reporter);
-      if (key === most_common_report.result) {
-        _response.reporter_reward_addresses.push(_report.reward_address);
-        _response.valid_reporters.push(_report.reporter);
+    if (_error_reporters.length >= this.reporter_required.threshold) {
+      // 3. Handle error
+      const _response = this.response_lookup.get(request_id);
+
+      _response.valid_reporters = _error_reporters;
+      _response.reporter_reward_addresses = []
+      // 4. Only error report will be rewarded
+      for (const _report of _valid_reports) {
+        if (_error_reporters.includes(_report.reporter)) {
+          _response.reporter_reward_addresses.push(_report.reward_address);
+        }
+      }
+      _response.error_code = 1;
+      this.response_lookup.set(request_id, _response);
+    } else {
+      // 3. Aggregate from multi reports
+      const most_common_report = this._aggregate_answer(_each_reporter_report);
+      near.log("most_common_report: ", most_common_report);
+
+      const result = most_common_report.result;
+      assert(_valid_reporters.length >= this.reporter_required.quorum, `Quorum: required ${this.reporter_required.quorum}, but got ${_valid_reporters.length}`);
+      assert(most_common_report.count >= this.reporter_required.threshold, `Threshold: required ${this.reporter_required.threshold}, but got ${most_common_report.count}`)
+
+      const _response = this.response_lookup.get(request_id);
+
+      _response.valid_reporters = [];
+      _response.reporter_reward_addresses = []
+      // 4. Filter most common reporter, only most common report will be rewarded
+      for (const _report of _valid_reports) {
+        const key = _each_reporter_result.get(_report.reporter);
+        if (key === most_common_report.result) {
+          _response.reporter_reward_addresses.push(_report.reward_address);
+          _response.valid_reporters.push(_report.reporter);
+        }
       }
-    }
 
-    _response.result = result;
-    this.response_lookup.set(request_id, _response);
-    return true;
+      _response.result = result;
+      this.response_lookup.set(request_id, _response);
+    }
   }
 
   _aggregate_answer(answers: string[]): { result: string, count: number } {
@@ -123,6 +149,11 @@ class OrmpAggregator extends Aggregator {
     return super._report({ request_id, answers, reward_address });
   }
 
+  @call({ payableFunction: true })
+  try_aggregate_external({ request_id }: { request_id: RequestId; }): NearPromise {
+    return super._try_aggregate({ request_id });
+  }
+
   @call({ payableFunction: true })
   sync_publish_config_to_remote({ chain_id, mpc_options }: { chain_id: ChainId; mpc_options: MpcOptions; }): NearPromise {
     return super._sync_publish_config_to_remote({ chain_id, mpc_options });
@@ -163,6 +194,11 @@ class OrmpAggregator extends Aggregator {
     super._set_staking_contract({ staking_contract });
   }
 
+  @call({})
+  set_max_result_length({ max_result_length }: { max_result_length: number; }): void {
+    super._set_max_result_length({ max_result_length });
+  }
+
   /// Views
 
   @view({})
@@ -206,6 +242,10 @@ class OrmpAggregator extends Aggregator {
     return super._get_data_sources()
   }
   @view({})
+  get_max_result_length(): number {
+    return super._get_max_result_length();
+  }
+  @view({})
   contract_source_metadata(): ContractSourceMetadata {
     return super._contract_source_metadata();
   }

xapi-consumer/contracts/ConsumerExample.sol

@@ -1,13 +1,14 @@
 // SPDX-License-Identifier: UNLICENSED
 pragma solidity ^0.8.24;
 
+import "./interfaces/IXAPIConsumer.sol";
 import "xapi/contracts/interfaces/IXAPI.sol";
 
-contract ConsumerExample {
+contract ConsumerExample is IXAPIConsumer {
     IXAPI public xapi;
 
     event RequestMade(uint256 requestId, string requestData);
-    event ConsumeResult(uint256 requestId, bytes responseData);
+    event ConsumeResult(uint256 requestId, bytes responseData, uint16 errorCode);
 
     constructor(address xapiAddress) {
         xapi = IXAPI(xapiAddress);
@@ -19,13 +20,13 @@ contract ConsumerExample {
 
     function makeRequest(address aggregator) external payable {
         string memory requestData = "{'hello':'world'}";
-        uint256 requestId = xapi.makeRequest{value: msg.value}(requestData, this.fulfillCallback.selector, aggregator);
+        uint256 requestId = xapi.makeRequest{value: msg.value}(aggregator, requestData, this.xapiCallback.selector);
         emit RequestMade(requestId, requestData);
     }
 
-    function fulfillCallback(uint256 requestId, ResponseData memory response) external {
+    function xapiCallback(uint256 requestId, ResponseData memory response) external {
         require(msg.sender == address(xapi), "Only XAPI can call this function");
 
-        emit ConsumeResult(requestId, response.result);
+        emit ConsumeResult(requestId, response.result, response.errorCode);
     }
-}
+}

xapi-consumer/contracts/interfaces/IXAPIConsumer.sol

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: UNLICENSED
+pragma solidity ^0.8.24;
+
+import "xapi/contracts/interfaces/IXAPI.sol";
+
+interface IXAPIConsumer {
+    function xapiCallback(uint256 requestId, ResponseData memory response) external;
+}

xapi-consumer/ignition/deployments/chain-11155111/deployed_addresses.json

@@ -1,3 +1,3 @@
 {
-  "ConsumerExampleModule#ConsumerExample": "0xcd3627925b1396104126Fa400715A01C09703D66"
+  "ConsumerExampleModule#ConsumerExample": "0x64A4d18E4e61b8a7274A9BF3D424250c5ff1c7F8"
 }

xapi-consumer/ignition/modules/ConsumerExample.deploy.js

@@ -3,7 +3,7 @@ const { buildModule } = require("@nomicfoundation/hardhat-ignition/modules");
 module.exports = buildModule("ConsumerExampleModule", (m) => {
     const deployer = m.getAccount(0);
 
-    const consumer = m.contract("ConsumerExample", ["0xcedBd5b942c37870D172fEF4FC1C568C0aB8c038"], {
+    const consumer = m.contract("ConsumerExample", ["0x844EB21C712914cC25Fa0Df8330f02b58495B250"], {
         from: deployer
     });
 

xapi-consumer/package.json

@@ -7,7 +7,7 @@
     "build": "npx hardhat compile",
     "deploy": "npx hardhat ignition deploy ./ignition/modules/ConsumerExample.deploy.js --network sepolia",
     "console": "npx hardhat console --network sepolia",
-    "verify": "npx hardhat verify --network sepolia 0xcd3627925b1396104126Fa400715A01C09703D66 '0xcedBd5b942c37870D172fEF4FC1C568C0aB8c038'"
+    "verify": "npx hardhat verify --network sepolia 0x64A4d18E4e61b8a7274A9BF3D424250c5ff1c7F8 '0x844EB21C712914cC25Fa0Df8330f02b58495B250'"
   },
   "dependencies": {
     "xapi": "file:../xapi"

xapi-consumer/yarn.lock

@@ -659,9 +659,9 @@
     "@nomicfoundation/solidity-analyzer-win32-x64-msvc" "0.1.2"
 
 "@openzeppelin/contracts@^5.0.2":
-  version "5.0.2"
-  resolved "https://registry.yarnpkg.com/@openzeppelin/contracts/-/contracts-5.0.2.tgz#b1d03075e49290d06570b2fd42154d76c2a5d210"
-  integrity sha512-ytPc6eLGcHHnapAZ9S+5qsdomhjo6QBHTDRRBFfTxXIpsicMhVPouPgmUPebZZZGX7vt9USA+Z+0M0dSVtSUEA==
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/@openzeppelin/contracts/-/contracts-5.1.0.tgz#4e61162f2a2bf414c4e10c45eca98ce5f1aadbd4"
+  integrity sha512-p1ULhl7BXzjjbha5aqst+QMLY+4/LCWADXOCsmLHRM77AqiPjnd9vvUN9sosUfhL9JGKpZ0TjEGxgvnizmWGSA==
 
 "@scure/base@~1.1.0", "@scure/base@~1.1.6":
   version "1.1.9"

xapi/contracts/XAPI.sol

@@ -12,7 +12,7 @@ contract XAPI is IXAPI, Ownable2Step {
 
     constructor() Ownable(msg.sender) {}
 
-    function makeRequest(string memory requestData, bytes4 callbackFunction, address exAggregator)
+    function makeRequest(address exAggregator, string memory requestData, bytes4 callbackFunction)
         external
         payable
         returns (uint256)
@@ -35,7 +35,7 @@ contract XAPI is IXAPI, Ownable2Step {
             payment: msg.value,
             aggregator: aggregatorConfig.aggregator,
             exAggregator: exAggregator,
-            response: ResponseData({reporters: new address[](0), result: new bytes(0)}),
+            response: ResponseData({reporters: new address[](0), result: new bytes(0), errorCode: 0}),
             requestData: requestData
         });
         emit RequestMade(