Implement a benchmarking mode

.github/workflows/common_checks.yaml

@@ -63,7 +63,7 @@ jobs:
     - name: Install dependencies
       run: pip install tomte[tox,cli]==0.2.14
     - name: Check copyright headers
-      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription
+      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci
     - name: License compatibility check
       run: tox -e liccheck
     - name: Check dependencies

Makefile

@@ -74,13 +74,13 @@ security:
 .PHONY: generators
 generators: clean-cache fix-abci-app-specs
 	tox -e abci-docstrings
-	tomte format-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription
+	tomte format-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci
 	autonomy packages lock
 	tox -e fix-doc-hashes
 
 .PHONY: common-checks-1
 common-checks-1:
-	tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription
+	tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci
 	tomte check-doc-links
 	tox -p -e check-hash -e check-packages -e check-doc-hashes -e analyse-service
 

README.md

@@ -57,7 +57,7 @@ Next, prepare the [Safe](https://safe.global/). The trader agent runs as part of
 which is an [autonomous service](https://docs.autonolas.network/open-autonomy/get_started/what_is_an_agent_service/) 
 represented on-chain in the [Autonolas Protocol](https://docs.autonolas.network/protocol/) by a [Safe](https://safe.global/) multisig. Follow the next steps to obtain a **Safe address** corresponding to your agent address:
 
-1. Visit https://registry.olas.network/services/mint and connect to the Gnosis network. We recommend connecting using a wallet with a Gnosis EOA account that you own.
+1. Visit https://registry.olas.network/gnosis/services/mint and connect to the Gnosis network. We recommend connecting using a wallet with a Gnosis EOA account that you own.
 2. Fill in the following fields:
     - *"Owner address"*: a Gnosis address for which you will be able to sign later using a supported wallet. If you want to use the address you are connected to, click on *"Prefill Address"*.
     - Click on *"Generate Hash & File"* and enter the value corresponding to the `service/valory/trader/0.1.0` key in [`packages.json`](https://github.com/valory-xyz/trader/blob/main/packages/packages.json)

packages/packages.json

@@ -12,13 +12,13 @@
         "contract/valory/service_staking_token/0.1.0": "bafybeid44l7qekvkwkvmfl4kcqchnaktttacp7lbx464mzqqs5cnefj35e",
         "contract/valory/transfer_nft_condition/0.1.0": "bafybeicgpoag2lymofz3vnen76q7gtig5hzimn32o57php4uerr6t25em4",
         "skill/valory/market_manager_abci/0.1.0": "bafybeidygkw7mwhbk3ry3au5c5265vms5eti375v5jthd4be5dfnnoache",
-        "skill/valory/decision_maker_abci/0.1.0": "bafybeieg3rkt7tfyam6kunhshnyoe5mo3cat7odph5mm6lswej5un3i3my",
-        "skill/valory/trader_abci/0.1.0": "bafybeidlsd5dgupxnonyflp5gvjhxtmycn4k3h7kry5dqmjp2g23s2imea",
-        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeictjqqnq5gxtxnripxefu6yd6foqw7ovgbr52nnqtldtgdk7jnxei",
-        "skill/valory/staking_abci/0.1.0": "bafybeicrp3dbvxcb2wa7kdgpkxu7aqji5fwq6icwcfw7anfcpqv26irsr4",
-        "skill/valory/check_stop_trading_abci/0.1.0": "bafybeibry7nmbug5bdkl4gza537wolyn72ju52v2patsfxovqe6c7c4nf4",
-        "agent/valory/trader/0.1.0": "bafybeigxipjd76mgsvi53ntvaxzo2r3rwffhbysoc6egjzlt67fcacceyu",
-        "service/valory/trader/0.1.0": "bafybeihyvufjuzzovbtxl747onc7axv625di76tknqjqouoaccluiha2ia"
+        "skill/valory/decision_maker_abci/0.1.0": "bafybeid4txlxu6ig7o3r6ufrsbegnjzsnunmrr5ozy6lv54yzvx6eg5vey",
+        "skill/valory/trader_abci/0.1.0": "bafybeibozxamrqtds3wxiye3fb3tz2uwtw7l7mydxqsmio4aldshtseig4",
+        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeifssqoybhgmuz6gwylkayajmzn763qsv6uqduzhe4ynbgnrteb7jq",
+        "skill/valory/staking_abci/0.1.0": "bafybeicsydq6fdansf7qrmrygzchl3h6rtkdw5rmx2jyrwecj4laj5nehy",
+        "skill/valory/check_stop_trading_abci/0.1.0": "bafybeidyc5fvw5wosbc3anxxxog5b67cfmvrsrltjh3cfllye3bb43r3z4",
+        "agent/valory/trader/0.1.0": "bafybeicnz4vh3tzszfu6zgpn433ipzd7evj6vtxlwnsfaahruav6pivijm",
+        "service/valory/trader/0.1.0": "bafybeihugrkov7ihw7u2hzv6cq4mrgdxjvjklqqkskfl7jrv65fdhtqnbu"
     },
     "third_party": {
         "protocol/open_aea/signing/1.0.0": "bafybeihv62fim3wl2bayavfcg3u5e5cxu3b7brtu4cn5xoxd6lqwachasi",

packages/valory/agents/trader/aea-config.yaml

@@ -27,6 +27,7 @@ contracts:
 - valory/agent_registry:0.1.0:bafybeibedc7ehebk3ikr4cowjbvgpxqpu65nforgqmraxqxiq5jv6rboqe
 - valory/service_staking_token:0.1.0:bafybeid44l7qekvkwkvmfl4kcqchnaktttacp7lbx464mzqqs5cnefj35e
 - valory/transfer_nft_condition:0.1.0:bafybeicgpoag2lymofz3vnen76q7gtig5hzimn32o57php4uerr6t25em4
+- valory/erc20:0.1.0:bafybeigvftdxjgnlsoemst5d57cor36idywk7bwcfj2bjqijxdxo3xpurq
 protocols:
 - open_aea/signing:1.0.0:bafybeihv62fim3wl2bayavfcg3u5e5cxu3b7brtu4cn5xoxd6lqwachasi
 - valory/abci:0.1.0:bafybeiaqmp7kocbfdboksayeqhkbrynvlfzsx4uy4x6nohywnmaig4an7u
@@ -43,12 +44,12 @@ skills:
 - valory/reset_pause_abci:0.1.0:bafybeidw4mbx3os3hmv7ley7b3g3gja7ydpitr7mxbjpwzxin2mzyt5yam
 - valory/termination_abci:0.1.0:bafybeihq6qtbwt6i53ayqym63vhjexkcppy26gguzhhjqywfmiuqghvv44
 - valory/transaction_settlement_abci:0.1.0:bafybeigtzlk4uakmd54rxnznorcrstsr52kta474lgrnvx5ovr546vj7sq
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeictjqqnq5gxtxnripxefu6yd6foqw7ovgbr52nnqtldtgdk7jnxei
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeifssqoybhgmuz6gwylkayajmzn763qsv6uqduzhe4ynbgnrteb7jq
 - valory/market_manager_abci:0.1.0:bafybeidygkw7mwhbk3ry3au5c5265vms5eti375v5jthd4be5dfnnoache
-- valory/decision_maker_abci:0.1.0:bafybeieg3rkt7tfyam6kunhshnyoe5mo3cat7odph5mm6lswej5un3i3my
-- valory/trader_abci:0.1.0:bafybeidlsd5dgupxnonyflp5gvjhxtmycn4k3h7kry5dqmjp2g23s2imea
-- valory/staking_abci:0.1.0:bafybeicrp3dbvxcb2wa7kdgpkxu7aqji5fwq6icwcfw7anfcpqv26irsr4
-- valory/check_stop_trading_abci:0.1.0:bafybeibry7nmbug5bdkl4gza537wolyn72ju52v2patsfxovqe6c7c4nf4
+- valory/decision_maker_abci:0.1.0:bafybeid4txlxu6ig7o3r6ufrsbegnjzsnunmrr5ozy6lv54yzvx6eg5vey
+- valory/trader_abci:0.1.0:bafybeibozxamrqtds3wxiye3fb3tz2uwtw7l7mydxqsmio4aldshtseig4
+- valory/staking_abci:0.1.0:bafybeicsydq6fdansf7qrmrygzchl3h6rtkdw5rmx2jyrwecj4laj5nehy
+- valory/check_stop_trading_abci:0.1.0:bafybeidyc5fvw5wosbc3anxxxog5b67cfmvrsrltjh3cfllye3bb43r3z4
 - valory/mech_interact_abci:0.1.0:bafybeigkvcluq2kejpxdcb54iwqtvwhov5elg3cv4v2yomwjxyu5u7g7hi
 customs:
 - valory/mike_strat:0.1.0:bafybeihjiol7f4ch4piwfikurdtfwzsh6qydkbsztpbwbwb2yrqdqf726m
@@ -199,7 +200,7 @@ models:
       slippage: ${float:0.01}
       redeem_round_timeout: ${float:3600.0}
       policy_epsilon: ${float:0.1}
-      policy_store_path: ${str:/data/}
+      store_path: ${str:/data/}
       irrelevant_tools: ${list:["openai-text-davinci-002", "openai-text-davinci-003",
         "openai-gpt-3.5-turbo", "openai-gpt-4", "stabilityai-stable-diffusion-v1-5",
         "stabilityai-stable-diffusion-xl-beta-v2-2-2", "stabilityai-stable-diffusion-512-v2-1",
@@ -223,6 +224,24 @@ models:
       contract_timeout: ${float:300.0}
       file_hash_to_strategies_json: ${list:[["bafybeihufqu2ra7vud4h6g2nwahx7mvdido7ff6prwnib2tdlc4np7dw24",["bet_amount_per_threshold"]],["bafybeif55cu7cf6znyma7kxus4wxa2doarhau2xmndo57iegshxorivwmq",["kelly_criterion"]]]}
       strategies_kwargs: ${list:[["bet_kelly_fraction",0.5],["floor_balance",500000000000000000],["bet_amount_per_threshold",{"0.0":0,"0.1":0,"0.2":0,"0.3":0,"0.4":0,"0.5":0,"0.6":60000000000000000,"0.7":90000000000000000,"0.8":100000000000000000,"0.9":1000000000000000000,"1.0":10000000000000000000}]]}
+  benchmarking_mode:
+    args:
+      enabled: ${bool:false}
+      native_balance: ${int:10000000000000000000}
+      collateral_balance: ${int:10000000000000000000}
+      mech_cost: ${int:10000000000000000}
+      pool_fee: ${int:20000000000000000}
+      sep: ${str:,}
+      dataset_filename: ${str:benchmark_data.csv}
+      question_field: ${str:question}
+      question_id_field: ${str:question_id}
+      answer_field: ${str:answer}
+      p_yes_field_part: ${str:p_yes_}
+      p_no_field_part: ${str:p_no_}
+      confidence_field_part: ${str:confidence_}
+      part_prefix_mode: ${bool:true}
+      bet_amount_field: ${str:collateral_amount}
+      results_filename: ${str:benchmarking_results.csv}
 ---
 public_id: valory/p2p_libp2p_client:0.1.0
 type: connection

packages/valory/services/trader/service.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeigtuothskwyvrhfosps2bu6suauycolj67dpuxqvnicdrdu7yhtvq
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeigxipjd76mgsvi53ntvaxzo2r3rwffhbysoc6egjzlt67fcacceyu
+agent: valory/trader:0.1.0:bafybeicnz4vh3tzszfu6zgpn433ipzd7evj6vtxlwnsfaahruav6pivijm
 number_of_agents: 4
 deployment: {}
 ---
@@ -104,7 +104,7 @@ type: skill
         redeeming_batch_size: ${REDEEMING_BATCH_SIZE:int:1}
         slippage: ${SLIPPAGE:float:0.01}
         policy_epsilon: ${POLICY_EPSILON:float:0.1}
-        policy_store_path: ${POLICY_STORE_PATH:str:/data/}
+        store_path: ${STORE_PATH:str:/data/}
         irrelevant_tools: ${IRRELEVANT_TOOLS:list:["openai-text-davinci-002", "openai-text-davinci-003",
           "openai-gpt-3.5-turbo", "openai-gpt-4", "stabilityai-stable-diffusion-v1-5",
           "stabilityai-stable-diffusion-xl-beta-v2-2-2", "stabilityai-stable-diffusion-512-v2-1",
@@ -133,6 +133,24 @@ type: skill
     benchmark_tool: &id004
       args:
         log_dir: ${LOG_DIR:str:/benchmarks}
+    benchmarking_mode:
+      args:
+        enabled: ${BENCHMARKING_MODE_ENABLED:bool:false}
+        native_balance: ${BENCHMARKING_MODE_NATIVE_BALANCE:int:10000000000000000000}
+        collateral_balance: ${BENCHMARKING_MODE_COLLATERAL_BALANCE:int:10000000000000000000}
+        mech_cost: ${BENCHMARKING_MODE_MECH_COST:int:10000000000000000}
+        pool_fee: ${BENCHMARKING_MODE_POOL_FEE:int:20000000000000000}
+        sep: ${BENCHMARKING_MODE_SEP:str:,}
+        dataset_filename: ${BENCHMARKING_MODE_DATASET_FILENAME:str:benchmark_data.csv}
+        question_field: ${BENCHMARKING_MODE_QUESTION_FIELD:str:question}
+        question_id_field: ${BENCHMARKING_MODE_QUESTION_ID_FIELD:str:question_id}
+        answer_field: ${BENCHMARKING_MODE_ANSWER_FIELD:str:answer}
+        p_yes_field_part: ${BENCHMARKING_MODE_P_YES_FIELD_PART:str:p_yes_}
+        p_no_field_part: ${BENCHMARKING_MODE_P_NO_FIELD_PART:str:p_no_}
+        confidence_field_part: ${BENCHMARKING_MODE_CONFIDENCE_FIELD_PART:str:confidence_}
+        part_prefix_mode: ${BENCHMARKING_MODE_PART_PREFIX_MODE:bool:true}
+        bet_amount_field: ${BENCHMARKING_MODE_BET_AMOUNT_FIELD:str:collateral_amount}
+        results_filename: ${BENCHMARKING_MODE_RESULTS_FILENAME:str:benchmarking_results.csv}
 1:
   models:
     params:
@@ -205,7 +223,7 @@ type: skill
         redeeming_batch_size: ${REDEEMING_BATCH_SIZE:int:1}
         slippage: ${SLIPPAGE:float:0.01}
         policy_epsilon: ${POLICY_EPSILON:float:0.1}
-        policy_store_path: ${POLICY_STORE_PATH:str:/data/}
+        store_path: ${STORE_PATH:str:/data/}
         irrelevant_tools: ${IRRELEVANT_TOOLS:list:["openai-text-davinci-002", "openai-text-davinci-003",
           "openai-gpt-3.5-turbo", "openai-gpt-4", "stabilityai-stable-diffusion-v1-5",
           "stabilityai-stable-diffusion-xl-beta-v2-2-2", "stabilityai-stable-diffusion-512-v2-1",
@@ -304,7 +322,7 @@ type: skill
         redeeming_batch_size: ${REDEEMING_BATCH_SIZE:int:1}
         slippage: ${SLIPPAGE:float:0.01}
         policy_epsilon: ${POLICY_EPSILON:float:0.1}
-        policy_store_path: ${POLICY_STORE_PATH:str:/data/}
+        store_path: ${STORE_PATH:str:/data/}
         irrelevant_tools: ${IRRELEVANT_TOOLS:list:["openai-text-davinci-002", "openai-text-davinci-003",
           "openai-gpt-3.5-turbo", "openai-gpt-4", "stabilityai-stable-diffusion-v1-5",
           "stabilityai-stable-diffusion-xl-beta-v2-2-2", "stabilityai-stable-diffusion-512-v2-1",
@@ -403,7 +421,7 @@ type: skill
         redeeming_batch_size: ${REDEEMING_BATCH_SIZE:int:1}
         slippage: ${SLIPPAGE:float:0.01}
         policy_epsilon: ${POLICY_EPSILON:float:0.1}
-        policy_store_path: ${POLICY_STORE_PATH:str:/data/}
+        store_path: ${STORE_PATH:str:/data/}
         irrelevant_tools: ${IRRELEVANT_TOOLS:list:["openai-text-davinci-002", "openai-text-davinci-003",
           "openai-gpt-3.5-turbo", "openai-gpt-4", "stabilityai-stable-diffusion-v1-5",
           "stabilityai-stable-diffusion-xl-beta-v2-2-2", "stabilityai-stable-diffusion-512-v2-1",

packages/valory/skills/check_stop_trading_abci/behaviours.py

@@ -23,20 +23,22 @@
 from typing import Generator, Set, Type, cast
 
 from packages.valory.contracts.mech.contract import Mech as MechContract
+from packages.valory.contracts.service_staking_token.contract import StakingState
 from packages.valory.skills.abstract_round_abci.base import get_name
 from packages.valory.skills.abstract_round_abci.behaviour_utils import BaseBehaviour
 from packages.valory.skills.abstract_round_abci.behaviours import AbstractRoundBehaviour
 from packages.valory.skills.check_stop_trading_abci.models import CheckStopTradingParams
-from packages.valory.skills.check_stop_trading_abci.payloads import CheckStopTradingPayload
+from packages.valory.skills.check_stop_trading_abci.payloads import (
+    CheckStopTradingPayload,
+)
 from packages.valory.skills.check_stop_trading_abci.rounds import (
-    CheckStopTradingRound,
     CheckStopTradingAbciApp,
+    CheckStopTradingRound,
 )
 from packages.valory.skills.staking_abci.behaviours import (
     StakingInteractBaseBehaviour,
     WaitableConditionType,
 )
-from packages.valory.contracts.service_staking_token.contract import StakingState
 
 
 # Liveness ratio from the staking contract is expressed in calls per 10**18 seconds.
@@ -46,6 +48,7 @@
 # satisfied, and the moment where the checkpoint is called.
 REQUIRED_MECH_REQUESTS_SAFETY_MARGIN = 1
 
+
 class CheckStopTradingBehaviour(StakingInteractBaseBehaviour):
     """A behaviour that checks stop trading conditions."""
 
@@ -110,16 +113,25 @@ def is_staking_kpi_met(self) -> Generator[None, None, bool]:
         liveness_ratio = self.liveness_ratio
         self.context.logger.debug(f"{liveness_ratio=}")
 
-        mech_requests_since_last_cp = mech_request_count - mech_request_count_on_last_checkpoint
+        mech_requests_since_last_cp = (
+            mech_request_count - mech_request_count_on_last_checkpoint
+        )
         self.context.logger.debug(f"{mech_requests_since_last_cp=}")
 
         current_timestamp = self.synced_timestamp
         self.context.logger.debug(f"{current_timestamp=}")
 
-        required_mech_requests = math.ceil(max(
-            (current_timestamp - last_ts_checkpoint) * liveness_ratio / LIVENESS_RATIO_SCALE_FACTOR,
-            (liveness_period) * liveness_ratio / LIVENESS_RATIO_SCALE_FACTOR
-        )) + REQUIRED_MECH_REQUESTS_SAFETY_MARGIN
+        required_mech_requests = (
+            math.ceil(
+                max(
+                    (current_timestamp - last_ts_checkpoint)
+                    * liveness_ratio
+                    / LIVENESS_RATIO_SCALE_FACTOR,
+                    liveness_period * liveness_ratio / LIVENESS_RATIO_SCALE_FACTOR,
+                )
+            )
+            + REQUIRED_MECH_REQUESTS_SAFETY_MARGIN
+        )
         self.context.logger.debug(f"{required_mech_requests=}")
 
         if mech_requests_since_last_cp >= required_mech_requests:
@@ -135,11 +147,11 @@ def _compute_stop_trading(self) -> Generator:
             self.context.logger.debug(f"{self.is_first_period=}")
             return False
 
-        stop_trading_conditions = []
-
         self.context.logger.debug(f"{self.params.disable_trading=}")
-        stop_trading_conditions.append(self.params.disable_trading)
+        if self.params.disable_trading:
+            return True
 
+        stop_trading_conditions = []
         self.context.logger.debug(f"{self.params.stop_trading_if_staking_kpi_met=}")
         if self.params.stop_trading_if_staking_kpi_met:
             staking_kpi_met = yield from self.is_staking_kpi_met()
@@ -153,9 +165,7 @@ def async_act(self) -> Generator:
         with self.context.benchmark_tool.measure(self.behaviour_id).local():
             stop_trading = yield from self._compute_stop_trading()
             self.context.logger.info(f"Computed {stop_trading=}")
-            payload = CheckStopTradingPayload(
-                self.context.agent_address, stop_trading
-            )
+            payload = CheckStopTradingPayload(self.context.agent_address, stop_trading)
 
         with self.context.benchmark_tool.measure(self.behaviour_id).consensus():
             yield from self.send_a2a_transaction(payload)