Merge pull request #382 from valory-xyz/feat/benchmarking-convergence

Feat/benchmarking convergence:

bets queue mechanism
nr mech calls limit support
hotfix in BetsManagerBehaviour for Benchmarking mode

.github/workflows/common_checks.yaml

@@ -63,7 +63,7 @@ jobs:
     - name: Install dependencies
       run: pip install tomte[tox,cli]==0.2.14
     - name: Check copyright headers
-      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci  --exclude-part http_server --exclude-part mech_marketplace
+      run: tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci  --exclude-part http_server --exclude-part mech_marketplace --exclude-part erc20
     - name: License compatibility check
       run: tox -e liccheck
     - name: Check dependencies

.gitignore

@@ -38,7 +38,7 @@ packages/valory/protocols/tendermint
 .idea
 **/__pycache__/
 *.DS_Store
-
+trader_backup/
 .mypy_cache
 /.tox/
 

Makefile

@@ -74,13 +74,13 @@ security:
 .PHONY: generators
 generators: clean-cache fix-abci-app-specs
 	tox -e abci-docstrings
-	tomte format-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci --exclude-part http_server --exclude-part mech_marketplace --exclude-part agent_registry
+	tomte format-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part http_server --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part erc20 --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part mech --exclude-part mech_marketplace --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part mech_interact_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part agent_registry
 	autonomy packages lock
 	tox -e fix-doc-hashes
 
 .PHONY: common-checks-1
 common-checks-1:
-	tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part mech --exclude-part mech_interact_abci --exclude-part http_server --exclude-part mech_marketplace --exclude-part agent_registry
+	tomte check-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part http_server --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part erc20 --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part mech --exclude-part mech_marketplace --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part mech_interact_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription -	tomte format-copyright --author valory --exclude-part abci --exclude-part http_client --exclude-part http_server --exclude-part ipfs --exclude-part ledger --exclude-part p2p_libp2p_client --exclude-part erc20 --exclude-part gnosis_safe --exclude-part gnosis_safe_proxy_factory --exclude-part mech --exclude-part mech_marketplace --exclude-part multisend --exclude-part service_registry --exclude-part protocols --exclude-part abstract_abci --exclude-part abstract_round_abci --exclude-part mech_interact_abci --exclude-part registration_abci --exclude-part reset_pause_abci --exclude-part termination_abci --exclude-part transaction_settlement_abci --exclude-part websocket_client --exclude-part contract_subscription --exclude-part agent_registry
 	tomte check-doc-links
 	tox -p -e check-hash -e check-packages -e check-doc-hashes -e analyse-service
 

packages/packages.json

@@ -9,20 +9,21 @@
         "contract/valory/realitio/0.1.0": "bafybeietgux6kkhdquspy35qera7gjwwqwrremmoeatjzwwokjb2lzsata",
         "contract/valory/realitio_proxy/0.1.0": "bafybeidx37xzjjmapwacedgzhum6grfzhp5vhouz4zu3pvpgdy5pgb2fr4",
         "contract/valory/conditional_tokens/0.1.0": "bafybeibnzmqmeph4cj5vfh3s622mo2o5627vjjwc6bptrhj4dk65mzgvhe",
+        "contract/valory/agent_registry/0.1.0": "bafybeibfqhrpvuaibi6pgwsrot3ybt77hf4itjw6pklwnpxi3wcrrg2gr4",
         "contract/valory/service_staking_token/0.1.0": "bafybeiarxffrwdq47rmydgffyxhe6ntfb5unp44yt47kxqg7fzc7vburni",
         "contract/valory/transfer_nft_condition/0.1.0": "bafybeicfl4jldwjf6ngk3x4b36krcqogzbgqzs4zkzxipmtgveeefus264",
         "contract/valory/mech_activity/0.1.0": "bafybeiey6n7fjehm5k4g5h4toygqq3zzyxadqj3zabf67jmn5vzw5kndre",
         "contract/valory/staking_token/0.1.0": "bafybeiabkkhjpybqpzdfc4vcbiz4vefctfpvwetbmo7pqpiuxprmpvnti4",
         "contract/valory/relayer/0.1.0": "bafybeibvqc3lwxtcnu6dgfkf7mzefdgtfyosyq2dow7ogyxsl25vkxjwea",
-        "skill/valory/market_manager_abci/0.1.0": "bafybeid75ggv2pk22gtf6ae5prtwlahmgkxjla3tgw5mykn7agcndrbs5e",
-        "skill/valory/decision_maker_abci/0.1.0": "bafybeiexu6ttsv6gqlwsxbn5zoq4fgjboiq5v65q3fwlamsxabha2xjihq",
-        "skill/valory/trader_abci/0.1.0": "bafybeifvlcs7ak6kjemueeqwv4uwrjgrcd4g2qumflaxkgcgrrzqfbnuae",
-        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeictbhf3hittfibpo6hr2435y4gtvppgmg4uu4d2xpudw3z2ek5hgm",
+        "skill/valory/market_manager_abci/0.1.0": "bafybeihmyqkzl3bm5zvjnc4auj32qjf3pk73scyq7mntmpsudqnisb4gey",
+        "skill/valory/decision_maker_abci/0.1.0": "bafybeigehrtalp3ppnog25yuvfcdt3gmrgvxb2zm2zyngxxfcvps2s3tza",
+        "skill/valory/trader_abci/0.1.0": "bafybeideek7oe3rbcnaymd4bnwsm2kull75wfdvljv5prvcsok5sgti7bq",
+        "skill/valory/tx_settlement_multiplexer_abci/0.1.0": "bafybeih2s2dwnyxm7xhd3knc3wlaejzyvfulxkipwnidmzxgx7t3ofqbpq",
         "skill/valory/staking_abci/0.1.0": "bafybeial5xtgzf37e2khvdrc2q2wa2xhirjwioi2szwvcgtupidjxhg7tq",
         "skill/valory/check_stop_trading_abci/0.1.0": "bafybeid7l74lmjnkeerkbvwhoo2l4cawb7c545rhcx3mjpsjux4zwy5wpm",
-        "agent/valory/trader/0.1.0": "bafybeiggtcerws6srpq7lljq54gnbmiqvoclre7jsjuhxqiiu3e7oop6yq",
-        "service/valory/trader/0.1.0": "bafybeiaaa3fufe4gwmrdeml5ixrqr42jz7paubqvbg7hucdqjvbbxwpbua",
-        "service/valory/trader_pearl/0.1.0": "bafybeigzwsotc2ku27gagstefantjvr2yr7usdytljwtb5cuupyt2jl2dm"
+        "agent/valory/trader/0.1.0": "bafybeicy63hsvgcui5p6vwjdgfwrm7ldvjxui4nyy7zykm675xjfnmdbza",
+        "service/valory/trader/0.1.0": "bafybeiawu46p7whmf6aiywqyechsolmhb56ew7c7pjphmrhzen7umk6lc4",
+        "service/valory/trader_pearl/0.1.0": "bafybeia3eztsz325jy2sbtilulfnl6pngz3bltkvogowx6xym3kr5re4ei"
     },
     "third_party": {
         "protocol/valory/acn_data_share/0.1.0": "bafybeih5ydonnvrwvy2ygfqgfabkr47s4yw3uqxztmwyfprulwfsoe7ipq",

packages/valory/agents/trader/aea-config.yaml

@@ -50,10 +50,10 @@ skills:
 - valory/reset_pause_abci:0.1.0:bafybeigebq46oqz2mx2iajupr6p5pgm6z5pvfye5w6zypsseuqtvta7b4a
 - valory/termination_abci:0.1.0:bafybeieurwmfernodqyczj5ertsgfbjtjnrlgvte7sli4sajnbopty7inu
 - valory/transaction_settlement_abci:0.1.0:bafybeifkftgkyzrxwxjdyqixpp7vk6aqmufikalmwx3kydtlg74tonu47u
-- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeictbhf3hittfibpo6hr2435y4gtvppgmg4uu4d2xpudw3z2ek5hgm
-- valory/market_manager_abci:0.1.0:bafybeid75ggv2pk22gtf6ae5prtwlahmgkxjla3tgw5mykn7agcndrbs5e
-- valory/decision_maker_abci:0.1.0:bafybeiexu6ttsv6gqlwsxbn5zoq4fgjboiq5v65q3fwlamsxabha2xjihq
-- valory/trader_abci:0.1.0:bafybeifvlcs7ak6kjemueeqwv4uwrjgrcd4g2qumflaxkgcgrrzqfbnuae
+- valory/tx_settlement_multiplexer_abci:0.1.0:bafybeih2s2dwnyxm7xhd3knc3wlaejzyvfulxkipwnidmzxgx7t3ofqbpq
+- valory/market_manager_abci:0.1.0:bafybeihmyqkzl3bm5zvjnc4auj32qjf3pk73scyq7mntmpsudqnisb4gey
+- valory/decision_maker_abci:0.1.0:bafybeigehrtalp3ppnog25yuvfcdt3gmrgvxb2zm2zyngxxfcvps2s3tza
+- valory/trader_abci:0.1.0:bafybeideek7oe3rbcnaymd4bnwsm2kull75wfdvljv5prvcsok5sgti7bq
 - valory/staking_abci:0.1.0:bafybeial5xtgzf37e2khvdrc2q2wa2xhirjwioi2szwvcgtupidjxhg7tq
 - valory/check_stop_trading_abci:0.1.0:bafybeid7l74lmjnkeerkbvwhoo2l4cawb7c545rhcx3mjpsjux4zwy5wpm
 - valory/mech_interact_abci:0.1.0:bafybeif2tpz2zet6p4z4vi3b254oxzyyzoe5tehj3me3znzt7h7otkpd54
@@ -268,6 +268,7 @@ models:
       bet_amount_field: ${str:collateral_amount}
       results_filename: ${str:benchmarking_results.csv}
       randomness: ${str:benchmarking_randomness}
+      nr_mech_calls: ${int:60}
   acc_info_fields:
     args:
       tool: ${str:tool}

packages/valory/services/trader/service.yaml

@@ -7,7 +7,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeigtuothskwyvrhfosps2bu6suauycolj67dpuxqvnicdrdu7yhtvq
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeiggtcerws6srpq7lljq54gnbmiqvoclre7jsjuhxqiiu3e7oop6yq
+agent: valory/trader:0.1.0:bafybeicy63hsvgcui5p6vwjdgfwrm7ldvjxui4nyy7zykm675xjfnmdbza
 number_of_agents: 4
 deployment:
   agent:
@@ -174,6 +174,7 @@ type: skill
         bet_amount_field: ${BENCHMARKING_MODE_BET_AMOUNT_FIELD:str:collateral_amount}
         results_filename: ${BENCHMARKING_MODE_RESULTS_FILENAME:str:benchmarking_results.csv}
         randomness: ${BENCHMARKING_MODE_RANDOMNESS:str:benchmarking_randomness}
+        nr_mech_calls: ${BENCHMARKING_MECH_CALLS:int:60}
     acc_info_fields: &id004
       args:
         tool: ${ACC_INFO_FIELDS_TOOL:str:tool}

packages/valory/services/trader_pearl/service.yaml

@@ -8,7 +8,7 @@ license: Apache-2.0
 fingerprint:
   README.md: bafybeibg7bdqpioh4lmvknw3ygnllfku32oca4eq5pqtvdrdsgw6buko7e
 fingerprint_ignore_patterns: []
-agent: valory/trader:0.1.0:bafybeiggtcerws6srpq7lljq54gnbmiqvoclre7jsjuhxqiiu3e7oop6yq
+agent: valory/trader:0.1.0:bafybeicy63hsvgcui5p6vwjdgfwrm7ldvjxui4nyy7zykm675xjfnmdbza
 number_of_agents: 1
 deployment:
   agent:

packages/valory/skills/decision_maker_abci/behaviours/base.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2023-2024 Valory AG
+#   Copyright 2023-2025 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -48,7 +48,6 @@
 from packages.valory.skills.decision_maker_abci.models import (
     AccuracyInfoFields,
     BenchmarkingMockData,
-    BenchmarkingMode,
     DecisionMakerParams,
     L0_END_FIELD,
     L0_START_FIELD,
@@ -171,11 +170,6 @@ def params(self) -> DecisionMakerParams:
         """Return the params."""
         return cast(DecisionMakerParams, self.context.params)
 
-    @property
-    def benchmarking_mode(self) -> BenchmarkingMode:
-        """Return the benchmarking mode configurations."""
-        return cast(BenchmarkingMode, self.context.benchmarking_mode)
-
     @property
     def mock_data(self) -> BenchmarkingMockData:
         """Return the mock data for the benchmarking mode."""

packages/valory/skills/decision_maker_abci/behaviours/bet_placement.py

@@ -213,6 +213,7 @@ def async_act(self) -> Generator:
         if self.benchmarking_mode.enabled:
             # simulate the bet placement
             with self.context.benchmark_tool.measure(self.behaviour_id).local():
+                self.update_bet_transaction_information()
                 payload = BetPlacementPayload(
                     agent, None, None, True, self.wallet_balance
                 )

packages/valory/skills/decision_maker_abci/behaviours/decision_receive.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2023-2024 Valory AG
+#   Copyright 2023-2025 Valory AG
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -49,6 +49,7 @@
     P_NO_FIELD,
     P_YES_FIELD,
     PredictionResponse,
+    QueueStatus,
 )
 from packages.valory.skills.mech_interact_abci.states.base import (
     MechInteractionResponse,
@@ -126,6 +127,7 @@ def _next_dataset_row(self) -> Optional[Dict[str, str]]:
         else:
             # no more bets available for this market
             msg = f"No more mock responses for the market with id: {sampled_bet_id}"
+            self.sampled_bet.queue_status = QueueStatus.BENCHMARKING_DONE
             self.context.logger.info(msg)
             self.shared_state.last_benchmarking_has_run = True
             self._rows_exceeded = True
@@ -514,6 +516,9 @@ def _is_profitable(
             else:
                 self._write_benchmark_results(prediction_response)
 
+            self.context.logger.info("Increasing Mech call count by 1")
+            self.shared_state.benchmarking_mech_calls += 1
+
         return is_profitable, bet_amount
 
     def _update_selected_bet(
@@ -569,6 +574,8 @@ def async_act(self) -> Generator:
                     prediction_response,
                     bet_amount,
                 )
+                self.context.logger.info("Increasing Mech call count by 1")
+                self.shared_state.benchmarking_mech_calls += 1
 
             if prediction_response is not None:
                 self.policy.tool_responded(

packages/valory/skills/decision_maker_abci/behaviours/sampling.py

@@ -138,6 +138,31 @@ def _sampled_bet_idx(self, bets: List[Bet]) -> int:
 
         return self.bets.index(sorted_bets[0])
 
+    def _sampling_benchmarking_bet(self, bets: List[Bet]) -> Optional[int]:
+        """Sample bet for benchmarking"""
+        to_process_bets, processed_bets, reprocessed_bets = self._get_bets_queue_wise(
+            bets
+        )
+
+        self.context.logger.info(f"TO_PROCESS_LEN: {len(to_process_bets)}")
+        self.context.logger.info(f"PROCESSED_LEN: {len(processed_bets)}")
+        self.context.logger.info(f"REPROCESSED_LEN: {len(reprocessed_bets)}")
+
+        self.context.logger.info(
+            f"MECH CALLS MADE: {self.shared_state.benchmarking_mech_calls}"
+        )
+
+        if (
+            self.shared_state.benchmarking_mech_calls
+            == self.benchmarking_mode.nr_mech_calls
+        ):
+            return None
+
+        bets_to_sort: List[Bet] = to_process_bets or processed_bets or reprocessed_bets
+        sorted_bets = self._sort_by_priority_logic(bets_to_sort)
+
+        return self.bets.index(sorted_bets[0])
+
     def _sample(self) -> Optional[int]:
         """Sample a bet, mark it as processed, and return its index."""
         # modify time "NOW" in benchmarking mode
@@ -164,6 +189,11 @@ def _sample(self) -> Optional[int]:
             self.context.logger.warning(msg)
             return None
 
+        if self.benchmarking_mode.enabled:
+            idx = self._sampling_benchmarking_bet(available_bets)
+            if not idx:
+                return None
+
         # sample a bet using the priority logic
         idx = self._sampled_bet_idx(available_bets)
         sampled_bet = self.bets[idx]
@@ -190,6 +220,8 @@ def _benchmarking_inc_day(self) -> Tuple[bool, bool]:
         if benchmarking_finished:
             self.context.logger.info("No more days to simulate in benchmarking mode.")
 
+        self.shared_state.benchmarking_mech_calls = 0
+
         day_increased = True
 
         return benchmarking_finished, day_increased
@@ -204,6 +236,9 @@ def async_act(self) -> Generator:
             # day increase simulation and benchmarking finished check
             if idx is None and self.benchmarking_mode.enabled:
                 benchmarking_finished, day_increased = self._benchmarking_inc_day()
+                for bet in self.bets:
+                    bet.queue_status = bet.queue_status.move_to_fresh()
+                    bet.queue_status = bet.queue_status.move_to_process()
 
             self.store_bets()
 

packages/valory/skills/decision_maker_abci/models.py

@@ -215,6 +215,9 @@ def __init__(self, *args: Any, skill_context: SkillContext, **kwargs: Any) -> No
         # the key is the market id/question_id
         self.bet_id_row_manager: Dict[str, List[int]] = {}
 
+        # mech call counter for benchmarking behaviour
+        self.benchmarking_mech_calls: int = 0
+
     @property
     def mock_question_id(self) -> Any:
         """Get the mock question id."""
@@ -497,39 +500,6 @@ def get_store_path(self, kwargs: Dict) -> Path:
         return Path(path)
 
 
-class BenchmarkingMode(Model, TypeCheckMixin):
-    """Configuration for the benchmarking mode."""
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Initialize the `BenchmarkingMode` object."""
-        self.enabled: bool = self._ensure("enabled", kwargs, bool)
-        self.native_balance: int = self._ensure("native_balance", kwargs, int)
-        self.collateral_balance: int = self._ensure("collateral_balance", kwargs, int)
-        self.mech_cost: int = self._ensure("mech_cost", kwargs, int)
-        self.pool_fee: int = self._ensure("pool_fee", kwargs, int)
-        self.sep: str = self._ensure("sep", kwargs, str)
-        self.dataset_filename: Path = Path(
-            self._ensure("dataset_filename", kwargs, str)
-        )
-        self.question_field: str = self._ensure("question_field", kwargs, str)
-        self.question_id_field: str = self._ensure("question_id_field", kwargs, str)
-        self.answer_field: str = self._ensure("answer_field", kwargs, str)
-        self.p_yes_field_part: str = self._ensure("p_yes_field_part", kwargs, str)
-        self.p_no_field_part: str = self._ensure("p_no_field_part", kwargs, str)
-        self.confidence_field_part: str = self._ensure(
-            "confidence_field_part", kwargs, str
-        )
-        # this is the mode for the p and confidence parts
-        # if the flag is `True`, then the field parts are used as prefixes, otherwise as suffixes
-        self.part_prefix_mode: bool = self._ensure("part_prefix_mode", kwargs, bool)
-        self.bet_amount_field: str = self._ensure("bet_amount_field", kwargs, str)
-        self.results_filename: Path = Path(
-            self._ensure("results_filename", kwargs, str)
-        )
-        self.randomness: str = self._ensure("randomness", kwargs, str)
-        super().__init__(*args, **kwargs)
-
-
 class AccuracyInfoFields(Model, TypeCheckMixin):
     """Configuration which holds the accuracy information file's fieldnames."""