From e7ab473f23efe3e88d9d32980cd65cea86c40553 Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Wed, 19 Jul 2023 13:44:24 +0300 Subject: [PATCH] Add two related tests about economic viability (#67) This PR adds a test around economic viability of the protocol. Specifically, the following are checked: 1. Fees collected - execution cost, per settlement. Results are aggregated per solver and reported once a day. 2. Fees collected - payout to solver, per settlement. Again, results are aggregated per solver and reported once a day. --------- Co-authored-by: Felix Henneke --- src/constants.py | 6 + src/daemon.py | 4 + .../cost_coverage_per_solver_test.py | 109 ++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 src/monitoring_tests/cost_coverage_per_solver_test.py diff --git a/src/constants.py b/src/constants.py index 70d0f2b..2491e3b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -19,6 +19,12 @@ # reference solver test SOLVER_TIME_LIMIT = 20 +# how many blocks are contained in a single day +DAY_BLOCK_INTERVAL = 7200 + +# cap parameter, per CIP-20, measured in ETH +CAP_PARAMETER = 0.01 + # requests SETTLEMENT_CONTRACT_ADDRESS = "0x9008D19f58AAbD9eD0D60971565AA8510560ab41" REQUEST_TIMEOUT = 5 diff --git a/src/daemon.py b/src/daemon.py index d33258b..4bb385e 100644 --- a/src/daemon.py +++ b/src/daemon.py @@ -21,6 +21,9 @@ from src.monitoring_tests.reference_solver_surplus_test import ( ReferenceSolverSurplusTest, ) +from src.monitoring_tests.cost_coverage_per_solver_test import ( + CostCoveragePerSolverTest, +) from src.constants import SLEEP_TIME_IN_SEC @@ -36,6 +39,7 @@ def main() -> None: ReferenceSolverSurplusTest(), PartialFillFeeQuoteTest(), PartialFillCostCoverageTest(), + CostCoveragePerSolverTest(), ] start_block: Optional[int] = None diff --git a/src/monitoring_tests/cost_coverage_per_solver_test.py b/src/monitoring_tests/cost_coverage_per_solver_test.py new file mode 100644 index 0000000..8571efc --- /dev/null +++ b/src/monitoring_tests/cost_coverage_per_solver_test.py @@ -0,0 +1,109 @@ +""" +Computing cost coverage per solver. +""" +# pylint: disable=logging-fstring-interpolation + +from typing import Any, Dict +from src.monitoring_tests.base_test import BaseTest +from src.apis.web3api import Web3API +from src.apis.orderbookapi import OrderbookAPI +from src.constants import DAY_BLOCK_INTERVAL, CAP_PARAMETER + + +class CostCoveragePerSolverTest(BaseTest): + """ + This test checks the following on a per solver basis: + 1. fees_collected (as perceived by solver) minus actual execution cost. + 2. total payout to solver minus fees collected. + The intent is to gain a better understanding of which solvers are more costly, + how much we are paying them etc. + + The results are reported once a day as a log. + """ + + def __init__(self) -> None: + super().__init__() + self.web3_api = Web3API() + self.orderbook_api = OrderbookAPI() + self.cost_coverage_per_solver: Dict[str, float] = {} + self.total_coverage_per_solver: Dict[str, float] = {} + self.original_block = self.web3_api.get_current_block_number() + + def cost_coverage(self, competition_data: dict[str, Any], gas_cost: float) -> bool: + """ + This function compares the fees, as perceived by the winning solver, and the actual + execution cost of the corresponding settlement. This is refered to as cost_coverage, + and is supposed to monitor how well the fees end up approximating the execution cost + of a solution. + We also look at the coverage, from the protocol perspective, of each settlement, i.e., + we compare the payout made to the solver with the fees collected. + """ + + solution = competition_data["solutions"][-1] + fees = float(int(solution["objective"]["fees"]) / 10**18) + surplus = float(int(solution["objective"]["surplus"]) / 10**18) + solver = solution["solver"] + ref_score = 0.0 + if len(competition_data["solutions"]) > 1: + second_best_sol = competition_data["solutions"][-2] + if "score" in second_best_sol: + ref_score_str = second_best_sol["score"] + elif "scoreDiscounted" in second_best_sol: + ref_score_str = second_best_sol["scoreDiscounted"] + else: + ref_score_str = second_best_sol["scoreProtocol"] + ref_score = float(int(ref_score_str) / 10**18) + payout = surplus + fees - ref_score + capped_payout = min(payout, gas_cost + CAP_PARAMETER) + if solver in self.cost_coverage_per_solver: + self.cost_coverage_per_solver[solver] += fees - gas_cost + self.total_coverage_per_solver[solver] += fees - capped_payout + else: + self.cost_coverage_per_solver[solver] = fees - gas_cost + self.total_coverage_per_solver[solver] = fees - capped_payout + + return True + + def run(self, tx_hash: str) -> bool: + """ + Wrapper function for the whole test. Checks if solver competition data is retrievable + and runs test, else returns False to add to list of unchecked hashes. + """ + solver_competition_data = self.orderbook_api.get_solver_competition_data( + tx_hash + ) + transaction = self.web3_api.get_transaction(tx_hash) + receipt = self.web3_api.get_receipt(tx_hash) + gas_cost = 0.0 + if transaction is not None and receipt is not None: + gas_used, gas_price = self.web3_api.get_batch_gas_costs( + transaction, receipt + ) + gas_cost = float(gas_used) * float(gas_price) / 10**18 + if gas_cost == 0 or solver_competition_data is None: + return False + + success = self.cost_coverage(solver_competition_data, gas_cost) + + ### This part takes care of the reporting once a day. + current_block = self.web3_api.get_current_block_number() + if self.original_block is None: + self.original_block = current_block + if current_block is None or self.original_block is None: + return success + if current_block - self.original_block > DAY_BLOCK_INTERVAL: + log_msg = ( + f'"Fees - gasCost" per solver from block {self.original_block} to {current_block}: ' + + str(self.cost_coverage_per_solver) + ) + self.logger.info(log_msg) + log_msg = ( + f'"Fees - payment" per solver from block {self.original_block} to {current_block}: ' + + str(self.total_coverage_per_solver) + ) + self.logger.info(log_msg) + self.original_block = current_block + for solver in self.cost_coverage_per_solver: + self.cost_coverage_per_solver[solver] = 0 + self.total_coverage_per_solver[solver] = 0 + return success