Change surplus based tests to use new data from orderbook (#91)

This PR addresses #90 for the competition surplus test and combinatorial
auction test.

The main changes are:
- Order information is fetched from the orderbook orders endpoint.
- Execution data is obtained via the new data in the solver competition
endpoint.
- Fees and costs are not explicitly part of the test anymore.

Before, information on trades (order data and order execution) were read
from calldata. For non-winning solutions this is not possible anymore.
For the winning solution it is still possible. But due to changes to the
ordering of orders in the competition data, it was not easy for me to
reconstruct the link from uid to trade. Thus, all information is not
obtained through the orderbook api.

In principle, information on fees for non-winning solutions could be
computed from clearing prices in combination with effective buy and sell
amount. This is not done here since fees do not matter for surplus. Also
clearing prices are not yet part of the data in the competition
endpoint.
Information on costs for non-winning solutions is and will not
available. A test of the form `fees >= costs` had to be removed due to
that.

The old code had a "bug" where only one execution per solver was
studied. This is fixed now and the test should work correctly with
solvers submitting multiple solutions. The fix required changing the
data format for alternative solutions from a `dict` to a `list`

src/apis/orderbookapi.py

@@ -7,6 +7,7 @@
 import json
 import requests
 from src.helper_functions import get_logger
+from src.models import Trade, OrderData, OrderExecution
 from src.constants import (
     header,
     REQUEST_TIMEOUT,
@@ -57,3 +58,66 @@ def get_solver_competition_data(self, tx_hash: str) -> Optional[dict[str, Any]]:
             )
             return None
         return solver_competition_data
+
+    def get_order_data(self, uid: str) -> dict[str, Any] | None:
+        """Get order data from uid.
+        The returned dict follows the schema outlined here:
+        https://api.cow.fi/docs/#/default/get_api_v1_orders__UID_
+        """
+        prod_endpoint_url = f"{PROD_BASE_URL}orders/{uid}"
+        barn_endpoint_url = f"{BARN_BASE_URL}orders/{uid}"
+        order_data: Optional[dict[str, Any]] = None
+        try:
+            json_order_data = requests.get(
+                prod_endpoint_url,
+                headers=header,
+                timeout=REQUEST_TIMEOUT,
+            )
+            if json_order_data.status_code == SUCCESS_CODE:
+                order_data = json_order_data.json()
+            elif json_order_data.status_code == FAIL_CODE:
+                barn_order_data = requests.get(
+                    barn_endpoint_url, headers=header, timeout=REQUEST_TIMEOUT
+                )
+                if barn_order_data.status_code == SUCCESS_CODE:
+                    order_data = barn_order_data.json()
+                else:
+                    return None
+        except requests.RequestException as err:
+            self.logger.warning(
+                f"Connection error while fetching order data. UID: {uid}, error: {err}"
+            )
+            return None
+        return order_data
+
+    def get_trade(
+        self, order_response: dict[str, Any], execution_response: dict[str, Any]
+    ) -> Trade:
+        """Create Trade from order and execution data."""
+        data = OrderData(
+            int(order_response["buyAmount"]),
+            int(order_response["sellAmount"]),
+            int(order_response["feeAmount"]),
+            order_response["buyToken"],
+            order_response["sellToken"],
+            order_response["kind"] == "sell",
+            order_response["partiallyFillable"],
+        )
+        execution = OrderExecution(
+            int(execution_response["buyAmount"]),
+            int(execution_response["sellAmount"]),
+            0,
+        )
+        return Trade(data, execution)
+
+    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade] | None:
+        """Get a dictionary mapping UIDs to trades in a solution."""
+        trades_dict: dict[str, Trade] = {}
+        for execution in solution["orders"]:
+            uid = execution["id"]
+            order_data = self.get_order_data(uid)
+            if order_data is None:
+                return None
+            trades_dict[uid] = self.get_trade(order_data, execution)
+
+        return trades_dict

src/monitoring_tests/combinatorial_auction_surplus_test.py

@@ -7,9 +7,7 @@
 from typing import Any
 from fractions import Fraction
 from src.monitoring_tests.base_test import BaseTest
-from src.apis.web3api import Web3API
 from src.apis.orderbookapi import OrderbookAPI
-from src.models import Trade
 from src.constants import SURPLUS_ABSOLUTE_DEVIATION_ETH
 
 
@@ -35,7 +33,6 @@ class CombinatorialAuctionSurplusTest(BaseTest):
 
     def __init__(self) -> None:
         super().__init__()
-        self.web3_api = Web3API()
         self.orderbook_api = OrderbookAPI()
 
     def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
@@ -51,15 +48,15 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
         """
 
         solutions = competition_data["solutions"]
-        winning_solution = competition_data["solutions"][-1]
 
-        aggregate_solutions = [
-            self.get_token_pairs_surplus(
+        aggregate_solutions: list[dict[tuple[str, str], Fraction]] = []
+        for solution in solutions:
+            aggregate_solution = self.get_token_pairs_surplus(
                 solution, competition_data["auction"]["prices"]
             )
-            for solution in solutions
-        ]
-        winning_aggregate_solution = aggregate_solutions[-1]
+            if aggregate_solution is None:
+                return False
+            aggregate_solutions.append(aggregate_solution)
 
         baseline_surplus = self.compute_baseline_surplus(aggregate_solutions)
         filter_mask = self.filter_solutions(aggregate_solutions, baseline_surplus)
@@ -80,18 +77,16 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
             sum(surplus for _, surplus in token_pair_surplus.items())
             for _, token_pair_surplus in winning_solvers.items()
         )
-        total_surplus = sum(
-            surplus for _, surplus in winning_aggregate_solution.items()
-        )
+        total_surplus = sum(surplus for _, surplus in aggregate_solutions[-1].items())
 
         a_abs_eth = total_combinatorial_surplus - total_surplus
 
         log_output = "\t".join(
             [
                 "Combinatorial auction surplus test:",
                 f"Tx Hash: {competition_data['transactionHash']}",
-                f"Winning Solver: {winning_solution['solver']}",
-                f"Winning surplus: {self.convert_fractions_to_floats(winning_aggregate_solution)}",
+                f"Winning Solver: {competition_data['solutions'][-1]['solver']}",
+                f"Winning surplus: {self.convert_fractions_to_floats(aggregate_solutions[-1])}",
                 f"Baseline surplus: {self.convert_fractions_to_floats(baseline_surplus)}",
                 f"Solutions filtering winner: {filter_mask[-1]}",
                 f"Solvers filtering winner: {solutions_filtering_winner}",
@@ -113,30 +108,16 @@ def run_combinatorial_auction(self, competition_data: dict[str, Any]) -> bool:
 
         return True
 
-    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade]:
-        """Get a dictionary mapping UIDs to trades in a solution."""
-        calldata = solution["callData"]
-        settlement = self.web3_api.get_settlement_from_calldata(calldata)
-        trades = self.web3_api.get_trades(settlement)
-        trades_dict = {
-            solution["orders"][i]["id"]: trade for (i, trade) in enumerate(trades)
-        }
-        return trades_dict
-
     def get_token_pairs_surplus(
         self, solution: dict[str, Any], prices: dict[str, float]
-    ) -> dict[tuple[str, str], Fraction]:
+    ) -> dict[tuple[str, str], Fraction] | None:
         """Aggregate surplus of a solution on the different token pairs.
         The result is a dict containing directed token pairs and the aggregated surplus on them.
-
-        Instead of surplus we use the minimum of surplus and the objective. This is more
-        conservative than just using objective. If fees are larger than costs, the objective is
-        larger than surplus and surplus is used for the comparison. If fees are larger than costs,
-        the objective is smaller than surplus and the objective is used instead of surplus for
-        filtering. This takes care of the case of solvers providing a lot of surplus but at really
-        large costs.
         """
-        trades_dict = self.get_uid_trades(solution)
+        trades_dict = self.orderbook_api.get_uid_trades(solution)
+        if trades_dict is None:
+            return None
+
         surplus_dict: dict[tuple[str, str], Fraction] = {}
         for uid in trades_dict:
             trade = trades_dict[uid]
@@ -159,14 +140,6 @@ def get_token_pairs_surplus(
                 + surplus_token_to_eth * trade.get_surplus()
             )
 
-        # use the minimum of surplus and objective in case there is only one token pair
-        if len(surplus_dict) == 1:
-            for token_pair in surplus_dict:
-                surplus_dict[token_pair] = min(
-                    surplus_dict[token_pair],
-                    Fraction(solution["objective"]["total"]) / 10**18,
-                )
-
         return surplus_dict
 
     def compute_baseline_surplus(

src/monitoring_tests/solver_competition_surplus_test.py

@@ -6,9 +6,8 @@
 from typing import Any
 from fractions import Fraction
 from src.monitoring_tests.base_test import BaseTest
-from src.apis.web3api import Web3API
 from src.apis.orderbookapi import OrderbookAPI
-from src.models import Trade
+from src.models import Trade, OrderExecution
 from src.constants import SURPLUS_ABSOLUTE_DEVIATION_ETH, SURPLUS_REL_DEVIATION
 
 
@@ -20,7 +19,6 @@ class SolverCompetitionSurplusTest(BaseTest):
 
     def __init__(self) -> None:
         super().__init__()
-        self.web3_api = Web3API()
         self.orderbook_api = OrderbookAPI()
 
     def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
@@ -32,7 +30,9 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
 
         solution = competition_data["solutions"][-1]
 
-        trades_dict = self.get_uid_trades(solution)
+        trades_dict = self.orderbook_api.get_uid_trades(solution)
+        if trades_dict is None:
+            return False
 
         for uid in trades_dict:
             trade = trades_dict[uid]
@@ -46,10 +46,10 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
             )
 
             trade_alt_dict = self.get_trade_alternatives(
-                uid, competition_data["solutions"][0:-1]
+                trade, uid, competition_data["solutions"][0:-1]
             )
 
-            for solver_alt, trade_alt in trade_alt_dict.items():
+            for solver_alt, trade_alt in trade_alt_dict:
                 a_abs = trade_alt.compare_surplus(trade)
                 a_abs_eth = a_abs * token_to_eth
                 a_rel = trade_alt.compare_price(trade)
@@ -82,35 +82,38 @@ def compare_orders_surplus(self, competition_data: dict[str, Any]) -> bool:
         return True
 
     def get_trade_alternatives(
-        self, uid: str, solution_alternatives: list[dict[str, Any]]
-    ) -> dict[str, Trade]:
+        self, trade: Trade, uid: str, solution_alternatives: list[dict[str, Any]]
+    ) -> list[tuple[str, Trade]]:
         """Compute surplus and exchange rate for an order with uid as settled in alternative
         solutions."""
-        trade_alt_dict: dict[str, Trade] = {}
+        trade_alt_list: list[tuple[str, Trade]] = []
+        order_data = trade.data
         for solution_alt in solution_alternatives:
-            if (
-                solution_alt["objective"]["fees"]
-                < 0.9 * solution_alt["objective"]["cost"]
-            ):
-                continue
-            trades_dict_alt = self.get_uid_trades(solution_alt)
+            executions_dict_alt = self.get_uid_order_execution(solution_alt)
             try:
-                trade_alt = trades_dict_alt[uid]
+                trade_alt = Trade(order_data, executions_dict_alt[uid])
             except KeyError:
                 continue
-            trade_alt_dict[solution_alt["solver"]] = trade_alt
-
-        return trade_alt_dict
-
-    def get_uid_trades(self, solution: dict[str, Any]) -> dict[str, Trade]:
-        """Get a dictionary mapping UIDs to trades in a solution."""
-        calldata = solution["callData"]
-        settlement = self.web3_api.get_settlement_from_calldata(calldata)
-        trades = self.web3_api.get_trades(settlement)
-        trades_dict = {
-            solution["orders"][i]["id"]: trade for (i, trade) in enumerate(trades)
-        }
-        return trades_dict
+            trade_alt_list.append((solution_alt["solver"], trade_alt))
+
+        return trade_alt_list
+
+    def get_uid_order_execution(
+        self, solution: dict[str, Any]
+    ) -> dict[str, OrderExecution]:
+        """Given a solution from the competition endpoint, compute the executin for all included
+        orders.
+        """
+        result: dict[str, OrderExecution] = {}
+        for order in solution["orders"]:
+            buy_amount = int(order["buyAmount"])
+            sell_amount = int(order["sellAmount"])
+            # fee amount is set to zero for the moment, could be computed from clearing prices
+            # and buy and sell token of the order
+            fee_amount = 0
+            order_execution = OrderExecution(buy_amount, sell_amount, fee_amount)
+            result[order["id"]] = order_execution
+        return result
 
     def run(self, tx_hash: str) -> bool:
         """

tests/e2e/combinatorial_auction_surplus_test.py

@@ -11,23 +11,8 @@
 class TestCombinatorialAuctionSurplus(unittest.TestCase):
     def test_surplus(self) -> None:
         surplus_test = CombinatorialAuctionSurplusTest()
-        # # Baseline EBBO error
-        # tx_hash = "0x4115f6f4abaea17f2ebef3a1e75c589c38cac048ff5116d406038e48ff7aeacd"
-        # # large settlement with bad execution for one of the orders
-        # tx_hash = "0xc22b1e4984b212e679d4af49c1622e7018c83d5e32ece590cf84a3e1950f9f18"
-        # # EBBO violations
-        # #
-        # tx_hash = "0x2ff69424f7bf8951ed5e7dd04b648380b0e73dbf7f0191c800651bc4b16a30c5"
-        # # combinatorial auction worse than current auction
-        # tx_hash = "0xb743b023ad838f04680fd321bf579c35931c4f886f664bd2b6e675c310a9c287"
-        # # combinatorial auction better than current auction
-        # tx_hash = "0x46639ae0e516bcad7b052fb6bfb6227d0aa2707e9882dd8d86bab2ab6aeee155"
-        # tx_hash = "0xe28b92ba73632d6b167fdb9bbfec10744ce208536901dd43379a6778c4408536"
-        # tx_hash = "0xad0ede9fd68481b8ef4722d069598898e01d61427ccb378ca4c82c772c6644e0"
-        # tx_hash = "0xead8f01e8e24fdc306fca8fcac5146edc22c27e49a7aad6134adc2ad50ba8581"
-        # tx_hash = "0x6200e744e5d6f9990271be53840c01044cc19f3a8526190e1eaac0bc5fefed85"
-        # uncovered bug with wrong scaling of objective
-        tx_hash = "0x97b2f8402d239e16b62b7cc2302ed77ac8fa40d63114ab6804041c9d3b9e6b81"
+        # CoW with liquidity order by Naive solver
+        tx_hash = "0x6b728195926e033ab92bbe7db51170c582ff57ba841aaaca3a9319cfe34491ff"
         self.assertTrue(surplus_test.run(tx_hash))
 
 

tests/e2e/surplus_test.py

@@ -9,22 +9,11 @@
 
 
 class TestSurplus(unittest.TestCase):
-    def test_surplus(self):
+    def test_surplus(self) -> None:
         surplus_test = SolverCompetitionSurplusTest()
-        # minor EBBO violation
-        tx_hash = "0xb2189d1a9fe31d15522f0110c0a2907354fbb1edccd1a6186ef0608fe5ad5722"
+        # new competition format: no alert or info
+        tx_hash = "0xc140a3adc9debfc00a45cc713afbac1bbe197ad2dd1d7fa5b4a36de1080a3d66"
         self.assertTrue(surplus_test.run(tx_hash))
-        # hash not in the data base
-        tx_hash = "0x999999999fe31d15522f0110c0a2907354fbb1edccd1a6186ef0608fe5ad5722"
-        self.assertFalse(surplus_test.run(tx_hash))
-        # surplus shift to partial fill
-        tx_hash = "0xf467a6a01f61fa608c1bc116e2f4f4df1b95461827b1e7700c1d36628875feab"
-        self.assertTrue(surplus_test.run(tx_hash))
-        # order combined with partial fill
-        tx_hash = "0x8e9f98cabf9b6ff4e001eda5efacfd70590a60bd03a499d8b02130b67b208eb1"
-        self.assertTrue(surplus_test.run(tx_hash))
-        # partial fill with competition
-        # look for this
 
 
 if __name__ == "__main__":