From 89b0a5a290fbdce42082519c434d3b24e85e762f Mon Sep 17 00:00:00 2001 From: sslivkoff Date: Thu, 1 Feb 2024 10:35:07 -0800 Subject: [PATCH] add mesc support --- ethereumetl/cli/export_all.py | 4 ++-- ethereumetl/cli/export_blocks_and_transactions.py | 4 ++-- ethereumetl/cli/export_contracts.py | 4 ++-- ethereumetl/cli/export_receipts_and_logs.py | 4 ++-- ethereumetl/cli/export_tokens.py | 4 ++-- ethereumetl/cli/extract_tokens.py | 3 ++- ethereumetl/cli/get_block_range_for_date.py | 4 ++-- ethereumetl/cli/get_block_range_for_timestamps.py | 4 ++-- ethereumetl/cli/stream.py | 3 ++- ethereumetl/providers/auto.py | 10 ++++++++++ ethereumetl/utils.py | 14 ++++++++++++++ setup.py | 1 + 12 files changed, 43 insertions(+), 16 deletions(-) diff --git a/ethereumetl/cli/export_all.py b/ethereumetl/cli/export_all.py index 60fc47f4f..43375420f 100644 --- a/ethereumetl/cli/export_all.py +++ b/ethereumetl/cli/export_all.py @@ -32,7 +32,7 @@ from ethereumetl.jobs.export_all_common import export_all_common from ethereumetl.providers.auto import get_provider_from_uri from ethereumetl.service.eth_service import EthService -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @@ -109,7 +109,7 @@ def get_partitions(start, end, partition_batch_size, provider_uri): @click.option('-e', '--end', required=True, type=str, help='End block/ISO date/Unix time') @click.option('-b', '--partition-batch-size', default=10000, show_default=True, type=int, help='The number of blocks to export in partition.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-o', '--output-dir', default='output', show_default=True, type=str, help='Output directory, partitioned in Hive style.') diff --git a/ethereumetl/cli/export_blocks_and_transactions.py b/ethereumetl/cli/export_blocks_and_transactions.py index 88564be96..d9ccb77f0 100644 --- a/ethereumetl/cli/export_blocks_and_transactions.py +++ b/ethereumetl/cli/export_blocks_and_transactions.py @@ -28,7 +28,7 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.providers.auto import get_provider_from_uri from ethereumetl.thread_local_proxy import ThreadLocalProxy -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @@ -37,7 +37,7 @@ @click.option('-s', '--start-block', default=0, show_default=True, type=int, help='Start block') @click.option('-e', '--end-block', required=True, type=int, help='End block') @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to export at a time.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.') diff --git a/ethereumetl/cli/export_contracts.py b/ethereumetl/cli/export_contracts.py index ae7ba765b..bafd24411 100644 --- a/ethereumetl/cli/export_contracts.py +++ b/ethereumetl/cli/export_contracts.py @@ -29,7 +29,7 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.thread_local_proxy import ThreadLocalProxy from ethereumetl.providers.auto import get_provider_from_uri -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @@ -40,7 +40,7 @@ help='The file containing contract addresses, one per line.') @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.') @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.') diff --git a/ethereumetl/cli/export_receipts_and_logs.py b/ethereumetl/cli/export_receipts_and_logs.py index 95ff491a8..cc2e1ae18 100644 --- a/ethereumetl/cli/export_receipts_and_logs.py +++ b/ethereumetl/cli/export_receipts_and_logs.py @@ -29,7 +29,7 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.thread_local_proxy import ThreadLocalProxy from ethereumetl.providers.auto import get_provider_from_uri -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @@ -38,7 +38,7 @@ @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of receipts to export at a time.') @click.option('-t', '--transaction-hashes', required=True, type=str, help='The file containing transaction hashes, one per line.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.') diff --git a/ethereumetl/cli/export_tokens.py b/ethereumetl/cli/export_tokens.py index 941276254..1660c4279 100644 --- a/ethereumetl/cli/export_tokens.py +++ b/ethereumetl/cli/export_tokens.py @@ -31,7 +31,7 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.thread_local_proxy import ThreadLocalProxy from ethereumetl.providers.auto import get_provider_from_uri -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @@ -41,7 +41,7 @@ help='The file containing token addresses, one per line.') @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.') @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-c', '--chain', default='ethereum', show_default=True, type=str, help='The chain network to connect to.') diff --git a/ethereumetl/cli/extract_tokens.py b/ethereumetl/cli/extract_tokens.py index 05e7d057a..209650016 100644 --- a/ethereumetl/cli/extract_tokens.py +++ b/ethereumetl/cli/extract_tokens.py @@ -33,6 +33,7 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.providers.auto import get_provider_from_uri from ethereumetl.thread_local_proxy import ThreadLocalProxy +from ethereumetl.utils import get_default_provider_uri from ethereumetl.web3_utils import build_web3 logging_basic_config() @@ -40,7 +41,7 @@ @click.command(context_settings=dict(help_option_names=['-h', '--help'])) @click.option('-c', '--contracts', type=str, required=True, help='The JSON file containing contracts.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.') diff --git a/ethereumetl/cli/get_block_range_for_date.py b/ethereumetl/cli/get_block_range_for_date.py index 970aa518c..e7993487e 100644 --- a/ethereumetl/cli/get_block_range_for_date.py +++ b/ethereumetl/cli/get_block_range_for_date.py @@ -30,13 +30,13 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.service.eth_service import EthService from ethereumetl.providers.auto import get_provider_from_uri -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @click.command(context_settings=dict(help_option_names=['-h', '--help'])) -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-d', '--date', required=True, type=lambda d: datetime.strptime(d, '%Y-%m-%d'), diff --git a/ethereumetl/cli/get_block_range_for_timestamps.py b/ethereumetl/cli/get_block_range_for_timestamps.py index ab218d77f..849e7da9c 100644 --- a/ethereumetl/cli/get_block_range_for_timestamps.py +++ b/ethereumetl/cli/get_block_range_for_timestamps.py @@ -29,13 +29,13 @@ from blockchainetl.logging_utils import logging_basic_config from ethereumetl.providers.auto import get_provider_from_uri from ethereumetl.service.eth_service import EthService -from ethereumetl.utils import check_classic_provider_uri +from ethereumetl.utils import check_classic_provider_uri, get_default_provider_uri logging_basic_config() @click.command(context_settings=dict(help_option_names=['-h', '--help'])) -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-s', '--start-timestamp', required=True, type=int, help='Start unix timestamp, in seconds.') diff --git a/ethereumetl/cli/stream.py b/ethereumetl/cli/stream.py index 217208dba..c6f0a03d5 100644 --- a/ethereumetl/cli/stream.py +++ b/ethereumetl/cli/stream.py @@ -29,12 +29,13 @@ from ethereumetl.providers.auto import get_provider_from_uri from ethereumetl.streaming.item_exporter_creator import create_item_exporters from ethereumetl.thread_local_proxy import ThreadLocalProxy +from ethereumetl.utils import get_default_provider_uri @click.command(context_settings=dict(help_option_names=['-h', '--help'])) @click.option('-l', '--last-synced-block-file', default='last_synced_block.txt', show_default=True, type=str, help='') @click.option('--lag', default=0, show_default=True, type=int, help='The number of blocks to lag behind the network.') -@click.option('-p', '--provider-uri', default='https://mainnet.infura.io', show_default=True, type=str, +@click.option('-p', '--provider-uri', default=get_default_provider_uri, show_default=True, type=str, help='The URI of the web3 provider e.g. ' 'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io') @click.option('-o', '--output', type=str, diff --git a/ethereumetl/providers/auto.py b/ethereumetl/providers/auto.py index e10d568db..d3c3b7ff2 100644 --- a/ethereumetl/providers/auto.py +++ b/ethereumetl/providers/auto.py @@ -23,6 +23,7 @@ from urllib.parse import urlparse +import mesc from web3 import IPCProvider, HTTPProvider from ethereumetl.providers.ipc import BatchIPCProvider @@ -32,6 +33,15 @@ def get_provider_from_uri(uri_string, timeout=DEFAULT_TIMEOUT, batch=False): + + if mesc.is_mesc_enabled(): + try: + endpoint = mesc.get_endpoint_by_query(uri_string, profile='ethereum_etl') + if endpoint is not None: + uri_string = endpoint['url'] + except Exception as e: + print('MESC configured improperly') + uri = urlparse(uri_string) if uri.scheme == 'file': if batch: diff --git a/ethereumetl/utils.py b/ethereumetl/utils.py index 1d2512b64..c2b59f7f5 100644 --- a/ethereumetl/utils.py +++ b/ethereumetl/utils.py @@ -24,6 +24,8 @@ import itertools import warnings +import mesc + from ethereumetl.misc.retriable_value_error import RetriableValueError @@ -142,3 +144,15 @@ def check_classic_provider_uri(chain, provider_uri): warnings.warn("ETC Chain not supported on Infura.io. Using https://ethereumclassic.network instead") return 'https://ethereumclassic.network' return provider_uri + + +def get_default_provider_uri(): + if mesc.is_mesc_enabled(): + try: + endpoint = mesc.get_default_endpoint(profile='ethereum_etl') + if endpoint is not None: + return endpoint['url'] + except Exception: + print('MESC configured improperly') + + return 'https://mainnet.infura.io' diff --git a/setup.py b/setup.py index ab6b8843d..b47b17a82 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ def read(fname): 'web3>=5.29,<6', 'eth-utils==1.10', 'eth-abi>=2.2.0,<3.0.0', + 'mesc==0.2.0', # TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi 'python-dateutil>=2.8.0,<3', 'click>=8.0.4,<9',