We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? # to your account
Currently s3fs and boto3 are required dependencies of dask_deltatable. Can they be put in an s3 or aws optional requirements list?
s3
aws
This diff should do the trick
diff --git a/dask_deltatable/core.py b/dask_deltatable/core.py index f833c16..1fa85ac 100644 --- a/dask_deltatable/core.py +++ b/dask_deltatable/core.py @@ -6,7 +6,6 @@ from urllib.parse import urlparse import dask import dask.dataframe as dd import pyarrow.parquet as pq -from boto3 import Session from dask.base import tokenize from dask.dataframe.io import from_delayed from dask.delayed import delayed @@ -172,6 +171,8 @@ class DeltaTableWrapper(object): def _read_from_catalog( database_name: str, table_name: str, **kwargs ) -> dd.core.DataFrame: + from boto3 import Session + if ("AWS_ACCESS_KEY_ID" not in os.environ) and ( "AWS_SECRET_ACCESS_KEY" not in os.environ ): diff --git a/requirements.txt b/requirements.txt index d12c835..5233228 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ dask[dataframe,distribuited] -boto3 deltalake==0.6.3 fsspec==2022.10.0 pyarrow==10.0.0 -s3fs==2022.10.0 diff --git a/setup.py b/setup.py index f4428f8..3fe0698 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ from setuptools import setup with open("README.md", "r", encoding="utf-8") as f: long_description = f.read() + setup( name="dask-deltatable", version="0.3rc", @@ -17,7 +18,10 @@ setup( long_description_content_type="text/markdown", python_requires=">=3.7", install_requires=open("requirements.txt").read().strip().split("\n"), - extras_require={"dev": ["pytest", "requests", "pytest-cov>=2.10.1"]}, + extras_require={ + "dev": ["pytest", "requests", "pytest-cov>=2.10.1"], + "s3": ["s3fs==2022.10.0", "boto3"], + }, package_data={"dask_deltatable": ["*.pyi" "__init__.pyi", "core.pyi"]}, include_package_data=True, zip_safe=False,
The text was updated successfully, but these errors were encountered:
Closed via #9
Sorry, something went wrong.
No branches or pull requests
Currently s3fs and boto3 are required dependencies of dask_deltatable. Can they be put in an
s3
oraws
optional requirements list?This diff should do the trick
The text was updated successfully, but these errors were encountered: