diff --git a/Dockerfile b/Dockerfile index bf07ca7..a2158d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7 AS base +FROM python:3.6 AS base ARG CI_USER_TOKEN ARG AZURE_BLOB_ACCOUNT_NAME ARG AZURE_BLOB_ACCOUNT_KEY diff --git a/Makefile b/Makefile index 25a360a..8dff814 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,7 @@ dist: clean ## builds source and wheel package python setup.py sdist python setup.py bdist_wheel ls -l dist + twine check dist/* install: clean ## install the package to the active Python's site-packages python setup.py install diff --git a/Pipfile b/Pipfile index 0cf6511..1a668df 100644 --- a/Pipfile +++ b/Pipfile @@ -6,8 +6,6 @@ name = "pypi" [packages] dask = "*" azure-storage = "*" -"unittest2" = "*" -tox = "*" [dev-packages] ipython = "*" @@ -20,7 +18,8 @@ cloudpickle = "*" numpy = "*" pandas = "*" toolz = "*" -pytest-cov = "*" +"unittest2" = "*" +twine = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index eaab0b2..07eba32 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6e1a30f42ed15483b6ccdd1aad927abdd9e4fe95f3a71e9d4ff7ddee741a59cc" + "sha256": "e80d7658bf365d8f6ca06cb37f8790a5cf995b26abc5ed213a475aa8a3f4d059" }, "pipfile-spec": 6, "requires": { @@ -16,13 +16,6 @@ ] }, "default": { - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "version": "==1.4.0" - }, "asn1crypto": { "hashes": [ "sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87", @@ -136,13 +129,6 @@ "index": "pypi", "version": "==1.0.0" }, - "filelock": { - "hashes": [ - "sha256:b8d5ca5ca1c815e1574aee746650ea7301de63d87935b3463d26368b76e31633", - "sha256:d610c1bb404daf85976d7a82eb2ada120f04671007266b708606565dd03b5be6" - ], - "version": "==3.0.10" - }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -150,27 +136,6 @@ ], "version": "==2.7" }, - "linecache2": { - "hashes": [ - "sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c", - "sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef" - ], - "version": "==1.0.0" - }, - "pluggy": { - "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" - ], - "version": "==0.8.0" - }, - "py": { - "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" - ], - "version": "==1.7.0" - }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" @@ -198,50 +163,12 @@ ], "version": "==1.11.0" }, - "toml": { - "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", - "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" - ], - "version": "==0.10.0" - }, - "tox": { - "hashes": [ - "sha256:513e32fdf2f9e2d583c2f248f47ba9886428c949f068ac54a0469cac55df5862", - "sha256:75fa30e8329b41b664585f5fb837e23ce1d7e6fa1f7811f2be571c990f9d911b" - ], - "index": "pypi", - "version": "==3.5.3" - }, - "traceback2": { - "hashes": [ - "sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030", - "sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23" - ], - "version": "==1.4.0" - }, - "unittest2": { - "hashes": [ - "sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8", - "sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579" - ], - "index": "pypi", - "version": "==1.1.0" - }, "urllib3": { "hashes": [ "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], "version": "==1.24.1" - }, - "virtualenv": { - "hashes": [ - "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", - "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" - ], - "version": "==16.1.0" } }, "develop": { @@ -252,6 +179,13 @@ ], "version": "==0.7.12" }, + "argparse": { + "hashes": [ + "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", + "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + ], + "version": "==1.4.0" + }, "atomicwrites": { "hashes": [ "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", @@ -280,6 +214,13 @@ ], "version": "==0.1.0" }, + "bleach": { + "hashes": [ + "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", + "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + ], + "version": "==3.0.2" + }, "certifi": { "hashes": [ "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", @@ -302,52 +243,6 @@ "index": "pypi", "version": "==0.6.1" }, - "coverage": { - "hashes": [ - "sha256:06123b58a1410873e22134ca2d88bd36680479fe354955b3579fb8ff150e4d27", - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:0d34245f824cc3140150ab7848d08b7e2ba67ada959d77619c986f2062e1f0e8", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:258b21c5cafb0c3768861a6df3ab0cfb4d8b495eee5ec660e16f928bf7385390", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:3ad59c84c502cd134b0088ca9038d100e8fb5081bbd5ccca4863f3804d81f61d", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:71afc1f5cd72ab97330126b566bbf4e8661aab7449f08895d21a5d08c6b051ff", - "sha256:7349c27128334f787ae63ab49d90bf6d47c7288c63a0a5dfaa319d4b4541dd2c", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:859714036274a75e6e57c7bab0c47a4602d2a8cfaaa33bbdb68c8359b2ed4f5c", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:869ef4a19f6e4c6987e18b315721b8b971f7048e6eaea29c066854242b4e98d9", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:977e2d9a646773cc7428cdd9a34b069d6ee254fadfb4d09b3f430e95472f3cf3", - "sha256:99bd767c49c775b79fdcd2eabff405f1063d9d959039c0bdd720527a7738748a", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" - ], - "version": "==4.5.2" - }, "decorator": { "hashes": [ "sha256:2c51dff8ef3c447388fe5e4453d24a2bf128d3a4c32af3fabef1f01c6851ab82", @@ -414,6 +309,13 @@ ], "version": "==2.10" }, + "linecache2": { + "hashes": [ + "sha256:4b26ff4e7110db76eeb6f5a7b64a82623839d595c2038eeda662f2a2db78e97c", + "sha256:e78be9c0a0dfcbac712fe04fbf92b96cddae80b1b842f24248214c8496f006ef" + ], + "version": "==1.0.0" + }, "markupsafe": { "hashes": [ "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", @@ -551,6 +453,13 @@ ], "version": "==0.7.5" }, + "pkginfo": { + "hashes": [ + "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", + "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + ], + "version": "==1.4.2" + }, "pluggy": { "hashes": [ "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", @@ -617,14 +526,6 @@ "index": "pypi", "version": "==4.0.1" }, - "pytest-cov": { - "hashes": [ - "sha256:513c425e931a0344944f84ea47f3956be0e416d95acbd897a44970c8d926d5d7", - "sha256:e360f048b7dae3f2f2a9a4d067b2dd6b6a015d384d1577c994a43f3f7cbad762" - ], - "index": "pypi", - "version": "==2.6.0" - }, "pytest-runner": { "hashes": [ "sha256:d23f117be39919f00dd91bffeb4f15e031ec797501b717a245e377aee0f577be", @@ -647,6 +548,13 @@ ], "version": "==2018.7" }, + "readme-renderer": { + "hashes": [ + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" + ], + "version": "==24.0" + }, "requests": { "hashes": [ "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", @@ -654,6 +562,13 @@ ], "version": "==2.20.1" }, + "requests-toolbelt": { + "hashes": [ + "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", + "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + ], + "version": "==0.8.0" + }, "setuptools-scm": { "hashes": [ "sha256:1191f2a136b5e86f7ca8ab00a97ef7aef997131f1f6d4971be69a1ef387d8b40", @@ -702,6 +617,20 @@ "index": "pypi", "version": "==0.9.0" }, + "tqdm": { + "hashes": [ + "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", + "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + ], + "version": "==4.28.1" + }, + "traceback2": { + "hashes": [ + "sha256:05acc67a09980c2ecfedd3423f7ae0104839eccb55fc645773e1caa0951c3030", + "sha256:8253cebec4b19094d67cc5ed5af99bf1dba1285292226e98a31929f87a5d6b23" + ], + "version": "==1.4.0" + }, "traitlets": { "hashes": [ "sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835", @@ -709,6 +638,22 @@ ], "version": "==4.3.2" }, + "twine": { + "hashes": [ + "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", + "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" + ], + "index": "pypi", + "version": "==1.12.1" + }, + "unittest2": { + "hashes": [ + "sha256:13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8", + "sha256:22882a0e418c284e1f718a822b3b022944d53d2d908e1690b319a9d3eb2c0579" + ], + "index": "pypi", + "version": "==1.1.0" + }, "urllib3": { "hashes": [ "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", @@ -722,6 +667,13 @@ "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], "version": "==0.1.7" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" } } } diff --git a/README.rst b/README.rst index b38b1a2..1527d04 100644 --- a/README.rst +++ b/README.rst @@ -11,23 +11,57 @@ Azure Blob Storage Backend for Dask :target: https://dask-azureblobfs.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status + Features -------- * Supports dask when your data files are stored in the cloud. - * Import `DaskAzureBlobFileSystem` + * Import ``DaskAzureBlobFileSystem`` + + * Use ``abfs://`` as protocol prefix and you are good to do. + +* For authentication, please read more on Usage_. + +* Support for key-value storage which is backed by azure storage. Create an instance of ``AzureBlobMap`` + + +Usage +----- + +Make the right imports:: + + from azureblobfs.dask import DaskAzureBlobFileSystem + import dask.dataframe as dd + +then put all data files in an azure storage container say ``clippy``, then you can read it:: + + data = dd.read_csv("abfs://noaa/clippy/weather*.csv") + max_by_state = data.groupby("states").max().compute() + +you would need to set your azure account name in environment variable ``AZURE_BLOB_ACCOUNT_NAME`` +(which in our above example is ``noaa``) and the account key in ``AZURE_BLOB_ACCOUNT_KEY``. + +If you don't want to use account key and instead want to use SAS, set it in the +environment variable ``AZURE_BLOB_SAS_TOKEN`` along with the connection string in the +environment variable ``AZURE_BLOB_CONNECTION_STRING``. + +Installation +------------ + +Just:: - * Use `abfs://` as protocol prefix and you are good to do. + pip install dask-azureblobfs -* For authentication, please read more on :ref:`Usage`. +or get the development version if you love to live dangerously:: -* Support for key-value storage which is backed by azure storage. Create an instance of `AzureBlobMap` + pip install git+https://github.com/manish/dask-azureblobfs@master#egg=dask-azureblobfs Credits ------- This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. +.. _Usage: https://dask-azureblobfs.readthedocs.io/en/latest/usage.html .. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage diff --git a/setup.py b/setup.py index bbc315d..d924313 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/manish/dask-azureblobfs', - version='0.1.0', + use_scm_version=True, zip_safe=False, )