Release: v0.2.5 (#255)

* delete nn

* Add support for loading train and test set in cifar10" (#193)

* Fix issue where tensor columns can't be indexed with pandas series (#195)

* Update cifar10 to support test set too (#196)

* Fix bacckwards compat issue with base_dir and gcs_image_column (#197)

* Support backwards compatibility with nn (#198)

* Bump version (#199)

* Update contributing to support new dev main structure (#203)

* Add args, kwargs to ColumnIOMixin._read_data (#204)

Co-authored-by: Jesse Vig <45317205+jessevig@users.noreply.github.com>

* Fix from_huggingface and add tests (#205)

closes #201

* allow_pickle=true when loading numpy block (#206)

* Add downloader to ImageColumn (#207)

* Remove default addition of index (#208)

* Remove default addition of index

* Fix provenance tests

* Add DEW contrib to registry (#209)

* Catch ConnectionResetError (#210)

* Add inaturalist to contrib (#211)

* Add inaturalist to contrib

* Add annotations to intarualist

* Fix issue where arraycolumns can't be saved with jsonlines (#214)

* Update the docs and add user guide. (#215)

* Add contrib for enron (#217)

* Fix PIL attribute error on list column representation (#218)

* mmap path bug fix (#219)

* Downgrade pytorch dependency bound (#220)

* Fix issue with subclassing datapanel _state_keys (#224)

* Use multiple slices instead of pa.Table.take in ArrowBlock (#226)

* Fix issue where boolean list can't index (#227)

* Add support for AudioColumn (#222)

* Add waterbirds (#228)

* Add use guide to indexing and stubs for remaining sections (#225)

* Docs/build fix (#230)

* Bump version (#231)

* Audioset DataPanel (#229)

* Add the audioset dataset

* Add AudioColumn to audioset datapanel

* Fix issue where old datapanels didn't have formatter state (#233)

* Make audioset datapanels relational (#235)

* Add coco, mir, and pascal (#239)

* Make write only write columns in datapanel (#240)

* Enforce contiguous index in pandas columns (#244)

* Fix issue where ray pickle fails on lazy loader (#245)

* Add support for groupby operation

* Reorganize the implementation of datasets (#246)

* Add support for persistent configuration (#247)

* Implement sort for data panel and columns (#237)

* Add emb module (#249)

* clusterby stuff

* Add clusterby

* clusterby stuff

* Add clusterby

* Add embed op (#248)

* Autoformat

Co-authored-by: Sam Randall <1billionmore@gmail.com>

* Reorganize ops code (#250)

* Update CI to include 3.9 and 3.10 and to drop 3.7

* Add sample (#251)

* Update ci.yml

* Add several HAPI datasets  (#252)

* Update styling of docs (#253)

* Bump version (#254)

* Remove fastbpe

Co-authored-by: Karan Goel <kgoel93@gmail.com>
Co-authored-by: Karan Goel <kgoel@cs.stanford.edu>
Co-authored-by: Jesse Vig <45317205+jessevig@users.noreply.github.com>
Co-authored-by: Khaled Saab <36782882+khaledsaab@users.noreply.github.com>
Co-authored-by: Priya2698 <52657555+Priya2698@users.noreply.github.com>
Co-authored-by: sam-randall <38796503+sam-randall@users.noreply.github.com>
Co-authored-by: Hannah Kim <61199762+hannahkim24@users.noreply.github.com>
Co-authored-by: Sam Randall <1billionmore@gmail.com>

.coveragerc

@@ -12,6 +12,6 @@ exclude_lines =
 ignore_errors = True
 omit =
     tests/*
-    meerkat/contrib/*
+    meerkat/datasets/*
     meerkat/ml/*
     setup.py

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ['3.7', '3.8']
+                python-version: ['3.8', '3.9', '3.10']
 
         steps:
             -   uses: actions/checkout@v2
@@ -43,7 +43,7 @@ jobs:
         runs-on: ubuntu-latest
         strategy:
             matrix:
-                python-version: ['3.7', '3.8']
+                python-version: ['3.8', '3.9', '3.10']
 
         steps:
             -   uses: actions/checkout@v2
@@ -72,7 +72,7 @@ jobs:
         strategy:
             matrix:
                 os: [ubuntu-latest] #, macos-latest]
-                python-version: ['3.7', '3.8']
+                python-version: ['3.8', '3.9', '3.10']
 
         steps:
             -   uses: actions/checkout@v2
@@ -105,7 +105,7 @@ jobs:
         strategy:
             matrix:
                 os: [ubuntu-latest] #, macos-latest]
-                python-version: ['3.7', '3.8']
+                python-version: ['3.8', '3.9', '3.10']
 
         steps:
             -   uses: actions/checkout@v2

.gitignore

@@ -146,3 +146,9 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
+# Giant file
+examples/datasets
+
+# History
+.history
+

.pre-commit-config.yaml

@@ -8,6 +8,8 @@ repos:
     hooks:
     - id: black
       language_version: python3
+      additional_dependencies: ['click==8.0.4']
+
 -   repo: https://gitlab.com/pycqa/flake8
     rev: 3.8.4
     hooks:

Makefile

@@ -20,7 +20,7 @@ test-cov:
 	pytest --cov=./ --cov-report=xml
 
 docs:
-	sphinx-apidoc ../meerkat -o source/apidocs -f --tocfile "API Docs"
+	sphinx-apidoc meerkat -o docs/source/apidocs -f --tocfile "apidoc"
 	sphinx-build -b html docs/source/ docs/build/html/
 
 docs-check:

docs/Makefile

@@ -1,6 +1,10 @@
 # Minimal makefile for Sphinx documentation
 #
 
+
+build_datasets_docs:
+	python source/datasets/build_datasets_docs.py 
+
 # You can set these variables from the command line, and also
 # from the environment for the first two.
 SPHINXOPTS    ?=
@@ -16,5 +20,5 @@ help:
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
+%: Makefile build_datasets_docs
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/requirements.txt

@@ -7,4 +7,4 @@ pydata_sphinx_theme
 ipython
 jupyter-sphinx
 sphinx-panels
-.
+.

docs/source/_static/css/custom.css

@@ -0,0 +1,7 @@
+html[data-theme="light"] {
+    /*****************************************************************************
+    * main colors
+    */
+    --pst-color-primary: rgb(19,7,83);
+    --pst-color-secondary: rgb(252,177,62);
+}

docs/source/apidocs/apidoc.rst

@@ -0,0 +1,7 @@
+meerkat
+=======
+
+.. toctree::
+   :maxdepth: 4
+
+   meerkat

docs/source/apidocs/index.rst

@@ -0,0 +1,7 @@
+API Reference
+=============
+
+.. toctree::
+   :maxdepth: 4
+
+   meerkat

docs/source/apidocs/meerkat.block.rst

@@ -1,11 +1,6 @@
 meerkat.block package
 =====================
 
-.. automodule:: meerkat.block
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 Submodules
 ----------
 
@@ -64,3 +59,11 @@ meerkat.block.tensor\_block module
    :members:
    :undoc-members:
    :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: meerkat.block
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/apidocs/meerkat.cells.rst

@@ -1,11 +1,6 @@
 meerkat.cells package
 =====================
 
-.. automodule:: meerkat.cells
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 Submodules
 ----------
 
@@ -25,18 +20,18 @@ meerkat.cells.spacy module
    :undoc-members:
    :show-inheritance:
 
-meerkat.cells.video module
---------------------------
+meerkat.cells.volume module
+---------------------------
 
-.. automodule:: meerkat.cells.video
+.. automodule:: meerkat.cells.volume
    :members:
    :undoc-members:
    :show-inheritance:
 
-meerkat.cells.volume module
----------------------------
+Module contents
+---------------
 
-.. automodule:: meerkat.cells.volume
+.. automodule:: meerkat.cells
    :members:
    :undoc-members:
    :show-inheritance:

docs/source/apidocs/meerkat.columns.rst

@@ -1,11 +1,6 @@
 meerkat.columns package
 =======================
 
-.. automodule:: meerkat.columns
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 Submodules
 ----------
 
@@ -25,6 +20,14 @@ meerkat.columns.arrow\_column module
    :undoc-members:
    :show-inheritance:
 
+meerkat.columns.audio\_column module
+------------------------------------
+
+.. automodule:: meerkat.columns.audio_column
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 meerkat.columns.cell\_column module
 -----------------------------------
 
@@ -33,6 +36,14 @@ meerkat.columns.cell\_column module
    :undoc-members:
    :show-inheritance:
 
+meerkat.columns.file\_column module
+-----------------------------------
+
+.. automodule:: meerkat.columns.file_column
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 meerkat.columns.image\_column module
 ------------------------------------
 
@@ -89,18 +100,18 @@ meerkat.columns.tensor\_column module
    :undoc-members:
    :show-inheritance:
 
-meerkat.columns.video\_column module
-------------------------------------
+meerkat.columns.volume\_column module
+-------------------------------------
 
-.. automodule:: meerkat.columns.video_column
+.. automodule:: meerkat.columns.volume_column
    :members:
    :undoc-members:
    :show-inheritance:
 
-meerkat.columns.volume\_column module
--------------------------------------
+Module contents
+---------------
 
-.. automodule:: meerkat.columns.volume_column
+.. automodule:: meerkat.columns
    :members:
    :undoc-members:
    :show-inheritance: