diff --git a/.circleci/config.yml b/.circleci/config.yml index ed5a05f5e9..3251d16844 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,12 @@ workflows: name: Linter | v3.10 | Linux mode: LINTER ################################ + ## DOC: All + ################################ + ################################ + - Doc: + name: Doc | v3.10 | Linux + ################################ #### PIP: Master ################################ ################################ @@ -178,8 +184,8 @@ jobs: RAY: << parameters.ray >> command: | "python<< parameters.v >>" -m venv test_evadb - pip install --upgrade pip source test_evadb/bin/activate + pip install --upgrade pip if [ $RAY = "ENABLED" ]; then if [ $PY_VERSION != "3.11" ]; then pip install ".[dev,ray,qdrant]" @@ -225,6 +231,47 @@ jobs: paths: - test_evadb + Doc: + parameters: + v: + type: string + default: "3.10" + mode: + type: string + default: "DOC" + resource_class: large + docker: + # https://circleci.com/docs/circleci-images#language-image-variants + - image: "cimg/python:<< parameters.v >>" + steps: + + - checkout + + # Restore pip wheel + - restore_cache: + keys: + - v1-pip-wheel_cache-python<< parameters.v >>-{{ checksum "docs/requirements.txt" }} + + - run: + name: Install EvaDB Doc dependencies + command: | + "python<< parameters.v >>" -m venv test_evadb_doc + source test_evadb_doc/bin/activate + pip install --upgrade pip + pip install -r docs/requirements.txt + + - run: + name: Test doc build and link validation + no_output_timeout: 10m # 10 minute timeout + command: | + source test_evadb_doc/bin/activate + bash script/test/test.sh -m "<< parameters.mode >>" + + - save_cache: + key: v1-pip-wheel_cache-python<< parameters.v >>-{{ checksum "docs/requirements.txt" }} + paths: + - test_evadb_doc + Postgres: parameters: v: diff --git a/docs/Makefile b/docs/Makefile index 6c27cca707..c17f086e1e 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= -vvv +SPHINXOPTS ?= -vvv -W SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build @@ -21,4 +21,4 @@ help: clean: rm -rf $(BUILDDIR)/* - rm -rf ./source/reference/python/doc/* \ No newline at end of file + rm -rf ./source/reference/python/doc/* diff --git a/docs/Readme.md b/docs/README.md similarity index 76% rename from docs/Readme.md rename to docs/README.md index fa252de75d..7bd445ca80 100644 --- a/docs/Readme.md +++ b/docs/README.md @@ -8,9 +8,8 @@ make html open _build/html/index.html ``` -To test links: +To further test external links: ``` -cd evadb/docs -sphinx-build . _build -b linkcheck -``` \ No newline at end of file +make linkcheck +``` diff --git a/docs/_toc.yml b/docs/_toc.yml index ae1a4950f4..b66a3932c7 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -8,7 +8,7 @@ parts: - file: source/overview/getting-started/install-guide title: Installation Guide - file: source/overview/concepts - # - file: source/overview/faq + #- file: source/overview/faq - caption: Use Cases chapters: @@ -83,7 +83,7 @@ parts: sections: - file: source/dev-guide/contribute/setup-dev title: Setup Environment - - file: source/contribute/contribute/testing + - file: source/dev-guide/contribute/testing title: Testing - file: source/dev-guide/contribute/submit-pr title: Submit a PR diff --git a/docs/conf.py b/docs/conf.py index 601fd64abe..1faef3ecc8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,7 @@ "sphinx_copybutton", "sphinx.ext.doctest", "sphinx.ext.coverage", - "sphinx.ext.autosectionlabel", +# "sphinx.ext.autosectionlabel", "sphinx.ext.autosummary", "sphinx.ext.autodoc", "sphinx.ext.autodoc.typehints", @@ -48,6 +48,9 @@ "versionwarning.extension", "IPython.sphinxext.ipython_console_highlighting", ] + +suppress_warnings = ["etoc.toctree", "myst.header"] + source_suffix = [".ipynb", ".html", ".md", ".rst"] autodoc_pydantic_model_show_json = False @@ -91,7 +94,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md", "images/reference/README.md"] # The name of the Pygments (syntax highlighting) style to use. # pygments_style = "lovelace" @@ -148,14 +151,21 @@ "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/solid.min.css", "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/brands.min.css"] -# Adding the Tutorial notebooks to ./docs/source/tutorials/ +# Check link: https://stackoverflow.com/questions/14492743/have-sphinx-report-broken-links/14735060#14735060 +nitpicky = True +# BUG: https://stackoverflow.com/questions/11417221/sphinx-autodoc-gives-warning-pyclass-reference-target-not-found-type-warning +nitpick_ignore_regex = [('py:class', r'.*')] + for i in os.listdir("../tutorials"): - if i.endswith(".ipynb"): + if i in [ + "02-object-detection.ipynb", + "03-emotion-analysis.ipynb", + "07-object-segmentation-huggingface.ipynb" + ]: shutil.copy(f"../tutorials/{i}", "./source/usecases/") -jupyter_execute_notebooks = "off" - +nb_execution_mode = "off" # -- Initialize Sphinx ---------------------------------------------- def setup(app): @@ -165,4 +175,4 @@ def setup(app): message=r".*Container node skipped.*", ) # Custom JS - app.add_js_file("js/top-navigation.js", defer="defer") \ No newline at end of file + app.add_js_file("js/top-navigation.js", defer="defer") diff --git a/docs/index.rst b/docs/index.rst index d607a4651b..579eaa1eef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -95,12 +95,12 @@ Key Features Try it out! ------------ -- `PrivateGPT `_ -- `Video Question Answering using ChatGPT `_ -- `Querying PDF documents `_ -- `Analyzing traffic flow at an intersection `_ -- `Examining the emotion palette of actors in a movie `_ -- `Classifying images based on their content `_ +- `PrivateGPT `_ +- `Video Question Answering using ChatGPT `_ +- `Querying PDF documents `_ +- `Analyzing traffic flow at an intersection `_ +- `Examining the emotion palette of actors in a movie `_ +- `Classifying images based on their content `_ diff --git a/docs/source/benchmarks/text_summarization.rst b/docs/source/benchmarks/text_summarization.rst index 0f93896b42..75af9a0fcf 100644 --- a/docs/source/benchmarks/text_summarization.rst +++ b/docs/source/benchmarks/text_summarization.rst @@ -1,9 +1,9 @@ Text summarization benchmark -==== +============================ In this benchmark, we compare the performance of text summarization between EvaDB and MindsDB on `CNN-DailyMail News `_. 1. Prepare dataset ----- +------------------ .. code-block:: bash @@ -11,7 +11,7 @@ In this benchmark, we compare the performance of text summarization between EvaD bash download_dataset.sh 2. Using EvaDB to summarize the CNN DailyMail News ----- +-------------------------------------------------- .. note:: @@ -24,10 +24,10 @@ In this benchmark, we compare the performance of text summarization between EvaD 3. Using MindsDB to summarize the CNN DailyMail News ----- +---------------------------------------------------- Prepare sqlite database for MindsDB -**** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: bash @@ -38,7 +38,8 @@ Prepare sqlite database for MindsDB Install MindsDB -**** +~~~~~~~~~~~~~~~ + Follow the `Setup for Source Code via pip `_ to install mindsdb. .. note:: @@ -52,11 +53,11 @@ After the installation, we use mysql cli to connect to MindsDB. Replace the port mysql -h 127.0.0.1 --port 47335 -u mindsdb -p Run Experiment -**** +~~~~~~~~~~~~~~ Connect the sqlite database we created before. -.. code-block:: sql +.. code-block:: text CREATE DATABASE sqlite_datasource WITH ENGINE = 'sqlite', @@ -66,7 +67,7 @@ Connect the sqlite database we created before. Create text summarization model and wait for its readiness. -.. code-block:: sql +.. code-block:: text CREATE MODEL mindsdb.hf_bart_sum_20 PREDICT PRED @@ -82,7 +83,7 @@ Create text summarization model and wait for its readiness. Use the model to summarize the CNN DailyMail news. -.. code-block:: sql +.. code-block:: text CREATE OR REPLACE TABLE sqlite_datasource.cnn_news_summary ( SELECT PRED @@ -92,8 +93,8 @@ Use the model to summarize the CNN DailyMail news. 4. Experiment results ----- -Below are nubmers from a server with 56 Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz and two Quadro P6000 GPU. +--------------------- +Below are numbers from a server with 56 Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz and two Quadro P6000 GPU. .. list-table:: Text summarization with ``sshleifer/distilbart-cnn-12-6`` on CNN-DailyMail News diff --git a/docs/source/dev-guide/contribute.rst b/docs/source/dev-guide/contribute.rst index 8f580a41c8..fc2f216d37 100644 --- a/docs/source/dev-guide/contribute.rst +++ b/docs/source/dev-guide/contribute.rst @@ -6,8 +6,8 @@ We welcome all kinds of contributions to EvaDB. - `Code reviews `_ - `Improving documentation `_ - `Tutorials and applications `_ -- `New features `__ +- :ref:`Extending EvaDB` This document goes over how to contribute to EvaDB in details. -.. tableofcontents:: \ No newline at end of file +.. tableofcontents:: diff --git a/docs/source/dev-guide/contribute/setup-dev.rst b/docs/source/dev-guide/contribute/setup-dev.rst index 0e9dd1225c..0811cca51f 100644 --- a/docs/source/dev-guide/contribute/setup-dev.rst +++ b/docs/source/dev-guide/contribute/setup-dev.rst @@ -27,10 +27,10 @@ After installing the package locally, you can make changes and run the test case .. note:: EvaDB provides multiple installation options for extending its functionalities. - Please see :doc:`Installation Guide ` for all options. + Please see :ref:`installation guide` for all options. Other options can be installed with the ``dev`` environment. .. code-block:: bash - pip install -e ".[dev,ludwig]" \ No newline at end of file + pip install -e ".[dev,ludwig]" diff --git a/docs/source/dev-guide/contribute/submit-pr.rst b/docs/source/dev-guide/contribute/submit-pr.rst index 59040fc4ff..1fc2917e07 100644 --- a/docs/source/dev-guide/contribute/submit-pr.rst +++ b/docs/source/dev-guide/contribute/submit-pr.rst @@ -18,7 +18,7 @@ Merge the most recent changes from the ``staging`` branch git pull . origin/staging Testing -------- +----------------- Run the :doc:`test script <./testing>` to ensure that all the test cases pass. @@ -35,4 +35,4 @@ Run the following command to ensure that code is properly formatted. .. code-block:: python - python script/formatting/formatter.py \ No newline at end of file + python script/formatting/formatter.py diff --git a/docs/source/dev-guide/extend.rst b/docs/source/dev-guide/extend.rst index 05d3d27a4d..69a119e7da 100644 --- a/docs/source/dev-guide/extend.rst +++ b/docs/source/dev-guide/extend.rst @@ -1,5 +1,7 @@ +.. _Extending EvaDB: + Extending EvaDB -==== +=============== This document details steps involved in extending EvaDB. .. tableofcontents:: diff --git a/docs/source/dev-guide/extend/new-command.rst b/docs/source/dev-guide/extend/new-command.rst index 337a78adc9..c5963c8aef 100644 --- a/docs/source/dev-guide/extend/new-command.rst +++ b/docs/source/dev-guide/extend/new-command.rst @@ -1,10 +1,10 @@ Operators / Commands -============= +==================== This document details the steps involved in adding support for a new operator (or command) in EvaDB. We illustrate the process using a DDL command. Command Handler ----- +--------------- An input query string is handled by **Parser**, **StatementTOPlanConverter**, **PlanGenerator**, and **PlanExecutor**. @@ -83,7 +83,7 @@ parser/ .. _2-statementtoplanconverter: 2. Statement To Plan Converter ---------------------------- +------------------------------ The part transforms the statement into corresponding logical plan. @@ -154,13 +154,13 @@ Optimizer .. _3-plangenerator: 3. Plan Generator ----------------- +------------------ The part transformed logical plan to physical plan. The modified files are stored under **Optimizer** and **Planner** folders. plan_nodes/ -~~~~~~~~ +~~~~~~~~~~~~ - ``[cmd]_plan.py`` - class [cmd]Plan, which stored information required for rename table. @@ -220,7 +220,7 @@ optimizer/rules .. _4-PlanExecutor: 4. Plan Executor --------------- +----------------- ``PlanExecutor`` uses data stored in physical plan to run the command. diff --git a/docs/source/dev-guide/extend/new-data-source.rst b/docs/source/dev-guide/extend/new-data-source.rst index d8d37483b9..af5730a9f3 100644 --- a/docs/source/dev-guide/extend/new-data-source.rst +++ b/docs/source/dev-guide/extend/new-data-source.rst @@ -1,19 +1,19 @@ Structured Data Source Integration -==== -This document details steps invovled in adding a new structured data source integration in EvaDB. +================================== +This document details steps involved in adding a new structured data source integration in EvaDB. Example Data Source Integration In EvaDB ----- +---------------------------------------- - `PostgreSQL `_ Create Data Source Handler ----- +-------------------------- 1. Create a new directory at `evadb/third_party/databases/ `_ -~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. note:: @@ -32,7 +32,7 @@ The *__init__.py* can contain copyright information. The *requirements.txt* cont EvaDB will only install a data source's specific dependency libraries when a connection to the data source is created by the user via, e.g., `CREATE DATABASE mydb_source WITH ENGINE = "mydb";`. 2. Implement the data source handler -~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In *mydb_handler.py*, you need to implement the `DBHandler` declared at `evadb/third_party/databases/types.py `_. There are 7 functions that you need to implement: @@ -76,7 +76,7 @@ You can check the PostgreSQL's handler example at `evadb/third_party/databases/p Register the Data Source Handler ----- +-------------------------------- Add your created data source handler in `get_database_handler` function at `evadb/third_party/databases/interface.py `_. Below is an example of registering the created mydb data source: diff --git a/docs/source/dev-guide/internals.rst b/docs/source/dev-guide/internals.rst deleted file mode 100644 index 88370d2204..0000000000 --- a/docs/source/dev-guide/internals.rst +++ /dev/null @@ -1,44 +0,0 @@ -EvaDB Internals -============= - -Path of a Query -------------------- - -The following code represents a sequence of operations that can be used to execute a query in a evaql database. found in `evadb/server/command_handler.py `_ - - Parse the query using the Parser() function provided by the evaql library. The result of this step will be a parsed representation of the query in the form of an abstract syntax tree (AST). - -.. code:: python - - stmt = Parser().parse(query)[0] - -Bind the parsed AST to a statement context using the StatementBinder() function. This step resolves references to schema objects and performs other semantic checks on the query. - -.. code:: python - - StatementBinder(StatementBinderContext()).bind(stmt) - -Convert the bound AST to a logical plan using the StatementToPlanConvertor() function. This step generates a logical plan that specifies the sequence of operations needed to execute the query. - -.. code:: python - - l_plan = StatementToPlanConvertor().visit(stmt) - -Generate a physical plan from the logical plan using the plan_generator.build() function. This step optimizes the logical plan and generates a physical plan that specifies how the query will be executed. - -.. code:: python - - p_plan = plan_generator.build(l_plan) - -Execute the physical plan using the PlanExecutor() function. This step retrieves the data from the database and produces the final output of the query. - -.. code:: python - - output = PlanExecutor(p_plan).execute_plan() - -Overall, this sequence of operations represents the path of query execution in a evaql database, from parsing the query to producing the final output. - -Topics ------- - -.. tableofcontents:: diff --git a/docs/source/dev-guide/internals/catalog.rst b/docs/source/dev-guide/internals/catalog.rst deleted file mode 100644 index a5236f5ffb..0000000000 --- a/docs/source/dev-guide/internals/catalog.rst +++ /dev/null @@ -1,36 +0,0 @@ -Catalog -======= - -Catalog Manager ---------------- - -`catalog_manager `_ provides a set of services to interact with a database that stores metadata about tables, columns, and user-defined functions (UDFs). Information like what is the data type in a certain column in a table, type of a table, its name, etc.. It contains functions to get, insert and delete catalog entries for Tables, UDFs, UDF IOs, Columns and Indexes. - -This data is stored in the evadb_catalog.db file which can be found in `~/.evadb//` folder. - -Catalog manager currently has 5 services in it: - -.. code:: python - - TableCatalogService() - ColumnCatalogService() - UdfCatalogService() - UdfIOCatalogService() - IndexCatalogService() - -Catalog Services ----------------- - -This class provides functionality related to a table catalog, including inserting, getting, deleting, and renaming table entries, as well as retrieving all entries. e.g. the TableCatalogService contains code to get, insert and delete a table. - -Catalog Models --------------- - -These contain the data model that is used by the catalog services. -Each model represents a table in the underlying database. - -.. |pic1| image:: ../../../images/reference/catalog.png - :width: 45% - :alt: Catalog Diagram - -|pic1| diff --git a/docs/source/dev-guide/packaging.rst b/docs/source/dev-guide/packaging.rst index 9b16b0f391..0394d11ccf 100644 --- a/docs/source/dev-guide/packaging.rst +++ b/docs/source/dev-guide/packaging.rst @@ -1,3 +1,5 @@ +:orphan: + .. _guide-packaging: Packaging @@ -5,8 +7,6 @@ Packaging This section describes practices to follow when packaging your own models or datasets to be used along with EvaDB. -.. tableofcontents:: - Models -------- diff --git a/docs/source/dev-guide/release.rst b/docs/source/dev-guide/release.rst index 3d690348ce..0d43cf7a84 100644 --- a/docs/source/dev-guide/release.rst +++ b/docs/source/dev-guide/release.rst @@ -1,5 +1,5 @@ EvaDB Release Guide ----- +------------------- Before you start, make sure you have :doc:`setup a PyPI account <./release/pypi-account>`. diff --git a/docs/source/dev-guide/release/pypi-account.rst b/docs/source/dev-guide/release/pypi-account.rst index d7723cc773..c28e94c91f 100644 --- a/docs/source/dev-guide/release/pypi-account.rst +++ b/docs/source/dev-guide/release/pypi-account.rst @@ -1,5 +1,5 @@ Setup PyPI Account -==== +================== Make sure you have `PyPI `_ account with maintainer access to the EvaDB project. Create a .pypirc in your home directory. It should look like this: @@ -15,4 +15,4 @@ Create a .pypirc in your home directory. It should look like this: username=YOUR_USERNAME password=YOUR_PASSWORD -Then run ``chmod 600 ./.pypirc`` so that only you can read/write the file. \ No newline at end of file +Then run ``chmod 600 ./.pypirc`` so that only you can read/write the file. diff --git a/docs/source/dev-guide/release/release-steps.rst b/docs/source/dev-guide/release/release-steps.rst index 5a6e6bea26..baf33e38b3 100644 --- a/docs/source/dev-guide/release/release-steps.rst +++ b/docs/source/dev-guide/release/release-steps.rst @@ -1,5 +1,5 @@ Release Steps -==== +============= 1. Ensure that you're in the top-level ``eva`` directory. 2. Ensure that your branch is in sync with the ``master`` branch: @@ -66,7 +66,7 @@ merged since the last release. pip install twine # if not installed twine upload dist/* -r pypi -11. A PR is automatically submitted (this will take a few hours) on [`conda-forge/eva-feedstock`](https://github.com/conda-forge/eva-feedstock) to update the version. +11. A PR is automatically submitted (this will take a few hours) on [`conda-forge/eva-feedstock`] to update the version. * A maintainer needs to accept and merge those changes. 12. Create a new release on Github. @@ -106,4 +106,4 @@ merged since the last release. * For the tag ``v0.9.1`` * For ``latest`` -Credits: `Snorkel `_ \ No newline at end of file +Credits: `Snorkel `_ diff --git a/docs/source/overview/command-line.rst b/docs/source/overview/command-line.rst index 74da7ae6c6..9f518926b7 100644 --- a/docs/source/overview/command-line.rst +++ b/docs/source/overview/command-line.rst @@ -1,5 +1,7 @@ +:orphan: + Command Line Client -==== +=================== Besides Python files and Jupyter notebooks, EvaDB also supports a command line interface for querying the data. This interface allows for quick querying from the terminal: diff --git a/docs/source/overview/concepts.rst b/docs/source/overview/concepts.rst index c6ac3a354b..f0e116686f 100644 --- a/docs/source/overview/concepts.rst +++ b/docs/source/overview/concepts.rst @@ -87,9 +87,9 @@ EvaDB accelerates AI pipelines using a collection of optimizations inspired by S EvaDB supports an AI-oriented query language for analyzing both structured and unstructured data. Here are some illustrative apps: -The `Getting Started `__ page shows how you can use EvaDB for different AI tasks and how you can easily extend EvaDB to support your custom deep learning model through user-defined functions. +The :ref:`Getting Started` page shows how you can use EvaDB for different AI tasks and how you can easily extend EvaDB to support your custom deep learning model through user-defined functions. -The `User Guides `__ section contains Jupyter Notebooks that demonstrate how to use various features of EvaDB. Each notebook includes a link to Google Colab, where you can run the code yourself. +The :ref:`User Guides` section contains Jupyter Notebooks that demonstrate how to use various features of EvaDB. Each notebook includes a link to Google Colab, where you can run the code yourself. @@ -130,4 +130,4 @@ That's it! You can now use the newly registered UDF anywhere in the query -- in # Here, we are also filtering based on the output of the function query2 = query2.filter("MnistImageClassifier(data).label = '6' AND id < 10") query2 = query2.select("data, MnistImageClassifier(data).label") - response = query2.df() \ No newline at end of file + response = query2.df() diff --git a/docs/source/overview/docker.rst b/docs/source/overview/docker.rst index 4e71d23d7d..cdbf5019bf 100644 --- a/docs/source/overview/docker.rst +++ b/docs/source/overview/docker.rst @@ -1,5 +1,7 @@ +:orphan: + Docker -==== +====== You can launch the EvaDB server using Docker either locally or on a server with GPUs: diff --git a/docs/source/overview/faq.rst b/docs/source/overview/faq.rst index c3468aac53..28b1e5a70b 100644 --- a/docs/source/overview/faq.rst +++ b/docs/source/overview/faq.rst @@ -1,3 +1,5 @@ +:orphan: + === FAQ === @@ -8,7 +10,7 @@ If you still have questions after reading this FAQ, ping us on `our Slack `__! Why am I not able to install EvaDB in my Python environment? -====================================== +============================================================ Ensure that the local Python version is >= 3.8 and <= 3.10. EvaDB cannot support 3.11 due to its `dependency on Ray `__. @@ -34,7 +36,7 @@ You can either kill that process or launch EvaDB server on another free port in evadb_server -p 9330 Why do I see no output from the server? -==================================== +======================================= If a query runs a complex vision task (such as object detection) on a long video, the query is expected to take a non-trivial amount of time to finish. You can check the status of the server by running ``top`` or ``pgrep``: @@ -48,4 +50,4 @@ pip install ray fails because of grpcio ======================================= Follow these instructions to install ``ray``: -https://github.com/ray-project/ray/issues/33039 \ No newline at end of file +https://github.com/ray-project/ray/issues/33039 diff --git a/docs/source/overview/getting-started.rst b/docs/source/overview/getting-started.rst index bb6b944000..5e692e2fcf 100644 --- a/docs/source/overview/getting-started.rst +++ b/docs/source/overview/getting-started.rst @@ -1,3 +1,4 @@ +.. _Getting Started: Getting Started ================= @@ -19,7 +20,7 @@ EvaDB supports Python (versions >= `3.8`). To install EvaDB, we recommend using Write Your AI App -------------------------- -Here is an `illustrative MNIST digit classification app `_ using EvaDB. +Here is an illustrative MNIST digit classification app using EvaDB. .. code-block:: python @@ -50,6 +51,7 @@ Here is an `illustrative MNIST digit classification app `_. -.. image:: ../../images/reference/mnist.png \ No newline at end of file +Check out our `Google Colab `_ for working example. + +.. image:: ../../images/reference/mnist.png diff --git a/docs/source/overview/getting-started/install-guide.rst b/docs/source/overview/getting-started/install-guide.rst index c054e8fd2c..24b63a1791 100644 --- a/docs/source/overview/getting-started/install-guide.rst +++ b/docs/source/overview/getting-started/install-guide.rst @@ -1,3 +1,5 @@ +.. _installation guide: + Installation Guide ================== @@ -56,4 +58,4 @@ EvaDB also allows users to improve the query performance by using ``ray`` to par .. code-block:: - pip install evadb[ray] \ No newline at end of file + pip install evadb[ray] diff --git a/docs/source/reference/evaql.rst b/docs/source/reference/evaql.rst index c2d84305ee..5f03fdd1e4 100644 --- a/docs/source/reference/evaql.rst +++ b/docs/source/reference/evaql.rst @@ -1,5 +1,5 @@ EvaDB Query Language Reference -============================ +=============================== EvaDB Query Language (EvaDB) is derived from SQL. It is tailored for AI-driven analytics. EvaDB allows users to invoke deep learning models in the form of user-defined functions (UDFs). diff --git a/docs/source/reference/evaql/create.rst b/docs/source/reference/evaql/create.rst index 58ad4aa4a4..edbae69e26 100644 --- a/docs/source/reference/evaql/create.rst +++ b/docs/source/reference/evaql/create.rst @@ -2,7 +2,7 @@ CREATE ====== CREATE TABLE ----- +------------ To create a table, specify the schema of the table. @@ -19,7 +19,7 @@ To create a table, specify the schema of the table. ); CREATE UDF ----- +---------- To register an user-defined function, specify the implementation details of the UDF. @@ -35,7 +35,7 @@ To register an user-defined function, specify the implementation details of the .. _create-udf-train: CREATE UDF via Training ----- +----------------------- To register an user-defined function by training a predication model. @@ -49,7 +49,7 @@ To register an user-defined function by training a predication model. 'tune_for_memory' False; CREATE MATERIALIZED VIEW ----- +------------------------ To create a view with materialized results -- like the outputs of deep learning model, use the following template: diff --git a/docs/source/reference/evaql/delete.rst b/docs/source/reference/evaql/delete.rst index b97d10869e..782935301f 100644 --- a/docs/source/reference/evaql/delete.rst +++ b/docs/source/reference/evaql/delete.rst @@ -2,10 +2,10 @@ DELETE ======= DELETE INTO TABLE ------ +----------------- Delete a tuple from a table based on a predicate. .. code:: mysql - DELETE FROM MyVideo WHERE id<10; \ No newline at end of file + DELETE FROM MyVideo WHERE id<10; diff --git a/docs/source/reference/evaql/drop.rst b/docs/source/reference/evaql/drop.rst index 6d9e04b05d..b9dfeb56ec 100644 --- a/docs/source/reference/evaql/drop.rst +++ b/docs/source/reference/evaql/drop.rst @@ -2,7 +2,7 @@ DROP ==== DROP TABLE ----- +---------- .. code:: mysql @@ -10,7 +10,7 @@ DROP TABLE DROP UDF ----- +-------- .. code:: mysql diff --git a/docs/source/reference/evaql/explain.rst b/docs/source/reference/evaql/explain.rst index 4971cb13cd..6a0b96bb1a 100644 --- a/docs/source/reference/evaql/explain.rst +++ b/docs/source/reference/evaql/explain.rst @@ -1,8 +1,8 @@ EXPLAIN -==== +======== EXPLAIN QUERY ----- +------------- List the query plan associated with a EvaDB query diff --git a/docs/source/reference/evaql/insert.rst b/docs/source/reference/evaql/insert.rst index af1e8bebdc..4d0914e8f2 100644 --- a/docs/source/reference/evaql/insert.rst +++ b/docs/source/reference/evaql/insert.rst @@ -6,20 +6,20 @@ TABLE MyVideo MyVideo Table schema -.. code:: mysql +.. code:: text CREATE TABLE MyVideo (id INTEGER, data NDARRAY FLOAT32(ANYDIM)); INSERT INTO TABLE ------ +----------------- Insert a tuple into a table. -.. code:: mysql +.. code:: text INSERT INTO MyVideo (id, data) VALUES (1, [[[40, 40, 40] , [40, 40, 40]], - [[40, 40, 40] , [40, 40, 40]]]); \ No newline at end of file + [[40, 40, 40] , [40, 40, 40]]]); diff --git a/docs/source/reference/evaql/load.rst b/docs/source/reference/evaql/load.rst index 5d5a84c0c5..9d53f2b7d4 100644 --- a/docs/source/reference/evaql/load.rst +++ b/docs/source/reference/evaql/load.rst @@ -4,7 +4,7 @@ LOAD .. _1-load-video-from-filesystem: LOAD VIDEO FROM FILESYSTEM ----- +-------------------------- .. code:: mysql @@ -20,7 +20,7 @@ When a video is loaded, there is no need to specify the schema for the video tab .. _2-load-video-from-s3: LOAD VIDEO FROM S3 ----- +------------------ .. code:: mysql @@ -32,7 +32,7 @@ The videos are downloaded to a directory that can be configured in the EvaDB con .. _3-load-image-from-file: LOAD IMAGE FROM FILESYSTEM ----- +-------------------------- .. code:: mysql @@ -44,7 +44,7 @@ the following schema with two columns: ``name`` and ``data``, that correspond to .. _4-load-the-csv-file: LOAD CSV ----- +-------- To **LOAD** a CSV file, we need to first specify the table schema. diff --git a/docs/source/reference/evaql/rename.rst b/docs/source/reference/evaql/rename.rst index 67dfc6736b..63875dd630 100644 --- a/docs/source/reference/evaql/rename.rst +++ b/docs/source/reference/evaql/rename.rst @@ -2,8 +2,8 @@ RENAME ====== RENAME TABLE ----- +------------ .. code:: sql - RENAME TABLE MyVideo TO MyVideo1; \ No newline at end of file + RENAME TABLE MyVideo TO MyVideo1; diff --git a/docs/source/reference/evaql/select.rst b/docs/source/reference/evaql/select.rst index 2c437cdfc0..bdd36e28df 100644 --- a/docs/source/reference/evaql/select.rst +++ b/docs/source/reference/evaql/select.rst @@ -2,7 +2,7 @@ SELECT ====== SELECT FRAMES WITH PREDICATES ----- +----------------------------- Search for frames with a car @@ -30,7 +30,7 @@ Search for frames containing greater than 3 cars ORDER BY id; SELECT WITH MULTIPLE UDFS ----- +------------------------- Compose multiple user-defined functions in a single query to construct semantically complex queries. diff --git a/docs/source/reference/evaql/show.rst b/docs/source/reference/evaql/show.rst index 2abcc1b776..77397cb96a 100644 --- a/docs/source/reference/evaql/show.rst +++ b/docs/source/reference/evaql/show.rst @@ -2,7 +2,7 @@ SHOW ==== SHOW UDFS ----- +--------- List the registered user-defined functions diff --git a/docs/source/reference/evaql/udf.rst b/docs/source/reference/evaql/udf.rst index 9c2693b9c7..3081821df6 100644 --- a/docs/source/reference/evaql/udf.rst +++ b/docs/source/reference/evaql/udf.rst @@ -1,8 +1,10 @@ +:orphan: + UDF === SHOW UDFS ----- +--------- Here is a list of built-in user-defined functions in EvaDB. @@ -22,7 +24,7 @@ FastRCNNObjectDetector is a model for detecting objects. MVITActionRecognition i ArrayCount and Crop are utility functions for counting the number of objects in an array and cropping a bounding box from an image, respectively. SELECT WITH MULTIPLE UDFS ----- +------------------------- Here is a query that illustrates how to use multiple UDFs in a single query. @@ -30,4 +32,4 @@ Here is a query that illustrates how to use multiple UDFs in a single query. SELECT id, bbox, EmotionDetector(Crop(data, bbox)) FROM HAPPY JOIN LATERAL UNNEST(FaceDetector(data)) AS Face(bbox, conf) - WHERE id < 15; \ No newline at end of file + WHERE id < 15; diff --git a/docs/source/reference/gpu.rst b/docs/source/reference/gpu.rst index 9381af9657..d5deebd605 100644 --- a/docs/source/reference/gpu.rst +++ b/docs/source/reference/gpu.rst @@ -1,5 +1,7 @@ +:orphan: + Configure GPU ------ +------------- 1. Queries in EvaDB use deep learning models that run much faster on a GPU as opposed to a CPU. If your workstation has a GPU, you can configure EvaDB to use the GPU during query execution. Use the following command to check your hardware capabilities: @@ -30,4 +32,4 @@ Output of `cuda:0` indicates the presence of a GPU. 0 indicates the index of the executor: gpus: {'127.0.1.1': [0]} -Here, `127.0.1.1` is the loopback address on which the EvaDB server is running. 0 refers to the GPU index to be used. \ No newline at end of file +Here, `127.0.1.1` is the loopback address on which the EvaDB server is running. 0 refers to the GPU index to be used. diff --git a/docs/source/reference/io.rst b/docs/source/reference/io.rst index 7dd63df9b4..b3232157db 100644 --- a/docs/source/reference/io.rst +++ b/docs/source/reference/io.rst @@ -1,3 +1,5 @@ +:orphan: + IO Descriptors ====================== EvaDB supports three key data types. The inputs and outputs of the user-defined functions (UDFs) must be of one of these types. diff --git a/docs/source/reference/udfs/hf.rst b/docs/source/reference/udfs/hf.rst index 2af377791a..174fee1171 100644 --- a/docs/source/reference/udfs/hf.rst +++ b/docs/source/reference/udfs/hf.rst @@ -19,7 +19,7 @@ EvaDB supports UDFS similar to `Pipelines `_ for working example. +Check out our `Integration Tests `_ for working example. diff --git a/docs/source/reference/udfs/openai.rst b/docs/source/reference/udfs/openai.rst index 5fea7cfcab..d229eff44a 100644 --- a/docs/source/reference/udfs/openai.rst +++ b/docs/source/reference/udfs/openai.rst @@ -24,4 +24,4 @@ EvaDB supports the following models for chat completion task: - "gpt-3.5-turbo" - "gpt-3.5-turbo-0301" -The chat completion UDF can be composed in interesting ways with other UDFs. Please refer to the `ChatGPT notebook `_ for an example of combining chat completion task with caption extraction and video summarization models from Hugging Face and feeding it to chat completion to ask questions about the results. +The chat completion UDF can be composed in interesting ways with other UDFs. Please check the `Google Colab `_ for an example of combining chat completion task with caption extraction and video summarization models from Hugging Face and feeding it to chat completion to ask questions about the results. diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 76f5c9ced3..db4b126cf1 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -1,3 +1,5 @@ +:orphan: + .. _guide-setup: Setup diff --git a/docs/source/usecases/image-classification.rst b/docs/source/usecases/image-classification.rst index 577e7f8eca..4a89695fec 100644 --- a/docs/source/usecases/image-classification.rst +++ b/docs/source/usecases/image-classification.rst @@ -1,10 +1,12 @@ +.. _image classification: + Image Classification Pipeline using EvaDB -==== +========================================= Assume the database has loaded a video ``mnist_video``. 1. Connect to EvaDB ----- +------------------- .. code-block:: python @@ -12,7 +14,7 @@ Assume the database has loaded a video ``mnist_video``. cursor = evadb.connect().cursor() 2. Register Image Classification Model as a Function in SQL ----- +----------------------------------------------------------- Create an image classification function from python source code. @@ -24,7 +26,7 @@ Create an image classification function from python source code. """).execute() 3. Execute Image Classification through SQL ----- +------------------------------------------- After the function is registered to EvaDB system, it can be directly called and used in SQL query. @@ -67,7 +69,7 @@ The result contains a projected ``label`` column, which indicates the digit of a ... ... 4. Optional: Process Only Segments of Videos based on Conditions ----- +----------------------------------------------------------------- Like normal SQL, you can also specify conditions to filter out some frames of the video. @@ -103,4 +105,4 @@ Now, the ``DataFrame`` only contains 2 rows after filtering. | 6 | +------------------------------+ -Check out our `Jupyter Notebook `_ for working example. \ No newline at end of file +Check out our `Jupyter Notebook `_ for working example. diff --git a/docs/source/usecases/qa-video.rst b/docs/source/usecases/qa-video.rst index 3145ca85b1..133eefca24 100644 --- a/docs/source/usecases/qa-video.rst +++ b/docs/source/usecases/qa-video.rst @@ -1,8 +1,8 @@ Q&A Application on Videos -==== +========================= 1. Connect to EvaDB ----- +------------------- .. code-block:: python @@ -10,7 +10,7 @@ Q&A Application on Videos cursor = evadb.connect().cursor() 2. Register Functions ----- +--------------------- Register speech-to-text **whisper** model from `HuggingFace` @@ -45,7 +45,7 @@ Register **OpenAI** LLM model ChatGPT function is a wrapper around OpenAI API call. You can also switch to other LLM models that can run locally. 3. Summarize Video in Text ----- +-------------------------- Create a table with text summary of the video. Text summarization is generated by running speech-to-text ``Whisper`` model from ``HuggingFace``. @@ -67,7 +67,7 @@ This results a table shown below. +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ 4. Q&A using ChatGPT ----- +--------------------- We can now embed the ChatGPT prompt inside SQL with text summary from the table as its knowledge base. @@ -86,4 +86,4 @@ This query returns a projected ``DataFrame``. | chatgpt.response | |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Based on the provided context, it seems that the video summary is related to the Ukraine-Russia war. It discusses how US oil companies are allegedly profiting from the war in Ukraine, similar to how they allegedly benefited from the war in Iraq. | - +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ \ No newline at end of file + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/usecases/similar-image-search.rst b/docs/source/usecases/similar-image-search.rst index 83f69e395b..5a02300541 100644 --- a/docs/source/usecases/similar-image-search.rst +++ b/docs/source/usecases/similar-image-search.rst @@ -1,5 +1,5 @@ Image Similarity Search Pipeline using EvaDB on Images -==== +====================================================== In this use case, we want to search similar images based on an image provided by the user. To implement this use case, we leverage EvaDB's capability of easily expressing feature extraction pipeline. Additionaly, we also leverage EvaDB's capability of building a similarity search index and searching the index to locate similar images through ``FAISS`` library. @@ -8,7 +8,7 @@ For this use case, we use a reddit image dataset that can be downloaded from `He We populate a table in the database that contains all images. 1. Connect to EvaDB ----- +------------------- .. code-block:: python @@ -16,7 +16,7 @@ We populate a table in the database that contains all images. cursor = evadb.connect().cursor() 2. Register SIFT as Function ----- +---------------------------- .. code-block:: python @@ -26,13 +26,13 @@ We populate a table in the database that contains all images. """).execute() 3. Search Similar Images ----- +------------------------ To locate images that have similar appearance, we will first build an index based on embeddings of images. Then, for the given image, EvaDB can find similar images by searching in the index. Build Index using ``FAISS`` -**** +*************************** The below query creates a new index on the projected column ``SiftFeatureExtractor(data)`` from the ``reddit_dataset`` table. @@ -45,7 +45,7 @@ The below query creates a new index on the projected column ``SiftFeatureExtract """).execute() Search Index for a Given Image -**** +******************************* EvaDB leverages the ``ORDER BY ... LIMIT ...`` SQL syntax to retrieve the top 5 similar images. In this example, ``Similarity(x, y)`` is a built-in function to calculate distance between ``x`` and ``y``. @@ -80,4 +80,4 @@ The ``DataFrame`` contains the top 5 similar images. +---------------------------------+ Check out our `Jupyter Notebook `_ for working example. -We also demonstrate more complicated features of EvaDB for similarity search. \ No newline at end of file +We also demonstrate more complicated features of EvaDB for similarity search. diff --git a/script/test/test.sh b/script/test/test.sh index 591ffa82de..bffd728f06 100644 --- a/script/test/test.sh +++ b/script/test/test.sh @@ -37,6 +37,22 @@ check_linter() { print_error_code $code "LINTER" } +check_doc_build() { + pushd docs + make html + code=$? + popd + print_error_code $code "DOC BUILD" +} + +check_doc_link() { + pushd docs + make linkcheck + code=$? + popd + print_error_code $code "DOC LINK CHECK" +} + unit_test() { PYTHONPATH="." pytest test/unit_tests/ --durations=20 --cov-report term-missing:skip-covered --cov-config=.coveragerc --cov-context=test --cov=evadb/ --capture=sys --tb=short -v -rsf --log-level=WARNING -m "not benchmark" code=$? @@ -117,6 +133,17 @@ then check_linter fi +################################################## +## DOC BUILD TESTS +################################################## + +if [[ ( "$OSTYPE" != "msys" ) && ( "$MODE" = "DOC" || "$MODE" = "ALL" ) ]]; +then + # Run black, isort, linter + check_doc_build + check_doc_link +fi + ################################################## ## UNIT TESTS ##################################################