georgia-tech-db · gaurav274 · Aug 30, 2023 · Aug 25, 2023 · Aug 26, 2023 · Aug 26, 2023
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,6 +15,12 @@ workflows:
             name: Linter | v3.10 | Linux
             mode: LINTER
         ################################
+        ## DOC: All
+        ################################
+        ################################
+        - Doc:
+            name: Doc | v3.10 | Linux
+        ################################
         #### PIP: Master
         ################################
         ################################
@@ -178,8 +184,8 @@ jobs:
             RAY: << parameters.ray >>
           command: |
             "python<< parameters.v >>" -m venv test_evadb
-            pip install --upgrade pip
             source test_evadb/bin/activate
+            pip install --upgrade pip
             if [ $RAY = "ENABLED" ]; then
               if [ $PY_VERSION != "3.11" ]; then
                 pip install ".[dev,ray,qdrant]"
@@ -225,6 +231,47 @@ jobs:
           paths:
             - test_evadb
 
+  Doc:
+    parameters:
+      v:
+        type: string
+        default: "3.10"
+      mode:
+        type: string
+        default: "DOC"
+    resource_class: large
+    docker:
+      # https://circleci.com/docs/circleci-images#language-image-variants
+      - image: "cimg/python:<< parameters.v >>"
+    steps:
+
+      - checkout
+
+      # Restore pip wheel
+      - restore_cache:
+          keys:
+            - v1-pip-wheel_cache-python<< parameters.v >>-{{ checksum "docs/requirements.txt" }}
+
+      - run:
+          name: Install EvaDB Doc dependencies
+          command: |
+            "python<< parameters.v >>" -m venv test_evadb_doc
+            source test_evadb_doc/bin/activate
+            pip install --upgrade pip
+            pip install -r docs/requirements.txt 
+
+      - run:
+          name: Test doc build and link validation
+          no_output_timeout: 10m # 10 minute timeout
+          command: |
+            source test_evadb_doc/bin/activate
+            bash script/test/test.sh -m "<< parameters.mode >>"
+
+      - save_cache:
+          key: v1-pip-wheel_cache-python<< parameters.v >>-{{ checksum "docs/requirements.txt" }}
+          paths:
+            - test_evadb_doc
+
   Postgres:
     parameters:
       v:

diff --git a/docs/Makefile b/docs/Makefile
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?= -vvv
+SPHINXOPTS    ?= -vvv -W
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
@@ -21,4 +21,4 @@ help:
 
 clean:
 	rm -rf $(BUILDDIR)/*
-	rm -rf ./source/reference/python/doc/*
+	rm -rf ./source/reference/python/doc/*
diff --git a/docs/Readme.md → docs/README.md b/docs/Readme.md → docs/README.md
@@ -8,9 +8,8 @@ make html
 open _build/html/index.html
 ```
 
-To test links:
+To further test external links:
 
 ```
-cd evadb/docs
-sphinx-build . _build -b linkcheck
-```
+make linkcheck
+```
diff --git a/docs/_toc.yml b/docs/_toc.yml
@@ -8,7 +8,7 @@ parts:
           - file: source/overview/getting-started/install-guide
             title: Installation Guide
       - file: source/overview/concepts
-      # - file: source/overview/faq
+      - file: source/overview/faq
 
   - caption: Use Cases
     chapters:
@@ -83,7 +83,7 @@ parts:
         sections:
           - file: source/dev-guide/contribute/setup-dev
             title: Setup Environment
-          - file: source/contribute/contribute/testing
+          - file: source/dev-guide/contribute/testing
             title: Testing
           - file: source/dev-guide/contribute/submit-pr
             title: Submit a PR

diff --git a/docs/conf.py b/docs/conf.py
@@ -31,7 +31,7 @@
     "sphinx_copybutton",
     "sphinx.ext.doctest",
     "sphinx.ext.coverage",
-    "sphinx.ext.autosectionlabel",
+#    "sphinx.ext.autosectionlabel",
     "sphinx.ext.autosummary",
     "sphinx.ext.autodoc",
     "sphinx.ext.autodoc.typehints",
@@ -48,6 +48,9 @@
     "versionwarning.extension",
     "IPython.sphinxext.ipython_console_highlighting",
 ]
+
+suppress_warnings = ["etoc.toctree", "myst.header"]
+
 source_suffix = [".ipynb", ".html", ".md", ".rst"]
 
 autodoc_pydantic_model_show_json = False
@@ -91,7 +94,7 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md", "images/reference/README.md"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 # pygments_style = "lovelace"
@@ -148,14 +151,21 @@
                     "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/solid.min.css",
                     "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/brands.min.css"]
 
-# Adding the Tutorial notebooks to ./docs/source/tutorials/
+# Check interal link: https://stackoverflow.com/questions/14492743/have-sphinx-report-broken-links/14735060#14735060
+nitpicky = True
+# BUG: https://stackoverflow.com/questions/11417221/sphinx-autodoc-gives-warning-pyclass-reference-target-not-found-type-warning
+nitpick_ignore_regex = [('py:class', r'.*')]
+
 
 for i in os.listdir("../tutorials"):
-    if i.endswith(".ipynb"):
+    if i in [
+        "02-object-detection.ipynb",
+        "03-emotion-analysis.ipynb", 
+        "07-object-segmentation-huggingface.ipynb"
+    ]:
         shutil.copy(f"../tutorials/{i}", "./source/usecases/")
 
-jupyter_execute_notebooks = "off"
-
+nb_execution_mode = "off"
 
 # -- Initialize Sphinx ----------------------------------------------
 def setup(app):
@@ -165,4 +175,4 @@ def setup(app):
         message=r".*Container node skipped.*",
     )
     # Custom JS
-    app.add_js_file("js/top-navigation.js", defer="defer")
+    app.add_js_file("js/top-navigation.js", defer="defer")
diff --git a/docs/index.rst b/docs/index.rst
@@ -95,12 +95,12 @@ Key Features
 Try it out!
 ------------
 
-- `PrivateGPT <https://evadb.readthedocs.io/en/stable/source/tutorials/13-privategpt.html>`_
-- `Video Question Answering using ChatGPT <https://evadb.readthedocs.io/en/stable/source/tutorials/08-chatgpt.html>`_
-- `Querying PDF documents <https://evadb.readthedocs.io/en/stable/source/tutorials/12-query-pdf.html>`_
-- `Analyzing traffic flow at an intersection <https://evadb.readthedocs.io/en/stable/source/tutorials/02-object-detection.html>`_
-- `Examining the emotion palette of actors in a movie <https://evadb.readthedocs.io/en/stable/source/tutorials/03-emotion-analysis.html>`_
-- `Classifying images based on their content <https://evadb.readthedocs.io/en/stable/source/tutorials/01-mnist.html>`_
+- `PrivateGPT <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/13-privategpt.ipynb>`_
+- `Video Question Answering using ChatGPT <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/08-chatgpt.ipynb>`_
+- `Querying PDF documents <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/12-query-pdf.ipynb>`_
+- `Analyzing traffic flow at an intersection <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/02-object-detection.ipynb>`_
+- `Examining the emotion palette of actors in a movie <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/03-emotion-analysis.ipynb>`_
+- `Classifying images based on their content <https://colab.research.google.com/github/georgia-tech-db/evadb/blob/master/tutorials/01-mnist.ipynb>`_
 
 
 
diff --git a/docs/source/benchmarks/text_summarization.rst b/docs/source/benchmarks/text_summarization.rst
@@ -1,17 +1,17 @@
 Text summarization benchmark 
-====
+============================
 In this benchmark, we compare the performance of text summarization between EvaDB and MindsDB on `CNN-DailyMail News <https://www.kaggle.com/datasets/gowrishankarp/newspaper-text-summarization-cnn-dailymail>`_.
 
 1. Prepare dataset
-----
+------------------
 
 .. code-block:: bash
 
    cd benchmark/text_summarization
    bash download_dataset.sh
 
 2. Using EvaDB to summarize the CNN DailyMail News
-----
+--------------------------------------------------
 
 .. note::
 
@@ -24,10 +24,10 @@ In this benchmark, we compare the performance of text summarization between EvaD
 
 
 3. Using MindsDB to summarize the CNN DailyMail News
-----
+----------------------------------------------------
 
 Prepare sqlite database for MindsDB
-****
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. code-block:: bash
 
@@ -38,7 +38,8 @@ Prepare sqlite database for MindsDB
 
 
 Install MindsDB
-****
+~~~~~~~~~~~~~~~
+
 Follow the `Setup for Source Code via pip <https://docs.mindsdb.com/setup/self-hosted/pip/source>`_ to install mindsdb.
 
 .. note::
@@ -52,11 +53,11 @@ After the installation, we use mysql cli to connect to MindsDB. Replace the port
    mysql -h 127.0.0.1 --port 47335 -u mindsdb -p
 
 Run Experiment
-****
+~~~~~~~~~~~~~~
 
 Connect the sqlite database we created before.
 
-.. code-block:: sql
+.. code-block:: text
 
    CREATE DATABASE sqlite_datasource
    WITH ENGINE = 'sqlite',
@@ -66,7 +67,7 @@ Connect the sqlite database we created before.
 
 Create text summarization model and wait for its readiness.
 
-.. code-block:: sql
+.. code-block:: text
 
    CREATE MODEL mindsdb.hf_bart_sum_20
    PREDICT PRED
@@ -82,7 +83,7 @@ Create text summarization model and wait for its readiness.
 
 Use the model to summarize the CNN DailyMail news.
 
-.. code-block:: sql
+.. code-block:: text
 
    CREATE OR REPLACE TABLE sqlite_datasource.cnn_news_summary (
      SELECT PRED
@@ -92,7 +93,7 @@ Use the model to summarize the CNN DailyMail news.
 
 
 4. Experiment results
-----
+---------------------
 Below are nubmers from a server with 56 Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz and two Quadro P6000 GPU.
 
 .. list-table:: Text summarization with ``sshleifer/distilbart-cnn-12-6`` on CNN-DailyMail News

diff --git a/docs/source/dev-guide/contribute.rst b/docs/source/dev-guide/contribute.rst
@@ -6,8 +6,8 @@ We welcome all kinds of contributions to EvaDB.
 -  `Code reviews <https://github.com/georgia-tech-db/evadb/pulls>`_
 -  `Improving documentation <https://github.com/georgia-tech-db/evadb/tree/master/docs>`_
 -  `Tutorials and applications <https://github.com/georgia-tech-db/evadb/tree/master/tutorials>`_
--  `New features <new-command.html>`__
+-  :ref:`Extending EvaDB`
 
 This document goes over how to contribute to EvaDB in details.
 
-.. tableofcontents::
+.. tableofcontents::
diff --git a/docs/source/dev-guide/contribute/setup-dev.rst b/docs/source/dev-guide/contribute/setup-dev.rst
@@ -27,10 +27,10 @@ After installing the package locally, you can make changes and run the test case
 .. note::
 
    EvaDB provides multiple installation options for extending its functionalities. 
-   Please see :doc:`Installation Guide <getting-started/install-guide>` for all options.
+   Please see :ref:`installation guide` for all options.
 
 Other options can be installed with the ``dev`` environment.
 
 .. code-block:: bash
 
-   pip install -e ".[dev,ludwig]"
+   pip install -e ".[dev,ludwig]"
diff --git a/docs/source/dev-guide/contribute/submit-pr.rst b/docs/source/dev-guide/contribute/submit-pr.rst
@@ -18,7 +18,7 @@ Merge the most recent changes from the ``staging`` branch
        git pull . origin/staging
 
 Testing
--------
+-----------------
 
 Run the :doc:`test script <./testing>` to ensure that all the test cases pass.
 
@@ -35,4 +35,4 @@ Run the following command to ensure that code is properly formatted.
 
 .. code-block:: python
 
-      python script/formatting/formatter.py 
+      python script/formatting/formatter.py 
diff --git a/docs/source/dev-guide/extend.rst b/docs/source/dev-guide/extend.rst
@@ -1,5 +1,7 @@
+.. _Extending EvaDB:
+
 Extending EvaDB
-====
+===============
 This document details steps involved in extending EvaDB.
 
 .. tableofcontents::
diff --git a/docs/source/dev-guide/extend/new-command.rst b/docs/source/dev-guide/extend/new-command.rst
@@ -1,10 +1,10 @@
 Operators / Commands
-=============
+====================
 
 This document details the steps involved in adding support for a new operator (or command) in EvaDB. We illustrate the process using a DDL command.
 
 Command Handler
-----
+---------------
 
 An input query string is handled by **Parser**,
 **StatementTOPlanConverter**, **PlanGenerator**, and **PlanExecutor**. 
@@ -83,7 +83,7 @@ parser/
 .. _2-statementtoplanconverter:
 
 2. Statement To Plan Converter
----------------------------
+------------------------------
 
 The part transforms the statement into corresponding logical plan.
 
@@ -154,13 +154,13 @@ Optimizer
 .. _3-plangenerator:
 
 3. Plan Generator
-----------------
+------------------
 
 The part transformed logical plan to physical plan. The modified files
 are stored under **Optimizer** and **Planner** folders.
 
 plan_nodes/
-~~~~~~~~
+~~~~~~~~~~~~
 
 -  ``[cmd]_plan.py`` - class [cmd]Plan, which stored information
    required for rename table.
@@ -220,7 +220,7 @@ optimizer/rules
 .. _4-PlanExecutor:
 
 4. Plan Executor
---------------
+-----------------
 
 ``PlanExecutor`` uses data stored in physical plan to run the command.