Merge branch 'main' into fix-immutable-tuples

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [simonw]

.github/workflows/deploy-latest.yml

@@ -31,7 +31,7 @@ jobs:
     - name: Run tests
       run: pytest
     - name: Build fixtures.db
-      run: python tests/fixtures.py fixtures.db fixtures.json plugins
+      run: python tests/fixtures.py fixtures.db fixtures.json plugins --extra-db-filename extra_database.db
     - name: Build docs.db
       run: |-
         cd docs
@@ -48,12 +48,12 @@ jobs:
       run: |-
         gcloud config set run/region us-central1
         gcloud config set project datasette-222320
-        datasette publish cloudrun fixtures.db \
+        datasette publish cloudrun fixtures.db extra_database.db \
             -m fixtures.json \
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
+            --extra-options="--setting template_debug 1 --crossdb" \
             --install=pysqlite3-binary \
             --service=datasette-latest
         # Deploy docs.db to a different service

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.2-slim-stretch as build
+FROM python:3.7.10-slim-stretch as build
 
 # Setup build dependencies
 RUN apt update \
@@ -7,7 +7,7 @@ RUN apt update \
 
 
 RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \
-    && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \
+    && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \
     && make && make install
 
 RUN wget "http://www.gaia-gis.it/gaia-sins/freexl-sources/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \
@@ -27,7 +27,7 @@ COPY . /datasette
 
 RUN pip install /datasette
 
-FROM python:3.7.2-slim-stretch
+FROM python:3.7.10-slim-stretch
 
 # Copy python dependencies and spatialite libraries
 COPY --from=build /usr/local/lib/ /usr/local/lib/

datasette/app.py

@@ -85,6 +85,9 @@
 
 app_root = Path(__file__).parent.parent
 
+# https://github.com/simonw/datasette/issues/283#issuecomment-781591015
+SQLITE_LIMIT_ATTACHED = 10
+
 Setting = collections.namedtuple("Setting", ("name", "default", "help"))
 SETTINGS = (
     Setting("default_page_size", 100, "Default page size for the table view"),
@@ -194,6 +197,7 @@ def __init__(
         version_note=None,
         config_dir=None,
         pdb=False,
+        crossdb=False,
     ):
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -208,16 +212,17 @@ def __init__(
             and (config_dir / "inspect-data.json").exists()
             and not inspect_data
         ):
-            inspect_data = json.load((config_dir / "inspect-data.json").open())
-            if not immutables:
+            inspect_data = json.loads((config_dir / "inspect-data.json").read_text())
+            if not immutables:            
                 immutable_filenames = [i["file"] for i in inspect_data.values()]
                 immutables = [
                     f for f in self.files if Path(f).name in immutable_filenames
                 ]
         self.inspect_data = inspect_data
         self.immutables = set(immutables or [])
         self.databases = collections.OrderedDict()
-        if memory or not self.files:
+        self.crossdb = crossdb
+        if memory or crossdb or not self.files:
             self.add_database(Database(self, is_memory=True), name="_memory")
         # memory_name is a random string so that each Datasette instance gets its own
         # unique in-memory named database - otherwise unit tests can fail with weird
@@ -264,7 +269,7 @@ def __init__(
         if config_dir and (config_dir / "config.json").exists():
             raise StartupError("config.json should be renamed to settings.json")
         if config_dir and (config_dir / "settings.json").exists() and not config:
-            config = json.load((config_dir / "settings.json").open())
+            config = json.loads((config_dir / "settings.json").read_text())
         self._settings = dict(DEFAULT_SETTINGS, **(config or {}))
         self.renderers = {}  # File extension -> (renderer, can_render) functions
         self.version_note = version_note
@@ -385,6 +390,9 @@ def add_database(self, db, name=None):
         self.databases[name] = db
         return db
 
+    def add_memory_database(self, memory_name):
+        return self.add_database(Database(self, memory_name=memory_name))
+
     def remove_database(self, name):
         self.databases.pop(name)
 
@@ -442,11 +450,10 @@ def plugin_config(self, plugin_name, database=None, table=None, fallback=True):
 
     def app_css_hash(self):
         if not hasattr(self, "_app_css_hash"):
-            self._app_css_hash = hashlib.sha1(
-                open(os.path.join(str(app_root), "datasette/static/app.css"))
-                .read()
-                .encode("utf8")
-            ).hexdigest()[:6]
+            with open(os.path.join(str(app_root), "datasette/static/app.css")) as fp:
+                self._app_css_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[
+                    :6
+                ]
         return self._app_css_hash
 
     async def get_canned_queries(self, database_name, actor):
@@ -499,6 +506,19 @@ def _prepare_connection(self, conn, database):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
         pm.hook.prepare_connection(conn=conn, database=database, datasette=self)
+        # If self.crossdb and this is _memory, connect the first SQLITE_LIMIT_ATTACHED databases
+        if self.crossdb and database == "_memory":
+            count = 0
+            for db_name, db in self.databases.items():
+                if count >= SQLITE_LIMIT_ATTACHED or db.is_memory:
+                    continue
+                sql = 'ATTACH DATABASE "file:{path}?{qs}" AS [{name}];'.format(
+                    path=db.path,
+                    qs="mode=ro" if db.is_mutable else "immutable=1",
+                    name=db_name,
+                )
+                conn.execute(sql)
+                count += 1
 
     def add_message(self, request, message, type=INFO):
         if not hasattr(request, "_messages"):

datasette/cli.py

@@ -12,7 +12,7 @@
 import sys
 from runpy import run_module
 import webbrowser
-from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, pm
+from .app import Datasette, DEFAULT_SETTINGS, SETTINGS, SQLITE_LIMIT_ATTACHED, pm
 from .utils import (
     StartupError,
     check_connection,
@@ -125,13 +125,13 @@ def cli():
 @sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
-    if inspect_file == "-":
-        out = sys.stdout
-    else:
-        out = open(inspect_file, "w")
     loop = asyncio.get_event_loop()
     inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
-    out.write(json.dumps(inspect_data, indent=2))
+    if inspect_file == "-":
+        sys.stdout.write(json.dumps(inspect_data, indent=2))
+    else:
+        with open(inspect_file, "w") as fp:
+            fp.write(json.dumps(inspect_data, indent=2))
 
 
 async def inspect_(files, sqlite_extensions):
@@ -223,6 +223,7 @@ def plugins(all, plugins_dir):
     "-p",
     "--port",
     default=8001,
+    type=click.IntRange(1, 65535),
     help="Port to run the server on, defaults to 8001",
 )
 @click.option("--title", help="Title for metadata")
@@ -329,6 +330,7 @@ def uninstall(packages, yes):
     "-p",
     "--port",
     default=8001,
+    type=click.IntRange(0, 65535),
     help="Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.",
 )
 @click.option(
@@ -408,6 +410,11 @@ def uninstall(packages, yes):
     is_flag=True,
     help="Create database files if they do not exist",
 )
+@click.option(
+    "--crossdb",
+    is_flag=True,
+    help="Enable cross-database joins using the /_memory database",
+)
 @click.option(
     "--ssl-keyfile",
     help="SSL key file",
@@ -440,6 +447,7 @@ def serve(
     pdb,
     open_browser,
     create,
+    crossdb,
     ssl_keyfile,
     ssl_certfile,
     return_instance=False,
@@ -467,7 +475,8 @@ def serve(
 
     inspect_data = None
     if inspect_file:
-        inspect_data = json.load(open(inspect_file))
+        with open(inspect_file) as fp:
+            inspect_data = json.load(fp)
 
     metadata_data = None
     if metadata:
@@ -497,6 +506,7 @@ def serve(
         secret=secret,
         version_note=version_note,
         pdb=pdb,
+        crossdb=crossdb,
     )
 
     # if files is a single directory, use that as config_dir=
@@ -589,3 +599,15 @@ async def check_databases(ds):
             raise click.UsageError(
                 f"Connection to {database.path} failed check: {str(e.args[0])}"
             )
+    # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning
+    if (
+        ds.crossdb
+        and len([db for db in ds.databases.values() if not db.is_memory])
+        > SQLITE_LIMIT_ATTACHED
+    ):
+        msg = (
+            "Warning: --crossdb only works with the first {} attached databases".format(
+                SQLITE_LIMIT_ATTACHED
+            )
+        )
+        click.echo(click.style(msg, bold=True, fg="yellow"), err=True)

datasette/database.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections import namedtuple
 from pathlib import Path
 import janus
 import queue
@@ -22,6 +23,8 @@
 
 connections = threading.local()
 
+AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
+
 
 class Database:
     def __init__(
@@ -78,7 +81,7 @@ def connect(self, write=False):
                 conn.execute("PRAGMA query_only=1")
             return conn
         if self.is_memory:
-            return sqlite3.connect(":memory:")
+            return sqlite3.connect(":memory:", uri=True)
         # mode=ro or immutable=1?
         if self.is_mutable:
             qs = "?mode=ro"
@@ -243,6 +246,12 @@ def mtime_ns(self):
             return None
         return Path(self.path).stat().st_mtime_ns
 
+    async def attached_databases(self):
+        results = await self.execute(
+            "select seq, name, file from pragma_database_list() where seq > 0"
+        )
+        return [AttachedDatabase(*row) for row in results.rows]
+
     async def table_exists(self, table):
         results = await self.execute(
             "select 1 from sqlite_master where type='table' and name=?", params=(table,)

datasette/facets.py

@@ -279,6 +279,7 @@ async def suggest(self):
             suggested_facet_sql = """
                 select distinct json_type({column})
                 from ({sql})
+                where {column} is not null and {column} != ''
             """.format(
                 column=escape_sqlite(column), sql=self.sql
             )
@@ -298,9 +299,13 @@ async def suggest(self):
                         v[0]
                         for v in await self.ds.execute(
                             self.database,
-                            "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format(
-                                column=escape_sqlite(column), sql=self.sql
-                            ),
+                            (
+                                "select {column} from ({sql}) "
+                                "where {column} is not null "
+                                "and {column} != '' "
+                                "and json_array_length({column}) > 0 "
+                                "limit 100"
+                            ).format(column=escape_sqlite(column), sql=self.sql),
                             self.params,
                             truncate=False,
                             custom_time_limit=self.ds.setting(

datasette/filters.py

@@ -150,7 +150,7 @@ class Filters:
                     "arraycontains",
                     "array contains",
                     """rowid in (
-            select {t}.rowid from {t}, json_each({t}.{c}) j
+            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
             where j.value = :{p}
         )""",
                     '{c} contains "{v}"',
@@ -159,7 +159,7 @@ class Filters:
                     "arraynotcontains",
                     "array does not contain",
                     """rowid not in (
-            select {t}.rowid from {t}, json_each({t}.{c}) j
+            select {t}.rowid from {t}, json_each([{t}].[{c}]) j
             where j.value = :{p}
         )""",
                     '{c} does not contain "{v}"',

datasette/publish/cloudrun.py

@@ -141,9 +141,11 @@ def cloudrun(
             if show_files:
                 if os.path.exists("metadata.json"):
                     print("=== metadata.json ===\n")
-                    print(open("metadata.json").read())
+                    with open("metadata.json") as fp:
+                        print(fp.read())
                 print("\n==== Dockerfile ====\n")
-                print(open("Dockerfile").read())
+                with open("Dockerfile") as fp:
+                    print(fp.read())
                 print("\n====================\n")
 
             image_id = f"gcr.io/{project}/{name}"

datasette/publish/heroku.py

@@ -171,9 +171,11 @@ def temporary_heroku_directory(
         os.chdir(tmp.name)
 
         if metadata_content:
-            open("metadata.json", "w").write(json.dumps(metadata_content, indent=2))
+            with open("metadata.json", "w") as fp:
+                fp.write(json.dumps(metadata_content, indent=2))
 
-        open("runtime.txt", "w").write("python-3.8.7")
+        with open("runtime.txt", "w") as fp:
+            fp.write("python-3.8.7")
 
         if branch:
             install = [
@@ -182,11 +184,11 @@ def temporary_heroku_directory(
         else:
             install = ["datasette"] + list(install)
 
-        open("requirements.txt", "w").write("\n".join(install))
+        with open("requirements.txt", "w") as fp:
+            fp.write("\n".join(install))
         os.mkdir("bin")
-        open("bin/post_compile", "w").write(
-            "datasette inspect --inspect-file inspect-data.json"
-        )
+        with open("bin/post_compile", "w") as fp:
+            fp.write("datasette inspect --inspect-file inspect-data.json")
 
         extras = []
         if template_dir:
@@ -218,7 +220,8 @@ def temporary_heroku_directory(
         procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format(
             quoted_files=quoted_files, extras=" ".join(extras)
         )
-        open("Procfile", "w").write(procfile_cmd)
+        with open("Procfile", "w") as fp:
+            fp.write(procfile_cmd)
 
         for path, filename in zip(file_paths, file_names):
             link_or_copy(path, os.path.join(tmp.name, filename))