From 751abbcc57c2b80275a9d507149dfe829a00493b Mon Sep 17 00:00:00 2001
From: Alex Garcia <alexsebastian.garcia@gmail.com>
Date: Wed, 14 Aug 2024 15:39:49 -0700
Subject: [PATCH] don't hide virtual table, hide shadow tables.
---
datasette/database.py | 76 ++++++++++++++++++++------------
tests/test_api.py | 46 +++++++++----------
tests/test_html.py | 3 +-
tests/test_internals_database.py | 42 ++++++++++++++++++
4 files changed, 116 insertions(+), 51 deletions(-)
diff --git a/datasette/database.py b/datasette/database.py
index 71c134d1d9..8b55f8f34e 100644
--- a/datasette/database.py
+++ b/datasette/database.py
@@ -20,6 +20,7 @@
table_columns,
table_column_details,
)
+from .utils.sqlite import sqlite_version
from .inspect import inspect_hash
connections = threading.local()
@@ -459,22 +460,56 @@ async def foreign_keys_for_table(self, table):
)
async def hidden_table_names(self):
- # Mark tables 'hidden' if they relate to FTS virtual tables
- hidden_tables = [
- r[0]
- for r in (
- await self.execute(
+ hidden_tables = []
+ # Add any tables marked as hidden in config
+ db_config = self.ds.config.get("databases", {}).get(self.name, {})
+ if "tables" in db_config:
+ hidden_tables += [
+ t for t in db_config["tables"] if db_config["tables"][t].get("hidden")
+ ]
+
+ if sqlite_version()[1] >= 37:
+ hidden_tables += [
+ x[0]
+ for x in await self.execute(
+ """
+ with shadow_tables as (
+ select name
+ from pragma_table_list
+ where [type] = 'shadow'
+ order by name
+ ),
+ core_tables as (
+ select name
+ from sqlite_master
+ WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
+ OR substr(name, 1, 1) == '_'
+ ),
+ combined as (
+ select name from shadow_tables
+ union all
+ select name from core_tables
+ )
+ select name from combined order by 1
"""
- select name from sqlite_master
- where rootpage = 0
- and (
- sql like '%VIRTUAL TABLE%USING FTS%'
- ) or name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
- or name like '\\_%' escape '\\'
- """
)
- ).rows
- ]
+ ]
+ else:
+ hidden_tables += [
+ x[0]
+ for x in await self.execute(
+ """
+ with final as (
+ select name
+ from sqlite_master
+ WHERE name in ('sqlite_stat1', 'sqlite_stat2', 'sqlite_stat3', 'sqlite_stat4')
+ OR substr(name, 1, 1) == '_'
+ ),
+ select name from final order by 1
+ """
+ )
+ ]
+
has_spatialite = await self.execute_fn(detect_spatialite)
if has_spatialite:
# Also hide Spatialite internal tables
@@ -503,19 +538,6 @@ async def hidden_table_names(self):
)
).rows
]
- # Add any tables marked as hidden in config
- db_config = self.ds.config.get("databases", {}).get(self.name, {})
- if "tables" in db_config:
- hidden_tables += [
- t for t in db_config["tables"] if db_config["tables"][t].get("hidden")
- ]
- # Also mark as hidden any tables which start with the name of a hidden table
- # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden
- for table_name in await self.table_names():
- for hidden_table in hidden_tables[:]:
- if table_name.startswith(hidden_table):
- hidden_tables.append(table_name)
- continue
return hidden_tables
diff --git a/tests/test_api.py b/tests/test_api.py
index 431ab5cee4..01c9bb79ad 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -389,6 +389,29 @@ async def test_database_page(ds_client):
},
"private": False,
},
+ {
+ "name": "searchable_fts",
+ "columns": [
+ "text1",
+ "text2",
+ "name with . and spaces",
+ ]
+ + (
+ [
+ "searchable_fts",
+ "docid",
+ "__langid",
+ ]
+ if supports_table_xinfo()
+ else []
+ ),
+ "primary_keys": [],
+ "count": 2,
+ "hidden": False,
+ "fts_table": "searchable_fts",
+ "foreign_keys": {"incoming": [], "outgoing": []},
+ "private": False,
+ },
{
"name": "searchable_tags",
"columns": ["searchable_id", "tag"],
@@ -525,29 +548,6 @@ async def test_database_page(ds_client):
"foreign_keys": {"incoming": [], "outgoing": []},
"private": False,
},
- {
- "name": "searchable_fts",
- "columns": [
- "text1",
- "text2",
- "name with . and spaces",
- ]
- + (
- [
- "searchable_fts",
- "docid",
- "__langid",
- ]
- if supports_table_xinfo()
- else []
- ),
- "primary_keys": [],
- "count": 2,
- "hidden": True,
- "fts_table": "searchable_fts",
- "foreign_keys": {"incoming": [], "outgoing": []},
- "private": False,
- },
{
"name": "searchable_fts_docsize",
"columns": ["docid", "size"],
diff --git a/tests/test_html.py b/tests/test_html.py
index 5b60d2f544..735a7ef7e0 100644
--- a/tests/test_html.py
+++ b/tests/test_html.py
@@ -39,13 +39,14 @@ def test_homepage(app_client_two_attached_databases):
assert "extra database" == h2.text.strip()
counts_p, links_p = h2.find_all_next("p")[:2]
assert (
- "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+ "4 rows in 2 tables, 3 rows in 3 hidden tables, 1 view" == counts_p.text.strip()
)
# We should only show visible, not hidden tables here:
table_links = [
{"href": a["href"], "text": a.text.strip()} for a in links_p.findAll("a")
]
assert [
+ {"href": r"/extra+database/searchable_fts", "text": "searchable_fts"},
{"href": r"/extra+database/searchable", "text": "searchable"},
{"href": r"/extra+database/searchable_view", "text": "searchable_view"},
] == table_links
diff --git a/tests/test_internals_database.py b/tests/test_internals_database.py
index 1c155cf3b3..70be0f4ed4 100644
--- a/tests/test_internals_database.py
+++ b/tests/test_internals_database.py
@@ -664,3 +664,45 @@ async def test_in_memory_databases_forbid_writes(app_client):
# Using db.execute_write() should work:
await db.execute_write("create table foo (t text)")
assert await db.table_names() == ["foo"]
+
+
+@pytest.mark.asyncio
+async def test_hidden_tables(app_client):
+ ds = app_client.ds
+ db = ds.add_database(Database(ds, is_memory=True, is_mutable=True))
+ assert await db.hidden_table_names() == []
+ await db.execute("create virtual table f using fts5(a)")
+ assert await db.hidden_table_names() == [
+ 'f_config',
+ 'f_content',
+ 'f_data',
+ 'f_docsize',
+ 'f_idx',
+ ]
+
+ await db.execute("create virtual table r using rtree(id, amin, amax)")
+ assert await db.hidden_table_names() == [
+ 'f_config',
+ 'f_content',
+ 'f_data',
+ 'f_docsize',
+ 'f_idx',
+ 'r_node',
+ 'r_parent',
+ 'r_rowid'
+ ]
+
+ await db.execute("create table _hideme(_)")
+ assert await db.hidden_table_names() == [
+ '_hideme',
+ 'f_config',
+ 'f_content',
+ 'f_data',
+ 'f_docsize',
+ 'f_idx',
+ 'r_node',
+ 'r_parent',
+ 'r_rowid'
+ ]
+
+