From 9559770b6d6e1b292c1a58eb6be2675eaf7ec61e Mon Sep 17 00:00:00 2001 From: hughcrt Date: Sun, 26 Jan 2025 17:50:27 +0000 Subject: [PATCH] ok --- packages/backend/src/api/v1/filters.ts | 7 ++--- packages/backend/src/create-indexes.ts | 30 ++++++++++++++----- packages/backend/src/index.ts | 2 +- ...rializedViews.ts => materialized-views.ts} | 5 ++-- packages/db/0060.sql | 22 ++++++++++++++ 5 files changed, 51 insertions(+), 15 deletions(-) rename packages/backend/src/jobs/{materializedViews.ts => materialized-views.ts} (83%) create mode 100644 packages/db/0060.sql diff --git a/packages/backend/src/api/v1/filters.ts b/packages/backend/src/api/v1/filters.ts index 28e99151..e777976d 100644 --- a/packages/backend/src/api/v1/filters.ts +++ b/packages/backend/src/api/v1/filters.ts @@ -45,12 +45,11 @@ filters.get("/metadata", async (ctx: Context) => { const rows = await sql` select distinct - jsonb_object_keys(metadata) as key + key from - run + metadata_cache where - project_id = ${projectId} - and metadata is not null + project_id = ${projectId} order by key; `; diff --git a/packages/backend/src/create-indexes.ts b/packages/backend/src/create-indexes.ts index 0644a509..e0f7dcf3 100644 --- a/packages/backend/src/create-indexes.ts +++ b/packages/backend/src/create-indexes.ts @@ -1,5 +1,6 @@ import sql from "./utils/db"; +// TODO: rename to "async migration" or something like this export async function createIndexes() { await sql`select pg_advisory_lock(123456789)`; @@ -11,7 +12,7 @@ export async function createIndexes() { operation, statement from - _db_migration_index + _db_migration_async where status in ('pending', 'failed') order @@ -24,7 +25,7 @@ export async function createIndexes() { try { await sql` update - _db_migration_index + _db_migration_async set status = 'in-progress' where @@ -33,6 +34,7 @@ export async function createIndexes() { await sql.unsafe(statement); + // TODO: rename to "create_index" and "drop_index" if (operation === "create") { const [validCheck] = await sql` select @@ -49,7 +51,7 @@ export async function createIndexes() { if (validCheck) { await sql` update - _db_migration_index + _db_migration_async set status = 'done' where @@ -60,7 +62,7 @@ export async function createIndexes() { await sql.unsafe(`drop index if exists ${name}`); await sql` update - _db_migration_index + _db_migration_async set status = 'failed' where @@ -83,7 +85,7 @@ export async function createIndexes() { if (!stillExists) { await sql` update - _db_migration_index + _db_migration_async set status = 'done' where @@ -93,7 +95,7 @@ export async function createIndexes() { } else { await sql` update - _db_migration_index + _db_migration_async set status = 'failed' where @@ -101,13 +103,25 @@ export async function createIndexes() { `; console.warn(`Index drop "${name}" failed; index still exists.`); } + } else if (operation === "create-materialized-view") { + await sql` + update + _db_migration_async + set + status = 'done' + where + id = ${id} + `; + console.log( + `Materialized view migration "${name}" completed successfully.`, + ); } } catch (err) { console.error(`Index migration "${name}" errored:`, err); - await sql.unsafe(`drop index if exists ${name}`); + await sql.unsafe(`drop index if exists ${name}`); // TODO: only drop if it was an index, not a materialized views await sql` update - _db_migration_index + _db_migration_async set status = 'failed' where diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index aac4507f..2317f51f 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -17,7 +17,7 @@ import ratelimit from "./utils/ratelimit"; import { initSentry, requestHandler, tracingMiddleWare } from "./utils/sentry"; import licenseMiddleware from "./utils/license"; import config from "./utils/config"; -import { startMaterializedViewRefreshJob } from "./jobs/materializedViews"; +import { startMaterializedViewRefreshJob } from "./jobs/materialized-views"; import { createIndexes } from "./create-indexes"; checkDbConnection(); diff --git a/packages/backend/src/jobs/materializedViews.ts b/packages/backend/src/jobs/materialized-views.ts similarity index 83% rename from packages/backend/src/jobs/materializedViews.ts rename to packages/backend/src/jobs/materialized-views.ts index 03f6c501..97c48265 100644 --- a/packages/backend/src/jobs/materializedViews.ts +++ b/packages/backend/src/jobs/materialized-views.ts @@ -2,11 +2,12 @@ import sql from "../utils/db"; import { sleep } from "../utils/misc"; export async function startMaterializedViewRefreshJob() { + // TODO: locks if (process.env.DISABLE_MATERIALIZED_VIEW_REFRESH) { return; } try { - const views = ["run_parent_feedback_cache"]; + const views = ["run_parent_feedback_cache", "metadata_cache"]; while (true) { for (const view of views) { @@ -18,7 +19,7 @@ export async function startMaterializedViewRefreshJob() { ); } - await sleep(1000); + await sleep(60000); } } catch (error) { console.error(error); diff --git a/packages/db/0060.sql b/packages/db/0060.sql new file mode 100644 index 00000000..dc726f2a --- /dev/null +++ b/packages/db/0060.sql @@ -0,0 +1,22 @@ +alter table _db_migration_index rename to _db_migration_async; + +insert into _db_migration_async (name, operation, statement) values +('metadata_cache', 'create-materialized-view', +'create materialized view metadata_cache as +select + run.project_id, + run.type, + jsonb_object_keys(run.metadata) as key, + now() as refreshed_at +from + run +group by + run.project_id, + run.type, + jsonb_object_keys(run.metadata);' +), +('metadata_cache_project_id_idx', 'create', 'create index concurrently if not exists metadata_cache_project_id_idx on metadata_cache(project_id)'), +('metadata_cache_project_id_type_key_idx', 'create', 'create unique index concurrently if not exists metadata_cache_project_id_type_key_idx on metadata_cache (project_id, type, key)'); + + +