From 49087fd4b60cce8473a0fb2f5c119cc3896daeeb Mon Sep 17 00:00:00 2001 From: Zhipeng He Date: Tue, 31 Jan 2023 15:29:36 +1000 Subject: [PATCH 1/3] Update validate scripts for mimic-iv-ed v2.2 --- mimic-iv-ed/buildmimic/mysql/validate.sql | 14 ++++---- mimic-iv-ed/buildmimic/mysql/validate_old.sql | 30 ++++++++++++++++ mimic-iv-ed/buildmimic/postgres/validate.sql | 15 ++++---- .../buildmimic/postgres/validate_old.sql | 34 +++++++++++++++++++ 4 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 mimic-iv-ed/buildmimic/mysql/validate_old.sql create mode 100644 mimic-iv-ed/buildmimic/postgres/validate_old.sql diff --git a/mimic-iv-ed/buildmimic/mysql/validate.sql b/mimic-iv-ed/buildmimic/mysql/validate.sql index d0108f685..e419efe5e 100644 --- a/mimic-iv-ed/buildmimic/mysql/validate.sql +++ b/mimic-iv-ed/buildmimic/mysql/validate.sql @@ -1,5 +1,5 @@ -- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. --- This check uses the MIMIC-IV-ED Demo, which is a subset of MIMIC-IV-ED. +-- For MIMIC-IV-ED v2.2, some stay_id and their related features are removed. SELECT CASE WHEN exp.row_count = obs.row_count @@ -10,12 +10,12 @@ SELECT , obs.row_count AS obs , exp.tbl as table_name FROM ( - SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL - SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL - SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL - SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL - SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL - SELECT 'vitalsign' AS tbl, 1646976 AS row_count + SELECT 'edstays' AS tbl, 425087 AS row_count UNION ALL + SELECT 'diagnosis' AS tbl, 899050 AS row_count UNION ALL + SELECT 'medrecon' AS tbl, 2987342 AS row_count UNION ALL + SELECT 'pyxis' AS tbl, 1586053 AS row_count UNION ALL + SELECT 'triage' AS tbl, 425087 AS row_count UNION ALL + SELECT 'vitalsign' AS tbl, 1564610 AS row_count ) exp INNER JOIN ( SELECT 'edstays' AS tbl, COUNT(*) AS row_count FROM edstays UNION ALL diff --git a/mimic-iv-ed/buildmimic/mysql/validate_old.sql b/mimic-iv-ed/buildmimic/mysql/validate_old.sql new file mode 100644 index 000000000..529df4353 --- /dev/null +++ b/mimic-iv-ed/buildmimic/mysql/validate_old.sql @@ -0,0 +1,30 @@ +-- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. +-- Only For MIMIC-IV-ED v1.0, v2.0 +SELECT + CASE + WHEN exp.row_count = obs.row_count + THEN 'PASSED' + ELSE 'FAILED' + END AS chk + , exp.row_count AS exp + , obs.row_count AS obs + , exp.tbl as table_name +FROM ( + SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL + SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL + SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL + SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL + SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL + SELECT 'vitalsign' AS tbl, 1646976 AS row_count +) exp +INNER JOIN ( + SELECT 'edstays' AS tbl, COUNT(*) AS row_count FROM edstays UNION ALL + SELECT 'diagnosis' AS tbl, COUNT(*) AS row_count FROM diagnosis UNION ALL + SELECT 'medrecon' AS tbl, COUNT(*) AS row_count FROM medrecon UNION ALL + SELECT 'pyxis' AS tbl, COUNT(*) AS row_count FROM pyxis UNION ALL + SELECT 'triage' AS tbl, COUNT(*) AS row_count FROM triage UNION ALL + SELECT 'vitalsign' AS tbl, COUNT(*) AS row_count FROM vitalsign +) obs + ON exp.tbl = obs.tbl +ORDER BY exp.tbl +; \ No newline at end of file diff --git a/mimic-iv-ed/buildmimic/postgres/validate.sql b/mimic-iv-ed/buildmimic/postgres/validate.sql index ed0333d93..b7e50a5f1 100644 --- a/mimic-iv-ed/buildmimic/postgres/validate.sql +++ b/mimic-iv-ed/buildmimic/postgres/validate.sql @@ -1,12 +1,13 @@ --- Validate the MIMIC-IV tables built correctly by checking against known row counts. +-- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. +-- For MIMIC-IV-ED v2.2, some stay_id and their related features are removed. WITH expected AS ( - SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL - SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL - SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL - SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL - SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL - SELECT 'vitalsign' AS tbl, 1646976 AS row_count + SELECT 'edstays' AS tbl, 425087 AS row_count UNION ALL + SELECT 'diagnosis' AS tbl, 899050 AS row_count UNION ALL + SELECT 'medrecon' AS tbl, 2987342 AS row_count UNION ALL + SELECT 'pyxis' AS tbl, 1586053 AS row_count UNION ALL + SELECT 'triage' AS tbl, 425087 AS row_count UNION ALL + SELECT 'vitalsign' AS tbl, 1564610 AS row_count ) , observed as ( diff --git a/mimic-iv-ed/buildmimic/postgres/validate_old.sql b/mimic-iv-ed/buildmimic/postgres/validate_old.sql new file mode 100644 index 000000000..2d18151b3 --- /dev/null +++ b/mimic-iv-ed/buildmimic/postgres/validate_old.sql @@ -0,0 +1,34 @@ +-- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. +-- Only For MIMIC-IV-ED v1.0, v2.0 +WITH expected AS +( + SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL + SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL + SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL + SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL + SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL + SELECT 'vitalsign' AS tbl, 1646976 AS row_count +) +, observed as +( + SELECT 'edstays' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.edstays UNION ALL + SELECT 'diagnosis' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.diagnosis UNION ALL + SELECT 'medrecon' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.medrecon UNION ALL + SELECT 'pyxis' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.pyxis UNION ALL + SELECT 'triage' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.triage UNION ALL + SELECT 'vitalsign' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.vitalsign +) +SELECT + exp.tbl + , exp.row_count AS expected_count + , obs.row_count AS observed_count + , CASE + WHEN exp.row_count = obs.row_count + THEN 'PASSED' + ELSE 'FAILED' + END AS ROW_COUNT_CHECK +FROM expected exp +INNER JOIN observed obs + ON exp.tbl = obs.tbl +ORDER BY exp.tbl +; \ No newline at end of file From d00cce7a2170b7cc2bdfa24d4638b7b242c299d5 Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Fri, 3 Feb 2023 19:33:15 -0500 Subject: [PATCH 2/3] update comment to reference full dataset --- mimic-iv-ed/buildmimic/mysql/validate.sql | 2 +- mimic-iv-ed/buildmimic/postgres/validate.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iv-ed/buildmimic/mysql/validate.sql b/mimic-iv-ed/buildmimic/mysql/validate.sql index e419efe5e..7a04c6a35 100644 --- a/mimic-iv-ed/buildmimic/mysql/validate.sql +++ b/mimic-iv-ed/buildmimic/mysql/validate.sql @@ -1,5 +1,5 @@ -- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. --- For MIMIC-IV-ED v2.2, some stay_id and their related features are removed. +-- Tested against MIMIC-IV-ED v2.2. SELECT CASE WHEN exp.row_count = obs.row_count diff --git a/mimic-iv-ed/buildmimic/postgres/validate.sql b/mimic-iv-ed/buildmimic/postgres/validate.sql index b7e50a5f1..bd81fcab4 100644 --- a/mimic-iv-ed/buildmimic/postgres/validate.sql +++ b/mimic-iv-ed/buildmimic/postgres/validate.sql @@ -1,5 +1,5 @@ -- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. --- For MIMIC-IV-ED v2.2, some stay_id and their related features are removed. +-- Tested against MIMIC-IV-ED v2.2. WITH expected AS ( SELECT 'edstays' AS tbl, 425087 AS row_count UNION ALL From 6908fa303d9322217518539fd2f1066f61b88b5a Mon Sep 17 00:00:00 2001 From: Alistair Johnson Date: Fri, 3 Feb 2023 19:33:38 -0500 Subject: [PATCH 3/3] remove scripts testing row count for older versions --- mimic-iv-ed/buildmimic/mysql/validate_old.sql | 30 ---------------- .../buildmimic/postgres/validate_old.sql | 34 ------------------- 2 files changed, 64 deletions(-) delete mode 100644 mimic-iv-ed/buildmimic/mysql/validate_old.sql delete mode 100644 mimic-iv-ed/buildmimic/postgres/validate_old.sql diff --git a/mimic-iv-ed/buildmimic/mysql/validate_old.sql b/mimic-iv-ed/buildmimic/mysql/validate_old.sql deleted file mode 100644 index 529df4353..000000000 --- a/mimic-iv-ed/buildmimic/mysql/validate_old.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. --- Only For MIMIC-IV-ED v1.0, v2.0 -SELECT - CASE - WHEN exp.row_count = obs.row_count - THEN 'PASSED' - ELSE 'FAILED' - END AS chk - , exp.row_count AS exp - , obs.row_count AS obs - , exp.tbl as table_name -FROM ( - SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL - SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL - SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL - SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL - SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL - SELECT 'vitalsign' AS tbl, 1646976 AS row_count -) exp -INNER JOIN ( - SELECT 'edstays' AS tbl, COUNT(*) AS row_count FROM edstays UNION ALL - SELECT 'diagnosis' AS tbl, COUNT(*) AS row_count FROM diagnosis UNION ALL - SELECT 'medrecon' AS tbl, COUNT(*) AS row_count FROM medrecon UNION ALL - SELECT 'pyxis' AS tbl, COUNT(*) AS row_count FROM pyxis UNION ALL - SELECT 'triage' AS tbl, COUNT(*) AS row_count FROM triage UNION ALL - SELECT 'vitalsign' AS tbl, COUNT(*) AS row_count FROM vitalsign -) obs - ON exp.tbl = obs.tbl -ORDER BY exp.tbl -; \ No newline at end of file diff --git a/mimic-iv-ed/buildmimic/postgres/validate_old.sql b/mimic-iv-ed/buildmimic/postgres/validate_old.sql deleted file mode 100644 index 2d18151b3..000000000 --- a/mimic-iv-ed/buildmimic/postgres/validate_old.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Validate the MIMIC-IV-ED tables built correctly by checking against known row counts. --- Only For MIMIC-IV-ED v1.0, v2.0 -WITH expected AS -( - SELECT 'edstays' AS tbl, 447712 AS row_count UNION ALL - SELECT 'diagnosis' AS tbl, 946692 AS row_count UNION ALL - SELECT 'medrecon' AS tbl, 3143791 AS row_count UNION ALL - SELECT 'pyxis' AS tbl, 1670590 AS row_count UNION ALL - SELECT 'triage' AS tbl, 447712 AS row_count UNION ALL - SELECT 'vitalsign' AS tbl, 1646976 AS row_count -) -, observed as -( - SELECT 'edstays' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.edstays UNION ALL - SELECT 'diagnosis' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.diagnosis UNION ALL - SELECT 'medrecon' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.medrecon UNION ALL - SELECT 'pyxis' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.pyxis UNION ALL - SELECT 'triage' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.triage UNION ALL - SELECT 'vitalsign' AS tbl, COUNT(*) AS row_count FROM mimiciv_ed.vitalsign -) -SELECT - exp.tbl - , exp.row_count AS expected_count - , obs.row_count AS observed_count - , CASE - WHEN exp.row_count = obs.row_count - THEN 'PASSED' - ELSE 'FAILED' - END AS ROW_COUNT_CHECK -FROM expected exp -INNER JOIN observed obs - ON exp.tbl = obs.tbl -ORDER BY exp.tbl -; \ No newline at end of file