[CHIP-1] Add query.json and patients.txt

- Add migrations via flyway - Change db schema for new fields - Add to API to support patient and query upload
hms-dbmi · Dec 31, 2024 · 1d8e4cc · 1d8e4cc
1 parent dd50a7c
commit 1d8e4cc
Show file tree

Hide file tree

Showing 16 changed files with 211 additions and 33 deletions.
diff --git a/uploader/README.md b/uploader/README.md
@@ -47,7 +47,7 @@ that Amazon provides you.
 
 5. Run
 
-Run `docker run --rm --env-file .env gic-data-uploader`. Here is an example of a
+Run `docker compose --profile production up -d `. Here is an example of a
 successful output, with logging prefixes omitted:
 
 ```
@@ -79,10 +79,6 @@ Verifying delete capabilities
 S3 connection verified.
 ```
 
-TODO:
-- Mount this study in service workbench
-  - Admin UI
-- Test all 4 workspace types
-  - Mount study
-  - R/W access
-  - 
+6. Run Migrations
+
+`docker compose --profile migrate up -d`
diff --git a/uploader/docker-compose.yml b/uploader/docker-compose.yml
@@ -14,8 +14,9 @@ services:
     volumes:
       - $DOCKER_CONFIG_DIR/aws_uploads/:/gic_query_results/
     networks:
-      - hpdsNet
       - picsure
+    profiles:
+      - "production"
   uploader-db:
     image: mysql:8.0
     container_name: uploader-db
@@ -39,11 +40,23 @@ services:
     volumes:
       - /usr/local/pic-sure-services/uploader/data/seed.sql:/docker-entrypoint-initdb.d/seed.sql:ro
       - uploader-db-data:/var/lib/mysql
+  migrations:
+    image: flyway/flyway:11-alpine
+    container_name: flyway
+    command: -url=jdbc:mysql://uploader-db:3306/${DATA_UPLOAD_DB_DATABASE}?allowPublicKeyRetrieval=true -schemas=${DATA_UPLOAD_DB_DATABASE} -user=${DATA_UPLOAD_DB_USER} -password=${DATA_UPLOAD_DB_PASS} -connectRetries=60 -validateMigrationNaming=true migrate
+    env_file:
+      - .env
+    volumes:
+      - ./flyway:/flyway/sql
+    networks:
+      - picsure
+    profiles:
+      - "migrate"
+    depends_on:
+      - "uploader-db"
 volumes:
   uploader-db-data:
 
 networks:
   picsure:
     external: true
-  hpdsNet:
-    external: true
diff --git a/uploader/flyway/V1__initial.sql b/uploader/flyway/V1__initial.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS `query_status` (
+    QUERY BINARY(16) NOT NULL,
+    GENOMIC_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent',
+    PHENOTYPIC_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent',
+    APPROVED DATETIME,
+    SITE VARCHAR(64) NOT NULL DEFAULT '',
+    PRIMARY KEY (`QUERY`)
+);
diff --git a/uploader/flyway/V2__add_patient.sql b/uploader/flyway/V2__add_patient.sql
@@ -0,0 +1,5 @@
+ALTER TABLE `query_status`
+    ADD PATIENT_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent';
+
+ALTER TABLE `query_status`
+    ADD QUERY_JSON_STATUS VARCHAR(64) NOT NULL DEFAULT 'Unsent';
diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java
@@ -105,4 +105,8 @@ private String createBody(Object query) {
             return null;
         }
     }
+
+    public boolean writePatientData(Query query) {
+        return writeData(query, "patients");
+    }
 }
diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java
@@ -5,6 +5,7 @@
 import java.time.LocalDate;
 
 public record DataUploadStatuses(
-    UploadStatus genomic, UploadStatus phenotypic, String queryId, @Nullable LocalDate approved, String site
+    UploadStatus genomic, UploadStatus phenotypic, UploadStatus patient, UploadStatus query,
+    String queryId, @Nullable LocalDate approved, String site
 ) {
 }
diff --git a/...r/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java b/...r/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java
@@ -12,13 +12,15 @@
 public class DataUploadStatusesMapper implements RowMapper<DataUploadStatuses> {
     @Override
     public DataUploadStatuses mapRow(ResultSet rs, int rowNum) throws SQLException {
-        UploadStatus genomicStatus = UploadStatus.fromString(rs.getString("GENOMIC_STATUS"));
-        UploadStatus phenotypicStatus = UploadStatus.fromString(rs.getString("PHENOTYPIC_STATUS"));
+        UploadStatus genomic = UploadStatus.fromString(rs.getString("GENOMIC_STATUS"));
+        UploadStatus pheno = UploadStatus.fromString(rs.getString("PHENOTYPIC_STATUS"));
+        UploadStatus patient = UploadStatus.fromString(rs.getString("PATIENT_STATUS"));
+        UploadStatus queryStatus = UploadStatus.fromString(rs.getString("QUERY_JSON_STATUS"));
         String query = fromDashlessString(rs.getString("QUERY")).toString();
         Date approved = rs.getDate("APPROVED");
         String site = rs.getString("SITE");
         return new DataUploadStatuses(
-                genomicStatus, phenotypicStatus, query, approved == null ? null : approved.toLocalDate(), site
+                genomic, pheno, patient, queryStatus, query, approved == null ? null : approved.toLocalDate(), site
         );
     }
 

diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java
@@ -1,5 +1,6 @@
 package edu.harvard.dbmi.avillach.dataupload.status;
 
+import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Repository;
@@ -18,7 +19,8 @@ public class StatusRepository {
     public Optional<DataUploadStatuses> getQueryStatus(String queryId) {
         String sql = """
                 SELECT
-                    GENOMIC_STATUS, PHENOTYPIC_STATUS, hex(QUERY) as QUERY, APPROVED, SITE
+                    GENOMIC_STATUS, PHENOTYPIC_STATUS, PATIENT_STATUS, QUERY_JSON_STATUS,
+                    hex(QUERY) as QUERY, APPROVED, SITE
                 FROM
                     query_status
                 WHERE
@@ -68,4 +70,24 @@ public void setSite(String picSureId, String site) {
         """;
         template.update(sql, picSureId.replace("-", ""), site, site);
     }
+
+    public void setPatientStatus(String queryId, UploadStatus status) {
+        String sql = """
+            INSERT INTO query_status
+                (query, patient_status)
+                VALUES (unhex(?), ?)
+                ON DUPLICATE KEY UPDATE patient_status=?
+        """;
+        template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
+    }
+
+    public void setQueryUploadStatus(String queryId, UploadStatus status) {
+        String sql = """
+            INSERT INTO query_status
+                (query, query_json_status)
+                VALUES (unhex(?), ?)
+                ON DUPLICATE KEY UPDATE query_json_status=?
+        """;
+        template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
+    }
 }
diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusService.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusService.java
@@ -44,4 +44,12 @@ public Optional<DataUploadStatuses> approve(String queryId, LocalDate approvalDa
     public void setSite(Query query, String site) {
         repository.setSite(query.getPicSureId(), site);
     }
+
+    public void setPatientStatus(Query query, UploadStatus uploadStatus) {
+        repository.setPatientStatus(query.getPicSureId(), uploadStatus);
+    }
+
+    public void setQueryUploadStatus(Query query, UploadStatus uploadStatus) {
+        repository.setQueryUploadStatus(query.getPicSureId(), uploadStatus);
+    }
 }
diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java
@@ -9,7 +9,7 @@
 import java.util.function.Function;
 
 public enum DataType {
-    Genomic("genomic_data.tsv"), Phenotypic("phenotypic_data.csv");
+    Genomic("genomic_data.tsv"), Phenotypic("phenotypic_data.csv"), Patient("patients.txt");
     public final String fileName;
 
     DataType(String fileName) {
@@ -20,13 +20,15 @@ public BiConsumer<Query, UploadStatus> getStatusSetter(StatusService statusServi
         return switch (this) {
             case Genomic -> statusService::setGenomicStatus;
             case Phenotypic -> statusService::setPhenotypicStatus;
+            case Patient -> statusService::setPatientStatus;
         };
     }
 
     public Function<Query, Boolean> getHPDSUpload(HPDSClient client) {
         return switch (this) {
             case Genomic -> client::writeGenomicData;
             case Phenotypic -> client::writePhenotypicData;
+            case Patient -> client::writePatientData;
         };
     }
 }
diff --git a/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java b/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java
@@ -1,5 +1,7 @@
 package edu.harvard.dbmi.avillach.dataupload.upload;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import edu.harvard.dbmi.avillach.dataupload.aws.AWSClientBuilder;
 import edu.harvard.dbmi.avillach.dataupload.aws.SiteAWSInfo;
 import edu.harvard.dbmi.avillach.dataupload.hpds.HPDSClient;
@@ -62,6 +64,8 @@ public class DataUploadService {
     @Autowired
     private Map<String, SiteAWSInfo> roleARNs;
 
+    private static final ObjectMapper mapper = new ObjectMapper();
+
     public DataUploadStatuses asyncUpload(Query query, String site, DataType dataType) {
         dataType.getStatusSetter(statusService).accept(query, UploadStatus.Queued);
         Thread.ofVirtual().start(() -> uploadData(query, dataType, site));
@@ -110,10 +114,41 @@ protected void uploadData(Query query, DataType dataType, String site) {
         } else {
             statusSetter.accept(query, UploadStatus.Error);
         }
+        uploadQueryJson(query, roleARNs.get(site));
         LOG.info("Releasing lock for  {} / {}", dataType, query.getPicSureId());
         uploadLock.release();
     }
 
+    private void uploadQueryJson(Query query, SiteAWSInfo site) {
+        UploadStatus queryUploadStatus = statusService.getStatus(query.getPicSureId())
+                                             .map(DataUploadStatuses::query)
+                                             .orElse(UploadStatus.Unsent);
+        if (queryUploadStatus == UploadStatus.Uploaded || queryUploadStatus == UploadStatus.Uploading) {
+            return;
+        }
+        statusService.setQueryUploadStatus(query, UploadStatus.Uploading);
+        LOG.info("Uploading query json for {}", query.getPicSureId());
+        try {
+            String queryJson = mapper.writeValueAsString(query);
+            LOG.info("Created query JSON. Writing to file.");
+            Path jsonPath = Path.of(sharingRoot.toString(), query.getPicSureId(), "query.json");
+            Files.writeString(jsonPath, queryJson);
+            if (!uploadFileFromPath(jsonPath, site, query.getPicSureId())) {
+                LOG.info("Failed to write query.json");
+                statusService.setQueryUploadStatus(query, UploadStatus.Error);
+            }
+            Files.delete(jsonPath);
+        } catch (JsonProcessingException e) {
+            statusService.setQueryUploadStatus(query, UploadStatus.Error);
+            LOG.info("Failed to get query json: ", e);
+        } catch (IOException e) {
+            statusService.setQueryUploadStatus(query, UploadStatus.Error);
+            LOG.info("Failed to write query json: ", e);
+        }
+        LOG.info("Successfully uploaded query.json for {} to {}", query.getPicSureId(), site.siteName());
+        statusService.setQueryUploadStatus(query, UploadStatus.Uploaded);
+    }
+
     private void deleteFile(Path data) {
         try {
             Files.delete(data);

diff --git a/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java
@@ -129,8 +129,8 @@ void shouldGetQueryId() {
 
         DataUploadStatuses actual = subject.getQueryStatus(query.getPicSureId()).orElseThrow();
         DataUploadStatuses expected = new DataUploadStatuses(
-            UploadStatus.Uploaded, UploadStatus.Error, "33613336-3934-3761-2d38-3233312d3131",
-            LocalDate.of(2022, 2, 22), "bch"
+            UploadStatus.Uploaded, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Uploading,
+            "33613336-3934-3761-2d38-3233312d3131", LocalDate.of(2022, 2, 22), "bch"
         );
 
         Assertions.assertEquals(expected, actual);
@@ -148,4 +148,30 @@ void shouldSetSite() {
 
         Assertions.assertEquals(expected, actual);
     }
+
+    @Test
+    void shouldSetPatient() {
+        Query query = new Query();
+        query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
+
+        subject.setPatientStatus(query.getPicSureId(), UploadStatus.Uploaded);
+        Optional<UploadStatus> actual = subject.getQueryStatus(query.getPicSureId())
+                                            .map(DataUploadStatuses::patient);
+        Optional<UploadStatus> expected = Optional.of(UploadStatus.Uploaded);
+
+        Assertions.assertEquals(expected, actual);
+    }
+
+    @Test
+    void shouldSetQueryStatus() {
+        Query query = new Query();
+        query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
+
+        subject.setQueryUploadStatus(query.getPicSureId(), UploadStatus.Error);
+        Optional<UploadStatus> actual = subject.getQueryStatus(query.getPicSureId())
+                                            .map(DataUploadStatuses::query);
+        Optional<UploadStatus> expected = Optional.of(UploadStatus.Error);
+
+        Assertions.assertEquals(expected, actual);
+    }
 }
diff --git a/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java b/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java
@@ -75,8 +75,10 @@ void shouldApprove() {
     void shouldGetQueryStatus() {
         Query q = new Query();
         q.setPicSureId(":)");
-        DataUploadStatuses statuses =
-            new DataUploadStatuses(UploadStatus.Error, UploadStatus.Error, ":)", LocalDate.now(), "bch");
+        DataUploadStatuses statuses = new DataUploadStatuses(
+            UploadStatus.Error, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Unsent,
+            ":)", LocalDate.now(), "bch"
+        );
         Mockito.when(repository.getQueryStatus(":)"))
             .thenReturn(Optional.of(statuses));
 
@@ -85,4 +87,24 @@ void shouldGetQueryStatus() {
 
         Assertions.assertEquals(expected, actual);
     }
+
+    @Test
+    void shouldSetPatientStatus() {
+        Query q = new Query();
+        q.setPicSureId(":)");
+
+        subject.setPatientStatus(q, UploadStatus.Uploading);
+
+        Mockito.verify(repository, Mockito.times(1)).setPatientStatus(":)", UploadStatus.Uploading);
+    }
+
+    @Test
+    void shouldSetQueryStatus() {
+        Query q = new Query();
+        q.setPicSureId(":)");
+
+        subject.setQueryUploadStatus(q, UploadStatus.Uploading);
+
+        Mockito.verify(repository, Mockito.times(1)).setQueryUploadStatus(":)", UploadStatus.Uploading);
+    }
 }
diff --git a/...r/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java b/...r/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java
@@ -40,10 +40,14 @@ public void init() {
     void shouldUpload() {
         Query query = new Query();
         query.setPicSureId("my id");
-        DataUploadStatuses before =
-            new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH, "bch");
-        DataUploadStatuses after =
-            new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, query.getPicSureId(), LocalDate.EPOCH, "bch");
+        DataUploadStatuses before = new DataUploadStatuses(
+            UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent,
+            query.getPicSureId(), LocalDate.EPOCH, "bch"
+        );
+        DataUploadStatuses after = new DataUploadStatuses(
+            UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent,
+            query.getPicSureId(), LocalDate.EPOCH, "bch"
+        );
         Mockito.when(statusService.getStatus(query.getPicSureId()))
             .thenReturn(Optional.of(before));
         Mockito.when(uploadService.asyncUpload(query, "bch", DataType.Genomic))
@@ -85,7 +89,7 @@ void shouldBlockUnapproved() {
         Query query = new Query();
         query.setPicSureId("my id");
         DataUploadStatuses nullApprovalDate =
-            new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch");
+            new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch");
         Mockito.when(statusService.getStatus(query.getPicSureId()))
             .thenReturn(Optional.of(nullApprovalDate));
 
@@ -99,7 +103,7 @@ void shouldBlockApprovedInFuture() {
         Query query = new Query();
         query.setPicSureId("my id");
         DataUploadStatuses nullApprovalDate =
-            new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch");
+            new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch");
         Mockito.when(statusService.getStatus(query.getPicSureId()))
             .thenReturn(Optional.of(nullApprovalDate));
 
@@ -113,7 +117,7 @@ void shouldNoOpWhenAlreadyUploading() {
         Query query = new Query();
         query.setPicSureId("my id");
         DataUploadStatuses uploading =
-            new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, query.getPicSureId(), LocalDate.EPOCH, "bch");
+            new DataUploadStatuses(UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH, "bch");
         Mockito.when(statusService.getStatus(query.getPicSureId()))
             .thenReturn(Optional.of(uploading));