Merge pull request #18 from KPMP/develop

Merge develop into master

README.md

@@ -24,7 +24,8 @@ To initialize the `.env` file, copy `wsi-worker/scripts/.env.example` and modify
 1. KPMP ID
 2. SVS filename without the `.svs` extension
 3. Package File ID from the Data Lake
-4. [Optional] stain type.  Values are 'he', 'pas', 'silver', 'tri', 'frz', 'tol' and 'cr'.  Defaults to 'pas'
+4. Slide type. Values are 'LM', 'EM', 'IF'.
+5. [Optional] stain type.  Values are 'he', 'pas', 'silver', 'tri', 'frz', 'tol' and 'cr'.  Defaults to 'pas'
 
 #### .env File
 1. `ENV_IMAGE`: defaults to `kingstonduo/wsi-worker`

changelog.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## Release 1.4 [unreleased]
+Brief summary of what's in this release:
+- updated scripts to take in slide type information so that we can add non-svs images
+- updated scripts to handle non-svs image pyramid building
+
+### Breaking changes
+
+Breaking changes include any database updates needed, if we need to edit any files on system (like .env or certs, etc). Things that are outside of the code itself that need changed for the system to work.
+- scripts require a new piece of information to determine what type of slide they are
+
+### Non-breaking changes
+
+Just a place to keep track of things that have changed in the code that we may want to pay special attention to when smoke testing, etc.

docker/wsi-worker/generate_mongo_records.js

@@ -8,12 +8,16 @@ const dbName = 'knowledgeEnvironment';
 const kpmpId = process.argv[2];
 const slideName = process.argv[3];
 const fileUUID = process.argv[4];
-const stainType = process.argv[5];
-const metadataFile = process.argv[6];
-let rawdata = fs.readFileSync(metadataFile);
-let metadata = JSON.parse(rawdata);
+const slideType = process.argv[5].toUpperCase();
+const stainType = process.argv[6];
+const metadataFile = process.argv[7];
 
 const addAndUpdateParticipants = function (db, callback) {
+	let metadata = null;
+	if (slideType === "LM" && fs.existsSync(metadataFile)) {
+		let rawdata = fs.readFileSync(metadataFile);
+		metadata = JSON.parse(rawdata);
+	}
 
 	let stainCollection = db.collection("stains");
 
@@ -25,6 +29,21 @@ const addAndUpdateParticipants = function (db, callback) {
 			stainsByType[stain.type] = stain;
 		});
 
+		let slideTypeFull = "";
+		switch (slideType) {
+			case "LM":
+				slideTypeFull = "(LM) Light Microscopy";
+				break;
+			case "EM":
+				slideTypeFull = "(EM) Electron Microscopy";
+				break;
+			case "IF":
+				slideTypeFull = "(IF) Immunofluorescence";
+				break;
+			default:
+				break;
+		}
+
 		let participantCollection = db.collection("patients");
 
 		participantCollection.find({ kpmp_id: kpmpId }).toArray(function (err, docs) {
@@ -37,24 +56,33 @@ const addAndUpdateParticipants = function (db, callback) {
 					let added = false;
 
 					slides.forEach(slide => {
-						if (slide.slideName === slideName) {
+						if (slideType === "LM" && slide.slideName === slideName) {
 							slide = slide['metadata'] = metadata;
 							participantCollection.update({_id: doc._id }, { $set: { slides: slides }});
 							console.log("updated slide with metadata");
 							exists = true;
 						}
 					});
-
 					if (!exists) {
-						if (stainsByType[stainType] !== null && stainsByType[stainType] !== undefined) {
-
+						if (stainsByType[stainType] !== null && stainsByType[stainType] !== undefined && slideType === "LM") {
 							slides.push({
 								_id: fileUUID,
 								slideName: slideName,
 								metadata: metadata,
-								stain: stainsByType[stainType]
+								stain: stainsByType[stainType],
+								slideType: slideTypeFull
+							});
+							console.log("--- adding new slide, fileUUID: " + fileUUID);
+							participantCollection.update({ _id: doc._id }, { $set: { slides: slides } });
+							added = true;
+						}
+						else if (slideType === "EM") {
+							slides.push({
+								_id: fileUUID,
+								slideName: slideName,
+								stain: { type: "other" },
+								slideType: slideTypeFull
 							});
-
 							console.log("--- adding new slide, fileUUID: " + fileUUID);
 							participantCollection.update({ _id: doc._id }, { $set: { slides: slides } });
 							added = true;
@@ -73,16 +101,34 @@ const addAndUpdateParticipants = function (db, callback) {
 				callback()
 			});
 			if (docs.length === 0) {
-				if (stainsByType[stainType] !== null && stainsByType[stainType] !== undefined) {
+				if (stainsByType[stainType] !== null && stainsByType[stainType] !== undefined && slideType === "LM") {
+
+					let participantRecord = {
+						kpmp_id: kpmpId,
+						label: kpmpId,
+						slides: [{
+							_id: fileUUID,
+							slideName: slideName,
+							stain: stainsByType[stainType],
+							slideType: slideTypeFull,
+							metadata: metadata
+						}]
+					};
 
+					console.log("--- adding new participant and slides, KPMP_ID: " + kpmpId);
+					participantCollection.insertOne(participantRecord, function () {
+						callback();
+					});
+				}
+				else if (slideType === "EM") {
 					let participantRecord = {
 						kpmp_id: kpmpId,
 						label: kpmpId,
 						slides: [{
 							_id: fileUUID,
 							slideName: slideName,
-							metadata: metadata,
-							stain: stainsByType[stainType]
+							stain: { type: "other" },
+							slideType: slideTypeFull,
 						}]
 					};
 

docker/wsi-worker/svs2dz

@@ -3,7 +3,7 @@
 source /usr/sbin/svs2dz-functions.sh
 assert_no_errors "!!! source load failure"
 
-validate_args $1 $2 $3 $4
+validate_args $1 $2 $3 $4 $5
 # no assertion needed; validate_args works differently
 
 call_vips $1 $2 $3 $4
@@ -15,10 +15,13 @@ assert_no_errors "!!! update_link_file failure"
 run_link_file
 assert_no_errors "!!! run_link_file failure"
 
-extract_metadata $1 $2 $3 $4
-assert_no_errors "!!! extract_metadata_ failure"
+if [ $4 = "LM" ]
+then
+    extract_metadata $1 $2 $3 $4
+    assert_no_errors "!!! extract_metadata failure"
+fi
 
-generate_mongo_records $1 $2 $3 $4
+generate_mongo_records $1 $2 $3 $4 $5
 assert_no_errors "!!! generate_mongo_records failure"
 
 echo "+++ done"

docker/wsi-worker/svs2dz-functions.sh

@@ -8,10 +8,10 @@ JOB_OUT_DIR=/data/job/out
 print_help() {
   cat << EOL
 --- Command help
-Usage: svs2dz "KPMP_ID" "FILENAME_NO_EXTENSION" "PKG_FILE_ID" ["STAIN_TYPE"]
+Usage: svs2dz "KPMP_ID" "FILENAME_NO_EXTENSION" "PKG_FILE_ID" "SLIDE_TYPE" ["STAIN_TYPE"]
 All arguments should be enclosed in double-quotes to escape spaces.
-Example: svs2dz "KPMP-Ex1" "KPMP-Ex1_TRI_1of1" "b9f7c729-8370-4f8d-9753-4a36e8ae57a4" "TRI"
-... The above example creates symlinks at <ENV_WSE_LINK_FROM_DIR>/files_KPMP-Ex1/KPMP-Ex1_TRI_1of1, stain type Trichrome
+Example: svs2dz "KPMP-Ex1" "KPMP-Ex1_TRI_1of1" "b9f7c729-8370-4f8d-9753-4a36e8ae57a4" "IF" "TRI"
+... The above example creates symlinks at <ENV_WSE_LINK_FROM_DIR>/files_KPMP-Ex1/KPMP-Ex1_TRI_1of1, slide type Immunofluorescence, stain type Trichrome
 EOL
 }
 
@@ -53,8 +53,12 @@ validate_args() {
   fi
 
   if [[ -z $4 ]]; then
+    print_error "!!! arg 4 must be slide type"
+  fi
+
+  if [[ -z $5 ]]; then
     echo "... optional stain not passed; defaulting to type 'pas'"
-    4=pas
+    $5 = "pas"
   fi
 
   if (( DID_ERROR > 0 )); then
@@ -65,9 +69,14 @@ validate_args() {
 
 call_vips() {
   mkdir -p $ENV_LINK_SRC_DIR/files_$1
-
-  echo "--- vips dzsave $JOB_IN_DIR/$2.svs $ENV_LINK_SRC_DIR/files_$1/$2"
-  vips dzsave $JOB_IN_DIR/$2.svs $ENV_LINK_SRC_DIR/files_$1/$2
+  if [ "$4" == "EM" ] || [ "$4" == "em" ]; then
+    echo "--vips dzsave $JOB_IN_DIR/$2.jpg $ENV_LINK_SRC_DIR/files_$1/$2"
+    vips dzsave $JOB_IN_DIR/$2.jpg $ENV_LINK_SRC_DIR/files_$1/$2
+  else
+    echo "--- vips dzsave $JOB_IN_DIR/$2.svs $ENV_LINK_SRC_DIR/files_$1/$2"
+    vips dzsave $JOB_IN_DIR/$2.svs $ENV_LINK_SRC_DIR/files_$1/$2
+  fi
+
   # Copy a consistently-well-sized DZ file out as our thumbnail
   cp $ENV_LINK_SRC_DIR/files_$1/$2_files/8/0_0.jpeg $ENV_LINK_SRC_DIR/files_$1/tn_$2.jpeg
 }
@@ -111,5 +120,5 @@ extract_metadata() {
 }
 
 generate_mongo_records() {
-  node /usr/sbin/generate_mongo_records.js $1 $2 $3 $4 $JOB_OUT_DIR/metadata.json
+  node /usr/sbin/generate_mongo_records.js $1 $2 $3 $4 $5 $JOB_OUT_DIR/metadata.json
 }

scripts/run-wsi-worker.sh

@@ -9,7 +9,8 @@ mkdir -p ${ENV_LINK_DST_DIR_HOST}
 # arg $1 must be KPMP ID
 # arg $2 must be input file name (no extension) directly inside $JOB_IN_DIR
 # arg $3 must be file ID
-# arg $4 is currently optional stain type (defaults to "pas")
+# arg $4 must be slide type (LM, EM, IF)
+# arg $5 is currently optional stain type (defaults to "pas")
 
 docker run \
   --env-file .env \
@@ -18,4 +19,4 @@ docker run \
   -v ${ENV_LINK_DST_DIR_HOST}:${ENV_LINK_DST_DIR} \
   -v ${ENV_JOB_IN_DIR}:/data/job/in \
   -v ${ENV_JOB_OUT_DIR}:/data/job/out \
-  ${ENV_IMAGE} $1 $2 $3 $4
+  ${ENV_IMAGE} $1 $2 $3 $4 $5