2D compatibility and true positives handling (#10)

---------

Co-authored-by: dbouget <david.bouget@sintef.no>
Co-authored-by: mathildefaanes <122807133+mathildefaanes@users.noreply.github.com>

Author: 3 people

.github/workflows/build_macos.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: macos-11
+          - os: macos-13
             TARGET: macos
             CMD_BUILD: python setup.py bdist_wheel --plat-name macosx_10_15_x86_64
 
@@ -42,7 +42,7 @@ jobs:
         run: ${{matrix.CMD_BUILD}}
 
       - name: Upload Python wheel
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Python wheel
           path: ${{github.workspace}}/dist/raidionicsval-*.whl
@@ -52,7 +52,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ macos-11, macos-12 ]
+        os: [ macos-13 ]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.os }}
 
@@ -63,7 +63,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "Python wheel"
 

.github/workflows/build_macos_arm.yml

@@ -38,7 +38,7 @@ jobs:
         run: python3 setup.py bdist_wheel --plat-name macosx_11_0_arm64
 
       - name: Upload Python wheel
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Python wheel
           path: ${{github.workspace}}/dist/raidionicsval-*.whl
@@ -58,7 +58,7 @@ jobs:
           default: ${{ matrix.python-version }}
 
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "Python wheel"
 
@@ -73,7 +73,7 @@ jobs:
         uses: actions/checkout@v1
 
       - name: k-fold cross-validation unit test
-        run: cd ${{github.workspace}}/tests && python validation_pipeline_test.py
+        run: cd ${{github.workspace}}/tests && python3 validation_pipeline_test.py
 
       - name: Segmentation study unit test
-        run: cd ${{github.workspace}}/tests && python studies_pipeline_test.py
+        run: cd ${{github.workspace}}/tests && python3 studies_pipeline_test.py

.github/workflows/build_ubuntu.yml

@@ -38,7 +38,7 @@ jobs:
         run: ${{matrix.CMD_BUILD}}
 
       - name: Upload Python wheel
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Python wheel
           path: ${{github.workspace}}/dist/raidionicsval-*.whl
@@ -48,7 +48,7 @@ jobs:
     needs: build
     strategy:
       matrix:
-        os: [ ubuntu-20.04, ubuntu-22.04 ]
+        os: [ ubuntu-20.04, ubuntu-22.04, ubuntu-24.04 ]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.os }}
 
@@ -59,7 +59,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "Python wheel"
 

.github/workflows/build_windows.yml

@@ -38,7 +38,7 @@ jobs:
         run: ${{matrix.CMD_BUILD}}
 
       - name: Upload Python wheel
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Python wheel
           path: ${{github.workspace}}/dist/raidionicsval-*.whl
@@ -59,7 +59,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Download artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: "Python wheel"
 

README.md

@@ -6,7 +6,9 @@
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/gist/dbouget/7560fe410db03e384a45ddc77bbe9a57/compute_validation_example.ipynb)
 
 The code corresponds to the Raidionics backend for running the k-fold cross-validation and metrics computation.
-The module can either be used as a Python library, as CLI, or as Docker container.
+The module can either be used as a Python library, as CLI, or as Docker container. It supports both 2D and 3D inputs,
+the only hard requirement is the expected folder structure to use as input.  
+:warning: For using custom structures, modifying the code [here](https://github.com/dbouget/validation_metrics_computation/blob/master/raidionicsval/Validation/kfold_model_validation.py#L155) is a good place to start.
 
 ## [Installation](https://github.com/dbouget/validation_metrics_computation#installation)
 

raidionicsval/Computation/dice_computation.py

@@ -27,15 +27,13 @@ def separate_dice_computation(args):
     t = np.round(args[0], 2)
     fold_number = args[1]
     gt = args[2]
-    detection_ni = args[3]
+    detection_raw = args[3]
     patient_id = args[4]
     volumes_extra = args[5]
     results = []
 
-    detection = deepcopy(detection_ni.get_fdata())
-    detection[detection < t] = 0
-    detection[detection >= t] = 1
-    detection = detection.astype('uint8')
+    detection = np.zeros(detection_raw.shape, dtype='uint8')
+    detection[detection_raw >= t] = 1
 
     # # Cleaning the too small objects that might be noise in the detection
     # if np.count_nonzero(detection) > 0:
@@ -52,20 +50,20 @@ def separate_dice_computation(args):
     if "pixelwise" in SharedResources.getInstance().validation_metric_spaces:
         pixelwise_results = __pixelwise_computation(gt, detection)
 
-    det_volume = np.round(np.count_nonzero(detection) * np.prod(detection_ni.header.get_zooms()) * 1e-3, 4)
+    det_volume = np.round(np.count_nonzero(detection) * np.prod(volumes_extra[2]) * 1e-3, 4)
 
     obj_val = InstanceSegmentationValidation(gt_image=gt, detection_image=detection)
     if "objectwise" in SharedResources.getInstance().validation_metric_spaces:
         try:
             # obj_val.set_trace_parameters(self.output_folder, fold_number, patient, t)
-            obj_val.spacing = detection_ni.header.get_zooms()
+            obj_val.spacing = volumes_extra[2]
             obj_val.run()
         except Exception as e:
             print('Issue computing instance segmentation parameters for patient {}'.format(patient_id))
             print(traceback.format_exc())
     instance_results = obj_val.instance_detection_results
 
-    results.append([fold_number, patient_id, t] + pixelwise_results + volumes_extra +
+    results.append([fold_number, patient_id, t] + pixelwise_results + volumes_extra[:-1] +
                    [det_volume] + instance_results + [len(obj_val.gt_candidates), len(obj_val.detection_candidates)])
 
     return results

raidionicsval/Studies/AbstractStudy.py

@@ -275,6 +275,8 @@ def compute_and_plot_metric_over_metric_categories(self, class_name: str, data=N
                 results_filename = os.path.join(self.input_folder, 'Validation', class_name + '_dice_scores.csv')
                 results = pd.read_csv(results_filename)
                 results.replace('inf', np.nan, inplace=True)
+                if category == 'True Positive':
+                    results = results.loc[results["True Positive"] == True]
             else:
                 results = deepcopy(data)
             total_thresholds = [np.round(x, 2) for x in list(np.unique(results['Threshold'].values))]
@@ -355,6 +357,8 @@ def compute_and_plot_categorical_metric_over_metric_categories(self, class_name:
                 results_filename = os.path.join(self.input_folder, 'Validation', class_name + '_dice_scores.csv')
                 results = pd.read_csv(results_filename)
                 results.replace('inf', np.nan, inplace=True)
+                if category == 'True Positive':
+                    results = results.loc[results["True Positive"] == True]
             else:
                 results = deepcopy(data)
             total_thresholds = [np.round(x, 2) for x in list(np.unique(results['Threshold'].values))]
@@ -446,6 +450,12 @@ def compute_fold_average_inner(self, folder, class_name, data=None, best_thresho
             unique_folds = np.unique(results['Fold'])
             nb_folds = len(unique_folds)
             metrics_per_fold = []
+            tp_volume_threshold = 0.
+            if len(SharedResources.getInstance().validation_true_positive_volume_thresholds) == 1:
+                tp_volume_threshold = SharedResources.getInstance().validation_true_positive_volume_thresholds[0]
+            else:
+                index_cl = SharedResources.getInstance().validation_class_names.find(class_name)
+                tp_volume_threshold = SharedResources.getInstance().validation_true_positive_volume_thresholds[index_cl]
 
             metric_names = list(results.columns[3:list(results.columns).index("#Det") + 1])
             if metrics is not None:
@@ -460,7 +470,8 @@ def compute_fold_average_inner(self, folder, class_name, data=None, best_thresho
             # Regarding the overlap threshold, should the patient discarded for recall be
             # used for other metrics computation?
             for f in unique_folds:
-                patientwise_metrics = compute_patientwise_fold_metrics(results, f, best_threshold, best_overlap)
+                patientwise_metrics = compute_patientwise_fold_metrics(results, f, best_threshold, best_overlap,
+                                                                       tp_volume_threshold)
                 fold_average_metrics, fold_std_metrics = compute_singe_fold_average_metrics(results, f, best_threshold,
                                                                                             best_overlap, metrics)
                 fold_metrics = []

raidionicsval/Studies/SegmentationStudy.py

@@ -50,13 +50,13 @@ def run(self):
             # Plotting the results based on the selection of categorical parameters
             for s in SharedResources.getInstance().studies_selections_categorical:
                 parsing = s.split(',')
-                metric1 = parsing[0]
-                metric2 = parsing[1]
-                if parsing[2] != '':
+                metric1 = parsing[0].strip()
+                metric2 = parsing[1].strip()
+                if parsing[2].strip() != '':
                     metric2_cutoff = [x for x in parsing[2].split('-')]
                 else:
                     metric2_cutoff = None
-                category = parsing[3]
+                category = parsing[3].strip()
                 self.compute_and_plot_categorical_metric_over_metric_categories(class_name=c, metric1=metric1,
                                                                                 metric2=metric2,
                                                                                 metric2_cutoffs=metric2_cutoff,

raidionicsval/Utils/PatientMetricsStructure.py

@@ -70,6 +70,22 @@ def extra_metrics(self) -> str:
     def class_names(self) -> List[str]:
         return self._class_names
 
+    @property
+    def ground_truth_filepaths(self) -> str:
+        return self._ground_truth_filepaths
+
+    @ground_truth_filepaths.setter
+    def ground_truth_filepaths(self, ground_truth_filepaths) -> None:
+        self._ground_truth_filepaths = ground_truth_filepaths
+
+    @property
+    def prediction_filepaths(self) -> str:
+        return self._prediction_filepaths
+
+    @prediction_filepaths.setter
+    def prediction_filepaths(self, prediction_filepaths) -> None:
+        self._prediction_filepaths = prediction_filepaths
+
     def init_from_file(self, study_folder: str):
         all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv')
 
@@ -136,7 +152,7 @@ def set_patient_filenames(self, filenames: dict) -> None:
             self._prediction_filepaths.append(filenames[c][1])
 
     def get_class_filenames(self, class_index: int) -> List[str]:
-        return [self._ground_truth_filepaths[class_index], self._prediction_filepaths[class_index]]
+        return [self._ground_truth_filepaths[class_index], self.prediction_filepaths[class_index]]
 
     def set_class_regular_metrics(self, class_name: str, results: list):
         self._class_metrics[class_name].set_results(results)

raidionicsval/Utils/io_converters.py

@@ -1,7 +1,11 @@
+import logging
 import os
 import pickle
 import pandas as pd
 import numpy as np
+import nibabel as nib
+from typing import Tuple, List
+from PIL import Image
 
 
 def get_fold_from_file(filename, fold_number):
@@ -52,3 +56,33 @@ def reload_optimal_validation_parameters(study_filename):
     optimums = study_df.iloc[-1]
 
     return optimums['Detection threshold'], optimums['Dice threshold']
+
+def open_image_file(input_filename: str) -> Tuple[np.ndarray, str, List]:
+    ext = '.' + '.'.join(input_filename.split('.')[1:])
+    input_array = None
+    input_specifics = []
+
+    if ext == ".nii" or ext == ".nii.gz":
+        input_ni = nib.load(input_filename)
+        if len(input_ni.shape) == 4:
+            input_ni = nib.four_to_three(input_ni)[0]
+        input_array = input_ni.get_fdata()[:]
+        input_specifics = [input_ni.affine, input_ni.header.get_zooms()]
+    elif ext in [".tif", ".tiff", ".png"]:
+        input_array = Image.open(input_filename)
+        input_specifics = [np.eye(4, dtype=int), [1., 1.]]
+    else:
+        logging.error("Working with an unknown file type: {}. Skipping...".format(ext))
+
+    return input_array, ext, input_specifics
+
+
+def save_image_file(output_array, output_filename: str, specifics: List = None) -> None:
+    ext = '.'.join(output_filename.split('.')[1:])
+
+    if ext == ".nii" or ext == ".nii.gz":
+        nib.save(nib.Nifti1Image(output_array, affine=specifics[0]), output_filename)
+    elif ext in [".tif", ".tiff", ".png"]:
+        Image.fromarray(output_array).save(output_filename)
+    else:
+        logging.error("Working with an unknown file type: {}. Skipping...".format(ext))