From be36e392062eadc4863d09867f3b03fcf1b02d9e Mon Sep 17 00:00:00 2001
From: Anton Schwaighofer <antonsc@microsoft.com>
Date: Mon, 21 Jun 2021 14:39:09 +0100
Subject: [PATCH] Bug fix for regression test (#496)

- The regresssion test code in #492 missed out files in AzureML, because the wrong function was called in the main workflow.
- Cleaner folder structure for regression test results
---
 .amlignore                                    |  2 +
 .../Template__Run_ML_on_AzureML.xml           |  2 +-
 CHANGELOG.md                                  |  1 +
 .../Statistics/wilcoxon_signed_rank_test.py   |  2 +-
 InnerEye/ML/baselines_util.py                 | 68 +++++++++----------
 InnerEye/ML/run_ml.py                         | 10 +--
 .../ML/visualizers/metrics_scatterplot.py     |  4 +-
 .../ML/visualizers/plot_cross_validation.py   |  2 +-
 .../{ => OUTPUT}/Train/epoch_metrics.csv      |  0
 .../{ => OUTPUT}/Val/epoch_metrics.csv        |  0
 .../Test/005/background.nii.gz                |  0
 .../Test/005/posterior_lung_l.nii.gz          |  0
 .../Test/005/uncertainty.nii.gz               |  0
 .../best_validation_epoch/Test/dataset_id.txt |  0
 .../Test/ground_truth_ids.txt                 |  0
 .../Test/image_channel_ids.txt                |  0
 .../best_validation_epoch/Test/metrics.csv    |  0
 .../Test/metrics_aggregates.csv               |  0
 .../Test/metrics_boxplot.png                  |  0
 .../Test/thumbnails/005_lung_l_slice_053.png  |  0
 .../Test/thumbnails/005_lung_r_slice_037.png  |  0
 .../thumbnails/005_spinalcord_slice_088.png   |  0
 .../Val/002/background.nii.gz                 |  0
 .../Val/002/posterior_lung_r.nii.gz           |  0
 .../best_validation_epoch/Val/metrics.csv     |  0
 .../Val/metrics_aggregates.csv                |  0
 .../Val/metrics_boxplot.png                   |  0
 .../Val/thumbnails/002_lung_l_slice_069.png   |  0
 .../Val/thumbnails/002_lung_r_slice_052.png   |  0
 .../thumbnails/002_spinalcord_slice_113.png   |  0
 .../patch_sampling/0_sampled_patches_dim0.png |  0
 .../patch_sampling/0_sampled_patches_dim1.png |  0
 .../patch_sampling/0_sampled_patches_dim2.png |  0
 .../model_inference_config.json               |  0
 .../{ => OUTPUT}/test_mse.txt                 |  0
 .../{ => OUTPUT}/Train/epoch_metrics.csv      |  0
 .../{ => OUTPUT}/Val/epoch_metrics.csv        |  0
 .../best_validation_epoch/Test/metrics.csv    |  0
 .../best_validation_epoch/Val/metrics.csv     |  0
 .../CrossValResults/Dice_Test_splits.jpg      |  3 -
 .../CrossValResults/Dice_Val_splits.jpg       |  3 -
 .../HausdorffDistance_mm_Test_splits.jpg      |  3 -
 .../HausdorffDistance_mm_Val_splits.jpg       |  3 -
 .../CrossValResults/MetricsAcrossAllRuns.csv  | 19 ------
 .../model_inference_config.json               |  2 +-
 .../{ => OUTPUT}/Train/epoch_metrics.csv      |  0
 .../{ => OUTPUT}/Val/epoch_metrics.csv        |  0
 Tests/ML/test_regression_tests.py             | 62 +++++++++--------
 docs/building_models.md                       |  8 +--
 49 files changed, 86 insertions(+), 108 deletions(-)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/Train/epoch_metrics.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/Val/epoch_metrics.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/005/background.nii.gz (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/005/posterior_lung_l.nii.gz (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/005/uncertainty.nii.gz (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/dataset_id.txt (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/ground_truth_ids.txt (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/image_channel_ids.txt (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/metrics.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/metrics_aggregates.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/metrics_boxplot.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/thumbnails/005_lung_l_slice_053.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/thumbnails/005_lung_r_slice_037.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Test/thumbnails/005_spinalcord_slice_088.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/002/background.nii.gz (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/002/posterior_lung_r.nii.gz (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/metrics.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/metrics_aggregates.csv (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/metrics_boxplot.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/thumbnails/002_lung_l_slice_069.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/thumbnails/002_lung_r_slice_052.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/best_validation_epoch/Val/thumbnails/002_spinalcord_slice_113.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/patch_sampling/0_sampled_patches_dim0.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/patch_sampling/0_sampled_patches_dim1.png (100%)
 rename RegressionTestResults/PR_BasicModel2Epochs/{ => OUTPUT}/patch_sampling/0_sampled_patches_dim2.png (100%)
 rename RegressionTestResults/PR_HelloContainer/AZUREML_OUTPUT/{ => final_model}/model_inference_config.json (100%)
 rename RegressionTestResults/PR_HelloContainer/{ => OUTPUT}/test_mse.txt (100%)
 rename RegressionTestResults/PR_Train2Nodes/{ => OUTPUT}/Train/epoch_metrics.csv (100%)
 rename RegressionTestResults/PR_Train2Nodes/{ => OUTPUT}/Val/epoch_metrics.csv (100%)
 rename RegressionTestResults/PR_Train2Nodes/{ => OUTPUT}/best_validation_epoch/Test/metrics.csv (100%)
 rename RegressionTestResults/PR_Train2Nodes/{ => OUTPUT}/best_validation_epoch/Val/metrics.csv (100%)
 delete mode 100644 RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Test_splits.jpg
 delete mode 100644 RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Val_splits.jpg
 delete mode 100644 RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Test_splits.jpg
 delete mode 100644 RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Val_splits.jpg
 delete mode 100644 RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/MetricsAcrossAllRuns.csv
 rename RegressionTestResults/PR_TrainEnsemble/{ => OUTPUT}/Train/epoch_metrics.csv (100%)
 rename RegressionTestResults/PR_TrainEnsemble/{ => OUTPUT}/Val/epoch_metrics.csv (100%)

diff --git a/.amlignore b/.amlignore
index e700f62d4..a8bd214ba 100644
--- a/.amlignore
+++ b/.amlignore
@@ -3,6 +3,8 @@
 .pytest_cache
 .mypy_cache
 __pycache__
+azure-pipelines
+.github
 datasets
 modelweights
 outputs
diff --git a/.idea/runConfigurations/Template__Run_ML_on_AzureML.xml b/.idea/runConfigurations/Template__Run_ML_on_AzureML.xml
index f8894bfff..8e58a0a7c 100644
--- a/.idea/runConfigurations/Template__Run_ML_on_AzureML.xml
+++ b/.idea/runConfigurations/Template__Run_ML_on_AzureML.xml
@@ -12,7 +12,7 @@
     <option name="ADD_CONTENT_ROOTS" value="true" />
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <option name="SCRIPT_NAME" value="InnerEye/ML/runner.py" />
-    <option name="PARAMETERS" value="--azureml=True --model=BasicModel2Epochs --train=True --tensorboard=True" />
+    <option name="PARAMETERS" value="--azureml --model=HelloContainer --train" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 54cd6aa6b..921117d48 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ created.
 jobs that run in AzureML.
 
 ### Changed
+- ([#496](https://github.com/microsoft/InnerEye-DeepLearning/pull/496)) All plots are now saved as PNG, rather than JPG.
 
 ### Fixed
 
diff --git a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py
index ca7ccc3bc..2cb043601 100644
--- a/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py
+++ b/InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py
@@ -309,7 +309,7 @@ def main() -> None:
     for line in lines:
         print(line)
     for basename, fig in plots.items():
-        fig.savefig(f"{basename}.jpg")
+        fig.savefig(f"{basename}.png")
 
 
 if __name__ == "__main__":
diff --git a/InnerEye/ML/baselines_util.py b/InnerEye/ML/baselines_util.py
index 11fc65780..1d2768826 100755
--- a/InnerEye/ML/baselines_util.py
+++ b/InnerEye/ML/baselines_util.py
@@ -29,10 +29,11 @@
 from InnerEye.ML.visualizers.metrics_scatterplot import write_to_scatterplot_directory
 from InnerEye.ML.visualizers.plot_cross_validation import convert_rows_for_comparisons, may_write_lines_to_file
 
+REGRESSION_TEST_OUTPUT_FOLDER = "OUTPUT"
 REGRESSION_TEST_AZUREML_FOLDER = "AZUREML_OUTPUT"
 REGRESSION_TEST_AZUREML_PARENT_FOLDER = "AZUREML_PARENT_OUTPUT"
 CONTENTS_MISMATCH = "Contents mismatch"
-MISSING_FILE = "Missing file"
+MISSING_FILE = "Missing"
 TEXT_FILE_SUFFIXES = [".txt", ".csv", ".json", ".html", ".md"]
 
 
@@ -222,56 +223,46 @@ def print_lines(prefix: str, lines: List[str]) -> None:
 
 def compare_folder_contents(expected_folder: Path,
                             actual_folder: Optional[Path] = None,
-                            run: Optional[Run] = None) -> None:
+                            run: Optional[Run] = None) -> List[str]:
     """
     Compares a set of files in a folder, against files in either the other folder or files stored in the given
     AzureML run. Each file that is present in the "expected" folder must be also present in the "actual" folder
     (or the AzureML run), with exactly the same contents, in the same folder structure.
     For example, if there is a file "<expected>/foo/bar/contents.txt", then there must also be a file
     "<actual>/foo/bar/contents.txt"
-    If a file is missing, or does not have the expected contents, an exception is raised.
     :param expected_folder: A folder with files that are expected to be present.
     :param actual_folder: The output folder with the actually produced files.
     :param run: An AzureML run
+    :return: A list of human readable error messages, with message and file path. If no errors are found, the list is
+    empty.
     """
-    logging.debug(f"Checking job output against expected files in folder {expected_folder}")
-    logging.debug(f"Current working directory: {Path.cwd()}")
     messages = []
-    if not expected_folder.is_dir():
-        raise ValueError(f"Folder with expected files does not exist: {expected_folder}")
     if run and is_offline_run_context(run):
         logging.warning("Skipping file comparison because the given run context is an AzureML offline run.")
-        return
+        return []
     files_in_run: List[str] = run.get_file_names() if run else []
     temp_folder = Path(tempfile.mkdtemp()) if run else None
     for file in expected_folder.rglob("*"):
         # rglob also returns folders, skip those
         if file.is_dir():
             continue
-        logging.debug(f"Checking file {file}")
         # All files stored in AzureML runs use Linux-style path
         file_relative = file.relative_to(expected_folder).as_posix()
-        if str(file_relative).startswith(REGRESSION_TEST_AZUREML_FOLDER) or \
-                str(file_relative).startswith(REGRESSION_TEST_AZUREML_PARENT_FOLDER):
-            continue
-        actual_file: Optional[Path] = None
         if actual_folder:
             actual_file = actual_folder / file_relative
-            if not actual_file.is_file():
-                actual_file = None
         elif temp_folder is not None and run is not None:
+            actual_file = temp_folder / file_relative
             if file_relative in files_in_run:
-                actual_file = temp_folder / file_relative
                 run.download_file(name=str(file_relative), output_file_path=str(actual_file))
-        message = compare_files(expected=file, actual=actual_file) if actual_file else "Missing file"
+        else:
+            raise ValueError("One of the two arguments run, actual_folder must be provided.")
+        message = compare_files(expected=file, actual=actual_file) if actual_file.exists() else MISSING_FILE
         if message:
-            logging.debug(f"Error: {message}")
             messages.append(f"{message}: {file_relative}")
+        logging.info(f"File {file_relative}: {message or 'OK'}")
     if temp_folder:
         shutil.rmtree(temp_folder)
-    if messages:
-        raise ValueError(f"Some expected files were missing or did not have the expected contents:{os.linesep}"
-                         f"{os.linesep.join(messages)}")
+    return messages
 
 
 def compare_folders_and_run_outputs(expected: Path, actual: Path) -> None:
@@ -286,17 +277,24 @@ def compare_folders_and_run_outputs(expected: Path, actual: Path) -> None:
     """
     if not expected.is_dir():
         raise ValueError(f"Folder with expected files does not exist: {expected}")
-    # First compare the normal output files that the run produces
-    compare_folder_contents(expected, actual)
-    # Compare the set of files in the magic folder with the outputs stored in the run context
-    azureml_folder = expected / REGRESSION_TEST_AZUREML_FOLDER
-    if azureml_folder.is_dir():
-        compare_folder_contents(azureml_folder, run=RUN_CONTEXT)
-    # Compare the set of files in the magic folder with the outputs stored in the run context of the parent run
-    azureml_parent_folder = expected / REGRESSION_TEST_AZUREML_PARENT_FOLDER
-    if azureml_parent_folder.is_dir():
-        if PARENT_RUN_CONTEXT is None:
-            raise ValueError(f"The set of expected test results in {expected} contains a folder "
-                             f"{REGRESSION_TEST_AZUREML_PARENT_FOLDER}, but the present run is not a cross-validation "
-                             "child run")
-        compare_folder_contents(azureml_parent_folder, run=PARENT_RUN_CONTEXT)
+    logging.debug(f"Current working directory: {Path.cwd()}")
+    messages = []
+    for (subfolder, message_prefix, actual_folder, run_to_compare) in \
+            [(REGRESSION_TEST_OUTPUT_FOLDER, "run output files", actual, None),
+             (REGRESSION_TEST_AZUREML_FOLDER, "AzureML outputs in present run", None, RUN_CONTEXT),
+             (REGRESSION_TEST_AZUREML_PARENT_FOLDER, "AzureML outputs in parent run", None, PARENT_RUN_CONTEXT)]:
+        folder = expected / subfolder
+        if folder.is_dir():
+            logging.info(f"Comparing results in {folder} against {message_prefix}:")
+            if actual_folder is None and run_to_compare is None:
+                raise ValueError(f"The set of expected test results in {expected} contains a folder "
+                                 f"{subfolder}, but there is no (parent) run to compare against.")
+            new_messages = compare_folder_contents(folder, actual_folder=actual_folder, run=run_to_compare)
+            if new_messages:
+                messages.append(f"Issues in {message_prefix}:")
+                messages.extend(new_messages)
+        else:
+            logging.info(f"Folder {subfolder} not found, skipping comparison against {message_prefix}.")
+    if messages:
+        raise ValueError(f"Some expected files were missing or did not have the expected contents:{os.linesep}"
+                         f"{os.linesep.join(messages)}")
diff --git a/InnerEye/ML/run_ml.py b/InnerEye/ML/run_ml.py
index f9a9a49d6..a46473285 100644
--- a/InnerEye/ML/run_ml.py
+++ b/InnerEye/ML/run_ml.py
@@ -34,7 +34,7 @@
     change_working_directory, get_best_epoch_results_path, is_windows, logging_section, logging_to_file, \
     print_exception, remove_file_or_directory
 from InnerEye.Common.fixed_paths import INNEREYE_PACKAGE_NAME, LOG_FILE_NAME, PYTHON_ENVIRONMENT_NAME
-from InnerEye.ML.baselines_util import compare_folder_contents
+from InnerEye.ML.baselines_util import compare_folders_and_run_outputs
 from InnerEye.ML.common import ModelExecutionMode
 from InnerEye.ML.config import SegmentationModelBase
 from InnerEye.ML.deep_learning_config import CHECKPOINT_FOLDER, DeepLearningConfig, FINAL_ENSEMBLE_MODEL_FOLDER, \
@@ -411,8 +411,8 @@ def run(self) -> None:
             # run context.
             logging.info("Comparing the current results against stored results")
             if self.is_normal_run_or_crossval_child_0():
-                compare_folder_contents(expected_folder=self.container.regression_test_folder,
-                                        actual_folder=self.container.outputs_folder)
+                compare_folders_and_run_outputs(expected=self.container.regression_test_folder,
+                                                actual=self.container.outputs_folder)
             else:
                 logging.info("Skipping because this is not cross-validation child run 0.")
 
@@ -712,9 +712,11 @@ def copy_file(source: Path, destination_file: str) -> None:
             else:
                 raise ValueError(f"Expected an absolute path to a checkpoint file, but got: {checkpoint}")
         model_folder.mkdir(parents=True, exist_ok=True)
+        # For reproducibility of the files used in regression tests, checkpoint paths need to be sorted.
+        checkpoints_sorted = sorted(relative_checkpoint_paths)
         model_inference_config = ModelInferenceConfig(model_name=self.container.model_name,
                                                       model_configs_namespace=self.config_namespace,
-                                                      checkpoint_paths=relative_checkpoint_paths)
+                                                      checkpoint_paths=checkpoints_sorted)
         # Inference configuration must live in the root folder of the registered model
         full_path_to_config = model_folder / fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME
         full_path_to_config.write_text(model_inference_config.to_json(), encoding='utf-8')  # type: ignore
diff --git a/InnerEye/ML/visualizers/metrics_scatterplot.py b/InnerEye/ML/visualizers/metrics_scatterplot.py
index a80bec21b..f9aab7f02 100644
--- a/InnerEye/ML/visualizers/metrics_scatterplot.py
+++ b/InnerEye/ML/visualizers/metrics_scatterplot.py
@@ -206,7 +206,7 @@ def to_dict(data: pd.DataFrame) -> Dict[str, Dict[str, float]]:
 
 def write_to_scatterplot_directory(root_folder: Path, plots: Dict[str, plt.Figure]) -> None:
     """
-    Writes a file root_folder/scatterplots/basename.jpg for every plot in plots with key "basename".
+    Writes a file root_folder/scatterplots/basename.png for every plot in plots with key "basename".
     :param root_folder: path to a folder
     :param plots: dictionary from plot basenames to plots (plt.Figure objects)
     """
@@ -217,7 +217,7 @@ def write_to_scatterplot_directory(root_folder: Path, plots: Dict[str, plt.Figur
     scatterplot_dir.mkdir(parents=True, exist_ok=True)
     logging.info(f"There are {len(plots)} plots to write to {scatterplot_dir}")
     for basename, fig in plots.items():
-        fig.savefig(scatterplot_dir / f"{basename}.jpg")
+        fig.savefig(scatterplot_dir / f"{basename}.png")
 
 
 def main() -> None:
diff --git a/InnerEye/ML/visualizers/plot_cross_validation.py b/InnerEye/ML/visualizers/plot_cross_validation.py
index b8fc8cac1..877004816 100644
--- a/InnerEye/ML/visualizers/plot_cross_validation.py
+++ b/InnerEye/ML/visualizers/plot_cross_validation.py
@@ -647,7 +647,7 @@ def plot_metrics(config: PlotCrossValidationConfig,
 
                 # save plot
                 suffix = f"_{sub_df_index}" if len(df_list) > 1 else ""
-                plot_dst = root / f"{metric_type}_{mode.value}_splits{suffix}.jpg"
+                plot_dst = root / f"{metric_type}_{mode.value}_splits{suffix}.png"
                 fig.savefig(plot_dst, bbox_inches='tight')
                 logging.info("Saved box-plots to: {}".format(plot_dst))
 
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/Train/epoch_metrics.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/Train/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/Train/epoch_metrics.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/Train/epoch_metrics.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/Val/epoch_metrics.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/Val/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/Val/epoch_metrics.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/Val/epoch_metrics.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/background.nii.gz b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/background.nii.gz
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/background.nii.gz
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/background.nii.gz
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/posterior_lung_l.nii.gz b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/posterior_lung_l.nii.gz
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/posterior_lung_l.nii.gz
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/posterior_lung_l.nii.gz
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/uncertainty.nii.gz b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/uncertainty.nii.gz
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/005/uncertainty.nii.gz
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/005/uncertainty.nii.gz
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/dataset_id.txt b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/dataset_id.txt
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/dataset_id.txt
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/dataset_id.txt
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/ground_truth_ids.txt b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/ground_truth_ids.txt
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/ground_truth_ids.txt
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/ground_truth_ids.txt
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/image_channel_ids.txt b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/image_channel_ids.txt
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/image_channel_ids.txt
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/image_channel_ids.txt
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics_aggregates.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics_aggregates.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics_aggregates.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics_aggregates.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics_boxplot.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics_boxplot.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/metrics_boxplot.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/metrics_boxplot.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_lung_l_slice_053.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_lung_l_slice_053.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_lung_l_slice_053.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_lung_l_slice_053.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_lung_r_slice_037.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_lung_r_slice_037.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_lung_r_slice_037.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_lung_r_slice_037.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_spinalcord_slice_088.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_spinalcord_slice_088.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Test/thumbnails/005_spinalcord_slice_088.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Test/thumbnails/005_spinalcord_slice_088.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/002/background.nii.gz b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/002/background.nii.gz
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/002/background.nii.gz
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/002/background.nii.gz
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/002/posterior_lung_r.nii.gz b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/002/posterior_lung_r.nii.gz
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/002/posterior_lung_r.nii.gz
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/002/posterior_lung_r.nii.gz
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics_aggregates.csv b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics_aggregates.csv
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics_aggregates.csv
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics_aggregates.csv
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics_boxplot.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics_boxplot.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/metrics_boxplot.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/metrics_boxplot.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_lung_l_slice_069.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_lung_l_slice_069.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_lung_l_slice_069.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_lung_l_slice_069.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_lung_r_slice_052.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_lung_r_slice_052.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_lung_r_slice_052.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_lung_r_slice_052.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_spinalcord_slice_113.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_spinalcord_slice_113.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/best_validation_epoch/Val/thumbnails/002_spinalcord_slice_113.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/best_validation_epoch/Val/thumbnails/002_spinalcord_slice_113.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim0.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim0.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim0.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim0.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim1.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim1.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim1.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim1.png
diff --git a/RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim2.png b/RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim2.png
similarity index 100%
rename from RegressionTestResults/PR_BasicModel2Epochs/patch_sampling/0_sampled_patches_dim2.png
rename to RegressionTestResults/PR_BasicModel2Epochs/OUTPUT/patch_sampling/0_sampled_patches_dim2.png
diff --git a/RegressionTestResults/PR_HelloContainer/AZUREML_OUTPUT/model_inference_config.json b/RegressionTestResults/PR_HelloContainer/AZUREML_OUTPUT/final_model/model_inference_config.json
similarity index 100%
rename from RegressionTestResults/PR_HelloContainer/AZUREML_OUTPUT/model_inference_config.json
rename to RegressionTestResults/PR_HelloContainer/AZUREML_OUTPUT/final_model/model_inference_config.json
diff --git a/RegressionTestResults/PR_HelloContainer/test_mse.txt b/RegressionTestResults/PR_HelloContainer/OUTPUT/test_mse.txt
similarity index 100%
rename from RegressionTestResults/PR_HelloContainer/test_mse.txt
rename to RegressionTestResults/PR_HelloContainer/OUTPUT/test_mse.txt
diff --git a/RegressionTestResults/PR_Train2Nodes/Train/epoch_metrics.csv b/RegressionTestResults/PR_Train2Nodes/OUTPUT/Train/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_Train2Nodes/Train/epoch_metrics.csv
rename to RegressionTestResults/PR_Train2Nodes/OUTPUT/Train/epoch_metrics.csv
diff --git a/RegressionTestResults/PR_Train2Nodes/Val/epoch_metrics.csv b/RegressionTestResults/PR_Train2Nodes/OUTPUT/Val/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_Train2Nodes/Val/epoch_metrics.csv
rename to RegressionTestResults/PR_Train2Nodes/OUTPUT/Val/epoch_metrics.csv
diff --git a/RegressionTestResults/PR_Train2Nodes/best_validation_epoch/Test/metrics.csv b/RegressionTestResults/PR_Train2Nodes/OUTPUT/best_validation_epoch/Test/metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_Train2Nodes/best_validation_epoch/Test/metrics.csv
rename to RegressionTestResults/PR_Train2Nodes/OUTPUT/best_validation_epoch/Test/metrics.csv
diff --git a/RegressionTestResults/PR_Train2Nodes/best_validation_epoch/Val/metrics.csv b/RegressionTestResults/PR_Train2Nodes/OUTPUT/best_validation_epoch/Val/metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_Train2Nodes/best_validation_epoch/Val/metrics.csv
rename to RegressionTestResults/PR_Train2Nodes/OUTPUT/best_validation_epoch/Val/metrics.csv
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Test_splits.jpg b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Test_splits.jpg
deleted file mode 100644
index 1ad9b8463..000000000
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Test_splits.jpg
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f88f75bdd30499b5f657787ead1b6f13d21a6a8a34d8380f1dc98c8447514ba8
-size 121759
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Val_splits.jpg b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Val_splits.jpg
deleted file mode 100644
index 9b95d9a1d..000000000
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/Dice_Val_splits.jpg
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:50fad029276d7db47260917c4eaddee4c1df03e328b67b22a242164b7e33ae28
-size 116685
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Test_splits.jpg b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Test_splits.jpg
deleted file mode 100644
index 513fefbc6..000000000
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Test_splits.jpg
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:689311af84739d3ceeaed08abb419b2994487491a11717d7df56ff5a55771ac6
-size 78792
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Val_splits.jpg b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Val_splits.jpg
deleted file mode 100644
index f77e1e0e4..000000000
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/HausdorffDistance_mm_Val_splits.jpg
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b081bfaea255cbdd4822b436ed22f5b10590439bed3d91a982ea714663b9b021
-size 73800
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/MetricsAcrossAllRuns.csv b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/MetricsAcrossAllRuns.csv
deleted file mode 100644
index 0d2d3993a..000000000
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/CrossValResults/MetricsAcrossAllRuns.csv
+++ /dev/null
@@ -1,19 +0,0 @@
-Patient,Structure,Dice,HausdorffDistance_mm,MeanDistance_mm,data_split,seriesId,institutionId,split,mode
-1,lung_l,0.0,inf,inf,Val,68364d72521f89f02c84043b57389e1c993723974bc8e6383b03e953c727ceb0,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-1,lung_r,0.0,inf,inf,Val,68364d72521f89f02c84043b57389e1c993723974bc8e6383b03e953c727ceb0,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-1,spinalcord,0.0,inf,inf,Val,68364d72521f89f02c84043b57389e1c993723974bc8e6383b03e953c727ceb0,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-2,lung_l,0.0,inf,inf,Val,ef497ff85bcffe2f6166fed168a84c9d7ef61d7f2d7de010ea019aebd4e337cd,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-2,lung_r,0.0,inf,inf,Val,ef497ff85bcffe2f6166fed168a84c9d7ef61d7f2d7de010ea019aebd4e337cd,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-2,spinalcord,0.0,inf,inf,Val,ef497ff85bcffe2f6166fed168a84c9d7ef61d7f2d7de010ea019aebd4e337cd,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Val
-0,lung_l,0.0,inf,inf,Val,3b9cc72ea4e2dfcfb55107e4c2c59a20d2da60c370227d6fceee93f7482b73cc,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Val
-0,lung_r,0.0,inf,inf,Val,3b9cc72ea4e2dfcfb55107e4c2c59a20d2da60c370227d6fceee93f7482b73cc,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Val
-0,spinalcord,0.0,160.22299999999998,98.037,Val,3b9cc72ea4e2dfcfb55107e4c2c59a20d2da60c370227d6fceee93f7482b73cc,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Val
-5,lung_l,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Test
-5,lung_r,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Test
-5,spinalcord,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_0,Test
-5,lung_l,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Test
-5,lung_r,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Test
-5,spinalcord,0.0,169.77,108.318,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71_1,Test
-5,lung_l,0.0,103.189,38.122,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71,Test
-5,lung_r,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71,Test
-5,spinalcord,0.0,inf,inf,Test,402ba5d42f37357f18f29af17b0846cbca4c430fea5b15a6baad5ec29dd6c9ba,b7f757fb-12e0-489e-a6da-f64895cdd229,refs_pull_492_merge:HD_344a9615-5125-4583-9d13-a234c8dc2d71,Test
diff --git a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/final_ensemble_model/model_inference_config.json b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/final_ensemble_model/model_inference_config.json
index 05440a162..9b590b5a4 100644
--- a/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/final_ensemble_model/model_inference_config.json
+++ b/RegressionTestResults/PR_TrainEnsemble/AZUREML_PARENT_OUTPUT/final_ensemble_model/model_inference_config.json
@@ -1 +1 @@
-{"model_name": "BasicModel2Epochs", "checkpoint_paths": ["checkpoints/best_checkpoint.ckpt", "checkpoints/OTHER_RUNS/1/best_checkpoint.ckpt"], "model_configs_namespace": "InnerEye.ML.configs.segmentation.BasicModel2Epochs"}
\ No newline at end of file
+{"model_name": "BasicModel2Epochs", "checkpoint_paths": ["checkpoints/OTHER_RUNS/1/best_checkpoint.ckpt", "checkpoints/best_checkpoint.ckpt"], "model_configs_namespace": "InnerEye.ML.configs.segmentation.BasicModel2Epochs"}
\ No newline at end of file
diff --git a/RegressionTestResults/PR_TrainEnsemble/Train/epoch_metrics.csv b/RegressionTestResults/PR_TrainEnsemble/OUTPUT/Train/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_TrainEnsemble/Train/epoch_metrics.csv
rename to RegressionTestResults/PR_TrainEnsemble/OUTPUT/Train/epoch_metrics.csv
diff --git a/RegressionTestResults/PR_TrainEnsemble/Val/epoch_metrics.csv b/RegressionTestResults/PR_TrainEnsemble/OUTPUT/Val/epoch_metrics.csv
similarity index 100%
rename from RegressionTestResults/PR_TrainEnsemble/Val/epoch_metrics.csv
rename to RegressionTestResults/PR_TrainEnsemble/OUTPUT/Val/epoch_metrics.csv
diff --git a/Tests/ML/test_regression_tests.py b/Tests/ML/test_regression_tests.py
index 89b963713..55db89b2d 100644
--- a/Tests/ML/test_regression_tests.py
+++ b/Tests/ML/test_regression_tests.py
@@ -3,6 +3,7 @@
 #  Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 #  ------------------------------------------------------------------------------------------
 import logging
+import uuid
 from pathlib import Path
 from unittest import mock
 
@@ -13,7 +14,7 @@
 from InnerEye.Common.output_directories import OutputFolderForTests
 from InnerEye.ML import baselines_util
 from InnerEye.ML.baselines_util import REGRESSION_TEST_AZUREML_FOLDER, REGRESSION_TEST_AZUREML_PARENT_FOLDER, \
-    compare_files, compare_folder_contents, compare_folders_and_run_outputs
+    REGRESSION_TEST_OUTPUT_FOLDER, compare_files, compare_folder_contents, compare_folders_and_run_outputs
 from InnerEye.ML.deep_learning_config import FINAL_MODEL_FOLDER
 from InnerEye.ML.run_ml import MLRunner
 from Tests.AfterTraining.test_after_training import FALLBACK_ENSEMBLE_RUN, FALLBACK_SINGLE_RUN, get_most_recent_run
@@ -36,7 +37,7 @@ def test_regression_test(test_output_dirs: OutputFolderForTests) -> None:
     """
     container = DummyContainerWithModel()
     container.local_dataset = test_output_dirs.root_dir
-    container.regression_test_folder = Path("foo")
+    container.regression_test_folder = Path(str(uuid.uuid4().hex))
     runner = MLRunner(container=container)
     runner.setup(use_mount_or_download_dataset=False)
     with pytest.raises(ValueError) as ex:
@@ -44,16 +45,6 @@ def test_regression_test(test_output_dirs: OutputFolderForTests) -> None:
     assert "Folder with expected files does not exist" in str(ex)
 
 
-def test_compare_folder_exists(test_output_dirs: OutputFolderForTests) -> None:
-    """
-    Test if the folder comparison method raises an exception on invalid input.
-    """
-    does_not_exist = test_output_dirs.root_dir / "foo"
-    with pytest.raises(ValueError) as ex:
-        compare_folder_contents(expected_folder=does_not_exist, actual_folder=test_output_dirs.root_dir)
-    assert "Folder with expected files does not exist" in str(ex)
-
-
 @pytest.mark.parametrize("file_extension", baselines_util.TEXT_FILE_SUFFIXES)
 def test_compare_files_text(test_output_dirs: OutputFolderForTests, file_extension: str) -> None:
     """
@@ -115,24 +106,39 @@ def test_compare_folder(test_output_dirs: OutputFolderForTests) -> None:
     # This file exists in both actual and expected, but has different contents, hence should create an error
     (expected / subfolder / mismatch).write_text("contents1")
     (actual / subfolder / mismatch).write_text("contents2")
-    # Create folders that hold the expected results in the AzureML run context. These should be ignored when doing
-    # the file-by-file comparison on the local output files.
-    for ignored_folder in [REGRESSION_TEST_AZUREML_FOLDER, REGRESSION_TEST_AZUREML_PARENT_FOLDER]:
-        azureml1 = expected / ignored_folder / ignored
-        azureml1.parent.mkdir()
-        azureml1.touch()
 
+    messages = compare_folder_contents(expected_folder=expected, actual_folder=actual)
+    all_messages = " ".join(messages)
+    # No issues expected
+    assert matching not in all_messages
+    assert extra not in all_messages
+    assert ignored not in all_messages
+    # Folders should be skipped in the comparison
+    assert f"{baselines_util.MISSING_FILE}: {subfolder}" not in messages
+    assert f"{baselines_util.MISSING_FILE}: {subfolder}/{missing}" in messages
+    assert f"{baselines_util.CONTENTS_MISMATCH}: {subfolder}/{mismatch}" in messages
+
+
+def test_compare_plain_outputs(test_output_dirs: OutputFolderForTests) -> None:
+    """
+    Test if we can compare that a set of files from the job outputs.
+    """
+    logging_to_stdout(log_level=logging.DEBUG)
+    expected = test_output_dirs.root_dir / REGRESSION_TEST_OUTPUT_FOLDER
+    actual = test_output_dirs.root_dir / "my_output"
+    for folder in [expected, actual]:
+        file1 = folder / "output.txt"
+        create_folder_and_write_text(file1, "Something")
+    # First comparison should pass
+    compare_folders_and_run_outputs(expected=expected, actual=actual)
+    # Now add a file to the set of expected files that does not exist in the run: comparison should now fail
+    no_such_file = "no_such_file.txt"
+    file2 = expected / no_such_file
+    create_folder_and_write_text(file2, "foo")
     with pytest.raises(ValueError) as ex:
-        compare_folder_contents(expected_folder=expected, actual_folder=actual)
+        compare_folders_and_run_outputs(expected=test_output_dirs.root_dir, actual=Path.cwd())
     message = ex.value.args[0].splitlines()
-    # No message expected
-    assert matching not in str(ex)
-    assert extra not in str(ex)
-    assert ignored not in str(ex)
-    # Folders should be skipped in the comparison
-    assert f"{baselines_util.MISSING_FILE}: {subfolder}" not in message
-    assert f"{baselines_util.MISSING_FILE}: {subfolder}/{missing}" in message
-    assert f"{baselines_util.CONTENTS_MISMATCH}: {subfolder}/{mismatch}" in message
+    assert f"{baselines_util.MISSING_FILE}: {no_such_file}" in message
 
 
 @pytest.mark.after_training_single_run
@@ -202,4 +208,4 @@ def test_compare_folder_against_parent_run(test_output_dirs: OutputFolderForTest
     # realize that the present run is a crossval child run
     with pytest.raises(ValueError) as ex:
         compare_folders_and_run_outputs(expected=test_output_dirs.root_dir, actual=Path.cwd())
-    assert "run is not a cross-validation child run" in str(ex)
+    assert "no (parent) run to compare against" in str(ex)
diff --git a/docs/building_models.md b/docs/building_models.md
index 3e2a60796..0ec3e7bb6 100755
--- a/docs/building_models.md
+++ b/docs/building_models.md
@@ -263,8 +263,8 @@ the `metrics.csv` files of the current run and the comparison run(s).
   indicates, for each structure, when the Dice scores for the second model are significantly better 
   or worse than the first. For full details, see the 
   [source code](../InnerEye/Common/Statistics/wilcoxon_signed_rank_test.py).
-  * A directory `scatterplots`, containing a `jpg` file for every pairing of the current model
-  with one of the baslines. Each one is named `AAA_vs_BBB.jpg`, where `AAA` and `BBB` are the run IDs
+  * A directory `scatterplots`, containing a `png` file for every pairing of the current model
+  with one of the baselines. Each one is named `AAA_vs_BBB.png`, where `AAA` and `BBB` are the run IDs
   of the two models. Each plot shows the Dice scores on the test set for the models.
   * For both segmentation and classification models an IPython Notebook `report.ipynb` will be generated in the
    `outputs` directory.
@@ -282,11 +282,11 @@ In addition, various scores and plots from the ensemble and from individual chil
 runs are uploaded to the parent run, in the `CrossValResults` directory. This contains:
 * Subdirectories named 0, 1, 2, ... for all the child runs including the zero'th one, as well
  as `ENSEMBLE`, containing their respective `epoch_NNN` directories.
-* Files `Dice_Test_Splits.jpg` and `Dice_Val_Splits.jpg`, containing box plots of the Dice scores
+* Files `Dice_Test_Splits.png` and `Dice_Val_Splits.png`, containing box plots of the Dice scores
   on those datasets for each structure and each (component and ensemble) model. These give a visual
   overview of the results in the `metrics.csv` files detailed above. When there are many different
   structures, several such plots are created, with a different subset of structures in each one.
-* Similarly, `HausdorffDistance_mm_Test_splits.jpg` and `HausdorffDistance_mm_Val_splits.jpg` contain
+* Similarly, `HausdorffDistance_mm_Test_splits.png` and `HausdorffDistance_mm_Val_splits.png` contain
   box plots of Hausdorff distances.
 * `MetricsAcrossAllRuns.csv` combines the data from all the `metrics.csv` files.
 * `Test_outliers.txt` and `Val_outliers.txt` highlight particular outlier scores (both Dice and