Add environment variable to force regression test harness to generate…

… expected results. (ubccr#848) The regression test harness used to automatically generate expected results if the tests did not match. This then marked the test as skipped. This is only desired behaiour if the developer wants to generate test data. In normal CI we want the test to fail if the actual does not match the expected. Also add an extra echo to the runtests.sh script to mitigate a bug in the shippable console folding. Also switch the order of the tests run so the test generation will function correctly (the algorithm relies on the public user being run first).
jtpalmer · Mar 13, 2019 · 79ee8f6 · 79ee8f6
1 parent 2b7770a
commit 79ee8f6
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php b/open_xdmod/modules/xdmod/regression_tests/lib/Controllers/UsageExplorerTest.php
@@ -69,7 +69,7 @@ public function testCsvExport($testName, $input, $expectedFile, $userRole)
             }
             $csvdata = preg_replace(self::$replaceRegex, self::$replacements, $csvdata);
 
-            if(!empty($expectedFile)){
+            if (getenv('REG_TEST_FORCE_GENERATION') !== '1') {
                 $expected = file_get_contents($expectedFile);
                 $expected = preg_replace(self::$replaceRegex, self::$replacements, $expected);
                 if($expected === $csvdata){
@@ -85,12 +85,12 @@ public function testCsvExport($testName, $input, $expectedFile, $userRole)
                     self::$messages[] = "$fullTestName IS ONLY ==";
                     return;
                 }
-                elseif(substr($expectedFile, -13) !== 'reference.csv'){
-                    throw new \PHPUnit_Framework_ExpectationFailedException(
-                        count($failures)." assertions failed:\n\t".implode("\n\t", $failures)
-                    );
-                }
+
+                throw new \PHPUnit_Framework_ExpectationFailedException(
+                    count($failures)." assertions failed:\n\t".implode("\n\t", $failures)
+                );
             }
+            // else running in test artifact generation mode
 
             $endpoint = parse_url(self::$helper->getSiteUrl());
             $outputDir = self::$baseDir .

diff --git a/open_xdmod/modules/xdmod/regression_tests/runtests.sh b/open_xdmod/modules/xdmod/regression_tests/runtests.sh
@@ -41,11 +41,11 @@ if [ "$REG_TEST_ALL" == "1" ]; then
     REG_TEST_USER_ROLE=cd $phpunit $CD lib/Controllers/UsageExplorerTest.php
     REG_TEST_USER_ROLE=cs $phpunit $CS lib/Controllers/UsageExplorerTest.php
 
+    $phpunit $PUB lib/Controllers/UsageExplorerCloudTest.php
     REG_TEST_USER_ROLE=usr $phpunit $REGUSER lib/Controllers/UsageExplorerCloudTest.php
     REG_TEST_USER_ROLE=pi $phpunit $PI lib/Controllers/UsageExplorerCloudTest.php
     REG_TEST_USER_ROLE=cd $phpunit $CD lib/Controllers/UsageExplorerCloudTest.php
     REG_TEST_USER_ROLE=cs $phpunit $CS lib/Controllers/UsageExplorerCloudTest.php
-    $phpunit $PUB lib/Controllers/UsageExplorerCloudTest.php
 else
     REG_TEST_USER_ROLE=usr $phpunit $REGUSER lib/Controllers/UsageExplorerTest.php & usrpid=$!
     REG_TEST_USER_ROLE=pi $phpunit $PI lib/Controllers/UsageExplorerTest.php & pipid=$!
@@ -68,5 +68,6 @@ else
             EXIT_STATUS=1
         fi
     done
+    echo "Parallel Tests Finished."
     exit $EXIT_STATUS
 fi