Enforce that there are no CSP violations when csp.rule is defined

src/main/java/org/jenkinsci/test/acceptance/junit/CspRule.java

@@ -2,6 +2,8 @@
 
 import com.google.inject.Inject;
 import com.google.inject.Injector;
+import java.util.List;
+import org.jenkinsci.test.acceptance.plugins.csp.ContentSecurityPolicyReport;
 import org.jenkinsci.test.acceptance.po.GlobalSecurityConfig;
 import org.jenkinsci.test.acceptance.po.Jenkins;
 import org.jenkinsci.test.acceptance.update_center.PluginSpec;
@@ -20,11 +22,9 @@ public Statement apply(final Statement base, final Description d) {
         return new Statement() {
             @Override
             public void evaluate() throws Throwable {
-                if (isEnabled()
-                        && d.getAnnotation(WithInstallWizard.class) == null
-                        && d.getTestClass().getAnnotation(WithInstallWizard.class) == null) {
-                    Jenkins jenkins = injector.getInstance(Jenkins.class);
+                Jenkins jenkins = injector.getInstance(Jenkins.class);
 
+                if (isEnabled() && !isSkipped()) {
                     PluginSpec plugin = new PluginSpec("csp");
                     jenkins.getPluginManager().installPlugins(plugin);
 
@@ -33,7 +33,28 @@ public void evaluate() throws Throwable {
                     security.disableCspReportOnly();
                     security.save();
                 }
-                base.evaluate();
+                try {
+                    base.evaluate();
+                } finally {
+                    // TODO enable for ArtifactoryPluginTest when JENKINS-74047 is resolved
+                    // TODO enable for SubversionPluginTest when JENKINS-73900 is resolved
+                    if (isEnabled()
+                            && !isSkipped()
+                            && !d.getTestClass().getName().equals("plugins.ArtifactoryPluginTest")
+                            && !d.getTestClass().getName().equals("plugins.SubversionPluginTest")) {
+                        ContentSecurityPolicyReport csp = new ContentSecurityPolicyReport(jenkins);
+                        csp.open();
+                        List<String> lines = csp.getReport();
+                        if (lines.size() > 2) {
+                            throw new AssertionError(String.join("\n", lines));
+                        }
+                    }
+                }
+            }
+
+            private boolean isSkipped() {
+                return d.getAnnotation(WithInstallWizard.class) != null
+                        || d.getTestClass().getAnnotation(WithInstallWizard.class) != null;
             }
 
             private static boolean isEnabled() {

src/main/java/org/jenkinsci/test/acceptance/plugins/csp/ContentSecurityPolicyReport.java

@@ -0,0 +1,36 @@
+package org.jenkinsci.test.acceptance.plugins.csp;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.jenkinsci.test.acceptance.po.Jenkins;
+import org.jenkinsci.test.acceptance.po.PageObject;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+
+public class ContentSecurityPolicyReport extends PageObject {
+    public ContentSecurityPolicyReport(Jenkins context) {
+        super(context, context.url("content-security-policy-reports/"));
+    }
+
+    public List<String> getReport() {
+        List<String> lines = new ArrayList<>();
+        WebElement table = find(By.className("bigtable"));
+        List<WebElement> headers = table.findElements(By.tagName("th"));
+        StringBuilder sb = new StringBuilder();
+        for (WebElement header : headers) {
+            sb.append(header.getText()).append("\t");
+        }
+        lines.add(sb.toString());
+        sb = new StringBuilder();
+        List<WebElement> rows = table.findElements(By.tagName("tr"));
+        for (WebElement row : rows) {
+            List<WebElement> cells = row.findElements(By.tagName("td"));
+            for (WebElement cell : cells) {
+                sb.append(cell.getText()).append("\t");
+            }
+            lines.add(sb.toString());
+            sb = new StringBuilder();
+        }
+        return lines;
+    }
+}

src/test/java/core/PublisherOrderTest.java

@@ -10,6 +10,7 @@
 import org.jenkinsci.test.acceptance.po.FreeStyleJob;
 import org.jenkinsci.test.acceptance.po.JUnitPublisher;
 import org.junit.Test;
+import org.openqa.selenium.By;
 
 @WithPlugins("junit")
 public class PublisherOrderTest extends AbstractJUnitTest {
@@ -59,5 +60,13 @@ public void testUnordered() {
         archiver.includes("another.txt");
         JUnitPublisher junit = upstream.addPublisher(JUnitPublisher.class);
         fingerprint.targets.set("yetanother");
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 }

src/test/java/plugins/ArtifactoryPluginTest.java

@@ -32,6 +32,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.jvnet.hudson.test.Issue;
+import org.openqa.selenium.By;
 
 /**
  * Checks the successful integration of Artifactory plugin.
@@ -65,6 +66,14 @@ public void check_config_is_persisted() {
                 hasContent(
                         Pattern.compile(
                                 "Error occurred while requesting version information: Connection( to http://localhost:4898)* refused")));
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 
     @Test

src/test/java/plugins/JobDslPluginTest.java

@@ -552,6 +552,10 @@ public void should_use_script_security() {
 
         // Build should succeed because script is approved now
         seedJob.scheduleBuild().shouldSucceed();
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin(ADMIN);
     }
 
     /**
@@ -622,6 +626,10 @@ public void should_use_script_approval() {
 
         // Build should succeed because script is approved now
         seedJob.scheduleBuild().shouldSucceed();
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin(ADMIN);
     }
 
     /**
@@ -651,6 +659,10 @@ public void should_use_grooy_sandbox_whitelisted_content() {
         // Build should succeed because the script runs in Groovy sandbox
         // and only Job DSL methods are used.
         seedJob.scheduleBuild().shouldSucceed();
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin(ADMIN);
     }
 
     /**
@@ -694,6 +706,10 @@ public void should_use_grooy_sandbox_no_whitelisted_content() {
 
         // Build should succeed because the not whitelisted content was approved.
         seedJob.scheduleBuild().shouldSucceed();
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin(ADMIN);
     }
 
     /**
@@ -725,6 +741,10 @@ public void should_run_grooy_sandbox_as_particular_user() {
         jenkins.login().doLogin(USER);
         // Build should succeed because now a particular user is specified
         seedJob.scheduleBuild().shouldSucceed();
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin(ADMIN);
     }
 
     /**

src/test/java/plugins/MailerPluginTest.java

@@ -15,6 +15,7 @@
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.openqa.selenium.By;
 
 @WithPlugins("mailer")
 @Category(DockerTest.class)
@@ -41,6 +42,14 @@ public void send_test_mail() throws IOException {
                 Pattern.compile("Test email #1"),
                 "admin@example.com",
                 Pattern.compile("This is test email #1 sent from Jenkins"));
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 
     @Test

src/test/java/plugins/MatrixAuthPluginTest.java

@@ -132,5 +132,9 @@ public void projectMatrixAuth() throws Exception {
         jenkins.login().doLogin("bob");
 
         assertTrue(j.open().getTitle().contains(j.name));
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        jenkins.login().doLogin("alice");
     }
 }

src/test/java/plugins/PlainCredentialsBindingTest.java

@@ -46,6 +46,7 @@
 import org.jenkinsci.test.acceptance.po.FreeStyleJob;
 import org.jenkinsci.test.acceptance.po.ShellBuildStep;
 import org.junit.Test;
+import org.openqa.selenium.By;
 
 /**
  * Tests the plain-credentials and credentials-binding plugins together.
@@ -69,11 +70,27 @@ public void globalSecretTextCredentialTest() throws URISyntaxException {
     @Test
     public void systemSecretFileCredentialTest() throws URISyntaxException {
         createAndUseCredential(SYSTEM_SCOPE, FileCredentials.class);
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 
     @Test
     public void systemSecretTextCredentialTest() throws URISyntaxException {
         createAndUseCredential(SYSTEM_SCOPE, StringCredentials.class);
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 
     @Test

src/test/java/plugins/ScriptSecurityPluginTest.java

@@ -114,6 +114,10 @@ public void scriptNeedsApproval() throws Exception {
             sa.find(job.name).approve();
         }
         shouldSucceed(job); // Script approved
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        login(ADMIN);
     }
 
     @Test
@@ -127,6 +131,10 @@ public void signatureNeedsApproval() throws Exception {
             sa.findSignature("getProperties").approve();
         }
         shouldSucceed(job); // Script approved
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        login(ADMIN);
     }
 
     @Test
@@ -140,6 +148,10 @@ public void pipelineNeedsApproval() throws Exception {
             sa.find(job.name).approve();
         }
         shouldSucceed(job); // Script approved
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        login(ADMIN);
     }
 
     @Test
@@ -153,5 +165,9 @@ public void pipelineSignatureNeedsApproval() throws Exception {
             sa.findSignature("getProperty").approve();
         }
         shouldSucceed(job); // Script approved
+
+        // Switch back to an admin user so that CspRule can check for violations.
+        jenkins.logout();
+        login(ADMIN);
     }
 }

src/test/java/plugins/SshSlavesPluginTest.java

@@ -50,6 +50,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.jvnet.hudson.test.Issue;
+import org.openqa.selenium.By;
 import org.openqa.selenium.NoSuchElementException;
 
 @WithPlugins({"ssh-slaves", "credentials", "ssh-credentials"})
@@ -146,6 +147,14 @@ public void newSlaveWithExistingCredential() throws Exception {
         l.host.set("127.0.0.1");
 
         l.credentialsId.select(String.format("%s (%s)", username, description));
+
+        /*
+         * Navigate back to the dashboard first to dismiss the alert so that CspRule can check for violations (see
+         * FormValidationTest).
+         */
+        jenkins.runThenConfirmAlert(() -> driver.findElement(By.xpath("//ol[@id=\"breadcrumbs\"]/li[1]/a"))
+                .click());
+        sleep(1000);
     }
 
     private void verifyValueForCredential(CredentialsPage cp, Control element, String expected) {