Fix #2695, improve Selenium tests

For CircleCI, start server before starting Selenium tests Make sure the pref for system-disabled is True before installing (to workaround #2712), and False before running tests Make channel configurable via $FIREFOX_CHANNEL Allow the tester to keep the window open with $NO_CLOSE Create a test that clicks the Screenshots button, skips onboarding, clicks Save Visible, and confirms a tab opens with a shot URL Make driver instantiation, which includes installing the add-on, async and blocking on add-on installation
mozilla-services · Apr 20, 2017 · 54aa574 · 54aa574
1 parent d44f34a
commit 54aa574
Show file tree

Hide file tree

Showing 3 changed files with 192 additions and 22 deletions.
diff --git a/circle.yml b/circle.yml
@@ -23,11 +23,12 @@ compile:
 test:
   override:
     - firefox -v
-    # addon tests
-    - npm test
-    # server tests
+    # Start the server before all tests:
     - node -e 'require("babel-polyfill"); require("./build/server/server");':
         background: true
+    # addon tests
+    - FIREFOX_CHANNEL=NIGHTLY npm test
+    # server tests
     - ./bin/test-request put '{}'
     - ./bin/load_test_exercise.py http://localhost:10080
 

diff --git a/package.json b/package.json
@@ -79,7 +79,8 @@
     "postlint": "nsp check -o summary",
     "nsp": "nsp check",
     "posttest": "npm run lint",
-    "test": "make zip && mocha test/",
+    "test": "make bootstrap_zip && mocha test/",
+    "test:selenium": "make bootstrap_zip && mocha test/test.js",
     "make_versions_exact": "node bin/build-scripts/make_versions_exact.js",
     "update_outdated": "david update && npm run make_versions_exact && npm update && npm outdated"
   }

diff --git a/test/test.js b/test/test.js
@@ -1,24 +1,42 @@
 /* globals describe, it, before, after */
+/* Environmental variables that help control this test:
+
+FIREFOX_CHANNEL = empty (default RELEASE)
+                  NIGHTLY
+                  AURORA (often Developer Edition)
+                  BETA
+                  RELEASE
+
+NO_CLOSE = if not empty then when the test is finished, the browser will not be closed
+
+*/
 
 const assert = require("assert");
 const firefox = require("selenium-webdriver/firefox");
 const webdriver = require("selenium-webdriver");
-// const FxRunnerUtils = require("fx-runner/lib/utils");
-// const Fs = require("fs-promise");
+const { By, until } = webdriver;
 const path = require("path");
 
 const SHOOTER_BUTTON_ID = "screenshots_mozilla_org-browser-action";
+const SLIDE_IFRAME_ID = "firefox-screenshots-onboarding-iframe";
+const PRESELECTION_IFRAME_ID = "firefox-screenshots-preselection-iframe";
+const backend = "http://localhost:10080";
 
 function addAddonToDriver(driver, location) {
+  driver.setContext(firefox.Context.CHROME);
   return driver.executeAsyncScript(`
 const FileUtils = Components.utils.import('resource://gre/modules/FileUtils.jsm').FileUtils;
 const { console } = Components.utils.import("resource://gre/modules/Console.jsm", {});
 const { AddonManager } = Components.utils.import("resource://gre/modules/AddonManager.jsm", {});
+const { Services } = Components.utils.import("resource://gre/modules/Services.jsm");
 const callback = arguments[arguments.length - 1];
+const { prefs } = Services;
+
+prefs.setBoolPref("extensions.screenshots.system-disabled", true);
 
 class AddonListener {
   onInstallEnded(install, addon) {
-    callback([addon.id, 0]);
+    callback([addon.version, 0]);
   }
 
   onInstallFailed(install) {
@@ -27,7 +45,7 @@ class AddonListener {
 
   onInstalled(addon) {
     AddonManager.removeAddonListener(this);
-    callback([addon.id, 0]);
+    callback([addon.version, 0]);
   }
 }
 
@@ -36,18 +54,34 @@ AddonManager.installTemporaryAddon(new FileUtils.File(arguments[0]))
   .catch((error) => {
     callback([null, error.toString()]);
   });
-
 `, location).then(([result, err]) => {
     if (!result && err) {
       throw new Error(`Failed to install add-on: ${err}`);
     }
+    console.info("    Installed add-on version:", result);
+  }).then(() => {
+    return driver.executeAsyncScript(`
+const { Services } = Components.utils.import("resource://gre/modules/Services.jsm");
+const { prefs } = Services;
+prefs.setBoolPref("extensions.screenshots.system-disabled", false);
+const callback = arguments[arguments.length - 1];
+callback();
+`);
+  }).then(() => {
+    driver.setContext(firefox.Context.CONTENT);
     return driver;
   });
 }
 
 function getDriver() {
   const profile = new firefox.Profile();
-  const options = new firefox.Options();
+  let channel = process.env.FIREFOX_CHANNEL || "RELEASE";
+  if (!(channel in firefox.Channel)) {
+    throw new Error(`Unknown channel: "${channel}"`);
+  }
+  const options = new firefox.Options()
+    .setBinary(firefox.Channel[channel]);
+  // FIXME: should assert somehow that we have Firefox 54+
   options.setProfile(profile);
 
   const builder = new webdriver.Builder()
@@ -56,35 +90,169 @@ function getDriver() {
 
   const driver = builder.build();
 
-  driver.setContext(firefox.Context.CHROME);
-  let fileLocation = path.join(process.cwd(), "build", "screenshots.zip");
-  addAddonToDriver(driver, fileLocation);
-  return driver;
+  let fileLocation = path.join(process.cwd(), "build", "screenshots-bootstrap.zip");
+  return addAddonToDriver(driver, fileLocation);
 }
 
-function getElementById(driver, id) {
+function getChromeElement(driver, selector) {
   driver.setContext(firefox.Context.CHROME);
   return driver.wait(
-    webdriver.until.elementLocated(
-      webdriver.By.id(id)), 1000);
+    webdriver.until.elementLocated(selector), 1000);
+}
+
+/** Calls finallyCallback() after the promise completes, successfully or not,
+    returning the resolved or rejected promise as normal */
+function promiseFinally(promise, finallyCallback) {
+  return promise.then((result) => {
+    finallyCallback();
+    return result;
+  }, (error) => {
+    finallyCallback();
+    throw error;
+  });
+}
+
+/** This will start Screenshots, and return a promise that
+    will be true if the onboarding slides are active, or false
+    if not.  An error will be returned if it does not start
+    successfully. */
+function startScreenshots(driver) {
+  return promiseFinally(
+    getChromeElement(driver, By.id(SHOOTER_BUTTON_ID)).then((button) => {
+      return button.click();
+    }),
+    () => {
+      driver.setContext(firefox.Context.CONTENT);
+    }
+  ).then(() => {
+    return driver.wait(() => {
+      return Promise.all([
+        driver.findElements(By.id(SLIDE_IFRAME_ID)),
+        driver.findElements(By.id(PRESELECTION_IFRAME_ID))
+      ]).then((results) => {
+        return results[0].length || results[1].length;
+      });
+    });
+  }).then(() => {
+    return driver.findElements(By.id(SLIDE_IFRAME_ID));
+  }).then((elements) => {
+    return !!elements.length;
+  });
+}
+
+function focusIframe(driver, iframeId) {
+  return driver.switchTo().defaultContent().then(() => {
+    return driver.findElement(By.id(iframeId))
+  }).then((iframe) => {
+    return driver.switchTo().frame(iframe);
+  });
+}
+
+function skipOnboarding(driver) {
+  return focusIframe(driver, SLIDE_IFRAME_ID).then(() => {
+    return driver.findElement(By.id("skip"));
+  }).then((skipLink) => {
+    return skipLink.click();
+  }).then(() => {
+    return driver.switchTo().defaultContent();
+  }).then(() => {
+    return driver.wait(until.elementLocated(By.id(PRESELECTION_IFRAME_ID)));
+  });
+}
+
+/** Waits when ready, calls creator to create the shot, waits for the resulting
+    promise to complete, then watches for the
+    new tab to open and load, and returns the shot URL.  The new tab
+    will be switched to by the driver */
+function expectCreatedShot(driver, creator) {
+  // We keep track of how many tabs we start with, so we can detect when
+  // a new tab is added (which signals that the saving worked)
+  let startingTabCount;
+  return driver.getAllWindowHandles().then((tabs) => {
+    startingTabCount = tabs.length;
+    return creator();
+  }).then(() => {
+    return driver.wait(() => {
+      return driver.getAllWindowHandles().then((tabs) => {
+        // On CircleCI there is consistently one weird tab with the id "22"
+        // It's not a normal tab, so we ignore it:
+        tabs = tabs.filter((t) => t != "22");
+        return tabs.length > startingTabCount;
+      });
+    });
+  }).then(() => {
+    return driver.getAllWindowHandles();
+  }).then((tabs) => {
+    tabs = tabs.filter((t) => t != "22");
+    if (tabs.length < startingTabCount) {
+      throw new Error("New tab did not open");
+    }
+    return driver.switchTo().window(tabs[tabs.length - 1]);
+  }).then(() => {
+    return driver.wait(() => {
+      return driver.getCurrentUrl().then((url) => {
+        return url != "about:blank" && !url.includes("/creating/")
+      });
+    });
+  }).then(() => {
+    return driver.getCurrentUrl();
+  });
 }
 
 describe("Test Screenshots", function() {
-  this.timeout(10000);
+  this.timeout(120000);
   let driver;
 
   before(function() {
-    driver = getDriver();
+    return getDriver().then((aDriver) => {
+      driver = aDriver;
+    });
   });
 
   after(function() {
-    return driver.quit();
+    if (!process.env.NO_CLOSE) {
+      return driver.quit();
+    }
+    console.info("Note: leaving browser open");
   });
 
   it("should find the add-on button", function() {
     this.timeout(15000);
-    return getElementById(driver, SHOOTER_BUTTON_ID)
+    return promiseFinally(
+      getChromeElement(driver, By.id(SHOOTER_BUTTON_ID))
       .then((button) => button.getAttribute("label"))
-      .then((label) => assert.equal(label, "Take a Screenshot"));
+      .then((label) => {
+        assert.equal(label, "Take a Screenshot");
+      }),
+      () => {
+        driver.setContext(firefox.Context.CONTENT);
+      });
   });
+
+  it("should take a shot", function() {
+    return driver.get(backend).then(() => {
+      return startScreenshots(driver);
+    }).then((onboarding) => {
+      if (!onboarding) {
+        throw new Error("Expected to get onboarding");
+      }
+      return skipOnboarding(driver);
+    }).then(() => {
+      return focusIframe(driver, PRESELECTION_IFRAME_ID);
+    }).then(() => {
+      return driver.wait(
+        until.elementLocated(By.css(".visible")));
+    }).then((visibleButton) => {
+      return expectCreatedShot(driver, () => {
+        visibleButton.click();
+      });
+    }).then((shotUrl) => {
+      assert(shotUrl.startsWith(backend), `Got url ${shotUrl} that doesn't start with ${backend}`);
+      let restUrl = shotUrl.substr(backend.length);
+      if (!/^\/[^/]+\/localhost$/.test(restUrl)) {
+        throw new Error(`Unexpected URL: ${shotUrl}`);
+      }
+    });
+  });
+
 });