Merge pull request #3712 from chenba/3282-thumbnails

Generate, upload, and display thumbnail of shots.
mozilla-services · Nov 27, 2017 · 6bafd7d · 6bafd7d
2 parents 61cd398 + c0496ab
commit 6bafd7d
Show file tree

Hide file tree

Showing 10 changed files with 314 additions and 125 deletions.
diff --git a/Makefile b/Makefile
@@ -88,7 +88,7 @@ build/%.html: %.html
 	cp $< $@
 
 .PHONY: addon
-addon: npm set_backend set_sentry addon/webextension/manifest.json addon/install.rdf addon_locales addon/webextension/build/shot.js addon/webextension/build/inlineSelectionCss.js addon/webextension/build/raven.js addon/webextension/build/onboardingCss.js addon/webextension/build/onboardingHtml.js addon/webextension/build/buildSettings.js
+addon: npm set_backend set_sentry addon/webextension/manifest.json addon/install.rdf addon_locales addon/webextension/build/shot.js addon/webextension/build/thumbnailGenerator.js addon/webextension/build/inlineSelectionCss.js addon/webextension/build/raven.js addon/webextension/build/onboardingCss.js addon/webextension/build/onboardingHtml.js addon/webextension/build/buildSettings.js
 
 $(VENV): bin/require.pip
 	virtualenv -p python2.7 $(VENV)
@@ -135,6 +135,10 @@ addon/webextension/build/shot.js: shared/shot.js
 	@mkdir -p $(@D)
 	./bin/build-scripts/modularize shot $< > $@
 
+addon/webextension/build/thumbnailGenerator.js: shared/thumbnailGenerator.js
+	@mkdir -p $(@D)
+	./bin/build-scripts/modularize thumbnailGenerator $< > $@
+
 addon/webextension/build/inlineSelectionCss.js: build/server/static/css/inline-selection.css
 	@mkdir -p $(@D)
 	./bin/build-scripts/css_to_js.py inlineSelectionCss $< > $@

diff --git a/addon/webextension/background/startBackground.js b/addon/webextension/background/startBackground.js
@@ -22,6 +22,7 @@ this.startBackground = (function() {
     "background/senderror.js",
     "build/raven.js",
     "build/shot.js",
+    "build/thumbnailGenerator.js",
     "background/analytics.js",
     "background/deviceInfo.js",
     "background/takeshot.js",

diff --git a/addon/webextension/background/takeshot.js b/addon/webextension/background/takeshot.js
@@ -1,4 +1,4 @@
-/* globals communication, shot, main, auth, catcher, analytics, buildSettings, blobConverters */
+/* globals communication, shot, main, auth, catcher, analytics, buildSettings, blobConverters, thumbnailGenerator */
 
 "use strict";
 
@@ -13,6 +13,7 @@ this.takeshot = (function() {
     shot.favicon = sender.tab.favIconUrl;
     let capturePromise = Promise.resolve();
     let openedTab;
+    let thumbnailBlob;
     if (!shot.clipNames().length) {
       // canvas.drawWindow isn't available, so we fall back to captureVisibleTab
       capturePromise = screenshotPage(selectedPos, scroll).then((dataUrl) => {
@@ -33,8 +34,9 @@ this.takeshot = (function() {
     }
     let convertBlobPromise = Promise.resolve();
     if (buildSettings.uploadBinary && !imageBlob) {
-      imageBlob = blobConverters.dataUrlToBlob(shot.getClip(shot.clipNames()[0]).image.url);
-      shot.getClip(shot.clipNames()[0]).image.url = "";
+      let clipImage = shot.getClip(shot.clipNames()[0]).image;
+      imageBlob = blobConverters.dataUrlToBlob(clipImage.url);
+      clipImage.url = "";
     } else if (!buildSettings.uploadBinary && imageBlob) {
       convertBlobPromise = blobConverters.blobToDataUrl(imageBlob).then((dataUrl) => {
         shot.getClip(shot.clipNames()[0]).image.url = dataUrl;
@@ -53,12 +55,24 @@ this.takeshot = (function() {
     }
     return catcher.watchPromise(capturePromise.then(() => {
       return convertBlobPromise;
+    }).then(() => {
+      if (buildSettings.uploadBinary) {
+        let blobToUrlPromise = blobConverters.blobToDataUrl(imageBlob);
+        return thumbnailGenerator.createThumbnailBlobFromPromise(shot, blobToUrlPromise);
+      }
+      return thumbnailGenerator.createThumbnailUrl(shot);
+    }).then((thumbnailImage) => {
+      if (buildSettings.uploadBinary) {
+        thumbnailBlob = thumbnailImage;
+      } else {
+        shot.thumbnail = thumbnailImage;
+      }
     }).then(() => {
       return browser.tabs.create({url: shot.creatingUrl})
     }).then((tab) => {
       openedTab = tab;
       sendEvent('internal', 'open-shot-tab');
-      return uploadShot(shot, imageBlob);
+      return uploadShot(shot, imageBlob, thumbnailBlob);
     }).then(() => {
       return browser.tabs.update(openedTab.id, {url: shot.viewUrl}).then(
         null,
@@ -131,29 +145,43 @@ this.takeshot = (function() {
   }
 
   /** Creates a multipart TypedArray, given {name: value} fields
-      and {name: blob} files
+      and a files array in the format of
+      [{fieldName: "NAME", filename: "NAME.png", blob: fileBlob}, {...}, ...]
 
       Returns {body, "content-type"}
       */
-  function createMultipart(fields, fileField, fileFilename, blob) {
+  function createMultipart(fields, files) {
     let boundary = "---------------------------ScreenshotBoundary" + Date.now();
-    return blobConverters.blobToArray(blob).then((blobAsBuffer) => {
-      let body = [];
-      for (let name in fields) {
-        body.push("--" + boundary);
-        body.push(`Content-Disposition: form-data; name="${name}"`);
-        body.push("");
-        body.push(fields[name]);
-      }
+    let body = [];
+    for (let name in fields) {
       body.push("--" + boundary);
-      body.push(`Content-Disposition: form-data; name="${fileField}"; filename="${fileFilename}"`);
-      body.push(`Content-Type: ${blob.type}`);
-      body.push("");
+      body.push(`Content-Disposition: form-data; name="${name}"`);
       body.push("");
-      body = body.join("\r\n");
-      let enc = new TextEncoder("utf-8");
-      body = enc.encode(body);
-      body = concatBuffers(body.buffer, blobAsBuffer);
+      body.push(fields[name]);
+    }
+    body.push("");
+    body = body.join("\r\n");
+    let enc = new TextEncoder("utf-8");
+    body = enc.encode(body).buffer;
+
+    let blobToArrayPromises = files.map(f => {
+      return blobConverters.blobToArray(f.blob);
+    });
+
+    return Promise.all(blobToArrayPromises).then(buffers => {
+      for (let i = 0; i < buffers.length; i++) {
+        let filePart = [];
+        filePart.push("--" + boundary);
+        filePart.push(`Content-Disposition: form-data; name="${files[i].fieldName}"; filename="${files[i].filename}"`);
+        filePart.push(`Content-Type: ${files[i].blob.type}`);
+        filePart.push("");
+        filePart.push("");
+        filePart = filePart.join("\r\n");
+        filePart = concatBuffers(enc.encode(filePart).buffer, buffers[i]);
+        body = concatBuffers(body, filePart);
+        body = concatBuffers(body, enc.encode("\r\n").buffer);
+      }
+
       let tail = `\r\n--${boundary}--`;
       tail = enc.encode(tail);
       body = concatBuffers(body, tail.buffer);
@@ -164,14 +192,18 @@ this.takeshot = (function() {
     });
   }
 
-  function uploadShot(shot, blob) {
+  function uploadShot(shot, blob, thumbnail) {
     let headers;
     return auth.authHeaders().then((_headers) => {
       headers = _headers;
       if (blob) {
+        let files = [ {fieldName: "blob", filename: "screenshot.png", blob} ];
+        if (thumbnail) {
+          files.push({fieldName: "thumbnail", filename: "thumbnail.png", blob: thumbnail});
+        }
         return createMultipart(
           {shot: JSON.stringify(shot.asJson())},
-          "blob", "screenshot.png", blob
+          files
         );
       }
       return {

diff --git a/docs/testing-the-api.md b/docs/testing-the-api.md
@@ -82,7 +82,7 @@ Authenticated.  Creates or updates a shot.  Takes a JSON body.  Looks like:
     "width": 1127,
     "height": 2086
   },
-  "fullScreenThumbnail": "data:...",
+  "thumbnail": "data:...",
   "clips": {
     "cplocgk9m1nc": {
       "createdDate": 1468983771992,

diff --git a/server/src/pages/shot/controller.js b/server/src/pages/shot/controller.js
@@ -3,6 +3,7 @@
 const sendEvent = require("../../browser-send-event.js");
 const page = require("./page").page;
 const { AbstractShot } = require("../../../shared/shot");
+const { createThumbnailUrl } = require("../../../shared/thumbnailGenerator");
 const { shotGaFieldForValue } = require("../../ab-tests.js");
 
 // This represents the model we are rendering:
@@ -119,33 +120,45 @@ exports.deleteShot = function(shot) {
 };
 
 exports.saveEdit = function(shot, shotUrl) {
-  var url = model.backend + "/api/save-edit";
-  var body = JSON.stringify({
+  let url = model.backend + "/api/save-edit";
+  let payload = {
     shotId: shot.id,
     _csrf: model.csrfToken,
     url: shotUrl
-  });
-  var req = new Request(url, {
-    method: 'POST',
-    credentials: 'include',
-    headers: new Headers({
-      'content-type': 'application/json'
-    }),
-    body
-  });
-  return fetch(req).then((resp) => {
-    if (!resp.ok) {
-      var errorMessage = "Error saving edited shot";
+  };
+
+  let postWith = body => {
+    let req = new Request(url, {
+      method: 'POST',
+      credentials: 'include',
+      headers: new Headers({
+        'content-type': 'application/json'
+      }),
+      body
+    });
+    return fetch(req).then((resp) => {
+      if (!resp.ok) {
+        let errorMessage = "Error saving edited shot";
+        window.alert(errorMessage);
+        window.Raven.captureException(new Error(`Error calling /api/save-edit: ${req.status} ${req.statusText}`));
+      } else {
+        location.reload();
+      }
+    }).catch((error) => {
+      let errorMessage = "Connection error";
       window.alert(errorMessage);
-      window.Raven.captureException(new Error(`Error calling /api/save-edit: ${req.status} ${req.statusText}`));
-    } else {
-      location.reload();
-    }
-  }).catch((error) => {
-    var errorMessage = "Connection error";
-    window.alert(errorMessage);
-    window.Raven.captureException(error);
-    throw error;
+      window.Raven.captureException(error);
+      throw error;
+    });
+  }
+
+  shot.getClip(shot.clipNames()[0]).image.url = shotUrl;
+
+  createThumbnailUrl(shot).then(thumbnail => {
+    payload.thumbnail = thumbnail;
+    return postWith(JSON.stringify(payload));
+  }).catch(() => {
+    return postWith(JSON.stringify(payload));
   });
 }
 

diff --git a/server/src/pages/shotindex/view.js b/server/src/pages/shotindex/view.js
@@ -282,12 +282,12 @@ class Card extends React.Component {
       // Some corrupted shot, we'll have to ignore it
       return null;
     }
-    if (clip && clip.image && clip.image.url) {
+    if (shot.thumbnail) {
+      imageUrl = shot.thumbnail;
+    } else if (clip && clip.image && clip.image.url) {
       imageUrl = clip.image.url;
     } else if (shot.images.length) {
       imageUrl = shot.images[0].url;
-    } else if (shot.fullScreenThumbnail) {
-      imageUrl = shot.fullScreenThumbnail;
     } else {
       imageUrl = defaultImageUrl;
     }