scheduler: upload scheduled query metadata (#504)

* remove setting of default refreshInterval from frontend

* add backend support for uploading metadata on create/update of queries

* avoid throwing on null cronInterval in mapCronToRefresh

* use mapCronToRefresh to set default refresh interval

* update jest tests to account for refreshInterval not being set by frontend

* add metadata specific error message

app/actions/sessions.js

@@ -166,7 +166,6 @@ export function createScheduledQuery(connectionId, payload = {}) {
         fid: payload.fid,
         filename: payload.filename,
         name: payload.name,
-        refreshInterval: payload.refreshInterval,
         query: payload.query,
         cronInterval: payload.cronInterval,
         connectionId
@@ -189,7 +188,6 @@ export function updateScheduledQuery(connectionId, payload = {}) {
       fid: payload.fid,
       filename: payload.filename,
       name: payload.name,
-      refreshInterval: payload.refreshInterval,
       query: payload.query,
       cronInterval: payload.cronInterval,
       connectionId

app/components/Settings/scheduler/create-modal.jsx

@@ -12,12 +12,7 @@ import TimedMessage from './timed-message.jsx';
 import CronPicker from '../cron-picker/cron-picker.jsx';
 import SQL from './sql.jsx';
 
-import {
-    getHighlightMode,
-    DEFAULT_REFRESH_INTERVAL,
-    WAITING_MESSAGE,
-    SAVE_WARNING
-} from '../../../constants/constants.js';
+import {getHighlightMode, WAITING_MESSAGE, SAVE_WARNING} from '../../../constants/constants.js';
 
 import './create-modal.css';
 
@@ -109,7 +104,6 @@ class CreateModal extends Component {
         this.props
             .onSubmit({
                 query: this.state.code,
-                refreshInterval: DEFAULT_REFRESH_INTERVAL,
                 filename: generateFilename(),
                 cronInterval: this.state.interval,
                 name: this.state.name ? this.state.name.trim() : ''

app/components/Settings/scheduler/preview-modal.jsx

@@ -13,12 +13,7 @@ import CronPicker from '../cron-picker/cron-picker.jsx';
 import {Row, Column} from '../../layout.jsx';
 import SQL from './sql.jsx';
 import {plotlyUrl} from '../../../utils/utils.js';
-import {
-    getHighlightMode,
-    DEFAULT_REFRESH_INTERVAL,
-    WAITING_MESSAGE,
-    SAVE_WARNING
-} from '../../../constants/constants.js';
+import {getHighlightMode, WAITING_MESSAGE, SAVE_WARNING} from '../../../constants/constants.js';
 import {getInitialCronMode} from '../cron-picker/cron-helpers.js';
 
 const NO_OP = () => {};
@@ -85,7 +80,7 @@ export class PreviewModal extends Component {
 
     onSubmit() {
         if (this.state.editing) {
-            const {connectionId, fid, requestor, uids, refreshInterval} = this.props.query;
+            const {connectionId, fid, requestor, uids} = this.props.query;
             const {code: query, cronInterval} = this.state;
             const name = this.state.name ? this.state.name.trim() : '';
 
@@ -98,8 +93,7 @@ export class PreviewModal extends Component {
                     uids,
                     query,
                     name,
-                    cronInterval,
-                    refreshInterval: refreshInterval || DEFAULT_REFRESH_INTERVAL
+                    cronInterval
                 })
                 .then(() => {
                     this.setState({

app/components/Settings/scheduler/request-error.jsx

@@ -50,6 +50,17 @@ function FormattedMessage(props) {
         );
     }
 
+    if (content.startsWith('MetadataError')) {
+        return (
+            <details style={outlineStyle}>
+                <summary style={outlineStyle}>
+                    An unexpected error occurred while uploading query metadata. Please try again now.
+                </summary>
+                <pre>{capitalize(content.replace('MetadataError: ', ''))}</pre>
+            </details>
+        );
+    }
+
     return content.slice(0, 100);
 }
 

app/constants/constants.js

@@ -2,12 +2,6 @@
 /* eslint-disable no-multi-str */
 import {concat} from 'ramda';
 
-/**
- * Default to '1 week' for `refreshInterval` for backwards compatability. For
- * older versions this will prevent issues such as `setInterval(runQuery, NaN)`
- */
-export const DEFAULT_REFRESH_INTERVAL = 7 * 24 * 60 * 60;
-
 export const DIALECTS = {
     MYSQL: 'mysql',
     MARIADB: 'mariadb',

backend/persistent/QueryScheduler.js

@@ -3,7 +3,7 @@ import * as scheduler from 'node-schedule';
 
 import {getConnectionById} from './Connections.js';
 import * as Connections from './datastores/Datastores.js';
-import { mapRefreshToCron } from '../utils/cronUtils.js';
+import { mapRefreshToCron, mapCronToRefresh } from '../utils/cronUtils.js';
 import Logger from '../logger';
 import {
     getQuery,
@@ -19,6 +19,7 @@ import {
     getCurrentUser,
     getGridMeta,
     newGrid,
+    patchGrid,
     updateGrid
 } from './plotly-api.js';
 
@@ -97,7 +98,7 @@ class QueryScheduler {
             requestor,
             fid,
             uids,
-            refreshInterval,
+            refreshInterval: refreshInterval || mapCronToRefresh(cronInterval),
             cronInterval,
             query,
             connectionId
@@ -142,9 +143,11 @@ class QueryScheduler {
         Object.keys(this.queryJobs).forEach(this.clearQuery);
     }
 
-    queryAndCreateGrid(filename, query, connectionId, requestor) {
+    queryAndCreateGrid(filename, query, connectionId, requestor, cronInterval, refreshInterval) {
         const {username, apiKey, accessToken} = getCredentials(requestor);
+        const formattedRefresh = refreshInterval || mapCronToRefresh(cronInterval);
         let startTime;
+        let createdJson;
 
         // Check if the user even exists
         if (!username || !(apiKey || accessToken)) {
@@ -204,16 +207,38 @@ class QueryScheduler {
                 Logger.log(`Error ${res.status} while creating a grid`, 2);
             }
 
-            return res.json().then((json) => {
-                Logger.log(`Grid ${json.file.fid} has been updated.`, 2);
-                return json;
+            return res.json();
+        }).then((json) => {
+            createdJson = json;
+            startTime = process.hrtime();
+
+            return patchGrid(
+                createdJson.file.fid,
+                requestor,
+                {
+                    metadata: {
+                        query,
+                        connectionId,
+                        connectorUrl: `https://${getSetting('CONNECTOR_HTTPS_DOMAIN')}:${getSetting('PORT_HTTPS')}`
+                    },
+                    refresh_interval: formattedRefresh
+                }
+            ).catch((e) => {
+                /*
+                * Warning: The front end looks for "MetadataError" in this error message. Don't change it!
+                */
+                throw new Error(`MetadataError: ${e.message}`);
             });
+        }).then(() => {
+            Logger.log(`Request to Plotly for creating a grid took ${process.hrtime(startTime)[0]} seconds`, 2);
+            Logger.log(`Grid ${createdJson.file.fid} has been updated.`, 2);
+            return createdJson;
         });
-
     }
 
-    queryAndUpdateGrid(fid, uids, query, connectionId, requestor) {
+    queryAndUpdateGrid(fid, uids, query, connectionId, requestor, cronInterval, refreshInterval) {
         const requestedDBConnections = getConnectionById(connectionId);
+        const formattedRefresh = refreshInterval || mapCronToRefresh(cronInterval);
         let startTime = process.hrtime();
 
         /*
@@ -337,13 +362,32 @@ class QueryScheduler {
 
             }
 
+            startTime = process.hrtime();
+
+            return patchGrid(
+                fid,
+                requestor,
+                {
+                    metadata: {
+                        query,
+                        connectionId,
+                        connectorUrl: `https://${getSetting('CONNECTOR_HTTPS_DOMAIN')}:${getSetting('PORT_HTTPS')}`
+                    },
+                    refresh_interval: formattedRefresh
+                }
+            ).catch((e) => {
+                /*
+                * Warning: The front end looks for "MetadataError" in this error message. Don't change it!
+                */
+                throw new Error(`MetadataError: ${e.message}`);
+            });
+        }).then((res) => {
+            Logger.log(`Request to Plotly for creating a grid took ${process.hrtime(startTime)[0]} seconds`, 2);
             return res.json().then(() => {
                 Logger.log(`Grid ${fid} has been updated.`, 2);
             });
         });
-
     }
-
 }
 
 export default QueryScheduler;

backend/persistent/plotly-api.js

@@ -152,6 +152,18 @@ export function deleteGrid(fid, requestor) {
     });
 }
 
+export function patchGrid(fid, requestor, body) {
+    const {username, apiKey, accessToken} = getCredentials(requestor);
+
+    return plotlyAPIRequest(`grids/${fid}`, {
+        method: 'PATCH',
+        username,
+        apiKey,
+        accessToken,
+        body
+    });
+}
+
 export function updateGrid(rows, fid, uids, requestor) {
     const {username, apiKey, accessToken} = getCredentials(requestor);
 

backend/routes.js

@@ -664,13 +664,22 @@ export default class Servers {
          * the endpoint `/queries/:fid`
          */
         server.post('/queries', function postQueriesHandler(req, res, next) {
-            const {filename, fid, uids, query, connectionId, requestor} = req.params;
+            const {
+                filename,
+                fid,
+                uids,
+                query,
+                connectionId,
+                requestor,
+                cronInterval = null,
+                refreshInterval = null
+            } = req.params;
 
             // If a filename has been provided,
             // make the query and create a new grid
             if (filename) {
                 return that.queryScheduler.queryAndCreateGrid(
-                    filename, query, connectionId, requestor
+                    filename, query, connectionId, requestor, cronInterval, refreshInterval
                 )
                 .then((newGridResponse) => {
                     const queryObject = {
@@ -691,7 +700,7 @@ export default class Servers {
             if (fid) {
                 return checkWritePermissions(fid, requestor).then(function () {
                     return that.queryScheduler.queryAndUpdateGrid(
-                        fid, uids, query, connectionId, requestor
+                        fid, uids, query, connectionId, requestor, cronInterval, refreshInterval
                     );
                 })
                 .then(() => {

backend/utils/cronUtils.js

@@ -22,6 +22,26 @@ export function mapRefreshToCron (refreshInterval) {
     return `${now.getMinutes()} ${now.getHours()} * * ${now.getDay()}`;
 }
 
+export function mapCronToRefresh (cronInterval) {
+    const DEFAULT_INTERVAL = 60 * 60 * 24 * 7; // default to weekly
+
+    if (!cronInterval) {
+        return DEFAULT_INTERVAL;
+    }
+
+    if (cronInterval === '* * * * *') {
+        return 60;
+    } else if (cronInterval === '*/5 * * * *') {
+        return 60 * 5;
+    } else if (cronInterval.match(/\S+? \* \* \* \*/)) {
+        return 60 * 60;
+    } else if (cronInterval.match(/\S+? \S+? \* \* \*/)) {
+        return 60 * 60 * 24;
+    }
+
+    return DEFAULT_INTERVAL;
+}
+
 function computeMinutes (now) {
     let currMinute = now.getMinutes() % 5; // start at 5 min offset
     const minutes = [];

test/app/components/Settings/scheduler/create-modal.test.jsx

@@ -18,7 +18,6 @@ global.document.createRange = function() {
 
 const wait = () => new Promise(resolve => setTimeout(resolve, 0));
 
-const {DEFAULT_REFRESH_INTERVAL} = require('../../../../../app/constants/constants');
 const CodeMirror = require('react-codemirror2').Controlled;
 const CreateModal = require('../../../../../app/components/Settings/scheduler/create-modal.jsx');
 const ErrorComponent = require('../../../../../app/components/error.jsx');
@@ -93,15 +92,13 @@ describe('Create Modal Test', () => {
                 // Once filename input is supported, this value should be: 'filename',
                 filename: expect.any(String),
                 query: 'SELECT * FROM foods',
-                refreshInterval: DEFAULT_REFRESH_INTERVAL,
                 cronInterval
             })
         );
         expect(onSubmit).toHaveBeenCalledWith({
             // Once filename input is supported, this value should be: 'filename',
             filename: expect.any(String),
             query: 'SELECT * FROM foods',
-            refreshInterval: DEFAULT_REFRESH_INTERVAL,
             cronInterval,
             name
         });

test/app/components/Settings/scheduler/preview-modal.jsx

@@ -17,7 +17,6 @@ global.document.createRange = function() {
 };
 
 const PreviewModal = require('../../../../../app/components/Settings/scheduler/preview-modal.jsx').default;
-const {DEFAULT_REFRESH_INTERVAL} = require('../../../../../app/constants/constants');
 
 describe('Preview Modal Tests', () => {
     beforeAll(() => {
@@ -155,9 +154,7 @@ describe('Preview Modal Tests', () => {
                 fid: 'fid:1',
                 requestor: 'user',
                 query: 'SELECT * FROM table',
-                cronInterval: '* * * * *',
-                // This is necessary to prevent older versions from breaking
-                refreshInterval: DEFAULT_REFRESH_INTERVAL
+                cronInterval: '* * * * *'
             })
         );
     });