Merge pull request #982 from OpenFn/feature/943-kobo-api

kobotoolbox: implement automatic pagination and change API signature

.changeset/wise-queens-decide.md

@@ -0,0 +1,15 @@
+---
+'@openfn/language-kobotoolbox': major
+---
+
+Update all functions in the main API
+
+## Migration guide
+
+- callbacks have removed from all functions. You can use `.then()` or `fn()` to
+  access the data returned by a function, ie, `getForms().then(state => state)`
+- `getForms(params, callback)` is now `getForms()`, and will download download
+  assets of type survey. Use `http.get('assets/')` to retrieve other assets (and
+  add query parameters)
+- `getSubmissions(params, callback)` is now `getSubmissions(formId, { query })`.
+- `getDeploymentInfo(params, callback)` is now `getDeploymentInfo(formId)`

packages/kobotoolbox/ast.json

@@ -2,12 +2,9 @@
   "operations": [
     {
       "name": "getForms",
-      "params": [
-        "options",
-        "callback"
-      ],
+      "params": [],
       "docs": {
-        "description": "Make a request to get the list of forms",
+        "description": "Make a request to fetch all survey forms accessible to the user's API token. The url is `/assets/?asset_type=survey`.",
         "tags": [
           {
             "title": "public",
@@ -16,35 +13,13 @@
           },
           {
             "title": "example",
-            "description": "getForms({}, state => {\n   console.log(state.data);\n   return state;\n});"
+            "description": "getForms();"
           },
           {
             "title": "function",
             "description": null,
             "name": null
           },
-          {
-            "title": "param",
-            "description": "Optional headers and query for the request",
-            "type": {
-              "type": "OptionalType",
-              "expression": {
-                "type": "NameExpression",
-                "name": "RequestOptions"
-              }
-            },
-            "name": "options",
-            "default": "{}"
-          },
-          {
-            "title": "param",
-            "description": "(Optional) Callback function to execute after fetching form list",
-            "type": {
-              "type": "NameExpression",
-              "name": "function"
-            },
-            "name": "callback"
-          },
           {
             "title": "returns",
             "description": null,
@@ -60,15 +35,21 @@
     {
       "name": "getSubmissions",
       "params": [
-        "params",
-        "callback"
+        "formId",
+        "options"
       ],
       "docs": {
         "description": "Get submissions for a specific form",
         "tags": [
           {
             "title": "example",
-            "description": "getSubmissions({formId: 'aXecHjmbATuF6iGFmvBLBX'}, state => {\n  console.log(state.data);\n  return state;\n});"
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX');",
+            "caption": "Get all submissions for a specific form"
+          },
+          {
+            "title": "example",
+            "description": "getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: \"2022-06-12T21:54:20\" } } });",
+            "caption": "Get form submissions with a query"
           },
           {
             "title": "function",
@@ -82,21 +63,25 @@
           },
           {
             "title": "param",
-            "description": "Form Id and data to make the fetch or filter",
+            "description": "Form Id to get the specific submissions",
             "type": {
               "type": "NameExpression",
-              "name": "object"
+              "name": "string"
             },
-            "name": "params"
+            "name": "formId"
           },
           {
             "title": "param",
-            "description": "(Optional) Callback function to execute after fetching form submissions",
+            "description": "Optional query params for the request",
             "type": {
-              "type": "NameExpression",
-              "name": "function"
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "object"
+              }
             },
-            "name": "callback"
+            "name": "options",
+            "default": "{}"
           },
           {
             "title": "returns",
@@ -113,15 +98,14 @@
     {
       "name": "getDeploymentInfo",
       "params": [
-        "params",
-        "callback"
+        "formId"
       ],
       "docs": {
         "description": "Get deployment information for a specific form",
         "tags": [
           {
             "title": "example",
-            "description": "getDeploymentInfo({formId: 'aXecHjmbATuF6iGFmvBLBX'}, state => {\n  console.log(state.data);\n  return state;\n});"
+            "description": "getDeploymentInfo('aXecHjmbATuF6iGFmvBLBX');"
           },
           {
             "title": "function",
@@ -135,21 +119,12 @@
           },
           {
             "title": "param",
-            "description": "Form Id and data to make the fetch or filter",
-            "type": {
-              "type": "NameExpression",
-              "name": "object"
-            },
-            "name": "params"
-          },
-          {
-            "title": "param",
-            "description": "(Optional) Callback function to execute after fetching form deployment information",
+            "description": "Form Id to get the deployment information",
             "type": {
               "type": "NameExpression",
-              "name": "function"
+              "name": "string"
             },
-            "name": "callback"
+            "name": "formId"
           },
           {
             "title": "returns",

packages/kobotoolbox/package.json

@@ -33,7 +33,6 @@
     "chai": "^3.4.0",
     "deep-eql": "^0.1.3",
     "esno": "^0.16.3",
-    "nock": "^12.0.3",
     "rimraf": "^3.0.2"
   },
   "type": "module",

packages/kobotoolbox/src/Adaptor.js

@@ -3,6 +3,7 @@ import { expandReferences } from '@openfn/language-common/util';
 
 import * as util from './Utils';
 
+
 /**
  * Options object
  * @typedef {Object} RequestOptions
@@ -38,81 +39,84 @@ export function execute(...operations) {
 }
 
 /**
- * Make a request to get the list of forms
+ * Make a request to fetch all survey forms accessible to the user's API token. Calls `/api/v2/assets/?asset_type=survey`.
  * @public
  * @example
- * getForms({}, state => {
- *    console.log(state.data);
- *    return state;
- * });
+ * getForms();
  * @function
- * @param {RequestOptions} [options={}] - Optional headers and query for the request
- * @param {function} callback - (Optional) Callback function to execute after fetching form list
+ * @state data - an array of form objects
  * @returns {Operation}
  */
-export function getForms(options = {}, callback) {
+export function getForms() {
   return async state => {
-    const [resolvedOptions] = expandReferences(state, options);
+    const url = `/assets/?asset_type=survey`;
 
-    const url = `/assets/?format=json`;
+    const response = await util.request(state, 'GET', url, {});
 
-    const response = await util.request(state, 'GET', url, resolvedOptions);
-    console.log('✓', response.body.count, 'forms fetched.');
-    return util.prepareNextState(state, response, callback);
+    console.log('✓', response.body.results.length, 'forms fetched.');
+    return util.prepareNextState(state, response);
   };
 }
 
 /**
- * Get submissions for a specific form
- * @example
- * getSubmissions({formId: 'aXecHjmbATuF6iGFmvBLBX'}, state => {
- *   console.log(state.data);
- *   return state;
- * });
+ * Get submissions for a specific form. Calls `/api/v2/assets/<id>/data/`.
+ * @example <caption>Get all submissions for a specific form</caption>
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX');
+ * @example <caption>Get form submissions with a query</caption>
+ * getSubmissions('aXecHjmbATuF6iGFmvBLBX', { query: { _submission_time:{ $gte: "2022-06-12T21:54:20" } } });
  * @function
  * @public
- * @param {object} params - Form Id and data to make the fetch or filter
- * @param {function} callback - (Optional) Callback function to execute after fetching form submissions
+ * @param {string} formId - Form Id to get the specific submissions
+ * @param {object} [options={}] - Optional query params for the request
+ * @state data - an array of submission objects
  * @returns {Operation}
  */
-export function getSubmissions(params, callback) {
+export function getSubmissions(formId, options = {}) {
   return async state => {
-    const [resolvedParams] = expandReferences(state, params);
-
-    const { formId } = resolvedParams;
-
-    const url = `/assets/${formId}/data/?format=json`;
-
-    const response = await util.request(state, 'GET', url, resolvedParams);
-    console.log('✓', response.body.count, 'forms fetched.');
-    return util.prepareNextState(state, response, callback);
+    const [resolvedFormId, resolvedOptions] = expandReferences(
+      state,
+      formId,
+      options
+    );
+
+    const url = `/assets/${resolvedFormId}/data/`;
+    const query = {};
+    if (resolvedOptions.query) {
+      if (typeof resolvedOptions.query == 'string') {
+        query.query = resolvedOptions.query;
+      } else {
+        query.query = JSON.stringify(resolvedOptions.query);
+      }
+    }
+
+    const response = await util.request(state, 'GET', url, {
+      paginate: true,
+      query,
+    });
+    console.log('✓', response.results.length, 'submissions fetched.');
+    return util.prepareNextState(state, response);
   };
 }
 
 /**
- * Get deployment information for a specific form
+ * Get deployment information for a specific form. Calls `/api/v2/assets/<id>/deployment/`.
  * @example
- * getDeploymentInfo({formId: 'aXecHjmbATuF6iGFmvBLBX'}, state => {
- *   console.log(state.data);
- *   return state;
- * });
+ * getDeploymentInfo('aXecHjmbATuF6iGFmvBLBX');
  * @function
  * @public
- * @param {object} params - Form Id and data to make the fetch or filter
- * @param {function} callback - (Optional) Callback function to execute after fetching form deployment information
+ * @param {string} formId - Form Id to get the deployment information
+ * @state data - an object containing deployment information
  * @returns {Operation}
  */
-export function getDeploymentInfo(params, callback) {
+export function getDeploymentInfo(formId) {
   return async state => {
-    const [resolvedParams] = expandReferences(state, params);
-
-    const { formId } = resolvedParams;
+    const [resolvedFormId] = expandReferences(state, formId);
 
-    const url = `/assets/${formId}/deployment/?format=json`;
+    const url = `/assets/${resolvedFormId}/deployment/`;
 
-    const response = await util.request(state, 'GET', url, resolvedParams);
+    const response = await util.request(state, 'GET', url, {});
     console.log('✓', 'deployment information fetched.');
-    return util.prepareNextState(state, response, callback);
+    return util.prepareNextState(state, response);
   };
 }
 
@@ -126,7 +130,7 @@ export {
   fields,
   fn,
   fnIf,
-  http, 
+  http,
   group,
   lastReferenceValue,
   merge,

packages/kobotoolbox/src/Utils.js

@@ -5,22 +5,30 @@ import {
   logResponse,
 } from '@openfn/language-common/util';
 
-export const prepareNextState = (state, response, callback = s => s) => {
+export const prepareNextState = (state, response) => {
   const { body, ...responseWithoutBody } = response;
-  const nextState = {
+
+  return {
     ...composeNextState(state, response.body),
     response: responseWithoutBody,
   };
-  return callback(nextState);
 };
 
 export async function request(state, method, path, opts) {
   const { baseURL, apiVersion, username, password } = state.configuration;
 
-  const { data = {}, query = {}, headers = {}, parseAs = 'json' } = opts;
+  const {
+    data = {},
+    query = {},
+    headers = {},
+    parseAs = 'json',
+    paginate = false,
+  } = opts;
 
   const authHeaders = makeBasicAuthHeader(username, password);
 
+  let start, limit;
+
   const options = {
     body: data,
     headers: {
@@ -36,5 +44,36 @@ export async function request(state, method, path, opts) {
     baseUrl: `${baseURL}/api/${apiVersion}`,
   };
 
-  return commonRequest(method, path, options).then(logResponse);
+  if (paginate) {
+    const results = [];
+
+    do {
+      const response = await commonRequest(method, path, options).then(
+        logResponse
+      );
+      results.push(...response.body.results);
+      if (response.body.next) {
+        const nextUrl = new URL(response.body.next);
+
+        const startDigit =
+          nextUrl.searchParams.get('start') !== null
+            ? nextUrl.searchParams.get('start')
+            : 0;
+
+        start = Number(startDigit);
+        limit = nextUrl.searchParams.get('limit');
+
+        options.query = {
+          ...options.query,
+          start,
+          limit,
+        };
+      } else {
+        break;
+      }
+    } while (true);
+    return { results };
+  } else {
+    return commonRequest(method, path, options).then(logResponse);
+  }
 }