diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..f47b5ce --- /dev/null +++ b/.eslintignore @@ -0,0 +1,4 @@ +node_modules +public +build +dist \ No newline at end of file diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000..7b6e9e3 --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,13 @@ +env: + es2021: true + node: true +extends: + - eslint:recommended + - plugin:@typescript-eslint/recommended +parser: '@typescript-eslint/parser' +parserOptions: + ecmaVersion: latest + sourceType: module +plugins: + - '@typescript-eslint' +rules: {} diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..7e15468 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +npm run lint-staged diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..f47b5ce --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +node_modules +public +build +dist \ No newline at end of file diff --git a/.prettierrc.yml b/.prettierrc.yml new file mode 100644 index 0000000..1f506bf --- /dev/null +++ b/.prettierrc.yml @@ -0,0 +1,9 @@ +printWidth: 100 +tabWidth: 4 +singleQuote: false +semi: true +jsxSingleQuote: false +quoteProps: as-needed +trailingComma: none +bracketSpacing: true +bracketSameLine: false diff --git a/package.json b/package.json index 0d50f2b..aa74973 100644 --- a/package.json +++ b/package.json @@ -1,91 +1,105 @@ { - "name": "mint-ensemble-manager", - "version": "4.1.0", - "description": "", - "scripts": { - "start": "ts-node -r node-localstorage/register ./dist/server.js", - "start:watch": "npx webpack -w", - "build": "npx webpack" - }, - "nodemonConfig": { - "ignore": [ - "**/*.test.ts", - "**/*.spec.ts", - ".git", - "node_modules" - ], - "watch": [ - "src" - ], - "exec": "npm run-script build && npm start", - "ext": "ts" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@apollo/client": "^3.7.1", - "@mintproject/modelcatalog_client": "^8.0.0", - "@types/fs-extra": "^8.0.1", - "@types/request": "^2.48.3", - "@types/request-promise-native": "^1.0.17", - "@types/webpack": "^5.28.0", - "@types/webpack-dev-middleware": "^3.7.2", - "apollo-boost": "^0.4.9", - "react-dom": "^18.2.0", - "bull": "^4.10.2", - "@bull-board/express": "^4.6.4", - "@bull-board/api": "^4.6.4", - "cors": "^2.8.5", - "cross-fetch": "^3.0.5", - "dockerode": "^3.2.1", - "dockerode-utils": "0.0.7", - "express": "^4.17.1", - "express-openapi": "^5.0.0", - "express-openapi-validator": "^5.0.0", - "fetch": "^1.1.0", - "form-data": "^2.3.3", - "fs-extra": "^8.1.0", - "graphql": "^15.3.0", - "graphql-tag": "^2.11.0", - "js-yaml": "^3.13.1", - "node-localstorage": "^2.2.1", - "nodemon-webpack-plugin": "^4.3.2", - "request": "^2.88.0", - "request-promise-native": "^1.0.8", - "graphql-ws": "^5.11.2", - "swagger-ui-express": "^4.1.3", - "ts-loader": "^9.4.1", - "tsconfig-paths": "^4.1.0", - "tslib": "^2.4.1", - "webpack-dev-middleware": "^3.7.2", - "yauzl": "^2.10.0" - }, - "devDependencies": { - "@types/body-parser": "^1.17.1", - "@types/bull": "^3.14.0", - "@types/cookie-parser": "^1.4.2", - "@types/cors": "^2.8.6", - "@types/dockerode": "^2.5.34", - "@types/express": "^4.17.2", - "@types/express-openapi": "^1.9.0", - "@types/js-yaml": "^3.12.1", - "@types/node": "^12.12.7", - "@types/swagger-ui-express": "^4.1.1", - "@types/yauzl": "^2.9.1", - "@typescript-eslint/eslint-plugin": "^5.44.0", - "@typescript-eslint/parser": "^5.44.0", - "body-parser": "^1.19.0", - "cookie-parser": "^1.4.4", - "copy-webpack-plugin": "^6.1.0", - "eslint": "^6.8.0", - "eslint-plugin-import": "^2.18.2", - "nodemon": "^2.0.20", - "ts-md5": "^1.3.1", - "ts-node": "^10.9.1", - "typescript": "^4.9.3", - "webpack": "^5.75.0", - "webpack-cli": "^5.0.0", - "webpack-node-externals": "^3.0.0" - } + "name": "mint-ensemble-manager", + "version": "4.1.0", + "description": "", + "scripts": { + "start": "ts-node -r node-localstorage/register ./dist/server.js", + "start:watch": "npx webpack -w", + "build": "npx webpack", + "test": "echo \"Error: no test specified\"", + "lint-staged": "lint-staged", + "eslint:fix": "eslint --quiet --fix", + "prettier:fix": "prettier --write --ignore-unknown", + "prepare": "husky && husky install" + }, + "nodemonConfig": { + "ignore": [ + "**/*.test.ts", + "**/*.spec.ts", + ".git", + "node_modules" + ], + "watch": [ + "src" + ], + "exec": "npm run-script build && npm start", + "ext": "ts" + }, + "lint-staged": { + "src/*.ts": [ + "prettier --write" + ] + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@apollo/client": "^3.7.1", + "@bull-board/api": "^4.6.4", + "@bull-board/express": "^4.6.4", + "@mintproject/modelcatalog_client": "^8.0.0", + "@types/fs-extra": "^8.0.1", + "@types/request": "^2.48.3", + "@types/request-promise-native": "^1.0.17", + "@types/webpack": "^5.28.0", + "@types/webpack-dev-middleware": "^3.7.2", + "apollo-boost": "^0.4.9", + "bull": "^4.10.2", + "cors": "^2.8.5", + "cross-fetch": "^3.0.5", + "dockerode": "^3.2.1", + "dockerode-utils": "0.0.7", + "express": "^4.17.1", + "express-openapi": "^5.0.0", + "express-openapi-validator": "^5.0.0", + "fetch": "^1.1.0", + "form-data": "^2.3.3", + "fs-extra": "^8.1.0", + "graphql": "^15.3.0", + "graphql-tag": "^2.11.0", + "graphql-ws": "^5.11.2", + "js-yaml": "^3.13.1", + "node-localstorage": "^2.2.1", + "nodemon-webpack-plugin": "^4.3.2", + "react-dom": "^18.2.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.8", + "swagger-ui-express": "^4.1.3", + "ts-loader": "^9.4.1", + "tsconfig-paths": "^4.1.0", + "tslib": "^2.4.1", + "webpack-dev-middleware": "^3.7.2", + "yauzl": "^2.10.0" + }, + "devDependencies": { + "@types/body-parser": "^1.17.1", + "@types/bull": "^3.14.0", + "@types/cookie-parser": "^1.4.2", + "@types/cors": "^2.8.6", + "@types/dockerode": "^2.5.34", + "@types/express": "^4.17.2", + "@types/express-openapi": "^1.9.0", + "@types/js-yaml": "^3.12.1", + "@types/node": "^12.12.7", + "@types/swagger-ui-express": "^4.1.1", + "@types/yauzl": "^2.9.1", + "@typescript-eslint/eslint-plugin": "^6.21.0", + "@typescript-eslint/parser": "^6.21.0", + "body-parser": "^1.19.0", + "cookie-parser": "^1.4.4", + "copy-webpack-plugin": "^6.1.0", + "eslint": "^8.0.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-import": "^2.18.2", + "husky": "^8.0.0", + "lint-staged": "^15.2.2", + "nodemon": "^2.0.20", + "prettier": "^3.2.5", + "ts-md5": "^1.3.1", + "ts-node": "^10.9.1", + "typescript": "^4.9.3", + "webpack": "^5.75.0", + "webpack-cli": "^5.0.0", + "webpack-node-externals": "^3.0.0" + } } diff --git a/src/api/api-doc.ts b/src/api/api-doc.ts index 9af87d6..8bdb2b7 100644 --- a/src/api/api-doc.ts +++ b/src/api/api-doc.ts @@ -1,137 +1,138 @@ module.exports = { - openapi: '3.0.0', + openapi: "3.0.0", info: { - title: 'Ensemble Manager API', - version: '3.0.0' + title: "Ensemble Manager API", + version: "3.0.0" }, - servers: [ - { url: 'http://localhost:3000/v1' }, - { url: 'https://ensemble.mint.isi.edu/v1' } - ], - components:{ - schemas: { - ModelThread: { - description: '', - required:[ - 'thread_id' - ], - type: 'object', - properties: { - thread_id: { - description: 'The Modeling thread id', - type: 'string' + servers: [{ url: "http://localhost:3000/v1" }, { url: "https://ensemble.mint.isi.edu/v1" }], + components: { + schemas: { + ModelThread: { + description: "", + required: ["thread_id"], + type: "object", + properties: { + thread_id: { + description: "The Modeling thread id", + type: "string" + }, + model_id: { + description: "The Model id (optional. Set to null to run all)", + type: "string" + } + } }, - model_id: { - description: 'The Model id (optional. Set to null to run all)', - type: 'string' - } - } - }, - NewModelThread: { - description: '', - required:[ - 'problem_statement', 'task', 'thread' - ], - type: 'object', - properties: { - problem_statement: { - $ref: '#/components/schemas/ProblemStatement' - }, - task: { - $ref: '#/components/schemas/Task' - }, - thread: { - $ref: '#/components/schemas/Thread' - } - } - }, - TimePeriod: { - type: 'object', - description: 'A time period', - properties: { - from: { - description: 'From date YYYY-MM-DD format', - type: 'string' - }, - to: { - description: 'to date YYYY-MM-DD format', - type: 'string' - } - } - }, - ProblemStatement: { - type: 'object', - description: 'A Problem Statement definition', - properties: { - id: { - description: 'The id of the problem statement (leave empty if creating a new problem statement)', - type: 'string' - }, - name: { - description: 'The name of the problem statement', - type: 'string' - }, - regionid: { - description: 'The top level region id of the problem statement (example south_sudan, ethiopia)', - type: 'string' + NewModelThread: { + description: "", + required: ["problem_statement", "task", "thread"], + type: "object", + properties: { + problem_statement: { + $ref: "#/components/schemas/ProblemStatement" + }, + task: { + $ref: "#/components/schemas/Task" + }, + thread: { + $ref: "#/components/schemas/Thread" + } + } }, - time_period: { - $ref: '#/components/schemas/TimePeriod' - } - } - }, - Task: { - type: 'object', - description: 'A Task definition', - properties: { - id: { - description: 'The id of the task (leave empty if creating a new task)', - type: 'string' - }, - name: { - description: 'The name of the task', - type: 'string' - }, - "indicatorid": { - description: 'The SVO label for the indicator/response (link to list upcoming). Example: grain~dry__mass-per-area_yield', - type: 'string' - }, - "interventionid": { - description: 'The SVO label for the intervention (if any). Example: crop__planting_start_time', - type: 'string' - }, - regionid: { - description: 'The specific region id for the task (browse https://dev.mint.isi.edu/ethiopia/regions to get the ids)', - type: 'string' - }, - time_period: { - $ref: '#/components/schemas/TimePeriod' - } - } - }, - Thread: { - type: 'object', - description: 'A Thread definition', - properties: { - name: { - description: 'The name of the thread (optional)', - type: 'string' + TimePeriod: { + type: "object", + description: "A time period", + properties: { + from: { + description: "From date YYYY-MM-DD format", + type: "string" + }, + to: { + description: "to date YYYY-MM-DD format", + type: "string" + } + } }, - modelid: { - description: 'The model id to use (browse here: https://dev.mint.isi.edu/ethiopia/models/explore. Example: cycles-0.10.2-alpha-collection-oromia)', - type: 'string' + ProblemStatement: { + type: "object", + description: "A Problem Statement definition", + properties: { + id: { + description: + "The id of the problem statement (leave empty if creating a new problem statement)", + type: "string" + }, + name: { + description: "The name of the problem statement", + type: "string" + }, + regionid: { + description: + "The top level region id of the problem statement (example south_sudan, ethiopia)", + type: "string" + }, + time_period: { + $ref: "#/components/schemas/TimePeriod" + } + } }, - datasets: { - description: 'A map of input name and dataset ids. Example { cycles_weather_soil: ac34f01b-1484-4403-98ea-3a380838cab1 }. To browse the datasets, go here: http://localhost:8080/ethiopia/datasets/browse', - type: 'object' + Task: { + type: "object", + description: "A Task definition", + properties: { + id: { + description: "The id of the task (leave empty if creating a new task)", + type: "string" + }, + name: { + description: "The name of the task", + type: "string" + }, + indicatorid: { + description: + "The SVO label for the indicator/response (link to list upcoming). Example: grain~dry__mass-per-area_yield", + type: "string" + }, + interventionid: { + description: + "The SVO label for the intervention (if any). Example: crop__planting_start_time", + type: "string" + }, + regionid: { + description: + "The specific region id for the task (browse https://dev.mint.isi.edu/ethiopia/regions to get the ids)", + type: "string" + }, + time_period: { + $ref: "#/components/schemas/TimePeriod" + } + } }, - parameters: { - description: 'A map of parameter name and parameter values. Example: { start_planting_day: [100, 107, 114] }', - type: 'object' + Thread: { + type: "object", + description: "A Thread definition", + properties: { + name: { + description: "The name of the thread (optional)", + type: "string" + }, + modelid: { + description: + "The model id to use (browse here: https://dev.mint.isi.edu/ethiopia/models/explore. Example: cycles-0.10.2-alpha-collection-oromia)", + type: "string" + }, + datasets: { + description: + "A map of input name and dataset ids. Example { cycles_weather_soil: ac34f01b-1484-4403-98ea-3a380838cab1 }. To browse the datasets, go here: http://localhost:8080/ethiopia/datasets/browse", + type: "object" + }, + parameters: { + description: + "A map of parameter name and parameter values. Example: { start_planting_day: [100, 107, 114] }", + type: "object" + } + } } - } - } - }, + } }, paths: {} -}; \ No newline at end of file +}; diff --git a/src/api/api-v1/paths/executionQueue.ts b/src/api/api-v1/paths/executionQueue.ts index 4e3243a..9dfa47a 100644 --- a/src/api/api-v1/paths/executionQueue.ts +++ b/src/api/api-v1/paths/executionQueue.ts @@ -2,62 +2,59 @@ import executionQueueService from "../services/executionQueueService"; - -export default function(executionsLocalService: any) { - let operations = { - GET, - DELETE +export default function (executionsLocalService: any) { + const operations = { + GET, + DELETE }; - + function GET(req: any, res: any, next: any) { executionQueueService.getExecutionQueue(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } }); } - - function DELETE(req:any, res: any, next: any) { + + function DELETE(req: any, res: any, next: any) { executionQueueService.emptyExecutionQueue().then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } - }); + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } + }); } - + // NOTE: We could also use a YAML string here. GET.apiDoc = { - summary: 'Get execution queue information', - operationId: 'getExecutionQueue', - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } + summary: "Get execution queue information", + operationId: "getExecutionQueue", + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" + } + } }; // NOTE: We could also use a YAML string here. DELETE.apiDoc = { - summary: 'Empty Local Execution Queue.', - operationId: 'emptyExecutionQueue', - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } + summary: "Empty Local Execution Queue.", + operationId: "emptyExecutionQueue", + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" + } + } }; - + return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/executions.ts b/src/api/api-v1/paths/executions.ts index 876316f..4f6cb74 100644 --- a/src/api/api-v1/paths/executions.ts +++ b/src/api/api-v1/paths/executions.ts @@ -1,45 +1,44 @@ // ./api/api-v1/paths/executions.ts - -export default function(executionsService: any) { - let operations = { - POST + +export default function (executionsService: any) { + const operations = { + POST }; - + function POST(req: any, res: any, next: any) { executionsService.submitExecution(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } }); } - + // NOTE: We could also use a YAML string here. POST.apiDoc = { - summary: 'Submit modeling thread for execution.', - operationId: 'submitExecution', - requestBody: { - description: 'Modeling thread scenario/subgoal/id', - required: true, - content: { - 'application/json': { - schema: { - $ref: '#/components/schemas/ModelThread' + summary: "Submit modeling thread for execution.", + operationId: "submitExecution", + requestBody: { + description: "Modeling thread scenario/subgoal/id", + required: true, + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ModelThread" + } + } + } + }, + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" } - } } - }, - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } }; - + return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/executionsLocal.ts b/src/api/api-v1/paths/executionsLocal.ts index a78a508..07bc117 100644 --- a/src/api/api-v1/paths/executionsLocal.ts +++ b/src/api/api-v1/paths/executionsLocal.ts @@ -1,110 +1,105 @@ // ./api/api-v1/paths/executionsLocal.ts - -export default function(executionsLocalService: any) { - let operations = { - POST, - DELETE, - EMPTY + +export default function (executionsLocalService: any) { + const operations = { + POST, + DELETE, + EMPTY }; - + function POST(req: any, res: any, next: any) { executionsLocalService.submitExecution(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } }); } function DELETE(req: any, res: any, next: any) { - executionsLocalService.deleteExecutionCache(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } - }); + executionsLocalService.deleteExecutionCache(req.body).then((result: any) => { + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } + }); } - - function EMPTY(req:any, res: any, next: any) { - executionsLocalService.emptyExecutionQueue().then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } - }); + + function EMPTY(req: any, res: any, next: any) { + executionsLocalService.emptyExecutionQueue().then((result: any) => { + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } + }); } - + // NOTE: We could also use a YAML string here. POST.apiDoc = { - summary: 'Submit modeling thread for local execution.', - operationId: 'submitLocalExecution', - requestBody: { - description: 'Modeling thread', - required: true, - content: { - 'application/json': { - schema: { - $ref: '#/components/schemas/ModelThread' + summary: "Submit modeling thread for local execution.", + operationId: "submitLocalExecution", + requestBody: { + description: "Modeling thread", + required: true, + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ModelThread" + } + } + } + }, + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" } - } } - }, - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } }; - - // NOTE: We could also use a YAML string here. DELETE.apiDoc = { - summary: 'Delete cached results, cached models and cached data for local execution.', - operationId: 'deleteLocalExecutionCache', - requestBody: { - description: 'Modeling thread', - required: true, - content: { - 'application/json': { - schema: { - $ref: '#/components/schemas/ModelThread' + summary: "Delete cached results, cached models and cached data for local execution.", + operationId: "deleteLocalExecutionCache", + requestBody: { + description: "Modeling thread", + required: true, + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ModelThread" + } + } + } + }, + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" } - } } - }, - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } }; // NOTE: We could also use a YAML string here. EMPTY.apiDoc = { - summary: 'Empty Local Execution Queue.', - operationId: 'emptyExecutionQueue', - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } + summary: "Empty Local Execution Queue.", + operationId: "emptyExecutionQueue", + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" + } + } }; - + return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/logs.ts b/src/api/api-v1/paths/logs.ts index f2a5412..4e3cfa4 100644 --- a/src/api/api-v1/paths/logs.ts +++ b/src/api/api-v1/paths/logs.ts @@ -1,39 +1,39 @@ // ./api/api-v1/paths/logs.ts - -export default function(logsService: any) { - let operations = { - GET + +export default function (logsService: any) { + const operations = { + GET }; - + function GET(req: any, res: any, next: any) { logsService.fetchLog(req.query.ensemble_id).then((result: string) => { res.status(200).json(result); }); } - + // NOTE: We could also use a YAML string here. GET.apiDoc = { - summary: 'Fetch logs for an execution.', - operationId: 'fetchLog', - parameters: [ - { - in: 'query', - name: 'ensemble_id', - required: true, - schema: { - type: 'string' + summary: "Fetch logs for an execution.", + operationId: "fetchLog", + parameters: [ + { + in: "query", + name: "ensemble_id", + required: true, + schema: { + type: "string" + } + } + ], + responses: { + "200": { + description: "Log Details" + }, + default: { + description: "An error occurred" } } - ], - responses: { - "200": { - description: "Log Details" - }, - default: { - description: 'An error occurred' - } - } }; - + return operations; -} \ No newline at end of file +} diff --git a/src/api/api-v1/paths/modelCache.ts b/src/api/api-v1/paths/modelCache.ts index 3a45773..6c024e4 100644 --- a/src/api/api-v1/paths/modelCache.ts +++ b/src/api/api-v1/paths/modelCache.ts @@ -1,32 +1,32 @@ // ./api/api-v1/paths/modelCache.ts -export default function(modelCacheService: any) { - let operations = { - DELETE +export default function (modelCacheService: any) { + const operations = { + DELETE }; - + function DELETE(req: any, res: any, next: any) { modelCacheService.deleteModel(req.query.model_id).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } - }); + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } + }); } - + // NOTE: We could also use a YAML string here. DELETE.apiDoc = { - summary: 'Delete cached models from graphQL. WARNING: This will also result in deletion of all executions for that model, even from other threads !', - operationId: 'deleteModel', + summary: + "Delete cached models from graphQL. WARNING: This will also result in deletion of all executions for that model, even from other threads !", + operationId: "deleteModel", parameters: [ { - in: 'query', - name: 'model_id', + in: "query", + name: "model_id", required: true, schema: { - type: 'string' + type: "string" } } ], @@ -35,10 +35,10 @@ export default function(modelCacheService: any) { description: "Successful response" }, default: { - description: 'An error occurred' + description: "An error occurred" } } }; return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/monitors.ts b/src/api/api-v1/paths/monitors.ts index 359ba2a..5bdc521 100644 --- a/src/api/api-v1/paths/monitors.ts +++ b/src/api/api-v1/paths/monitors.ts @@ -1,88 +1,88 @@ // ./api/api-v1/paths/monitors.ts - -export default function(monitorsService: any) { - let operations = { - POST, - GET + +export default function (monitorsService: any) { + const operations = { + POST, + GET }; - + function POST(req: any, res: any, next: any) { monitorsService.submitMonitor(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } }); } function GET(req: any, res: any, next: any) { - monitorsService.fetchRunStatus(req.query.scenario_id, req.query.thread_id).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } - }); - } - + monitorsService + .fetchRunStatus(req.query.scenario_id, req.query.thread_id) + .then((result: any) => { + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } + }); + } + // NOTE: We could also use a YAML string here. POST.apiDoc = { - summary: 'Submit modeling thread for monitoring.', - operationId: 'submitMonitor', - requestBody: { - description: 'Modeling thread scenario/subgoal/id', - required: true, - content: { - 'application/json': { - schema: { - $ref: '#/components/schemas/ModelThread' + summary: "Submit modeling thread for monitoring.", + operationId: "submitMonitor", + requestBody: { + description: "Modeling thread scenario/subgoal/id", + required: true, + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ModelThread" + } + } + } + }, + responses: { + "201": { + description: "Successful response" + }, + default: { + description: "An error occurred" } - } } - }, - responses: { - "201": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } }; GET.apiDoc = { - summary: 'Fetch execution status of modeling thread', - operationId: 'fetchRunStatus', - parameters: [ - { - in: 'query', - name: 'scenario_id', - required: true, - schema: { - type: 'string' - } - }, - { - in: 'query', - name: 'thread_id', - required: true, - schema: { - type: 'string' + summary: "Fetch execution status of modeling thread", + operationId: "fetchRunStatus", + parameters: [ + { + in: "query", + name: "scenario_id", + required: true, + schema: { + type: "string" + } + }, + { + in: "query", + name: "thread_id", + required: true, + schema: { + type: "string" + } + } + ], + responses: { + "200": { + description: "Thread Details" + }, + default: { + description: "An error occurred" } } - ], - responses: { - "200": { - description: "Thread Details" - }, - default: { - description: 'An error occurred' - } - } }; - + return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/registration.ts b/src/api/api-v1/paths/registration.ts index 0c244ee..08a4fea 100644 --- a/src/api/api-v1/paths/registration.ts +++ b/src/api/api-v1/paths/registration.ts @@ -1,45 +1,44 @@ // ./api/api-v1/paths/registration.ts - -export default function(registrationService: any) { - let operations = { - POST + +export default function (registrationService: any) { + const operations = { + POST }; - + function POST(req: any, res: any, next: any) { registrationService.registerExecutionOutputs(req.body).then((result: any) => { - if(result.result == "error") { - res.status(406).json(result); - } - else { - res.status(202).json(result); - } + if (result.result == "error") { + res.status(406).json(result); + } else { + res.status(202).json(result); + } }); } - + // NOTE: We could also use a YAML string here. POST.apiDoc = { - summary: 'Register outputs of modeling thread in data catalog.', - operationId: 'registerExecutionOutputs', - requestBody: { - description: 'Modeling thread scenario/subgoal/id', - required: true, - content: { - 'application/json': { - schema: { - $ref: '#/components/schemas/ModelThread' + summary: "Register outputs of modeling thread in data catalog.", + operationId: "registerExecutionOutputs", + requestBody: { + description: "Modeling thread scenario/subgoal/id", + required: true, + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/ModelThread" + } + } + } + }, + responses: { + "202": { + description: "Successful response" + }, + default: { + description: "An error occurred" } - } } - }, - responses: { - "202": { - description: "Successful response" - }, - default: { - description: 'An error occurred' - } - } }; - + return operations; - } \ No newline at end of file +} diff --git a/src/api/api-v1/paths/threads.ts b/src/api/api-v1/paths/threads.ts index 8ffb02e..0f45131 100644 --- a/src/api/api-v1/paths/threads.ts +++ b/src/api/api-v1/paths/threads.ts @@ -1,7 +1,7 @@ // ./api/api-v1/paths/threads.ts export default function (threadsService: any) { - let operations = { + const operations = { POST }; @@ -9,8 +9,7 @@ export default function (threadsService: any) { threadsService.createThread(req.body).then((result: any) => { if (result.result == "error") { res.status(406).json(result); - } - else { + } else { res.status(202).json(result); } }); @@ -18,15 +17,15 @@ export default function (threadsService: any) { // NOTE: We could also use a YAML string here. POST.apiDoc = { - summary: 'Create modeling thread in MINT.', - operationId: 'createThread', + summary: "Create modeling thread in MINT.", + operationId: "createThread", requestBody: { - description: 'New modeling thread details', + description: "New modeling thread details", required: true, content: { - 'application/json': { + "application/json": { schema: { - $ref: '#/components/schemas/NewModelThread' + $ref: "#/components/schemas/NewModelThread" } } } @@ -36,7 +35,7 @@ export default function (threadsService: any) { description: "Successful response" }, default: { - description: 'An error occurred' + description: "An error occurred" } } }; diff --git a/src/api/api-v1/services/executionQueueService.ts b/src/api/api-v1/services/executionQueueService.ts index 5eb77dc..def7b10 100644 --- a/src/api/api-v1/services/executionQueueService.ts +++ b/src/api/api-v1/services/executionQueueService.ts @@ -5,28 +5,28 @@ import { createResponse } from "./util"; // ./api-v1/services/executionsLocalService.js const cleanQueue = (queue: Queue.Queue) => { queue.empty(); - queue.clean(0, 'delayed'); - queue.clean(0, 'wait'); - queue.clean(0, 'active'); - queue.clean(0, 'completed'); - queue.clean(0, 'failed'); + queue.clean(0, "delayed"); + queue.clean(0, "wait"); + queue.clean(0, "active"); + queue.clean(0, "completed"); + queue.clean(0, "failed"); - let multi = queue.multi(); - multi.del(queue.toKey('repeat')); + const multi = queue.multi(); + multi.del(queue.toKey("repeat")); multi.exec(); -} +}; const executionQueueService = { async emptyExecutionQueue() { - let executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); + const executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); cleanQueue(executionQueue); return createResponse("success", "Queues emptied"); }, async getExecutionQueue(thread: any) { - let executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); - let count = await executionQueue.getActiveCount(); + const executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); + const count = await executionQueue.getActiveCount(); return createResponse("success", "Number of Active Jobs: " + count); } }; -export default executionQueueService; \ No newline at end of file +export default executionQueueService; diff --git a/src/api/api-v1/services/executionsLocalService.ts b/src/api/api-v1/services/executionsLocalService.ts index 1460bc4..24d9c54 100644 --- a/src/api/api-v1/services/executionsLocalService.ts +++ b/src/api/api-v1/services/executionsLocalService.ts @@ -1,42 +1,49 @@ import { getThread } from "../../../classes/graphql/graphql_functions"; import { Thread } from "../../../classes/mint/mint-types"; -import { saveAndRunExecutionsLocally, deleteExecutableCacheLocally } from "../../../classes/mint/mint-local-functions"; +import { + saveAndRunExecutionsLocally, + deleteExecutableCacheLocally +} from "../../../classes/mint/mint-local-functions"; import { fetchMintConfig } from "../../../classes/mint/mint-functions"; import { KeycloakAdapter } from "../../../config/keycloak-adapter"; import { createResponse } from "./util"; const executionsLocalService = { async submitExecution(threadmodel: any) { - let prefs = await fetchMintConfig(); + const prefs = await fetchMintConfig(); KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password); - let thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { - if(thread) { - let ok = await saveAndRunExecutionsLocally(thread, threadmodel.model_id, prefs); + const thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { + if (thread) { + const ok = await saveAndRunExecutionsLocally(thread, threadmodel.model_id, prefs); if (ok) { - return createResponse("success", - "Thread " + threadmodel.thread_id + " submitted for execution !"); + return createResponse( + "success", + "Thread " + threadmodel.thread_id + " submitted for execution !" + ); + } else { + return createResponse( + "failure", + "Thread " + threadmodel.thread_id + " had errors while submitting execution !" + ); } - else { - return createResponse("failure", "Thread " + threadmodel.thread_id + " had errors while submitting execution !"); - } - } - else { + } else { return createResponse("failure", "Thread " + threadmodel.thread_id + " not found !"); } }, async deleteExecutionCache(threadmodel: any) { - let mint_prefs = await fetchMintConfig(); - let thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { - if(thread) { + const mint_prefs = await fetchMintConfig(); + const thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { + if (thread) { deleteExecutableCacheLocally(thread, threadmodel.model_id, mint_prefs); - return createResponse("success", - "Thread " + threadmodel.thread_id + " execution cache deleted !"); - } - else { + return createResponse( + "success", + "Thread " + threadmodel.thread_id + " execution cache deleted !" + ); + } else { return createResponse("failure", "Thread " + threadmodel.thread_id + " not found !"); } } }; -export default executionsLocalService; \ No newline at end of file +export default executionsLocalService; diff --git a/src/api/api-v1/services/executionsService.ts b/src/api/api-v1/services/executionsService.ts index 508e82e..c3226ee 100644 --- a/src/api/api-v1/services/executionsService.ts +++ b/src/api/api-v1/services/executionsService.ts @@ -6,17 +6,18 @@ import { createResponse } from "./util"; // ./api-v1/services/executionsService.js const executionsService = { async submitExecution(threadmodel: any) { - let thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { - if(thread) { - let mint_prefs = await fetchMintConfig(); + const thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { + if (thread) { + const mint_prefs = await fetchMintConfig(); saveAndRunExecutions(thread, threadmodel.model_id, mint_prefs); - return createResponse("success", - "Thread " + threadmodel.thread_id + " submitted for execution !"); - } - else { + return createResponse( + "success", + "Thread " + threadmodel.thread_id + " submitted for execution !" + ); + } else { return createResponse("failure", "Thread " + threadmodel.thread_id + " not found !"); } } }; -export default executionsService; \ No newline at end of file +export default executionsService; diff --git a/src/api/api-v1/services/logsService.ts b/src/api/api-v1/services/logsService.ts index 737441f..3a8aaff 100644 --- a/src/api/api-v1/services/logsService.ts +++ b/src/api/api-v1/services/logsService.ts @@ -8,28 +8,25 @@ import { createResponse } from "./util"; // ./api-v1/services/logsService.js - const logsService = { async fetchLog(execution_id: string) { try { - let prefs = await fetchMintConfig(); - KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password) + const prefs = await fetchMintConfig(); + KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password); - let ensemble: Execution = await getExecution(execution_id); - if(!ensemble.execution_engine || ensemble.execution_engine == "wings") { - let log = await fetchWingsRunLog(ensemble.runid, prefs); + const ensemble: Execution = await getExecution(execution_id); + if (!ensemble.execution_engine || ensemble.execution_engine == "wings") { + const log = await fetchWingsRunLog(ensemble.runid, prefs); return log; - } - else if(ensemble.execution_engine == "localex") { - let log = fetchLocalRunLog(execution_id, prefs); + } else if (ensemble.execution_engine == "localex") { + const log = fetchLocalRunLog(execution_id, prefs); return log; } - } - catch (error) { + } catch (error) { console.log(error); return createResponse("failure", error); } } }; -export default logsService; \ No newline at end of file +export default logsService; diff --git a/src/api/api-v1/services/modelCacheService.ts b/src/api/api-v1/services/modelCacheService.ts index a88d769..8ca6e0d 100644 --- a/src/api/api-v1/services/modelCacheService.ts +++ b/src/api/api-v1/services/modelCacheService.ts @@ -9,25 +9,22 @@ import { createResponse } from "./util"; const modelCacheService = { async deleteModel(model_id: string) { try { - let prefs = await fetchMintConfig(); - KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password) + const prefs = await fetchMintConfig(); + KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password); - let model = await getModel(model_id); + const model = await getModel(model_id); if (model) { deleteModelCache(model, prefs); - return createResponse("success", - "Model " + model_id + " model cache deleted !"); - } - else { + return createResponse("success", "Model " + model_id + " model cache deleted !"); + } else { return createResponse("failure", "Model " + model_id + " not found !"); } - } - catch (error) { + } catch (error) { console.log(error); return createResponse("failure", error); } } }; -export default modelCacheService; \ No newline at end of file +export default modelCacheService; diff --git a/src/api/api-v1/services/monitorsService.ts b/src/api/api-v1/services/monitorsService.ts index 35e17ea..f34fc3e 100644 --- a/src/api/api-v1/services/monitorsService.ts +++ b/src/api/api-v1/services/monitorsService.ts @@ -7,24 +7,23 @@ import { createResponse } from "./util"; const monitorsService = { async submitMonitor(threadmodel: any) { - let thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { - if(thread) { - let mint_prefs = await fetchMintConfig(); + const thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { + if (thread) { + const mint_prefs = await fetchMintConfig(); monitorAllExecutions(thread, threadmodel.model_id, mint_prefs); - return createResponse("success", - "Thread " + threadmodel.thread_id + " submitted for monitoring !"); - } - else { + return createResponse( + "success", + "Thread " + threadmodel.thread_id + " submitted for monitoring !" + ); + } else { return createResponse("failure", "Thread " + threadmodel.thread_id + " not found !"); } }, async fetchRunStatus(threadid: string) { - let thread = await getThread(threadid); - if(thread) - return thread.execution_summary; - else - return createResponse("failure", "Thread not found"); + const thread = await getThread(threadid); + if (thread) return thread.execution_summary; + else return createResponse("failure", "Thread not found"); } }; -export default monitorsService; \ No newline at end of file +export default monitorsService; diff --git a/src/api/api-v1/services/registrationService.ts b/src/api/api-v1/services/registrationService.ts index ef1a1c4..bbed49e 100644 --- a/src/api/api-v1/services/registrationService.ts +++ b/src/api/api-v1/services/registrationService.ts @@ -8,22 +8,25 @@ import { KeycloakAdapter } from "../../../config/keycloak-adapter"; const registrationService = { async registerExecutionOutputs(threadmodel: any) { try { - let prefs = await fetchMintConfig(); - let thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { - let ok = await registerExecutionResults(thread, threadmodel.model_id, prefs); + const prefs = await fetchMintConfig(); + const thread: Thread = await getThread(threadmodel.thread_id); //.then((thread: Thread) => { + const ok = await registerExecutionResults(thread, threadmodel.model_id, prefs); if (ok) { - return createResponse("success", - "Thread " + threadmodel.thread_id + " outputs registered in the data catalog !"); + return createResponse( + "success", + "Thread " + threadmodel.thread_id + " outputs registered in the data catalog !" + ); + } else { + return createResponse( + "failure", + "Thread " + threadmodel.thread_id + " had errors while registering outputs !" + ); } - else { - return createResponse("failure", "Thread " + threadmodel.thread_id + " had errors while registering outputs !"); - } - } - catch (error) { + } catch (error) { console.log(error); return createResponse("failure", error); } - } + } }; -export default registrationService; \ No newline at end of file +export default registrationService; diff --git a/src/api/api-v1/services/threadsService.ts b/src/api/api-v1/services/threadsService.ts index 7cc1d62..26513e3 100644 --- a/src/api/api-v1/services/threadsService.ts +++ b/src/api/api-v1/services/threadsService.ts @@ -1,37 +1,56 @@ -import { Thread, ProblemStatement, Task, Model, Dataset, ExecutionSummary, ThreadInfo, Region, ThreadModelExecutions, Execution, MintPermission, ThreadModelMap, ModelEnsembleMap, Dataslice, DataMap, IdMap, ProblemStatementEvent, TaskEvent } from "../../../classes/mint/mint-types"; +import { + Thread, + ProblemStatement, + Task, + Dataset, + ExecutionSummary, + Region, + Dataslice, + DataMap, + IdMap, + ProblemStatementEvent, + TaskEvent +} from "../../../classes/mint/mint-types"; import { fetchModelFromCatalog } from "../../../classes/mint/model-catalog-functions"; import { queryDatasetDetails } from "../../../classes/mint/data-catalog-functions"; -import { addProblemStatement, addTask, addThread, getProblemStatement, getRegionDetails, getTask, getThread, getTotalConfigurations, setThreadData, setThreadModels, setThreadParameters } from "../../../classes/graphql/graphql_functions"; +import { + addProblemStatement, + addTask, + addThread, + getProblemStatement, + getRegionDetails, + getTask, + getThread, + getTotalConfigurations, + setThreadData, + setThreadModels, + setThreadParameters +} from "../../../classes/graphql/graphql_functions"; import { getCreateEvent, uuidv4 } from "../../../classes/graphql/graphql_adapter"; import { fetchMintConfig } from "../../../classes/mint/mint-functions"; -import { ModelConfigurationSetup } from '@mintproject/modelcatalog_client'; +import { ModelConfigurationSetup } from "@mintproject/modelcatalog_client"; import { KeycloakAdapter } from "../../../config/keycloak-adapter"; import { createResponse } from "./util"; // ./api-v1/services/threadsService.js -function flatten(array) -{ - if(array.length == 0) - return array; - else if(Array.isArray(array[0])) - return flatten(array[0]).concat(flatten(array.slice(1))); - else - return [array[0]].concat(flatten(array.slice(1))); +function flatten(array) { + if (array.length == 0) return array; + else if (Array.isArray(array[0])) return flatten(array[0]).concat(flatten(array.slice(1))); + else return [array[0]].concat(flatten(array.slice(1))); } const threadsService = { async createThread(desc: any) { - let mint_prefs = await fetchMintConfig(); - KeycloakAdapter.signIn(mint_prefs.graphql.username, mint_prefs.graphql.password) + const mint_prefs = await fetchMintConfig(); + KeycloakAdapter.signIn(mint_prefs.graphql.username, mint_prefs.graphql.password); // Create Problem Statement if needed - let prob_desc = desc.problem_statement; - let prob : ProblemStatement = null; - if(prob_desc.id) { + const prob_desc = desc.problem_statement; + let prob: ProblemStatement = null; + if (prob_desc.id) { prob = await getProblemStatement(prob_desc.id); - } - else { + } else { prob = { name: prob_desc.name, regionid: prob_desc.regionid, @@ -41,40 +60,38 @@ const threadsService = { }, tasks: {}, events: [getCreateEvent("Problem Statement from API") as ProblemStatementEvent], - permissions: [{read: true, write: true, execute: true, owner: false, userid: "*"}] - } + permissions: [{ read: true, write: true, execute: true, owner: false, userid: "*" }] + }; prob.id = await addProblemStatement(prob); } // Create Task if needed - let task_desc = desc.task; - let task : Task = null; - let time_period = task_desc.time_period ? task_desc.time_period : prob_desc.time_period; - if(task_desc.id) { + const task_desc = desc.task; + let task: Task = null; + const time_period = task_desc.time_period ? task_desc.time_period : prob_desc.time_period; + if (task_desc.id) { task = await getTask(task_desc.id); - } - else { + } else { task = { name: task_desc.name, regionid: task_desc.regionid, problem_statement_id: prob.id, - response_variables: [ task_desc.indicatorid ], - driving_variables: task_desc.interventionid ? [ task_desc.interventionid ] : [], + response_variables: [task_desc.indicatorid], + driving_variables: task_desc.interventionid ? [task_desc.interventionid] : [], dates: { start_date: new Date(time_period.from), end_date: new Date(time_period.to) }, events: [getCreateEvent("Task from API") as TaskEvent], - permissions: [{read: true, write: true, execute: true, owner: false, userid: "*"}] - } + permissions: [{ read: true, write: true, execute: true, owner: false, userid: "*" }] + }; task.id = await addTask(prob, task); } - // Create Thread - let thread_desc = desc.thread; - let thread_name = thread_desc.name ? thread_desc.name : null; - let thread_notes = "Added thread from API"; + const thread_desc = desc.thread; + const thread_name = thread_desc.name ? thread_desc.name : null; + const thread_notes = "Added thread from API"; let thread = { name: thread_name, @@ -86,64 +103,76 @@ const threadsService = { models: {}, execution_summary: {}, events: [getCreateEvent(thread_notes)], - permissions: [{read: true, write: true, execute: true, owner: false, userid: "*"}] + permissions: [{ read: true, write: true, execute: true, owner: false, userid: "*" }] } as Thread; // Store Thread (no data or models yet) thread.id = await addThread(task, thread); // Fetch region details - let region: Region = await getRegionDetails(task.regionid); + const region: Region = await getRegionDetails(task.regionid); /* Set Thread Model */ - let model: ModelConfigurationSetup = await fetchModelFromCatalog(task.response_variables, [], desc.thread.modelid, mint_prefs); + const model: ModelConfigurationSetup = await fetchModelFromCatalog( + task.response_variables, + [], + desc.thread.modelid, + mint_prefs + ); await setThreadModels([model], "Added models", thread); - + thread = await getThread(thread.id); /* Set Thread Data */ - let data: DataMap = {}; - let model_ensembles = thread.model_ensembles; + const data: DataMap = {}; + const model_ensembles = thread.model_ensembles; // Fetch dataset details from the Data Catalog - for(var i=0; i { - if(pres.hasStandardVariable) - return pres.hasStandardVariable.map((sv)=>sv.label) + if (!(datasetids instanceof Array)) datasetids = [datasetids]; + for (let j = 0; j < datasetids.length; j++) { + const dsid = datasetids[j]; + const variables_arr = input_file.hasPresentation.map((pres) => { + if (pres.hasStandardVariable) + return pres.hasStandardVariable.map((sv) => sv.label); }); let variables = flatten(variables_arr); variables = variables.filter((v) => v); - let dataset : Dataset = await queryDatasetDetails(model.id, input_file.id, - variables, dsid, thread.dates, region, mint_prefs); - let sliceid = uuidv4(); - let dataslice = { + const dataset: Dataset = await queryDatasetDetails( + model.id, + input_file.id, + variables, + dsid, + thread.dates, + region, + mint_prefs + ); + const sliceid = uuidv4(); + const dataslice = { id: sliceid, total_resources: dataset.resources.length, - selected_resources: dataset.resources.filter((res)=>res.selected).length, + selected_resources: dataset.resources.filter((res) => res.selected) + .length, resources: dataset.resources, time_period: thread.dates, name: dataset.name, dataset: dataset, resources_loaded: dataset.resources_loaded - } as Dataslice + } as Dataslice; data[sliceid] = dataslice; model_ensembles[model.id].bindings[input_file.id].push(sliceid!); - } + } } } } @@ -151,20 +180,18 @@ const threadsService = { await setThreadData(data, model_ensembles, "Setting thread data via API", thread); // Set Thread Parameters - let execution_summary : IdMap = {}; - for(var i=0; i = {}; + for (var i = 0; i < model.hasParameter.length; i++) { + const input_parameter = model.hasParameter[i]; + if (!input_parameter.hasFixedValue || input_parameter.hasFixedValue.length == 0) { let value = thread_desc.parameters[input_parameter.label[0]]; - if(!value) - value = input_parameter.hasDefaultValue[0]; - if (!(value instanceof Array)) - value = [ value ]; + if (!value) value = input_parameter.hasDefaultValue[0]; + if (!(value instanceof Array)) value = [value]; model_ensembles[model.id].bindings[input_parameter.id] = value; } } // Get total number of configs to run - let totalconfigs = getTotalConfigurations(model, model_ensembles[model.id].bindings, data); + const totalconfigs = getTotalConfigurations(model, model_ensembles[model.id].bindings, data); execution_summary[model.id] = { total_runs: totalconfigs, submitted_runs: 0, @@ -172,16 +199,21 @@ const threadsService = { successful_runs: 0 } as ExecutionSummary; - await setThreadParameters(model_ensembles, execution_summary, "Setting thread parameters via API", thread); + await setThreadParameters( + model_ensembles, + execution_summary, + "Setting thread parameters via API", + thread + ); // Return details of newly created thread - let modelthread = { + const modelthread = { problem_statement_id: prob.id, task_id: task.id, thread_id: thread.id - } + }; return createResponse("success", JSON.stringify(modelthread)); - }, + } }; -export default threadsService; \ No newline at end of file +export default threadsService; diff --git a/src/classes/graphql/graphql_adapter.ts b/src/classes/graphql/graphql_adapter.ts index 5804981..1f82ea9 100644 --- a/src/classes/graphql/graphql_adapter.ts +++ b/src/classes/graphql/graphql_adapter.ts @@ -1,61 +1,81 @@ -import { Task, Thread, ProblemStatementInfo, ProblemStatement, ThreadInfo, MintEvent, ModelEnsembleMap, - ModelIOBindings, Execution, ExecutionSummary, DataMap, ThreadModelMap, Dataslice, ModelIO, Dataset, Model, ModelParameter, DataResource, Variable, MintPermission } from "../mint/mint-types" - -import * as crypto from 'crypto'; +import { + Task, + Thread, + ProblemStatementInfo, + ProblemStatement, + ThreadInfo, + MintEvent, + ModelEnsembleMap, + ModelIOBindings, + Execution, + ExecutionSummary, + DataMap, + ThreadModelMap, + Dataslice, + ModelIO, + Dataset, + Model, + ModelParameter, + DataResource, + Variable, + MintPermission +} from "../mint/mint-types"; + +import * as crypto from "crypto"; import { Region } from "../mint/mint-types"; import { KeycloakAdapter } from "../../config/keycloak-adapter"; export const uuidv4 = () => { - return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { - var r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8); - return v.toString(16); + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) { + const r = (Math.random() * 16) | 0, + v = c == "x" ? r : (r & 0x3) | 0x8; + return v.toString(16); }); -} +}; -export const fromTimestampIntegerToString = (timestamp: number) : string => { - return new Date(timestamp).toISOString().replace(/\.000Z$/, ''); -} +export const fromTimestampIntegerToString = (timestamp: number): string => { + return new Date(timestamp).toISOString().replace(/\.000Z$/, ""); +}; -export const fromTimestampIntegerToReadableString = (timestamp: number) : string => { - return fromTimestampIntegerToString(timestamp).replace(/T/,' at ').replace(/\..+$/,''); -} +export const fromTimestampIntegerToReadableString = (timestamp: number): string => { + return fromTimestampIntegerToString(timestamp).replace(/T/, " at ").replace(/\..+$/, ""); +}; -export const fromTimestampIntegerToDateString = (timestamp: number) : string => { - return fromTimestampIntegerToString(timestamp).replace(/T.*$/,''); -} +export const fromTimestampIntegerToDateString = (timestamp: number): string => { + return fromTimestampIntegerToString(timestamp).replace(/T.*$/, ""); +}; -export const toDateString = (date: Date) : string => { - if(!date) - return null; - if(typeof date === 'string') { - return (date+"").split('T')[0] +export const toDateString = (date: Date): string => { + if (!date) return null; + if (typeof date === "string") { + return (date + "").split("T")[0]; } - return date.toISOString().split('T')[0] -} + return date.toISOString().split("T")[0]; +}; -export const toDateTimeString = (date: Date) : string => { - if(!date) - return null; +export const toDateTimeString = (date: Date): string => { + if (!date) return null; return date.toLocaleDateString() + " " + date.toLocaleTimeString(); -} +}; export const regionToGQL = (region: Region) => { - let regionobj = { + const regionobj = { name: region.name, category_id: region.category_id, geometries: { - data: region.geometries.map((geom) => { return { "geometry": geom }}) + data: region.geometries.map((geom) => { + return { geometry: geom }; + }) }, model_catalog_uri: region.model_catalog_uri }; - if (!region.id) - regionobj["id"] = getAutoID() + if (!region.id) regionobj["id"] = getAutoID(); return regionobj; -} +}; -export const regionFromGQL = (regionobj: any) : Region => { - let region = { +export const regionFromGQL = (regionobj: any): Region => { + const region = { id: regionobj.id, name: regionobj.name, category_id: regionobj.category_id, @@ -63,10 +83,10 @@ export const regionFromGQL = (regionobj: any) : Region => { model_catalog_uri: regionobj.model_catalog_uri } as Region; return region; -} +}; export const variableFromGQL = (varobj: any) => { - let variable = { + const variable = { id: varobj.id, name: varobj.name, categories: (varobj.categories ?? []).map((catobj) => catobj["category"]), @@ -76,49 +96,49 @@ export const variableFromGQL = (varobj: any) => { intervention: varobj.intervention } as Variable; return variable; -} +}; export const eventToGQL = (event: MintEvent) => { - let eventobj = { + const eventobj = { event: event.event, userid: event.userid, timestamp: event.timestamp.toISOString(), notes: event.notes }; return eventobj; -} +}; -export const eventFromGQL = (eventobj: any) : MintEvent => { - let event = { +export const eventFromGQL = (eventobj: any): MintEvent => { + const event = { event: eventobj.event, userid: eventobj.userid, timestamp: new Date(eventobj.timestamp), notes: eventobj.notes } as MintEvent; return event; -} +}; -export const permissionFromGQL = (permobj: any) : MintPermission => { - let permission = { +export const permissionFromGQL = (permobj: any): MintPermission => { + const permission = { userid: permobj.user_id, read: permobj.read ?? false, write: permobj.write ?? false, execute: permobj?.execute ?? false } as MintPermission; return permission; -} +}; export const permissionToGQL = (permission: MintPermission) => { - let permissionobj = { + const permissionobj = { user_id: permission.userid, read: permission.read ?? false, write: permission.write ?? false }; return permissionobj; -} +}; export const problemStatementToGQL = (problem_statement: ProblemStatementInfo) => { - let problemobj = { + const problemobj = { id: getAutoID(), name: problem_statement.name, start_date: toDateString(problem_statement.dates.start_date), @@ -136,10 +156,10 @@ export const problemStatementToGQL = (problem_statement: ProblemStatementInfo) = } }; return problemobj; -} +}; export const problemStatementUpdateToGQL = (problem_statement: ProblemStatementInfo) => { - let problemobj = { + const problemobj = { id: problem_statement.id, name: problem_statement.name, start_date: toDateString(problem_statement.dates.start_date), @@ -157,11 +177,11 @@ export const problemStatementUpdateToGQL = (problem_statement: ProblemStatementI } }; return problemobj; -} +}; -export const problemStatementFromGQL = (problem: any) : ProblemStatement => { - let details = { - id : problem["id"], +export const problemStatementFromGQL = (problem: any): ProblemStatement => { + const details = { + id: problem["id"], regionid: problem["region_id"], name: problem["name"], dates: { @@ -173,18 +193,18 @@ export const problemStatementFromGQL = (problem: any) : ProblemStatement => { tasks: {}, preview: problem["preview"] } as ProblemStatement; - if(problem["tasks"]) { - problem["tasks"].forEach((task:any) => { - let fbtask = taskFromGQL(task); + if (problem["tasks"]) { + problem["tasks"].forEach((task: any) => { + const fbtask = taskFromGQL(task); fbtask.problem_statement_id = problem["id"]; details.tasks[fbtask.id] = fbtask; - }) + }); } return details; -} +}; export const taskToGQL = (task: Task, problem_statement: ProblemStatementInfo) => { - let taskGQL = { + const taskGQL = { id: getAutoID(), name: task.name, problem_statement_id: problem_statement.id, @@ -194,7 +214,7 @@ export const taskToGQL = (task: Task, problem_statement: ProblemStatementInfo) = response_variable_id: task.response_variables[0], driving_variable_id: task.driving_variables.length > 0 ? task.driving_variables[0] : null, events: { - data: task.events.map(eventToGQL), + data: task.events.map(eventToGQL) }, permissions: { data: (task.permissions || []).map(permissionToGQL), @@ -205,10 +225,10 @@ export const taskToGQL = (task: Task, problem_statement: ProblemStatementInfo) = } }; return taskGQL; -} +}; export const taskUpdateToGQL = (task: Task) => { - let taskGQL = { + const taskGQL = { id: task.id, name: task.name, problem_statement_id: task.problem_statement_id, @@ -218,7 +238,7 @@ export const taskUpdateToGQL = (task: Task) => { response_variable_id: task.response_variables[0], driving_variable_id: task.driving_variables.length > 0 ? task.driving_variables[0] : null, events: { - data: task.events.map(eventToGQL), + data: task.events.map(eventToGQL) }, permissions: { data: (task.permissions || []).map(permissionToGQL), @@ -228,13 +248,13 @@ export const taskUpdateToGQL = (task: Task) => { } } }; - + return taskGQL; -} +}; -export const taskFromGQL = (task: any) : Task => { - let taskobj = { - id : task["id"], +export const taskFromGQL = (task: any): Task => { + const taskobj = { + id: task["id"], problem_statement_id: task["problem_statement_id"], regionid: task["region_id"], name: task["name"], @@ -248,18 +268,18 @@ export const taskFromGQL = (task: any) : Task => { events: task["events"].map(eventFromGQL), permissions: task["permissions"].map(permissionFromGQL) } as Task; - if(task["threads"]) { - task["threads"].forEach((thread:any) => { - let fbthread = threadInfoFromGQL(thread); + if (task["threads"]) { + task["threads"].forEach((thread: any) => { + const fbthread = threadInfoFromGQL(thread); fbthread.task_id = task["id"]; taskobj.threads[fbthread.id] = fbthread; }); } return taskobj; -} +}; export const threadInfoToGQL = (thread: ThreadInfo, taskid: string, regionid: string) => { - let threadobj = { + const threadobj = { id: getAutoID(), name: thread.name, task_id: taskid, @@ -267,9 +287,10 @@ export const threadInfoToGQL = (thread: ThreadInfo, taskid: string, regionid: st end_date: toDateString(thread.dates.end_date), region_id: regionid, response_variable_id: thread.response_variables[0], - driving_variable_id: thread.driving_variables.length > 0 ? thread.driving_variables[0] : null, + driving_variable_id: + thread.driving_variables.length > 0 ? thread.driving_variables[0] : null, events: { - data: thread.events.map(eventToGQL), + data: thread.events.map(eventToGQL) }, permissions: { data: (thread.permissions || []).map(permissionToGQL), @@ -280,19 +301,20 @@ export const threadInfoToGQL = (thread: ThreadInfo, taskid: string, regionid: st } }; return threadobj; -} +}; -export const threadInfoUpdateToGQL = (thread: ThreadInfo) => { - let threadobj = { +export const threadInfoUpdateToGQL = (thread: ThreadInfo) => { + const threadobj = { id: thread.id, task_id: thread.task_id, name: thread.name, start_date: toDateString(thread.dates.start_date), end_date: toDateString(thread.dates.end_date), response_variable_id: thread.response_variables[0], - driving_variable_id: thread.driving_variables.length > 0 ? thread.driving_variables[0] : null, + driving_variable_id: + thread.driving_variables.length > 0 ? thread.driving_variables[0] : null, events: { - data: thread.events.map(eventToGQL), + data: thread.events.map(eventToGQL) }, permissions: { data: (thread.permissions || []).map(permissionToGQL), @@ -303,11 +325,11 @@ export const threadInfoUpdateToGQL = (thread: ThreadInfo) => { } }; return threadobj; -} +}; export const threadInfoFromGQL = (thread: any) => { return { - id : thread["id"], + id: thread["id"], name: thread["name"], dates: { start_date: new Date(thread["start_date"]), @@ -315,15 +337,16 @@ export const threadInfoFromGQL = (thread: any) => { }, regionid: thread["region_id"], driving_variables: thread.driving_variable_id != null ? [thread.driving_variable_id] : [], - response_variables: thread.response_variable_id != null ? [thread.response_variable_id] : [], + response_variables: + thread.response_variable_id != null ? [thread.response_variable_id] : [], events: thread["events"].map(eventFromGQL), permissions: thread["permissions"].map(permissionFromGQL) } as ThreadInfo; -} +}; export const threadFromGQL = (thread: any) => { - let fbthread = { - id : thread["id"], + const fbthread = { + id: thread["id"], task_id: thread["task_id"], regionid: thread["region_id"], name: thread["name"], @@ -332,26 +355,27 @@ export const threadFromGQL = (thread: any) => { end_date: new Date(thread["end_date"]) }, driving_variables: thread.driving_variable_id != null ? [thread.driving_variable_id] : [], - response_variables: thread.response_variable_id != null ? [thread.response_variable_id] : [], + response_variables: + thread.response_variable_id != null ? [thread.response_variable_id] : [], execution_summary: {}, events: thread["events"].map(eventFromGQL), models: {}, data: {}, model_ensembles: {} } as Thread; - - thread["thread_data"].forEach((tm:any) => { - let m = tm["dataslice"]; - let dataslice : Dataslice = datasliceFromGQL(m); + + thread["thread_data"].forEach((tm: any) => { + const m = tm["dataslice"]; + const dataslice: Dataslice = datasliceFromGQL(m); fbthread.data[dataslice.id] = dataslice; - }) + }); - thread["thread_models"].forEach((tm:any) => { - let m = tm["model"]; - let model : Model = modelFromGQL(m); + thread["thread_models"].forEach((tm: any) => { + const m = tm["model"]; + const model: Model = modelFromGQL(m); fbthread.models[model.id] = model; - let model_ensemble = modelEnsembleFromGQL(tm["data_bindings"], tm["parameter_bindings"]); + const model_ensemble = modelEnsembleFromGQL(tm["data_bindings"], tm["parameter_bindings"]); fbthread.model_ensembles[model.id] = { id: tm["id"], bindings: model_ensemble @@ -362,9 +386,9 @@ export const threadFromGQL = (thread: any) => { // Set summary changed to true, to load the executions initially //fbthread.execution_summary[model.id].changed = true; }); - }) + }); return fbthread; -} +}; export const threadModelExecutionSummaryFromGQL = (tmex: any) => { return { @@ -382,65 +406,63 @@ export const threadModelExecutionSummaryFromGQL = (tmex: any) => { submitted_for_publishing: tmex["submitted_for_publishing"], submitted_for_registration: tmex["submitted_for_registration"] } as ExecutionSummary; -} +}; export const threadModelsToGQL = (models: Model[], threadid: string) => { return models.map((model) => { return { - "thread_id": threadid, - "model": { - "data": modelToGQL(model), - "on_conflict": { - "constraint": "model_pkey", - "update_columns": ["name"] + thread_id: threadid, + model: { + data: modelToGQL(model), + on_conflict: { + constraint: "model_pkey", + update_columns: ["name"] } } }; }); -} +}; export const getTotalConfigs = (model: Model, bindings: ModelIOBindings, thread: Thread) => { let totalconfigs = 1; model.input_files.map((io) => { - if(!io.value) { + if (!io.value) { // Expand a dataset to it's constituent resources // FIXME: Create a collection if the model input has dimensionality of 1 - if(bindings[io.id]) { + if (bindings[io.id]) { let numresources = 0; bindings[io.id].map((dsid) => { - let ds = thread.data[dsid]; + const ds = thread.data[dsid]; numresources += ds.selected_resources; }); totalconfigs *= numresources; } - } - else { + } else { totalconfigs *= (io.value.resources as any[]).length; } - }) - + }); + // Add adjustable parameters to the input ids model.input_parameters.map((io) => { - if(!io.value) - totalconfigs *= bindings[io.id].length; + if (!io.value) totalconfigs *= bindings[io.id].length; }); return totalconfigs; -} +}; export const datasliceFromGQL = (d: any) => { - let ds = d["dataset"]; + const ds = d["dataset"]; return { id: d["id"], name: ds["name"], total_resources: d["total_resources"]?.aggregate?.count ?? 0, selected_resources: d["selected_resources"]?.aggregate?.count ?? 0, - resources: (d["resources"] ?? []).map((resobj:any) => { - let res = resourceFromGQL(resobj["resource"]); + resources: (d["resources"] ?? []).map((resobj: any) => { + const res = resourceFromGQL(resobj["resource"]); res.selected = resobj["selected"]; return res; }), - resources_loaded: (d["resources"]?.length > 0) ? true : false, + resources_loaded: d["resources"]?.length > 0 ? true : false, time_period: { start_date: ds["start_date"], end_date: ds["end_date"] @@ -451,7 +473,7 @@ export const datasliceFromGQL = (d: any) => { name: ds["name"] } as Dataset } as Dataslice; -} +}; export const modelFromGQL = (m: any) => { m = Object.assign({}, m); @@ -468,31 +490,32 @@ export const modelFromGQL = (m: any) => { }); delete m["parameters"]; return m; -} +}; export const modelIOFromGQL = (model_io: any) => { - let io = model_io["model_io"]; - let fixed_ds = (io["fixed_bindings"] && io["fixed_bindings"].length > 0) ? - { - id: io.id + "_fixed_dataset", - name: io.name + "_fixed_dataset", - resources: io["fixed_bindings"].map((res:any) => { - return res["resource"] - }) - } as Dataslice - : null; + const io = model_io["model_io"]; + const fixed_ds = + io["fixed_bindings"] && io["fixed_bindings"].length > 0 + ? ({ + id: io.id + "_fixed_dataset", + name: io.name + "_fixed_dataset", + resources: io["fixed_bindings"].map((res: any) => { + return res["resource"]; + }) + } as Dataslice) + : null; return { id: io["id"], name: io["name"], type: io["type"], value: fixed_ds, position: model_io["position"], - variables: io["variables"].map((varobj:any) => { - let v = varobj["variable"]; + variables: io["variables"].map((varobj: any) => { + const v = varobj["variable"]; return v["id"]; }) - } as ModelIO -} + } as ModelIO; +}; export const modelParameterFromGQL = (p: any) => { return { @@ -508,30 +531,28 @@ export const modelParameterFromGQL = (p: any) => { unit: p["unit"], value: p["fixed_value"], position: p["position"] - } as ModelParameter -} + } as ModelParameter; +}; export const modelEnsembleFromGQL = (dbs: any[], pbs: any[]): ModelIOBindings => { - let bindings = {} as ModelIOBindings; + const bindings = {} as ModelIOBindings; dbs.forEach((db) => { let binding = bindings[db["model_io"]["id"]]; - if(!binding) - binding = []; + if (!binding) binding = []; binding.push(db["dataslice_id"]); bindings[db["model_io"]["id"]] = binding; - }) + }); pbs.forEach((pb) => { let binding = bindings[pb["model_parameter"]["id"]]; - if(!binding) - binding = []; + if (!binding) binding = []; binding.push(pb["parameter_value"]); - bindings[pb["model_parameter"]["id"]] = binding; - }) + bindings[pb["model_parameter"]["id"]] = binding; + }); return bindings; -} +}; -export const executionToGQL = (ex: Execution) : any => { - let exobj = { +export const executionToGQL = (ex: Execution): any => { + const exobj = { id: ex.id, model_id: ex.modelid, status: ex.status, @@ -548,29 +569,28 @@ export const executionToGQL = (ex: Execution) : any => { results: { data: [] as any } - } + }; Object.keys(ex.bindings).forEach((ioid) => { - let binding = ex.bindings[ioid]; - if (typeof(binding) == 'string') { + const binding = ex.bindings[ioid]; + if (typeof binding == "string") { exobj.parameter_bindings.data.push({ model_parameter_id: ioid, - parameter_value: binding+"", - }) - } - else { + parameter_value: binding + "" + }); + } else { exobj.data_bindings.data.push({ model_io_id: ioid, resource_id: binding["id"] - }) + }); } - }) + }); exobj.results.data = executionResultsToGQL(ex.results); return exobj; -} +}; -export const executionFromGQL = (ex: any, emulator=false) : Execution => { - let exobj = { - id: ex.id.replace(/\-/g,''), +export const executionFromGQL = (ex: any, emulator = false): Execution => { + const exobj = { + id: ex.id.replace(/\-/g, ""), modelid: ex.model_id, status: ex.status, start_time: new Date(ex.start_time), @@ -581,20 +601,23 @@ export const executionFromGQL = (ex: any, emulator=false) : Execution => { bindings: {}, results: {} } as Execution; - ex.parameter_bindings.forEach((param:any) => { - exobj.bindings[(emulator ? param.model_parameter.name : param.model_parameter_id)] = param.parameter_value; + ex.parameter_bindings.forEach((param: any) => { + exobj.bindings[emulator ? param.model_parameter.name : param.model_parameter_id] = + param.parameter_value; }); - ex.data_bindings.forEach((data:any) => { - exobj.bindings[(emulator ? data.model_io.name : data.model_io_id)] = data.resource as DataResource; + ex.data_bindings.forEach((data: any) => { + exobj.bindings[emulator ? data.model_io.name : data.model_io_id] = + data.resource as DataResource; }); - ex.results.forEach((data:any) => { - exobj.results[(emulator ? data.model_output.name : data.model_io_id)] = data.resource as DataResource; + ex.results.forEach((data: any) => { + exobj.results[emulator ? data.model_output.name : data.model_io_id] = + data.resource as DataResource; }); return exobj; -} +}; export const resourceToGQL = (resource: DataResource) => { - let resourceobj = { + const resourceobj = { id: resource.id, name: resource.name, url: resource.url, @@ -602,10 +625,10 @@ export const resourceToGQL = (resource: DataResource) => { end_date: resource.time_period?.end_date }; return resourceobj; -} +}; -export const resourceFromGQL = (resourceobj: any) : DataResource => { - let resource = { +export const resourceFromGQL = (resourceobj: any): DataResource => { + const resource = { id: resourceobj.id, name: resourceobj.name, url: resourceobj.url, @@ -616,8 +639,7 @@ export const resourceFromGQL = (resourceobj: any) : DataResource => { } } as DataResource; return resource; -} - +}; export const getCreateEvent = (notes: string) => { return { @@ -626,7 +648,7 @@ export const getCreateEvent = (notes: string) => { userid: KeycloakAdapter.getUser().email, notes: notes } as MintEvent; -} +}; export const getUpdateEvent = (notes: string) => { return { @@ -635,353 +657,346 @@ export const getUpdateEvent = (notes: string) => { userid: KeycloakAdapter.getUser().email, notes: notes } as MintEvent; -} +}; -export const getCustomEvent = (event:string, notes: string) => { +export const getCustomEvent = (event: string, notes: string) => { return { event: event, timestamp: new Date(), userid: KeycloakAdapter.getUser().email, notes: notes } as MintEvent; -} - +}; const getNamespacedId = (namespace, id) => { - if(id.indexOf(namespace) == 0) - return id; - return namespace + id -} + if (id.indexOf(namespace) == 0) return id; + return namespace + id; +}; export const modelToGQL = (m: Model) => { - let namespace = m.id.replace(/(^.*\/).*$/, "$1"); + const namespace = m.id.replace(/(^.*\/).*$/, "$1"); return { - "id": m.id, - "name": m.name, - "category": m.category, - "description": m.description, - "region_name": m.region_name, - "type": m.model_type, - "model_configuration": getNamespacedId(namespace, m.model_configuration), - "model_version": getNamespacedId(namespace, m.model_version), - "model_name": getNamespacedId(namespace, m.model_name), - "dimensionality": m.dimensionality, - "parameter_assignment": m.parameter_assignment, - "parameter_assignment_details": m.parameter_assignment_details, - "calibration_target_variable": m.calibration_target_variable, - "spatial_grid_resolution": m.spatial_grid_resolution, - "spatial_grid_type": m.spatial_grid_type, - "output_time_interval": m.output_time_interval, - "code_url": m.code_url, - "usage_notes": m.usage_notes, - "software_image": m.software_image, - "inputs": { - "data": m.input_files.map((input) => modelInputOutputToGQL(input)), - "on_conflict": { - "constraint": "model_input_pkey", - "update_columns": ["model_id"] + id: m.id, + name: m.name, + category: m.category, + description: m.description, + region_name: m.region_name, + type: m.model_type, + model_configuration: getNamespacedId(namespace, m.model_configuration), + model_version: getNamespacedId(namespace, m.model_version), + model_name: getNamespacedId(namespace, m.model_name), + dimensionality: m.dimensionality, + parameter_assignment: m.parameter_assignment, + parameter_assignment_details: m.parameter_assignment_details, + calibration_target_variable: m.calibration_target_variable, + spatial_grid_resolution: m.spatial_grid_resolution, + spatial_grid_type: m.spatial_grid_type, + output_time_interval: m.output_time_interval, + code_url: m.code_url, + usage_notes: m.usage_notes, + software_image: m.software_image, + inputs: { + data: m.input_files.map((input) => modelInputOutputToGQL(input)), + on_conflict: { + constraint: "model_input_pkey", + update_columns: ["model_id"] } }, - "parameters": { - "data": m.input_parameters.map((param) => modelParameterToGQL(param)), - "on_conflict": { - "constraint": "model_parameter_pkey", - "update_columns": ["model_id"] + parameters: { + data: m.input_parameters.map((param) => modelParameterToGQL(param)), + on_conflict: { + constraint: "model_parameter_pkey", + update_columns: ["model_id"] } }, - "outputs": { - "data": m.output_files.map((output) => modelInputOutputToGQL(output)), - "on_conflict": { - "constraint": "model_output_pkey", - "update_columns": ["model_id"] + outputs: { + data: m.output_files.map((output) => modelInputOutputToGQL(output)), + on_conflict: { + constraint: "model_output_pkey", + update_columns: ["model_id"] } } }; -} +}; export const getAutoID = () => { - const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - let autoId = '' + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let autoId = ""; for (let i = 0; i < 20; i++) { - autoId += chars.charAt(Math.floor(Math.random() * chars.length)) + autoId += chars.charAt(Math.floor(Math.random() * chars.length)); } return autoId; -} +}; const getMd5Hash = (str2hash) => { - return crypto.createHash('md5').update(str2hash).digest("hex"); -} + return crypto.createHash("md5").update(str2hash).digest("hex"); +}; const getModelIOFixedBindings = (io) => { - let fixed_bindings_data = [] + const fixed_bindings_data = []; if ("value" in io && "resources" in io["value"]) { io["value"]["resources"].forEach((res: any) => { - if (!("name" in res)) - res["name"] = res["url"].replace(/^.*\/(.*?)$/, "$1"); + if (!("name" in res)) res["name"] = res["url"].replace(/^.*\/(.*?)$/, "$1"); fixed_bindings_data.push({ - "resource": { - "data": { - "id": getMd5Hash(res["url"]), - "name": res["name"], - "url": res["url"] + resource: { + data: { + id: getMd5Hash(res["url"]), + name: res["name"], + url: res["url"] }, - "on_conflict": { - "constraint": "resource_pkey", - "update_columns": ["name"] + on_conflict: { + constraint: "resource_pkey", + update_columns: ["name"] } } - }) + }); }); } return { - "data": fixed_bindings_data, - "on_conflict": { - "constraint": "model_input_bindings_pkey", - "update_columns": ["resource_id"] + data: fixed_bindings_data, + on_conflict: { + constraint: "model_input_bindings_pkey", + update_columns: ["resource_id"] } - } -} + }; +}; const getVariableData = (variableid) => { return { - "data": { - "id": variableid + data: { + id: variableid }, - "on_conflict": { - "constraint": "variable_pkey", - "update_columns": ["description"] + on_conflict: { + constraint: "variable_pkey", + update_columns: ["description"] } - } -} + }; +}; const modelIOToGQL = (io: any) => { - let fixed_bindings = getModelIOFixedBindings(io) + const fixed_bindings = getModelIOFixedBindings(io); return { - "id": io["id"], - "name": io["name"], - "type": io["type"], - "format": io["format"], - "fixed_bindings": fixed_bindings, - "variables": { - "data": io["variables"].map((v) => { - return { - "variable": getVariableData(v) + id: io["id"], + name: io["name"], + type: io["type"], + format: io["format"], + fixed_bindings: fixed_bindings, + variables: { + data: io["variables"].map((v) => { + return { + variable: getVariableData(v) }; }), - "on_conflict": { - "constraint": "model_io_variable_pkey", - "update_columns": ["variable_id"] + on_conflict: { + constraint: "model_io_variable_pkey", + update_columns: ["variable_id"] } } - } -} + }; +}; const modelInputOutputToGQL = (io: any) => { return { - "position": io["position"], - "model_io": { - "data": modelIOToGQL(io), - "on_conflict": { - "constraint": "model_io_pkey", - "update_columns": ["id"] + position: io["position"], + model_io: { + data: modelIOToGQL(io), + on_conflict: { + constraint: "model_io_pkey", + update_columns: ["id"] } } - } -} + }; +}; const modelParameterToGQL = (input: ModelParameter) => { - if ("default" in input && input["default"]) - input["default"] = input["default"] + ""; - if ("value" in input && input["value"]) - input["fixed_value"] = input["value"] + ""; - delete input["value"] - return input -} + if ("default" in input && input["default"]) input["default"] = input["default"] + ""; + if ("value" in input && input["value"]) input["fixed_value"] = input["value"] + ""; + delete input["value"]; + return input; +}; const getModelDataBindings = (model, model_ensemble: ThreadModelMap) => { - let dataBindings = [] + const dataBindings = []; model["input_files"].forEach((ifile) => { - let inputid = ifile["id"] + const inputid = ifile["id"]; if (inputid in model_ensemble.bindings) { model_ensemble.bindings[inputid].forEach((sliceid) => { dataBindings.push({ - "thread_model_id": model_ensemble.id, - "model_io_id": inputid, - "dataslice_id": sliceid - }) + thread_model_id: model_ensemble.id, + model_io_id: inputid, + dataslice_id: sliceid + }); }); } }); return dataBindings; -} +}; const getModelParameterBindings = (model, model_ensemble: ThreadModelMap) => { - let parameterBindings = []; + const parameterBindings = []; model["input_parameters"].forEach((iparam) => { - let inputid = iparam["id"] + const inputid = iparam["id"]; if (inputid in model_ensemble.bindings) { model_ensemble.bindings[inputid].forEach((paramvalue) => { parameterBindings.push({ - "thread_model_id": model_ensemble.id, - "model_parameter_id": inputid, - "parameter_value": paramvalue + "" + thread_model_id: model_ensemble.id, + model_parameter_id: inputid, + parameter_value: paramvalue + "" }); }); } }); - return parameterBindings -} + return parameterBindings; +}; const getSpatialCoverageGeometry = (coverage) => { - if(!coverage) - return null; - let value = coverage["value"] + if (!coverage) return null; + const value = coverage["value"]; if (coverage["type"] == "Point") { return { - "type": "Point", - "coordinates": [ - parseFloat(value["x"]), parseFloat(value["y"]) - ] - } + type: "Point", + coordinates: [parseFloat(value["x"]), parseFloat(value["y"])] + }; } if (coverage["type"] == "BoundingBox") { return { - "type": "Polygon", - "coordinates": [ + type: "Polygon", + coordinates: [ [ - [ parseFloat(value["xmin"]), parseFloat(value["ymin"]) ], - [ parseFloat(value["xmax"]), parseFloat(value["ymin"]) ], - [ parseFloat(value["xmax"]), parseFloat(value["ymax"]) ], - [ parseFloat(value["xmin"]), parseFloat(value["ymax"]) ], - [ parseFloat(value["xmin"]), parseFloat(value["ymin"]) ] + [parseFloat(value["xmin"]), parseFloat(value["ymin"])], + [parseFloat(value["xmax"]), parseFloat(value["ymin"])], + [parseFloat(value["xmax"]), parseFloat(value["ymax"])], + [parseFloat(value["xmin"]), parseFloat(value["ymax"])], + [parseFloat(value["xmin"]), parseFloat(value["ymin"])] ] - ] - } + ] + }; } -} - +}; const getDates = (dates) => { - let start = dates["start_date"] - let end = dates["end_date"] + const start = dates["start_date"]; + const end = dates["end_date"]; return { - "start_date" : toDateString(start), - "end_date": toDateString(end) - } -} + start_date: toDateString(start), + end_date: toDateString(end) + }; +}; const getResourceData = (data) => { - let dates = getDates(data["time_period"]) + const dates = getDates(data["time_period"]); return { - "data": { - "id": getMd5Hash(data["url"]), - "dcid": data["id"], - "name": data["name"], - "spatial_coverage": getSpatialCoverageGeometry(data["spatial_coverage"]), - "start_date": dates?.start_date, - "end_date": dates?.end_date, - "url": data["url"] + data: { + id: getMd5Hash(data["url"]), + dcid: data["id"], + name: data["name"], + spatial_coverage: getSpatialCoverageGeometry(data["spatial_coverage"]), + start_date: dates?.start_date, + end_date: dates?.end_date, + url: data["url"] }, - "on_conflict": { - "constraint": "resource_pkey", - "update_columns": ["name"] + on_conflict: { + constraint: "resource_pkey", + update_columns: ["name"] } - } -} + }; +}; const getDatasliceResourceData = (data) => { return { - "resource": getResourceData(data), - "selected": data["selected"] ?? false - } -} + resource: getResourceData(data), + selected: data["selected"] ?? false + }; +}; const getDatasliceData = (data: Dataslice, thread: Thread) => { - let dsname = data.name; - let threadname = thread.name; + const dsname = data.name; + const threadname = thread.name; - let slicename = dsname + " for thread: " + threadname; - let sliceid = data["id"] ?? uuidv4(); // Change to using md5 hash of sorted resource ids + const slicename = dsname + " for thread: " + threadname; + const sliceid = data["id"] ?? uuidv4(); // Change to using md5 hash of sorted resource ids return { - "id": sliceid, - "name": slicename, - "region_id": thread.regionid, - "start_date": toDateString(thread.dates?.start_date), - "end_date": toDateString(thread.dates?.end_date), - "resource_count": data.dataset.resource_count, - "dataset": { - "data": { - "id": data.dataset.id, - "name": dsname, + id: sliceid, + name: slicename, + region_id: thread.regionid, + start_date: toDateString(thread.dates?.start_date), + end_date: toDateString(thread.dates?.end_date), + resource_count: data.dataset.resource_count, + dataset: { + data: { + id: data.dataset.id, + name: dsname }, - "on_conflict": { - "constraint": "dataset_pkey", - "update_columns": ["name"] + on_conflict: { + constraint: "dataset_pkey", + update_columns: ["name"] } }, - "resources": { - "data": data.resources.map((res) => getDatasliceResourceData(res)), - "on_conflict": { - "constraint": "dataslice_resource_pkey", - "update_columns": ["dataslice_id"] + resources: { + data: data.resources.map((res) => getDatasliceResourceData(res)), + on_conflict: { + constraint: "dataslice_resource_pkey", + update_columns: ["dataslice_id"] } } - } -} + }; +}; const getThreadDataslice = (data: Dataslice, thread: Thread) => { return { - "thread_id": thread.id, - "dataslice": { - "data": getDatasliceData(data, thread), - "on_conflict": { - "constraint": "dataslice_pkey", - "update_columns": ["id"] + thread_id: thread.id, + dataslice: { + data: getDatasliceData(data, thread), + on_conflict: { + constraint: "dataslice_pkey", + update_columns: ["id"] } } - } -} - -export const threadDataBindingsToGQL = (data: DataMap, - model_ensemble: ModelEnsembleMap, thread: Thread) => { - let dataslices = [] + }; +}; + +export const threadDataBindingsToGQL = ( + data: DataMap, + model_ensemble: ModelEnsembleMap, + thread: Thread +) => { + const dataslices = []; Object.keys(data).map((sliceid) => { - let dataslice = getThreadDataslice(data[sliceid], thread) + const dataslice = getThreadDataslice(data[sliceid], thread); dataslices.push(dataslice); }); let thread_model_io = []; Object.keys(model_ensemble).forEach((modelid) => { - let model = thread.models[modelid]; - let tmio = getModelDataBindings(model, model_ensemble[modelid]); + const model = thread.models[modelid]; + const tmio = getModelDataBindings(model, model_ensemble[modelid]); thread_model_io = thread_model_io.concat(tmio); - }) - + }); + return { data: dataslices, model_io: thread_model_io - } -} + }; +}; -export const threadParameterBindingsToGQL = ( - model_ensemble: ModelEnsembleMap, thread: Thread) => { +export const threadParameterBindingsToGQL = (model_ensemble: ModelEnsembleMap, thread: Thread) => { let thread_model_params = []; Object.keys(model_ensemble).forEach((modelid) => { - let model = thread.models[modelid]; - let tmparams = getModelParameterBindings(model, model_ensemble[modelid]); + const model = thread.models[modelid]; + const tmparams = getModelParameterBindings(model, model_ensemble[modelid]); thread_model_params = thread_model_params.concat(tmparams); - }) + }); return thread_model_params; -} +}; export const executionResultsToGQL = (results: any) => { - let data: any = []; + const data: any = []; Object.keys(results).forEach((outid) => { - let result = results[outid]; + const result = results[outid]; data.push({ model_io_id: outid, resource: getResourceData(result) }); - }) + }); return data; -} \ No newline at end of file +}; diff --git a/src/classes/graphql/graphql_functions.ts b/src/classes/graphql/graphql_functions.ts index 1f633f7..2f768a6 100644 --- a/src/classes/graphql/graphql_functions.ts +++ b/src/classes/graphql/graphql_functions.ts @@ -1,52 +1,71 @@ import { - ProblemStatement, Thread, Task, Model, DataResource, - Execution, ExecutionSummary, MintPreferences, ModelIOBindings, ThreadModelMap, ProblemStatementInfo, ThreadInfo, DataMap, ModelEnsembleMap, IdMap, Region, BoundingBox } from '../mint/mint-types'; -import { ModelConfigurationSetup } from '@mintproject/modelcatalog_client'; - -import { GraphQL } from '../../config/graphql'; - -import getProblemStatementGQL from './queries/problem-statement/get.graphql'; -import getTaskGQL from './queries/task/get.graphql'; -import getThreadGQL from './queries/thread/get.graphql'; - -import newProblemStatementGQL from './queries/problem-statement/new.graphql'; -import newTaskGQL from './queries/task/new.graphql'; -import newThreadGQL from './queries/thread/new.graphql'; - -import updateProblemStatementGQL from './queries/problem-statement/update.graphql'; -import updateTaskGQL from './queries/task/update.graphql'; -import updateThreadModelGQL from './queries/thread/update-models.graphql'; -import updateThreadDataGQL from './queries/thread/update-datasets.graphql'; -import updateThreadParametersGQL from './queries/thread/update-parameters.graphql'; -import updateThreadInfoGQL from './queries/thread/update-info.graphql'; - -import getExecutionGQL from './queries/execution/get.graphql'; -import listSuccessfulIdsGQL from './queries/execution/list-successful-ids.graphql'; -import listExistingIdStatusGQL from './queries/execution/list-existing-ids.graphql'; -import getExecutionsGQL from './queries/execution/list.graphql'; -import setExecutionsGQL from './queries/execution/new.graphql'; -import updateExecutionStatusResultsGQL from './queries/execution/update-status-results.graphql'; -import updateExecutionStatusGQL from './queries/execution/update-status-results.graphql'; -import deleteExecutionsGQL from './queries/execution/delete.graphql'; - -import getRegionDetailsGQL from './queries/region/get.graphql'; - -import updateExecutionSummary from './queries/execution/update-execution-summary.graphql'; -import incFailedRunsGQL from './queries/execution/increment-failed-runs.graphql'; -import incSuccessfulRunsGQL from './queries/execution/increment-successful-runs.graphql'; -import incSubmittedRunsGQL from './queries/execution/increment-submitted-runs.graphql'; -import incRegisteredRunsGQL from './queries/execution/increment-registered-runs.graphql'; - -import listThreadModelExecutionIdsGQL from './queries/execution/list-thread-model-executions.graphql'; -import newThreadModelExecutionsGQL from './queries/execution/new-thread-model-executions.graphql'; -import deleteThreadModelExecutionsGQL from './queries/execution/delete-thread-model-executions.graphql'; - -import getModelGQL from './queries/model/get.graphql'; -import deleteModelGQL from './queries/model/delete.graphql'; - -import { problemStatementFromGQL, taskFromGQL, threadFromGQL, + ProblemStatement, + Thread, + Task, + Model, + DataResource, + Execution, + ExecutionSummary, + MintPreferences, + ModelIOBindings, + ThreadModelMap, + ProblemStatementInfo, + ThreadInfo, + DataMap, + ModelEnsembleMap, + IdMap, + Region, + BoundingBox +} from "../mint/mint-types"; +import { ModelConfigurationSetup } from "@mintproject/modelcatalog_client"; + +import { GraphQL } from "../../config/graphql"; + +import getProblemStatementGQL from "./queries/problem-statement/get.graphql"; +import getTaskGQL from "./queries/task/get.graphql"; +import getThreadGQL from "./queries/thread/get.graphql"; + +import newProblemStatementGQL from "./queries/problem-statement/new.graphql"; +import newTaskGQL from "./queries/task/new.graphql"; +import newThreadGQL from "./queries/thread/new.graphql"; + +import updateProblemStatementGQL from "./queries/problem-statement/update.graphql"; +import updateTaskGQL from "./queries/task/update.graphql"; +import updateThreadModelGQL from "./queries/thread/update-models.graphql"; +import updateThreadDataGQL from "./queries/thread/update-datasets.graphql"; +import updateThreadParametersGQL from "./queries/thread/update-parameters.graphql"; +import updateThreadInfoGQL from "./queries/thread/update-info.graphql"; + +import getExecutionGQL from "./queries/execution/get.graphql"; +import listSuccessfulIdsGQL from "./queries/execution/list-successful-ids.graphql"; +import listExistingIdStatusGQL from "./queries/execution/list-existing-ids.graphql"; +import getExecutionsGQL from "./queries/execution/list.graphql"; +import setExecutionsGQL from "./queries/execution/new.graphql"; +import updateExecutionStatusResultsGQL from "./queries/execution/update-status-results.graphql"; +import updateExecutionStatusGQL from "./queries/execution/update-status-results.graphql"; +import deleteExecutionsGQL from "./queries/execution/delete.graphql"; + +import getRegionDetailsGQL from "./queries/region/get.graphql"; + +import updateExecutionSummary from "./queries/execution/update-execution-summary.graphql"; +import incFailedRunsGQL from "./queries/execution/increment-failed-runs.graphql"; +import incSuccessfulRunsGQL from "./queries/execution/increment-successful-runs.graphql"; +import incSubmittedRunsGQL from "./queries/execution/increment-submitted-runs.graphql"; +import incRegisteredRunsGQL from "./queries/execution/increment-registered-runs.graphql"; + +import listThreadModelExecutionIdsGQL from "./queries/execution/list-thread-model-executions.graphql"; +import newThreadModelExecutionsGQL from "./queries/execution/new-thread-model-executions.graphql"; +import deleteThreadModelExecutionsGQL from "./queries/execution/delete-thread-model-executions.graphql"; + +import getModelGQL from "./queries/model/get.graphql"; +import deleteModelGQL from "./queries/model/delete.graphql"; + +import { + problemStatementFromGQL, + taskFromGQL, + threadFromGQL, executionFromGQL, - executionToGQL, + executionToGQL, problemStatementToGQL, taskToGQL, threadInfoToGQL, @@ -59,160 +78,168 @@ import { problemStatementFromGQL, taskFromGQL, threadFromGQL, executionResultsToGQL, regionFromGQL, eventToGQL, - modelFromGQL} from './graphql_adapter'; -import { Md5 } from 'ts-md5'; + modelFromGQL +} from "./graphql_adapter"; +import { Md5 } from "ts-md5"; import { KeycloakAdapter } from "../../config/keycloak-adapter"; - -export const getProblemStatement = async(problem_statement_id: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getProblemStatement = async ( + problem_statement_id: string +): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getProblemStatementGQL, variables: { id: problem_statement_id } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { - console.log("ERROR"); - console.log(result); - } - else { - let problem = result.data.problem_statement_by_pk; - if(problem) { - return problemStatementFromGQL(problem); + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const problem = result.data.problem_statement_by_pk; + if (problem) { + return problemStatementFromGQL(problem); + } } - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }); +}; -export const getTask = async(taskid: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getTask = async (taskid: string): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getTaskGQL, variables: { id: taskid } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { - console.log("ERROR"); - console.log(result); - } - else { - let task = result.data.task_by_pk; - if(task) { - return taskFromGQL(task); + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const task = result.data.task_by_pk; + if (task) { + return taskFromGQL(task); + } } - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }); +}; -export const getThread = async(threadid: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getThread = async (threadid: string): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getThreadGQL, - fetchPolicy: 'no-cache', + fetchPolicy: "no-cache", variables: { id: threadid } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { - console.log("ERROR"); - console.log(result); - } - else { - let thread = result.data.thread_by_pk; - if(thread) { - return threadFromGQL(thread); + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const thread = result.data.thread_by_pk; + if (thread) { + return threadFromGQL(thread); + } } - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }); +}; -export const getModel = async(modelid: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getModel = async (modelid: string): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getModelGQL, - fetchPolicy: 'no-cache', + fetchPolicy: "no-cache", variables: { id: modelid } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { - console.log("ERROR"); - console.log(result); - } - else { - let model = result.data.model_by_pk; - if(model) { - return modelFromGQL(model); + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const model = result.data.model_by_pk; + if (model) { + return modelFromGQL(model); + } } - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }); +}; const MAX_CONFIGURATIONS = 1000000; -export const getTotalConfigurations = (model: ModelConfigurationSetup, bindings: ModelIOBindings, data: DataMap) => { +export const getTotalConfigurations = ( + model: ModelConfigurationSetup, + bindings: ModelIOBindings, + data: DataMap +) => { let totalconfigs = 1; model.hasInput.map((io) => { - if(!io.hasFixedResource || io.hasFixedResource.length == 0) { + if (!io.hasFixedResource || io.hasFixedResource.length == 0) { // Expand a dataset to it's constituent resources // FIXME: Create a collection if the model input has dimensionality of 1 - if(bindings[io.id]) { - let nexecution : any[] = []; + if (bindings[io.id]) { + let nexecution: any[] = []; bindings[io.id].map((dsid) => { - let ds = data[dsid]; + const ds = data[dsid]; let selected_resources = ds.resources.filter((res) => res.selected); // Fix for older saved resources - if(!selected_resources || selected_resources.length == 0) + if (!selected_resources || selected_resources.length == 0) selected_resources = ds.resources; nexecution = nexecution.concat(selected_resources); }); totalconfigs *= nexecution.length; } - } - else { + } else { totalconfigs *= (io.hasFixedResource as any[]).length; } - }) - + }); + // Add adjustable parameters to the input ids model.hasParameter.map((io) => { - if(!io.hasFixedValue || io.hasFixedValue.length == 0) - totalconfigs *= (bindings[io.id]?.length ?? 1); + if (!io.hasFixedValue || io.hasFixedValue.length == 0) + totalconfigs *= bindings[io.id]?.length ?? 1; }); return totalconfigs; -} +}; -const cartProd = (lists : any[]) => { - let ps : any[] = [], - acc : any [][] = [ - [] - ], +const cartProd = (lists: any[]) => { + let ps: any[] = [], + acc: any[][] = [[]], i = lists.length; while (i--) { let subList = lists[i], @@ -220,222 +247,228 @@ const cartProd = (lists : any[]) => { while (j--) { let x = subList[j], k = acc.length; - while (k--) ps.push([x].concat(acc[k])) - }; + while (k--) ps.push([x].concat(acc[k])); + } acc = ps; ps = []; - }; + } return acc.reverse(); }; -export const getModelInputConfigurations = ( - threadModel: ThreadModelMap, - inputIds: string[]) => { - let dataBindings = threadModel.bindings; - let inputBindings : any[] = []; +export const getModelInputConfigurations = (threadModel: ThreadModelMap, inputIds: string[]) => { + const dataBindings = threadModel.bindings; + const inputBindings: any[] = []; let totalproducts = 1; inputIds.map((inputid) => { inputBindings.push(dataBindings[inputid]); - if(dataBindings[inputid]) - totalproducts *= dataBindings[inputid].length; + if (dataBindings[inputid]) totalproducts *= dataBindings[inputid].length; }); - if(totalproducts < MAX_CONFIGURATIONS) { + if (totalproducts < MAX_CONFIGURATIONS) { return cartProd(inputBindings); - } - else { + } else { return null; } -} +}; export const getModelInputBindings = (model: Model, thread: Thread, region: Region) => { - let me = thread.model_ensembles[model.id]; - let threadModel = { + const me = thread.model_ensembles[model.id]; + const threadModel = { id: me.id, bindings: Object.assign({}, me.bindings) } as ThreadModelMap; - let inputIds : any[] = []; + const inputIds: any[] = []; model.input_files.map((io) => { inputIds.push(io.id); - if(!io.value) { + if (!io.value) { // Expand a dataset to it's constituent "selected" resources // FIXME: Create a collection if the model input has dimensionality of 1 - if(threadModel.bindings[io.id]) { - let nexecution : any[] = []; + if (threadModel.bindings[io.id]) { + let nexecution: any[] = []; threadModel.bindings[io.id].map((dsid) => { - let ds = thread.data[dsid]; + const ds = thread.data[dsid]; let selected_resources = ds.resources.filter((res) => res.selected); // Fix for older saved resources - if(selected_resources.length == 0) - selected_resources = ds.resources; + if (selected_resources.length == 0) selected_resources = ds.resources; nexecution = nexecution.concat(selected_resources); }); threadModel.bindings[io.id] = nexecution; } - } - else { + } else { threadModel.bindings[io.id] = io.value.resources as any[]; } - }) - + }); + // Add adjustable parameters to the input ids model.input_parameters.map((io) => { inputIds.push(io.id); - if(io.value) { + if (io.value) { // If this is a non-adjustable parameter, set the binding value to the fixed value threadModel.bindings[io.id] = [io.value]; } - + // HACK: Add region id to __region_geojson (Not replacing ) - if(threadModel.bindings[io.id] && threadModel.bindings[io.id][0] == "__region_geojson") { - threadModel.bindings[io.id] = ["__region_geojson:"+region.id]; + if (threadModel.bindings[io.id] && threadModel.bindings[io.id][0] == "__region_geojson") { + threadModel.bindings[io.id] = ["__region_geojson:" + region.id]; } - }) + }); return [threadModel, inputIds]; }; - export const deleteModel = async (model_id: string) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: deleteModelGQL, variables: { id: model_id } }); -} +}; -/* +/* Executions */ -export const getExecution = async(executionid: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getExecution = async (executionid: string): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getExecutionGQL, variables: { id: executionid } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { - console.log("ERROR"); - console.log(result); - } - else { - let execution = result.data.execution_by_pk; - if(execution) { - return executionFromGQL(execution); + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const execution = result.data.execution_by_pk; + if (execution) { + return executionFromGQL(execution); + } } - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + return null; + }) + .catch((e) => { + console.log("ERROR"); + console.log(e); + return null; + }); +}; // Get Executions -export const getExecutions = (executionids: string[]) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getExecutions = (executionids: string[]): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: getExecutionsGQL, variables: { ids: executionids } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.execution.map((ex: any) => executionFromGQL(ex)); + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.execution.map((ex:any) => executionFromGQL(ex)); - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; -export const getMatchingExecution = (executions: Execution[], execution: Execution, hashes: string[]) => { - let hash = getExecutionHash(execution); - let index = hashes.indexOf(hash); - if(index >= 0) { +export const getMatchingExecution = ( + executions: Execution[], + execution: Execution, + hashes: string[] +) => { + const hash = getExecutionHash(execution); + const index = hashes.indexOf(hash); + if (index >= 0) { return executions[index]; } return null; -} +}; -export const getExecutionHash = (execution: Execution) : string => { +export const getExecutionHash = (execution: Execution): string => { let str = execution.modelid; - let varids = Object.keys(execution.bindings).sort(); + const varids = Object.keys(execution.bindings).sort(); varids.map((varid) => { - let binding = execution.bindings[varid]; - let bindingid = (binding !== null && typeof binding === 'object') ? (binding as DataResource).id : binding; + const binding = execution.bindings[varid]; + const bindingid = + binding !== null && typeof binding === "object" + ? (binding as DataResource).id + : binding; str += varid + "=" + bindingid + "&"; - }) + }); return Md5.hashStr(str).toString(); -} +}; // List Existing Execution Ids -export const listExistingExecutionIdStatus = (executionids: string[]) : Promise> => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const listExistingExecutionIdStatus = ( + executionids: string[] +): Promise> => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: listExistingIdStatusGQL, variables: { ids: executionids } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + const idstatus = {}; + result.data.execution.forEach((ex: any) => { + idstatus[ex["id"].replace(/-/g, "")] = ex["status"]; + }); + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - let idstatus = {} - result.data.execution.forEach((ex:any) => { - idstatus[ex["id"].replace(/-/g, "")] = ex["status"]; - }); - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; // List Successful Execution Ids -export const listSuccessfulExecutionIds = (executionids: string[]) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const listSuccessfulExecutionIds = (executionids: string[]): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: listSuccessfulIdsGQL, variables: { ids: executionids } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.execution.map((ex: any) => ex["id"].replace(/-/g, "")); + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.execution.map((ex:any) => ex["id"].replace(/-/g, "")); - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; // Update Executions export const setExecutions = (executions: Execution[], thread_model_id: string) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let exobjs = executions.map((ex) => executionToGQL(ex)) - let exids = executions.map((ex) => ex.id) + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const exobjs = executions.map((ex) => executionToGQL(ex)); + const exids = executions.map((ex) => ex.id); return APOLLO_CLIENT.mutate({ mutation: setExecutionsGQL, variables: { @@ -447,11 +480,11 @@ export const setExecutions = (executions: Execution[], thread_model_id: string) console.log("ERROR"); console.log(e); }); -} +}; // Update Execution status and results only export const updateExecutionStatusAndResults = (execution: Execution) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: updateExecutionStatusResultsGQL, variables: { @@ -460,17 +493,17 @@ export const updateExecutionStatusAndResults = (execution: Execution) => { end_time: execution.end_time, run_progress: execution.run_progress, status: execution.status, - results: executionResultsToGQL(execution.results).map((exres:any) => { + results: executionResultsToGQL(execution.results).map((exres: any) => { exres["execution_id"] = execution.id; return exres; }) } }); -} +}; // Update Execution status only export const updateExecutionStatus = (execution: Execution) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: updateExecutionStatusGQL, variables: { @@ -480,53 +513,53 @@ export const updateExecutionStatus = (execution: Execution) => { status: execution.status } }); -} +}; // Delete Executions export const deleteExecutions = (executionids: string[]) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: deleteExecutionsGQL, variables: { ids: executionids } }); -} - +}; -/* - Thread Model Execution Mappings +/* + Thread Model Execution Mappings */ -export const getThreadModelExecutionIds = async (thread_model_id: string) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const getThreadModelExecutionIds = async (thread_model_id: string): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.query({ query: listThreadModelExecutionIdsGQL, variables: { threadModelId: thread_model_id } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.thread_model_by_pk.executions.map((ex: any) => ex.execution_id); + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.thread_model_by_pk.executions.map((ex:any) => ex.execution_id); - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); -} + console.log(e); + return null; + }); +}; export const setThreadModelExecutionIds = (thread_model_id: string, executionids: string[]) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let tmexids = executionids.map((exid) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const tmexids = executionids.map((exid) => { return { thread_model_id: thread_model_id, execution_id: exid - } + }; }); return APOLLO_CLIENT.mutate({ mutation: newThreadModelExecutionsGQL, @@ -534,23 +567,26 @@ export const setThreadModelExecutionIds = (thread_model_id: string, executionids threadModelExecutions: tmexids } }); -} +}; export const deleteThreadModelExecutionIds = async (thread_model_id: string) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: deleteThreadModelExecutionsGQL, variables: { threadModelId: thread_model_id } }); -} +}; -/* - Thread Model Execution Summaries +/* + Thread Model Execution Summaries */ -export const setThreadModelExecutionSummary = (thread_model_id: string, summary: ExecutionSummary) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const setThreadModelExecutionSummary = ( + thread_model_id: string, + summary: ExecutionSummary +) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: updateExecutionSummary, variables: { @@ -561,8 +597,8 @@ export const setThreadModelExecutionSummary = (thread_model_id: string, summary: }; // Increment thread submitted runs -export const incrementThreadModelSubmittedRuns = (thread_model_id: string, num: number = 1) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const incrementThreadModelSubmittedRuns = (thread_model_id: string, num: number = 1) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: incSubmittedRunsGQL, variables: { @@ -573,8 +609,8 @@ export const incrementThreadModelSubmittedRuns = (thread_model_id: string, num: }; // Increment thread successful runs -export const incrementThreadModelSuccessfulRuns = (thread_model_id: string, num: number = 1) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const incrementThreadModelSuccessfulRuns = (thread_model_id: string, num: number = 1) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: incSuccessfulRunsGQL, variables: { @@ -585,8 +621,8 @@ export const incrementThreadModelSuccessfulRuns = (thread_model_id: string, num: }; // Increment thread failed runs -export const incrementThreadModelFailedRuns = (thread_model_id: string, num: number = 1) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const incrementThreadModelFailedRuns = (thread_model_id: string, num: number = 1) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: incFailedRunsGQL, variables: { @@ -597,8 +633,8 @@ export const incrementThreadModelFailedRuns = (thread_model_id: string, num: num }; // Increment thread outputs registered runs -export const incrementThreadModelRegisteredRuns = (thread_model_id: string, num: number = 1) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); +export const incrementThreadModelRegisteredRuns = (thread_model_id: string, num: number = 1) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: incRegisteredRunsGQL, variables: { @@ -610,118 +646,125 @@ export const incrementThreadModelRegisteredRuns = (thread_model_id: string, num: /* Update Functions */ // Add ProblemStatement -export const addProblemStatement = (problem_statement:ProblemStatementInfo) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let problemobj = problemStatementToGQL(problem_statement); +export const addProblemStatement = (problem_statement: ProblemStatementInfo): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const problemobj = problemStatementToGQL(problem_statement); return APOLLO_CLIENT.mutate({ mutation: newProblemStatementGQL, variables: { object: problemobj } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.insert_problem_statement.returning[0].id; + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.insert_problem_statement.returning[0].id; - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; // Add Task -export const addTask = (problem_statement: ProblemStatementInfo, task: Task) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let taskobj = taskToGQL(task, problem_statement); +export const addTask = (problem_statement: ProblemStatementInfo, task: Task): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const taskobj = taskToGQL(task, problem_statement); return APOLLO_CLIENT.mutate({ mutation: newTaskGQL, variables: { object: taskobj } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.insert_task.returning[0].id; + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.insert_task.returning[0].id; - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; // Add Task -export const addTaskWithThread = (problem_statement: ProblemStatementInfo, task: Task, thread: ThreadInfo) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let taskobj = taskToGQL(task, problem_statement); - let threadobj = threadInfoToGQL(thread, task.id, task.regionid); +export const addTaskWithThread = ( + problem_statement: ProblemStatementInfo, + task: Task, + thread: ThreadInfo +): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const taskobj = taskToGQL(task, problem_statement); + const threadobj = threadInfoToGQL(thread, task.id, task.regionid); taskobj["threads"] = { data: [threadobj] - } + }; return APOLLO_CLIENT.mutate({ mutation: newTaskGQL, variables: { object: taskobj } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return [ + result.data.insert_task.returning[0].id, + result.data.insert_task.returning[0].threads[0].id + ]; + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return [ - result.data.insert_task.returning[0].id, - result.data.insert_task.returning[0].threads[0].id - ]; - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; // Add Thread -export const addThread = (task:Task, thread: ThreadInfo) : Promise => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let threadobj = threadInfoToGQL(thread, task.id, task.regionid); +export const addThread = (task: Task, thread: ThreadInfo): Promise => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const threadobj = threadInfoToGQL(thread, task.id, task.regionid); //console.log(threadobj); return APOLLO_CLIENT.mutate({ mutation: newThreadGQL, variables: { object: threadobj } - }).then((result) => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + } else { + return result.data.insert_thread.returning[0].id; + } + return null; + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - } - else { - return result.data.insert_thread.returning[0].id; - } - return null; - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }; - // Update ProblemStatement -export const updateProblemStatement = (problem_statement: ProblemStatementInfo) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let problemobj = problemStatementUpdateToGQL(problem_statement); +export const updateProblemStatement = (problem_statement: ProblemStatementInfo) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const problemobj = problemStatementUpdateToGQL(problem_statement); return APOLLO_CLIENT.mutate({ mutation: updateProblemStatementGQL, variables: { @@ -731,9 +774,9 @@ export const updateProblemStatement = (problem_statement: ProblemStatementInfo) }; // Update Task -export const updateTask = (task: Task) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let taskobj = taskUpdateToGQL(task); +export const updateTask = (task: Task) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const taskobj = taskUpdateToGQL(task); return APOLLO_CLIENT.mutate({ mutation: updateTaskGQL, variables: { @@ -743,26 +786,30 @@ export const updateTask = (task: Task) => { }; export const updateThreadInformation = (threadinfo: ThreadInfo) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let threadobj = threadInfoUpdateToGQL(threadinfo); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const threadobj = threadInfoUpdateToGQL(threadinfo); return APOLLO_CLIENT.mutate({ mutation: updateThreadInfoGQL, variables: { object: threadobj } }); -} +}; -export const setThreadModels = (models: ModelConfigurationSetup[], notes: string, thread: Thread) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let threadmodelsobj = models.map((model) => { +export const setThreadModels = ( + models: ModelConfigurationSetup[], + notes: string, + thread: Thread +) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const threadmodelsobj = models.map((model) => { return { model_id: model.id, thread_id: thread.id }; }); - let event = getCustomEvent("SELECT_MODELS", notes); - let eventobj = eventToGQL(event); + const event = getCustomEvent("SELECT_MODELS", notes); + const eventobj = eventToGQL(event); eventobj["thread_id"] = thread.id; return APOLLO_CLIENT.mutate({ mutation: updateThreadModelGQL, @@ -774,14 +821,18 @@ export const setThreadModels = (models: ModelConfigurationSetup[], notes: string }); }; -export const setThreadData = (datasets: DataMap, model_ensembles: ModelEnsembleMap, - notes: string, thread: Thread) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); - let bindings = threadDataBindingsToGQL(datasets, model_ensembles, thread); - let event = getCustomEvent("SELECT_DATA", notes); - let eventobj = eventToGQL(event); +export const setThreadData = ( + datasets: DataMap, + model_ensembles: ModelEnsembleMap, + notes: string, + thread: Thread +) => { + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const bindings = threadDataBindingsToGQL(datasets, model_ensembles, thread); + const event = getCustomEvent("SELECT_DATA", notes); + const eventobj = eventToGQL(event); eventobj["thread_id"] = thread.id; - + return APOLLO_CLIENT.mutate({ mutation: updateThreadDataGQL, variables: { @@ -793,20 +844,23 @@ export const setThreadData = (datasets: DataMap, model_ensembles: ModelEnsembleM }); }; -export const setThreadParameters = (model_ensembles: ModelEnsembleMap, - execution_summary: IdMap, - notes: string, thread: Thread) => { - let bindings = threadParameterBindingsToGQL(model_ensembles, thread); - let event = getCustomEvent("SELECT_PARAMETERS", notes); - let eventobj = eventToGQL(event); +export const setThreadParameters = ( + model_ensembles: ModelEnsembleMap, + execution_summary: IdMap, + notes: string, + thread: Thread +) => { + const bindings = threadParameterBindingsToGQL(model_ensembles, thread); + const event = getCustomEvent("SELECT_PARAMETERS", notes); + const eventobj = eventToGQL(event); eventobj["thread_id"] = thread.id; - let summaries = []; + const summaries = []; Object.keys(execution_summary).forEach((modelid) => { - let summary = execution_summary[modelid]; + const summary = execution_summary[modelid]; summary["thread_model_id"] = model_ensembles[modelid].id; summaries.push(summary); - }) - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + }); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return APOLLO_CLIENT.mutate({ mutation: updateThreadParametersGQL, variables: { @@ -820,58 +874,58 @@ export const setThreadParameters = (model_ensembles: ModelEnsembleMap, // Get details about a particular region/subregion export const getRegionDetails = (regionid: string) => { - let APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); + const APOLLO_CLIENT = GraphQL.instance(KeycloakAdapter.getUser()); return new Promise((resolve, reject) => { APOLLO_CLIENT.query({ query: getRegionDetailsGQL, variables: { id: regionid } - }).then(result => { - if(!result || (result.errors && result.errors.length > 0)) { + }) + .then((result) => { + if (!result || (result.errors && result.errors.length > 0)) { + console.log("ERROR"); + console.log(result); + reject(); + } else { + const region = regionFromGQL(result.data.region_by_pk); + region.bounding_box = _calculateBoundingBox(region.geometries); + resolve(region); + } + }) + .catch((e) => { console.log("ERROR"); - console.log(result); - reject(); - } - else { - let region = regionFromGQL(result.data.region_by_pk); - region.bounding_box = _calculateBoundingBox(region.geometries) - resolve(region); - } - }).catch((e) => { - console.log("ERROR"); - console.log(e); - return null; - }); + console.log(e); + return null; + }); }); }; const _calculateBoundingBox = (geometries: any[]) => { - var xmin=99999, ymin=99999, xmax=-99999, ymax=-99999; + let xmin = 99999, + ymin = 99999, + xmax = -99999, + ymax = -99999; geometries.forEach((geometry) => { let coords_list = geometry.coordinates; - if(geometry.type == "MultiPolygon") { + if (geometry.type == "MultiPolygon") { coords_list = coords_list.flat(1); } coords_list.map((coords: any) => { coords.map((c: any) => { - if(c[0] < xmin) - xmin = c[0]; - if(c[1] < ymin) - ymin = c[1]; - if(c[0] > xmax) - xmax = c[0]; - if(c[1] > ymax) - ymax = c[1]; - }) - }) + if (c[0] < xmin) xmin = c[0]; + if (c[1] < ymin) ymin = c[1]; + if (c[0] > xmax) xmax = c[0]; + if (c[1] > ymax) ymax = c[1]; + }); + }); }); return { - xmin: xmin-0.01, - ymin: ymin-0.01, - xmax: xmax+0.01, - ymax: ymax+0.01 + xmin: xmin - 0.01, + ymin: ymin - 0.01, + xmax: xmax + 0.01, + ymax: ymax + 0.01 } as BoundingBox; -} \ No newline at end of file +}; diff --git a/src/classes/localex/docker-functions.ts b/src/classes/localex/docker-functions.ts index f4a4c10..d0ffcca 100644 --- a/src/classes/localex/docker-functions.ts +++ b/src/classes/localex/docker-functions.ts @@ -1,22 +1,27 @@ -import Dockerode = require("dockerode") -import { imageExists, containerExec, pullImageAsync, waitForOutput } from 'dockerode-utils'; +import Dockerode = require("dockerode"); +import { imageExists, containerExec, pullImageAsync, waitForOutput } from "dockerode-utils"; -let dockerode = new Dockerode(); +const dockerode = new Dockerode(); -export const pullImage = async (image: string, version:string ="latest") => { - let exists = await imageExists(dockerode, image); - if(!exists) { - console.log("Pulling docker image: "+ image); +export const pullImage = async (image: string, version: string = "latest") => { + const exists = await imageExists(dockerode, image); + if (!exists) { + console.log("Pulling docker image: " + image); await pullImageAsync(dockerode, image); } -} +}; -export const runImage = async (cmd: string[], image: string, logstream:any, - workingDirectory: string, folderBindings: string[] ) => { +export const runImage = async ( + cmd: string[], + image: string, + logstream: any, + workingDirectory: string, + folderBindings: string[] +) => { return dockerode.run(image, cmd, logstream, { - 'WorkingDir': workingDirectory, - 'Binds': folderBindings, - 'EntryPoint': "", - 'User': "1000:1000" + WorkingDir: workingDirectory, + Binds: folderBindings, + EntryPoint: "", + User: "1000:1000" }); -} +}; diff --git a/src/classes/localex/local-execution-functions.ts b/src/classes/localex/local-execution-functions.ts index a69d616..145e686 100644 --- a/src/classes/localex/local-execution-functions.ts +++ b/src/classes/localex/local-execution-functions.ts @@ -1,5 +1,20 @@ -import { Thread, Execution, MintPreferences, DataResource, Model, ModelIO, ModelParameter, Wcm } from "../mint/mint-types"; -import { Component, ComponentSeed, ComponentParameterBindings, ComponentDataBindings, ComponentParameterTypes } from "./local-execution-types"; +import { + Thread, + Execution, + MintPreferences, + DataResource, + Model, + ModelIO, + ModelParameter, + Wcm +} from "../mint/mint-types"; +import { + Component, + ComponentSeed, + ComponentParameterBindings, + ComponentDataBindings, + ComponentParameterTypes +} from "./local-execution-types"; import path from "path"; import fs from "fs-extra"; @@ -14,10 +29,10 @@ import { Md5 } from "ts-md5"; import { getConfiguration } from "../mint/mint-functions"; import { Region } from "../mint/mint-types"; -let prefs = getConfiguration(); +const prefs = getConfiguration(); -let executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); -executionQueue.process(prefs.localex.parallelism, __dirname + '/execution.js'); +const executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); +executionQueue.process(prefs.localex.parallelism, __dirname + "/execution.js"); // You can listen to global events to get notified when jobs are processed /*executionQueue.on('global:completed', (jobId, result) => { @@ -27,19 +42,19 @@ executionQueue.process(prefs.localex.parallelism, __dirname + '/execution.js'); const _downloadFile = (url: string, filepath: string): Promise => { const file = fs.createWriteStream(filepath); return new Promise((resolve, reject) => { - request.get(url).on('response', (res) => { + request.get(url).on("response", (res) => { res.pipe(file); - res.on('end', function () { + res.on("end", function () { resolve(); }); }); }); -} +}; // TODO: Unzip the wcm zip file const _unzipFile = (zipfilename: string, dirname: string): Promise => { return new Promise((resolve, reject) => { - if (!fs.existsSync(dirname)){ + if (!fs.existsSync(dirname)) { fs.mkdirsSync(dirname); } yauzl.open(zipfilename, { lazyEntries: true }, function (err, zipfile) { @@ -56,8 +71,8 @@ const _unzipFile = (zipfilename: string, dirname: string): Promise => { return; } // If this is a directory, then copy its contents to dirname - if(entry.fileName.indexOf("/") >= 0) { - let filename = entry.fileName.substr(entry.fileName.indexOf("/") + 1); + if (entry.fileName.indexOf("/") >= 0) { + const filename = entry.fileName.substr(entry.fileName.indexOf("/") + 1); if (!filename) { zipfile.readEntry(); return; @@ -65,7 +80,7 @@ const _unzipFile = (zipfilename: string, dirname: string): Promise => { if (/\/$/.test(filename)) { // Directories zipfile.readEntry(); - if(!fs.existsSync(dirname + "/" + filename)) + if (!fs.existsSync(dirname + "/" + filename)) fs.mkdirsSync(dirname + "/" + filename); } else { // Files @@ -74,19 +89,17 @@ const _unzipFile = (zipfilename: string, dirname: string): Promise => { readStream.on("end", function () { zipfile.readEntry(); }); - let filepath = dirname + "/" + filename; - let outstream = fs.createWriteStream(filepath) + const filepath = dirname + "/" + filename; + const outstream = fs.createWriteStream(filepath); readStream.pipe(outstream); - readStream.on('end', () => { + readStream.on("end", () => { // Make it executable outstream.close(); - if(fs.existsSync(filepath)) - fs.chmodSync(filepath, "755"); + if (fs.existsSync(filepath)) fs.chmodSync(filepath, "755"); }); }); } } - }); zipfile.once("end", function () { resolve(dirname); @@ -94,70 +107,67 @@ const _unzipFile = (zipfilename: string, dirname: string): Promise => { }); }); }); -} +}; const _downloadAndUnzipToDirectory = (url: string, modeldir: string, compname: string) => { - let zipfile = modeldir + ".zip"; + const zipfile = modeldir + ".zip"; return new Promise((resolve, reject) => { _downloadFile(url, zipfile).then(() => { // Unzip file if (fs.existsSync(zipfile)) { - _unzipFile(zipfile, modeldir).then(() => { - resolve(); - }).catch((e) => { - console.log(e); - reject(); - }) - } - else { + _unzipFile(zipfile, modeldir) + .then(() => { + resolve(); + }) + .catch((e) => { + console.log(e); + reject(); + }); + } else { reject(); } }); }); -} +}; const _downloadCwlToDirectory = (url: string, modeldir: string) => { - let cwlfile = modeldir + '/run.cwl' + const cwlfile = modeldir + "/run.cwl"; return new Promise((resolve, reject) => { _downloadFile(url, cwlfile).then(() => { // Unzip file if (fs.existsSync(cwlfile)) { - resolve(); - } - else { + resolve(); + } else { reject(); } }); }); -} +}; const _downloadWCM = async (url: string, prefs: MintPreferences) => { - let hashdir = Md5.hashStr(url).toString(); - + const hashdir = Md5.hashStr(url).toString(); + // Get zip file name from url - let plainurl = url.replace(/\?.*$/, ''); - let component_file = plainurl.replace(/.+\//, ""); - let extension = path.extname(component_file) - let compname = path.basename(component_file, extension) + const plainurl = url.replace(/\?.*$/, ""); + const component_file = plainurl.replace(/.+\//, ""); + const extension = path.extname(component_file); + const compname = path.basename(component_file, extension); - let codedir = prefs.localex.codedir + "/" + hashdir; - if(!fs.existsSync(codedir)) - fs.mkdirsSync(codedir); + const codedir = prefs.localex.codedir + "/" + hashdir; + if (!fs.existsSync(codedir)) fs.mkdirsSync(codedir); - let modeldir = codedir + "/" + compname; - let src_dir = modeldir + "/" + "src" + const modeldir = codedir + "/" + compname; + const src_dir = modeldir + "/" + "src"; if (!fs.existsSync(src_dir)) { - if (extension == ".zip") - await _downloadAndUnzipToDirectory(url, modeldir, compname); - else if (extension == ".cwl"){ - if(!fs.existsSync(modeldir)) - fs.mkdirsSync(modeldir) - fs.mkdirsSync(src_dir) + if (extension == ".zip") await _downloadAndUnzipToDirectory(url, modeldir, compname); + else if (extension == ".cwl") { + if (!fs.existsSync(modeldir)) fs.mkdirsSync(modeldir); + fs.mkdirsSync(src_dir); await _downloadCwlToDirectory(url, src_dir); } } return modeldir; -} +}; const _getModelDetailsFromYAML = (modeldir: string) => { const wcmYamlFileName = modeldir + "/wings-component.yml"; @@ -170,28 +180,28 @@ const _getModelDetailsFromYAML = (modeldir: string) => { } else { throw new Error("The component is not a valid WINGS component."); } - - let comp: Component = { + + const comp: Component = { rundir: modeldir + "/src", inputs: [], - outputs: [], + outputs: [] }; - let yml = yaml.safeLoad(fs.readFileSync(wcmYamlFileName, 'utf8')) as Wcm; - let wings = yml["wings"] + const yml = yaml.safeLoad(fs.readFileSync(wcmYamlFileName, "utf8")) as Wcm; + const wings = yml["wings"]; wings.inputs.map((input: any) => { comp.inputs.push(input); - }) + }); wings.outputs.map((output: any) => { comp.outputs.push(output); - }) + }); return comp; -} +}; const _getModelIODetails = (io: ModelIO, iotype: string) => { if (!io.position) { return null; } - let pfx = (iotype == "input") ? "-i" : "-o"; + const pfx = iotype == "input" ? "-i" : "-o"; return { id: io.id, role: io.name, @@ -199,8 +209,8 @@ const _getModelIODetails = (io: ModelIO, iotype: string) => { isParam: false, format: io.format, type: io.type - } -} + }; +}; const _getModelParamDetails = (param: ModelParameter) => { if (!param.position) { @@ -212,69 +222,60 @@ const _getModelParamDetails = (param: ModelParameter) => { prefix: "-p" + param.position, isParam: true, type: param.type - } -} + }; +}; const _getModelDetails = (model: Model, modeldir: string) => { - let comp: Component = { + const comp: Component = { rundir: modeldir + "/src", softwareImage: model.software_image, inputs: [], - outputs: [], + outputs: [] }; let okinput = true; let okparam = true; let okoutput = true; model.input_files.map((input) => { - let details = _getModelIODetails(input, "input"); - if (!details){ + const details = _getModelIODetails(input, "input"); + if (!details) { okinput = false; console.error("Input file missing position: " + input.id); - } - else - comp.inputs.push(details); - }) + } else comp.inputs.push(details); + }); model.input_parameters.map((param) => { - let details = _getModelParamDetails(param); - if (!details){ + const details = _getModelParamDetails(param); + if (!details) { okparam = false; console.error("Input parameter missing position: " + param.id); - } - else - comp.inputs.push(details); - }) + } else comp.inputs.push(details); + }); model.output_files.map((output) => { - let details = _getModelIODetails(output, "output"); - if (!details){ + const details = _getModelIODetails(output, "output"); + if (!details) { okoutput = false; console.error("Output file missing position: " + output.id); - } - else - comp.outputs.push(details); - }) - if (okoutput) - return comp; - else - return null; -} - + } else comp.outputs.push(details); + }); + if (okoutput) return comp; + else return null; +}; export const getModelCacheDirectory = (url: string, prefs: MintPreferences) => { - let hashdir = Md5.hashStr(url).toString(); + const hashdir = Md5.hashStr(url).toString(); // Get zip file name from url - let plainurl = url.replace(/\?.*$/, ''); - let zipfile = plainurl.replace(/.+\//, ""); - let compname = zipfile.replace(/\.zip/i, ""); + const plainurl = url.replace(/\?.*$/, ""); + const zipfile = plainurl.replace(/.+\//, ""); + const compname = zipfile.replace(/\.zip/i, ""); - let codedir = prefs.localex.codedir; - let modeldir = codedir + "/" + hashdir + "/" + compname; + const codedir = prefs.localex.codedir; + const modeldir = codedir + "/" + hashdir + "/" + compname; return modeldir; -} +}; export const loadModelWCM = async (url: string, model: Model, prefs: MintPreferences) => { - let modeldir = await _downloadWCM(url, prefs); - if(model.software_image != null) { + const modeldir = await _downloadWCM(url, prefs); + if (model.software_image != null) { // Pull docker image if needed await pullImage(model.software_image); } @@ -284,141 +285,141 @@ export const loadModelWCM = async (url: string, model: Model, prefs: MintPrefere details = _getModelDetailsFromYAML(modeldir); } return details; -} - +}; const _getRegionGeoJson = (region: Region) => { - let geojson = {"type":"FeatureCollection","features":[]}; + const geojson = { type: "FeatureCollection", features: [] }; region.geometries.map((geom) => { - let feature = {"type": "Feature", "geometry": geom}; - geojson["features"].push(feature) + const feature = { type: "Feature", geometry: geom }; + geojson["features"].push(feature); }); - return JSON.stringify(geojson) -} + return JSON.stringify(geojson); +}; // Create Jobs (Seeds) and Queue them -export const queueModelExecutionsLocally = - async (thread: Thread, - modelid: string, - component: Component, - region: Region, - executions: Execution[], - prefs: MintPreferences): Promise[]> => { - - let seeds: ComponentSeed[] = []; - let registered_resources: any = {}; - let downloadInputPromises = []; - - let model = thread.models[modelid]; - let thread_model_id = thread.model_ensembles[modelid].id; - - // Get all input dataset bindings and parameter bindings - executions.map((execution) => { - let bindings = execution.bindings; - let datasets: ComponentDataBindings = {}; - let parameters: ComponentParameterBindings = {}; - let paramtypes: ComponentParameterTypes = {}; - - // Get input datasets - model.input_files.map((io: ModelIO) => { - let resources: DataResource[] = []; - let dsid = null; - if (bindings[io.id]) { - // We have a dataset binding from the user for it - resources = [bindings[io.id] as DataResource]; - } - else if (io.value) { - // There is a hardcoded value in the model itself - dsid = io.value.id; - resources = io.value.resources; - } - if (resources.length > 0) { - let type = io.type.replace(/^.*#/, ''); - let newresources: any = {}; - resources.map((res) => { - let resid = res.id; - let resname = res.name; - if (res.url) { - resname = res.url.replace(/^.*(#|\/)/, ''); - resname = resname.replace(/^([0-9])/, '_$1'); - if (!resid) - resid = resname; - } - newresources[resid] = { - id: resid, - url: res.url, - name: resname, - time_period: res.time_period, - spatial_coverage: res.spatial_coverage - } as DataResource - registered_resources[resid] = [resname, type, res.url]; - }) - datasets[io.id] = resources.map((res) => newresources[res.id]); - } - }); - - // Get Input parameters - model.input_parameters.map((ip) => { - if (ip.value) { - parameters[ip.id] = ip.value.toString(); - } - else if (bindings[ip.id]) { - let value = bindings[ip.id]; - parameters[ip.id] = value.toString(); - } - // HACK: Replace region geojson - if(parameters[ip.id].match(/__region_geojson:(.+)/)) { - let region_geojson = _getRegionGeoJson(region); - parameters[ip.id] = region_geojson; - } +export const queueModelExecutionsLocally = async ( + thread: Thread, + modelid: string, + component: Component, + region: Region, + executions: Execution[], + prefs: MintPreferences +): Promise[]> => { + const seeds: ComponentSeed[] = []; + const registered_resources: any = {}; + const downloadInputPromises = []; + + const model = thread.models[modelid]; + const thread_model_id = thread.model_ensembles[modelid].id; + + // Get all input dataset bindings and parameter bindings + executions.map((execution) => { + const bindings = execution.bindings; + const datasets: ComponentDataBindings = {}; + const parameters: ComponentParameterBindings = {}; + const paramtypes: ComponentParameterTypes = {}; + + // Get input datasets + model.input_files.map((io: ModelIO) => { + let resources: DataResource[] = []; + let dsid = null; + if (bindings[io.id]) { + // We have a dataset binding from the user for it + resources = [bindings[io.id] as DataResource]; + } else if (io.value) { + // There is a hardcoded value in the model itself + dsid = io.value.id; + resources = io.value.resources; + } + if (resources.length > 0) { + const type = io.type.replace(/^.*#/, ""); + const newresources: any = {}; + resources.map((res) => { + let resid = res.id; + let resname = res.name; + if (res.url) { + resname = res.url.replace(/^.*(#|\/)/, ""); + resname = resname.replace(/^([0-9])/, "_$1"); + if (!resid) resid = resname; + } + newresources[resid] = { + id: resid, + url: res.url, + name: resname, + time_period: res.time_period, + spatial_coverage: res.spatial_coverage + } as DataResource; + registered_resources[resid] = [resname, type, res.url]; + }); + datasets[io.id] = resources.map((res) => newresources[res.id]); + } + }); - paramtypes[ip.id] = ip.type; - }); + // Get Input parameters + model.input_parameters.map((ip) => { + if (ip.value) { + parameters[ip.id] = ip.value.toString(); + } else if (bindings[ip.id]) { + const value = bindings[ip.id]; + parameters[ip.id] = value.toString(); + } + // HACK: Replace region geojson + if (parameters[ip.id].match(/__region_geojson:(.+)/)) { + const region_geojson = _getRegionGeoJson(region); + parameters[ip.id] = region_geojson; + } - seeds.push({ - component: component, - execution: execution, - datasets: datasets, - parameters: parameters, - paramtypes: paramtypes - } as ComponentSeed); + paramtypes[ip.id] = ip.type; }); - // Add Download Job to Queue (if it doesn't already exist) - for (let resid in registered_resources) { - let args = registered_resources[resid]; - let inputpath = prefs.localex.datadir + "/" + args[0]; - let inputurl = args[2]; - if (!fs.existsSync(inputpath)) - downloadInputPromises.push(_downloadFile(inputurl, inputpath)); - } + seeds.push({ + component: component, + execution: execution, + datasets: datasets, + parameters: parameters, + paramtypes: paramtypes + } as ComponentSeed); + }); - // Download all datasets - if (downloadInputPromises.length > 0) - await Promise.all(downloadInputPromises); - - // Once all Downloads are finished, Add all execution jobs (seeds) to queue - let numseeds = seeds.length; - let priority = numseeds < 10 ? 1 : - numseeds < 50 ? 2 : numseeds < 200 ? 3 : - numseeds < 500 ? 4 : 5; - - return Promise.all(seeds.map((seed) => executionQueue.add({ - seed: seed, - prefs: prefs.localex, - thread_id: thread.id, - thread_model_id: thread_model_id - }, { - priority: priority - //jobId: seed.execution.id, - //removeOnComplete: true, - //attempts: 2 - }))); -} + // Add Download Job to Queue (if it doesn't already exist) + for (const resid in registered_resources) { + const args = registered_resources[resid]; + const inputpath = prefs.localex.datadir + "/" + args[0]; + const inputurl = args[2]; + if (!fs.existsSync(inputpath)) + downloadInputPromises.push(_downloadFile(inputurl, inputpath)); + } + + // Download all datasets + if (downloadInputPromises.length > 0) await Promise.all(downloadInputPromises); + + // Once all Downloads are finished, Add all execution jobs (seeds) to queue + const numseeds = seeds.length; + const priority = + numseeds < 10 ? 1 : numseeds < 50 ? 2 : numseeds < 200 ? 3 : numseeds < 500 ? 4 : 5; + + return Promise.all( + seeds.map((seed) => + executionQueue.add( + { + seed: seed, + prefs: prefs.localex, + thread_id: thread.id, + thread_model_id: thread_model_id + }, + { + priority: priority + //jobId: seed.execution.id, + //removeOnComplete: true, + //attempts: 2 + } + ) + ) + ); +}; export const fetchLocalRunLog = (executionid: string, prefs: MintPreferences) => { - let logstdout = prefs.localex.logdir + "/" + executionid + ".log"; - if(fs.existsSync(logstdout)) - return fs.readFileSync(logstdout).toString(); + const logstdout = prefs.localex.logdir + "/" + executionid + ".log"; + if (fs.existsSync(logstdout)) return fs.readFileSync(logstdout).toString(); return "Job not yet started running"; -} +}; diff --git a/src/classes/localex/local-execution-types.ts b/src/classes/localex/local-execution-types.ts index 73eaf69..f8bd3a9 100644 --- a/src/classes/localex/local-execution-types.ts +++ b/src/classes/localex/local-execution-types.ts @@ -1,38 +1,38 @@ import { Execution, DataResource } from "../mint/mint-types"; export interface ComponentSeed { - component: Component, - execution: Execution, - datasets: ComponentDataBindings, - parameters: ComponentParameterBindings, - paramtypes: ComponentParameterTypes, + component: Component; + execution: Execution; + datasets: ComponentDataBindings; + parameters: ComponentParameterBindings; + paramtypes: ComponentParameterTypes; } export interface ComponentDataBindings { - [inputid: string] : DataResource[] + [inputid: string]: DataResource[]; } export interface ComponentParameterBindings { - [inputid: string] : string + [inputid: string]: string; } export interface ComponentParameterTypes { - [inputid: string] : string + [inputid: string]: string; } export interface Component { - rundir: string, - softwareImage?: string, - inputs: ComponentArgument[], - outputs: ComponentArgument[] + rundir: string; + softwareImage?: string; + inputs: ComponentArgument[]; + outputs: ComponentArgument[]; } export interface ComponentArgument { - id: string, - type: string, - role: string, - prefix: string, - format?: string, - isParam?: boolean, - dimensionality?: number, - paramDefaultValue?: any, - testValue?: string -} \ No newline at end of file + id: string; + type: string; + role: string; + prefix: string; + format?: string; + isParam?: boolean; + dimensionality?: number; + paramDefaultValue?: any; + testValue?: string; +} diff --git a/src/classes/localex/seed-execution.ts b/src/classes/localex/seed-execution.ts index c91fc03..3ad828a 100644 --- a/src/classes/localex/seed-execution.ts +++ b/src/classes/localex/seed-execution.ts @@ -3,7 +3,13 @@ import os, { type } from "os"; import fs from "fs-extra"; import { Md5 } from "ts-md5"; import child_process from "child_process"; -import { incrementThreadModelSubmittedRuns, incrementThreadModelSuccessfulRuns, incrementThreadModelFailedRuns, updateExecutionStatusAndResults, updateExecutionStatus } from "../graphql/graphql_functions"; +import { + incrementThreadModelSubmittedRuns, + incrementThreadModelSuccessfulRuns, + incrementThreadModelFailedRuns, + updateExecutionStatusAndResults, + updateExecutionStatus +} from "../graphql/graphql_functions"; import { Component, ComponentSeed, ComponentArgument } from "./local-execution-types"; import { runImage } from "./docker-functions"; import { Container } from "dockerode"; @@ -17,39 +23,37 @@ module.exports = async (job: any) => { // Run the model seed (model config + bindings) // Clone the job data object - job.data = JSON.parse(JSON.stringify(job.data)); + job.data = JSON.parse(JSON.stringify(job.data)); - var seed: ComponentSeed = job.data.seed; - var localex: LocalExecutionPreferences = job.data.prefs; - var thread_model_id: string = job.data.thread_model_id; + const seed: ComponentSeed = job.data.seed; + const localex: LocalExecutionPreferences = job.data.prefs; + const thread_model_id: string = job.data.thread_model_id; - let prefs = await fetchMintConfig() - await KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password) + const prefs = await fetchMintConfig(); + await KeycloakAdapter.signIn(prefs.graphql.username, prefs.graphql.password); // Only increment submitted runs if this isn't a retry - if(seed.execution.status != "FAILURE" && !DEVMODE) + if (seed.execution.status != "FAILURE" && !DEVMODE) incrementThreadModelSubmittedRuns(thread_model_id); seed.execution.start_time = new Date(); // Initialize log file - if (! fs.existsSync(localex.logdir)) - fs.mkdirsSync(localex.logdir) - let logstdout = localex.logdir + "/" + seed.execution.id + ".log"; + if (!fs.existsSync(localex.logdir)) fs.mkdirsSync(localex.logdir); + const logstdout = localex.logdir + "/" + seed.execution.id + ".log"; fs.removeSync(logstdout); // Setup execution let error = null; - let comp : Component = seed.component; - let inputdir = localex.datadir; + const comp: Component = seed.component; + const inputdir = localex.datadir; let outputdir = localex.datadir; // Create temporary directory - let ostmp = localex.tempdir; - if (! fs.existsSync(ostmp)) - fs.mkdirsSync(ostmp) - let tmpprefix = ostmp + "/" + seed.execution.modelid.replace(/.*\//, ''); - let tempdir = fs.mkdtempSync(tmpprefix); + const ostmp = localex.tempdir; + if (!fs.existsSync(ostmp)) fs.mkdirsSync(ostmp); + const tmpprefix = ostmp + "/" + seed.execution.modelid.replace(/.*\//, ""); + const tempdir = fs.mkdtempSync(tmpprefix); console.log(tempdir); try { @@ -59,9 +63,9 @@ module.exports = async (job: any) => { // Set the execution engine used for this execution seed.execution.execution_engine = "localex"; - // Data/Parameter arguments to the script (will be setup below) - let args: string[] = []; - let plainargs: string[] = []; + // Data/Parameter arguments to the script (will be setup below) + const args: string[] = []; + const plainargs: string[] = []; // Default invocation is via Bash let command = "bash"; @@ -70,53 +74,49 @@ module.exports = async (job: any) => { args.push("./run"); // Set the Input file/parameter arguments for the command - let time_period : DateRange = null; - let spatial_coverage : any = null; + let time_period: DateRange = null; + let spatial_coverage: any = null; comp.inputs.map((input: ComponentArgument) => { args.push(input.prefix); plainargs.push(input.prefix); if (input.isParam) { //let paramtype = seed.paramtypes[input.id]; let paramvalue = seed.parameters[input.id]; - if (!paramvalue) - paramvalue = input.paramDefaultValue; + if (!paramvalue) paramvalue = input.paramDefaultValue; args.push(paramvalue); plainargs.push(paramvalue); - } - else { - let datasets = seed.datasets[input.id] || []; + } else { + const datasets = seed.datasets[input.id] || []; datasets.map((ds: DataResource) => { // Copy input files to tempdir - let ifile = inputdir + "/" + ds.name; - let newifile = tempdir + "/" + ds.name; + const ifile = inputdir + "/" + ds.name; + const newifile = tempdir + "/" + ds.name; fs.symlinkSync(ifile, newifile); //fs.copyFileSync(ifile, newifile); - args.push(ds.name) + args.push(ds.name); plainargs.push(ds.name); // Copy over spatio-temporal metadata from inputs - if(ds.time_period) - time_period = ds.time_period; - if(ds.spatial_coverage) - spatial_coverage = ds.spatial_coverage; + if (ds.time_period) time_period = ds.time_period; + if (ds.spatial_coverage) spatial_coverage = ds.spatial_coverage; }); } - }) - + }); + // Set the output file arguments for the command // Create the output file suffix based on a hash of inputs - let opsuffix = "-" + Md5.hashAsciiStr(seed.execution.modelid + plainargs.join()); - let results: any = {}; + const opsuffix = "-" + Md5.hashAsciiStr(seed.execution.modelid + plainargs.join()); + const results: any = {}; comp.outputs.map((output: ComponentArgument) => { args.push(output.prefix); - let opid = output.id + opsuffix; + const opid = output.id + opsuffix; let opfilename = output.role + opsuffix; - if(output.format) { + if (output.format) { opfilename += "." + output.format; } - let opfilepath = outputdir + "/" + opfilename; + const opfilepath = outputdir + "/" + opfilename; args.push(opfilename); - let opfileurl = opfilepath.replace(localex.datadir, localex.dataurl); + const opfileurl = opfilepath.replace(localex.datadir, localex.dataurl); results[output.id] = { id: opid, name: opfilename, @@ -124,7 +124,7 @@ module.exports = async (job: any) => { role: output.role, time_period: time_period ?? {}, spatial_coverage: spatial_coverage - } as DataResource + } as DataResource; }); let logstream = fs.createWriteStream(logstdout); @@ -135,85 +135,91 @@ module.exports = async (job: any) => { // Check if this component requires a docker image via the model definition // - or via the older pegasus job properties file - let softwareImage = comp.softwareImage; + const softwareImage = comp.softwareImage; let statusCode = 0; - let cwl_file = comp.rundir + "/run.cwl"; - let cwl_outputs: any = {} + const cwl_file = comp.rundir + "/run.cwl"; + let cwl_outputs: any = {}; let is_cwl = false; - if (! fs.existsSync(tempdir)) - fs.mkdirsSync(tempdir) + if (!fs.existsSync(tempdir)) fs.mkdirsSync(tempdir); if (fs.existsSync(cwl_file)) { - console.log("Running cwl:" ) + console.log("Running cwl:"); is_cwl = true; // Create cwl output directory - let output_suffix_cwl = Md5.hashAsciiStr(seed.execution.modelid + plainargs.join()); - outputdir = outputdir + '/' + output_suffix_cwl; + const output_suffix_cwl = Md5.hashAsciiStr(seed.execution.modelid + plainargs.join()); + outputdir = outputdir + "/" + output_suffix_cwl; if (!fs.existsSync(outputdir)) { - fs.mkdirsSync(outputdir) + fs.mkdirsSync(outputdir); } - let cwl_values_file = write_cwl_values(comp, seed, results, inputdir, tempdir, outputdir, plainargs) - let cwl_args: string[] = []; - let cwl_command = "cwltool" - cwl_args.push("--no-read-only") - cwl_args.push("--copy-outputs") - cwl_args.push("--no-match-user") + const cwl_values_file = write_cwl_values( + comp, + seed, + results, + inputdir, + tempdir, + outputdir, + plainargs + ); + const cwl_args: string[] = []; + const cwl_command = "cwltool"; + cwl_args.push("--no-read-only"); + cwl_args.push("--copy-outputs"); + cwl_args.push("--no-match-user"); //cwl_args.push("--user-space-docker-cmd") //cwl_args.push("docker") - cwl_args.push(cwl_file) - cwl_args.push(cwl_values_file) - console.log("running a new execution " + logstdout) - console.log("temporary directory " + tempdir) - + cwl_args.push(cwl_file); + cwl_args.push(cwl_values_file); + console.log("running a new execution " + logstdout); + console.log("temporary directory " + tempdir); + console.log(cwl_command + " " + cwl_args.join(" ") + "\n"); - let spawnResult = child_process.spawnSync(cwl_command, cwl_args, { + const spawnResult = child_process.spawnSync(cwl_command, cwl_args, { cwd: tempdir, shell: true, maxBuffer: 1024 * 1024 * 50 // 50 MB of log cutoff }); - + // Write log file - logstream = fs.createWriteStream(logstdout, { 'flags': 'a' }); + logstream = fs.createWriteStream(logstdout, { flags: "a" }); logstream.write("\n------- STDOUT ---------\n"); logstream.write(spawnResult.stdout); - if (spawnResult.error) - logstream.write(spawnResult.error.message); + if (spawnResult.error) logstream.write(spawnResult.error.message); logstream.write("\n------- STDERR ---------\n"); - logstream.write(spawnResult.stderr); + logstream.write(spawnResult.stderr); logstream.close(); if (spawnResult.error) { error = spawnResult.error.message; } statusCode = spawnResult.status; - if (statusCode == 0){ - cwl_outputs = JSON.parse(spawnResult.stdout.toString()) + if (statusCode == 0) { + cwl_outputs = JSON.parse(spawnResult.stdout.toString()); } - } - else if (softwareImage != null) { - console.log("Running as a Docker Image:" ) - logstream = fs.createWriteStream(logstdout, { 'flags': 'a' }); - + } else if (softwareImage != null) { + console.log("Running as a Docker Image:"); + logstream = fs.createWriteStream(logstdout, { flags: "a" }); + // Run command in docker image - let folderBindings = [`${tempdir}:${tempdir}`, `${localex.datadir}:${localex.datadir}`]; - let data = await runImage(args, softwareImage, logstream, tempdir, folderBindings); - var output = data[0]; - var container: Container = data[1]; + const folderBindings = [`${tempdir}:${tempdir}`, `${localex.datadir}:${localex.datadir}`]; + const data = await runImage(args, softwareImage, logstream, tempdir, folderBindings); + const output = data[0]; + const container: Container = data[1]; statusCode = output.StatusCode; - + // Clean up logstream.close(); - await container.remove({force: true}); - } - else { - console.log("Running as a Singularity Information") - let pegasus_jobprops_file = comp.rundir + "/__pegasus-job.properties"; + await container.remove({ force: true }); + } else { + console.log("Running as a Singularity Information"); + const pegasus_jobprops_file = comp.rundir + "/__pegasus-job.properties"; if (fs.existsSync(pegasus_jobprops_file)) { - let jobprops = fs.readFileSync(pegasus_jobprops_file); - let matches: RegExpMatchArray = jobprops.toString().match(/SingularityImage = "(.+)"/); + const jobprops = fs.readFileSync(pegasus_jobprops_file); + const matches: RegExpMatchArray = jobprops + .toString() + .match(/SingularityImage = "(.+)"/); if (matches.length > 1) { command = "singularity"; args.push("exec"); @@ -222,20 +228,19 @@ module.exports = async (job: any) => { } // Spawn the process & pipe stdout and stderr - let spawnResult = child_process.spawnSync(command, args, { + const spawnResult = child_process.spawnSync(command, args, { cwd: tempdir, shell: true, maxBuffer: 1024 * 1024 * 50 // 50 MB of log cutoff }); - + // Write log file - logstream = fs.createWriteStream(logstdout, { 'flags': 'a' }); + logstream = fs.createWriteStream(logstdout, { flags: "a" }); logstream.write("\n------- STDOUT ---------\n"); logstream.write(spawnResult.stdout); - if (spawnResult.error) - logstream.write(spawnResult.error.message); + if (spawnResult.error) logstream.write(spawnResult.error.message); logstream.write("\n------- STDERR ---------\n"); - logstream.write(spawnResult.stderr); + logstream.write(spawnResult.stderr); logstream.close(); if (spawnResult.error) { error = spawnResult.error.message; @@ -243,26 +248,23 @@ module.exports = async (job: any) => { statusCode = spawnResult.status; } - // Check for Errors & Process Results - if(statusCode != 0) { + if (statusCode != 0) { error = "Execution returned with non-zero status code"; - } - else { + } else { // Process Results Object.values(results).map((result: any) => { // Rename temporary output files to desired output name let desired_output_file = null; let tmp_output_file = null; - if(is_cwl) { - result.name = result.role + if (is_cwl) { + result.name = result.role; if (result.role !== undefined && result.role in cwl_outputs) { - let cwl_output = cwl_outputs[result.role]; - desired_output_file = outputdir + '/' + cwl_output['basename']; - tmp_output_file = cwl_output['path'] + const cwl_output = cwl_outputs[result.role]; + desired_output_file = outputdir + "/" + cwl_output["basename"]; + tmp_output_file = cwl_output["path"]; } - } - else { + } else { tmp_output_file = tempdir + "/" + result.name; desired_output_file = outputdir + "/" + result.name; } @@ -271,92 +273,88 @@ module.exports = async (job: any) => { if (fs.existsSync(tmp_output_file)) { fs.copyFileSync(tmp_output_file, desired_output_file); } - let url = desired_output_file.replace(localex.datadir, localex.dataurl); + const url = desired_output_file.replace(localex.datadir, localex.dataurl); result.url = url; } }); seed.execution.results = results; } - + // Remove temporary directory - fs.removeSync(tempdir) - } - catch(e) { + fs.removeSync(tempdir); + } catch (e) { error = "ERROR: " + e; - let logstream = fs.createWriteStream(logstdout); + const logstream = fs.createWriteStream(logstdout); logstream.write("ERROR in Execution: \n"); logstream.write(error + "\n"); - logstream.close(); + logstream.close(); } // Set the execution status - seed.execution.status = "SUCCESS"; + seed.execution.status = "SUCCESS"; seed.execution.end_time = new Date(); seed.execution.run_progress = 1; - if(error) { + if (error) { seed.execution.status = "FAILURE"; } // Update execution status and results in backend - if(!DEVMODE) - await updateExecutionStatusAndResults(seed.execution); + if (!DEVMODE) await updateExecutionStatusAndResults(seed.execution); // Return job execution or error - if(seed.execution.status == "SUCCESS") { - if(!DEVMODE) - await incrementThreadModelSuccessfulRuns(thread_model_id); + if (seed.execution.status == "SUCCESS") { + if (!DEVMODE) await incrementThreadModelSuccessfulRuns(thread_model_id); return Promise.resolve(seed.execution); - } - else { - if(!DEVMODE) - await incrementThreadModelFailedRuns(thread_model_id); + } else { + if (!DEVMODE) await incrementThreadModelFailedRuns(thread_model_id); return Promise.reject(new Error(error)); } - -} - -const write_cwl_values = (comp: Component, seed: any, results: any, inputdir: string, - tempdir: string, outputdir: string, plainargs: string[]) => { - let execution_dir = comp.rundir +}; + +const write_cwl_values = ( + comp: Component, + seed: any, + results: any, + inputdir: string, + tempdir: string, + outputdir: string, + plainargs: string[] +) => { + const execution_dir = comp.rundir; interface CwlValueFile { - class: string, - location: string + class: string; + location: string; } - let data : Record = {} + const data: Record = {}; comp.inputs.map((input: any) => { if (input.isParam) { - let paramtype = seed.paramtypes[input.id]; + const paramtype = seed.paramtypes[input.id]; let paramvalue = seed.parameters[input.id]; - if (!paramvalue) - paramvalue = input.paramDefaultValue; - if (paramtype == "int") - paramvalue = parseInt(paramvalue); - else if (paramtype == "float") - paramvalue = parseFloat(paramvalue); + if (!paramvalue) paramvalue = input.paramDefaultValue; + if (paramtype == "int") paramvalue = parseInt(paramvalue); + else if (paramtype == "float") paramvalue = parseFloat(paramvalue); else if (paramtype == "boolean") - paramvalue = (paramvalue.toString().toLowerCase() == "true") - else - paramvalue = paramvalue.toString(); - data[input.role] = paramvalue - } - else { - let datasets = seed.datasets[input.id] || []; + paramvalue = paramvalue.toString().toLowerCase() == "true"; + else paramvalue = paramvalue.toString(); + data[input.role] = paramvalue; + } else { + const datasets = seed.datasets[input.id] || []; datasets.map((ds: string) => { // Copy input files to tempdir - data[input.role] = {"class": "File", "location": ds["url"]} + data[input.role] = { class: "File", location: ds["url"] }; }); - console.log(datasets) + console.log(datasets); } - }) + }); // Set output file names Object.values(results).map((result: any) => { - data[result.role] = {"class": "File", "location": result.name} - }) + data[result.role] = { class: "File", location: result.name }; + }); - let valuesFile = tempdir + "/values.yml"; - let ymlStr = yaml.safeDump(data); - fs.writeFileSync(valuesFile, ymlStr, 'utf8') + const valuesFile = tempdir + "/values.yml"; + const ymlStr = yaml.safeDump(data); + fs.writeFileSync(valuesFile, ymlStr, "utf8"); console.log("writing the values file " + valuesFile); - return valuesFile -} + return valuesFile; +}; diff --git a/src/classes/mint/data-catalog-functions.ts b/src/classes/mint/data-catalog-functions.ts index 4ebeee1..8b5f109 100644 --- a/src/classes/mint/data-catalog-functions.ts +++ b/src/classes/mint/data-catalog-functions.ts @@ -2,48 +2,50 @@ import { DateRange, Region, MintPreferences, DatasetQueryParameters, Dataset } f import * as rp from "request-promise-native"; const getDatasetsFromDCResponse = (obj: any, queryParameters: DatasetQueryParameters) => { - let datasets = obj.datasets.map((ds: any) => { - let dmeta = ds['dataset_metadata']; + const datasets = obj.datasets.map((ds: any) => { + const dmeta = ds["dataset_metadata"]; return { - id: ds['dataset_id'], - name: ds['dataset_name'] || '', - region: '', + id: ds["dataset_id"], + name: ds["dataset_name"] || "", + region: "", variables: queryParameters.variables, - time_period: dmeta['temporal_coverage'] ? { - start_date: (dmeta['temporal_coverage']['start_time'] ? - new Date(dmeta['temporal_coverage']['start_time']) - : null), - end_date: (dmeta['temporal_coverage']['end_time'] ? - new Date(dmeta['temporal_coverage']['end_time']) - : null), - } : null, - description: dmeta['dataset_description'] || '', - version: dmeta['version'] || '', - limitations: dmeta['limitations'] || '', + time_period: dmeta["temporal_coverage"] + ? { + start_date: dmeta["temporal_coverage"]["start_time"] + ? new Date(dmeta["temporal_coverage"]["start_time"]) + : null, + end_date: dmeta["temporal_coverage"]["end_time"] + ? new Date(dmeta["temporal_coverage"]["end_time"]) + : null + } + : null, + description: dmeta["dataset_description"] || "", + version: dmeta["version"] || "", + limitations: dmeta["limitations"] || "", source: { - name: dmeta['source'] || '', - url: dmeta['source_url'] || '', - type: dmeta['source_type'] || '', + name: dmeta["source"] || "", + url: dmeta["source_url"] || "", + type: dmeta["source_type"] || "" }, - is_cached: dmeta['is_cached'] || false, - resource_count: dmeta['resource_count'] || 0, + is_cached: dmeta["is_cached"] || false, + resource_count: dmeta["resource_count"] || 0, datatype: dmeta["datatype"] || dmeta["data_type"] || "", categories: dmeta["category_tags"] || [], - resources: [] as any[] - } + resources: [] as any[] + }; }); return datasets; -} +}; const getDatasetResourceListFromDCResponse = (obj: any) => { - let resources : any[] = []; + const resources: any[] = []; obj.resources.map((row: any) => { - let rmeta = row["resource_metadata"]; - let tcover = rmeta["temporal_coverage"]; - let scover = rmeta["spatial_coverage"]; - let tcoverstart = tcover ? new Date(tcover["start_time"]) : null; - let tcoverend = tcover ? new Date(tcover["end_time"]) : null; - + const rmeta = row["resource_metadata"]; + const tcover = rmeta["temporal_coverage"]; + const scover = rmeta["spatial_coverage"]; + const tcoverstart = tcover ? new Date(tcover["start_time"]) : null; + const tcoverend = tcover ? new Date(tcover["end_time"]) : null; + resources.push({ id: row["resource_id"], name: row["resource_name"], @@ -54,55 +56,62 @@ const getDatasetResourceListFromDCResponse = (obj: any) => { }, spatial_coverage: scover, selected: true - }); + }); }); return resources; -} +}; // Query Dataset details and fetch dataset resources for a region and time-period -export const queryDatasetDetails = async (modelid: string, inputid: string, driving_variables: string[], - dsid: string, dates: DateRange, region: Region, prefs:MintPreferences ) : Promise => { - +export const queryDatasetDetails = async ( + modelid: string, + inputid: string, + driving_variables: string[], + dsid: string, + dates: DateRange, + region: Region, + prefs: MintPreferences +): Promise => { return new Promise((resolve, reject) => { - let dsQueryData = { + const dsQueryData = { standard_variable_names__in: driving_variables, spatial_coverage__intersects: region.geometries[0], - end_time__gte: dates.start_date.toISOString().replace(/\.\d{3}Z$/,''), - start_time__lte: dates.end_date.toISOString().replace(/\.\d{3}Z$/,''), + end_time__gte: dates.start_date.toISOString().replace(/\.\d{3}Z$/, ""), + start_time__lte: dates.end_date.toISOString().replace(/\.\d{3}Z$/, ""), dataset_ids__in: [dsid], limit: 100 - } + }; - let resQueryData : any = { + const resQueryData: any = { dataset_id: dsid, filter: { spatial_coverage__intersects: region.geometries[0], - end_time__gte: dates.start_date.toISOString().replace(/\.\d{3}Z$/,''), - start_time__lte: dates.end_date.toISOString().replace(/\.\d{3}Z$/,'') + end_time__gte: dates.start_date.toISOString().replace(/\.\d{3}Z$/, ""), + start_time__lte: dates.end_date.toISOString().replace(/\.\d{3}Z$/, "") }, limit: 5000 - } + }; rp.post({ url: prefs.data_catalog_api + "/datasets/find", - headers: {'Content-Type': 'application/json'}, + headers: { "Content-Type": "application/json" }, body: dsQueryData, json: true }).then((obj) => { - let datasets: Dataset[] = getDatasetsFromDCResponse(obj, - {variables: driving_variables} as DatasetQueryParameters); - - if(datasets.length > 0) { - let ds = datasets[0]; + const datasets: Dataset[] = getDatasetsFromDCResponse(obj, { + variables: driving_variables + } as DatasetQueryParameters); + + if (datasets.length > 0) { + const ds = datasets[0]; rp.post({ url: prefs.data_catalog_api + "/datasets/dataset_resources", - headers: {'Content-Type': 'application/json'}, + headers: { "Content-Type": "application/json" }, body: resQueryData, json: true }).then((resobj) => { ds.resources = getDatasetResourceListFromDCResponse(resobj); resolve(ds); - }) + }); } else { reject(); } @@ -110,18 +119,23 @@ export const queryDatasetDetails = async (modelid: string, inputid: string, driv }); }; -export const registerDataset = async (dataset: Dataset, prefs: MintPreferences) : Promise => { +export const registerDataset = async ( + dataset: Dataset, + prefs: MintPreferences +): Promise => { // If datacatalog is CKAN, we need to convert the dataset object to CKAN format and convert if (prefs.data_catalog_type == "CKAN") { return await registerCKANDataset(dataset, prefs); - } - else { + } else { // Register in internal data catalog } }; -export const registerCKANDataset = async (dataset: Dataset, prefs: MintPreferences) : Promise => { - let ckan_dataset = { +export const registerCKANDataset = async ( + dataset: Dataset, + prefs: MintPreferences +): Promise => { + const ckan_dataset = { name: dataset.name, title: dataset.name, notes: dataset.description, @@ -129,35 +143,37 @@ export const registerCKANDataset = async (dataset: Dataset, prefs: MintPreferenc private: false, version: dataset.version, extras: [ - { + { key: "spatial", value: dataset.spatial_coverage || "" } ], - tags: dataset.variables.map((v) => { - return { name: "stdvar." + v } + tags: dataset.variables.map((v) => { + return { name: "stdvar." + v }; }), resources: dataset.resources.map((res) => { return { name: res.name, url: res.url - } + }; }) }; return new Promise((resolve, reject) => { rp.post({ url: prefs.data_catalog_api + "/api/action/package_create", headers: { - 'Content-Type': 'application/json', - 'Authorization': prefs.data_catalog_key + "Content-Type": "application/json", + Authorization: prefs.data_catalog_key }, body: ckan_dataset, json: true - }).then((obj) => { - resolve(true); - }).catch((err) => { - console.log(err); - reject(err); - }); + }) + .then((obj) => { + resolve(true); + }) + .catch((err) => { + console.log(err); + reject(err); + }); }); -} \ No newline at end of file +}; diff --git a/src/classes/mint/date-utils.ts b/src/classes/mint/date-utils.ts index a5f2540..f8fdaaa 100644 --- a/src/classes/mint/date-utils.ts +++ b/src/classes/mint/date-utils.ts @@ -1,11 +1,11 @@ -export const fromTimestampIntegerToString = (timestamp: number) : string => { - return new Date(timestamp).toISOString().replace(/\.000Z$/, ''); -} +export const fromTimestampIntegerToString = (timestamp: number): string => { + return new Date(timestamp).toISOString().replace(/\.000Z$/, ""); +}; -export const fromTimestampIntegerToReadableString = (timestamp: number) : string => { - return fromTimestampIntegerToString(timestamp).replace(/T/,' at ').replace(/\..+$/,''); -} +export const fromTimestampIntegerToReadableString = (timestamp: number): string => { + return fromTimestampIntegerToString(timestamp).replace(/T/, " at ").replace(/\..+$/, ""); +}; -export const fromTimestampIntegerToDateString = (timestamp: number) : string => { - return fromTimestampIntegerToString(timestamp).replace(/T.*$/,''); -} \ No newline at end of file +export const fromTimestampIntegerToDateString = (timestamp: number): string => { + return fromTimestampIntegerToString(timestamp).replace(/T.*$/, ""); +}; diff --git a/src/classes/mint/mint-functions.ts b/src/classes/mint/mint-functions.ts index e6fd60b..f76e714 100644 --- a/src/classes/mint/mint-functions.ts +++ b/src/classes/mint/mint-functions.ts @@ -1,12 +1,32 @@ -import { Thread, ProblemStatement, MintPreferences, ExecutionSummary, Execution, - ModelIOBindings, ThreadModelMap } from "./mint-types"; -import { setupModelWorkflow, fetchWingsTemplate, loginToWings, runModelExecutions, - fetchWingsRunsStatuses, fetchWingsRunResults } from "../wings/wings-functions"; -import { getModelInputBindings, getModelInputConfigurations, deleteThreadModelExecutionIds, - setThreadModelExecutionIds, getExecutionHash, listSuccessfulExecutionIds, - getThreadModelExecutionIds, getExecutions, setExecutions, getThread, - setThreadModelExecutionSummary, - getRegionDetails} from "../graphql/graphql_functions"; +import { + Thread, + MintPreferences, + ExecutionSummary, + Execution, + ThreadModelMap +} from "./mint-types"; +import { + setupModelWorkflow, + fetchWingsTemplate, + loginToWings, + runModelExecutions, + fetchWingsRunsStatuses, + fetchWingsRunResults +} from "../wings/wings-functions"; +import { + getModelInputBindings, + getModelInputConfigurations, + deleteThreadModelExecutionIds, + setThreadModelExecutionIds, + getExecutionHash, + listSuccessfulExecutionIds, + getThreadModelExecutionIds, + getExecutions, + setExecutions, + getThread, + setThreadModelExecutionSummary, + getRegionDetails +} from "../graphql/graphql_functions"; import fs from "fs-extra"; @@ -14,13 +34,13 @@ import { DEVMODE } from "../../config/app"; import { DEVHOMEDIR } from "../../config/app"; import { PORT } from "../../config/app"; -export const getConfiguration = () : MintPreferences => { - let config_file = process.env.ENSEMBLE_MANAGER_CONFIG_FILE +export const getConfiguration = (): MintPreferences => { + let config_file = process.env.ENSEMBLE_MANAGER_CONFIG_FILE; if (!config_file) { - config_file = __dirname + "/config/config.json" + config_file = __dirname + "/config/config.json"; } - let prefs = JSON.parse(fs.readFileSync(config_file, 'utf8')) as MintPreferences; + const prefs = JSON.parse(fs.readFileSync(config_file, "utf8")) as MintPreferences; if (prefs.graphql && !prefs.graphql.secret && process.env.HASURA_GRAPHQL_ADMIN_SECRET) { prefs.graphql.secret = process.env.HASURA_GRAPHQL_ADMIN_SECRET; } @@ -28,98 +48,99 @@ export const getConfiguration = () : MintPreferences => { }; export const fetchMintConfig = async () => { - let prefs = getConfiguration(); - if(prefs.execution_engine == "wings") { - let res = await fetch(prefs.wings.server + "/config"); - let wdata = await res.json(); - prefs.wings.export_url = wdata["internal_server"] + const prefs = getConfiguration(); + if (prefs.execution_engine == "wings") { + const res = await fetch(prefs.wings.server + "/config"); + const wdata = await res.json(); + prefs.wings.export_url = wdata["internal_server"]; prefs.wings.storage = wdata["storage"]; prefs.wings.dotpath = wdata["dotpath"]; prefs.wings.onturl = wdata["ontology"]; } - if(DEVMODE) { + if (DEVMODE) { prefs.ensemble_manager_api = "http://localhost:" + PORT + "/v1"; prefs.localex.datadir = DEVHOMEDIR + "/data"; prefs.localex.codedir = DEVHOMEDIR + "/code"; prefs.localex.logdir = DEVHOMEDIR + "/logs"; prefs.localex.dataurl = "file://" + DEVHOMEDIR + "/data"; prefs.localex.logurl = "file://" + DEVHOMEDIR + "/logs"; - } + } return prefs; }; -export const saveAndRunExecutions = async( - thread: Thread, - modelid: string, - prefs: MintPreferences) => { - +export const saveAndRunExecutions = async ( + thread: Thread, + modelid: string, + prefs: MintPreferences +) => { // Setup Model for execution on Wings - await loginToWings(prefs); // Login to Wings now Happens at the top app level + await loginToWings(prefs); // Login to Wings now Happens at the top app level - for(let pmodelid in thread.model_ensembles) { - if(!modelid || (modelid == pmodelid)) + for (const pmodelid in thread.model_ensembles) { + if (!modelid || modelid == pmodelid) await saveAndRunExecutionsForModel(pmodelid, thread, prefs); } console.log("Finished sending all executions for execution"); monitorAllExecutions(thread, modelid, prefs); -} - -export const saveAndRunExecutionsForModel = async(modelid: string, - thread: Thread, - prefs: MintPreferences) => { - if(!thread.execution_summary) - thread.execution_summary = {}; - - let thread_model_id = thread.model_ensembles[modelid].id; - let model = thread.models[modelid]; - - let thread_region = await getRegionDetails(thread.regionid) - let execution_details = getModelInputBindings(model, thread, thread_region); - let threadModel = execution_details[0] as ThreadModelMap; - let inputIds = execution_details[1] as string[]; - let configs = getModelInputConfigurations(threadModel, inputIds); - - if(configs != null) { +}; + +export const saveAndRunExecutionsForModel = async ( + modelid: string, + thread: Thread, + prefs: MintPreferences +) => { + if (!thread.execution_summary) thread.execution_summary = {}; + + const thread_model_id = thread.model_ensembles[modelid].id; + const model = thread.models[modelid]; + + const thread_region = await getRegionDetails(thread.regionid); + const execution_details = getModelInputBindings(model, thread, thread_region); + const threadModel = execution_details[0] as ThreadModelMap; + const inputIds = execution_details[1] as string[]; + const configs = getModelInputConfigurations(threadModel, inputIds); + + if (configs != null) { // Delete existing thread model execution ids deleteThreadModelExecutionIds(thread_model_id); // Setup Model for execution on Wings - let workflowid = await setupModelWorkflow(model, thread, prefs); - let tpl_package = await fetchWingsTemplate(workflowid, prefs); + const workflowid = await setupModelWorkflow(model, thread, prefs); + const tpl_package = await fetchWingsTemplate(workflowid, prefs); + + const datasets = {}; // Map of datasets to be registered (passed to Wings to keep track) - let datasets = {}; // Map of datasets to be registered (passed to Wings to keep track) - // Setup some book-keeping to help in searching for results - let execution_summary = { + const execution_summary = { total_runs: configs.length, - workflow_name: workflowid.replace(/.+#/, ''), + workflow_name: workflowid.replace(/.+#/, ""), submission_time: new Date() - } as ExecutionSummary + } as ExecutionSummary; setThreadModelExecutionSummary(thread_model_id, execution_summary); - + // Work in batches - let batchSize = 100; // Deal with executions from firebase in this batch size + const batchSize = 100; // Deal with executions from firebase in this batch size let batchid = 0; // Use to create batchids in firebase for storing execution ids - let executionBatchSize = 10; // Run workflows in Wings in batches - + const executionBatchSize = 10; // Run workflows in Wings in batches + // Create executions in batches - for(let i=0; i { - let inputBindings : any = {}; - for(let j=0; j eid); // Filter for null/undefined execution ids + const all_execution_ids: any[] = await listSuccessfulExecutionIds(executionids); + const current_execution_ids = all_execution_ids.filter((eid) => eid); // Filter for null/undefined execution ids // Run executions in smaller batches - for(let i=0; i current_execution_ids.indexOf(execution.id) < 0); - if(eslice_nr.length > 0) { - let runids = await runModelExecutions(thread, eslice_nr, datasets, tpl_package, prefs); - for(let j=0; j current_execution_ids.indexOf(execution.id) < 0 + ); + if (eslice_nr.length > 0) { + const runids = await runModelExecutions( + thread, + eslice_nr, + datasets, + tpl_package, + prefs + ); + for (let j = 0; j < eslice_nr.length; j++) { eslice_nr[j].runid = runids[j]; eslice_nr[j].status = "WAITING"; eslice_nr[j].execution_engine = "wings"; @@ -162,16 +191,16 @@ export const saveAndRunExecutionsForModel = async(modelid: string, } } console.log("Finished submitting all executions for model: " + modelid); -} - -export const monitorAllExecutions = async( - thread: Thread, - modelid: string, - prefs: MintPreferences) => { +}; - let currentTimeout = 30000; // Check every 30 seconds +export const monitorAllExecutions = async ( + thread: Thread, + modelid: string, + prefs: MintPreferences +) => { + const currentTimeout = 30000; // Check every 30 seconds - console.log("Start monitoring for "+thread.id); + console.log("Start monitoring for " + thread.id); checkStatusAllExecutions(thread, modelid, prefs).then(() => { console.log("Status checking finished"); @@ -179,89 +208,92 @@ export const monitorAllExecutions = async( thread = thr; let done = true; Object.keys(thread.model_ensembles).map((modelid) => { - let summary = thread.execution_summary[modelid]; - if(summary.total_runs != (summary.successful_runs + summary.failed_runs)) { + const summary = thread.execution_summary[modelid]; + if (summary.total_runs != summary.successful_runs + summary.failed_runs) { done = false; } - }) + }); // FIXME: Check for a global shared variable if a request comes to abort for this thread - if(!done) { + if (!done) { setTimeout(() => { - monitorAllExecutions(thread, modelid, prefs) + monitorAllExecutions(thread, modelid, prefs); }, currentTimeout); } else { - console.log("Finished Monitoring for "+thread.id+". Thread runs have finished") + console.log("Finished Monitoring for " + thread.id + ". Thread runs have finished"); } - }) + }); }); -} - -export const checkStatusAllExecutions = async( - thread: Thread, - modelid: string, - prefs: MintPreferences) => { +}; +export const checkStatusAllExecutions = async ( + thread: Thread, + modelid: string, + prefs: MintPreferences +) => { // Setup Model for execution on Wings - await loginToWings(prefs); // Login to Wings now Happens at the top app level + await loginToWings(prefs); // Login to Wings now Happens at the top app level - for(let pmodelid in thread.model_ensembles) { - if(!modelid || modelid==pmodelid) + for (const pmodelid in thread.model_ensembles) { + if (!modelid || modelid == pmodelid) await checkStatusAllExecutionsForModel(pmodelid, thread, prefs); } console.log("Finished checking executions"); -} - -export const checkStatusAllExecutionsForModel = async( - modelid: string, - thread: Thread, - prefs: MintPreferences) => { - let model = thread.models[modelid]; - let thread_model_id = thread.model_ensembles[modelid].id; - let summary = thread.execution_summary[modelid]; - +}; + +export const checkStatusAllExecutionsForModel = async ( + modelid: string, + thread: Thread, + prefs: MintPreferences +) => { + const model = thread.models[modelid]; + const thread_model_id = thread.model_ensembles[modelid].id; + const summary = thread.execution_summary[modelid]; + // await loginToWings(prefs); // Login to Wings handled at the top now - + // FIXME: Some problem with the submission times - let runtimeInfos : any = await fetchWingsRunsStatuses(summary.workflow_name, - Math.floor(summary.submission_time.getTime()), summary.total_runs, prefs); + const runtimeInfos: any = await fetchWingsRunsStatuses( + summary.workflow_name, + Math.floor(summary.submission_time.getTime()), + summary.total_runs, + prefs + ); let start = 0; - let pageSize = 100; + const pageSize = 100; let numSuccessful = 0; let numFailed = 0; let numRunning = 0; - let threadModelExecutionIds = await getThreadModelExecutionIds(thread_model_id); + const threadModelExecutionIds = await getThreadModelExecutionIds(thread_model_id); - while(true) { - let executionids = threadModelExecutionIds.slice(start, start+pageSize); - let executions = await getExecutions(executionids); + while (true) { + const executionids = threadModelExecutionIds.slice(start, start + pageSize); + const executions = await getExecutions(executionids); start += pageSize; - if(!executions || executions.length == 0) - break; + if (!executions || executions.length == 0) break; - let changed_executions : Execution[] = []; + const changed_executions: Execution[] = []; executions.map((execution) => { // Check if the execution is not already finished (probably from another run) - if(execution.status == "WAITING" || execution.status == "RUNNING") { - let runtimeInfo = runtimeInfos[execution.runid]; - if(runtimeInfo) { - if(runtimeInfo.status != execution.status) { - if(runtimeInfo.status == "SUCCESS" || runtimeInfo.status == "FAILURE") { + if (execution.status == "WAITING" || execution.status == "RUNNING") { + const runtimeInfo = runtimeInfos[execution.runid]; + if (runtimeInfo) { + if (runtimeInfo.status != execution.status) { + if (runtimeInfo.status == "SUCCESS" || runtimeInfo.status == "FAILURE") { execution.run_progress = 1; } execution.status = runtimeInfo.status; changed_executions.push(execution); } - } - else { + } else { // Execution not yet submitted //console.log(execution); } } - switch(execution.status) { + switch (execution.status) { case "RUNNING": numRunning++; break; @@ -274,24 +306,27 @@ export const checkStatusAllExecutionsForModel = async( } }); - let finished_executions = changed_executions.filter((execution) => execution.status == "SUCCESS"); + const finished_executions = changed_executions.filter( + (execution) => execution.status == "SUCCESS" + ); // Fetch Results of executions that have finished - let results = await Promise.all(finished_executions.map((execution) => { - return fetchWingsRunResults(execution, prefs); - })); - for(let i=0; i { + return fetchWingsRunResults(execution, prefs); + }) + ); + for (let i = 0; i < finished_executions.length; i++) { + if (results[i]) finished_executions[i].results = results[i]; } // Update all executions setExecutions(changed_executions, thread_model_id); } - + summary.successful_runs = numSuccessful; summary.failed_runs = numFailed; summary.submitted_runs = numRunning + numSuccessful + numFailed; - + await setThreadModelExecutionSummary(thread_model_id, summary); -} \ No newline at end of file +}; diff --git a/src/classes/mint/mint-local-functions.ts b/src/classes/mint/mint-local-functions.ts index 4ca5b69..fe3aa47 100644 --- a/src/classes/mint/mint-local-functions.ts +++ b/src/classes/mint/mint-local-functions.ts @@ -1,9 +1,37 @@ -import { Thread, Model, ThreadModelMap, ProblemStatement, MintPreferences, ExecutionSummary, Execution, Dataset } from "./mint-types"; -import { getModelInputConfigurations, deleteThreadModelExecutionIds, setThreadModelExecutionIds, - getExecutionHash, getThreadModelExecutionIds, getExecutions, - setExecutions, deleteExecutions, - setThreadModelExecutionSummary, getModelInputBindings, listSuccessfulExecutionIds, incrementThreadModelSubmittedRuns, incrementThreadModelSuccessfulRuns, getRegionDetails, deleteModel, updateExecutionStatus, incrementThreadModelRegisteredRuns } from "../graphql/graphql_functions"; -import { loadModelWCM, getModelCacheDirectory, queueModelExecutionsLocally } from "../localex/local-execution-functions"; +import { + Thread, + Model, + ThreadModelMap, + ProblemStatement, + MintPreferences, + ExecutionSummary, + Execution, + Dataset +} from "./mint-types"; +import { + getModelInputConfigurations, + deleteThreadModelExecutionIds, + setThreadModelExecutionIds, + getExecutionHash, + getThreadModelExecutionIds, + getExecutions, + setExecutions, + deleteExecutions, + setThreadModelExecutionSummary, + getModelInputBindings, + listSuccessfulExecutionIds, + incrementThreadModelSubmittedRuns, + incrementThreadModelSuccessfulRuns, + getRegionDetails, + deleteModel, + updateExecutionStatus, + incrementThreadModelRegisteredRuns +} from "../graphql/graphql_functions"; +import { + loadModelWCM, + getModelCacheDirectory, + queueModelExecutionsLocally +} from "../localex/local-execution-functions"; import fs from "fs-extra"; @@ -11,13 +39,13 @@ import { DEVMODE } from "../../config/app"; import { registerDataset } from "./data-catalog-functions"; export const saveAndRunExecutionsLocally = async ( - thread: Thread, - modelid: string, - prefs: MintPreferences) => { - + thread: Thread, + modelid: string, + prefs: MintPreferences +) => { let ok = false; - for(let pmodelid in thread.model_ensembles) { - if(!modelid || (modelid == pmodelid)) { + for (const pmodelid in thread.model_ensembles) { + if (!modelid || modelid == pmodelid) { ok = await saveAndRunExecutionsForModelLocally(pmodelid, thread, prefs); if (!ok) { return false; @@ -34,80 +62,76 @@ export const saveAndRunExecutionsLocally = async ( if (ok) { console.log("Finished sending all executions for local execution"); return true; - } - else { + } else { return false; } -} - -export const saveAndRunExecutionsForModelLocally = async(modelid: string, - thread: Thread, - prefs: MintPreferences) => { +}; +export const saveAndRunExecutionsForModelLocally = async ( + modelid: string, + thread: Thread, + prefs: MintPreferences +) => { try { - if(!thread.execution_summary) - thread.execution_summary = {}; - - let model = thread.models[modelid]; - let thread_model_id = thread.model_ensembles[modelid].id; - - let thread_region = await getRegionDetails(thread.regionid) - let execution_details = getModelInputBindings(model, thread, thread_region); - - let threadModel = execution_details[0] as ThreadModelMap; - let inputIds = execution_details[1] as string[]; + if (!thread.execution_summary) thread.execution_summary = {}; + + const model = thread.models[modelid]; + const thread_model_id = thread.model_ensembles[modelid].id; + + const thread_region = await getRegionDetails(thread.regionid); + const execution_details = getModelInputBindings(model, thread, thread_region); + + const threadModel = execution_details[0] as ThreadModelMap; + const inputIds = execution_details[1] as string[]; // This is the part that creates all different run configurations // - Cross product of all input collections // - TODO: Change to allow flexibility - let configs = getModelInputConfigurations(threadModel, inputIds); + const configs = getModelInputConfigurations(threadModel, inputIds); - let datadir = prefs.localex.datadir; + const datadir = prefs.localex.datadir; - if(!fs.existsSync(datadir)) - fs.mkdirsSync(datadir); + if (!fs.existsSync(datadir)) fs.mkdirsSync(datadir); - if(configs != null) { + if (configs != null) { // Pre-Run Setup // Reset execution summary - let summary = { + const summary = { total_runs: configs.length, - submitted_runs : 0, + submitted_runs: 0, failed_runs: 0, successful_runs: 0, workflow_name: "", // No workflow. Local execution submitted_for_execution: true, submission_time: new Date() - } as ExecutionSummary + } as ExecutionSummary; + + if (!DEVMODE) await setThreadModelExecutionSummary(thread_model_id, summary); - if(!DEVMODE) - await setThreadModelExecutionSummary(thread_model_id, summary); - // Load the component model - let component = await loadModelWCM(model.code_url, model, prefs); + const component = await loadModelWCM(model.code_url, model, prefs); // Delete existing thread execution ids - if(!DEVMODE) - await deleteThreadModelExecutionIds(thread_model_id); - + if (!DEVMODE) await deleteThreadModelExecutionIds(thread_model_id); + // Work in batches - let batchSize = 500; // Store executions in the database in batches + const batchSize = 500; // Store executions in the database in batches // Create executions in batches - for(let i=0; i { - let inputBindings : any = {}; - for(let j=0; j successful_execution_ids.indexOf(e.id) < 0); + const successful_execution_ids: string[] = DEVMODE + ? [] + : await listSuccessfulExecutionIds(executionids); + const executions_to_be_run = executions.filter( + (e) => successful_execution_ids.indexOf(e.id) < 0 + ); // Create Executions and Thread Model Mappings to those executions - if(!DEVMODE) { + if (!DEVMODE) { await setExecutions(executions_to_be_run, thread_model_id); await setThreadModelExecutionIds(thread_model_id, executionids); - - let num_already_run = successful_execution_ids.length; - if(num_already_run > 0) { + + const num_already_run = successful_execution_ids.length; + if (num_already_run > 0) { await incrementThreadModelSubmittedRuns(thread_model_id, num_already_run); await incrementThreadModelSuccessfulRuns(thread_model_id, num_already_run); } } // Queue the model executions - queueModelExecutionsLocally(thread, modelid, component, thread_region, executions_to_be_run, prefs); + queueModelExecutionsLocally( + thread, + modelid, + component, + thread_region, + executions_to_be_run, + prefs + ); } } console.log("Finished submitting all executions for model: " + modelid); return true; - } - catch(e) { + } catch (e) { console.log(e); } return false; -} +}; -export const deleteExecutableCacheLocally = async( - thread: Thread, +export const deleteExecutableCacheLocally = async ( + thread: Thread, modelid: string, - prefs: MintPreferences) => { - - for(let pmodelid in thread.model_ensembles) { - if(!modelid || (modelid == pmodelid)) + prefs: MintPreferences +) => { + for (const pmodelid in thread.model_ensembles) { + if (!modelid || modelid == pmodelid) await deleteExecutableCacheForModelLocally(pmodelid, thread, prefs); } console.log("Finished deleting all execution cache for local execution"); //monitorAllEnsembles(thread, problem_statement, prefs); -} +}; export const deleteModelInputCacheLocally = ( thread: Thread, modelid: string, - prefs: MintPreferences) => { - + prefs: MintPreferences +) => { // Delete the selected datasets - for(let dsid in thread.data) { - let ds = thread.data[dsid]; + for (const dsid in thread.data) { + const ds = thread.data[dsid]; ds.resources.map((res) => { - let file = prefs.localex.datadir + "/" + res.name; - if(fs.existsSync(file)) { - fs.remove(file) + const file = prefs.localex.datadir + "/" + res.name; + if (fs.existsSync(file)) { + fs.remove(file); } - }) + }); } // Also delete any model setup hardcoded input datasets - let model = thread.models[modelid]; + const model = thread.models[modelid]; model.input_files.map((io) => { - if(io.value) { + if (io.value) { // There is a hardcoded value in the model itself - let resources = io.value.resources; - if(resources.length > 0) { - let type = io.type.replace(/^.*#/, ''); + const resources = io.value.resources; + if (resources.length > 0) { + const type = io.type.replace(/^.*#/, ""); resources.map((res) => { - if(res.url) { - let filename = res.url.replace(/^.*(#|\/)/, ''); - filename = filename.replace(/^([0-9])/, '_$1'); - let file = prefs.localex.datadir + "/" + filename; - if(fs.existsSync(file)) { - fs.remove(file) + if (res.url) { + let filename = res.url.replace(/^.*(#|\/)/, ""); + filename = filename.replace(/^([0-9])/, "_$1"); + const file = prefs.localex.datadir + "/" + filename; + if (fs.existsSync(file)) { + fs.remove(file); } } - }) + }); } } }); +}; -} - -export const deleteModelCache = ( - model: Model, - prefs: MintPreferences) => { - // Delete cached model directory and zip file - let modeldir = getModelCacheDirectory(model.code_url, prefs); - if(modeldir != null) { - console.log("Deleting model directory: " + modeldir) - fs.remove(modeldir); - fs.remove(modeldir + ".zip"); - } - - deleteModel(model.id); - -} +export const deleteModelCache = (model: Model, prefs: MintPreferences) => { + // Delete cached model directory and zip file + const modeldir = getModelCacheDirectory(model.code_url, prefs); + if (modeldir != null) { + console.log("Deleting model directory: " + modeldir); + fs.remove(modeldir); + fs.remove(modeldir + ".zip"); + } -export const deleteExecutableCacheForModelLocally = async(modelid: string, - thread: Thread, - prefs: MintPreferences) => { + deleteModel(model.id); +}; - let model = thread.models[modelid]; - let thread_model_id = thread.model_ensembles[modelid].id; +export const deleteExecutableCacheForModelLocally = async ( + modelid: string, + thread: Thread, + prefs: MintPreferences +) => { + const model = thread.models[modelid]; + const thread_model_id = thread.model_ensembles[modelid].id; - let all_execution_ids = await getThreadModelExecutionIds(thread_model_id); + const all_execution_ids = await getThreadModelExecutionIds(thread_model_id); // Delete existing thread execution ids (*NOT* deleting global execution records .. Only clearing list of the thread's execution id mappings) deleteThreadModelExecutionIds(thread_model_id); // Work in batches - let batchSize = 500; + const batchSize = 500; // Process executions in batches - for(let i=0; i { - + prefs: MintPreferences +) => { let ok = false; - for(let pmodelid in thread.model_ensembles) { - if(!modelid || (modelid == pmodelid)) { + for (const pmodelid in thread.model_ensembles) { + if (!modelid || modelid == pmodelid) { ok = await registerModelExecutionResults(pmodelid, thread, prefs); if (!ok) { return false; @@ -284,83 +314,80 @@ export const registerExecutionResults = async ( if (ok) { console.log("Finished registering all execution outputs"); return true; - } - else { + } else { return false; } -} - -export const registerModelExecutionResults = async(modelid: string, - thread: Thread, - prefs: MintPreferences) => { +}; +export const registerModelExecutionResults = async ( + modelid: string, + thread: Thread, + prefs: MintPreferences +) => { try { - if(!thread.execution_summary) - thread.execution_summary = {}; - - let model = thread.models[modelid]; - let thread_model_id = thread.model_ensembles[modelid].id; - + if (!thread.execution_summary) thread.execution_summary = {}; + + const model = thread.models[modelid]; + const thread_model_id = thread.model_ensembles[modelid].id; + await setThreadModelExecutionSummary(thread_model_id, { submitted_for_ingestion: true, submitted_for_publishing: true, submitted_for_registration: true } as ExecutionSummary); - let all_execution_ids = await getThreadModelExecutionIds(thread_model_id); + const all_execution_ids = await getThreadModelExecutionIds(thread_model_id); // Work in batches - let batchSize = 500; + const batchSize = 500; // Process executions in batches - for(let i=0; i { - - let executions = await getExecutions(executionids); + prefs: MintPreferences +) => { + const executions = await getExecutions(executionids); executions.map(async (execution) => { // Only register outputs of successful executions - if(execution.status == "SUCCESS") { + if (execution.status == "SUCCESS") { // Copy any input's spatial/temporal input (if any) let spatial = null; let temporal = null; - for(let inputid in execution.bindings) { - let input = execution.bindings[inputid]; - if(input["spatial_coverage"]) { - spatial = JSON.stringify(input["spatial_coverage"]) + for (const inputid in execution.bindings) { + const input = execution.bindings[inputid]; + if (input["spatial_coverage"]) { + spatial = JSON.stringify(input["spatial_coverage"]); } - if(input["start_date"] && input["end_date"]) { + if (input["start_date"] && input["end_date"]) { temporal = input["start_date"] + " to " + input["end_date"]; } } - let promises = []; + const promises = []; // Register outputs in the data catalog model.output_files.map((outf) => { if (execution.results[outf.id]) { - // Register output with the appropriate variable and spatio-temporal information from input (if transferred) - let output = execution.results[outf.id]; - let ds = { + // Register output with the appropriate variable and spatio-temporal information from input (if transferred) + const output = execution.results[outf.id]; + const ds = { name: `${outf.name}-${execution.id}`, variables: outf.variables, datatype: output.type, @@ -368,22 +395,24 @@ export const registerExecutionOutputsInCatalog = async( is_cached: false, resource_repr: null, dataset_repr: null, - resources: [{ - name: `${output.name}-${execution.id}`, - url: output.url - }], + resources: [ + { + name: `${output.name}-${execution.id}`, + url: output.url + } + ], resource_count: 1, - spatial_coverage: spatial, - } as Dataset + spatial_coverage: spatial + } as Dataset; promises.push(registerDataset(ds, prefs)); } }); - let values = await Promise.all(promises); + const values = await Promise.all(promises); if (values.length == model.output_files.length) { console.log("Finished registering outputs for execution: " + execution.id); } } }); -} \ No newline at end of file +}; diff --git a/src/classes/mint/mint-types.ts b/src/classes/mint/mint-types.ts index cb944b3..7431d83 100644 --- a/src/classes/mint/mint-types.ts +++ b/src/classes/mint/mint-types.ts @@ -1,247 +1,247 @@ -import { env } from "process" +import { env } from "process"; export interface IdMap { - [id: string]: T + [id: string]: T; } export interface IdNameObject { - id?: string - name?: string + id?: string; + name?: string; } export interface User { - email: string - uid: string - region: string - graph: string + email: string; + uid: string; + region: string; + graph: string; } export interface AppState { - page: string, - subpage: string, - user?: User, - prefs?: UserPreferences + page: string; + subpage: string; + user?: User; + prefs?: UserPreferences; } export interface UserPreferences { - mint: MintPreferences + mint: MintPreferences; } export interface MintPreferences { - data_catalog_api: string, - data_catalog_type: string, - data_catalog_key: string, - data_catalog_extra: any, - - model_catalog_api?: string, - ensemble_manager_api: string, - ingestion_api: string, - visualization_url: string, - execution_engine?: "wings" | "localex", + data_catalog_api: string; + data_catalog_type: string; + data_catalog_key: string; + data_catalog_extra: any; + + model_catalog_api?: string; + ensemble_manager_api: string; + ingestion_api: string; + visualization_url: string; + execution_engine?: "wings" | "localex"; // Local Execution - localex?: LocalExecutionPreferences, - // Wings Execution - wings?: WingsPreferences, - graphql?: GraphQLPreferences, - wings_api?: string, + localex?: LocalExecutionPreferences; + // Wings Execution + wings?: WingsPreferences; + graphql?: GraphQLPreferences; + wings_api?: string; //maps - google_maps_key: string, + google_maps_key: string; //auth - auth_server: string, - auth_realm: string, - auth_client_id: string, + auth_server: string; + auth_realm: string; + auth_client_id: string; //old - apiKey: string, - authDomain: string, - databaseURL: string, - projectId: string, - storageBucket: string, - messagingSenderId: string, - appId: string, - cromo_api?: string + apiKey: string; + authDomain: string; + databaseURL: string; + projectId: string; + storageBucket: string; + messagingSenderId: string; + appId: string; + cromo_api?: string; } export interface GraphQLPreferences { - endpoint: string, - enable_ssl: boolean, - secret: string, - username: string, - password: string + endpoint: string; + enable_ssl: boolean; + secret: string; + username: string; + password: string; } export interface WingsPreferences { - server: string, - domain: string, - username: string, - password: string, - datadir: string, - dataurl: string + server: string; + domain: string; + username: string; + password: string; + datadir: string; + dataurl: string; // The following is retrieved from wings itself - export_url?: string, - storage?: string, - dotpath?: string, - onturl?: string, + export_url?: string; + storage?: string; + dotpath?: string; + onturl?: string; } export interface LocalExecutionPreferences { - datadir: string, - dataurl: string, - logdir: string, - logurl: string, - codedir: string, - tempdir: string, - parallelism: number + datadir: string; + dataurl: string; + logdir: string; + logurl: string; + codedir: string; + tempdir: string; + parallelism: number; } export type RegionMap = IdMap; export interface Region extends IdNameObject { - bounding_box?: BoundingBox, - model_catalog_uri?: string, - category_id: string, - geometries: any[] + bounding_box?: BoundingBox; + model_catalog_uri?: string; + category_id: string; + geometries: any[]; } export interface RegionsState { - regions?: RegionMap, - top_region_ids?: string[], - sub_region_ids?: IdMap, - categories?: IdMap - bbox_preview?: BoundingBox[] + regions?: RegionMap; + top_region_ids?: string[]; + sub_region_ids?: IdMap; + categories?: IdMap; + bbox_preview?: BoundingBox[]; } export interface RegionCategory extends IdNameObject { - citation?: string, - subcategories?: RegionCategory[] + citation?: string; + subcategories?: RegionCategory[]; } export interface BoundingBox { - xmin: number - xmax: number - ymin: number - ymax: number + xmin: number; + xmax: number; + ymin: number; + ymax: number; } export interface Point { - x: number, - y: number + x: number; + y: number; } export interface Dataset extends IdNameObject { - region: string, - variables: string[], - datatype: string, - time_period: DateRange, - description: string, - version: string, - limitations: string, - source: Source, - categories?: string[], - is_cached?: boolean, - resource_repr?: any, - dataset_repr?: any, - resources: DataResource[], - resources_loaded?: boolean, - resource_count?: number, - spatial_coverage?: any, -}; + region: string; + variables: string[]; + datatype: string; + time_period: DateRange; + description: string; + version: string; + limitations: string; + source: Source; + categories?: string[]; + is_cached?: boolean; + resource_repr?: any; + dataset_repr?: any; + resources: DataResource[]; + resources_loaded?: boolean; + resource_count?: number; + spatial_coverage?: any; +} export interface Dataslice extends IdNameObject { - dataset: Dataset, - total_resources?: number, - selected_resources?: number, - resources: DataResource[], - resources_loaded?: boolean, - time_period: DateRange, - spatial_coverage?: any + dataset: Dataset; + total_resources?: number; + selected_resources?: number; + resources: DataResource[]; + resources_loaded?: boolean; + time_period: DateRange; + spatial_coverage?: any; } export interface DataResource extends IdNameObject { - url: string - time_period?: DateRange, - spatial_coverage?: any - selected? : boolean + url: string; + time_period?: DateRange; + spatial_coverage?: any; + selected?: boolean; } export interface DatasetDetail extends Dataset { - documentation: string, - image: string + documentation: string; + image: string; } export interface Source { - name: string, - url: string, - type: string + name: string; + url: string; + type: string; } export interface DatasetQueryParameters { - spatialCoverage?: BoundingBox, - dateRange?: DateRange, - name?: string, - variables?: string[] + spatialCoverage?: BoundingBox; + dateRange?: DateRange; + name?: string; + variables?: string[]; } export interface DatasetsState { - model_datasets?: ModelDatasets - query_datasets?: DatasetsWithStatus - region_datasets?: DatasetsWithStatus - dataset?: DatasetWithStatus - datasets?: IdMap + model_datasets?: ModelDatasets; + query_datasets?: DatasetsWithStatus; + region_datasets?: DatasetsWithStatus; + dataset?: DatasetWithStatus; + datasets?: IdMap; } export interface DatasetsWithStatus { - loading: boolean, - datasets: Dataset[] + loading: boolean; + datasets: Dataset[]; } export interface DatasetWithStatus { - loading: boolean, - dataset: Dataset + loading: boolean; + dataset: Dataset; } -export type ModelInputDatasets = Map -export type ModelDatasets = Map +export type ModelInputDatasets = Map; +export type ModelDatasets = Map; export interface Model extends IdNameObject { - localname?: string, - region_name: string, - description?: string, - category: string, - input_files: ModelIO[], - input_parameters: ModelParameter[], - output_files: ModelIO[], - code_url?: string, - model_type?: string, - model_name?: string, - model_version?: string, - model_configuration?:string, - parameter_assignment?: string, - parameter_assignment_details?: string, - software_image?: string, - calibration_target_variable?: string, - modeled_processes?: string[], - dimensionality?: number|string, - spatial_grid_type?: string, - spatial_grid_resolution?: string, - output_time_interval?: string, - usage_notes?: string, - indicators?: string, - hasRegion?: any -}; - -const getLastPart = (s:string) => { - let sp = s.split('/'); + localname?: string; + region_name: string; + description?: string; + category: string; + input_files: ModelIO[]; + input_parameters: ModelParameter[]; + output_files: ModelIO[]; + code_url?: string; + model_type?: string; + model_name?: string; + model_version?: string; + model_configuration?: string; + parameter_assignment?: string; + parameter_assignment_details?: string; + software_image?: string; + calibration_target_variable?: string; + modeled_processes?: string[]; + dimensionality?: number | string; + spatial_grid_type?: string; + spatial_grid_resolution?: string; + output_time_interval?: string; + usage_notes?: string; + indicators?: string; + hasRegion?: any; +} + +const getLastPart = (s: string) => { + const sp = s.split("/"); if (sp && sp.length > 0) return sp.pop(); - return ''; -} + return ""; +}; -export const getPathFromModel = (m:Model) => { +export const getPathFromModel = (m: Model) => { let path = ""; - let model = getLastPart(m.model_name); + const model = getLastPart(m.model_name); if (model) { path += "/" + model; - let version = getLastPart(m.model_version); + const version = getLastPart(m.model_version); if (version) { path += "/" + version; - let cfg = getLastPart(m.model_configuration) + const cfg = getLastPart(m.model_configuration); if (cfg) { path += "/" + cfg; path += "/" + getLastPart(m.id); @@ -249,268 +249,274 @@ export const getPathFromModel = (m:Model) => { } } return path; -} +}; export interface ModelIO extends IdNameObject { - type?: string, - variables: string[], - value?: Dataslice, - position?: number, - format?: string + type?: string; + variables: string[]; + value?: Dataslice; + position?: number; + format?: string; } export interface ModelParameter extends IdNameObject { - type: string, - description?: string, - min?: string, - max?: string, - unit?: string, - default?: string, - value?: string, - adjustment_variable?: string, - accepted_values?: string[], - position?: number + type: string; + description?: string; + min?: string; + max?: string; + unit?: string; + default?: string; + value?: string; + adjustment_variable?: string; + accepted_values?: string[]; + position?: number; } export interface ModelDetail extends Model { - documentation: string, - lookupTable: string, - image: string + documentation: string; + lookupTable: string; + image: string; } export interface ModelsState { - models: VariableModels, - model: ModelDetail | null, - loading: boolean + models: VariableModels; + model: ModelDetail | null; + loading: boolean; } -export type VariableModels = Map // Map of response variable to models +export type VariableModels = Map; // Map of response variable to models export interface ModelingState { - problem_statements?: ProblemStatementList - problem_statement?: ProblemStatement - thread?: Thread - execution_summaries?: ThreadModelExecutionSummary - executions?: ThreadModelExecutions + problem_statements?: ProblemStatementList; + problem_statement?: ProblemStatement; + thread?: Thread; + execution_summaries?: ThreadModelExecutionSummary; + executions?: ThreadModelExecutions; } export interface ExecutionsWithStatus { - loading: boolean, - changed: boolean, - executions: Execution[] + loading: boolean; + changed: boolean; + executions: Execution[]; } export type ModelExecutions = { - [modelid: string]: ExecutionsWithStatus -} + [modelid: string]: ExecutionsWithStatus; +}; export type ThreadModelExecutions = { - [threadid: string]: ModelExecutions -} - + [threadid: string]: ModelExecutions; +}; export type ThreadModelExecutionSummary = { - [threadid: string]: ModelExecutionSummary -} + [threadid: string]: ModelExecutionSummary; +}; export type ModelExecutionSummary = { - [modelid: string]: ExecutionSummary -} + [modelid: string]: ExecutionSummary; +}; export interface MintPermission { - userid: string, - read: boolean, - write: boolean, - execute: boolean, - owner: boolean + userid: string; + read: boolean; + write: boolean; + execute: boolean; + owner: boolean; } export interface MintEvent { - event: string, - userid: string - timestamp: Date - notes: string + event: string; + userid: string; + timestamp: Date; + notes: string; } export interface ProblemStatementEvent extends MintEvent { - event: "CREATE" | "UPDATE" | "ADD_TASK" | "DELETE_TASK" + event: "CREATE" | "UPDATE" | "ADD_TASK" | "DELETE_TASK"; } export interface TaskEvent extends MintEvent { - event: "CREATE" | "UPDATE" | "ADD_THREAD" | "DELETE_THREAD" + event: "CREATE" | "UPDATE" | "ADD_THREAD" | "DELETE_THREAD"; } export interface ThreadEvent extends MintEvent { - event: "CREATE" | "UPDATE" | "SELECT_DATA" | "SELECT_MODELS" | "SELECT_PARAMETERS" | "EXECUTE" | "INGEST" | "VISUALIZE" + event: + | "CREATE" + | "UPDATE" + | "SELECT_DATA" + | "SELECT_MODELS" + | "SELECT_PARAMETERS" + | "EXECUTE" + | "INGEST" + | "VISUALIZE"; } export interface ProblemStatementList { - problem_statement_ids: string[] - problem_statements: IdMap - unsubscribe?: Function + problem_statement_ids: string[]; + problem_statements: IdMap; + unsubscribe?: Function; } export interface ProblemStatementInfo extends IdNameObject { - regionid: string - dates: DateRange - events?: ProblemStatementEvent[] - permissions?: MintPermission[] - preview?: string[] + regionid: string; + dates: DateRange; + events?: ProblemStatementEvent[]; + permissions?: MintPermission[]; + preview?: string[]; } export interface DateRange { - start_date: Date - end_date: Date + start_date: Date; + end_date: Date; } export interface ProblemStatement extends ProblemStatementInfo { - tasks: IdMap - changed?: boolean - unsubscribe?: Function + tasks: IdMap; + changed?: boolean; + unsubscribe?: Function; } export interface ThreadInfo extends IdNameObject { - dates?: DateRange - task_id: string - driving_variables: string[] - response_variables: string[] - regionid?: string - events?: ThreadEvent[] - permissions?: MintPermission[] + dates?: DateRange; + task_id: string; + driving_variables: string[]; + response_variables: string[]; + regionid?: string; + events?: ThreadEvent[]; + permissions?: MintPermission[]; } export interface ThreadList { - thread_ids: string[] - threads: IdMap - unsubscribe?: Function + thread_ids: string[]; + threads: IdMap; + unsubscribe?: Function; } export interface Thread extends ThreadInfo { - models?: ModelMap - data?: DataMap - model_ensembles?: ModelEnsembleMap + models?: ModelMap; + data?: DataMap; + model_ensembles?: ModelEnsembleMap; //data_transformations?: IdMap - model_dt_ensembles?: ModelEnsembleMap - execution_summary: IdMap - visualizations?: Visualization[] - events: ThreadEvent[] - changed?: boolean - refresh?: boolean - unsubscribe?: Function + model_dt_ensembles?: ModelEnsembleMap; + execution_summary: IdMap; + visualizations?: Visualization[]; + events: ThreadEvent[]; + changed?: boolean; + refresh?: boolean; + unsubscribe?: Function; } export interface Visualization { - type: string, - url: string + type: string; + url: string; } export interface ComparisonFeature { - name: string, - fn: Function + name: string; + fn: Function; } -export type ModelMap = IdMap +export type ModelMap = IdMap; -export type DataMap = IdMap +export type DataMap = IdMap; export interface TaskList { - task_ids: string[] - tasks: IdMap - unsubscribe?: Function + task_ids: string[]; + tasks: IdMap; + unsubscribe?: Function; } export interface Task extends IdNameObject { - problem_statement_id: string, - dates?: DateRange, - response_variables: string[], - driving_variables: string[], - regionid?: string, - threads?: IdMap - events?: TaskEvent[] - permissions?: MintPermission[] - unsubscribe?: Function + problem_statement_id: string; + dates?: DateRange; + response_variables: string[]; + driving_variables: string[]; + regionid?: string; + threads?: IdMap; + events?: TaskEvent[]; + permissions?: MintPermission[]; + unsubscribe?: Function; } // Mapping of model id to data ensembles export interface ModelEnsembleMap { - [modelid: string]: ThreadModelMap + [modelid: string]: ThreadModelMap; } export interface ThreadModelMap { - id: string, - bindings: ModelIOBindings + id: string; + bindings: ModelIOBindings; } // Mapping of model input to list of values (data ids or parameter values) export interface ModelIOBindings { - [inputid: string]: string[] + [inputid: string]: string[]; } export interface ExecutionSummary { - workflow_name?: string - changed?: boolean - unsubscribe?: Function + workflow_name?: string; + changed?: boolean; + unsubscribe?: Function; - submitted_for_execution: boolean - submission_time: Date - total_runs: number - submitted_runs: number - successful_runs: number - failed_runs: number + submitted_for_execution: boolean; + submission_time: Date; + total_runs: number; + submitted_runs: number; + successful_runs: number; + failed_runs: number; - submitted_for_ingestion: boolean - fetched_run_outputs: number // Run data fetched for ingestion - ingested_runs: number // Run data ingested in the Visualization database + submitted_for_ingestion: boolean; + fetched_run_outputs: number; // Run data fetched for ingestion + ingested_runs: number; // Run data ingested in the Visualization database - submitted_for_registration: boolean - registered_runs: number // Registered in the data catalog + submitted_for_registration: boolean; + registered_runs: number; // Registered in the data catalog - submitted_for_publishing: boolean - published_runs: number // Published in the provenance catalog + submitted_for_publishing: boolean; + published_runs: number; // Published in the provenance catalog } export interface Execution { - id?: string - modelid: string - bindings: InputBindings - runid?: string - start_time: Date - end_time?: Date - execution_engine?: "wings" | "localex" - status: "FAILURE" | "SUCCESS" | "RUNNING" | "WAITING", - run_progress?: number // 0 to 100 (percentage done) - results: any[] // Chosen results after completed run - selected: boolean + id?: string; + modelid: string; + bindings: InputBindings; + runid?: string; + start_time: Date; + end_time?: Date; + execution_engine?: "wings" | "localex"; + status: "FAILURE" | "SUCCESS" | "RUNNING" | "WAITING"; + run_progress?: number; // 0 to 100 (percentage done) + results: any[]; // Chosen results after completed run + selected: boolean; } export interface InputBindings { - [input: string]: string | DataResource + [input: string]: string | DataResource; } export type VariableMap = IdMap; export interface Variable extends IdNameObject { - description: string, - is_adjustment_variable: boolean, - is_indicator: boolean, - categories: string[], - intervention: Intervention + description: string; + is_adjustment_variable: boolean; + is_indicator: boolean; + categories: string[]; + intervention: Intervention; } export interface Intervention extends IdNameObject { - description: string + description: string; } export interface VariablesState { - variables?: VariableMap + variables?: VariableMap; } export interface Wcm { - wings: any ; + wings: any; } export interface WingsWcm { - inputs: ModelIO[], - outputs: ModelIO[], - -} \ No newline at end of file + inputs: ModelIO[]; + outputs: ModelIO[]; +} diff --git a/src/classes/mint/model-catalog-functions.ts b/src/classes/mint/model-catalog-functions.ts index 1478ca7..43367de 100644 --- a/src/classes/mint/model-catalog-functions.ts +++ b/src/classes/mint/model-catalog-functions.ts @@ -1,17 +1,19 @@ import { Model, MintPreferences, ModelIO, Dataset, ModelParameter, Dataslice } from "./mint-types"; import * as rp from "request-promise-native"; -import { ModelConfigurationSetup } from '@mintproject/modelcatalog_client'; +import { ModelConfigurationSetup } from "@mintproject/modelcatalog_client"; import { GraphQL } from "../../config/graphql"; -import { KeycloakAdapter } from '../../config/keycloak-adapter'; +import { KeycloakAdapter } from "../../config/keycloak-adapter"; -// Query Model Catalog By Variables, +// Query Model Catalog By Variables, // - Filter by driving variables and model id/name (match with calibration) // - Return only 1 model -export const fetchModelFromCatalog = (response_variables: string[], - driving_variables: string[], modelid: string, - prefs: MintPreferences) : Promise => { - - let username = KeycloakAdapter.getUser().email; +export const fetchModelFromCatalog = ( + response_variables: string[], + driving_variables: string[], + modelid: string, + prefs: MintPreferences +): Promise => { + const username = KeycloakAdapter.getUser().email; return new Promise((resolve, reject) => { rp.get({ url: prefs.model_catalog_api + "custom/modelconfigurationsetups/variable", @@ -19,27 +21,30 @@ export const fetchModelFromCatalog = (response_variables: string[], json: true }).then((setups) => { let found = false; - for(var i=0; i { resolve(setup); - }) + }); break; // Return only 1 match } } - if(!found) { + if (!found) { reject(); } - }) + }); }); -}; \ No newline at end of file +}; diff --git a/src/classes/wings/wings-functions.ts b/src/classes/wings/wings-functions.ts index fbc1ee3..e78c41d 100644 --- a/src/classes/wings/wings-functions.ts +++ b/src/classes/wings/wings-functions.ts @@ -1,406 +1,494 @@ import { getResource, postFormResource, postJSONResource } from "./xhr-requests"; -import {IdMap, MintPreferences, Execution, Thread, Model, DataResource} from "../mint/mint-types"; -import { WingsComponent, WingsTemplatePackage, WingsTemplate, WingsDataBindings, WingsParameterBindings, WingsTemplateMetadata, WingsPort, WingsPortRole, WingsDataVariable, WingsParameterVariable, WingsNode, WingsComponentVariable, WingsTemplateSeed, WingsParameterTypes } from "./wings-types"; - -export const loginToWings = async(config: MintPreferences) : Promise => { - let config_wings = config.wings; +import { IdMap, MintPreferences, Execution, Thread, Model, DataResource } from "../mint/mint-types"; +import { + WingsComponent, + WingsTemplatePackage, + WingsTemplate, + WingsDataBindings, + WingsParameterBindings, + WingsTemplateMetadata, + WingsPort, + WingsPortRole, + WingsDataVariable, + WingsParameterVariable, + WingsNode, + WingsComponentVariable, + WingsTemplateSeed, + WingsParameterTypes +} from "./wings-types"; + +export const loginToWings = async (config: MintPreferences): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { getResource({ - url: config_wings.server + '/login', - onLoad: function(e: any) { - var txt = e.target.responseText; - if(txt.match(/j_security_check/)) { - var data = { - j_username: config_wings.username, - j_password: config_wings.password - }; - postFormResource({ - url: config_wings.server + '/j_security_check', - onLoad: function(e2: any) { - getResource({ - // Try to access the login page again - url: config_wings.server + '/login', - onLoad: function(e3: any) { - var txt2 = e2.target.responseText; - if(txt2.match(/j_security_check/)) { - reject(); - } - else { - // Success: Logged in - resolve(); - //console.log("Success !! Logged in"); + url: config_wings.server + "/login", + onLoad: function (e: any) { + const txt = e.target.responseText; + if (txt.match(/j_security_check/)) { + const data = { + j_username: config_wings.username, + j_password: config_wings.password + }; + postFormResource( + { + url: config_wings.server + "/j_security_check", + onLoad: function (e2: any) { + getResource({ + // Try to access the login page again + url: config_wings.server + "/login", + onLoad: function (e3: any) { + const txt2 = e2.target.responseText; + if (txt2.match(/j_security_check/)) { + reject(); + } else { + // Success: Logged in + resolve(); + //console.log("Success !! Logged in"); + } + }, + onError: function () { + console.log("Cannot login"); + reject(); + } + }); + }, + onError: function () { + console.log("Cannot login"); + getResource({ + url: config_wings.server + "/login", + onLoad: function (e2: any) { + const match = /USER_ID\s*=\s*"(.+)"\s*;/.exec( + e2.target.responseText + ); + if (match) { + const userid = match[1]; + resolve(); + //console.log("Logged in as " + userid + " !"); + } + }, + onError: function () { + reject(); + } + }); } }, - onError: function() { - console.log("Cannot login"); - reject(); - } - }); - }, - onError: function() { - console.log("Cannot login"); - getResource({ - url: config_wings.server + '/login', - onLoad: function(e2: any) { - var match = /USER_ID\s*=\s*"(.+)"\s*;/.exec(e2.target.responseText); - if(match) { - let userid = match[1]; - resolve(); - //console.log("Logged in as " + userid + " !"); - } - }, - onError: function(){ - reject(); - } - }); - } - }, data, true); - } else { - var match = /USER_ID\s*=\s*"(.+)"\s*;/.exec(txt); - if(match) { - let userid = match[1]; - resolve(); - //console.log("Already Logged in as " + userid + " !"); + data, + true + ); + } else { + const match = /USER_ID\s*=\s*"(.+)"\s*;/.exec(txt); + if (match) { + const userid = match[1]; + resolve(); + //console.log("Already Logged in as " + userid + " !"); + } } - } }, - onError: function(e: any) { + onError: function (e: any) { reject(); console.log("Cannot connect to wings"); } - }); + }); }); -} +}; -export const fetchWingsComponent = async(cname: string, config: MintPreferences) : Promise => { - let config_wings = config.wings; +export const fetchWingsComponent = async ( + cname: string, + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - var exurl = config_wings.export_url + "/export/users/" + config_wings.username + "/" + config_wings.domain; - var cid = exurl + "/components/library.owl#" + cname; + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const exurl = + config_wings.export_url + + "/export/users/" + + config_wings.username + + "/" + + config_wings.domain; + const cid = exurl + "/components/library.owl#" + cname; getResource({ url: purl + "/components/getComponentJSON?cid=" + escape(cid), - onLoad: function(e: any) { - let comp = JSON.parse(e.target.responseText) as WingsComponent; + onLoad: function (e: any) { + const comp = JSON.parse(e.target.responseText) as WingsComponent; resolve(comp); }, - onError: function(e: any) { + onError: function (e: any) { reject("Could not get component"); } }); }); -} +}; -export const logoutFromWings = async(config: MintPreferences) : Promise => { - let config_wings = config.wings; +export const logoutFromWings = async (config: MintPreferences): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { getResource({ - url: config_wings.server + '/jsp/login/logout.jsp', - onLoad: function(e: any) { - resolve(); - //console.log("Logged out"); - }, - onError: function(e: any) { - reject("Could not logout !"); - //console.log("Could not logout !"); - } + url: config_wings.server + "/jsp/login/logout.jsp", + onLoad: function (e: any) { + resolve(); + //console.log("Logged out"); + }, + onError: function (e: any) { + reject("Could not logout !"); + //console.log("Could not logout !"); + } }); }); -} +}; -export const fetchWingsTemplatesList = async(config: MintPreferences) : Promise => { - let config_wings = config.wings; +export const fetchWingsTemplatesList = async (config: MintPreferences): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; getResource({ url: purl + "/workflows/getTemplatesListJSON", - onLoad: function(e: any) { - let list = JSON.parse(e.target.responseText) + onLoad: function (e: any) { + const list = JSON.parse(e.target.responseText); resolve(list); }, - onError: function(e: any) { + onError: function (e: any) { reject("Could not get templates list"); } }); }); -} +}; -export const fetchWingsTemplate = async(tid: string, config: MintPreferences) : Promise => { - let config_wings = config.wings; +export const fetchWingsTemplate = async ( + tid: string, + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; //var exurl = config_wings.export_url + "/export/users/" + config_wings.username + "/" + config_wings.domain; //var tid = exurl + "/workflows/" + tname + ".owl#" + tname; getResource({ url: purl + "/workflows/getEditorJSON?template_id=" + escape(tid), - onLoad: function(e: any) { - let tpl = JSON.parse(e.target.responseText) as WingsTemplatePackage; + onLoad: function (e: any) { + const tpl = JSON.parse(e.target.responseText) as WingsTemplatePackage; resolve(tpl); }, - onError: function(e: any) { + onError: function (e: any) { reject("Could not get template"); } }); }); -} +}; -export const saveWingsTemplate = async(tpl: WingsTemplatePackage, config: MintPreferences) : Promise => { +export const saveWingsTemplate = async ( + tpl: WingsTemplatePackage, + config: MintPreferences +): Promise => { //TODO: Get a MD5 Hash for template to check if it is already saved. // - To avoid cluttering up template library - let config_wings = config.wings; + const config_wings = config.wings; // Get url prefix for operations return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - var data = { + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const data = { template_id: tpl.template.id, constraints_json: JSON.stringify(tpl.constraints), json: JSON.stringify(tpl.template) }; - postFormResource({ - url: purl + "/workflows/saveTemplateJSON", - onLoad: function(e: any) { - resolve(); + postFormResource( + { + url: purl + "/workflows/saveTemplateJSON", + onLoad: function (e: any) { + resolve(); + }, + onError: function () { + reject("Cannot save"); + } }, - onError: function() { - reject("Cannot save"); - } - }, data, false); + data, + false + ); }); -} +}; -export const layoutWingsTemplate = async(tpl: WingsTemplate, config: MintPreferences) - : Promise => { - - let config_wings = config.wings; +export const layoutWingsTemplate = async ( + tpl: WingsTemplate, + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { // Get url prefix for operations - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - postJSONResource({ - url: purl + "/workflows/layoutTemplate", - onLoad: function(e: any) { - var ntpl = JSON.parse(e.target.responseText) as WingsTemplatePackage; - resolve(ntpl); + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + postJSONResource( + { + url: purl + "/workflows/layoutTemplate", + onLoad: function (e: any) { + const ntpl = JSON.parse(e.target.responseText) as WingsTemplatePackage; + resolve(ntpl); + }, + onError: function () { + reject("Cannot layout template"); + } }, - onError: function() { - reject("Cannot layout template"); - } - }, tpl); + tpl + ); }); -} +}; -export const elaborateWingsTemplate = async(tpl: WingsTemplatePackage, config: MintPreferences) - : Promise => { - - let config_wings = config.wings; +export const elaborateWingsTemplate = async ( + tpl: WingsTemplatePackage, + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { // Get url prefix for operations - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - var data = { + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const data = { template_id: tpl.template.id, constraints_json: JSON.stringify(tpl.constraints), json: JSON.stringify(tpl.template) - } - postFormResource({ - url: purl + "/plan/elaborateTemplateJSON", - onLoad: function(e:any) { - var ntpl = JSON.parse(e.target.responseText) as WingsTemplate; - resolve(ntpl); + }; + postFormResource( + { + url: purl + "/plan/elaborateTemplateJSON", + onLoad: function (e: any) { + const ntpl = JSON.parse(e.target.responseText) as WingsTemplate; + resolve(ntpl); + }, + onError: function () { + reject("Cannot elaborate template"); + } }, - onError: function() { - reject("Cannot elaborate template"); - } - }, data, true); + data, + true + ); }); -} +}; const _getComponentBindings = (tpl: WingsTemplatePackage) => { - var cbindings = {} as any; - for(var nid in tpl.template.Nodes) { - var c = tpl.template.Nodes[nid].componentVariable; + const cbindings = {} as any; + for (const nid in tpl.template.Nodes) { + const c = tpl.template.Nodes[nid].componentVariable; cbindings[c.id] = c.binding.id; } return cbindings; -} +}; -export const expandAndRunWingsWorkflow = async( - tpl: WingsTemplatePackage, +export const expandAndRunWingsWorkflow = async ( + tpl: WingsTemplatePackage, dataBindings: WingsDataBindings, parameterBindings: WingsParameterBindings, parameterTypes: WingsParameterBindings, - config: MintPreferences) - : Promise => { - - let config_wings = config.wings; + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { // Get url prefix for operations - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - var data = { + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const data = { templateId: tpl.template.id, parameterBindings: parameterBindings, parameterTypes: parameterTypes, componentBindings: _getComponentBindings(tpl), - dataBindings: dataBindings - } - postJSONResource({ - url: purl + "/executions/expandAndRunWorkflow", - onLoad: function(e: any) { - var runid = e.target.responseText; - if(runid) { - resolve(runid); - } - else { - reject("Could not run workflow"); + dataBindings: dataBindings + }; + postJSONResource( + { + url: purl + "/executions/expandAndRunWorkflow", + onLoad: function (e: any) { + const runid = e.target.responseText; + if (runid) { + resolve(runid); + } else { + reject("Could not run workflow"); + } + }, + onError: function () { + reject("Cannot run workflow"); } - }, - onError: function() { - reject("Cannot run workflow"); - } - }, - data); + }, + data + ); }); -} - -export const fetchWingsRunsStatuses = (template_name: string, start_time: number, total_runs: number, config: MintPreferences) - : Promise> => { +}; + +export const fetchWingsRunsStatuses = ( + template_name: string, + start_time: number, + total_runs: number, + config: MintPreferences +): Promise> => { return new Promise>((resolve, reject) => { let statuses = {} as Map; - let promises = []; - for(let i=0; i { vals.map((val) => { - statuses = Object.assign({}, statuses, val); + statuses = Object.assign({}, statuses, val); }); resolve(statuses); - }) - }) -} - -const _fetchWingsRunsStatuses = (template_name: string, start_time: number, start: number, limit: number, config: MintPreferences) - : Promise> => { - - let config_wings = config.wings; + }); + }); +}; + +const _fetchWingsRunsStatuses = ( + template_name: string, + start_time: number, + start: number, + limit: number, + config: MintPreferences +): Promise> => { + const config_wings = config.wings; return new Promise>((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; getResource({ - url: purl + "/executions/getRunListSimple?pattern="+template_name+"&start="+start+"&limit="+limit+"&sort=startTime&dir=ASC&started_after="+start_time, - onLoad: function(e: any) { - let runsjson = JSON.parse(e.target.responseText); - if(runsjson.success) { - let statuses: any = {}; - let runslist : any[] = runsjson.rows; + url: + purl + + "/executions/getRunListSimple?pattern=" + + template_name + + "&start=" + + start + + "&limit=" + + limit + + "&sort=startTime&dir=ASC&started_after=" + + start_time, + onLoad: function (e: any) { + const runsjson = JSON.parse(e.target.responseText); + if (runsjson.success) { + const statuses: any = {}; + const runslist: any[] = runsjson.rows; runslist.map((runjson) => { - let runid = runjson.id; + const runid = runjson.id; statuses[runid] = runjson.runtimeInfo; - }) + }); resolve(statuses); } }, - onError: function() { + onError: function () { reject("Cannot fetch runs"); } }); }); -} +}; -export const fetchWingsRunResults = (execution: Execution, config: MintPreferences) - : Promise => { - - let config_wings = config.wings; +export const fetchWingsRunResults = ( + execution: Execution, + config: MintPreferences +): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - if(!execution.runid) { + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + if (!execution.runid) { reject(); return; } - let data = { - run_id: execution.runid, + const data = { + run_id: execution.runid }; - postFormResource({ - url: purl + "/executions/getRunPlan", - onLoad: function(e: any) { - let ex = JSON.parse(e.target.responseText); - if(execution.status == "SUCCESS") { - // Look for outputs that aren't inputs to any other steps - let outputfiles : any = {}; - ex.plan.steps.map((step: any) => { - step.outputFiles.map((file: any) => { - outputfiles[file.id] = file; - }) - }) - ex.plan.steps.map((step: any) => { - step.inputFiles.map((file: any) => { - delete outputfiles[file.id]; - }) - }) - resolve(outputfiles); + postFormResource( + { + url: purl + "/executions/getRunPlan", + onLoad: function (e: any) { + const ex = JSON.parse(e.target.responseText); + if (execution.status == "SUCCESS") { + // Look for outputs that aren't inputs to any other steps + const outputfiles: any = {}; + ex.plan.steps.map((step: any) => { + step.outputFiles.map((file: any) => { + outputfiles[file.id] = file; + }); + }); + ex.plan.steps.map((step: any) => { + step.inputFiles.map((file: any) => { + delete outputfiles[file.id]; + }); + }); + resolve(outputfiles); + } + }, + onError: function () { + reject("Cannot get run details"); } }, - onError: function() { - reject("Cannot get run details"); - } - }, data, true); + data, + true + ); }); -} - -export const fetchWingsRunLog = (runid: string, config: MintPreferences) - : Promise => { +}; - let config_wings = config.wings; +export const fetchWingsRunLog = (runid: string, config: MintPreferences): Promise => { + const config_wings = config.wings; return new Promise((resolve, reject) => { - var purl = config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; - let data = { - run_id: runid, + const purl = + config_wings.server + "/users/" + config_wings.username + "/" + config_wings.domain; + const data = { + run_id: runid }; - postFormResource({ - url: purl + "/executions/getRunDetails", - onLoad: function(e: any) { - try { - let response = JSON.parse(e.target.responseText); - let ex = response.execution; - let log = ex.runtimeInfo.log; - log += "\n----------------------------------------------\n"; - ex.queue.steps.map((step: any) => { - log += step.id.replace(/.*#/, '') + "\n" - log += step.runtimeInfo.log; + postFormResource( + { + url: purl + "/executions/getRunDetails", + onLoad: function (e: any) { + try { + const response = JSON.parse(e.target.responseText); + const ex = response.execution; + let log = ex.runtimeInfo.log; log += "\n----------------------------------------------\n"; - }) - resolve(log); - } - catch(e) { - reject(); + ex.queue.steps.map((step: any) => { + log += step.id.replace(/.*#/, "") + "\n"; + log += step.runtimeInfo.log; + log += "\n----------------------------------------------\n"; + }); + resolve(log); + } catch (e) { + reject(); + } + }, + onError: function () { + reject("Cannot get run details"); } }, - onError: function() { - reject("Cannot get run details"); - } - }, data, true); + data, + true + ); }); -} - -export const createSingleComponentTemplate = (comp: WingsComponent, config: MintPreferences) : WingsTemplate => { - let config_wings = config.wings; - var cname = comp.id.replace(/^.+#/, ''); - var exurl = config_wings.export_url + "/export/users/" + config_wings.username + "/" + config_wings.domain; - var tname = "workflow_" + cname; - var tns = exurl + "/workflows/" + tname + ".owl#"; - var tid = tns + tname; - - var storage = config_wings.storage; - var dotpath = config_wings.dotpath; - var ontpfx = config_wings.onturl; - - var clibns = exurl + "/components/library.owl#"; - var usfx = "/users/" + config_wings.username + "/" + config_wings.domain; - var purl = config_wings.server + usfx; - var pdir = storage + usfx; - - let tpl : WingsTemplate = { +}; + +export const createSingleComponentTemplate = ( + comp: WingsComponent, + config: MintPreferences +): WingsTemplate => { + const config_wings = config.wings; + const cname = comp.id.replace(/^.+#/, ""); + const exurl = + config_wings.export_url + + "/export/users/" + + config_wings.username + + "/" + + config_wings.domain; + const tname = "workflow_" + cname; + const tns = exurl + "/workflows/" + tname + ".owl#"; + const tid = tns + tname; + + const storage = config_wings.storage; + const dotpath = config_wings.dotpath; + const ontpfx = config_wings.onturl; + + const clibns = exurl + "/components/library.owl#"; + const usfx = "/users/" + config_wings.username + "/" + config_wings.domain; + const purl = config_wings.server + usfx; + const pdir = storage + usfx; + + const tpl: WingsTemplate = { id: tid, Nodes: {}, Links: {}, @@ -417,38 +505,38 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint } as WingsTemplateMetadata, rules: {}, props: { - "lib.concrete.url": exurl + "/components/library.owl", - "lib.domain.execution.url": exurl + "/executions/library.owl", - "lib.domain.code.storage": pdir + "/code/library", - "domain.workflows.dir.url": exurl + "/workflows", - "user.id": config_wings.username, - "tdb.repository.dir": storage + "/TDB", - "viewer.id": config_wings.username, - "domain.executions.dir.url": exurl + "/executions", - "lib.domain.data.url": exurl + "/data/library.owl", - "ont.domain.data.url": exurl + "/data/ontology.owl", - "lib.abstract.url": exurl + "/components/abstract.owl", - "lib.provenance.url": config_wings.export_url + "/export/common/provenance/library.owl", - "ont.data.url": ontpfx + "/data.owl", - "lib.domain.data.storage": pdir + "/data", - "lib.domain.workflow.url": exurl + "/workflows/library.owl", - "lib.resource.url": config_wings.export_url + "/export/common/resource/library.owl", - "ont.component.url": ontpfx + "/component.owl", - "ont.workflow.url": ontpfx + "/workflow.owl", - "ont.dir.url": ontpfx, - "dot.path": dotpath, - "ont.domain.component.ns": clibns, - "ont.execution.url": ontpfx + "/execution.owl", - "ont.resource.url": ontpfx + "/resource.owl" + "lib.concrete.url": exurl + "/components/library.owl", + "lib.domain.execution.url": exurl + "/executions/library.owl", + "lib.domain.code.storage": pdir + "/code/library", + "domain.workflows.dir.url": exurl + "/workflows", + "user.id": config_wings.username, + "tdb.repository.dir": storage + "/TDB", + "viewer.id": config_wings.username, + "domain.executions.dir.url": exurl + "/executions", + "lib.domain.data.url": exurl + "/data/library.owl", + "ont.domain.data.url": exurl + "/data/ontology.owl", + "lib.abstract.url": exurl + "/components/abstract.owl", + "lib.provenance.url": config_wings.export_url + "/export/common/provenance/library.owl", + "ont.data.url": ontpfx + "/data.owl", + "lib.domain.data.storage": pdir + "/data", + "lib.domain.workflow.url": exurl + "/workflows/library.owl", + "lib.resource.url": config_wings.export_url + "/export/common/resource/library.owl", + "ont.component.url": ontpfx + "/component.owl", + "ont.workflow.url": ontpfx + "/workflow.owl", + "ont.dir.url": ontpfx, + "dot.path": dotpath, + "ont.domain.component.ns": clibns, + "ont.execution.url": ontpfx + "/execution.owl", + "ont.resource.url": ontpfx + "/resource.owl" } }; - let nodeid = tns + cname + "_node"; - let inputPorts : IdMap = {}; - let outputPorts : IdMap = {}; + const nodeid = tns + cname + "_node"; + const inputPorts: IdMap = {}; + const outputPorts: IdMap = {}; comp.inputs.map((arg) => { - let portid = nodeid + "_" + arg.role; + const portid = nodeid + "_" + arg.role; inputPorts[portid] = { id: portid, role: { @@ -457,11 +545,11 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint dimensionality: arg.dimensionality, id: portid + "_role" } as WingsPortRole - } - let varid = tns + arg.role; + }; + const varid = tns + arg.role; tpl.Variables[varid] = { id: varid, - type: arg.isParam ? 2 : 1, + type: arg.isParam ? 2 : 1 } as WingsDataVariable | WingsParameterVariable; tpl.inputRoles[varid] = { @@ -471,16 +559,16 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint id: varid + "_trole" }; - let linkid = portid + "_input"; + const linkid = portid + "_input"; tpl.Links[linkid] = { id: linkid, toNode: { id: nodeid }, toPort: { id: portid }, variable: { id: varid } - } + }; }); comp.outputs.map((arg) => { - let portid = nodeid + "_" + arg.role; + const portid = nodeid + "_" + arg.role; outputPorts[portid] = { id: portid, role: { @@ -489,13 +577,13 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint dimensionality: arg.dimensionality, id: portid + "_role" } as WingsPortRole - } - let varid = tns + arg.role; + }; + const varid = tns + arg.role; tpl.Variables[varid] = { id: varid, - type: arg.isParam ? 2 : 1, + type: arg.isParam ? 2 : 1 } as WingsDataVariable | WingsParameterVariable; - + tpl.outputRoles[varid] = { type: arg.isParam ? 2 : 1, roleid: portid, @@ -503,16 +591,16 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint id: varid + "_trole" }; - let linkid = portid + "_output"; + const linkid = portid + "_output"; tpl.Links[linkid] = { id: linkid, fromNode: { id: nodeid }, fromPort: { id: portid }, variable: { id: varid } - } - }); + }; + }); - let node : WingsNode = { + const node: WingsNode = { id: nodeid, componentVariable: { id: nodeid + "_comp", @@ -526,23 +614,25 @@ export const createSingleComponentTemplate = (comp: WingsComponent, config: Mint inputPorts: inputPorts, outputPorts: outputPorts, crule: { - type: 'WTYPE' + type: "WTYPE" }, prule: { - type: 'STYPE' + type: "STYPE" } - } + }; tpl.Nodes[nodeid] = node; return tpl; -} +}; /* WCM API based Wings calls */ -export const registerWingsComponent = async(name: string, uri: string, config: MintPreferences) - : Promise => { - - let config_wings = config.wings; - let data = { +export const registerWingsComponent = async ( + name: string, + uri: string, + config: MintPreferences +): Promise => { + const config_wings = config.wings; + const data = { id: name, model_catalog_uri: uri, wings_instance: { @@ -552,26 +642,33 @@ export const registerWingsComponent = async(name: string, uri: string, config: M username: config_wings.username, password: config_wings.password } - } + }; return new Promise((resolve, reject) => { - postJSONResource({ - url: config.wings_api + "/components", - onLoad: function(e: any) { - let compjson = JSON.parse(e.target.responseText); - resolve(compjson.id); + postJSONResource( + { + url: config.wings_api + "/components", + onLoad: function (e: any) { + const compjson = JSON.parse(e.target.responseText); + resolve(compjson.id); + }, + onError: function () { + reject("Cannot create component"); + } }, - onError: function() { - reject("Cannot create component"); - } - }, data); + data + ); }); -} - -export const registerWingsDataset = async(dcid: string, name: string, type: string, uri: string, config: MintPreferences) - : Promise => { - - let config_wings = config.wings; - let data = { +}; + +export const registerWingsDataset = async ( + dcid: string, + name: string, + type: string, + uri: string, + config: MintPreferences +): Promise => { + const config_wings = config.wings; + const data = { data_catalog_id: dcid, id: name, type: type, @@ -583,148 +680,151 @@ export const registerWingsDataset = async(dcid: string, name: string, type: stri username: config_wings.username, password: config_wings.password } - } + }; return new Promise((resolve, reject) => { - postJSONResource({ - url: config.wings_api + "/datasets", - onLoad: function(e: any) { - resolve(); + postJSONResource( + { + url: config.wings_api + "/datasets", + onLoad: function (e: any) { + resolve(); + }, + onError: function () { + reject("Cannot create component"); + } }, - onError: function() { - reject("Cannot create component"); - } - }, data); + data + ); }); -} - -const _createModelTemplate = ( - cname: string, - prefs: MintPreferences) : Promise => { +}; +const _createModelTemplate = (cname: string, prefs: MintPreferences): Promise => { return new Promise((resolve, reject) => { - let config = prefs.wings; - let expfx = config.export_url + "/export/users/" + config.username + "/" + config.domain; + const config = prefs.wings; + const expfx = config.export_url + "/export/users/" + config.username + "/" + config.domain; - - let tname = "workflow_" + cname; - let tns = expfx + "/workflows/" + tname + ".owl#"; - let tid = tns + tname; + const tname = "workflow_" + cname; + const tns = expfx + "/workflows/" + tname + ".owl#"; + const tid = tns + tname; fetchWingsTemplatesList(prefs).then((list) => { - if(list.indexOf(tid) >= 0) { + if (list.indexOf(tid) >= 0) { //console.log(tid + " template already exists"); resolve(tid); - } - else { + } else { // Create template fetchWingsComponent(cname, prefs).then((comp) => { - let tpl = createSingleComponentTemplate(comp, prefs); + const tpl = createSingleComponentTemplate(comp, prefs); layoutWingsTemplate(tpl, prefs).then((tpl_package) => { saveWingsTemplate(tpl_package, prefs).then(() => { console.log("Template saved as " + tpl.id); resolve(tpl.id); - }) + }); }); }); } }); }); -} +}; const _runModelTemplates = ( seeds: WingsTemplateSeed[], tpl_package: WingsTemplatePackage, - prefs: MintPreferences) : Promise => { - - let config = prefs.wings; - let expfx = config.export_url + "/export/users/" + config.username + "/" + config.domain; + prefs: MintPreferences +): Promise => { + const config = prefs.wings; + const expfx = config.export_url + "/export/users/" + config.username + "/" + config.domain; return Promise.all( seeds.map((seed) => { - let tns = seed.tid.replace(/#.*$/, "#"); - - let dataBindings = {} as WingsDataBindings; - let parameterBindings = {} as WingsParameterBindings; - let parameterTypes = {} as WingsParameterTypes; - for(let varname in seed.datasets) { - let varid = tns + varname; - dataBindings[varid] = seed.datasets[varname].map((ds: string)=> expfx + "/data/library.owl#" + ds); + const tns = seed.tid.replace(/#.*$/, "#"); + + const dataBindings = {} as WingsDataBindings; + const parameterBindings = {} as WingsParameterBindings; + const parameterTypes = {} as WingsParameterTypes; + for (const varname in seed.datasets) { + const varid = tns + varname; + dataBindings[varid] = seed.datasets[varname].map( + (ds: string) => expfx + "/data/library.owl#" + ds + ); } - for(let varname in seed.parameters) { - let varid = tns + varname; + for (const varname in seed.parameters) { + const varid = tns + varname; parameterBindings[varid] = seed.parameters[varname]; - parameterTypes[varid] = "http://www.w3.org/2001/XMLSchema#" + seed.paramtypes[varname]; + parameterTypes[varid] = + "http://www.w3.org/2001/XMLSchema#" + seed.paramtypes[varname]; } - return expandAndRunWingsWorkflow(tpl_package, - dataBindings, - parameterBindings, - parameterTypes, prefs); + return expandAndRunWingsWorkflow( + tpl_package, + dataBindings, + parameterBindings, + parameterTypes, + prefs + ); }) ); -} +}; -export const setupModelWorkflow = async(model: Model, thread: Thread, prefs: MintPreferences) => { - let cname = model.model_configuration; - let compid = await registerWingsComponent(cname, model.code_url, prefs); - let compname = compid.replace(/^.*#/, ''); - let templateid = await _createModelTemplate(compname, prefs); +export const setupModelWorkflow = async (model: Model, thread: Thread, prefs: MintPreferences) => { + const cname = model.model_configuration; + const compid = await registerWingsComponent(cname, model.code_url, prefs); + const compname = compid.replace(/^.*#/, ""); + const templateid = await _createModelTemplate(compname, prefs); return templateid; -} +}; -export const runModelExecutions = async(thread: Thread, - executions: Execution[], +export const runModelExecutions = async ( + thread: Thread, + executions: Execution[], existing_registered_resources: any, tpl_package: WingsTemplatePackage, - prefs: MintPreferences) => { - let registerDatasetPromises = []; - let seeds : WingsTemplateSeed[] = []; - let registered_resources: any = {}; + prefs: MintPreferences +) => { + const registerDatasetPromises = []; + const seeds: WingsTemplateSeed[] = []; + const registered_resources: any = {}; // Get all input dataset bindings and parameter bindings executions.map((execution) => { - let model = thread.models[execution.modelid]; - let bindings = execution.bindings; - let datasets : any = {}; - let parameters : any = {}; - let paramtypes : any= {}; + const model = thread.models[execution.modelid]; + const bindings = execution.bindings; + const datasets: any = {}; + const parameters: any = {}; + const paramtypes: any = {}; // Get input datasets model.input_files.map((io) => { - let resources : DataResource[] = []; + let resources: DataResource[] = []; let dsid = null; - if(bindings[io.id]) { + if (bindings[io.id]) { // We have a dataset binding from the user for it - resources = [ bindings[io.id] as DataResource ]; - } - else if(io.value) { + resources = [bindings[io.id] as DataResource]; + } else if (io.value) { // There is a hardcoded value in the model itself dsid = io.value.id; resources = io.value.resources; } - if(resources.length > 0) { - let type = io.type.replace(/^.*#/, ''); + if (resources.length > 0) { + const type = io.type.replace(/^.*#/, ""); resources.map((res) => { - if(res.url) { - res.name = res.url.replace(/^.*(#|\/)/, ''); - res.name = res.name.replace(/^([0-9])/, '_$1'); - if(!res.id) - res.id = res.name; + if (res.url) { + res.name = res.url.replace(/^.*(#|\/)/, ""); + res.name = res.name.replace(/^([0-9])/, "_$1"); + if (!res.id) res.id = res.name; } - if(!existing_registered_resources[res.id]) { + if (!existing_registered_resources[res.id]) { registered_resources[res.id] = [res.name, type, res.url]; } - }) + }); datasets[io.name] = resources.map((res) => res.name); } }); // Get Input parameters model.input_parameters.map((ip) => { - if(ip.value) { + if (ip.value) { parameters[ip.name] = ip.value; - } - else if(bindings[ip.id]) { - let value = bindings[ip.id]; + } else if (bindings[ip.id]) { + const value = bindings[ip.id]; parameters[ip.name] = value; } paramtypes[ip.name] = ip.type; @@ -736,34 +836,33 @@ export const runModelExecutions = async(thread: Thread, parameters: parameters, paramtypes: paramtypes } as WingsTemplateSeed); - }) + }); // Register any datasets that need to be registered - for(let resid in registered_resources) { - let args = registered_resources[resid]; + for (const resid in registered_resources) { + const args = registered_resources[resid]; existing_registered_resources[resid] = args; registerDatasetPromises.push(registerWingsDataset(resid, args[0], args[1], args[2], prefs)); } // Register all datasets - if(registerDatasetPromises.length > 0) - await Promise.all(registerDatasetPromises); + if (registerDatasetPromises.length > 0) await Promise.all(registerDatasetPromises); - let runids = await _runModelTemplates(seeds, tpl_package, prefs); + const runids = await _runModelTemplates(seeds, tpl_package, prefs); return runids; -} +}; export const matchVariables = (variables1: string[], variables2: string[], fullmatch: boolean) => { -let matched = fullmatch ? true: false; + let matched = fullmatch ? true : false; variables1.map((var1compound) => { var1compound.split(/\s*,\s/).map((var1) => { if (!fullmatch && variables2.indexOf(var1) >= 0) { matched = true; } - if(fullmatch && variables2.indexOf(var1) < 0) { + if (fullmatch && variables2.indexOf(var1) < 0) { matched = false; } }); }); return matched; -} +}; diff --git a/src/classes/wings/wings-types.ts b/src/classes/wings/wings-types.ts index 7244424..464db34 100644 --- a/src/classes/wings/wings-types.ts +++ b/src/classes/wings/wings-types.ts @@ -2,159 +2,159 @@ import { IdNameObject, IdMap } from "../mint/mint-types"; /* Defining all Wings Types */ export interface WingsTemplatePackage { - template: WingsTemplate, - constraints: WingsTemplateConstraint[] + template: WingsTemplate; + constraints: WingsTemplateConstraint[]; } export interface WingsTemplateConstraint { - subject: any, - predicate: any, - object: any + subject: any; + predicate: any; + object: any; } export interface WingsTemplate extends IdNameObject { - Nodes: IdMap, - Links: IdMap, - Variables: IdMap, - inputRoles: IdMap, - outputRoles: IdMap, + Nodes: IdMap; + Links: IdMap; + Variables: IdMap; + inputRoles: IdMap; + outputRoles: IdMap; - version: number, - onturl: string, - wflowns: string, - props: Object, - rules: Object, - subtemplates: Object, - metadata: WingsTemplateMetadata, + version: number; + onturl: string; + wflowns: string; + props: Object; + rules: Object; + subtemplates: Object; + metadata: WingsTemplateMetadata; } export interface WingsTemplateMetadata { - contributors: string[], - createdFrom: string[], - lastUpdateTime: string, - documentation: string + contributors: string[]; + createdFrom: string[]; + lastUpdateTime: string; + documentation: string; } export interface WingsNode extends IdNameObject { - inputPorts: IdMap, - outputPorts: IdMap, - componentVariable: WingsComponentVariable, - crule: WingsNodeRule, - prule: WingsNodeRule, + inputPorts: IdMap; + outputPorts: IdMap; + componentVariable: WingsComponentVariable; + crule: WingsNodeRule; + prule: WingsNodeRule; } export interface WingsNodeRule { - type: "STYPE" | "WTYPE", - expr?: WingsNodeRuleExpression + type: "STYPE" | "WTYPE"; + expr?: WingsNodeRuleExpression; } export interface WingsNodeRuleExpression { - op?: "XPRODUCT" | "NWISE" | "SHIFT" | "REDUCEDIM" | "INCREASEDIM", - args?: string[] | WingsNodeRuleExpression[] + op?: "XPRODUCT" | "NWISE" | "SHIFT" | "REDUCEDIM" | "INCREASEDIM"; + args?: string[] | WingsNodeRuleExpression[]; } export interface WingsVariable extends IdNameObject { - type: number, - comment: string, - binding: URIBinding | URIBinding[] | ValueBinding | ValueBinding[], - autofill: boolean, - breakpoint: boolean + type: number; + comment: string; + binding: URIBinding | URIBinding[] | ValueBinding | ValueBinding[]; + autofill: boolean; + breakpoint: boolean; } export interface WingsDataVariable extends WingsVariable { - type: 1, - unknown?: boolean, - inactive?: boolean, - derivedFrom?: string, - binding: URIBinding[] | URIBinding + type: 1; + unknown?: boolean; + inactive?: boolean; + derivedFrom?: string; + binding: URIBinding[] | URIBinding; } export interface WingsParameterVariable extends WingsVariable { - type: 2, - unknown?: boolean, - inactive?: boolean, - derivedFrom?: string, - binding: ValueBinding[] | ValueBinding + type: 2; + unknown?: boolean; + inactive?: boolean; + derivedFrom?: string; + binding: ValueBinding[] | ValueBinding; } export interface WingsComponentVariable extends WingsVariable { - type: 3, - isConcrete: boolean, - binding: URIBinding + type: 3; + isConcrete: boolean; + binding: URIBinding; } export interface WingsComponent extends IdNameObject { - type: number, - binding: URIBinding, - inputs: WingsComponentArgument[], - outputs: WingsComponentArgument[], - rules: string[], - inheritedRules: string[], - requirement: WingsComponentRequirement + type: number; + binding: URIBinding; + inputs: WingsComponentArgument[]; + outputs: WingsComponentArgument[]; + rules: string[]; + inheritedRules: string[]; + requirement: WingsComponentRequirement; } export interface WingsComponentArgument extends IdNameObject { - type: string, - role: string, - prefix: string, - isParam?: boolean, - dimensionality?: number, - paramDefaultValue?: any + type: string; + role: string; + prefix: string; + isParam?: boolean; + dimensionality?: number; + paramDefaultValue?: any; } export interface WingsComponentRequirement { - storageGB: number, - memoryGB: number, - needs64bit: boolean, - softwareIds: string[] + storageGB: number; + memoryGB: number; + needs64bit: boolean; + softwareIds: string[]; } export interface WingsPort extends IdNameObject { - role: WingsPortRole, + role: WingsPortRole; } export interface WingsPortRole extends IdNameObject { - type: number, - roleid: string, - dimensionality?: number + type: number; + roleid: string; + dimensionality?: number; } export interface WingsLink extends IdNameObject { - fromNode?: IdNameObject, - toNode?: IdNameObject, - fromPort?: IdNameObject, - toPort?: IdNameObject, - variable?: IdNameObject + fromNode?: IdNameObject; + toNode?: IdNameObject; + fromPort?: IdNameObject; + toPort?: IdNameObject; + variable?: IdNameObject; } export interface URIBinding extends IdNameObject { - type: string + type: string; } export interface ValueBinding { - type: string, - value: any + type: string; + value: any; } export interface WingsPlannerResults { - success: boolean, - data: WingsPlannerData + success: boolean; + data: WingsPlannerData; } export interface WingsPlannerData { - explanations: string[], - error: boolean, + explanations: string[]; + error: boolean; } -export interface WingsPlannerExpansionsResults extends WingsPlannerResults{ - data: WingsWorkflowExpansions +export interface WingsPlannerExpansionsResults extends WingsPlannerResults { + data: WingsWorkflowExpansions; } export interface WingsWorkflowExpansions extends WingsPlannerData { - seed: WingsTemplatePackage, - templates: WingsTemplatePackage[] + seed: WingsTemplatePackage; + templates: WingsTemplatePackage[]; } export interface WingsDataBindings { - [inputid: string] : string[] + [inputid: string]: string[]; } export interface WingsParameterBindings { - [inputid: string] : string + [inputid: string]: string; } export interface WingsParameterTypes { - [inputid: string] : string + [inputid: string]: string; } export interface WingsTemplateSeed { - tid?: string, - datasets: any, - parameters: any, - paramtypes: any, + tid?: string; + datasets: any; + parameters: any; + paramtypes: any; } diff --git a/src/classes/wings/xhr-requests.ts b/src/classes/wings/xhr-requests.ts index d537618..a47c5c0 100644 --- a/src/classes/wings/xhr-requests.ts +++ b/src/classes/wings/xhr-requests.ts @@ -5,11 +5,10 @@ const cookieJar = request.jar(); function _sendRequest(rq: any, options: any, storeCredentials: boolean) { options.jar = cookieJar; options.uri = rq.url; - request(options, function(err:any, resp:any, body:any) { - if(err) { + request(options, function (err: any, resp: any, body: any) { + if (err) { rq.onError(err); - } - else { + } else { rq.onLoad({ target: { responseText: body @@ -20,19 +19,17 @@ function _sendRequest(rq: any, options: any, storeCredentials: boolean) { } export function getResource(rq: any) { - let options = { + const options = { method: "GET" }; _sendRequest(rq, options, false); - } -export function postJSONResource( - rq: any, data: Object) { - let headers = { - "Content-type": "application/json", - } as any; - let options = { +export function postJSONResource(rq: any, data: Object) { + const headers = { + "Content-type": "application/json" + } as any; + const options = { headers: headers, method: "POST", body: JSON.stringify(data) @@ -40,9 +37,8 @@ export function postJSONResource( _sendRequest(rq, options, false); } -export function putJSONResource( - rq: any, data: Object) { - let options = { +export function putJSONResource(rq: any, data: Object) { + const options = { headers: { "Content-type": "application/json" }, @@ -52,16 +48,14 @@ export function putJSONResource( _sendRequest(rq, options, false); } -export function postFormResource(rq: any, keyvalues: any, - storeCredentials: boolean) { +export function postFormResource(rq: any, keyvalues: any, storeCredentials: boolean) { // Crate form data - var data = ""; - for (var key in keyvalues) { - if (data) - data += "&"; + let data = ""; + for (const key in keyvalues) { + if (data) data += "&"; data += key + "=" + encodeURIComponent(keyvalues[key]); } - let options = { + const options = { headers: { "Content-type": "application/x-www-form-urlencoded" }, @@ -72,7 +66,7 @@ export function postFormResource(rq: any, keyvalues: any, } export function deleteResource(rq: any) { - let options = { + const options = { headers: { "Content-type": "application/x-www-form-urlencoded" }, @@ -80,4 +74,3 @@ export function deleteResource(rq: any) { }; _sendRequest(rq, options, false); } - \ No newline at end of file diff --git a/src/config/app.ts b/src/config/app.ts index 914d0a5..3bd1027 100644 --- a/src/config/app.ts +++ b/src/config/app.ts @@ -1,6 +1,6 @@ -export const PORT = 3000 +export const PORT = 3000; export const VERSION = "v1"; export const DEVMODE = false; // We need a code, logs and data directory under the DEVHOMEDIR -export const DEVHOMEDIR = "/Users/varun/mintproject"; +export const DEVHOMEDIR = "/Users/varun/mintproject"; diff --git a/src/config/graphql.ts b/src/config/graphql.ts index 70a6a41..9f83748 100644 --- a/src/config/graphql.ts +++ b/src/config/graphql.ts @@ -1,39 +1,39 @@ -import 'cross-fetch/polyfill'; -import { ApolloClient, createHttpLink, InMemoryCache, NormalizedCacheObject } from '@apollo/client'; -import { User } from '../classes/mint/mint-types'; +import "cross-fetch/polyfill"; +import { ApolloClient, createHttpLink, InMemoryCache, NormalizedCacheObject } from "@apollo/client"; +import { User } from "../classes/mint/mint-types"; -import { KeycloakAdapter } from './keycloak-adapter'; -import { getConfiguration } from '../classes/mint/mint-functions'; +import { KeycloakAdapter } from "./keycloak-adapter"; +import { getConfiguration } from "../classes/mint/mint-functions"; export class GraphQL { - static client : ApolloClient; - static userId; + static client: ApolloClient; + static userId; - static instance = (auth:User) => { - if(GraphQL.client != null && auth.email && auth.email == GraphQL.userId) - return GraphQL.client - GraphQL.userId = auth?.email; + static instance = (auth: User) => { + if (GraphQL.client != null && auth.email && auth.email == GraphQL.userId) + return GraphQL.client; + GraphQL.userId = auth?.email; - // Create the Apollo GraphQL Client - GraphQL.client = new ApolloClient({ - link: GraphQL.getHTTPSLink(), - cache: new InMemoryCache() - }); - return GraphQL.client; - } + // Create the Apollo GraphQL Client + GraphQL.client = new ApolloClient({ + link: GraphQL.getHTTPSLink(), + cache: new InMemoryCache() + }); + return GraphQL.client; + }; - static getHTTPSLink() { - let prefs = getConfiguration(); - - // Normal HTTP Link - let protocol = prefs.graphql.enable_ssl? "https://" : "http://" - let uri = protocol + prefs.graphql.endpoint - - return createHttpLink({ - uri: uri, - headers: prefs.graphql.secret ? - { "X-Hasura-Admin-Secret": prefs.graphql.secret } : KeycloakAdapter.getAccessTokenHeader() - }); - } -} + static getHTTPSLink() { + const prefs = getConfiguration(); + + // Normal HTTP Link + const protocol = prefs.graphql.enable_ssl ? "https://" : "http://"; + const uri = protocol + prefs.graphql.endpoint; + return createHttpLink({ + uri: uri, + headers: prefs.graphql.secret + ? { "X-Hasura-Admin-Secret": prefs.graphql.secret } + : KeycloakAdapter.getAccessTokenHeader() + }); + } +} diff --git a/src/config/keycloak-adapter.ts b/src/config/keycloak-adapter.ts index 78890dc..b0ef914 100644 --- a/src/config/keycloak-adapter.ts +++ b/src/config/keycloak-adapter.ts @@ -1,58 +1,63 @@ -import { getConfiguration } from '../classes/mint/mint-functions'; -import { User } from '../classes/mint/mint-types'; +import { getConfiguration } from "../classes/mint/mint-functions"; +import { User } from "../classes/mint/mint-types"; -let prefs = getConfiguration(); +const prefs = getConfiguration(); interface tokenResponse { - access_token: string, - expires_in: number, - 'not-before-policy': number, - refresh_expires_in: number, - refresh_token: string, - scope: string, - session_state: string, - token_type: string -}; + access_token: string; + expires_in: number; + "not-before-policy": number; + refresh_expires_in: number; + refresh_token: string; + scope: string; + session_state: string; + token_type: string; +} interface keycloakProfile { - region: string, - graph: string, + region: string; + graph: string; } interface decodedToken { - sub: string, - email: string, - profile: keycloakProfile, - preferred_username: string + sub: string; + email: string; + profile: keycloakProfile; + preferred_username: string; } export class KeycloakAdapter { - private static server : string = prefs.auth_server; - private static realm : string = prefs.auth_realm; - private static clientId : string = prefs.auth_client_id; + private static server: string = prefs.auth_server; + private static realm: string = prefs.auth_realm; + private static clientId: string = prefs.auth_client_id; // Return values - private static accessToken : string; - private static refreshToken : string; - private static sessionState : string; - private static expiresIn : number; - private static refreshExpiresIn : number; + private static accessToken: string; + private static refreshToken: string; + private static sessionState: string; + private static expiresIn: number; + private static refreshExpiresIn: number; // user data - private static username : string; - private static userid : string; - private static email : string; - private static region : string; - private static graph : string; - - private static getTokenUri () : string { - return KeycloakAdapter.server + "realms/" + KeycloakAdapter.realm + "/protocol/openid-connect/token"; + private static username: string; + private static userid: string; + private static email: string; + private static region: string; + private static graph: string; + + private static getTokenUri(): string { + return ( + KeycloakAdapter.server + + "realms/" + + KeycloakAdapter.realm + + "/protocol/openid-connect/token" + ); } - private static nodejs_atob (base64): string { - let buff = Buffer.from(base64, 'base64'); - return buff.toString('utf-8'); + private static nodejs_atob(base64): string { + const buff = Buffer.from(base64, "base64"); + return buff.toString("utf-8"); } - private static saveTokenResponse (tkn: tokenResponse) : void { + private static saveTokenResponse(tkn: tokenResponse): void { KeycloakAdapter.accessToken = tkn.access_token; KeycloakAdapter.refreshToken = tkn.refresh_token; KeycloakAdapter.sessionState = tkn.session_state; @@ -61,21 +66,23 @@ export class KeycloakAdapter { KeycloakAdapter.setLocalStorage(); //Decode token - let decoded : decodedToken = JSON.parse(KeycloakAdapter.nodejs_atob(KeycloakAdapter.accessToken.split(".")[1])); + const decoded: decodedToken = JSON.parse( + KeycloakAdapter.nodejs_atob(KeycloakAdapter.accessToken.split(".")[1]) + ); KeycloakAdapter.username = decoded.preferred_username; - KeycloakAdapter.userid = decoded.sub - KeycloakAdapter.email = decoded.email + KeycloakAdapter.userid = decoded.sub; + KeycloakAdapter.email = decoded.email; //FIXME KeycloakAdapter.region = decoded.profile ? decoded.profile.region : undefined; KeycloakAdapter.graph = decoded.profile ? decoded.profile.graph : undefined; } - public static signIn (username: string, password:string) : Promise { + public static signIn(username: string, password: string): Promise { if (!username || !password) { return; } - let uri : string = KeycloakAdapter.getTokenUri(); - let data = { + const uri: string = KeycloakAdapter.getTokenUri(); + const data = { client_id: KeycloakAdapter.clientId, grant_type: "password", username: username, @@ -83,22 +90,24 @@ export class KeycloakAdapter { }; return new Promise((resolve, reject) => { - let req : Promise = fetch(uri, { - method: 'POST', - headers: {'Content-Type': "application/x-www-form-urlencoded"}, + const req: Promise = fetch(uri, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams(data) }); req.catch(reject); - req.then((response:Response) => { + req.then((response: Response) => { if (response.status === 200) { - let jsn = response.json(); + const jsn = response.json(); jsn.catch(reject); - jsn.then((tkn:tokenResponse) => { + jsn.then((tkn: tokenResponse) => { KeycloakAdapter.saveTokenResponse(tkn); //Logged in, we need to start to auto refresh - setTimeout(() => { KeycloakAdapter.refresh(KeycloakAdapter.refreshToken) }, KeycloakAdapter.refreshExpiresIn * 100); + setTimeout(() => { + KeycloakAdapter.refresh(KeycloakAdapter.refreshToken); + }, KeycloakAdapter.refreshExpiresIn * 100); resolve(); - }) + }); } else { reject(); } @@ -106,27 +115,29 @@ export class KeycloakAdapter { }); } - public static refresh (token?: string) : Promise { - let uri : string = KeycloakAdapter.getTokenUri(); - let data = { + public static refresh(token?: string): Promise { + const uri: string = KeycloakAdapter.getTokenUri(); + const data = { client_id: KeycloakAdapter.clientId, grant_type: "refresh_token", refresh_token: token ? token : KeycloakAdapter.refreshToken - } + }; return new Promise((resolve, reject) => { - let req : Promise = fetch(uri, { - method: 'POST', - headers: {'Content-Type': "application/x-www-form-urlencoded"}, + const req: Promise = fetch(uri, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams(data) }); req.catch(reject); - req.then((response:Response) => { + req.then((response: Response) => { if (response.status === 200) { - response.json().then((tkn : tokenResponse) => { + response.json().then((tkn: tokenResponse) => { KeycloakAdapter.saveTokenResponse(tkn); // Everything is ok autorefresh - setTimeout(() => { KeycloakAdapter.refresh(KeycloakAdapter.refreshToken) }, KeycloakAdapter.refreshExpiresIn * 100); + setTimeout(() => { + KeycloakAdapter.refresh(KeycloakAdapter.refreshToken); + }, KeycloakAdapter.refreshExpiresIn * 100); resolve(true); }); } else { @@ -136,17 +147,17 @@ export class KeycloakAdapter { }); } - private static setLocalStorage () : void { - localStorage.setItem('access-token', KeycloakAdapter.accessToken); - localStorage.setItem('refresh-token', KeycloakAdapter.refreshToken); + private static setLocalStorage(): void { + localStorage.setItem("access-token", KeycloakAdapter.accessToken); + localStorage.setItem("refresh-token", KeycloakAdapter.refreshToken); } - private static clearLocalStorage () : void { - localStorage.removeItem('access-token'); - localStorage.removeItem('refresh-token'); + private static clearLocalStorage(): void { + localStorage.removeItem("access-token"); + localStorage.removeItem("refresh-token"); } - public static signOut () : void { + public static signOut(): void { KeycloakAdapter.username = undefined; KeycloakAdapter.userid = undefined; KeycloakAdapter.email = undefined; @@ -161,17 +172,18 @@ export class KeycloakAdapter { KeycloakAdapter.clearLocalStorage(); } - public static loadFromLocalStorage () : Promise { + public static loadFromLocalStorage(): Promise { return new Promise((resolve, reject) => { - let accessToken : string = localStorage.getItem('access-token'); - let refreshToken : string = localStorage.getItem('refresh-token'); + const accessToken: string = localStorage.getItem("access-token"); + const refreshToken: string = localStorage.getItem("refresh-token"); if (accessToken && refreshToken) { //Check if access token is still valid, if not, try to refresh. - if (false) { //KeycloakAdapter.checkToken()) { + if (false) { + //KeycloakAdapter.checkToken()) { } else { - let ref : Promise = KeycloakAdapter.refresh(refreshToken); + const ref: Promise = KeycloakAdapter.refresh(refreshToken); ref.catch(() => resolve(false)); - ref.then((b:boolean) => resolve(b)); + ref.then((b: boolean) => resolve(b)); } } else { resolve(false); @@ -179,28 +191,26 @@ export class KeycloakAdapter { }); } - public static getAccessToken () { - if (KeycloakAdapter.accessToken) - return KeycloakAdapter.accessToken; + public static getAccessToken() { + if (KeycloakAdapter.accessToken) return KeycloakAdapter.accessToken; return undefined; } - public static getAccessTokenHeader () { + public static getAccessTokenHeader() { if (KeycloakAdapter.accessToken) - return {'Authorization': "Bearer " + KeycloakAdapter.accessToken}; + return { Authorization: "Bearer " + KeycloakAdapter.accessToken }; return undefined; } - public static getUser () : User { - if(KeycloakAdapter.username) { + public static getUser(): User { + if (KeycloakAdapter.username) { return { email: KeycloakAdapter.username, uid: KeycloakAdapter.userid, region: KeycloakAdapter.region, graph: KeycloakAdapter.graph } as User; - } - else { + } else { // Secret based login. Return a fake user return { email: "mint@isi.edu", @@ -209,23 +219,32 @@ export class KeycloakAdapter { } } - public static updateProfile (u: User) : void { - let uri : string = KeycloakAdapter.server + "admin/realms/" + KeycloakAdapter.realm + "/users/" + KeycloakAdapter.userid; - let data = { - "attributes": { - "region": u.region, - "graph": u.graph + public static updateProfile(u: User): void { + const uri: string = + KeycloakAdapter.server + + "admin/realms/" + + KeycloakAdapter.realm + + "/users/" + + KeycloakAdapter.userid; + const data = { + attributes: { + region: u.region, + graph: u.graph } - } + }; //FIXME: editing profile requires user role } - public static checkToken () { - let uri : string = KeycloakAdapter.server + "realms/" + KeycloakAdapter.realm + "/protocol/openid-connect/userinfo"; + public static checkToken() { + const uri: string = + KeycloakAdapter.server + + "realms/" + + KeycloakAdapter.realm + + "/protocol/openid-connect/userinfo"; - let req = fetch(uri, { - method: 'GET', - credentials: 'include', + const req = fetch(uri, { + method: "GET", + credentials: "include", headers: KeycloakAdapter.getAccessTokenHeader() }); //FIXME: we should be able to verify the token here! diff --git a/src/config/redis.ts b/src/config/redis.ts index a49f0bd..1e64597 100644 --- a/src/config/redis.ts +++ b/src/config/redis.ts @@ -1,2 +1,2 @@ -export const REDIS_URL = process.env.REDIS_URL || 'redis://127.0.0.1:7379'; +export const REDIS_URL = process.env.REDIS_URL || "redis://127.0.0.1:7379"; export const EXECUTION_QUEUE_NAME = "LocalExecution"; diff --git a/src/server.ts b/src/server.ts index 3770490..3237fd1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,40 +4,39 @@ import path from "path"; import cors from "cors"; import swaggerUi from "swagger-ui-express"; -import v1ExecutionsService from './api/api-v1/services/executionsService'; -import v1ExecutionsLocalService from './api/api-v1/services/executionsLocalService'; -import v1RegistrationService from './api/api-v1/services/registrationService'; -import v1ExecutionQueueService from './api/api-v1/services/executionQueueService'; -import v1MonitorsService from './api/api-v1/services/monitorsService'; -import v1LogsService from './api/api-v1/services/logsService'; -import v1ThreadsService from './api/api-v1/services/threadsService'; -import v1ModelCacheService from './api/api-v1/services/modelCacheService'; - +import v1ExecutionsService from "./api/api-v1/services/executionsService"; +import v1ExecutionsLocalService from "./api/api-v1/services/executionsLocalService"; +import v1RegistrationService from "./api/api-v1/services/registrationService"; +import v1ExecutionQueueService from "./api/api-v1/services/executionQueueService"; +import v1MonitorsService from "./api/api-v1/services/monitorsService"; +import v1LogsService from "./api/api-v1/services/logsService"; +import v1ThreadsService from "./api/api-v1/services/threadsService"; +import v1ModelCacheService from "./api/api-v1/services/modelCacheService"; import { initialize } from "express-openapi"; import { getResource } from "./classes/wings/xhr-requests"; import Queue from "bull"; -const { createBullBoard } = require('@bull-board/api'); -const { BullAdapter } = require('@bull-board/api/bullAdapter'); -const { ExpressAdapter } = require('@bull-board/express'); +const { createBullBoard } = require("@bull-board/api"); +const { BullAdapter } = require("@bull-board/api/bullAdapter"); +const { ExpressAdapter } = require("@bull-board/express"); import { EXECUTION_QUEUE_NAME, REDIS_URL } from "./config/redis"; import { PORT, VERSION } from "./config/app"; // Main Express Server const app = express(); -const port = PORT; +const port = PORT; const version = VERSION; -const dashboard_url = '/admin/queues'; +const dashboard_url = "/admin/queues"; // Setup API -var v1ApiDoc = require('./api/api-doc'); +const v1ApiDoc = require("./api/api-doc"); app.use(bodyParser.json()); app.use(cors()); initialize({ - app, + app, apiDoc: v1ApiDoc, dependencies: { executionsService: v1ExecutionsService, @@ -49,8 +48,8 @@ initialize({ logsService: v1LogsService, modelCacheService: v1ModelCacheService }, - paths: path.resolve(__dirname, './api/api-v1/paths'), - routesGlob: '**/*.{ts,js}', + paths: path.resolve(__dirname, "./api/api-v1/paths"), + routesGlob: "**/*.{ts,js}", routesIndexFileRegExp: /(?:index)?\.[tj]s$/ }); @@ -62,30 +61,28 @@ app.use(((err, req, res, next) => { }) as express.ErrorRequestHandler); // Setup Queue -let executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); +const executionQueue = new Queue(EXECUTION_QUEUE_NAME, REDIS_URL); // Setup Bull Dashboard const serverAdapter = new ExpressAdapter(); serverAdapter.setBasePath(dashboard_url); const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({ - queues: [new BullAdapter(executionQueue)], - serverAdapter: serverAdapter, + queues: [new BullAdapter(executionQueue)], + serverAdapter: serverAdapter }); -app.use(dashboard_url, serverAdapter.getRouter()) +app.use(dashboard_url, serverAdapter.getRouter()); // Express start app.listen(port, () => { // Serve Swagger UI getResource({ - url: 'http://localhost:' + port + '/' + version + '/api-docs', + url: "http://localhost:" + port + "/" + version + "/api-docs", onError: () => {}, - onLoad: (resp:any) => { - var apidoc = JSON.parse(resp.target.responseText); - app.use('/' + version + '/ui', swaggerUi.serve, swaggerUi.setup(apidoc)); + onLoad: (resp: any) => { + const apidoc = JSON.parse(resp.target.responseText); + app.use("/" + version + "/ui", swaggerUi.serve, swaggerUi.setup(apidoc)); } - }) + }); }); - - diff --git a/src/typings/graphql.d.ts b/src/typings/graphql.d.ts index 98eb236..0a21aa7 100644 --- a/src/typings/graphql.d.ts +++ b/src/typings/graphql.d.ts @@ -1,6 +1,6 @@ -declare module '*.graphql' { - import { DocumentNode } from 'graphql' - const Schema: DocumentNode - - export = Schema -} \ No newline at end of file +declare module "*.graphql" { + import { DocumentNode } from "graphql"; + const Schema: DocumentNode; + + export = Schema; +} diff --git a/src/typings/json.d.ts b/src/typings/json.d.ts index 01e0305..4b3471f 100644 --- a/src/typings/json.d.ts +++ b/src/typings/json.d.ts @@ -1,4 +1,4 @@ -declare module "*.json" -{ const value: any; - export default value; -} \ No newline at end of file +declare module "*.json" { + const value: any; + export default value; +}