From 9d456d4dcd854612fa149a40374cec88be07dd2d Mon Sep 17 00:00:00 2001 From: Ali Deishidi Date: Wed, 17 Nov 2021 13:41:06 +0000 Subject: [PATCH 1/5] feat(server): fetch all data in one request --- src/executor/index.ts | 61 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/src/executor/index.ts b/src/executor/index.ts index 158f795..fdb5a74 100644 --- a/src/executor/index.ts +++ b/src/executor/index.ts @@ -69,6 +69,30 @@ async function fetchTableData ( return res.json() } +async function fetchTablesData ( +tableDefinitions: Array, + headers: any, + queryAst: any, + method: 'POST' | 'GET' = 'POST' +): Promise { + const res = await fetch( + 'https://api.ali-dev.factorialhr.com/business_intelligence/tables', { + headers: headers, + method: method, + body: JSON.stringify({ + query_ast: queryAst, + names: tableDefinitions.map((tableDefinition: TableDefinition) => tableDefinition.name) + }) + } + ) + + if (!res.ok) { + return Promise.reject(new Error(`Error with the request. Status code: ${res.status}`)) + } + + return res.json() +} + async function populateTables ( db: IDatabaseAdapter, usedTables: Array, @@ -81,14 +105,44 @@ async function populateTables ( ) => usedTables.includes(tableDefinition.name)) const promises = filteredTableDefinition.map(async (tableDefinition: TableDefinition) => { + const data = await fetchTableData(tableDefinition, headers, queryAst) + syncData(tableDefinition, data, db) + }) + return Promise.all(promises) +} + +async function populateTablesInOneHTTPRequest ( + db: IDatabaseAdapter, + usedTables: Array, + headers: any, + schema: any, + queryAst: any +) { + const filteredTableDefinition = schema.filter(( + tableDefinition: TableDefinition + ) => usedTables.includes(tableDefinition.name)) + const tablesData = await fetchTablesData(filteredTableDefinition, headers, queryAst) + filteredTableDefinition.map((tableDefinition: TableDefinition) => { + populateData( + tableDefinition, + tablesData.select(tableData => tableDefinition.name === tableData.name).body, + db + ) + }) +} + +function syncData( + tableDefinition: TableDefinition, + data: any, + db: IDatabaseAdapter + ) { + console.log(data) const schemas = parseSchema(tableDefinition.fields).join(', ') if (!tableDefinition.autodiscover) { db.createTable(tableDefinition, schemas) } - const data = await fetchTableData(tableDefinition, headers, queryAst) - const resultKey = tableDefinition.resultKey const dataPointer = resultKey ? data[resultKey] : data const fixedData = dataPointer.map((field: any) => flattenObject(field, '_')) @@ -111,9 +165,6 @@ async function populateTables ( } else { db.storeToDb(tableDefinition, fixedData) } - }) - - return Promise.all(promises) } const DEFAULT_CONFIG = { From 6f0ae3b764d09af9c58497d64be49bbc1bdf8464 Mon Sep 17 00:00:00 2001 From: Ali Deishidi Date: Thu, 18 Nov 2021 14:57:35 +0000 Subject: [PATCH 2/5] feat(runner_branch): decide run based on input --- README.md | 3 +++ src/executor/index.ts | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 30bbc6a..8569563 100644 --- a/README.md +++ b/README.md @@ -152,3 +152,6 @@ yarn cli query "SELECT 1;" # or installed global tentaclesql query "SELECT 1;" ``` + +### Bulk fetch +`BULK_FETCH=true` and `BULK_FETCH_URL=url` to fetch all data in one HTTP request. #TODO: explain in details diff --git a/src/executor/index.ts b/src/executor/index.ts index fdb5a74..e24f9dd 100644 --- a/src/executor/index.ts +++ b/src/executor/index.ts @@ -76,7 +76,7 @@ tableDefinitions: Array, method: 'POST' | 'GET' = 'POST' ): Promise { const res = await fetch( - 'https://api.ali-dev.factorialhr.com/business_intelligence/tables', { + process.env.BULK_FETCH_URL, { headers: headers, method: method, body: JSON.stringify({ @@ -121,11 +121,12 @@ async function populateTablesInOneHTTPRequest ( const filteredTableDefinition = schema.filter(( tableDefinition: TableDefinition ) => usedTables.includes(tableDefinition.name)) - const tablesData = await fetchTablesData(filteredTableDefinition, headers, queryAst) + const remoteData = await fetchTablesData(filteredTableDefinition, headers, queryAst) filteredTableDefinition.map((tableDefinition: TableDefinition) => { - populateData( + const targetTable = remoteData.find(tableData => tableData.name === tableDefinition.name) + syncData( tableDefinition, - tablesData.select(tableData => tableDefinition.name === tableData.name).body, + targetTable.data, db ) }) @@ -136,7 +137,6 @@ function syncData( data: any, db: IDatabaseAdapter ) { - console.log(data) const schemas = parseSchema(tableDefinition.fields).join(', ') if (!tableDefinition.autodiscover) { @@ -172,6 +172,33 @@ const DEFAULT_CONFIG = { schema: [] } +async function runPopulateTables( + db: IDatabaseAdapter, + usedTables: Array, + headers: any, + schema: any, + ast: any + ) { + if (process.env.BULK_FETCH) { + await populateTablesInOneHTTPRequest( + db, + usedTables, + headers, + schema, + ast + ) + } else { + await populateTables( + db, + usedTables, + headers, + schema, + ast + ) + } +} + + async function executor ( sql: string, parameters: Parameters, @@ -204,7 +231,7 @@ async function executor ( const headersWithHost = getHost() ? { ...headers, host: getHost() } : { ...headers } headersWithHost['user-agent'] = `tentaclesql/${version}` - await populateTables( + await runPopulateTables( db, usedTables, headers, From aefa5b2f088c575478cddaff7bd669a590c7577d Mon Sep 17 00:00:00 2001 From: Ali Deishidi Date: Thu, 18 Nov 2021 15:35:15 +0000 Subject: [PATCH 3/5] style: fix --- src/executor/index.ts | 73 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/src/executor/index.ts b/src/executor/index.ts index e24f9dd..46f2f06 100644 --- a/src/executor/index.ts +++ b/src/executor/index.ts @@ -70,20 +70,20 @@ async function fetchTableData ( } async function fetchTablesData ( -tableDefinitions: Array, + tableDefinitions: Array, headers: any, queryAst: any, method: 'POST' | 'GET' = 'POST' ): Promise { const res = await fetch( process.env.BULK_FETCH_URL, { - headers: headers, - method: method, - body: JSON.stringify({ - query_ast: queryAst, - names: tableDefinitions.map((tableDefinition: TableDefinition) => tableDefinition.name) - }) - } + headers: headers, + method: method, + body: JSON.stringify({ + query_ast: queryAst, + names: tableDefinitions.map((tableDefinition: TableDefinition) => tableDefinition.name) + }) + } ) if (!res.ok) { @@ -122,7 +122,7 @@ async function populateTablesInOneHTTPRequest ( tableDefinition: TableDefinition ) => usedTables.includes(tableDefinition.name)) const remoteData = await fetchTablesData(filteredTableDefinition, headers, queryAst) - filteredTableDefinition.map((tableDefinition: TableDefinition) => { + filteredTableDefinition.forEach((tableDefinition: TableDefinition) => { const targetTable = remoteData.find(tableData => tableData.name === tableDefinition.name) syncData( tableDefinition, @@ -132,39 +132,39 @@ async function populateTablesInOneHTTPRequest ( }) } -function syncData( +function syncData ( tableDefinition: TableDefinition, data: any, db: IDatabaseAdapter - ) { - const schemas = parseSchema(tableDefinition.fields).join(', ') - - if (!tableDefinition.autodiscover) { - db.createTable(tableDefinition, schemas) - } +) { + const schemas = parseSchema(tableDefinition.fields).join(', ') - const resultKey = tableDefinition.resultKey - const dataPointer = resultKey ? data[resultKey] : data - const fixedData = dataPointer.map((field: any) => flattenObject(field, '_')) + if (!tableDefinition.autodiscover) { + db.createTable(tableDefinition, schemas) + } - if (fixedData.length === 0) return + const resultKey = tableDefinition.resultKey + const dataPointer = resultKey ? data[resultKey] : data + const fixedData = dataPointer.map((field: any) => flattenObject(field, '_')) - // No support for booleans :/ - mutateDataframe(fixedData, (row, k) => { - if (typeof row[k] === 'boolean') row[k] = row[k] ? 'TRUE' : 'FALSE' - }) + if (fixedData.length === 0) return - if (tableDefinition.autodiscover) { - const dynamicDefinition = { - name: tableDefinition.name, - fields: Object.keys(fixedData[0]).map((key) => ({ key: key })) - } + // No support for booleans :/ + mutateDataframe(fixedData, (row, k) => { + if (typeof row[k] === 'boolean') row[k] = row[k] ? 'TRUE' : 'FALSE' + }) - db.createTable(dynamicDefinition, schemas) - db.storeToDb(dynamicDefinition, fixedData) - } else { - db.storeToDb(tableDefinition, fixedData) + if (tableDefinition.autodiscover) { + const dynamicDefinition = { + name: tableDefinition.name, + fields: Object.keys(fixedData[0]).map((key) => ({ key: key })) } + + db.createTable(dynamicDefinition, schemas) + db.storeToDb(dynamicDefinition, fixedData) + } else { + db.storeToDb(tableDefinition, fixedData) + } } const DEFAULT_CONFIG = { @@ -172,13 +172,13 @@ const DEFAULT_CONFIG = { schema: [] } -async function runPopulateTables( +async function runPopulateTables ( db: IDatabaseAdapter, usedTables: Array, headers: any, schema: any, ast: any - ) { +) { if (process.env.BULK_FETCH) { await populateTablesInOneHTTPRequest( db, @@ -186,7 +186,7 @@ async function runPopulateTables( headers, schema, ast - ) + ) } else { await populateTables( db, @@ -198,7 +198,6 @@ async function runPopulateTables( } } - async function executor ( sql: string, parameters: Parameters, From fd74de109f15ae5ccb88ebb2cd38a9f564971021 Mon Sep 17 00:00:00 2001 From: Ali Deishidi Date: Thu, 18 Nov 2021 15:54:19 +0000 Subject: [PATCH 4/5] style: fix --- README.md | 4 +++- src/executor/index.ts | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8569563..711e7e8 100644 --- a/README.md +++ b/README.md @@ -154,4 +154,6 @@ tentaclesql query "SELECT 1;" ``` ### Bulk fetch -`BULK_FETCH=true` and `BULK_FETCH_URL=url` to fetch all data in one HTTP request. #TODO: explain in details +By default Tentacle sends one HTTP request for each table data, however you can change this and fetch all table data in one HTTP request. To enable this you need to pass following paramaters: +- `BULK_FETCH=true` +- `BULK_FETCH_URL=url` diff --git a/src/executor/index.ts b/src/executor/index.ts index 46f2f06..8d41113 100644 --- a/src/executor/index.ts +++ b/src/executor/index.ts @@ -75,6 +75,9 @@ async function fetchTablesData ( queryAst: any, method: 'POST' | 'GET' = 'POST' ): Promise { + if (process.env.BULK_FETCH_URL == undefined ) { + return Error(`Bulk fetch requested but bulk fetch url is not defined.`) + } const res = await fetch( process.env.BULK_FETCH_URL, { headers: headers, @@ -123,7 +126,7 @@ async function populateTablesInOneHTTPRequest ( ) => usedTables.includes(tableDefinition.name)) const remoteData = await fetchTablesData(filteredTableDefinition, headers, queryAst) filteredTableDefinition.forEach((tableDefinition: TableDefinition) => { - const targetTable = remoteData.find(tableData => tableData.name === tableDefinition.name) + const targetTable = remoteData.find((tableData: any) => tableData.name === tableDefinition.name) syncData( tableDefinition, targetTable.data, From 38f1625e5082134fa2506220f4455bae8badcacf Mon Sep 17 00:00:00 2001 From: Ali Deishidi Date: Thu, 18 Nov 2021 15:59:24 +0000 Subject: [PATCH 5/5] style: fix --- src/executor/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/executor/index.ts b/src/executor/index.ts index 8d41113..94ba340 100644 --- a/src/executor/index.ts +++ b/src/executor/index.ts @@ -75,8 +75,8 @@ async function fetchTablesData ( queryAst: any, method: 'POST' | 'GET' = 'POST' ): Promise { - if (process.env.BULK_FETCH_URL == undefined ) { - return Error(`Bulk fetch requested but bulk fetch url is not defined.`) + if (process.env.BULK_FETCH_URL === undefined) { + return Error('Bulk fetch requested but bulk fetch url is not defined.') } const res = await fetch( process.env.BULK_FETCH_URL, {