From b293898ed1bf785e0fac130e4bd3518cc8e53beb Mon Sep 17 00:00:00 2001 From: Hugo Martinez Date: Thu, 9 Feb 2023 11:46:09 +0100 Subject: [PATCH] Integration of downloading files --- .env.dist | 3 ++ downloader.js | 87 +++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 6 ++++ package.json | 1 + 4 files changed, 97 insertions(+) create mode 100644 downloader.js diff --git a/.env.dist b/.env.dist index 2e36199..2bdfaba 100644 --- a/.env.dist +++ b/.env.dist @@ -7,3 +7,6 @@ SLACK_WEBHOOK="" TWILIO_ACCOUND_SID="" TWILIO_AUTH_TOKEN="" TWILIO_PHONE_VOICE="" +METABASE_URL="" +METABASE_MAIL="" +METABASE_PWD="" diff --git a/downloader.js b/downloader.js new file mode 100644 index 0000000..106acd2 --- /dev/null +++ b/downloader.js @@ -0,0 +1,87 @@ +const axios = require('axios') +const fs = require('fs') + +const reviewsDownloader = async () => { + const response = await axios({ + method: 'post', + url: `https://${process.env.METABASE_URL}/api/session`, + data: { username: `${process.env.METABASE_MAIL}`, password: `${process.env.METABASE_PWD}` } + }) + console.log(response.data.id) + + const datasetReviews = await axios.post( + `https://${process.env.METABASE_URL}/api/dataset/json`, + new URLSearchParams({ + query: '{"database":2,"query":{"source-table":6},"type":"query","middleware":{"js-int-to-string?":true,"add-default-userland-constraints?":true}}', + visualization_settings: '{"column_settings":{},"table.pivot":false,"table.pivot_column":"completed_at","table.cell_column":"do_it_for_me","table.columns":[{"name":"id","fieldRef":["field",48,null],"enabled":true},{"name":"google_place_id","fieldRef":["field",52,null],"enabled":true},{"name":"created_at","fieldRef":["field",49,{"temporal-unit":"default"}],"enabled":true},{"name":"completed_at","fieldRef":["field",51,{"temporal-unit":"default"}],"enabled":true},{"name":"type","fieldRef":["field",47,null],"enabled":true},{"name":"do_it_for_me","fieldRef":["field",50,null],"enabled":true}],"table.column_formatting":[]}' + }), + { + headers: { + authority: `${process.env.METABASE_URL}`, + accept: '*/*', + 'accept-language': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7', + 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8', + cookie: `metabase.DEVICE=50796a71-d0ec-49c3-9dcd-d453b57c5f53; _ga=GA1.2.938353226.1673289574; metabase.TIMEOUT=alive; metabase.SESSION=${response.data.id}; _gid=GA1.2.1794784591.1675763957; _gat=1`, + origin: `https://${process.env.METABASE_URL}`, + referer: `https://${process.env.METABASE_URL}/question`, + 'sec-ch-ua': '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36' + } + } + ) + + const json = JSON.stringify(datasetReviews.data) + + fs.writeFile('reviews.json', json, 'utf8', function (res, err) { + console.log(res, err) + }) +} + +const placesDownloader = async () => { + const response = await axios({ + method: 'post', + url: `https://${process.env.METABASE_URL}/api/session`, + data: { username: `${process.env.METABASE_MAIL}`, password: `${process.env.METABASE_PWD}` } + }) + console.log(response.data.id) + + const datasetPlaces = await axios.post( + 'https://o2t37cyszh.execute-api.eu-west-3.amazonaws.com/api/dataset/json', + new URLSearchParams({ + query: '{"database":2,"query":{"source-table":7},"type":"query","middleware":{"js-int-to-string?":true,"add-default-userland-constraints?":true}}', + visualization_settings: '{"column_settings":{},"table.pivot":false,"table.pivot_column":"report_count","table.cell_column":"latitude","table.columns":[{"name":"google_place_id","fieldRef":["field",44,null],"enabled":true},{"name":"name","fieldRef":["field",38,null],"enabled":true},{"name":"google_place_url","fieldRef":["field",46,null],"enabled":true},{"name":"address","fieldRef":["field",41,null],"enabled":true},{"name":"phone_number","fieldRef":["field",45,null],"enabled":true},{"name":"created_at","fieldRef":["field",43,{"temporal-unit":"default"}],"enabled":true},{"name":"report_count","fieldRef":["field",39,null],"enabled":true},{"name":"latitude","fieldRef":["field",42,null],"enabled":true},{"name":"longitude","fieldRef":["field",40,null],"enabled":true}],"table.column_formatting":[]}' + }), + { + headers: { + authority: 'o2t37cyszh.execute-api.eu-west-3.amazonaws.com', + accept: '*/*', + 'accept-language': 'fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7', + 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8', + cookie: 'metabase.DEVICE=50796a71-d0ec-49c3-9dcd-d453b57c5f53; _ga=GA1.2.938353226.1673289574; metabase.TIMEOUT=alive; metabase.SESSION=475122a0-579c-45cd-bc49-f5e84783c5c0; _gid=GA1.2.1794784591.1675763957; _gat=1', + origin: `https://${process.env.METABASE_URL}`, + referer: `https://${process.env.METABASE_URL}/question`, + 'sec-ch-ua': '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36' + } + } + ) + + const json = JSON.stringify(datasetPlaces.data) + + fs.writeFile('places.json', json, 'utf8', function (res, err) { + console.log(res, err) + }) +} + +reviewsDownloader() +placesDownloader() diff --git a/package-lock.json b/package-lock.json index 3f15db6..9b649b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "curl": "^0.1.4", "dataframe-js": "^1.4.4", "dotenv": "^16.0.3", + "fs": "^0.0.1-security", "google-libphonenumber": "^3.2.32", "got": "^12.5.3", "ovh": "^2.0.3", @@ -3732,6 +3733,11 @@ "node": ">= 14.17" } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", diff --git a/package.json b/package.json index c807a47..2a6bdda 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "curl": "^0.1.4", "dataframe-js": "^1.4.4", "dotenv": "^16.0.3", + "fs": "^0.0.1-security", "google-libphonenumber": "^3.2.32", "got": "^12.5.3", "ovh": "^2.0.3",