diff --git a/src/db/migrations/20190116144239-org-ein-pad-zeros.js b/src/db/migrations/20190116144239-org-ein-pad-zeros.js new file mode 100644 index 0000000..94a8587 --- /dev/null +++ b/src/db/migrations/20190116144239-org-ein-pad-zeros.js @@ -0,0 +1,19 @@ +'use strict'; + +module.exports = { + up: (queryInterface, Sequelize) => { + return queryInterface.sequelize + .query(`update organization set ein=null where ein='0'`) + .then(() => + queryInterface.sequelize.query( + `update organization set ein=lpad(ein, 9, '0')` + ) + ); + }, + + down: (queryInterface, Sequelize) => { + return queryInterface.sequelize + .query(`update organization set ein='0' where ein is null`) + .then(() => `update organization set ein=trim(leading '0' from ein)`); + }, +}; diff --git a/src/scripts/orgImporter.ts b/src/scripts/orgImporter.ts index b8df817..794fb8f 100644 --- a/src/scripts/orgImporter.ts +++ b/src/scripts/orgImporter.ts @@ -30,7 +30,7 @@ export const orgs = require(`${DATADIR}/orgs.json`).orgs; const importOrg = async drupalOrg => { const cleansed = { name: drupalOrg.title, - ein: drupalOrg.field_ein, + ein: drupalOrg.field_ein || null, duns: drupalOrg.field_duns != 0 ? drupalOrg.field_duns : null, stateCorpId: drupalOrg.field_state_corp_id ? drupalOrg.field_state_corp_id.value @@ -116,4 +116,10 @@ export const doImport = async () => { } }; -doImport(); +doImport() + .then(() => process.exit(0)) + .catch(e => { + console.error('ERROR'); + console.error(e); + process.exit(1); + }); diff --git a/src/scripts/tagImporter.ts b/src/scripts/tagImporter.ts index 72d8e94..78cad8a 100644 --- a/src/scripts/tagImporter.ts +++ b/src/scripts/tagImporter.ts @@ -11,28 +11,55 @@ const DATADIR = `${ process.env.HOME }/gnl/data.detroitledger.org/profiles/gnl_profile/exporters`; -const createBasicParser = (model: Sequelize.Model) => (err, data) => - data.map(datum => model.create(datum)); - -[ - { - file: 'grantTag.csv', - parser: createBasicParser(db.GrantTag), - }, - { - file: 'organizationTag.csv', - parser: createBasicParser(db.OrganizationTag), - }, - { - file: 'nteeGrantType.csv', - parser: createBasicParser(db.NteeGrantType), - }, - { - file: 'nteeOrganizationType.csv', - parser: createBasicParser(db.NteeOrganizationType), - }, -].map(cfg => - fs - .createReadStream(`${DATADIR}/${cfg.file}`) - .pipe(csvParse({ delimiter: ',', columns: true }, cfg.parser)) -); +function doImport() { + const promises = [ + { + file: 'grantTag.csv', + model: db.GrantTag, + }, + { + file: 'organizationTag.csv', + model: db.OrganizationTag, + }, + { + file: 'nteeGrantType.csv', + model: db.NteeGrantType, + }, + { + file: 'nteeOrganizationType.csv', + model: db.NteeOrganizationType, + }, + ].map( + cfg => + new Promise((resolve, reject) => { + const stream = fs + .createReadStream(`${DATADIR}/${cfg.file}`) + .pipe(csvParse({ delimiter: ',', columns: true })); + + stream.on('readable', async () => { + let chunk; + + chunk = stream.read(); + + while (chunk !== null) { + await cfg.model.create(chunk); + chunk = stream.read(); + } + + resolve(); + }); + + stream.on('error', reject); + }) + ); + + return Promise.all(promises); +} + +doImport() + .then(() => process.exit(0)) + .catch(e => { + console.error('ERROR'); + console.error(e); + process.exit(1); + });