Merge pull request #29 from detroitledger/27-null-eins

import empty organization ein as null

src/db/migrations/20190116144239-org-ein-pad-zeros.js

@@ -0,0 +1,19 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query(`update organization set ein=null where ein='0'`)
+      .then(() =>
+        queryInterface.sequelize.query(
+          `update organization set ein=lpad(ein, 9, '0')`
+        )
+      );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query(`update organization set ein='0' where ein is null`)
+      .then(() => `update organization set ein=trim(leading '0' from ein)`);
+  },
+};

src/scripts/orgImporter.ts

@@ -30,7 +30,7 @@ export const orgs = require(`${DATADIR}/orgs.json`).orgs;
 const importOrg = async drupalOrg => {
   const cleansed = {
     name: drupalOrg.title,
-    ein: drupalOrg.field_ein,
+    ein: drupalOrg.field_ein || null,
     duns: drupalOrg.field_duns != 0 ? drupalOrg.field_duns : null,
     stateCorpId: drupalOrg.field_state_corp_id
       ? drupalOrg.field_state_corp_id.value
@@ -116,4 +116,10 @@ export const doImport = async () => {
   }
 };
 
-doImport();
+doImport()
+  .then(() => process.exit(0))
+  .catch(e => {
+    console.error('ERROR');
+    console.error(e);
+    process.exit(1);
+  });

src/scripts/tagImporter.ts

@@ -11,28 +11,55 @@ const DATADIR = `${
   process.env.HOME
 }/gnl/data.detroitledger.org/profiles/gnl_profile/exporters`;
 
-const createBasicParser = (model: Sequelize.Model<any, any>) => (err, data) =>
-  data.map(datum => model.create(datum));
-
-[
-  {
-    file: 'grantTag.csv',
-    parser: createBasicParser(db.GrantTag),
-  },
-  {
-    file: 'organizationTag.csv',
-    parser: createBasicParser(db.OrganizationTag),
-  },
-  {
-    file: 'nteeGrantType.csv',
-    parser: createBasicParser(db.NteeGrantType),
-  },
-  {
-    file: 'nteeOrganizationType.csv',
-    parser: createBasicParser(db.NteeOrganizationType),
-  },
-].map(cfg =>
-  fs
-    .createReadStream(`${DATADIR}/${cfg.file}`)
-    .pipe(csvParse({ delimiter: ',', columns: true }, cfg.parser))
-);
+function doImport() {
+  const promises = [
+    {
+      file: 'grantTag.csv',
+      model: db.GrantTag,
+    },
+    {
+      file: 'organizationTag.csv',
+      model: db.OrganizationTag,
+    },
+    {
+      file: 'nteeGrantType.csv',
+      model: db.NteeGrantType,
+    },
+    {
+      file: 'nteeOrganizationType.csv',
+      model: db.NteeOrganizationType,
+    },
+  ].map(
+    cfg =>
+      new Promise((resolve, reject) => {
+        const stream = fs
+          .createReadStream(`${DATADIR}/${cfg.file}`)
+          .pipe(csvParse({ delimiter: ',', columns: true }));
+
+        stream.on('readable', async () => {
+          let chunk;
+
+          chunk = stream.read();
+
+          while (chunk !== null) {
+            await cfg.model.create(chunk);
+            chunk = stream.read();
+          }
+
+          resolve();
+        });
+
+        stream.on('error', reject);
+      })
+  );
+
+  return Promise.all(promises);
+}
+
+doImport()
+  .then(() => process.exit(0))
+  .catch(e => {
+    console.error('ERROR');
+    console.error(e);
+    process.exit(1);
+  });