From 8fd3b7afd21200f769269a2c4116d8fd6c5ed487 Mon Sep 17 00:00:00 2001
From: Benjamin Chodoroff <bc@thermitic.net>
Date: Tue, 15 Jan 2019 21:44:39 -0500
Subject: [PATCH 1/3] import empty organization ein as null

fixes #27

also, fix up tag importer so it exits properly
---
 src/scripts/orgImporter.ts | 10 ++++--
 src/scripts/tagImporter.ts | 74 +++++++++++++++++++++++++-------------
 2 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/src/scripts/orgImporter.ts b/src/scripts/orgImporter.ts
index b8df817..794fb8f 100644
--- a/src/scripts/orgImporter.ts
+++ b/src/scripts/orgImporter.ts
@@ -30,7 +30,7 @@ export const orgs = require(`${DATADIR}/orgs.json`).orgs;
 const importOrg = async drupalOrg => {
   const cleansed = {
     name: drupalOrg.title,
-    ein: drupalOrg.field_ein,
+    ein: drupalOrg.field_ein || null,
     duns: drupalOrg.field_duns != 0 ? drupalOrg.field_duns : null,
     stateCorpId: drupalOrg.field_state_corp_id
       ? drupalOrg.field_state_corp_id.value
@@ -116,4 +116,10 @@ export const doImport = async () => {
   }
 };
 
-doImport();
+doImport()
+  .then(() => process.exit(0))
+  .catch(e => {
+    console.error('ERROR');
+    console.error(e);
+    process.exit(1);
+  });
diff --git a/src/scripts/tagImporter.ts b/src/scripts/tagImporter.ts
index 72d8e94..fe21961 100644
--- a/src/scripts/tagImporter.ts
+++ b/src/scripts/tagImporter.ts
@@ -11,28 +11,52 @@ const DATADIR = `${
   process.env.HOME
 }/gnl/data.detroitledger.org/profiles/gnl_profile/exporters`;
 
-const createBasicParser = (model: Sequelize.Model<any, any>) => (err, data) =>
-  data.map(datum => model.create(datum));
-
-[
-  {
-    file: 'grantTag.csv',
-    parser: createBasicParser(db.GrantTag),
-  },
-  {
-    file: 'organizationTag.csv',
-    parser: createBasicParser(db.OrganizationTag),
-  },
-  {
-    file: 'nteeGrantType.csv',
-    parser: createBasicParser(db.NteeGrantType),
-  },
-  {
-    file: 'nteeOrganizationType.csv',
-    parser: createBasicParser(db.NteeOrganizationType),
-  },
-].map(cfg =>
-  fs
-    .createReadStream(`${DATADIR}/${cfg.file}`)
-    .pipe(csvParse({ delimiter: ',', columns: true }, cfg.parser))
-);
+function doImport() {
+  const promises = [
+    {
+      file: 'grantTag.csv',
+      model: db.GrantTag,
+    },
+    {
+      file: 'organizationTag.csv',
+      model: db.OrganizationTag,
+    },
+    {
+      file: 'nteeGrantType.csv',
+      model: db.NteeGrantType,
+    },
+    {
+      file: 'nteeOrganizationType.csv',
+      model: db.NteeOrganizationType,
+    },
+  ].map(
+    cfg =>
+      new Promise((resolve, reject) => {
+        const stream = fs
+          .createReadStream(`${DATADIR}/${cfg.file}`)
+          .pipe(csvParse({ delimiter: ',', columns: true }));
+
+        stream.on('readable', async () => {
+          let chunk;
+
+          while (null !== (chunk = stream.read())) {
+            await cfg.model.create(chunk);
+          }
+
+          resolve();
+        });
+
+        stream.on('error', reject);
+      })
+  );
+
+  return Promise.all(promises);
+}
+
+doImport()
+  .then(() => process.exit(0))
+  .catch(e => {
+    console.error('ERROR');
+    console.error(e);
+    process.exit(1);
+  });

From 1a4b96752beb8d998fd7d989fb9a17f30a2d4964 Mon Sep 17 00:00:00 2001
From: Benjamin Chodoroff <bc@thermitic.net>
Date: Wed, 16 Jan 2019 10:21:20 -0500
Subject: [PATCH 2/3] add migration to clean up org eins

---
 .../20190116144239-org-ein-pad-zeros.js       | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 src/db/migrations/20190116144239-org-ein-pad-zeros.js

diff --git a/src/db/migrations/20190116144239-org-ein-pad-zeros.js b/src/db/migrations/20190116144239-org-ein-pad-zeros.js
new file mode 100644
index 0000000..94a8587
--- /dev/null
+++ b/src/db/migrations/20190116144239-org-ein-pad-zeros.js
@@ -0,0 +1,19 @@
+'use strict';
+
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query(`update organization set ein=null where ein='0'`)
+      .then(() =>
+        queryInterface.sequelize.query(
+          `update organization set ein=lpad(ein, 9, '0')`
+        )
+      );
+  },
+
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.sequelize
+      .query(`update organization set ein='0' where ein is null`)
+      .then(() => `update organization set ein=trim(leading '0' from ein)`);
+  },
+};

From a51ee444fc01c10dda3b265d65f89786e2dbdf86 Mon Sep 17 00:00:00 2001
From: Benjamin Chodoroff <bc@thermitic.net>
Date: Fri, 25 Jan 2019 09:04:20 -0500
Subject: [PATCH 3/3] nicer syntax

---
 src/scripts/tagImporter.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/scripts/tagImporter.ts b/src/scripts/tagImporter.ts
index fe21961..78cad8a 100644
--- a/src/scripts/tagImporter.ts
+++ b/src/scripts/tagImporter.ts
@@ -39,8 +39,11 @@ function doImport() {
         stream.on('readable', async () => {
           let chunk;
 
-          while (null !== (chunk = stream.read())) {
+          chunk = stream.read();
+
+          while (chunk !== null) {
             await cfg.model.create(chunk);
+            chunk = stream.read();
           }
 
           resolve();