Added parsing date with SQL format

clearlydefined/crawler#450
qtomlinson · Mar 8, 2022 · 2b168c8 · 2b168c8
1 parent de88433
commit 2b168c8
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 4 deletions.
diff --git a/lib/utils.js b/lib/utils.js
@@ -111,12 +111,16 @@ function simplifyAttributions(entries) {
   return uniqBy(sorted, value => trim(value, '.')).reverse()
 }
 
+const dateTimeFormats = ['MM-dd-yyyy']
 function utcDateTime(dateAndTime) {
   const utcOpt = { zone: 'utc' }
   let result = DateTime.fromISO(dateAndTime, utcOpt)
-  if (!result.isValid) {
-    //fromISO includes parsing pattern 'yyyy-MM-dd' (https://moment.github.io/luxon/#/parsing?id=iso-8601)
-    result = DateTime.fromFormat(dateAndTime, 'MM-dd-yyyy', utcOpt)
+  if (!result.isValid) result = DateTime.fromRFC2822(dateAndTime, utcOpt)
+  if (!result.isValid) result = DateTime.fromHTTP(dateAndTime, utcOpt)
+  if (!result.isValid) result = DateTime.fromSQL(dateAndTime, utcOpt)
+
+  for (let index = 0; !result.isValid && index < dateTimeFormats.length; index++) {
+    result = DateTime.fromFormat(dateAndTime, dateTimeFormats[index])
   }
   return result.isValid ? result : null
 }

diff --git a/test/lib/util.js b/test/lib/util.js
@@ -431,7 +431,8 @@ describe('Utils extractDate', () => {
       '1900-01-01:T00:00:00': null,
       '9999-01-01:T00:00:00': null,
       '1900-01-01': null,
-      '9999-01-01': null
+      '9999-01-01': null,
+      '2018-05-28 07:26:25 UTC': '2018-05-28'
     }
 
     for (const timestamp of Object.getOwnPropertyNames(inputs)) {