add csv support #948

alibaba · Nov 24, 2022 · 36419b4 · 36419b4
1 parent 0ec3ba5
commit 36419b4
Show file tree

Hide file tree

Showing 36 changed files with 499,606 additions and 1,363 deletions.
diff --git a/adapter/src/main/java/com/alibaba/fastjson2/adapter/jackson/databind/ObjectWriter.java b/adapter/src/main/java/com/alibaba/fastjson2/adapter/jackson/databind/ObjectWriter.java
@@ -2,10 +2,13 @@
 
 import com.alibaba.fastjson2.JSONWriter;
 import com.alibaba.fastjson2.adapter.jackson.core.JsonFactory;
+import com.alibaba.fastjson2.csv.CSVWriter;
 import com.alibaba.fastjson2.util.TypeUtils;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
 
 public class ObjectWriter {
     final SerializationConfig config;
@@ -26,6 +29,14 @@ public ObjectWriter(
 
     public String writeValueAsString(Object value)
             throws IOException {
+        if (jsonFactory.isCSV()) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            CSVWriter writer = CSVWriter.of(out);
+            writer.writeRowObject(value);
+            writer.close();
+            String str = new String(out.toByteArray(), StandardCharsets.UTF_8);
+            return str;
+        }
         JSONWriter jsonWriter = jsonFactory.createJSONWriter();
         JSONWriter.Context context = jsonWriter.getContext();
         context.getObjectWriter(objectType, objectClass)

diff --git a/adapter/src/main/java/com/alibaba/fastjson2/adapter/jackson/dataformat/csv/CsvFactory.java b/adapter/src/main/java/com/alibaba/fastjson2/adapter/jackson/dataformat/csv/CsvFactory.java
@@ -1,16 +1,9 @@
 package com.alibaba.fastjson2.adapter.jackson.dataformat.csv;
 
-import com.alibaba.fastjson2.JSONFactory;
-import com.alibaba.fastjson2.JSONWriter;
 import com.alibaba.fastjson2.adapter.jackson.core.JsonFactory;
 
 public class CsvFactory
         extends JsonFactory {
-    public JSONWriter createJSONWriter() {
-        JSONWriter.Context context = JSONFactory.createWriteContext(JSONWriter.Feature.BeanToArray);
-        return JSONWriter.ofCSV(context);
-    }
-
     public boolean isCSV() {
         return true;
     }

diff --git a/...t/java/com/alibaba/fastjson2/adapter/jackson/databind/flink/JacksonMapperFactoryTest.java b/...t/java/com/alibaba/fastjson2/adapter/jackson/databind/flink/JacksonMapperFactoryTest.java
@@ -17,7 +17,7 @@ public class JacksonMapperFactoryTest {
     public void test() throws Exception {
         final CsvMapper mapper = JacksonMapperFactory.createCsvMapper();
 
-        final String instantString = "2022-08-07T12:00:33.107787800Z";
+        final String instantString = "2022-08-07T12:00:33.1077878Z";
         final Instant instant = Instant.parse(instantString);
         final String instantCsv = String.format("%s\n", instantString);
 

diff --git a/benchmark/src/main/java/com/alibaba/fastjson2/DoubleValidate.java b/benchmark/src/main/java/com/alibaba/fastjson2/DoubleValidate.java
@@ -1,6 +1,7 @@
 package com.alibaba.fastjson2;
 
 import com.alibaba.fastjson2.util.IOUtils;
+import com.alibaba.fastjson2.util.TypeUtils;
 
 import java.text.NumberFormat;
 import java.time.LocalDateTime;
@@ -33,7 +34,7 @@ public static void main(String[] args) throws Exception {
             for (long j = 0; j < max; ++j) {
                 IOUtils.getChars(j, index, chars);
 
-                double d0 = FloatingDecimal.parseDouble(chars, 0, index);
+                double d0 = TypeUtils.parseDouble(chars, 0, index);
                 double d1 = j / JSONFactory.SMALL_10_POW[p];
                 if (d0 != d1) {
                     String str = new String(chars, 0, index);

diff --git a/benchmark/src/main/java/com/alibaba/fastjson2/FloatValidate.java b/benchmark/src/main/java/com/alibaba/fastjson2/FloatValidate.java
@@ -1,6 +1,7 @@
 package com.alibaba.fastjson2;
 
 import com.alibaba.fastjson2.util.IOUtils;
+import com.alibaba.fastjson2.util.TypeUtils;
 
 import java.text.NumberFormat;
 import java.util.Arrays;
@@ -32,7 +33,7 @@ public static void main(String[] args) throws Exception {
             for (long j = 0; j < max; ++j) {
                 IOUtils.getChars(j, index, chars);
 
-                float f0 = FloatingDecimal.parseFloat(chars, 0, index);
+                float f0 = TypeUtils.parseFloat(chars, 0, index);
                 float f1 = (float) (j / JSONFactory.SMALL_10_POW[p]);
                 if (f0 != f1) {
                     String str = new String(chars, 0, index);

diff --git a/benchmark/src/main/java/com/alibaba/fastjson2/benchmark/CSVBig38M.java b/benchmark/src/main/java/com/alibaba/fastjson2/benchmark/CSVBig38M.java
@@ -7,6 +7,8 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.lang.reflect.Type;
+import java.math.BigDecimal;
 import java.net.URL;
 
 public class CSVBig38M {
@@ -42,4 +44,27 @@ public void readLines(Blackhole bh) throws IOException {
         }
         bh.consume(rowCount);
     }
+
+    @Benchmark
+    public void readLineValues(Blackhole bh) throws IOException {
+        URL resource = Thread.currentThread().getContextClassLoader().getResource("organised_Gen.csv");
+        if (resource == null) {
+            return;
+        }
+
+        File file = new File(resource.getFile());
+        Type[] types = new Type[] {
+                Integer.class, Integer.class, Integer.class, String.class, String.class, String.class, BigDecimal.class
+        };
+        CSVParser parser = CSVParser.of(file, types);
+        int rowCount = 0;
+        while (true) {
+            Object[] line = parser.readLineValues();
+            if (line == null) {
+                break;
+            }
+            rowCount++;
+        }
+        bh.consume(rowCount);
+    }
 }
diff --git a/benchmark/src/test/java/com/alibaba/fastjson2/FloatingDecimalTest.java b/benchmark/src/test/java/com/alibaba/fastjson2/FloatingDecimalTest.java
@@ -1,5 +1,6 @@
 package com.alibaba.fastjson2;
 
+import com.alibaba.fastjson2.util.TypeUtils;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -10,11 +11,11 @@ public void test0() {
         String s0 = "A123.456789000B";
         String s1 = s0.substring(1, s0.length() - 1);
         char[] chars = s0.toCharArray();
-        float f0 = FloatingDecimal.parseFloat(chars, 1, chars.length - 2);
+        float f0 = TypeUtils.parseFloat(chars, 1, chars.length - 2);
         float f1 = Float.parseFloat(s1);
         assertEquals(f1, f0);
 
-        double d0 = FloatingDecimal.parseDouble(chars, 1, chars.length - 2);
+        double d0 = TypeUtils.parseDouble(chars, 1, chars.length - 2);
         double d1 = Double.parseDouble(s1);
         assertEquals(d1, d0);
     }

diff --git a/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBig38MTest.java b/benchmark/src/test/java/com/alibaba/fastjson2/benchmark/CSVBig38MTest.java
@@ -14,7 +14,7 @@ public static void rowCount() throws Exception {
             }
             long millis = System.currentTimeMillis() - start;
             System.out.println("CSVBig38M-rowCount millis : " + millis);
-            // zulu8.62.0.19 : 419 393
+            // zulu8.62.0.19 : 336
         }
     }
 
@@ -26,11 +26,23 @@ public static void readLines() throws Exception {
             }
             long millis = System.currentTimeMillis() - start;
             System.out.println("CSVBig38M-readLines millis : " + millis);
-            // zulu8.62.0.19 :
+            // zulu8.62.0.19 : 2218
+        }
+    }
+
+    public static void readLineValues() throws Exception {
+        for (int j = 0; j < COUNT; j++) {
+            long start = System.currentTimeMillis();
+            for (int i = 0; i < 10; ++i) {
+                benchmark.readLineValues(BH);
+            }
+            long millis = System.currentTimeMillis() - start;
+            System.out.println("CSVBig38M-readLines millis : " + millis);
+            // zulu8.62.0.19 : 2267
         }
     }
 
     public static void main(String[] args) throws Exception {
-        readLines();
+        readLineValues();
     }
 }