More extensive stats for n-grams

Jaybit0 · Sep 2, 2024 · 05f5033 · 05f5033
1 parent f4e53ba
commit 05f5033
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java b/src/main/java/org/apache/sysds/utils/stats/NGramBuilder.java
@@ -19,6 +19,8 @@
 
 package org.apache.sysds.utils.stats;
 
+import org.apache.commons.lang3.function.TriFunction;
+
 import java.lang.reflect.Array;
 import java.util.Arrays;
 import java.util.Comparator;

diff --git a/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java b/src/test/java/org/apache/sysds/test/applications/ApplyTransformTest.java
@@ -19,6 +19,8 @@
 
 package org.apache.sysds.test.applications;
 
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -27,6 +29,9 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.sysds.api.DMLScript;
+import org.apache.sysds.utils.Statistics;
+import org.apache.sysds.utils.stats.NGramBuilder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -176,5 +181,17 @@ public void testApplyTransform() {
 			if(!XDML.containsKey(cell4)) success = false;
 			else success = success && (dummy_coding_maps != " ") ? (XDML.get(cell4).doubleValue() == 1) : (XDML.get(cell4).doubleValue() == 2);
 		 }
+
+		NGramBuilder<String, Statistics.NGramStats>[] builders = Statistics.mergeNGrams();
+		for (int i = 0; i < builders.length; i++) {
+			try (FileWriter writer = new FileWriter("/Users/janniklindemann/Dev/MScThesis/NGramAnalysis/" + TEST_NAME + testCtr + "_" + DMLScript.STATISTICS_NGRAM_SIZES[i] + "-grams.csv")) {
+				writer.write(NGramBuilder.toCSV(new String[] { "N-Gram", "Time[s]", "StdDev(Time[s])/Mean(Time[s])", "Count" }, builders[i].getTopK(100000, Statistics.NGramStats.getComparator(), true), e -> Statistics.getNGramStdDevs(e.getStats(), 5).replace("-", "").replace(",", ";")));
+			} catch (IOException e) {
+
+			}
+		}
+		testCtr++;
 	 }
+
+	public static int testCtr = 1;
 }