Skip to content

Commit

Permalink
feat: Array functions
Browse files Browse the repository at this point in the history 
Signed-off-by: Andreas Reichel <andreas@manticore-projects.com>
  • Loading branch information
@manticore-projects
manticore-projects committed Apr 3, 2024
1 parent 0c32a05 commit d13d7a2
Show file tree
Hide file tree
Showing 7 changed files with 382 additions and 32 deletions.
67 changes: 42 additions & 25 deletions src/main/java/com/manticore/transpiler/ExpressionTranspiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ enum TranspiledFunction {

, SAFE_ADD, SAFE_DIVIDE, SAFE_MULTIPLY, SAFE_NEGATE, SAFE_SUBTRACT, TRUNC

, ARRAY_CONCAT_AGG, COUNTIF, LOGICAL_AND, LOGICAL_OR
, ARRAY_CONCAT_AGG, COUNTIF, LOGICAL_AND, LOGICAL_OR, ARRAY, ARRAY_CONCAT, ARRAY_TO_STRING


, NVL, UNNEST;
Expand All @@ -118,7 +118,11 @@ public static TranspiledFunction from(Function f) {
}

enum UnsupportedFunction {
ASINH, ACOSH, COSH, SINH, COTH, COSINE_DISTANCE, CSC, CSCH, EUCLIDEAN_DISTANCE, SEC, SECH;
ASINH, ACOSH, COSH, SINH, COTH, COSINE_DISTANCE, CSC, CSCH, EUCLIDEAN_DISTANCE, SEC, SECH

, APPROX_QUANTILES, APPROX_TOP_COUNT, APPROX_TOP_SUM

;

@SuppressWarnings({"PMD.EmptyCatchBlock"})
public static UnsupportedFunction from(String name) {
Expand Down Expand Up @@ -687,10 +691,7 @@ THEN Instr( source_value, Regexp_Extract( source_value, reg_exp ) )
new WhenClause(new Function("REGEXP_MATCHES", parameters.get(0), parameters.get(1)),
new Function("INSTR", parameters.get(0),
new Function("REGEXP_EXTRACT", parameters.get(0), parameters.get(1))));
CaseExpression caseExpression = new CaseExpression(new LongValue(0), when);
visit(caseExpression);

rewrittenExpression = caseExpression;
rewrittenExpression = new CaseExpression(new LongValue(0), when);
break;
case REGEXP_REPLACE:
// pass through
Expand Down Expand Up @@ -734,11 +735,9 @@ THEN Instr( source_value, Regexp_Extract( source_value, reg_exp ) )
function.setParameters(new Function("Decode", parameters.get(0)));
break;
case UNICODE:
Function ifFunction = new Function("If",
rewrittenExpression = new Function("If",
new EqualsTo(new Function("Length$$", parameters.get(0)), new LongValue(0)),
new LongValue(0), function.withName(function.getName() + "$$"));
visit(ifFunction);
rewrittenExpression = ifFunction;
break;
case DIV:
case IEEE_DIVIDE:
Expand Down Expand Up @@ -843,10 +842,37 @@ public void visit(Column column) {
case LOGICAL_OR:
function.setName("Bool_Or");
break;

case ARRAY:
function.setName("List_Sort");
function.setParameters(new Function("Array$$", parameters));
break;

case ARRAY_CONCAT:
rewrittenExpression =
Concat.concat(parameters.toArray(new Expression[parameters.size()]));
break;

case ARRAY_TO_STRING:
if (parameters != null) {
switch (parameters.size()) {
case 3:
Expression p1 = parameters.get(0);
Expression p2 = parameters.get(1);

// turn it into a Lambda replacing the NULL values with 3rd parameter
p1 = new Function("List_Transform", p1, new LambdaExpression("x",
new Function("Coalesce", new Column("x"), parameters.get(2))));
function.setParameters(p1, p2);
}
}
break;
}
}
if (rewrittenExpression == null) {
super.visit(function);
} else {
rewrittenExpression.accept(this);
}
}

Expand Down Expand Up @@ -897,6 +923,8 @@ public void visit(Column column) {
}
if (rewrittenExpression == null) {
super.visit(function);
} else {
rewrittenExpression.accept(this);
}
}

Expand All @@ -922,7 +950,6 @@ case typeof(bytes)
CaseExpression caseExpression = new CaseExpression(new LongValue(-1), whenChar, whenBLOB)
.withSwitchExpression(new Function("typeOf", parameters.get(0)));

visit(caseExpression);
return caseExpression;
}
}
Expand All @@ -949,7 +976,6 @@ private Expression rewritePad(Function function, ExpressionList<?> parameters) {
CaseExpression caseExpression = new CaseExpression(whenChar)
.withSwitchExpression(new Function("typeOf", parameters.get(0)));

visit(caseExpression);
return caseExpression;
}
}
Expand All @@ -971,7 +997,6 @@ private Expression rewriteContainsSubStr(ExpressionList<?> parameters) {
.withLeftExpression(new Function("nfc_normalize", parameters.get(0)))
.withRightExpression(new Function("nfc_normalize", concat));

visit(like);
return like;
case 3:
throw new RuntimeException(
Expand All @@ -995,7 +1020,6 @@ private Expression rewriteCodePointsToBytes(ExpressionList<?> parameters) {

Parenthesis p = new Parenthesis(select);

visit(p);
return p;
}

Expand All @@ -1012,7 +1036,6 @@ private Expression rewriteCodePointsToString(ExpressionList<?> parameters) {

Parenthesis p = new Parenthesis(select);

visit(p);
return p;
}

Expand Down Expand Up @@ -1076,7 +1099,6 @@ private Expression rewriteParseDateFunction(Function function, ExpressionList<?>

CastExpression castExpression = new CastExpression().withLeftExpression(function)
.withType(new ColDataType(dateTimeType.name()));
visit(castExpression);

return castExpression;
default:
Expand Down Expand Up @@ -1237,7 +1259,6 @@ private Expression rewriteDateTruncFunction(Function function, ExpressionList<?>
function.setName("DATE_TRUNC$$");
CastExpression castExpression = new CastExpression().withLeftExpression(function)
.withType(new ColDataType(dateTimeType.name()));
visit(castExpression);
return castExpression;
}
break;
Expand Down Expand Up @@ -1268,7 +1289,6 @@ private Expression rewriteDateTruncFunction(Function function, ExpressionList<?>
function.setName("DATE_TRUNC$$");
CastExpression castExpression = new CastExpression().withLeftExpression(function)
.withType(new ColDataType(DateTimeLiteralExpression.DateTime.TIMESTAMPTZ.name()));
visit(castExpression);
return castExpression;
}
return null;
Expand Down Expand Up @@ -1361,7 +1381,6 @@ private Expression rewriteDateFunction(Function function, ExpressionList<?> para
warning("timezone not supported");
castExpression = new CastExpression("Cast").withLeftExpression(parameters.get(0))
.withType(new ColDataType().withDataType("DATE"));
visit(castExpression);
break;
case 3:
function.setName("MAKE_DATE");
Expand All @@ -1383,7 +1402,6 @@ private Expression rewriteDateTimeFunction(ExpressionList<?> parameters) {
new Addition().withLeftExpression(dateFuncttion).withRightExpression(timeFuncttion);
castExpression = new CastExpression("Cast").withLeftExpression(add)
.withType(new ColDataType().withDataType("DATETIME"));
visit(castExpression);
break;
case 2:
if (parameters.get(0) instanceof DateTimeLiteralExpression
Expand All @@ -1392,7 +1410,6 @@ private Expression rewriteDateTimeFunction(ExpressionList<?> parameters) {
.withRightExpression(parameters.get(1));
castExpression = new CastExpression("Cast").withLeftExpression(add)
.withType(new ColDataType().withDataType("DATETIME"));
visit(castExpression);
} else if (parameters.get(0) instanceof DateTimeLiteralExpression
&& ((DateTimeLiteralExpression) parameters.get(0))
.getType() == DateTimeLiteralExpression.DateTime.TIMESTAMP
Expand All @@ -1401,9 +1418,8 @@ private Expression rewriteDateTimeFunction(ExpressionList<?> parameters) {
warning("timezone not supported");
castExpression = new CastExpression("Cast").withLeftExpression(parameters.get(0))
.withType(new ColDataType().withDataType("DATETIME"));
visit(castExpression);
} else {
// @todo: veryify if this needs to be ammended
// @todo: verify if this needs to be amended
throw new RuntimeException("Unsupported: DATETIME(string, string) is not supported yet.");
}
break;
Expand All @@ -1421,7 +1437,6 @@ private Expression rewriteTimeFunction(Function function, ExpressionList<?> para
warning("timezone not supported");
castExpression = new CastExpression("Cast").withLeftExpression(parameters.get(0))
.withType(new ColDataType().withDataType("TIME"));
visit(castExpression);
break;
case 3:
function.setName("MAKE_TIME");
Expand All @@ -1446,14 +1461,12 @@ private Expression rewriteTimestampFunction(ExpressionList<?> parameters) {
case 1:
castExpression = new CastExpression("Cast").withLeftExpression(parameters.get(0))
.withType(new ColDataType().withDataType(timestampType));
visit(castExpression);
return castExpression;
case 2:
castExpression = new CastExpression("Cast").withLeftExpression(parameters.get(0))
.withType(new ColDataType().withDataType(timestampType));
TimezoneExpression timezoneExpression =
new TimezoneExpression(castExpression, parameters.get(1));
visit(timezoneExpression);
return timezoneExpression;
}
}
Expand Down Expand Up @@ -1608,9 +1621,13 @@ public void visit(StructType structType) {
}


// @todo: complete the data type mapping
// implement an Enum on Big Query allowed data types
public final static ColDataType rewriteType(ColDataType colDataType) {
if (colDataType.getDataType().equalsIgnoreCase("BYTES")) {
colDataType.setDataType("BLOB");
} else if (colDataType.getDataType().equalsIgnoreCase("FLOAT64")) {
colDataType.setDataType("FLOAT");
}
return colDataType;
}
Expand Down
Binary file modified src/main/resources/doc/JSQLTranspiler.ods
View file
Binary file not shown.
15 changes: 8 additions & 7 deletions src/test/java/com/manticore/transpiler/JSQLTranspilerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ class JSQLTranspilerTest {
public static final FilenameFilter FILENAME_FILTER = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(".sql");
String filename = name.toLowerCase().trim();
return name.endsWith(".sql") && !name.startsWith("disabled");
}
};

Expand Down Expand Up @@ -157,7 +158,7 @@ static Map<File, List<SQLTest>> getSqlTestMap(File[] testFiles,
k = line.substring(2).trim().toLowerCase();
}

if (line.toLowerCase().replaceAll("\\s", "").startsWith("--provided")) {
if (line.toLowerCase().replaceAll("\\s", "").startsWith("--provid")) {
if (test.providedSqlStr != null
&& (test.expectedTally >= 0 || test.expectedResult != null)) {
LOGGER.fine("Found multiple test descriptions in " + file.getName());
Expand All @@ -176,20 +177,20 @@ static Map<File, List<SQLTest>> getSqlTestMap(File[] testFiles,
endContent = startContent && !line.startsWith("--")
&& (line.trim().endsWith(";")
|| (k.equalsIgnoreCase("count") || k.equalsIgnoreCase("tally")) && line.isEmpty()
|| k.startsWith("result") && line.isEmpty());
|| (k.startsWith("result") || k.startsWith("return")) && line.isEmpty());

if (startContent && !line.isEmpty()) {
stringBuilder.append(line).append("\n");
}

if (endContent) {
if (k.equalsIgnoreCase("provided")) {
if (k.startsWith("provid")) {
test.providedSqlStr = stringBuilder.toString();
} else if (k.equalsIgnoreCase("expected")) {
} else if (k.startsWith("expect")) {
test.expectedSqlStr = stringBuilder.toString();
} else if (k.equalsIgnoreCase("count") || k.equalsIgnoreCase("tally")) {
} else if (k.startsWith("count") || k.startsWith("tally")) {
test.expectedTally = Integer.parseInt(stringBuilder.toString().trim());
} else if (k.startsWith("result")) {
} else if (k.startsWith("result") || k.startsWith("return")) {
test.expectedResult = stringBuilder.toString().trim();
}
stringBuilder.setLength(0);
Expand Down
Loading

0 comments on commit d13d7a2

Please # to comment.