Skip to content

Commit

Permalink
Merge pull request #33 from civitaspo/support_floating_point_numbers
Browse files Browse the repository at this point in the history
Support type conversion from floating point numbers to integers
  • Loading branch information
civitaspo authored Sep 12, 2017
2 parents 4bbbdbf + 76f0fa1 commit 5cb82cd
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,13 @@ else if (Types.LONG.equals(expandedJsonColumn.getColumn().getType())) {
pageBuilder.setLong(expandedJsonColumn.getColumn(), Long.parseLong(finalValue));
}
catch (NumberFormatException e) {
throw new JsonValueInvalidException(String.format("Failed to parse '%s' as long", finalValue), e);
// ad-hoc workaround for exponential notation
try {
pageBuilder.setLong(expandedJsonColumn.getColumn(), (long) Double.parseDouble(finalValue));
}
catch (NumberFormatException e2) {
throw new JsonValueInvalidException(String.format("Failed to parse '%s' as long", finalValue), e);
}
}
}
else if (Types.TIMESTAMP.equals(expandedJsonColumn.getColumn().getType())) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -966,6 +966,55 @@ public void run(TaskSource taskSource, Schema outputSchema)
});
}

@Test
public void testParseNumbersInExponentialNotation()
{
final String configYaml = "" +
"type: expand_json\n" +
"json_column_name: _c1\n" +
"root: $.\n" +
"expanded_columns:\n" +
" - {name: _j0, type: double}\n" +
" - {name: _j1, type: long}\n";
ConfigSource config = getConfigFromYaml(configYaml);
final Schema schema = schema("_c1", STRING);

expandJsonFilterPlugin.transaction(config, schema, new Control()
{
@Override
public void run(TaskSource taskSource, Schema outputSchema)
{
MockPageOutput mockPageOutput = new MockPageOutput();

String doubleFloatingPoint = "-1.234e-5";
double doubleFixedPoint = -0.00001234; // Use in Asserting.
String longFloatingPoint = "12345e3";
long longFixedPoint = 12_345_000L; // Use in Asserting.

String data = String.format(
"{\"_j0\":%s, \"_j1\":%s}",
doubleFloatingPoint,
longFloatingPoint);

try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, data, c1Data)) {
pageOutput.add(page);
}

pageOutput.finish();
}

PageReader pageReader = new PageReader(outputSchema);

for (Page page : mockPageOutput.pages) {
pageReader.setPage(page);
assertEquals(doubleFixedPoint, pageReader.getDouble(outputSchema.getColumn(0)), 0.0);
assertEquals(longFixedPoint, pageReader.getLong(outputSchema.getColumn(1)));
}
}
});
}

private static Schema schema(Object... nameAndTypes)
{
Schema.Builder builder = Schema.builder();
Expand Down

0 comments on commit 5cb82cd

Please # to comment.