Skip to content

Commit 12b473b

Browse files
compheadcomphead
authored
Change input for to_timestamp function to be seconds rather than nanoseconds, add to_timestamp_nanos (#7844)
* Change input for `to_timestamp` function * docs * fix examples * output `to_timestamp` signature as ns
1 parent 0911f15 commit 12b473b

File tree

14 files changed

+130
-32
lines changed

14 files changed

+130
-32
lines changed

‎datafusion/core/tests/sql/expr.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -639,7 +639,7 @@ async fn test_uuid_expression() -> Result<()> {
639639
async fn test_extract_date_part() -> Result<()> {
640640
test_expression!("date_part('YEAR', CAST('2000-01-01' AS DATE))", "2000.0");
641641
test_expression!(
642-
"EXTRACT(year FROM to_timestamp('2020-09-08T12:00:00+00:00'))",
642+
"EXTRACT(year FROM timestamp '2020-09-08T12:00:00+00:00')",
643643
"2020.0"
644644
);
645645
test_expression!("date_part('QUARTER', CAST('2000-01-01' AS DATE))", "1.0");
@@ -686,35 +686,35 @@ async fn test_extract_date_part() -> Result<()> {
686686
"12.0"
687687
);
688688
test_expression!(
689-
"EXTRACT(second FROM to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
689+
"EXTRACT(second FROM timestamp '2020-09-08T12:00:12.12345678+00:00')",
690690
"12.12345678"
691691
);
692692
test_expression!(
693-
"EXTRACT(millisecond FROM to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
693+
"EXTRACT(millisecond FROM timestamp '2020-09-08T12:00:12.12345678+00:00')",
694694
"12123.45678"
695695
);
696696
test_expression!(
697-
"EXTRACT(microsecond FROM to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
697+
"EXTRACT(microsecond FROM timestamp '2020-09-08T12:00:12.12345678+00:00')",
698698
"12123456.78"
699699
);
700700
test_expression!(
701-
"EXTRACT(nanosecond FROM to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
701+
"EXTRACT(nanosecond FROM timestamp '2020-09-08T12:00:12.12345678+00:00')",
702702
"1.212345678e10"
703703
);
704704
test_expression!(
705-
"date_part('second', to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
705+
"date_part('second', timestamp '2020-09-08T12:00:12.12345678+00:00')",
706706
"12.12345678"
707707
);
708708
test_expression!(
709-
"date_part('millisecond', to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
709+
"date_part('millisecond', timestamp '2020-09-08T12:00:12.12345678+00:00')",
710710
"12123.45678"
711711
);
712712
test_expression!(
713-
"date_part('microsecond', to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
713+
"date_part('microsecond', timestamp '2020-09-08T12:00:12.12345678+00:00')",
714714
"12123456.78"
715715
);
716716
test_expression!(
717-
"date_part('nanosecond', to_timestamp('2020-09-08T12:00:12.12345678+00:00'))",
717+
"date_part('nanosecond', timestamp '2020-09-08T12:00:12.12345678+00:00')",
718718
"1.212345678e10"
719719
);
720720

‎datafusion/core/tests/sql/timestamp.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -742,7 +742,7 @@ async fn test_arrow_typeof() -> Result<()> {
742742
"+-----------------------------------------------------------------------+",
743743
"| arrow_typeof(date_trunc(Utf8(\"microsecond\"),to_timestamp(Int64(61)))) |",
744744
"+-----------------------------------------------------------------------+",
745-
"| Timestamp(Nanosecond, None) |",
745+
"| Timestamp(Second, None) |",
746746
"+-----------------------------------------------------------------------+",
747747
];
748748
assert_batches_eq!(expected, &actual);

‎datafusion/expr/src/built_in_function.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,8 @@ pub enum BuiltinScalarFunction {
266266
ToTimestampMillis,
267267
/// to_timestamp_micros
268268
ToTimestampMicros,
269+
/// to_timestamp_nanos
270+
ToTimestampNanos,
269271
/// to_timestamp_seconds
270272
ToTimestampSeconds,
271273
/// from_unixtime
@@ -444,6 +446,7 @@ impl BuiltinScalarFunction {
444446
BuiltinScalarFunction::ToTimestamp => Volatility::Immutable,
445447
BuiltinScalarFunction::ToTimestampMillis => Volatility::Immutable,
446448
BuiltinScalarFunction::ToTimestampMicros => Volatility::Immutable,
449+
BuiltinScalarFunction::ToTimestampNanos => Volatility::Immutable,
447450
BuiltinScalarFunction::ToTimestampSeconds => Volatility::Immutable,
448451
BuiltinScalarFunction::Translate => Volatility::Immutable,
449452
BuiltinScalarFunction::Trim => Volatility::Immutable,
@@ -755,6 +758,7 @@ impl BuiltinScalarFunction {
755758
BuiltinScalarFunction::ToTimestamp => Ok(Timestamp(Nanosecond, None)),
756759
BuiltinScalarFunction::ToTimestampMillis => Ok(Timestamp(Millisecond, None)),
757760
BuiltinScalarFunction::ToTimestampMicros => Ok(Timestamp(Microsecond, None)),
761+
BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)),
758762
BuiltinScalarFunction::ToTimestampSeconds => Ok(Timestamp(Second, None)),
759763
BuiltinScalarFunction::FromUnixtime => Ok(Timestamp(Second, None)),
760764
BuiltinScalarFunction::Now => {
@@ -995,6 +999,18 @@ impl BuiltinScalarFunction {
995999
],
9961000
self.volatility(),
9971001
),
1002+
BuiltinScalarFunction::ToTimestampNanos => Signature::uniform(
1003+
1,
1004+
vec![
1005+
Int64,
1006+
Timestamp(Nanosecond, None),
1007+
Timestamp(Microsecond, None),
1008+
Timestamp(Millisecond, None),
1009+
Timestamp(Second, None),
1010+
Utf8,
1011+
],
1012+
self.volatility(),
1013+
),
9981014
BuiltinScalarFunction::ToTimestampSeconds => Signature::uniform(
9991015
1,
10001016
vec![
@@ -1431,6 +1447,7 @@ fn aliases(func: &BuiltinScalarFunction) -> &'static [&'static str] {
14311447
BuiltinScalarFunction::ToTimestampMillis => &["to_timestamp_millis"],
14321448
BuiltinScalarFunction::ToTimestampMicros => &["to_timestamp_micros"],
14331449
BuiltinScalarFunction::ToTimestampSeconds => &["to_timestamp_seconds"],
1450+
BuiltinScalarFunction::ToTimestampNanos => &["to_timestamp_nanos"],
14341451
BuiltinScalarFunction::FromUnixtime => &["from_unixtime"],
14351452

14361453
// hashing functions

‎datafusion/expr/src/expr_fn.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,12 @@ scalar_expr!(
834834
date,
835835
"converts a string to a `Timestamp(Microseconds, None)`"
836836
);
837+
scalar_expr!(
838+
ToTimestampNanos,
839+
to_timestamp_nanos,
840+
date,
841+
"converts a string to a `Timestamp(Nanoseconds, None)`"
842+
);
837843
scalar_expr!(
838844
ToTimestampSeconds,
839845
to_timestamp_seconds,

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1501,7 +1501,7 @@ mod tests {
15011501
test_evaluate(expr, lit("foobarbaz"));
15021502

15031503
// Check non string arguments
1504-
// to_timestamp("2020-09-08T12:00:00+00:00") --> timestamp(1599566400000000000i64)
1504+
// to_timestamp("2020-09-08T12:00:00+00:00") --> timestamp(1599566400i64)
15051505
let expr =
15061506
call_fn("to_timestamp", vec![lit("2020-09-08T12:00:00+00:00")]).unwrap();
15071507
test_evaluate(expr, lit_timestamp_nano(1599566400000000000i64));

‎datafusion/physical-expr/src/datetime_expressions.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,15 @@ pub fn to_timestamp_micros(args: &[ColumnarValue]) -> Result<ColumnarValue> {
154154
)
155155
}
156156

157+
/// to_timestamp_nanos SQL function
158+
pub fn to_timestamp_nanos(args: &[ColumnarValue]) -> Result<ColumnarValue> {
159+
handle::<TimestampNanosecondType, _, TimestampNanosecondType>(
160+
args,
161+
string_to_timestamp_nanos_shim,
162+
"to_timestamp_nanos",
163+
)
164+
}
165+
157166
/// to_timestamp_seconds SQL function
158167
pub fn to_timestamp_seconds(args: &[ColumnarValue]) -> Result<ColumnarValue> {
159168
handle::<TimestampSecondType, _, TimestampSecondType>(
@@ -962,7 +971,7 @@ mod tests {
962971
let mut string_builder = StringBuilder::with_capacity(2, 1024);
963972
let mut ts_builder = TimestampNanosecondArray::builder(2);
964973

965-
string_builder.append_value("2020-09-08T13:42:29.190855Z");
974+
string_builder.append_value("2020-09-08T13:42:29.190855");
966975
ts_builder.append_value(1599572549190855000);
967976

968977
string_builder.append_null();

‎datafusion/physical-expr/src/functions.rs

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,20 @@ pub fn create_physical_expr(
7474
// so we don't have to pay a per-array/batch cost.
7575
BuiltinScalarFunction::ToTimestamp => {
7676
Arc::new(match input_phy_exprs[0].data_type(input_schema) {
77-
Ok(DataType::Int64) | Ok(DataType::Timestamp(_, None)) => {
78-
|col_values: &[ColumnarValue]| {
79-
cast_column(
80-
&col_values[0],
81-
&DataType::Timestamp(TimeUnit::Nanosecond, None),
82-
None,
83-
)
84-
}
85-
}
77+
Ok(DataType::Int64) => |col_values: &[ColumnarValue]| {
78+
cast_column(
79+
&col_values[0],
80+
&DataType::Timestamp(TimeUnit::Second, None),
81+
None,
82+
)
83+
},
84+
Ok(DataType::Timestamp(_, None)) => |col_values: &[ColumnarValue]| {
85+
cast_column(
86+
&col_values[0],
87+
&DataType::Timestamp(TimeUnit::Nanosecond, None),
88+
None,
89+
)
90+
},
8691
Ok(DataType::Utf8) => datetime_expressions::to_timestamp,
8792
other => {
8893
return internal_err!(
@@ -129,6 +134,25 @@ pub fn create_physical_expr(
129134
}
130135
})
131136
}
137+
BuiltinScalarFunction::ToTimestampNanos => {
138+
Arc::new(match input_phy_exprs[0].data_type(input_schema) {
139+
Ok(DataType::Int64) | Ok(DataType::Timestamp(_, None)) => {
140+
|col_values: &[ColumnarValue]| {
141+
cast_column(
142+
&col_values[0],
143+
&DataType::Timestamp(TimeUnit::Nanosecond, None),
144+
None,
145+
)
146+
}
147+
}
148+
Ok(DataType::Utf8) => datetime_expressions::to_timestamp_nanos,
149+
other => {
150+
return internal_err!(
151+
"Unsupported data type {other:?} for function to_timestamp_nanos"
152+
);
153+
}
154+
})
155+
}
132156
BuiltinScalarFunction::ToTimestampSeconds => Arc::new({
133157
match input_phy_exprs[0].data_type(input_schema) {
134158
Ok(DataType::Int64) | Ok(DataType::Timestamp(_, None)) => {

‎datafusion/proto/proto/datafusion.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,7 @@ enum ScalarFunction {
620620
ArrayEmpty = 115;
621621
ArrayPopBack = 116;
622622
StringToArray = 117;
623+
ToTimestampNanos = 118;
623624
}
624625

625626
message ScalarFunctionNode {

‎datafusion/proto/src/generated/pbjson.rs

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/proto/src/generated/prost.rs

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/proto/src/logical_plan/from_proto.rs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ use datafusion_expr::{
5454
random, regexp_match, regexp_replace, repeat, replace, reverse, right, round, rpad,
5555
rtrim, sha224, sha256, sha384, sha512, signum, sin, sinh, split_part, sqrt,
5656
starts_with, strpos, substr, substring, tan, tanh, to_hex, to_timestamp_micros,
57-
to_timestamp_millis, to_timestamp_seconds, translate, trim, trunc, upper, uuid,
57+
to_timestamp_millis, to_timestamp_nanos, to_timestamp_seconds, translate, trim,
58+
trunc, upper, uuid,
5859
window_frame::regularize,
5960
AggregateFunction, Between, BinaryExpr, BuiltInWindowFunction, BuiltinScalarFunction,
6061
Case, Cast, Expr, GetFieldAccess, GetIndexedField, GroupingSet,
@@ -521,6 +522,7 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction {
521522
ScalarFunction::Substr => Self::Substr,
522523
ScalarFunction::ToHex => Self::ToHex,
523524
ScalarFunction::ToTimestampMicros => Self::ToTimestampMicros,
525+
ScalarFunction::ToTimestampNanos => Self::ToTimestampNanos,
524526
ScalarFunction::ToTimestampSeconds => Self::ToTimestampSeconds,
525527
ScalarFunction::Now => Self::Now,
526528
ScalarFunction::CurrentDate => Self::CurrentDate,
@@ -1592,6 +1594,9 @@ pub fn parse_expr(
15921594
ScalarFunction::ToTimestampMicros => {
15931595
Ok(to_timestamp_micros(parse_expr(&args[0], registry)?))
15941596
}
1597+
ScalarFunction::ToTimestampNanos => {
1598+
Ok(to_timestamp_nanos(parse_expr(&args[0], registry)?))
1599+
}
15951600
ScalarFunction::ToTimestampSeconds => {
15961601
Ok(to_timestamp_seconds(parse_expr(&args[0], registry)?))
15971602
}

‎datafusion/proto/src/logical_plan/to_proto.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1522,6 +1522,7 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction {
15221522
BuiltinScalarFunction::Substr => Self::Substr,
15231523
BuiltinScalarFunction::ToHex => Self::ToHex,
15241524
BuiltinScalarFunction::ToTimestampMicros => Self::ToTimestampMicros,
1525+
BuiltinScalarFunction::ToTimestampNanos => Self::ToTimestampNanos,
15251526
BuiltinScalarFunction::ToTimestampSeconds => Self::ToTimestampSeconds,
15261527
BuiltinScalarFunction::Now => Self::Now,
15271528
BuiltinScalarFunction::CurrentDate => Self::CurrentDate,

‎datafusion/sqllogictest/test_files/timestamps.slt

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ SELECT to_timestamp_micros(ts) FROM ts_data_secs LIMIT 3
217217

218218
# to nanos
219219
query P
220-
SELECT to_timestamp(ts) FROM ts_data_secs LIMIT 3
220+
SELECT to_timestamp_nanos(ts) FROM ts_data_secs LIMIT 3
221221
----
222222
2020-09-08T13:42:29
223223
2020-09-08T12:42:29
@@ -244,7 +244,7 @@ SELECT to_timestamp_seconds(ts) FROM ts_data_micros LIMIT 3
244244
2020-09-08T11:42:29
245245

246246

247-
# Original column is micros, convert to nanos and check timestamp
247+
# Original column is micros, convert to seconds and check timestamp
248248

249249
query P
250250
SELECT to_timestamp(ts) FROM ts_data_micros LIMIT 3
@@ -266,7 +266,7 @@ SELECT from_unixtime(ts / 1000000000) FROM ts_data LIMIT 3;
266266
# to_timestamp
267267

268268
query I
269-
SELECT COUNT(*) FROM ts_data_nanos where ts > to_timestamp('2020-09-08T12:00:00+00:00')
269+
SELECT COUNT(*) FROM ts_data_nanos where ts > timestamp '2020-09-08T12:00:00+00:00'
270270
----
271271
2
272272

@@ -375,15 +375,14 @@ set datafusion.optimizer.skip_failed_rules = true
375375
query P
376376
select to_timestamp(a) from (select to_timestamp(1) as a) A;
377377
----
378-
1970-01-01T00:00:00.000000001
378+
1970-01-01T00:00:01
379379

380380
# cast_to_timestamp_seconds_twice
381381
query P
382382
select to_timestamp_seconds(a) from (select to_timestamp_seconds(1) as a)A
383383
----
384384
1970-01-01T00:00:01
385385

386-
387386
# cast_to_timestamp_millis_twice
388387
query P
389388
select to_timestamp_millis(a) from (select to_timestamp_millis(1) as a)A;
@@ -396,18 +395,30 @@ select to_timestamp_micros(a) from (select to_timestamp_micros(1) as a)A;
396395
----
397396
1970-01-01T00:00:00.000001
398397

398+
# cast_to_timestamp_nanos_twice
399+
query P
400+
select to_timestamp_nanos(a) from (select to_timestamp_nanos(1) as a)A;
401+
----
402+
1970-01-01T00:00:00.000000001
403+
399404
# to_timestamp_i32
400405
query P
401406
select to_timestamp(cast (1 as int));
402407
----
403-
1970-01-01T00:00:00.000000001
408+
1970-01-01T00:00:01
404409

405410
# to_timestamp_micros_i32
406411
query P
407412
select to_timestamp_micros(cast (1 as int));
408413
----
409414
1970-01-01T00:00:00.000001
410415

416+
# to_timestamp_nanos_i32
417+
query P
418+
select to_timestamp_nanos(cast (1 as int));
419+
----
420+
1970-01-01T00:00:00.000000001
421+
411422
# to_timestamp_millis_i32
412423
query P
413424
select to_timestamp_millis(cast (1 as int));
@@ -1776,3 +1787,9 @@ query B
17761787
SELECT TIMESTAMPTZ '2020-01-01 00:00:00Z' = TIMESTAMP '2020-01-01'
17771788
----
17781789
true
1790+
1791+
# verify to_timestamp edge cases to be in sync with postgresql
1792+
query PPPPP
1793+
SELECT to_timestamp(null), to_timestamp(-62125747200), to_timestamp(0), to_timestamp(1926632005177), to_timestamp(1926632005)
1794+
----
1795+
NULL 0001-04-25T00:00:00 1970-01-01T00:00:00 +63022-07-16T12:59:37 2031-01-19T23:33:25

0 commit comments

Comments
 (0)