From f0791d9d3a9b58bede20d606513661ded9c70221 Mon Sep 17 00:00:00 2001 From: qooba Date: Tue, 20 Apr 2021 02:00:04 +0200 Subject: [PATCH 1/5] fix materialize for None Signed-off-by: qooba --- sdk/python/feast/infra/provider.py | 2 +- sdk/python/feast/type_map.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 8d1d6639f76..37710752eb6 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -257,7 +257,7 @@ def _coerce_datetime(ts): feature_dict = {} for feature in feature_view.features: idx = table.column_names.index(feature.name) - value = python_value_to_proto_value(row[idx]) + value = python_value_to_proto_value(row[idx],feature.dtype) feature_dict[feature.name] = value event_timestamp_idx = table.column_names.index( feature_view.input.event_timestamp_column diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 5afb5f2e408..f81b0e665a4 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -313,8 +313,8 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue: raise Exception(f"Unsupported data type: ${str(type(value))}") -def python_value_to_proto_value(value: Any) -> ProtoValue: - value_type = python_type_to_feast_value_type("", value) +def python_value_to_proto_value(value: Any, feature_type: ValueType = None) -> ProtoValue: + value_type = python_type_to_feast_value_type("", value) if value else feature_type return _python_value_to_proto_value(value_type, value) From a0e27dba7a50b8e029c01e8615a7b5a25de7fa4d Mon Sep 17 00:00:00 2001 From: qooba Date: Tue, 20 Apr 2021 02:19:59 +0200 Subject: [PATCH 2/5] fix materialize for None Signed-off-by: qooba --- sdk/python/feast/infra/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 37710752eb6..6cd5fe94019 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -257,7 +257,7 @@ def _coerce_datetime(ts): feature_dict = {} for feature in feature_view.features: idx = table.column_names.index(feature.name) - value = python_value_to_proto_value(row[idx],feature.dtype) + value = python_value_to_proto_value(row[idx], feature.dtype) feature_dict[feature.name] = value event_timestamp_idx = table.column_names.index( feature_view.input.event_timestamp_column From 6ce100575074f85a274749efb60b015548110f17 Mon Sep 17 00:00:00 2001 From: qooba Date: Tue, 20 Apr 2021 22:28:18 +0200 Subject: [PATCH 3/5] fix materialize for None Signed-off-by: qooba --- sdk/python/feast/type_map.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index f81b0e665a4..d755d56b968 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -313,7 +313,9 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue: raise Exception(f"Unsupported data type: ${str(type(value))}") -def python_value_to_proto_value(value: Any, feature_type: ValueType = None) -> ProtoValue: +def python_value_to_proto_value( + value: Any, feature_type: ValueType = None +) -> ProtoValue: value_type = python_type_to_feast_value_type("", value) if value else feature_type return _python_value_to_proto_value(value_type, value) From 5165b564602b4a9b4c356777a47190c26c48777f Mon Sep 17 00:00:00 2001 From: qooba Date: Fri, 23 Apr 2021 02:14:42 +0200 Subject: [PATCH 4/5] fix materialize for None - add test Signed-off-by: qooba --- .../test_offline_online_store_consistency.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 102e9f4fed8..012a6d57a53 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -26,7 +26,7 @@ def create_dataset() -> pd.DataFrame: ts = pd.Timestamp(now).round("ms") data = { "id": [1, 2, 1, 3, 3], - "value": [0.1, 0.2, 0.3, 4, 5], + "value": [0.1, None, 0.3, 4, 5], "ts_1": [ ts - timedelta(hours=4), ts, @@ -153,7 +153,11 @@ def check_offline_and_online_features( response_dict = fs.get_online_features( [f"{fv.name}:value"], [{"driver": driver_id}] ).to_dict() - assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6 + + if expected_value: + assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6 + else: + assert response_dict[f"{fv.name}__value"][0] is None # Check offline store df = fs.get_historical_features( @@ -163,7 +167,11 @@ def check_offline_and_online_features( feature_refs=[f"{fv.name}:value"], ).to_df() - assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6 + if expected_value: + assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6 + else: + df = df.where(pd.notnull(df), None) + assert df.to_dict()[f"{fv.name}__value"][0] is None def run_offline_online_store_consistency_test( @@ -181,6 +189,10 @@ def run_offline_online_store_consistency_test( fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3 ) + check_offline_and_online_features( + fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None + ) + # check prior value for materialize_incremental() check_offline_and_online_features( fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4 From 15c8c4609df2783bc0ff2de2ef57cb5c71b21208 Mon Sep 17 00:00:00 2001 From: qooba Date: Fri, 7 May 2021 23:37:58 +0200 Subject: [PATCH 5/5] Fix materialize - correct lint errors Signed-off-by: qooba --- sdk/python/tests/test_offline_online_store_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py index 012a6d57a53..e0d5a645f4c 100644 --- a/sdk/python/tests/test_offline_online_store_consistency.py +++ b/sdk/python/tests/test_offline_online_store_consistency.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime, timedelta from pathlib import Path -from typing import Iterator, Tuple, Union +from typing import Iterator, Optional, Tuple, Union import pandas as pd import pytest @@ -147,7 +147,7 @@ def check_offline_and_online_features( fv: FeatureView, driver_id: int, event_timestamp: datetime, - expected_value: float, + expected_value: Optional[float], ) -> None: # Check online store response_dict = fs.get_online_features(