Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Fix materialize for None #1481

Merged
merged 5 commits into from
May 10, 2021
Merged

Fix materialize for None #1481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f0791d9
fix materialize for None
qooba Apr 20, 2021
a0e27db
fix materialize for None
qooba Apr 20, 2021
6ce1005
fix materialize for None
qooba Apr 20, 2021
5165b56
fix materialize for None - add test
qooba Apr 23, 2021
15c8c46
Fix materialize - correct lint errors
qooba May 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sdk/python/feast/infra/provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ def _coerce_datetime(ts):
feature_dict = {}
for feature in feature_view.features:
idx = table.column_names.index(feature.name)
value = python_value_to_proto_value(row[idx])
value = python_value_to_proto_value(row[idx], feature.dtype)
feature_dict[feature.name] = value
event_timestamp_idx = table.column_names.index(
feature_view.input.event_timestamp_column
Expand Down
6 changes: 4 additions & 2 deletions sdk/python/feast/type_map.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,10 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
raise Exception(f"Unsupported data type: ${str(type(value))}")


def python_value_to_proto_value(value: Any) -> ProtoValue:
value_type = python_type_to_feast_value_type("", value)
def python_value_to_proto_value(
value: Any, feature_type: ValueType = None
) -> ProtoValue:
value_type = python_type_to_feast_value_type("", value) if value else feature_type
return _python_value_to_proto_value(value_type, value)


Expand Down
22 changes: 17 additions & 5 deletions sdk/python/tests/test_offline_online_store_consistency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import uuid
from datetime import datetime, timedelta
from pathlib import Path
from typing import Iterator, Tuple, Union
from typing import Iterator, Optional, Tuple, Union

import pandas as pd
import pytest
Expand All @@ -26,7 +26,7 @@ def create_dataset() -> pd.DataFrame:
ts = pd.Timestamp(now).round("ms")
data = {
"id": [1, 2, 1, 3, 3],
"value": [0.1, 0.2, 0.3, 4, 5],
"value": [0.1, None, 0.3, 4, 5],
"ts_1": [
ts - timedelta(hours=4),
ts,
Expand Down Expand Up @@ -147,13 +147,17 @@ def check_offline_and_online_features(
fv: FeatureView,
driver_id: int,
event_timestamp: datetime,
expected_value: float,
expected_value: Optional[float],
) -> None:
# Check online store
response_dict = fs.get_online_features(
[f"{fv.name}:value"], [{"driver": driver_id}]
).to_dict()
assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6

if expected_value:
assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6
else:
assert response_dict[f"{fv.name}__value"][0] is None

# Check offline store
df = fs.get_historical_features(
Expand All @@ -163,7 +167,11 @@ def check_offline_and_online_features(
feature_refs=[f"{fv.name}:value"],
).to_df()

assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
if expected_value:
assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
else:
df = df.where(pd.notnull(df), None)
assert df.to_dict()[f"{fv.name}__value"][0] is None


def run_offline_online_store_consistency_test(
Expand All @@ -181,6 +189,10 @@ def run_offline_online_store_consistency_test(
fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3
)

check_offline_and_online_features(
fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None
)

# check prior value for materialize_incremental()
check_offline_and_online_features(
fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4
Expand Down