Skip to content

feat: Data Source Api Update #2468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 9 commits into from
Mar 30, 2022
Merged

feat: Data Source Api Update #2468

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
13b2d93
Fix
kevjumba Mar 30, 2022
292eb2f
File source
kevjumba Mar 30, 2022
d694c39
Redshift source
kevjumba Mar 30, 2022
132d60c
Rest of them.
kevjumba Mar 30, 2022
c68d5fd
Fix typo
kevjumba Mar 30, 2022
6bc3b32
Fix
kevjumba Mar 30, 2022
7b6bf49
Fix lint
kevjumba Mar 30, 2022
e4bd319
Revert small change
kevjumba Mar 30, 2022
ce43b41
Address review comments
kevjumba Mar 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Redshift source 
Signed-off-by: Kevin Zhang <kzhang@tecton.ai>
  • Loading branch information
@kevjumba
kevjumba committed Mar 30, 2022
commit d694c399d61b72bc6ca6aecbbe5da96d3a5efd1d
20 changes: 20 additions & 0 deletions sdk/python/feast/infra/offline_stores/redshift_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from re import M
import warnings
from typing import Callable, Dict, Iterable, Optional, Tuple

Expand All @@ -24,6 +25,9 @@ def __init__(
date_partition_column: Optional[str] = "",
query: Optional[str] = None,
name: Optional[str] = None,
description: Optional[str] = "",
tags: Optional[Dict[str, str]] = None,
owner: Optional[str] = "",
):
"""
Creates a RedshiftSource object.
Expand All @@ -40,6 +44,10 @@ def __init__(
date_partition_column (optional): Timestamp column used for partitioning.
query (optional): The query to be executed to obtain the features.
name (optional): Name for the source. Defaults to the table_ref if not specified.
description (optional):: A human-readable description.
tags (optional):: A dictionary of key-value pairs to store arbitrary metadata.
owner (optional):: The owner of the data source, typically the email of the primary
maintainer.
"""
# The default Redshift schema is named "public".
_schema = "public" if table and not schema else schema
Expand Down Expand Up @@ -68,6 +76,9 @@ def __init__(
created_timestamp_column,
field_mapping,
date_partition_column,
description=description,
tags=tags,
owner=owner,
)

@staticmethod
Expand All @@ -89,6 +100,9 @@ def from_proto(data_source: DataSourceProto):
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
query=data_source.redshift_options.query,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)

# Note: Python requires redefining hash in child classes that override __eq__
Expand All @@ -109,6 +123,9 @@ def __eq__(self, other):
and self.event_timestamp_column == other.event_timestamp_column
and self.created_timestamp_column == other.created_timestamp_column
and self.field_mapping == other.field_mapping
and self.description == other.description
and self.tags == other.tags
and self.owner == other.owner
)

@property
Expand Down Expand Up @@ -137,6 +154,9 @@ def to_proto(self) -> DataSourceProto:
type=DataSourceProto.BATCH_REDSHIFT,
field_mapping=self.field_mapping,
redshift_options=self.redshift_options.to_proto(),
description=self.description,
tags=self.tags,
owner=self.owner,
)

data_source_proto.event_timestamp_column = self.event_timestamp_column
Expand Down