From 9076bd810b33a238cf2ff3c821fd93302516b93b Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 16:20:15 -0700 Subject: [PATCH 01/15] bump feast version to latest --- feast | 2 +- provider/sdk/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/feast b/feast index b1ccf8d..bdecb10 160000 --- a/feast +++ b/feast @@ -1 +1 @@ -Subproject commit b1ccf8dd1535f721aee8bea937ee38feff80bec5 +Subproject commit bdecb10367aad79ec11265b3e15eceeb8b9a040a diff --git a/provider/sdk/setup.py b/provider/sdk/setup.py index 91df5a2..0ed16e6 100644 --- a/provider/sdk/setup.py +++ b/provider/sdk/setup.py @@ -29,7 +29,7 @@ setup( python_requires=">=3.7.0", packages=find_packages(exclude=("tests",)), install_requires=[ - "feast[redis]==0.18.1", + "feast[redis]==0.21.3", "azure-storage-blob>=0.37.0", "azure-identity>=1.6.1", "SQLAlchemy>=1.4.19", From d6a09ce11c7879a2e4f68cd961078c7bbdc139f1 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 16:54:55 -0700 Subject: [PATCH 02/15] update test pipeline --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7362a27..e1ac3c0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,6 +38,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - python -m pip install -e feast/sdk/python[ci] + python -m pip install -e feast[ci] - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests From 1149588f8e6932e0f2bb259166251284458c0bc7 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 17:57:22 -0700 Subject: [PATCH 03/15] update azure provider --- .../feast_azure_provider/azure_provider.py | 53 ++++++++++++++++++- .../sdk/feast_azure_provider/mssqlserver.py | 4 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/provider/sdk/feast_azure_provider/azure_provider.py b/provider/sdk/feast_azure_provider/azure_provider.py index ea4a5f9..09e5560 100644 --- a/provider/sdk/feast_azure_provider/azure_provider.py +++ b/provider/sdk/feast_azure_provider/azure_provider.py @@ -1,14 +1,16 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from datetime import datetime +from datetime import datetime, timedelta from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import pandas import pyarrow as pa from tqdm import tqdm +from feast import FeatureService from feast.entity import Entity +from feast.feature_logging import FeatureServiceLoggingSource from feast.feature_view import FeatureView from feast.infra.offline_stores.offline_store import RetrievalJob from feast.infra.offline_stores.offline_utils import get_offline_store_from_config @@ -24,6 +26,8 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.registry import Registry from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDataset +from feast.usage import RatioSampler, log_exceptions_and_usage, set_usage_attribute +from feast.utils import make_tzaware DEFAULT_BATCH_SIZE = 10_000 @@ -78,6 +82,7 @@ class AzureProvider(Provider): if self.online_store: self.online_store.online_write_batch(config, table, data, progress) + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) def online_read( self, config: RepoConfig, @@ -198,3 +203,49 @@ class AzureProvider(Provider): end_date=make_tzaware(dataset.max_event_timestamp + timedelta(seconds=1)), # type: ignore ) + def write_feature_service_logs( + self, + feature_service: FeatureService, + logs: Union[pa.Table, str], + config: RepoConfig, + registry: Registry, + ): + assert ( + feature_service.logging_config is not None + ), "Logging should be configured for the feature service before calling this function" + + self.offline_store.write_logged_features( + config=config, + data=logs, + source=FeatureServiceLoggingSource(feature_service, config.project), + logging_config=feature_service.logging_config, + registry=registry, + ) + + def retrieve_feature_service_logs( + self, + feature_service: FeatureService, + start_date: datetime, + end_date: datetime, + config: RepoConfig, + registry: Registry, + ) -> RetrievalJob: + assert ( + feature_service.logging_config is not None + ), "Logging should be configured for the feature service before calling this function" + + logging_source = FeatureServiceLoggingSource(feature_service, config.project) + schema = logging_source.get_schema(registry) + logging_config = feature_service.logging_config + ts_column = logging_source.get_log_timestamp_column() + columns = list(set(schema.names) - {ts_column}) + + return self.offline_store.pull_all_from_table_or_query( + config=config, + data_source=logging_config.destination.to_data_source(), + join_key_columns=[], + feature_name_columns=columns, + timestamp_field=ts_column, + start_date=make_tzaware(start_date), + end_date=make_tzaware(end_date), + ) \ No newline at end of file diff --git a/provider/sdk/feast_azure_provider/mssqlserver.py b/provider/sdk/feast_azure_provider/mssqlserver.py index 8fe257b..79b3f04 100644 --- a/provider/sdk/feast_azure_provider/mssqlserver.py +++ b/provider/sdk/feast_azure_provider/mssqlserver.py @@ -26,8 +26,10 @@ from sqlalchemy.orm import Session, sessionmaker from feast import errors from feast.data_source import DataSource + from .mssqlserver_source import MsSqlServerSource from feast.feature_view import FeatureView +from feast.infra.offline_stores.file_source import SavedDatasetFileStorage from feast.infra.offline_stores.offline_store import ( OfflineStore, RetrievalJob, @@ -135,7 +137,7 @@ class MsSqlServerOfflineStore(OfflineStore): == "feast_azure_provider.mssqlserver.MsSqlServerOfflineStore" ) from_expression = data_source.get_table_query_string().replace("`", "") - + timestamps = [event_timestamp_column] field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) query = f""" From 47fe728c0708cd7ff2b7fd0834726bfa87d882b1 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 18:35:00 -0700 Subject: [PATCH 04/15] update submodule branch --- .gitmodules | 1 + feast | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index ef77ae7..8340615 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "feast"] path = feast url = https://github.com/feast-dev/feast.git + branch = v0.21-branch diff --git a/feast b/feast index bdecb10..bdd47d0 160000 --- a/feast +++ b/feast @@ -1 +1 @@ -Subproject commit bdecb10367aad79ec11265b3e15eceeb8b9a040a +Subproject commit bdd47d041c7c8d5f71cb8bcd7fa0c24a45e8eb0f From 978432d474eb839438291c9061a5b94adf23a878 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 19:19:49 -0700 Subject: [PATCH 05/15] sync test pipeline --- .github/workflows/test.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e1ac3c0..85d190e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,6 +18,11 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Setup Go + id: setup-go + uses: actions/setup-go@v2 + with: + go-version: 1.17.7 - name: include submodule run: git submodule update --init # - name: Get pip cache dir @@ -34,10 +39,12 @@ jobs: # key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles('**/setup.py') }} # restore-keys: | # ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Install pip-tools + run: pip install pip-tools - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install db-dtypes - python -m pip install -e feast[ci] + make install feast/install-python-ci-dependencies - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests From f667717757ec954b2e6840658ed175c8533e785c Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 19:22:14 -0700 Subject: [PATCH 06/15] fix test pipeline --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 85d190e..7659465 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,6 +45,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - make install feast/install-python-ci-dependencies + make feast/install-python-ci-dependencies - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests From 859f19b025894247697b385fbfcaa0a75d33a707 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 19:37:09 -0700 Subject: [PATCH 07/15] fix --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7659465..946ce13 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,6 +45,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - make feast/install-python-ci-dependencies + make -f feast/Makefile install-python-ci-dependencies - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests From eb51eabd7c4fe64b0dd0f9e74d614fe25be2ac8d Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 21:33:05 -0700 Subject: [PATCH 08/15] use makefile --- .github/workflows/test.yaml | 4 ++-- Makefile | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 946ce13..97cd8a0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,7 +44,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install db-dtypes - make -f feast/Makefile install-python-ci-dependencies + python -m pip install db-dtypes + make install-python-ci-dependencies - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..23dc5d8 --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +# Python SDK + +install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies + python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + COMPILE_GO=true python feast/setup.py develop + +# Go SDK & embedded + +install-go-proto-dependencies: + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 + +install-go-ci-dependencies: + # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line + # ToDo: there should be a better way to install gopy + go get github.com/go-python/gopy + go install golang.org/x/tools/cmd/goimports@latest + go install github.com/go-python/gopy@latest + python -m pip install pybindgen==0.22.0 \ No newline at end of file From 2e14e6fcbacf932f3cead1d43fdfbe5b47d48f5b Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 22:03:56 -0700 Subject: [PATCH 09/15] fix pipeline failure --- .github/workflows/test.yaml | 1 + Makefile | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 97cd8a0..0b8473f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,5 +46,6 @@ jobs: python -m pip install --upgrade pip python -m pip install db-dtypes make install-python-ci-dependencies + python -m pip install -e feast[ci] - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests diff --git a/Makefile b/Makefile index 23dc5d8..becbf00 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt - COMPILE_GO=true python feast/setup.py develop # Go SDK & embedded From 2276e054b5513d485f5932021f4f49600e54f3eb Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 22:15:24 -0700 Subject: [PATCH 10/15] fix --- .github/workflows/test.yaml | 2 +- Makefile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0b8473f..c9a682f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - make install-python-ci-dependencies + make compile-go-lib python -m pip install -e feast[ci] - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests diff --git a/Makefile b/Makefile index becbf00..a6e5cc3 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ # Python SDK +compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies + COMPILE_GO=True python setup.py build_ext --inplace + install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt From ff59443aeb218691dbbabfa0c053bfab25818908 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 22:18:04 -0700 Subject: [PATCH 11/15] fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a6e5cc3..a8308f1 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # Python SDK compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - COMPILE_GO=True python setup.py build_ext --inplace + COMPILE_GO=True python feast/setup.py build_ext --inplace install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt From 303997cd2e86536ee0a7cda3bdf40b268f737563 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 22:23:07 -0700 Subject: [PATCH 12/15] fix --- .github/workflows/test.yaml | 3 +-- Makefile | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9a682f..97cd8a0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - make compile-go-lib - python -m pip install -e feast[ci] + make install-python-ci-dependencies - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests diff --git a/Makefile b/Makefile index a8308f1..a34453c 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + COMPILE_GO=True python feast/setup.py develop --inplace # Go SDK & embedded From b7fd9978c8148d5adf8ff990e54def46d1f24790 Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 22:55:42 -0700 Subject: [PATCH 13/15] fix --- .github/workflows/test.yaml | 2 ++ Makefile | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 97cd8a0..1f6f1cb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,6 +45,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes + cd ./feast make install-python-ci-dependencies + cd .. - name: Run Feast unit tests run: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 feast/sdk/python/tests diff --git a/Makefile b/Makefile index a34453c..c5756dd 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,8 @@ compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies COMPILE_GO=True python feast/setup.py build_ext --inplace install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies - python -m piptools sync feast/sdk/python/requirements/py$(PYTHON)-ci-requirements.txt - COMPILE_GO=True python feast/setup.py develop --inplace + python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt + COMPILE_GO=True python setup.py develop # Go SDK & embedded From 9af21faf03582a235d4b1adaf99a576a3efb791d Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Wed, 15 Jun 2022 23:08:38 -0700 Subject: [PATCH 14/15] remove unused file --- .github/workflows/test.yaml | 2 +- Makefile | 25 ------------------------- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 Makefile diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1f6f1cb..208b6f3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install db-dtypes - cd ./feast + cd feast make install-python-ci-dependencies cd .. - name: Run Feast unit tests diff --git a/Makefile b/Makefile deleted file mode 100644 index c5756dd..0000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -# Python SDK - -compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - COMPILE_GO=True python feast/setup.py build_ext --inplace - -install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies - python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt - COMPILE_GO=True python setup.py develop - -# Go SDK & embedded - -install-go-proto-dependencies: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 - -install-go-ci-dependencies: - # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line - # ToDo: there should be a better way to install gopy - go get github.com/go-python/gopy - go install golang.org/x/tools/cmd/goimports@latest - go install github.com/go-python/gopy@latest - python -m pip install pybindgen==0.22.0 \ No newline at end of file From 591411d871686bf32193bd22224d7ccacb03f28e Mon Sep 17 00:00:00 2001 From: Runhan Li Date: Thu, 16 Jun 2022 16:39:28 -0700 Subject: [PATCH 15/15] updates to make feast azure compatible with latest feast --- .../sdk/feast_azure_provider/mssqlserver.py | 12 +++---- .../mssqlserver_source.py | 34 ++++++++++++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/provider/sdk/feast_azure_provider/mssqlserver.py b/provider/sdk/feast_azure_provider/mssqlserver.py index 79b3f04..440604f 100644 --- a/provider/sdk/feast_azure_provider/mssqlserver.py +++ b/provider/sdk/feast_azure_provider/mssqlserver.py @@ -416,7 +416,7 @@ def get_feature_view_query_context( join_keys = [] entity_selections = [] reverse_field_mapping = { - v: k for k, v in feature_view.input.field_mapping.items() + v: k for k, v in feature_view.source.field_mapping.items() } for entity_name in feature_view.entities: entity = registry.get_entity(entity_name, project) @@ -431,17 +431,17 @@ def get_feature_view_query_context( else: ttl_seconds = 0 - assert isinstance(feature_view.input, MsSqlServerSource) + assert isinstance(feature_view.source, MsSqlServerSource) - event_timestamp_column = feature_view.input.event_timestamp_column - created_timestamp_column = feature_view.input.created_timestamp_column + event_timestamp_column = feature_view.source.event_timestamp_column + created_timestamp_column = feature_view.source.created_timestamp_column context = FeatureViewQueryContext( name=feature_view.name, ttl=ttl_seconds, entities=join_keys, features=features, - table_ref=feature_view.input.table_ref, + table_ref=feature_view.source.table_ref, event_timestamp_column=reverse_field_mapping.get( event_timestamp_column, event_timestamp_column ), @@ -449,7 +449,7 @@ def get_feature_view_query_context( created_timestamp_column, created_timestamp_column ), # TODO: Make created column optional and not hardcoded - table_subquery=feature_view.input.get_table_query_string().replace("`", ""), + table_subquery=feature_view.source.get_table_query_string().replace("`", ""), entity_selections=entity_selections, ) query_context.append(context) diff --git a/provider/sdk/feast_azure_provider/mssqlserver_source.py b/provider/sdk/feast_azure_provider/mssqlserver_source.py index a9de632..eb5dfa8 100644 --- a/provider/sdk/feast_azure_provider/mssqlserver_source.py +++ b/provider/sdk/feast_azure_provider/mssqlserver_source.py @@ -100,6 +100,10 @@ class MsSqlServerSource(DataSource): field_mapping: Optional[Dict[str, str]] = None, date_partition_column: Optional[str] = "", connection_str: Optional[str] = "", + description: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, + owner: Optional[str] = None, + name: Optional[str] = None ): self._mssqlserver_options = MsSqlServerOptions( connection_str=connection_str, table_ref=table_ref @@ -107,11 +111,15 @@ class MsSqlServerSource(DataSource): self._connection_str = connection_str super().__init__( - event_timestamp_column - or self._infer_event_timestamp_column("TIMESTAMP|DATETIME"), - created_timestamp_column, - field_mapping, - date_partition_column, + created_timestamp_column = created_timestamp_column, + field_mapping = field_mapping, + date_partition_column = date_partition_column, + description = description, + tags = tags, + owner = owner, + name = name, + timestamp_field = event_timestamp_column + or self._infer_event_timestamp_column("TIMESTAMP|DATETIME") ) def __eq__(self, other): @@ -121,12 +129,20 @@ class MsSqlServerSource(DataSource): ) return ( - self.mssqlserver_options.connection_str + self.name == other.name + and self.mssqlserver_options.connection_str == other.mssqlserver_options.connection_str - and self.event_timestamp_column == other.event_timestamp_column + and self.timestamp_field == other.timestamp_field and self.created_timestamp_column == other.created_timestamp_column and self.field_mapping == other.field_mapping ) + + def __hash__(self): + return hash(( + self.name, + self.mssqlserver_options.connection_str, + self.timestamp_field, + self.created_timestamp_column)) @property def table_ref(self): @@ -153,7 +169,7 @@ class MsSqlServerSource(DataSource): field_mapping=dict(data_source.field_mapping), table_ref=options["table_ref"], connection_str=options["connection_string"], - event_timestamp_column=data_source.event_timestamp_column, + event_timestamp_column=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, date_partition_column=data_source.date_partition_column, ) @@ -165,7 +181,7 @@ class MsSqlServerSource(DataSource): custom_options=self.mssqlserver_options.to_proto(), ) - data_source_proto.event_timestamp_column = self.event_timestamp_column + data_source_proto.timestamp_field = self.timestamp_field data_source_proto.created_timestamp_column = self.created_timestamp_column data_source_proto.date_partition_column = self.date_partition_column