mozilla
/
bigquery-etl
зеркало из https://github.com/mozilla/bigquery-etl.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

CI fixes for supporting private UDFs in bigquery-etl - DENG-735 (#3631) 

* Minimize stub normalize_search_engine UDF and usage in search_clients_last_seen tests
* Move sql tests downstream of private-generate-sql and copy UDFs into sql-dir for tests
This commit is contained in:
Alexander 2023-03-09 16:54:42 -05:00 коммит произвёл GitHub
Родитель ba6a281eef
Коммит edcfe758f7
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 15 добавлений и 12 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

11
.circleci/config.yml
Просмотреть файл

@ -361,6 +361,11 @@ jobs:
- *java_deps
- *attach_generated_sql
- *copy_generated_sql
- run:
name: Install rsync
command: |
apt update
apt install -y rsync
- add_ssh_keys:
fingerprints: "22:b9:3c:1b:82:ab:3f:e4:b5:79:70:d1:7b:b9:28:d2"
- run:
@ -378,6 +383,11 @@ jobs:
PATHS="$(git diff --no-index --name-only --diff-filter=d generated-sql/sql sql)" || true
echo $PATHS
mkdir private-sql
find /tmp/workspace/private-generated-sql/sql -type d -name 'udf' -exec cp -r --parents '{}' private-sql \;
rsync --archive private-sql/tmp/workspace/private-generated-sql/sql/ sql/
PATH="venv/bin:$PATH" script/bqetl stage deploy \
--dataset-suffix=$CIRCLE_SHA1 \
--remove-updated-artifacts \
@ -633,6 +643,7 @@ workflows:
- deploy-changes-to-stage:
requires:
- generate-sql
- private-generate-sql
- verify-requirements
- test-sql:
context: data-eng-circleci-tests

8
sql/moz-fx-data-shared-prod/udf/normalize_search_engine/udf.sql
Просмотреть файл

@ -8,20 +8,12 @@ CREATE OR REPLACE FUNCTION udf.normalize_search_engine(engine STRING) AS (
CASE
WHEN engine IS NULL
THEN NULL
WHEN STARTS_WITH(LOWER(engine), 'engine1')
THEN 'Engine1'
WHEN STARTS_WITH(LOWER(engine), 'engine2')
THEN 'Engine2'
WHEN STARTS_WITH(LOWER(engine), 'engine3')
THEN 'Engine3'
ELSE 'Other'
END
);
-- Test
SELECT
assert.equals('Engine1', udf.normalize_search_engine('engine1')),
assert.equals('Engine2', udf.normalize_search_engine('Engine2-abc')),
assert.equals('Other', udf.normalize_search_engine('not-Engine1')),
assert.equals('Other', udf.normalize_search_engine('engine')),
assert.null(udf.normalize_search_engine(NULL))

6
tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_last_seen_v1/test_single_day/expect.ndjson
Просмотреть файл

@ -1,3 +1,3 @@
{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine2", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}, {"key": "Engine3", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}

2
tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_last_seen_v1/test_single_day/search_clients_last_seen_v1.ndjson
Просмотреть файл

@ -1,2 +1,2 @@
{"submission_date":"2019-10-01", "client_id": "client2", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 2}
{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}