CI fixes for supporting private UDFs in bigquery-etl - DENG-735 (#3631)

* Minimize stub normalize_search_engine UDF and usage in search_clients_last_seen tests
* Move sql tests downstream of private-generate-sql and copy UDFs into sql-dir for tests

.circleci/config.yml

@@ -361,6 +361,11 @@ jobs:
       - *java_deps
       - *attach_generated_sql
       - *copy_generated_sql
+      - run:
+          name: Install rsync
+          command: |
+            apt update
+            apt install -y rsync
       - add_ssh_keys:
           fingerprints: "22:b9:3c:1b:82:ab:3f:e4:b5:79:70:d1:7b:b9:28:d2"
       - run:
@@ -378,6 +383,11 @@ jobs:
 
             PATHS="$(git diff --no-index --name-only --diff-filter=d generated-sql/sql sql)" || true
             echo $PATHS
+
+            mkdir private-sql
+            find /tmp/workspace/private-generated-sql/sql -type d -name 'udf' -exec cp -r --parents '{}' private-sql \;
+            rsync --archive private-sql/tmp/workspace/private-generated-sql/sql/ sql/
+
             PATH="venv/bin:$PATH" script/bqetl stage deploy \
               --dataset-suffix=$CIRCLE_SHA1 \
               --remove-updated-artifacts \
@@ -633,6 +643,7 @@ workflows:
       - deploy-changes-to-stage:
           requires:
             - generate-sql
+            - private-generate-sql
       - verify-requirements
       - test-sql:
           context: data-eng-circleci-tests

sql/moz-fx-data-shared-prod/udf/normalize_search_engine/udf.sql

@@ -8,20 +8,12 @@ CREATE OR REPLACE FUNCTION udf.normalize_search_engine(engine STRING) AS (
   CASE
     WHEN engine IS NULL
       THEN NULL
-    WHEN STARTS_WITH(LOWER(engine), 'engine1')
-      THEN 'Engine1'
-    WHEN STARTS_WITH(LOWER(engine), 'engine2')
-      THEN 'Engine2'
-    WHEN STARTS_WITH(LOWER(engine), 'engine3')
-      THEN 'Engine3'
     ELSE 'Other'
   END
 );
 
 -- Test
 SELECT
-  assert.equals('Engine1', udf.normalize_search_engine('engine1')),
-  assert.equals('Engine2', udf.normalize_search_engine('Engine2-abc')),
   assert.equals('Other', udf.normalize_search_engine('not-Engine1')),
   assert.equals('Other', udf.normalize_search_engine('engine')),
   assert.null(udf.normalize_search_engine(NULL))

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_last_seen_v1/test_single_day/expect.ndjson

@@ -1,3 +1,3 @@
-{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
+{"submission_date":"2019-10-02","client_id":"client1","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "RE", "user_pref_browser_search_region": "RE", "app_version": "69.0.1", "default_search_engine": "other-Yahoo Web", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 0, "active_addons_count_mean": 1, "max_concurrent_tab_count_max": 1, "tab_open_event_count_sum": 2, "active_hours_sum": 2, "subsession_hours_sum": 2, "sessions_started_on_this_day": 2, "total_searches":6, "tagged_searches": 0, "ad_click": 1, "search_with_ads": 0, "organic": 0, "sap": 6, "profile_creation_date": 18171, "os": "Windows_NT", "os_version": "1", "channel": "release"}
-{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine2", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}, {"key": "Engine3", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
+{"submission_date":"2019-10-02","client_id":"client2","days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAQ==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQ==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "sample_id": 84, "country": "EE", "user_pref_browser_search_region": "EE", "app_version": "69.0", "default_search_engine": "other", "default_search_engine_data_load_path": "c0ffee", "locale": "en-US", "profile_age_in_days": 2, "active_addons_count_mean": 2, "max_concurrent_tab_count_max": 2, "tab_open_event_count_sum": 4, "active_hours_sum": 4, "subsession_hours_sum": 4, "sessions_started_on_this_day": 4, "total_searches": 4, "tagged_searches": 2, "ad_click": 0, "search_with_ads": 0, "organic": 0, "sap": 2, "tagged_sap": 2, "profile_creation_date": 17764, "os":"Linux", "default_search_engine_data_submission_url": "https://duckduckgo.com/html/", "os_version": "2", "channel": "release"}
-{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
+{"submission_date":"2019-10-02", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}

tests/sql/moz-fx-data-shared-prod/search_derived/search_clients_last_seen_v1/test_single_day/search_clients_last_seen_v1.ndjson

@@ -1,2 +1,2 @@
 {"submission_date":"2019-10-01", "client_id": "client2", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 2}
-{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Engine1", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}
+{"submission_date":"2019-10-01", "client_id": "client3", "days_seen_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAA==", "days_tagged_searched_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_searched_with_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_clicked_ads_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "days_created_profile_bytes": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", "engine_searches": [{"key": "Other", "value": {"total_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0], "tagged_searches": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "search_with_ads": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ad_click": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}}], "ad_click": 0, "sample_id": 84 , "search_with_ads": 0, "tagged_searches": 0, "total_searches": 0, "profile_age_in_days": 3}