Add option to publish metadata to GCS to entrypoint script

bigquery_etl/parse_metadata.py

@@ -55,6 +55,8 @@ class Metadata:
             try:
                 metadata = yaml.safe_load(yaml_stream)
 
+                print(metadata["friendly_name"])
+
                 friendly_name = metadata.get("friendly_name", None)
                 description = metadata.get("description", None)
 

script/entrypoint

@@ -33,6 +33,12 @@ elif [ "$1" = "query" ]; then
     # For queries with public_json set in the metadata, the results will be
     # exported as JSON to Cloud storage.
     python script/publish_public_data_json --query_file="${@: -1}" ${@:1:$#-1}
+elif [ "$1" = "publish_gcs_metadata" ]; then
+    # For an invocation like:
+    #     publish_gcs_metadata [options]
+    # we dispatch to a script that publishes metadata files to GCS for datasets
+    # that have previsously been made public on GCS.
+    exec script/publish_public_data_gcs_metadata ${@:2}
 elif [ "$XCOM_PUSH" = "true" ]; then
     # KubernetesPodOperator will extract the contents of /airflow/xcom/return.json as an xcom
     # if the xcom_push parameter is true

tests/public_data/all_datasets.json

@@ -1 +1 @@
-{"test": {"non_incremental_query": {"v1": {"friendly_name": "Test table for a non-incremental query\n", "description": "Test table for a non-incremental query\n", "incremental": false, "incremental_export": false, "review_link": "https://bugzilla.mozilla.org/show_bug.cgi?id=1999999", "files_uri": "http://test.endpoint.mozilla.com/api/v1/tables/test/non_incremental_query/v1/files"}}, "incremental_query": {"v1": {"friendly_name": "Test table for an incremental query", "description": "Test table for an incremental query\n", "incremental": true, "incremental_export": true, "files_uri": "http://test.endpoint.mozilla.com/api/v1/tables/test/incremental_query/v1/files"}}}}
+{"test": {"non_incremental_query": {"v1": {"friendly_name": "Test table for a non-incremental query", "description": "Test table for a non-incremental query", "incremental": false, "incremental_export": false, "review_link": "https://bugzilla.mozilla.org/show_bug.cgi?id=1999999", "files_uri": "http://test.endpoint.mozilla.com/api/v1/tables/test/non_incremental_query/v1/files"}}, "incremental_query": {"v1": {"friendly_name": "Test table for an incremental query", "description": "Test table for an incremental query", "incremental": true, "incremental_export": true, "files_uri": "http://test.endpoint.mozilla.com/api/v1/tables/test/incremental_query/v1/files"}}}}

tests/public_data/test_publish_gcs_metadata.py

@@ -84,8 +84,8 @@ class TestPublishGcsMetadata(object):
         result = gcs_table_metadata.table_metadata_to_json()
 
         assert len(result.items()) == 6
-        assert result["description"] == "Test table for a non-incremental query\n"
-        assert result["friendly_name"] == "Test table for a non-incremental query\n"
+        assert result["description"] == "Test table for a non-incremental query"
+        assert result["friendly_name"] == "Test table for a non-incremental query"
         assert result["incremental"] == False
         assert result["incremental_export"] == False
         assert (

tests/public_data/test_sql/test/incremental_query_v1/metadata.yaml

@@ -1,6 +1,5 @@
-friendly_name: Test table for an incremental query
-description: >
-  Test table for an incremental query
+friendly_name: "Test table for an incremental query"
+description: "Test table for an incremental query"
 owners:
   - ascholtz@mozilla.com
 labels:

tests/public_data/test_sql/test/non_incremental_query_v1/metadata.yaml

@@ -1,7 +1,5 @@
-friendly_name: >
-  Test table for a non-incremental query
-description: >
-  Test table for a non-incremental query
+friendly_name: "Test table for a non-incremental query"
+description: "Test table for a non-incremental query"
 owners:
   - ascholtz@mozilla.com
 labels: