diff --git a/bigquery_etl/cli/udf.py b/bigquery_etl/cli/udf.py index 3abfe34434..6a9436dfe9 100644 --- a/bigquery_etl/cli/udf.py +++ b/bigquery_etl/cli/udf.py @@ -21,7 +21,7 @@ from ..util.common import project_dirs UDF_NAME_RE = re.compile(r"^(?P[a-zA-z0-9_]+)\.(?P[a-zA-z0-9_]+)$") UDF_DATASET_RE = re.compile(r"^(?P[a-zA-z0-9_]+)$") UDF_FILE_RE = re.compile(r"(^.*/|^)([a-zA-Z0-9_]+)/([a-zA-Z0-9_]+)/udf\.sql$") -DEFAULT_UDF_DEPENDENCY_DIR = "lib/" +DEFAULT_UDF_DEPENDENCY_DIR = "udf_js_lib/" DEFAULT_GCS_BUCKET = "moz-fx-data-prod-bigquery-etl" DEFAULT_GCS_PATH = "" DEFAULT_PROJECT_ID = "moz-fx-data-shared-prod" diff --git a/bigquery_etl/routine/publish_routines.py b/bigquery_etl/routine/publish_routines.py index 8dd398e6b6..8df9b5a562 100644 --- a/bigquery_etl/routine/publish_routines.py +++ b/bigquery_etl/routine/publish_routines.py @@ -15,9 +15,10 @@ from bigquery_etl.routine.parse_routine import ( accumulate_dependencies, ) -DEFAULT_UDF_DEPENDENCY_DIR = "lib/" +DEFAULT_UDF_DEPENDENCY_DIR = "udf_js_lib/" DEFAULT_GCS_BUCKET = "moz-fx-data-prod-bigquery-etl" DEFAULT_GCS_PATH = "" +DEFAULT_PROJECT = "sql/moz-fx-data-shared-prod" SQL_DIR = "sql/" OPTIONS_LIB_RE = re.compile(r'library = "gs://[^"]+/([^"]+)"') @@ -33,6 +34,12 @@ parser.add_argument( help="Project to publish UDFs to. " "If not set, publish UDFs for all projects except mozfun.", ) +parser.add_argument( + "--target", + default=DEFAULT_PROJECT, + required=False, + help="Path to project directory.", +) parser.add_argument( "--dependency-dir", "--dependency_dir", @@ -63,14 +70,15 @@ def main(): """Publish routine.""" args = parser.parse_args() - if args.project_id is not None: - projects = [args.project_id] + if args.target is not None: + projects = [args.target] else: projects = project_dirs() for project in projects: publish( - os.path.basename(project), + args.target, + args.project_id, os.path.join(SQL_DIR, project, args.dependency_dir), args.gcs_bucket, args.gcs_path, @@ -78,14 +86,16 @@ def main(): ) -def publish(project_id, dependency_dir, gcs_bucket, gcs_path, public): +def publish(target, project_id, dependency_dir, gcs_bucket, gcs_path, public): """Publish routines in the provided directory.""" client = bigquery.Client(project_id) if dependency_dir and os.path.exists(dependency_dir): - push_dependencies_to_gcs(gcs_bucket, gcs_path, dependency_dir, project_id) + push_dependencies_to_gcs( + gcs_bucket, gcs_path, dependency_dir, os.path.basename(target) + ) - raw_routines = read_routine_dir(os.path.join(SQL_DIR, project_id)) + raw_routines = read_routine_dir(target) published_routines = [] diff --git a/script/publish_public_udfs b/script/publish_public_udfs index 6f86cabde1..774f2da6f3 100755 --- a/script/publish_public_udfs +++ b/script/publish_public_udfs @@ -5,4 +5,4 @@ cd "$(dirname "$0")/.." exec python3 -m bigquery_etl.routine.publish_routines --project_id=mozfun \ - --dependency_dir=lib/ --gcs-bucket=mozfun --public=True "$@" + --dependency_dir=lib/ --gcs-bucket=mozfun --target=sql/mozfun --public=True "$@" diff --git a/sql/moz-fx-data-shared-prod/lib/atob.js b/sql/moz-fx-data-shared-prod/udf_js_lib/atob.js similarity index 100% rename from sql/moz-fx-data-shared-prod/lib/atob.js rename to sql/moz-fx-data-shared-prod/udf_js_lib/atob.js diff --git a/sql/moz-fx-data-shared-prod/lib/gunzip.min.js b/sql/moz-fx-data-shared-prod/udf_js_lib/gunzip.min.js similarity index 100% rename from sql/moz-fx-data-shared-prod/lib/gunzip.min.js rename to sql/moz-fx-data-shared-prod/udf_js_lib/gunzip.min.js diff --git a/tests/routine/test_publish_routine.py b/tests/routine/test_publish_routine.py index 09d02715bd..5ebe6213f5 100644 --- a/tests/routine/test_publish_routine.py +++ b/tests/routine/test_publish_routine.py @@ -11,12 +11,12 @@ class TestPublishRoutine: udf_dir = TEST_DIR / "data" / "test_sql" / "moz-fx-data-test-project" / "udf" @mock.patch("google.cloud.bigquery.Client") - def test_publish_udf_with_description(self, mock_client): + def test_publish_routine_with_description(self, mock_client): raw_routine = parse_routine.RawRoutine.from_file( self.udf_dir / "test_shift_28_bits_one_day" / "udf.sql" ) mock_client.query = MagicMock() - publish_routines.publish_udf( + publish_routines.publish_routine( raw_routine, mock_client, "test-project", "", "", [], False ) query = ( @@ -33,7 +33,7 @@ class TestPublishRoutine: self.udf_dir / "test_js_udf" / "udf.sql" ) mock_client.query = MagicMock() - publish_routines.publish_udf( + publish_routines.publish_routine( raw_routine, mock_client, "test-project", "", "", [], False ) query = (