2649 строки
93 KiB
Python
2649 строки
93 KiB
Python
import json
|
|
import os
|
|
from datetime import date, timedelta
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import yaml
|
|
from click.testing import CliRunner
|
|
from google.api_core.exceptions import NotFound
|
|
|
|
from bigquery_etl.backfill.parse import (
|
|
BACKFILL_FILE,
|
|
DEFAULT_REASON,
|
|
DEFAULT_WATCHER,
|
|
Backfill,
|
|
BackfillStatus,
|
|
)
|
|
from bigquery_etl.backfill.utils import (
|
|
BACKFILL_DESTINATION_DATASET,
|
|
BACKFILL_DESTINATION_PROJECT,
|
|
get_backfill_backup_table_name,
|
|
get_backfill_file_from_qualified_table_name,
|
|
get_backfill_staging_qualified_table_name,
|
|
get_entries_from_qualified_table_name,
|
|
get_qualified_table_name_to_entries_map_by_project,
|
|
qualified_table_name_matching,
|
|
validate_metadata_workgroups,
|
|
)
|
|
from bigquery_etl.cli.backfill import (
|
|
complete,
|
|
create,
|
|
info,
|
|
initiate,
|
|
scheduled,
|
|
validate,
|
|
)
|
|
from bigquery_etl.cli.stage import QUERY_FILE
|
|
from bigquery_etl.deploy import FailedDeployException
|
|
|
|
DEFAULT_STATUS = BackfillStatus.INITIATE
|
|
VALID_REASON = "test_reason"
|
|
VALID_WATCHER = "test@example.org"
|
|
VALID_BACKFILL = Backfill(
|
|
date(2021, 5, 4),
|
|
date(2021, 1, 3),
|
|
date(2021, 5, 3),
|
|
[date(2021, 2, 3)],
|
|
VALID_REASON,
|
|
[VALID_WATCHER],
|
|
DEFAULT_STATUS,
|
|
)
|
|
BACKFILL_YAML_TEMPLATE = (
|
|
"2021-05-04:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" excluded_dates:\n"
|
|
" - 2021-02-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
|
|
VALID_WORKGROUP_ACCESS = [
|
|
dict(
|
|
role="roles/bigquery.dataViewer",
|
|
members=["workgroup:mozilla-confidential"],
|
|
)
|
|
]
|
|
|
|
TABLE_METADATA_CONF = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"owners": ["test@example.org"],
|
|
"workgroup_access": VALID_WORKGROUP_ACCESS,
|
|
}
|
|
|
|
TABLE_METADATA_CONF_EMPTY_WORKGROUP = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"owners": ["test@example.org"],
|
|
"workgroup_access": [],
|
|
}
|
|
|
|
TABLE_METADATA_CONF_DEPENDS_ON_PAST = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"owners": ["test@example.org"],
|
|
"workgroup_access": VALID_WORKGROUP_ACCESS,
|
|
"scheduling": {"depends_on_past": True},
|
|
}
|
|
|
|
DATASET_METADATA_CONF = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"dataset_base_acl": "derived",
|
|
"workgroup_access": VALID_WORKGROUP_ACCESS,
|
|
}
|
|
|
|
DATASET_METADATA_CONF_EMPTY_WORKGROUP = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"dataset_base_acl": "derived",
|
|
"workgroup_access": [],
|
|
"default_table_workgroup_access": VALID_WORKGROUP_ACCESS,
|
|
}
|
|
|
|
PARTITIONED_TABLE_METADATA = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"owners": ["test@example.org"],
|
|
"workgroup_access": VALID_WORKGROUP_ACCESS,
|
|
"bigquery": {
|
|
"time_partitioning": {
|
|
"type": "day",
|
|
"field": "submission_date",
|
|
"require_partition_filter": True,
|
|
}
|
|
},
|
|
}
|
|
|
|
DEFAULT_BILLING_PROJECT = "moz-fx-data-backfill-slots"
|
|
VALID_BILLING_PROJECT = "moz-fx-data-backfill-1"
|
|
INVALID_BILLING_PROJECT = "mozdata"
|
|
|
|
|
|
class TestBackfill:
|
|
@pytest.fixture
|
|
def runner(self):
|
|
return CliRunner()
|
|
|
|
def test_create_backfill(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfill_file = SQL_DIR + "/" + BACKFILL_FILE
|
|
backfill = Backfill.entries_from_file(backfill_file)[0]
|
|
|
|
assert backfill.entry_date == date.today()
|
|
assert backfill.start_date == date(2021, 3, 1)
|
|
assert backfill.end_date == date.today()
|
|
assert backfill.watchers == [DEFAULT_WATCHER]
|
|
assert backfill.reason == DEFAULT_REASON
|
|
assert backfill.status == DEFAULT_STATUS
|
|
assert backfill.billing_project is None
|
|
|
|
def test_create_backfill_with_billing_project(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
f"--billing_project={VALID_BILLING_PROJECT}",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfill_file = SQL_DIR + "/" + BACKFILL_FILE
|
|
backfill = Backfill.entries_from_file(backfill_file)[0]
|
|
|
|
assert backfill.entry_date == date.today()
|
|
assert backfill.start_date == date(2021, 3, 1)
|
|
assert backfill.end_date == date.today()
|
|
assert backfill.watchers == [DEFAULT_WATCHER]
|
|
assert backfill.reason == DEFAULT_REASON
|
|
assert backfill.status == DEFAULT_STATUS
|
|
assert backfill.billing_project == VALID_BILLING_PROJECT
|
|
|
|
def test_create_backfill_with_invalid_billing_project_should_fail(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
f"--billing_project={INVALID_BILLING_PROJECT}",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
assert "Invalid billing project" in str(result.exception)
|
|
|
|
def test_create_backfill_depends_on_past_should_fail(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF_DEPENDS_ON_PAST))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
assert (
|
|
"Tables that depend on past are currently not supported."
|
|
in result.output
|
|
)
|
|
|
|
def test_create_backfill_with_invalid_watcher(self, runner):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
invalid_watcher = "test.org"
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
"--watcher=" + invalid_watcher,
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid" in str(result.exception)
|
|
assert "watchers" in str(result.exception)
|
|
|
|
def test_create_backfill_with_invalid_path(self, runner):
|
|
with runner.isolated_filesystem():
|
|
invalid_path = "test.test_query_v1"
|
|
result = runner.invoke(create, [invalid_path, "--start_date=2021-03-01"])
|
|
assert result.exit_code == 2
|
|
assert "Invalid" in result.output
|
|
assert "path" in result.output
|
|
|
|
def test_create_backfill_with_invalid_start_date_greater_than_end_date(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
invalid_start_date = "2021-05-01"
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=" + invalid_start_date,
|
|
"--end_date=2021-03-01",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid start date" in str(result.exception)
|
|
|
|
def test_create_backfill_with_invalid_excluded_dates_before_start_date(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
invalid_exclude_date = "2021-03-01"
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-05-01",
|
|
"--exclude=" + invalid_exclude_date,
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid excluded dates" in str(result.exception)
|
|
|
|
def test_create_backfill_with_excluded_dates_after_end_date(self, runner):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
invalid_exclude_date = "2021-07-01"
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-05-01",
|
|
"--end_date=2021-06-01",
|
|
"--exclude=" + invalid_exclude_date,
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid excluded dates" in str(result.exception)
|
|
|
|
def test_create_backfill_entry_with_all_params(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2021-03-01",
|
|
"--end_date=2021-03-10",
|
|
"--exclude=2021-03-05",
|
|
"--watcher=test@example.org",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfill_file = SQL_DIR + "/" + BACKFILL_FILE
|
|
backfill = Backfill.entries_from_file(backfill_file)[0]
|
|
|
|
assert backfill.start_date == date(2021, 3, 1)
|
|
assert backfill.end_date == date(2021, 3, 10)
|
|
assert backfill.watchers == [VALID_WATCHER]
|
|
assert backfill.reason == DEFAULT_REASON
|
|
assert backfill.status == DEFAULT_STATUS
|
|
|
|
def test_create_backfill_with_exsting_entry(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_entry_1 = Backfill(
|
|
date(2021, 5, 3),
|
|
date(2021, 1, 3),
|
|
date(2021, 5, 3),
|
|
[date(2021, 2, 3)],
|
|
VALID_REASON,
|
|
[VALID_WATCHER],
|
|
BackfillStatus.COMPLETE,
|
|
)
|
|
|
|
backfill_entry_2 = Backfill(
|
|
date.today(),
|
|
date(2023, 3, 1),
|
|
date(2023, 3, 10),
|
|
[],
|
|
DEFAULT_REASON,
|
|
[DEFAULT_WATCHER],
|
|
DEFAULT_STATUS,
|
|
)
|
|
|
|
backfill_file = (
|
|
Path("sql/moz-fx-data-shared-prod/test/test_query_v1") / BACKFILL_FILE
|
|
)
|
|
backfill_file.write_text(backfill_entry_1.to_yaml())
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfills = Backfill.entries_from_file(backfill_file)
|
|
assert backfills[0] == backfill_entry_1
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2023-03-01",
|
|
"--end_date=2023-03-10",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
|
|
backfills = Backfill.entries_from_file(backfill_file)
|
|
|
|
assert backfills[1] == backfill_entry_1
|
|
assert backfills[0] == backfill_entry_2
|
|
|
|
def test_create_backfill_with_existing_entry_with_initiate_status_should_fail(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_entry_1 = Backfill(
|
|
date(2021, 5, 3),
|
|
date(2021, 1, 3),
|
|
date(2021, 5, 3),
|
|
[date(2021, 2, 3)],
|
|
VALID_REASON,
|
|
[VALID_WATCHER],
|
|
DEFAULT_STATUS,
|
|
)
|
|
|
|
backfill_file = (
|
|
Path("sql/moz-fx-data-shared-prod/test/test_query_v1") / BACKFILL_FILE
|
|
)
|
|
backfill_file.write_text(backfill_entry_1.to_yaml())
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfills = Backfill.entries_from_file(backfill_file)
|
|
assert backfills[0] == backfill_entry_1
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--start_date=2023-03-01",
|
|
"--end_date=2023-03-10",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
|
|
assert (
|
|
"Backfill entries cannot contain more than one entry with Initiate status"
|
|
in str(result.exception)
|
|
)
|
|
|
|
def test_validate_backfill(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(BACKFILL_YAML_TEMPLATE)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
def test_validate_backfill_with_billing_project(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_text = (
|
|
BACKFILL_YAML_TEMPLATE + f" billing_project: {VALID_BILLING_PROJECT}"
|
|
)
|
|
backfill_file.write_text(backfill_text)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
def test_validate_backfill_with_invalid_billing_project_should_fail(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_text = (
|
|
BACKFILL_YAML_TEMPLATE + f" billing_project: {INVALID_BILLING_PROJECT}"
|
|
)
|
|
backfill_file.write_text(backfill_text)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid billing project" in str(result.exception)
|
|
|
|
def test_validate_backfill_depends_on_past_should_fail(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF_DEPENDS_ON_PAST))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(BACKFILL_YAML_TEMPLATE)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert (
|
|
"Tables that depend on past are currently not supported"
|
|
in result.output
|
|
)
|
|
|
|
def test_validate_backfill_invalid_table_name(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert (
|
|
"Qualified table name must be named like: <project>.<dataset>.<table>"
|
|
in str(result.exception)
|
|
)
|
|
|
|
def test_validate_backfill_non_existing_table_name(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v2",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "does not exist" in result.output
|
|
|
|
def test_validate_backfill_invalid_default_reason(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
VALID_REASON, DEFAULT_REASON
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Default reason" in result.output
|
|
|
|
def test_validate_backfill_empty_reason(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_reason = ""
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
VALID_REASON, invalid_reason
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert (
|
|
"Reason in backfill entry should not be empty"
|
|
in result.exception.args[0]
|
|
)
|
|
|
|
def test_validate_backfill_invalid_watcher(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_watcher = "test@example"
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
VALID_WATCHER, invalid_watcher
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid" in str(result.exception)
|
|
assert "watchers" in str(result.exception)
|
|
|
|
def test_validate_backfill_empty_watcher(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_watcher = ""
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
VALID_WATCHER, invalid_watcher
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid" in str(result.exception)
|
|
assert "watchers" in str(result.exception)
|
|
|
|
def test_validate_backfill_watchers_duplicated(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_watchers = " - test@example.org\n" " - test@example.org\n"
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
" - " + VALID_WATCHER, invalid_watchers
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Duplicate watcher" in result.exception.args[0]
|
|
|
|
def test_validate_backfill_invalid_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_status = "INVALIDSTATUS"
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
DEFAULT_STATUS.value, invalid_status
|
|
)
|
|
backfill_file.write_text(invalid_backfill)
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert invalid_status in str(result.exception)
|
|
|
|
def test_validate_backfill_duplicate_entry_dates(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
|
|
duplicate_entry_date = "2021-05-05"
|
|
invalid_backfill = BACKFILL_YAML_TEMPLATE.replace(
|
|
"2021-05-04", duplicate_entry_date
|
|
)
|
|
backfill_file.write_text(invalid_backfill + "\n" + invalid_backfill)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Backfill entry already exists" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_entry_date(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_entry_date = (date.today() + timedelta(days=1)).strftime("%Y-%m-%d")
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-05-04", invalid_entry_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "can't be in the future" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_start_date_greater_than_end_date(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_start_date = "2021-05-04"
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-01-03", invalid_start_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid start date" in str(result.exception)
|
|
|
|
#
|
|
def test_validate_backfill_invalid_start_date_greater_than_entry_date(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_start_date = "2021-05-05"
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-01-03", invalid_start_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid start date" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_end_date_greater_than_entry_date(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_end_date = "2021-05-05"
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-05-03", invalid_end_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid end date" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_excluded_dates_less_than_start_date(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_excluded_date = "2021-01-02"
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-02-03", invalid_excluded_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid excluded dates" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_excluded_dates_greater_than_end_date(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
invalid_excluded_date = "2021-05-04"
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE.replace("2021-02-03", invalid_excluded_date)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Invalid excluded dates" in str(result.exception)
|
|
|
|
def test_validate_backfill_invalid_excluded_dates_duplicated(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
duplicate_excluded_dates = " - 2021-02-03\n" " - 2021-02-03\n"
|
|
backfill_file.write_text(
|
|
(
|
|
"2021-05-04:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" excluded_dates:\n"
|
|
+ duplicate_excluded_dates
|
|
+ " reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "duplicate excluded dates" in result.exception.args[0]
|
|
|
|
def test_validate_backfill_invalid_excluded_dates_not_sorted(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
duplicate_excluded_dates = " - 2021-02-04\n" " - 2021-02-03\n"
|
|
backfill_file.write_text(
|
|
"2021-05-04:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" excluded_dates:\n"
|
|
+ duplicate_excluded_dates
|
|
+ " reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "excluded dates not sorted" in result.exception.args[0]
|
|
|
|
def test_validate_backfill_entries_not_sorted(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2023-05-04:\n"
|
|
" start_date: 2020-01-03\n"
|
|
" end_date: 2020-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Complete\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
validate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "entries are not sorted" in result.output
|
|
|
|
def test_backfill_info_one_table_all_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
qualified_table_name_1 = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Complete\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v2"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
qualified_table_name_2 = "moz-fx-data-shared-prod.test.test_query_v2"
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
qualified_table_name_2,
|
|
"--start_date=2021-04-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
info,
|
|
[qualified_table_name_1],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert qualified_table_name_1 in result.output
|
|
assert BackfillStatus.INITIATE.value in result.output
|
|
assert "total of 2 backfill(s)" in result.output
|
|
assert qualified_table_name_2 not in result.output
|
|
assert BackfillStatus.COMPLETE.value in result.output
|
|
|
|
def test_backfill_info_one_table_initiate_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
info,
|
|
[qualified_table_name, "--status=Initiate"],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert qualified_table_name in result.output
|
|
assert BackfillStatus.INITIATE.value in result.output
|
|
assert "total of 2 backfill(s)" in result.output
|
|
assert BackfillStatus.COMPLETE.value not in result.output
|
|
|
|
def test_backfill_info_all_tables_all_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
qualified_table_name_1 = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Complete\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v2"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
qualified_table_name_2 = "moz-fx-data-shared-prod.test.test_query_v2"
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
qualified_table_name_2,
|
|
"--start_date=2021-04-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
info,
|
|
[],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert qualified_table_name_1 in result.output
|
|
assert qualified_table_name_2 in result.output
|
|
assert BackfillStatus.INITIATE.value in result.output
|
|
assert "total of 3 backfill(s)" in result.output
|
|
assert BackfillStatus.COMPLETE.value in result.output
|
|
|
|
def test_backfill_info_all_tables_with_initiate_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
qualified_table_name_1 = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v2"
|
|
qualified_table_name_2 = "moz-fx-data-shared-prod.test.test_query_v2"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Initiate\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
info,
|
|
[
|
|
"--status=Initiate",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert qualified_table_name_1 in result.output
|
|
assert qualified_table_name_2 in result.output
|
|
assert BackfillStatus.INITIATE.value in result.output
|
|
assert "total of 4 backfill(s)" in result.output
|
|
assert BackfillStatus.COMPLETE.value not in result.output
|
|
|
|
def test_backfill_info_with_invalid_path(self, runner):
|
|
with runner.isolated_filesystem():
|
|
invalid_path = "moz-fx-data-shared-prod.test.test_query_v2"
|
|
result = runner.invoke(info, [invalid_path])
|
|
|
|
assert result.exit_code == 2
|
|
assert "Invalid" in result.output
|
|
assert "path" in result.output
|
|
|
|
def test_backfill_info_with_invalid_qualified_table_name(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
invalid_qualified_table_name = "mozdata.test_query_v2"
|
|
result = runner.invoke(info, [invalid_qualified_table_name])
|
|
|
|
assert result.exit_code == 1
|
|
assert "Qualified table name must be named like:" in str(result.exception)
|
|
|
|
def test_backfill_info_one_table_invalid_status(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE + "\n"
|
|
"2021-05-03:\n"
|
|
" start_date: 2021-01-03\n"
|
|
" end_date: 2021-05-03\n"
|
|
" reason: test_reason\n"
|
|
" watchers:\n"
|
|
" - test@example.org\n"
|
|
" status: Complete\n"
|
|
)
|
|
|
|
assert BACKFILL_FILE in os.listdir(SQL_DIR)
|
|
|
|
result = runner.invoke(
|
|
info,
|
|
[qualified_table_name, "--status=testing"],
|
|
)
|
|
|
|
assert result.exit_code == 2
|
|
assert "Invalid value for '--status'" in result.output
|
|
|
|
def test_get_entries_from_qualified_table_name(self, runner):
|
|
with runner.isolated_filesystem():
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
qualified_table_name,
|
|
"--start_date=2021-03-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
backfill_file = SQL_DIR + "/" + BACKFILL_FILE
|
|
backfill = Backfill.entries_from_file(backfill_file)[0]
|
|
|
|
assert backfill.entry_date == date.today()
|
|
assert backfill.start_date == date(2021, 3, 1)
|
|
assert backfill.end_date == date.today()
|
|
assert backfill.watchers == [DEFAULT_WATCHER]
|
|
assert backfill.reason == DEFAULT_REASON
|
|
assert backfill.status == DEFAULT_STATUS
|
|
|
|
backfills = get_entries_from_qualified_table_name(
|
|
"sql", qualified_table_name
|
|
)
|
|
|
|
expected_backfill = Backfill(
|
|
date.today(),
|
|
date(2021, 3, 1),
|
|
date.today(),
|
|
[],
|
|
DEFAULT_REASON,
|
|
[DEFAULT_WATCHER],
|
|
DEFAULT_STATUS,
|
|
)
|
|
|
|
assert backfills == [expected_backfill]
|
|
|
|
def test_get_qualified_table_name_to_entries_map_by_project(self, runner):
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
qualified_table_name_1 = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
qualified_table_name_1,
|
|
"--start_date=2021-03-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name_2 = "moz-fx-data-shared-prod.test.test_query_v2"
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v2"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = runner.invoke(
|
|
create,
|
|
[
|
|
qualified_table_name_2,
|
|
"--start_date=2021-03-01",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert BACKFILL_FILE in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v2"
|
|
)
|
|
|
|
backfills_dict = get_qualified_table_name_to_entries_map_by_project(
|
|
"sql", "moz-fx-data-shared-prod"
|
|
)
|
|
|
|
expected_backfill = Backfill(
|
|
date.today(),
|
|
date(2021, 3, 1),
|
|
date.today(),
|
|
[],
|
|
DEFAULT_REASON,
|
|
[DEFAULT_WATCHER],
|
|
DEFAULT_STATUS,
|
|
)
|
|
|
|
assert qualified_table_name_1 in backfills_dict
|
|
assert backfills_dict[qualified_table_name_1] == [expected_backfill]
|
|
assert qualified_table_name_2 in backfills_dict
|
|
|
|
def test_get_backfill_file_from_qualified_table_name(self, runner):
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
actual_backfill_file = get_backfill_file_from_qualified_table_name(
|
|
"sql", qualified_table_name
|
|
)
|
|
expected_backfill_file = Path(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/backfill.yaml"
|
|
)
|
|
|
|
assert actual_backfill_file == expected_backfill_file
|
|
|
|
def test_get_backfill_staging_qualified_table_name(self, runner):
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
backfill_table_id = "test__test_query_v1_2023_05_30"
|
|
|
|
actual_backfill_staging = get_backfill_staging_qualified_table_name(
|
|
qualified_table_name, date.fromisoformat("2023-05-30")
|
|
)
|
|
expected_backfill_staging = f"{BACKFILL_DESTINATION_PROJECT}.{BACKFILL_DESTINATION_DATASET}.{backfill_table_id}"
|
|
|
|
assert actual_backfill_staging == expected_backfill_staging
|
|
|
|
def test_get_backfill_backup_table_name(self, runner):
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
cloned_table_id = "test__test_query_v1_backup_2023_05_30"
|
|
|
|
actual_backfill_staging = get_backfill_backup_table_name(
|
|
qualified_table_name, date.fromisoformat("2023-05-30")
|
|
)
|
|
expected_backfill_backup = f"{BACKFILL_DESTINATION_PROJECT}.{BACKFILL_DESTINATION_DATASET}.{cloned_table_id}"
|
|
|
|
assert actual_backfill_staging == expected_backfill_backup
|
|
|
|
def test_validate_metadata_workgroups_invalid_table_workgroup_and_valid_dataset_workgroup(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
invalid_workgroup_access = [
|
|
dict(
|
|
role="roles/bigquery.dataViewer",
|
|
members=["workgroup:invalid_workgroup"],
|
|
)
|
|
]
|
|
|
|
metadata_conf = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"owners": ["test@example.org"],
|
|
"workgroup_access": invalid_workgroup_access,
|
|
}
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(metadata_conf))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert result
|
|
|
|
def test_validate_metadata_workgroups_empty_table_workgroup_and_valid_dataset_workgroups(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert result
|
|
|
|
def test_validate_metadata_workgroups_valid_table_workgroup_and_invalid_dataset_workgroup(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
invalid_workgroup_access = [
|
|
dict(
|
|
role="roles/bigquery.dataViewer",
|
|
members=["workgroup:invalid_workgroup"],
|
|
)
|
|
]
|
|
|
|
dataset_metadata_conf = {
|
|
"friendly_name": "test",
|
|
"description": "test",
|
|
"dataset_base_acl": "derived",
|
|
"workgroup_access": invalid_workgroup_access,
|
|
}
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(dataset_metadata_conf))
|
|
|
|
assert "dataset_metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test"
|
|
)
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert result
|
|
|
|
def test_validate_metadata_workgroups_valid_table_workgroup_and_empty_dataset_workgroup(
|
|
self, runner
|
|
):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
assert "dataset_metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test"
|
|
)
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert result
|
|
|
|
def test_validate_metadata_workgroups_missing_table_metadata(self, runner):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
assert "metadata.yaml" not in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
assert "dataset_metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test"
|
|
)
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert result
|
|
|
|
def test_validate_metadata_workgroups_missing_dataset_metadata(self, runner):
|
|
with pytest.raises(ValueError) as e:
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
assert "dataset_metadata.yaml" not in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test"
|
|
)
|
|
|
|
validate_metadata_workgroups("sql", qualified_table_name)
|
|
|
|
assert e.type == ValueError
|
|
|
|
def test_validate_metadata_workgroups_empty_workgroups(self, runner):
|
|
with runner.isolated_filesystem():
|
|
os.makedirs("sql/moz-fx-data-shared-prod/test/test_query_v1")
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
assert "query.sql" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
assert "metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
assert "dataset_metadata.yaml" in os.listdir(
|
|
"sql/moz-fx-data-shared-prod/test"
|
|
)
|
|
|
|
result = validate_metadata_workgroups("sql", qualified_table_name)
|
|
assert not result
|
|
|
|
def test_qualified_table_name_matching(self, runner):
|
|
qualified_table_name = "moz-fx-data-shared-prod.test.test_query_v1"
|
|
project_id, dataset_id, table_id = qualified_table_name_matching(
|
|
qualified_table_name
|
|
)
|
|
|
|
assert project_id == "moz-fx-data-shared-prod"
|
|
assert dataset_id == "test"
|
|
assert table_id == "test_query_v1"
|
|
|
|
def test_qualified_table_name_matching_invalid_name(self, runner):
|
|
with pytest.raises(AttributeError) as e:
|
|
invalid_name = "test.test_query_v1"
|
|
qualified_table_name_matching(invalid_name)
|
|
|
|
assert "Qualified table name must be named like" in str(e.value)
|
|
|
|
@patch("google.cloud.bigquery.Client.get_table")
|
|
def test_backfill_scheduled(self, get_table, runner):
|
|
get_table.side_effect = [
|
|
None, # Check that staging data exists
|
|
NotFound( # Check that clone does not exist
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test_query_v1_backup_2021_05_03"
|
|
"not found"
|
|
),
|
|
NotFound( # Check that staging data does not exist
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test_query_v1_2021_05_04"
|
|
"not found"
|
|
),
|
|
]
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE
|
|
+ """
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-05-03
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Complete"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
scheduled,
|
|
[
|
|
"--json_path=tmp.json",
|
|
"--status=Complete",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert "1 backfill(s) require processing." in result.output
|
|
assert Path("tmp.json").exists()
|
|
assert len(json.loads(Path("tmp.json").read_text())) == 1
|
|
|
|
result = runner.invoke(
|
|
scheduled,
|
|
[
|
|
"--json_path=tmp.json",
|
|
"--status=Initiate",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert "1 backfill(s) require processing." in result.output
|
|
|
|
@patch("google.cloud.bigquery.Client.get_table")
|
|
def test_backfill_scheduled_depends_on_past_should_fail(self, get_table, runner):
|
|
get_table.side_effect = [
|
|
None, # Check that staging data exists
|
|
NotFound( # Check that clone does not exist
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test_query_v1_backup_2021_05_03"
|
|
"not found"
|
|
),
|
|
NotFound( # Check that staging data does not exist
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test_query_v1_2021_05_04"
|
|
"not found"
|
|
),
|
|
]
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(TABLE_METADATA_CONF_DEPENDS_ON_PAST))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
BACKFILL_YAML_TEMPLATE
|
|
+ """
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-05-03
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Complete"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
scheduled,
|
|
[
|
|
"--json_path=tmp.json",
|
|
"--status=Complete",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert "0 backfill(s) require processing." in result.output
|
|
assert Path("tmp.json").exists()
|
|
assert len(json.loads(Path("tmp.json").read_text())) == 0
|
|
|
|
result = runner.invoke(
|
|
scheduled,
|
|
[
|
|
"--json_path=tmp.json",
|
|
"--status=Initiate",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert "0 backfill(s) require processing." in result.output
|
|
|
|
@patch("google.cloud.bigquery.Client.get_table")
|
|
@patch("google.cloud.bigquery.Client.copy_table")
|
|
@patch("google.cloud.bigquery.Client.delete_table")
|
|
def test_complete_partitioned_backfill(
|
|
self, delete_table, copy_table, get_table, runner
|
|
):
|
|
get_table.side_effect = [
|
|
None, # Check that staging data exists
|
|
NotFound( # Check that clone does not exist
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1_backup_2021_05_03"
|
|
"not found"
|
|
),
|
|
]
|
|
copy_table.side_effect = None
|
|
delete_table.side_effect = None
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-13
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Complete"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
complete,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
assert copy_table.call_count == 12 # one for backup, 11 for partitions
|
|
for i, call in enumerate(copy_table.call_args_list[1:]):
|
|
d = date(2021, 1, 3) + timedelta(days=i)
|
|
assert call.args == (
|
|
f'moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1_2021_05_03${d.strftime("%Y%m%d")}',
|
|
f'moz-fx-data-shared-prod.test.test_query_v1${d.strftime("%Y%m%d")}',
|
|
)
|
|
assert delete_table.call_count == 1
|
|
|
|
@patch("google.cloud.bigquery.Client")
|
|
@patch("subprocess.check_call")
|
|
@patch("bigquery_etl.cli.backfill.deploy_table")
|
|
@patch("bigquery_etl.cli.backfill.Schema.from_query_file")
|
|
def test_initiate_partitioned_backfill(
|
|
self, mock_from_query_file, mock_deploy_table, check_call, mock_client, runner
|
|
):
|
|
backfill_staging_table_name = (
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1"
|
|
)
|
|
|
|
mock_client().get_table.side_effect = [
|
|
NotFound( # Check that staging data does not exist
|
|
f"{backfill_staging_table_name}_backup_2021_05_03" "not found"
|
|
),
|
|
None, # Check that production data exists during dry run
|
|
None, # Check that production data exists
|
|
]
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
query_path = Path(SQL_DIR) / QUERY_FILE
|
|
|
|
with open(query_path, "w") as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/schema.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump({"fields": [{"name": "f0_", "type": "INTEGER"}]}))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-08
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Initiate"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
initiate,
|
|
["moz-fx-data-shared-prod.test.test_query_v1", "--parallelism=0"],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
|
|
assert not mock_from_query_file.called # schema exists
|
|
|
|
mock_deploy_table.assert_called_with(
|
|
query_file=query_path,
|
|
destination_table=f"{backfill_staging_table_name}_2021_05_03",
|
|
respect_dryrun_skip=False,
|
|
)
|
|
|
|
expected_submission_date_params = [
|
|
f"--parameter=submission_date:DATE:2021-01-0{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
expected_destination_table_params = [
|
|
f"--destination_table=moz-fx-data-shared-prod:backfills_staging_derived.test__test_query_v1_2021_05_03$2021010{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
# this is inspecting calls to the underlying subprocess.check_call(["bq]"...)
|
|
assert check_call.call_count == 12 # 6 for dry run, 6 for backfill
|
|
for call in check_call.call_args_list:
|
|
submission_date_params = [
|
|
arg for arg in call.args[0] if "--parameter=submission_date" in arg
|
|
]
|
|
assert len(submission_date_params) == 1
|
|
assert submission_date_params[0] in expected_submission_date_params
|
|
assert f"--project_id={DEFAULT_BILLING_PROJECT}" in call.args[0]
|
|
destination_table_params = [
|
|
arg for arg in call.args[0] if "--destination_table" in arg
|
|
]
|
|
assert destination_table_params[0] in expected_destination_table_params
|
|
|
|
@patch("google.cloud.bigquery.Client")
|
|
@patch("subprocess.check_call")
|
|
@patch("bigquery_etl.cli.backfill.deploy_table")
|
|
@patch("bigquery_etl.cli.backfill.Schema.from_query_file")
|
|
def test_initiate_partitioned_backfill_without_schema_should_pass(
|
|
self, mock_from_query_file, mock_deploy_table, check_call, mock_client, runner
|
|
):
|
|
backfill_staging_table_name = (
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1"
|
|
)
|
|
|
|
mock_client().get_table.side_effect = [
|
|
NotFound( # Check that staging data does not exist
|
|
f"{backfill_staging_table_name}_backup_2021_05_03" "not found"
|
|
),
|
|
None, # Check that production data exists during dry run
|
|
None, # Check that production data exists
|
|
]
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
query_path = Path(SQL_DIR) / QUERY_FILE
|
|
|
|
with open(query_path, "w") as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-08
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Initiate"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
initiate,
|
|
["moz-fx-data-shared-prod.test.test_query_v1", "--parallelism=0"],
|
|
)
|
|
|
|
assert result.exit_code == 0
|
|
|
|
mock_from_query_file.assert_called_with(
|
|
query_file=query_path,
|
|
respect_skip=False,
|
|
sql_dir="sql/",
|
|
)
|
|
|
|
mock_deploy_table.assert_called_with(
|
|
query_file=query_path,
|
|
destination_table=f"{backfill_staging_table_name}_2021_05_03",
|
|
respect_dryrun_skip=False,
|
|
)
|
|
|
|
expected_submission_date_params = [
|
|
f"--parameter=submission_date:DATE:2021-01-0{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
expected_destination_table_params = [
|
|
f"--destination_table=moz-fx-data-shared-prod:backfills_staging_derived.test__test_query_v1_2021_05_03$2021010{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
# this is inspecting calls to the underlying subprocess.check_call(["bq]"...)
|
|
assert check_call.call_count == 12 # 6 for dry run, 6 for backfill
|
|
for call in check_call.call_args_list:
|
|
submission_date_params = [
|
|
arg for arg in call.args[0] if "--parameter=submission_date" in arg
|
|
]
|
|
assert len(submission_date_params) == 1
|
|
assert submission_date_params[0] in expected_submission_date_params
|
|
assert f"--project_id={DEFAULT_BILLING_PROJECT}" in call.args[0]
|
|
destination_table_params = [
|
|
arg for arg in call.args[0] if "--destination_table" in arg
|
|
]
|
|
assert destination_table_params[0] in expected_destination_table_params
|
|
|
|
@patch("google.cloud.bigquery.Client")
|
|
@patch("subprocess.check_call")
|
|
@patch("bigquery_etl.cli.backfill.deploy_table")
|
|
@patch("bigquery_etl.cli.backfill.Schema.from_query_file")
|
|
def test_initiate_partitioned_backfill_with_valid_billing_project_from_entry(
|
|
self, mock_from_query_file, mock_deploy_table, check_call, mock_client, runner
|
|
):
|
|
backfill_staging_table_name = (
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1"
|
|
)
|
|
|
|
mock_client().get_table.side_effect = [
|
|
NotFound( # Check that staging data does not exist
|
|
f"{backfill_staging_table_name}_backup_2021_05_03" "not found"
|
|
),
|
|
None, # Check that production data exists during dry run
|
|
None, # Check that production data exists
|
|
]
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
query_path = Path(SQL_DIR) / QUERY_FILE
|
|
|
|
with open(query_path, "w") as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/schema.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump({"fields": [{"name": "f0_", "type": "INTEGER"}]}))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
f"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-08
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Initiate
|
|
billing_project: {VALID_BILLING_PROJECT}
|
|
"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
initiate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--parallelism=0",
|
|
],
|
|
)
|
|
|
|
assert not mock_from_query_file.called # schema exists
|
|
|
|
mock_deploy_table.assert_called_with(
|
|
query_file=query_path,
|
|
destination_table=f"{backfill_staging_table_name}_2021_05_03",
|
|
respect_dryrun_skip=False,
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
expected_submission_date_params = [
|
|
f"--parameter=submission_date:DATE:2021-01-0{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
expected_destination_table_params = [
|
|
f"--destination_table=moz-fx-data-shared-prod:backfills_staging_derived.test__test_query_v1_2021_05_03$2021010{day}"
|
|
for day in range(3, 9)
|
|
]
|
|
|
|
# this is inspecting calls to the underlying subprocess.check_call(["bq]"...)
|
|
assert check_call.call_count == 12 # 6 for dry run, 6 for backfill
|
|
for call in check_call.call_args_list:
|
|
submission_date_params = [
|
|
arg for arg in call.args[0] if "--parameter=submission_date" in arg
|
|
]
|
|
assert len(submission_date_params) == 1
|
|
assert submission_date_params[0] in expected_submission_date_params
|
|
assert f"--project_id={VALID_BILLING_PROJECT}" in call.args[0]
|
|
destination_table_params = [
|
|
arg for arg in call.args[0] if "--destination_table" in arg
|
|
]
|
|
assert destination_table_params[0] in expected_destination_table_params
|
|
|
|
@patch("google.cloud.bigquery.Client")
|
|
@patch("bigquery_etl.cli.backfill.deploy_table")
|
|
@patch("bigquery_etl.cli.backfill.Schema.from_query_file")
|
|
def test_initiate_backfill_with_failed_deploy(
|
|
self, mock_from_query_file, mock_deploy_table, mock_client, runner
|
|
):
|
|
backfill_staging_table_name = (
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1"
|
|
)
|
|
|
|
mock_deploy_table.side_effect = FailedDeployException("Unable to deploy table")
|
|
|
|
mock_client().get_table.side_effect = [
|
|
NotFound( # Check that staging data does not exist
|
|
f"{backfill_staging_table_name}_backup_2021_05_03" "not found"
|
|
),
|
|
None, # Check that production data exists during dry run
|
|
None, # Check that production data exists
|
|
]
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
query_path = Path(SQL_DIR) / QUERY_FILE
|
|
|
|
with open(query_path, "w") as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-08
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Initiate
|
|
"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
initiate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--parallelism=0",
|
|
],
|
|
)
|
|
|
|
mock_from_query_file.assert_called_with(
|
|
query_file=query_path,
|
|
respect_skip=False,
|
|
sql_dir="sql/",
|
|
)
|
|
|
|
mock_deploy_table.assert_called_with(
|
|
query_file=query_path,
|
|
destination_table=f"{backfill_staging_table_name}_2021_05_03",
|
|
respect_dryrun_skip=False,
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "Backfill initiate failed to deploy" in str(result.exception)
|
|
|
|
@patch("google.cloud.bigquery.Client")
|
|
def test_initiate_partitioned_backfill_with_invalid_billing_project_from_entry_should_fail(
|
|
self, mock_client, runner
|
|
):
|
|
backfill_staging_table_name = (
|
|
"moz-fx-data-shared-prod.backfills_staging_derived.test__test_query_v1"
|
|
)
|
|
|
|
mock_client().get_table.side_effect = [
|
|
NotFound( # Check that staging data does not exist
|
|
f"{backfill_staging_table_name}_backup_2021_05_03" "not found"
|
|
),
|
|
None, # Check that production data exists during dry run
|
|
None, # Check that production data exists
|
|
]
|
|
|
|
with runner.isolated_filesystem():
|
|
SQL_DIR = "sql/moz-fx-data-shared-prod/test/test_query_v1"
|
|
os.makedirs(SQL_DIR)
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/query.sql", "w"
|
|
) as f:
|
|
f.write("SELECT 1")
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/test_query_v1/metadata.yaml",
|
|
"w",
|
|
) as f:
|
|
f.write(yaml.dump(PARTITIONED_TABLE_METADATA))
|
|
|
|
with open(
|
|
"sql/moz-fx-data-shared-prod/test/dataset_metadata.yaml", "w"
|
|
) as f:
|
|
f.write(yaml.dump(DATASET_METADATA_CONF_EMPTY_WORKGROUP))
|
|
|
|
backfill_file = Path(SQL_DIR) / BACKFILL_FILE
|
|
backfill_file.write_text(
|
|
f"""
|
|
2021-05-03:
|
|
start_date: 2021-01-03
|
|
end_date: 2021-01-08
|
|
reason: test_reason
|
|
watchers:
|
|
- test@example.org
|
|
status: Initiate
|
|
billing_project: {INVALID_BILLING_PROJECT}
|
|
"""
|
|
)
|
|
|
|
result = runner.invoke(
|
|
initiate,
|
|
[
|
|
"moz-fx-data-shared-prod.test.test_query_v1",
|
|
"--parallelism=0",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
assert "Invalid billing project" in str(result.exception)
|