Merge branch 'main' into shredder_mitigation_kpi_search
This commit is contained in:
Коммит
87929c9541
|
@ -6,11 +6,11 @@ repos:
|
|||
- id: no-commit-to-branch
|
||||
args: [--branch, main, --branch, master]
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.8.0
|
||||
rev: 24.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/PyCQA/flake8
|
||||
rev: 7.0.0
|
||||
rev: 4.0.1
|
||||
hooks:
|
||||
- id: flake8
|
||||
exclude: dags/
|
||||
|
@ -20,7 +20,7 @@ repos:
|
|||
- id: yamllint
|
||||
args: [-c, .yamllint.yaml, .]
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: 5.13.2
|
||||
rev: 5.12.0
|
||||
hooks:
|
||||
- id: isort
|
||||
# This pre-commit hook has been archived on Nov 2023
|
||||
|
@ -35,9 +35,8 @@ repos:
|
|||
hooks:
|
||||
- id: mypy
|
||||
exclude: sql/.+/.+/.+/query\.py$
|
||||
additional_dependencies: [types-pytz==2024.2.0.20240913,
|
||||
additional_dependencies: [types-pytz==2024.2.0.20241221,
|
||||
types-ujson==5.10.0.20240515,
|
||||
types-python-dateutil==2.9.0.20240906,
|
||||
types-requests==2.32.0.20240914,
|
||||
types-attrs==19.1.0,
|
||||
types-python-dateutil==2.9.0.20241206,
|
||||
types-requests==2.32.0.20241016,
|
||||
types-PyYAML==6.0.12.20240917]
|
||||
|
|
|
@ -10,7 +10,11 @@ import click
|
|||
from dateutil.relativedelta import relativedelta
|
||||
from google.cloud import bigquery
|
||||
|
||||
from bigquery_etl.metadata.parse_metadata import DatasetMetadata, Metadata
|
||||
from bigquery_etl.metadata.parse_metadata import (
|
||||
DatasetMetadata,
|
||||
Metadata,
|
||||
WorkgroupAccessMetadata,
|
||||
)
|
||||
from bigquery_etl.metadata.publish_metadata import publish_metadata
|
||||
|
||||
from ..cli.utils import (
|
||||
|
@ -57,6 +61,9 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
|
|||
table_metadata_files = paths_matching_name_pattern(
|
||||
name, sql_dir, project_id=project_id, files=["metadata.yaml"]
|
||||
)
|
||||
retained_dataset_roles = ConfigLoader.get(
|
||||
"deprecation", "retain_dataset_roles", fallback=[]
|
||||
)
|
||||
|
||||
# create and populate the dataset metadata yaml file if it does not exist
|
||||
for table_metadata_file in table_metadata_files:
|
||||
|
@ -83,14 +90,41 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
|
|||
# this overwrites existing workgroups
|
||||
table_metadata.workgroup_access = []
|
||||
table_metadata_updated = True
|
||||
dataset_metadata.workgroup_access = []
|
||||
dataset_metadata.workgroup_access = [
|
||||
workgroup
|
||||
for workgroup in dataset_metadata.workgroup_access
|
||||
if workgroup.get("role") in retained_dataset_roles
|
||||
]
|
||||
dataset_metadata_updated = True
|
||||
else:
|
||||
if table_metadata.workgroup_access is None:
|
||||
table_metadata.workgroup_access = (
|
||||
dataset_metadata.default_table_workgroup_access
|
||||
)
|
||||
table_metadata_updated = True
|
||||
table_metadata.workgroup_access = []
|
||||
|
||||
for (
|
||||
default_workgroup_access
|
||||
) in dataset_metadata.default_table_workgroup_access:
|
||||
role_exists = False
|
||||
for i, table_workgroup_access in enumerate(
|
||||
table_metadata.workgroup_access
|
||||
):
|
||||
if table_workgroup_access.role == default_workgroup_access.get(
|
||||
"role"
|
||||
):
|
||||
role_exists = True
|
||||
table_metadata.workgroup_access[i].members = sorted(
|
||||
set(table_workgroup_access.members)
|
||||
| set(default_workgroup_access.get("members", []))
|
||||
)
|
||||
table_metadata_updated = True
|
||||
|
||||
if not role_exists:
|
||||
table_metadata.workgroup_access.append(
|
||||
WorkgroupAccessMetadata(
|
||||
role=default_workgroup_access["role"],
|
||||
members=default_workgroup_access.get("members", []),
|
||||
)
|
||||
)
|
||||
table_metadata_updated = True
|
||||
|
||||
if dataset_metadata_updated:
|
||||
dataset_metadata.write(dataset_metadata_path)
|
||||
|
|
|
@ -790,8 +790,8 @@ def run(name, project_id, sql_dir, workspace, base_url, marker):
|
|||
click.echo(
|
||||
f"Trigger metric runs for {project}.{dataset}.{table}: {metric_ids}"
|
||||
)
|
||||
response = client.run_metric_batch(metric_ids=metric_ids)
|
||||
for metric_info in response.metric_infos:
|
||||
response = client.run_metric_batch_async(metric_ids=metric_ids)
|
||||
for metric_info in response:
|
||||
latest_metric_run = metric_info.latest_metric_runs[-1]
|
||||
if (
|
||||
latest_metric_run
|
||||
|
|
|
@ -38,6 +38,21 @@ except ImportError:
|
|||
from backports.cached_property import cached_property # type: ignore
|
||||
|
||||
|
||||
QUERY_PARAMETER_TYPE_VALUES = {
|
||||
"DATE": "2019-01-01",
|
||||
"DATETIME": "2019-01-01 00:00:00",
|
||||
"TIMESTAMP": "2019-01-01 00:00:00",
|
||||
"STRING": "foo",
|
||||
"BOOL": True,
|
||||
"FLOAT64": 1,
|
||||
"FLOAT": 1,
|
||||
"INT64": 1,
|
||||
"INTEGER": 1,
|
||||
"NUMERIC": 1,
|
||||
"BIGNUMERIC": 1,
|
||||
}
|
||||
|
||||
|
||||
def get_credentials(auth_req: Optional[GoogleAuthRequest] = None):
|
||||
"""Get GCP credentials."""
|
||||
auth_req = auth_req or GoogleAuthRequest()
|
||||
|
@ -204,28 +219,30 @@ class DryRun:
|
|||
sql = self.content
|
||||
else:
|
||||
sql = self.get_sql()
|
||||
if self.metadata:
|
||||
# use metadata to rewrite date-type query params as submission_date
|
||||
date_params = [
|
||||
query_param
|
||||
for query_param in (
|
||||
self.metadata.scheduling.get("date_partition_parameter"),
|
||||
*(
|
||||
param.split(":", 1)[0]
|
||||
for param in self.metadata.scheduling.get("parameters", [])
|
||||
if re.fullmatch(r"[^:]+:DATE:{{.*ds.*}}", param)
|
||||
),
|
||||
|
||||
query_parameters = []
|
||||
scheduling_metadata = self.metadata.scheduling if self.metadata else {}
|
||||
if date_partition_parameter := scheduling_metadata.get(
|
||||
"date_partition_parameter", "submission_date"
|
||||
):
|
||||
query_parameters.append(
|
||||
bigquery.ScalarQueryParameter(
|
||||
date_partition_parameter,
|
||||
"DATE",
|
||||
QUERY_PARAMETER_TYPE_VALUES["DATE"],
|
||||
)
|
||||
if query_param and query_param != "submission_date"
|
||||
]
|
||||
if date_params:
|
||||
pattern = re.compile(
|
||||
"@("
|
||||
+ "|".join(date_params)
|
||||
# match whole query parameter names
|
||||
+ ")(?![a-zA-Z0-9_])"
|
||||
)
|
||||
for parameter in scheduling_metadata.get("parameters", []):
|
||||
parameter_name, parameter_type, _ = parameter.strip().split(":", 2)
|
||||
parameter_type = parameter_type.upper() or "STRING"
|
||||
query_parameters.append(
|
||||
bigquery.ScalarQueryParameter(
|
||||
parameter_name,
|
||||
parameter_type,
|
||||
QUERY_PARAMETER_TYPE_VALUES.get(parameter_type),
|
||||
)
|
||||
sql = pattern.sub("@submission_date", sql)
|
||||
)
|
||||
|
||||
project = basename(dirname(dirname(dirname(self.sqlfile))))
|
||||
dataset = basename(dirname(dirname(self.sqlfile)))
|
||||
try:
|
||||
|
@ -234,6 +251,10 @@ class DryRun:
|
|||
"project": self.project or project,
|
||||
"dataset": self.dataset or dataset,
|
||||
"query": sql,
|
||||
"query_parameters": [
|
||||
query_parameter.to_api_repr()
|
||||
for query_parameter in query_parameters
|
||||
],
|
||||
}
|
||||
|
||||
if self.table:
|
||||
|
@ -257,11 +278,7 @@ class DryRun:
|
|||
dry_run=True,
|
||||
use_query_cache=False,
|
||||
default_dataset=f"{project}.{dataset}",
|
||||
query_parameters=[
|
||||
bigquery.ScalarQueryParameter(
|
||||
"submission_date", "DATE", "2019-01-01"
|
||||
)
|
||||
],
|
||||
query_parameters=query_parameters,
|
||||
)
|
||||
job = self.client.query(sql, job_config=job_config)
|
||||
try:
|
||||
|
|
|
@ -8,11 +8,12 @@ from pathlib import Path
|
|||
|
||||
import click
|
||||
import yaml
|
||||
from google.cloud import bigquery
|
||||
|
||||
from bigquery_etl.config import ConfigLoader
|
||||
from bigquery_etl.schema import SCHEMA_FILE, Schema
|
||||
|
||||
from ..util import standard_args
|
||||
from ..util import extract_from_query_path, standard_args
|
||||
from ..util.common import extract_last_group_by_from_query, project_dirs
|
||||
from .parse_metadata import DatasetMetadata, Metadata
|
||||
|
||||
|
@ -106,6 +107,7 @@ def validate_change_control(
|
|||
def validate_shredder_mitigation(query_dir, metadata):
|
||||
"""Check queries with shredder mitigation label comply with requirements."""
|
||||
has_shredder_mitigation = SHREDDER_MITIGATION_LABEL in metadata.labels
|
||||
table_not_empty = True
|
||||
|
||||
if has_shredder_mitigation:
|
||||
schema_file = Path(query_dir) / SCHEMA_FILE
|
||||
|
@ -116,7 +118,44 @@ def validate_shredder_mitigation(query_dir, metadata):
|
|||
query_file = Path(query_dir) / "query.sql"
|
||||
query_group_by = extract_last_group_by_from_query(sql_path=query_file)
|
||||
|
||||
# Validate that the query group by is as required.
|
||||
# Validate that this label is only applied to tables in version 1 if they're not empty
|
||||
# If the table is empty, it should be backfilled before applying the label.
|
||||
project, dataset, table = extract_from_query_path(Path(query_dir))
|
||||
client = bigquery.Client(project=project)
|
||||
error_message = (
|
||||
f"The shredder-mitigation label can only be applied to existing and "
|
||||
f"non-empty tables.\nEnsure that the table `{project}.{dataset}.{table}` is deployed"
|
||||
f" and run a managed backfill without mitigation before applying this label to"
|
||||
f" a new or empty table."
|
||||
f"\n\nSubsequent backfills then can use the [shredder mitigation process]"
|
||||
f"(https://mozilla.github.io/bigquery-etl/cookbooks/creating_a_derived_dataset/#initiating-the-backfill)."
|
||||
)
|
||||
|
||||
# Check that the table exists and it's not empty.
|
||||
query_table_is_not_empty = (
|
||||
f"SELECT EXISTS (SELECT 1 "
|
||||
f"FROM `{project}.{dataset}.INFORMATION_SCHEMA.TABLES` "
|
||||
f"WHERE table_name = '{table}' LIMIT 1) AS not_empty;"
|
||||
)
|
||||
|
||||
try:
|
||||
table_not_empty = client.query(query_table_is_not_empty).result()
|
||||
except Exception:
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Table {project}.{dataset}.{table} not found or inaccessible."
|
||||
f" for validation. Please check that the name is correct and if the table"
|
||||
f" is in a private repository, ensure that it exists and has data before"
|
||||
f" running a backfill with shredder mitigation.",
|
||||
fg="yellow",
|
||||
)
|
||||
)
|
||||
|
||||
if table_not_empty is None or table_not_empty is False:
|
||||
click.echo(click.style(error_message, fg="yellow"))
|
||||
return False
|
||||
|
||||
# Validate that the query group by is explicit and as required.
|
||||
integers_in_group_by = False
|
||||
for e in query_group_by:
|
||||
try:
|
||||
|
@ -213,11 +252,11 @@ def validate_retention_policy_based_on_table_type(metadata, path):
|
|||
else None
|
||||
)
|
||||
|
||||
retention_exclusion_list = set(
|
||||
ConfigLoader.get("retention_exclusion_list", fallback=[])
|
||||
)
|
||||
# retention_exclusion_list = set(
|
||||
# ConfigLoader.get("retention_exclusion_list", fallback=[])
|
||||
# )
|
||||
|
||||
normalized_path = str(Path(path).parent)
|
||||
# normalized_path = str(Path(path).parent)
|
||||
if expiration_days is not None and table_type == "aggregate":
|
||||
click.echo(
|
||||
click.style(
|
||||
|
@ -226,18 +265,19 @@ def validate_retention_policy_based_on_table_type(metadata, path):
|
|||
)
|
||||
)
|
||||
is_valid = False
|
||||
if (
|
||||
expiration_days is None
|
||||
and table_type == "client_level"
|
||||
and normalized_path not in retention_exclusion_list
|
||||
):
|
||||
click.echo(
|
||||
click.style(
|
||||
f"ERROR: Table at {path} is an client level table and needs expiration_days to be set",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
is_valid = False
|
||||
# The below line of code should be uncommented when the retention project is completed
|
||||
# if (
|
||||
# expiration_days is None
|
||||
# and table_type == "client_level"
|
||||
# and normalized_path not in retention_exclusion_list
|
||||
# ):
|
||||
# click.echo(
|
||||
# click.style(
|
||||
# f"ERROR: Table at {path} is an client level table and needs expiration_days to be set",
|
||||
# fg="red",
|
||||
# )
|
||||
# )
|
||||
# is_valid = False
|
||||
return is_valid
|
||||
|
||||
|
||||
|
|
|
@ -622,7 +622,7 @@ class Task:
|
|||
task.is_bigeye_check = True
|
||||
task.depends_on_past = False
|
||||
task.destination_table = None
|
||||
task.retries = 0
|
||||
task.retries = 1
|
||||
task.depends_on_fivetran = []
|
||||
task.referenced_tables = None
|
||||
task.depends_on = []
|
||||
|
|
|
@ -308,6 +308,7 @@ with DAG('{{ name }}', default_args=default_args{%+ if schedule_interval != None
|
|||
{#+ Avoid having concurrent tasks for ETLs that target the whole table. -#}
|
||||
task_concurrency=1,
|
||||
{%+ endif -%}
|
||||
execution_timeout=datetime.timedelta(hours=1),
|
||||
{%+ else -%}
|
||||
{{ task.task_name }} = bigquery_etl_query(
|
||||
{% if name == "bqetl_default" -%}
|
||||
|
|
|
@ -19,6 +19,7 @@ from ..util.bigquery_id import qualified_table_id
|
|||
|
||||
SHARED_PROD = "moz-fx-data-shared-prod"
|
||||
GLEAN_SCHEMA_ID = "glean_ping_1"
|
||||
GLEAN_MIN_SCHEMA_ID = "glean-min_ping_1"
|
||||
GLEAN_APP_LISTINGS_URL = "https://probeinfo.telemetry.mozilla.org/v2/glean/app-listings"
|
||||
|
||||
|
||||
|
@ -91,6 +92,7 @@ DeleteIndex = dict[DeleteTarget, DeleteSource | tuple[DeleteSource, ...]]
|
|||
|
||||
CLIENT_ID = "client_id"
|
||||
GLEAN_CLIENT_ID = "client_info.client_id"
|
||||
GLEAN_USAGE_PROFILE_ID = "metrics.uuid.usage_profile_id"
|
||||
IMPRESSION_ID = "impression_id"
|
||||
USER_ID = "user_id"
|
||||
POCKET_ID = "pocket_id"
|
||||
|
@ -205,9 +207,6 @@ DELETE_TARGETS: DeleteIndex = {
|
|||
client_id_target(
|
||||
table="telemetry_derived.clients_last_seen_joined_v1"
|
||||
): DESKTOP_SRC,
|
||||
client_id_target(
|
||||
table="telemetry_derived.clients_profile_per_install_affected_v1"
|
||||
): DESKTOP_SRC,
|
||||
client_id_target(table="telemetry_derived.core_clients_daily_v1"): DESKTOP_SRC,
|
||||
client_id_target(table="telemetry_derived.core_clients_last_seen_v1"): DESKTOP_SRC,
|
||||
client_id_target(table="telemetry_derived.event_events_v1"): DESKTOP_SRC,
|
||||
|
@ -298,8 +297,6 @@ DELETE_TARGETS: DeleteIndex = {
|
|||
DeleteTarget(table="telemetry_stable.sync_v4", field=SYNC_IDS): SYNC_SOURCES,
|
||||
DeleteTarget(table="telemetry_stable.sync_v5", field=SYNC_IDS): SYNC_SOURCES,
|
||||
# fxa
|
||||
client_id_target(table="firefox_accounts_derived.events_daily_v1"): FXA_SRC,
|
||||
client_id_target(table="firefox_accounts_derived.funnel_events_source_v1"): FXA_SRC,
|
||||
user_id_target(
|
||||
table="firefox_accounts_derived.fxa_amplitude_export_v1"
|
||||
): FXA_HMAC_SRC,
|
||||
|
@ -533,20 +530,23 @@ def find_glean_targets(
|
|||
"""
|
||||
datasets = {dataset.dataset_id for dataset in client.list_datasets(project)}
|
||||
|
||||
glean_stable_tables = [
|
||||
table
|
||||
for tables in pool.map(
|
||||
client.list_tables,
|
||||
[
|
||||
bigquery.DatasetReference(project, dataset_id)
|
||||
for dataset_id in datasets
|
||||
if dataset_id.endswith("_stable")
|
||||
],
|
||||
chunksize=1,
|
||||
)
|
||||
for table in tables
|
||||
if table.labels.get("schema_id") == GLEAN_SCHEMA_ID
|
||||
]
|
||||
def stable_tables_by_schema(schema_id):
|
||||
return [
|
||||
table
|
||||
for tables in pool.map(
|
||||
client.list_tables,
|
||||
[
|
||||
bigquery.DatasetReference(project, dataset_id)
|
||||
for dataset_id in datasets
|
||||
if dataset_id.endswith("_stable")
|
||||
],
|
||||
chunksize=1,
|
||||
)
|
||||
for table in tables
|
||||
if table.labels.get("schema_id") == schema_id
|
||||
]
|
||||
|
||||
glean_stable_tables = stable_tables_by_schema(GLEAN_SCHEMA_ID)
|
||||
|
||||
channel_to_app_name = get_glean_channel_to_app_name_mapping()
|
||||
|
||||
|
@ -627,6 +627,16 @@ def find_glean_targets(
|
|||
manually_added_tables = {target.table for target in DELETE_TARGETS.keys()}
|
||||
skipped_tables = manually_added_tables | GLEAN_IGNORE_LIST
|
||||
|
||||
# Handle custom deletion requests for usage_reporting pings
|
||||
glean_min_stable_tables = stable_tables_by_schema(GLEAN_MIN_SCHEMA_ID)
|
||||
usage_reporting_sources: dict[str, tuple[DeleteSource, ...]] = defaultdict(tuple)
|
||||
for table in glean_min_stable_tables:
|
||||
if table.table_id == "usage_deletion_request_v1":
|
||||
source = DeleteSource(
|
||||
qualified_table_id(table), GLEAN_USAGE_PROFILE_ID, project
|
||||
)
|
||||
usage_reporting_sources[table.dataset_id] += (source,)
|
||||
|
||||
return {
|
||||
**{
|
||||
# glean stable tables that have a source
|
||||
|
@ -675,6 +685,18 @@ def find_glean_targets(
|
|||
and not table.table_id.startswith(derived_source_prefix)
|
||||
and qualified_table_id(table) not in skipped_tables
|
||||
},
|
||||
**{
|
||||
# usage_reporting tables via custom usage_deletion_request ping
|
||||
DeleteTarget(
|
||||
table=qualified_table_id(table),
|
||||
# field must be repeated for each deletion source
|
||||
field=(GLEAN_USAGE_PROFILE_ID,)
|
||||
* len(usage_reporting_sources[table.dataset_id]),
|
||||
): usage_reporting_sources[table.dataset_id]
|
||||
for table in glean_min_stable_tables
|
||||
if table.table_id == "usage_reporting_v1"
|
||||
and table.dataset_id in usage_reporting_sources
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -14,6 +14,7 @@ from typing import Callable, Iterable, Optional, Tuple, Union
|
|||
|
||||
from google.api_core.exceptions import NotFound
|
||||
from google.cloud import bigquery
|
||||
from google.cloud.bigquery import QueryJob
|
||||
|
||||
from ..format_sql.formatter import reformat
|
||||
from ..util import standard_args
|
||||
|
@ -148,6 +149,16 @@ parser.add_argument(
|
|||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeleteJobResults:
|
||||
"""Subset of a bigquery job object retaining only fields that are needed."""
|
||||
|
||||
job_id: str
|
||||
total_bytes_processed: Optional[int]
|
||||
num_dml_affected_rows: Optional[int]
|
||||
destination: str
|
||||
|
||||
|
||||
def record_state(client, task_id, job, dry_run, start_date, end_date, state_table):
|
||||
"""Record the job for task_id in state_table."""
|
||||
if state_table is not None:
|
||||
|
@ -180,11 +191,11 @@ def wait_for_job(
|
|||
create_job,
|
||||
check_table_existence=False,
|
||||
**state_kwargs,
|
||||
):
|
||||
) -> DeleteJobResults:
|
||||
"""Get a job from state or create a new job, and wait for the job to complete."""
|
||||
job = None
|
||||
if task_id in states:
|
||||
job = client.get_job(**FULL_JOB_ID_RE.fullmatch(states[task_id]).groupdict())
|
||||
job = client.get_job(**FULL_JOB_ID_RE.fullmatch(states[task_id]).groupdict()) # type: ignore[union-attr]
|
||||
if job.errors:
|
||||
logging.info(f"Previous attempt failed, retrying for {task_id}")
|
||||
job = None
|
||||
|
@ -209,7 +220,20 @@ def wait_for_job(
|
|||
if not dry_run and not job.ended:
|
||||
logging.info(f"Waiting on {full_job_id(job)} for {task_id}")
|
||||
job.result()
|
||||
return job
|
||||
|
||||
try:
|
||||
bytes_processed = job.total_bytes_processed
|
||||
except AttributeError:
|
||||
bytes_processed = 0
|
||||
|
||||
return DeleteJobResults(
|
||||
job_id=job.job_id,
|
||||
total_bytes_processed=bytes_processed,
|
||||
num_dml_affected_rows=(
|
||||
job.num_dml_affected_rows if isinstance(job, QueryJob) else None
|
||||
),
|
||||
destination=job.destination,
|
||||
)
|
||||
|
||||
|
||||
def get_task_id(target, partition_id):
|
||||
|
@ -419,13 +443,12 @@ def delete_from_partition_with_sampling(
|
|||
copy_job.total_bytes_processed = sum(
|
||||
[r.total_bytes_processed for r in results]
|
||||
)
|
||||
copy_job.num_dml_affected_rows = None
|
||||
return copy_job
|
||||
else:
|
||||
# copy job doesn't have dry runs so dry run base partition for byte estimate
|
||||
return client.query(
|
||||
f"SELECT * FROM {target_table}",
|
||||
job_config=bigquery.QueryJobConfig(dry_run=True, use_legacy_sql=True),
|
||||
f"SELECT * FROM `{sql_table_id(target)}` WHERE {partition.condition}",
|
||||
job_config=bigquery.QueryJobConfig(dry_run=True),
|
||||
)
|
||||
|
||||
return partial(
|
||||
|
@ -789,21 +812,13 @@ def main():
|
|||
jobs_by_table[tasks[i].table].append(job)
|
||||
bytes_processed = rows_deleted = 0
|
||||
for table, jobs in jobs_by_table.items():
|
||||
table_bytes_processed = sum(
|
||||
job.total_bytes_processed or 0
|
||||
for job in jobs
|
||||
if isinstance(job, bigquery.QueryJob)
|
||||
)
|
||||
table_bytes_processed = sum(job.total_bytes_processed or 0 for job in jobs)
|
||||
bytes_processed += table_bytes_processed
|
||||
table_id = sql_table_id(table)
|
||||
if args.dry_run:
|
||||
logging.info(f"Would scan {table_bytes_processed} bytes from {table_id}")
|
||||
else:
|
||||
table_rows_deleted = sum(
|
||||
job.num_dml_affected_rows or 0
|
||||
for job in jobs
|
||||
if isinstance(job, bigquery.QueryJob)
|
||||
)
|
||||
table_rows_deleted = sum(job.num_dml_affected_rows or 0 for job in jobs)
|
||||
rows_deleted += table_rows_deleted
|
||||
logging.info(
|
||||
f"Scanned {table_bytes_processed} bytes and "
|
||||
|
|
|
@ -56,7 +56,10 @@ def project_dirs(project_id=None, sql_dir=None) -> List[str]:
|
|||
sql_dir = ConfigLoader.get("default", "sql_dir", fallback="sql")
|
||||
if project_id is None:
|
||||
return [
|
||||
os.path.join(sql_dir, project_dir) for project_dir in os.listdir(sql_dir)
|
||||
os.path.join(sql_dir, project_dir)
|
||||
for project_dir in os.listdir(sql_dir)
|
||||
# sql/ can contain files like bigconfig.yml
|
||||
if os.path.isdir(os.path.join(sql_dir, project_dir))
|
||||
]
|
||||
else:
|
||||
return [os.path.join(sql_dir, project_id)]
|
||||
|
|
|
@ -15,6 +15,7 @@ default:
|
|||
- moz-fx-data-pioneer-nonprod
|
||||
- moz-fx-data-pioneer-prod
|
||||
- moz-fx-data-glam-prod-fca7
|
||||
- moz-fx-glam-prod
|
||||
public_project: mozilla-public-data
|
||||
|
||||
monitoring:
|
||||
|
@ -42,6 +43,7 @@ dry_run:
|
|||
- sql/moz-fx-data-shared-prod/accounts_backend_external/nonprod_emails_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_backend_external/accounts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_backend_external/emails_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/table_storage_v1/query.sql
|
||||
# uses time travel, will error on dates prior to the time travel window
|
||||
- sql/moz-fx-data-shared-prod/accounts_backend_derived/monitoring_db_counts_v1/query.sql
|
||||
- sql/moz-fx-data-shared-prod/accounts_db_external/**/*.sql
|
||||
|
@ -199,6 +201,7 @@ dry_run:
|
|||
# Dataset sql/glam-fenix-dev:glam_etl was not found
|
||||
- sql/glam-fenix-dev/glam_etl/**/*.sql
|
||||
- sql/moz-fx-data-glam-prod-fca7/glam_etl/**/*.sql
|
||||
- sql/moz-fx-glam-prod/glam_etl/**/*.sql
|
||||
# Source table does not have a verifiable schema
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_run_v1/view.sql
|
||||
- sql/moz-fx-data-shared-prod/monitoring_derived/outerbounds_cost_per_flow_v1/view.sql
|
||||
|
@ -214,6 +217,9 @@ dry_run:
|
|||
# Tests
|
||||
- sql/moz-fx-data-test-project/test/simple_view/view.sql
|
||||
|
||||
deprecation:
|
||||
retain_dataset_roles: # the following roles are retain permissions on datasets when a dataset table is deprecated
|
||||
- roles/bigquery.dataEditor # operational access
|
||||
|
||||
format:
|
||||
skip:
|
||||
|
@ -337,6 +343,49 @@ generate:
|
|||
events_tables: # overwrite event tables
|
||||
pine: # Probe info returns pings that don't have tables, only use events_v1
|
||||
- events_v1
|
||||
bigconfig:
|
||||
skip_apps:
|
||||
- firefox_echo_show
|
||||
- firefox_fire_tv
|
||||
- thunderbird_android
|
||||
- org_mozilla_social_nightly
|
||||
- mozilla_vpn
|
||||
- mozillavpn_backend_cirrus
|
||||
- accounts_backend
|
||||
- accounts_cirrus
|
||||
- burnham
|
||||
- firefox_crashreporter
|
||||
- firefox_reality
|
||||
- firefox_reality_pc
|
||||
- lockwise_android
|
||||
- mach
|
||||
- monitor_backend
|
||||
- monitor_cirrus
|
||||
- moso_mastodon_backend
|
||||
- mozphab
|
||||
- mozregression
|
||||
skip_app_metrics:
|
||||
- thunderbird_desktop
|
||||
bqetl_checks:
|
||||
skip_apps:
|
||||
- mozilla_vpn
|
||||
- mozillavpn_backend_cirrus
|
||||
- accounts_backend
|
||||
- accounts_cirrus
|
||||
- burnham
|
||||
- firefox_crashreporter
|
||||
- firefox_reality
|
||||
- firefox_reality_pc
|
||||
- lockwise_android
|
||||
- mach
|
||||
- monitor_backend
|
||||
- monitor_cirrus
|
||||
- moso_mastodon_backend
|
||||
- mozphab
|
||||
- mozregression
|
||||
stable_views:
|
||||
skip_datasets:
|
||||
- mlhackweek_search
|
||||
|
||||
retention_exclusion_list:
|
||||
- sql/moz-fx-data-shared-prod/search_derived/acer_cohort_v1
|
||||
|
|
63
dags.yaml
63
dags.yaml
|
@ -431,7 +431,7 @@ bqetl_monitoring:
|
|||
used for monitoring of the data platform.
|
||||
default_args:
|
||||
owner: ascholtz@mozilla.com
|
||||
email: ["ascholtz@mozilla.com"]
|
||||
email: ["ascholtz@mozilla.com", "kwindau@mozilla.com"]
|
||||
start_date: "2018-10-30"
|
||||
retries: 2
|
||||
retry_delay: 30m
|
||||
|
@ -1490,7 +1490,7 @@ bqetl_glam_refresh_aggregates:
|
|||
- impact/tier_2
|
||||
- repo/bigquery-etl
|
||||
|
||||
bqetl_glam_refresh_aggregates_fog_release:
|
||||
bqetl_glam_refresh_aggregates_release:
|
||||
default_args:
|
||||
depends_on_past: false
|
||||
email:
|
||||
|
@ -1501,7 +1501,7 @@ bqetl_glam_refresh_aggregates_fog_release:
|
|||
retries: 2
|
||||
retry_delay: 30m
|
||||
start_date: '2024-12-10'
|
||||
description: Update GLAM FOG release tables that serve aggregated data.
|
||||
description: Update GLAM FOG and Fenix release tables that serve aggregated data.
|
||||
repo: bigquery-etl
|
||||
schedule_interval: 0 18 * * 6
|
||||
tags:
|
||||
|
@ -2037,3 +2037,60 @@ bqetl_event_aggregates:
|
|||
- impact/tier_3
|
||||
repo: bigquery-etl
|
||||
schedule_interval: 40 16 * * *
|
||||
|
||||
bqetl_fx_cert_error_privacy_dashboard:
|
||||
description: |
|
||||
This DAG builds the desktop event_aggregates table for unique users count
|
||||
default_args:
|
||||
depends_on_past: false
|
||||
owner: akommasani@mozilla.com
|
||||
email:
|
||||
- telemetry-alerts@mozilla.com
|
||||
- akommasani@mozilla.com
|
||||
email_on_failure: true
|
||||
email_on_retry: false
|
||||
start_date: "2024-12-19"
|
||||
retries: 2
|
||||
retry_delay: 5m
|
||||
tags:
|
||||
- impact/tier_3
|
||||
repo: bigquery-etl
|
||||
schedule_interval: 40 16 * * *
|
||||
|
||||
bqetl_default_browser_aggregates:
|
||||
default_args:
|
||||
depends_on_past: false
|
||||
email:
|
||||
- telemetry-alerts@mozilla.com
|
||||
- wichan@mozilla.com
|
||||
email_on_failure: true
|
||||
email_on_retry: false
|
||||
owner: wichan@mozilla.com
|
||||
retries: 2
|
||||
retry_delay: 30m
|
||||
start_date: '2024-12-20'
|
||||
description: |
|
||||
This DAG builds daily aggregate tables for default browser tables
|
||||
repo: bigquery-etl
|
||||
schedule_interval: 0 22 * * *
|
||||
tags:
|
||||
- impact/tier_3
|
||||
|
||||
bqetl_dsktp_acqstn_fnnl:
|
||||
description: |
|
||||
This DAG builds the Desktop Acquisiton Funnel aggregate table
|
||||
default_args:
|
||||
depends_on_past: false
|
||||
owner: kwindau@mozilla.com
|
||||
email:
|
||||
- telemetry-alerts@mozilla.com
|
||||
- kwindau@mozilla.com
|
||||
email_on_failure: true
|
||||
email_on_retry: false
|
||||
start_date: "2025-01-08"
|
||||
retries: 2
|
||||
retry_delay: 5m
|
||||
tags:
|
||||
- impact/tier_2
|
||||
repo: bigquery-etl
|
||||
schedule_interval: 30 11 * * *
|
||||
|
|
|
@ -126,7 +126,7 @@ The following is an example to update a new field in `telemetry_derived.clients_
|
|||
|
||||
## Remove a field from a table schema
|
||||
|
||||
Deleting a field from an existing table schema should be done only when is totally neccessary. If you decide to delete it:
|
||||
Deleting a field from an existing table schema should be done only when is totally necessary. If you decide to delete it:
|
||||
1. Validate if there is data in the column and make sure data it is either backed up or it can be reprocessed.
|
||||
1. Follow [Big Query docs](https://cloud.google.com/bigquery/docs/managing-table-schemas#deleting_columns_from_a_tables_schema_definition) recommendations for deleting.
|
||||
1. If the column size exceeds the allowed limit, consider setting the field as NULL. See this [search_clients_daily_v8](https://github.com/mozilla/bigquery-etl/pull/2463) PR for an example.
|
||||
|
|
|
@ -268,11 +268,13 @@ For our example:
|
|||
bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD>
|
||||
```
|
||||
|
||||
- If the backfill requires [shredder_mitigation](https://docs.telemetry.mozilla.org/cookbooks/data_modeling/shredder_mitigation) to maintain metrics stable, use the `--shredder_mitigation` parameter in the backfill command:
|
||||
|
||||
```bash
|
||||
bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD> --shredder_mitigation
|
||||
```
|
||||
- If the table's metadata has the label `shredder_mitigation: true`, use the process to run a [backfill with shredder_mitigation](https://docs.telemetry.mozilla.org/cookbooks/data_modeling/shredder_mitigation#running-a-managed-backfill-with-shredder-mitigation):
|
||||
- Bump the version of the query.
|
||||
- Make the necessary updates to the new version of the query and schema.
|
||||
- Create the managed backfill for the new version of the query, including the parameter `--shredder_mitigation`.
|
||||
```bash
|
||||
bqetl backfill create <project>.<dataset>.<table> --start_date=<YYYY-MM-DD> --end_date=<YYYY-MM-DD> --shredder_mitigation
|
||||
```
|
||||
|
||||
2. Fill out the missing details:
|
||||
- Watchers: Mozilla Emails for users that should be notified via Slack about backfill progress.
|
||||
|
|
|
@ -16,7 +16,6 @@ dry_run:
|
|||
views:
|
||||
skip_validation:
|
||||
- sql/moz-fx-data-test-project/test/simple_view/view.sql
|
||||
- sql/moz-fx-data-shared-prod/mlhackweek_search/events/view.sql
|
||||
- sql/moz-fx-data-shared-prod/**/client_deduplication/view.sql
|
||||
# - ...
|
||||
skip_publishing:
|
||||
|
|
|
@ -11,7 +11,7 @@ description = "Tooling for building derived datasets in BigQuery"
|
|||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dynamic = ["dependencies"]
|
||||
version = "2024.5.1"
|
||||
version = "2025.1.1"
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/mozilla/bigquery-etl"
|
||||
|
|
|
@ -1,34 +1,35 @@
|
|||
attrs==24.3.0
|
||||
authlib==1.3.2
|
||||
bigeye-sdk==0.4.94
|
||||
authlib==1.4.0
|
||||
bigeye-sdk==0.4.97
|
||||
black==24.10.0
|
||||
cattrs==24.1.2
|
||||
click==8.1.7
|
||||
click==8.1.8
|
||||
cryptography==44.0.0
|
||||
exceptiongroup==1.2.2 # for backwards compatibility with python < 3.11
|
||||
flake8<5 # pytest-flake8 does not support flake8 5+
|
||||
gcsfs==2024.10.0
|
||||
gcsfs==2024.12.0
|
||||
gcloud==0.18.3
|
||||
gitpython==3.1.43
|
||||
gitpython==3.1.44
|
||||
google-auth>=2.30.0 # To try to fix "Compute Engine Metadata server call to universe/universe_domain returned 404" errors.
|
||||
google-cloud-bigquery==3.27.0
|
||||
google-cloud-bigquery==3.29.0
|
||||
google-cloud-bigquery-storage[fastavro]==2.27.0
|
||||
google-cloud-datacatalog-lineage==0.3.11
|
||||
google-cloud-storage==2.19.0
|
||||
Jinja2==3.1.4
|
||||
Jinja2==3.1.5
|
||||
jsonschema==4.23.0
|
||||
PyJWT==2.10.1
|
||||
markdown-include==0.8.1
|
||||
mdx_truly_sane_lists==1.3
|
||||
mkdocs==1.6.1
|
||||
mkdocs-material==9.5.49
|
||||
mkdocs-material==9.5.50
|
||||
mkdocs-awesome-pages-plugin==2.9.3
|
||||
mozilla-metric-config-parser==2024.11.1
|
||||
mozilla-schema-generator==0.5.1
|
||||
pandas==2.2.3
|
||||
pathos==0.3.3
|
||||
pip-tools==7.4.1
|
||||
pre-commit==4.0.1
|
||||
pyarrow==18.1.0
|
||||
pre-commit==4.1.0
|
||||
pyarrow==19.0.0
|
||||
pytest-black==0.6.0
|
||||
pytest-flake8==1.3.0
|
||||
pytest-isort==4.0.0
|
||||
|
@ -42,16 +43,16 @@ smart_open==6.4.0
|
|||
sqlglot==25.28.0
|
||||
sqlparse==0.5.3
|
||||
stripe==6.4.0
|
||||
symbolic==12.12.4
|
||||
symbolic==12.13.3
|
||||
siggen==2.2.20241029
|
||||
tomli==2.2.1 # for backwards compatibility with python < 3.11
|
||||
types-python-dateutil==2.9.0.20241206
|
||||
types-pytz==2024.2.0.20241003
|
||||
types-PyYAML==6.0.12.20240917
|
||||
types-pytz==2024.2.0.20241221
|
||||
types-PyYAML==6.0.12.20241230
|
||||
types-requests==2.32.0.20241016
|
||||
types-ujson==5.10.0.20240515
|
||||
typing==3.7.4.3
|
||||
ujson==5.10.0
|
||||
yamllint==1.35.1
|
||||
requests==2.32.3
|
||||
responses==0.25.3
|
||||
responses==0.25.6
|
||||
|
|
274
requirements.txt
274
requirements.txt
|
@ -121,9 +121,9 @@ attrs==24.3.0 \
|
|||
# mozilla-metric-config-parser
|
||||
# pytest-mypy
|
||||
# referencing
|
||||
authlib==1.3.2 \
|
||||
--hash=sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2 \
|
||||
--hash=sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc
|
||||
authlib==1.4.0 \
|
||||
--hash=sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2 \
|
||||
--hash=sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3
|
||||
# via -r requirements.in
|
||||
babel==2.12.1 \
|
||||
--hash=sha256:b4246fb7677d3b98f501a39d43396d3cafdc8eadb045f4a31be01863f655c610 \
|
||||
|
@ -132,9 +132,9 @@ babel==2.12.1 \
|
|||
betterproto[compiler]==1.2.5 \
|
||||
--hash=sha256:74a3ab34646054f674d236d1229ba8182dc2eae86feb249b8590ef496ce9803d
|
||||
# via bigeye-sdk
|
||||
bigeye-sdk==0.4.94 \
|
||||
--hash=sha256:d17e50629ecf3273bc7ca8e4ff9883d8fb59e1111776a02ef9cdc07399e1306c \
|
||||
--hash=sha256:ff230eb0fcfdfd72932ec83975ede59c7f1c2d2fed89286fb70463f3752fcef7
|
||||
bigeye-sdk==0.4.97 \
|
||||
--hash=sha256:90d44f8cfc3170e984f39a684f24e38b7fa2268172b82a6bf601c96a5c1dfe48 \
|
||||
--hash=sha256:fd2a0db64486a3c2a3dfc49ae66c8604d2d7a4b5ac8be22f4779edaf49417af1
|
||||
# via -r requirements.in
|
||||
black==24.10.0 \
|
||||
--hash=sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f \
|
||||
|
@ -278,9 +278,9 @@ charset-normalizer==2.1.1 \
|
|||
--hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \
|
||||
--hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f
|
||||
# via requests
|
||||
click==8.1.7 \
|
||||
--hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
|
||||
--hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
|
||||
click==8.1.8 \
|
||||
--hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \
|
||||
--hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a
|
||||
# via
|
||||
# -r requirements.in
|
||||
# black
|
||||
|
@ -298,40 +298,36 @@ commonmark==0.9.1 \
|
|||
--hash=sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60 \
|
||||
--hash=sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9
|
||||
# via rich
|
||||
cryptography==42.0.4 \
|
||||
--hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \
|
||||
--hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \
|
||||
--hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \
|
||||
--hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \
|
||||
--hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \
|
||||
--hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \
|
||||
--hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \
|
||||
--hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \
|
||||
--hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \
|
||||
--hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \
|
||||
--hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \
|
||||
--hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \
|
||||
--hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \
|
||||
--hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \
|
||||
--hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \
|
||||
--hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \
|
||||
--hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \
|
||||
--hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \
|
||||
--hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \
|
||||
--hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \
|
||||
--hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \
|
||||
--hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \
|
||||
--hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \
|
||||
--hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \
|
||||
--hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \
|
||||
--hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \
|
||||
--hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \
|
||||
--hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \
|
||||
--hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \
|
||||
--hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \
|
||||
--hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \
|
||||
--hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449
|
||||
cryptography==44.0.0 \
|
||||
--hash=sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7 \
|
||||
--hash=sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731 \
|
||||
--hash=sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b \
|
||||
--hash=sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc \
|
||||
--hash=sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543 \
|
||||
--hash=sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c \
|
||||
--hash=sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591 \
|
||||
--hash=sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede \
|
||||
--hash=sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb \
|
||||
--hash=sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f \
|
||||
--hash=sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123 \
|
||||
--hash=sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c \
|
||||
--hash=sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c \
|
||||
--hash=sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285 \
|
||||
--hash=sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd \
|
||||
--hash=sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092 \
|
||||
--hash=sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa \
|
||||
--hash=sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289 \
|
||||
--hash=sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02 \
|
||||
--hash=sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64 \
|
||||
--hash=sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053 \
|
||||
--hash=sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417 \
|
||||
--hash=sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e \
|
||||
--hash=sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e \
|
||||
--hash=sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7 \
|
||||
--hash=sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756 \
|
||||
--hash=sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4
|
||||
# via
|
||||
# -r requirements.in
|
||||
# authlib
|
||||
# secretstorage
|
||||
decorator==5.1.1 \
|
||||
|
@ -348,9 +344,9 @@ dill==0.3.9 \
|
|||
# via
|
||||
# multiprocess
|
||||
# pathos
|
||||
distlib==0.3.6 \
|
||||
--hash=sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46 \
|
||||
--hash=sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e
|
||||
distlib==0.3.9 \
|
||||
--hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
|
||||
--hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
|
||||
# via virtualenv
|
||||
exceptiongroup==1.2.2 \
|
||||
--hash=sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b \
|
||||
|
@ -402,9 +398,9 @@ fastavro==1.9.0 \
|
|||
--hash=sha256:f803c33f4fd4e3bfc17bbdbf3c036fbcb92a1f8e6bd19a035800518479ce6b36 \
|
||||
--hash=sha256:fb7e3a058a169d2c8bd19dfcbc7ae14c879750ce49fbaf3c436af683991f7eae
|
||||
# via google-cloud-bigquery-storage
|
||||
filelock==3.9.0 \
|
||||
--hash=sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de \
|
||||
--hash=sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d
|
||||
filelock==3.16.1 \
|
||||
--hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
|
||||
--hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
|
||||
# via
|
||||
# pytest-mypy
|
||||
# virtualenv
|
||||
|
@ -492,16 +488,16 @@ frozenlist==1.3.3 \
|
|||
# via
|
||||
# aiohttp
|
||||
# aiosignal
|
||||
fsspec==2024.10.0 \
|
||||
--hash=sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871 \
|
||||
--hash=sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493
|
||||
fsspec==2024.12.0 \
|
||||
--hash=sha256:670700c977ed2fb51e0d9f9253177ed20cbde4a3e5c0283cc5385b5870c8533f \
|
||||
--hash=sha256:b520aed47ad9804237ff878b504267a3b0b441e97508bd6d2d8774e3db85cee2
|
||||
# via gcsfs
|
||||
gcloud==0.18.3 \
|
||||
--hash=sha256:0af2dec59fce20561752f86e42d981c6a255e306a6c5e5d1fa3d358a8857e4fb
|
||||
# via -r requirements.in
|
||||
gcsfs==2024.10.0 \
|
||||
--hash=sha256:5df54cfe568e8fdeea5aafa7fed695cdc69a9a674e991ca8c1ce634f5df1d314 \
|
||||
--hash=sha256:bb2d23547e61203ea2dda5fa6c4b91a0c34b74ebe8bb6ab1926f6c33381bceb2
|
||||
gcsfs==2024.12.0 \
|
||||
--hash=sha256:e672413922108300ebc1fe78b8f99f3c7c1b94e7e088f5a6dc88de6d5a93d156 \
|
||||
--hash=sha256:ec88e48f77e466723705458af85dda238e43aa69fac071efd98829d06e9f095a
|
||||
# via -r requirements.in
|
||||
ghp-import==2.1.0 \
|
||||
--hash=sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619 \
|
||||
|
@ -511,9 +507,9 @@ gitdb==4.0.10 \
|
|||
--hash=sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a \
|
||||
--hash=sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7
|
||||
# via gitpython
|
||||
gitpython==3.1.43 \
|
||||
--hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \
|
||||
--hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff
|
||||
gitpython==3.1.44 \
|
||||
--hash=sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110 \
|
||||
--hash=sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269
|
||||
# via
|
||||
# -r requirements.in
|
||||
# mozilla-metric-config-parser
|
||||
|
@ -531,9 +527,9 @@ google-api-core[grpc]==2.17.1 \
|
|||
# google-cloud-core
|
||||
# google-cloud-datacatalog-lineage
|
||||
# google-cloud-storage
|
||||
google-auth==2.37.0 \
|
||||
--hash=sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00 \
|
||||
--hash=sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0
|
||||
google-auth==2.38.0 \
|
||||
--hash=sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4 \
|
||||
--hash=sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a
|
||||
# via
|
||||
# -r requirements.in
|
||||
# gcsfs
|
||||
|
@ -548,9 +544,9 @@ google-auth-oauthlib==0.8.0 \
|
|||
--hash=sha256:40cc612a13c3336d5433e94e2adb42a0c88f6feb6c55769e44500fc70043a576 \
|
||||
--hash=sha256:81056a310fb1c4a3e5a7e1a443e1eb96593c6bbc55b26c0261e4d3295d3e6593
|
||||
# via gcsfs
|
||||
google-cloud-bigquery==3.27.0 \
|
||||
--hash=sha256:379c524054d7b090fa56d0c22662cc6e6458a6229b6754c0e7177e3a73421d2c \
|
||||
--hash=sha256:b53b0431e5ba362976a4cd8acce72194b4116cdf8115030c7b339b884603fcc3
|
||||
google-cloud-bigquery==3.29.0 \
|
||||
--hash=sha256:5453a4eabe50118254eda9778f3d7dad413490de5f7046b5e66c98f5a1580308 \
|
||||
--hash=sha256:fafc2b455ffce3bcc6ce0e884184ef50b6a11350a83b91e327fadda4d5566e72
|
||||
# via -r requirements.in
|
||||
google-cloud-bigquery-storage[fastavro]==2.27.0 \
|
||||
--hash=sha256:3bfa8f74a61ceaffd3bfe90be5bbef440ad81c1c19ac9075188cccab34bffc2b \
|
||||
|
@ -767,9 +763,9 @@ jeepney==0.8.0 \
|
|||
# via
|
||||
# keyring
|
||||
# secretstorage
|
||||
jinja2==3.1.4 \
|
||||
--hash=sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369 \
|
||||
--hash=sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d
|
||||
jinja2==3.1.5 \
|
||||
--hash=sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb \
|
||||
--hash=sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb
|
||||
# via
|
||||
# -r requirements.in
|
||||
# betterproto
|
||||
|
@ -934,9 +930,9 @@ mkdocs-get-deps==0.2.0 \
|
|||
--hash=sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c \
|
||||
--hash=sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134
|
||||
# via mkdocs
|
||||
mkdocs-material==9.5.49 \
|
||||
--hash=sha256:3671bb282b4f53a1c72e08adbe04d2481a98f85fed392530051f80ff94a9621d \
|
||||
--hash=sha256:c3c2d8176b18198435d3a3e119011922f3e11424074645c24019c2dcf08a360e
|
||||
mkdocs-material==9.5.50 \
|
||||
--hash=sha256:ae5fe16f3d7c9ccd05bb6916a7da7420cf99a9ce5e33debd9d40403a090d5825 \
|
||||
--hash=sha256:f24100f234741f4d423a9d672a909d859668a4f404796be3cf035f10d6050385
|
||||
# via -r requirements.in
|
||||
mkdocs-material-extensions==1.3 \
|
||||
--hash=sha256:0297cc48ba68a9fdd1ef3780a3b41b534b0d0df1d1181a44676fda5f464eeadc \
|
||||
|
@ -1209,9 +1205,9 @@ pip-tools==7.4.1 \
|
|||
--hash=sha256:4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9 \
|
||||
--hash=sha256:864826f5073864450e24dbeeb85ce3920cdfb09848a3d69ebf537b521f14bcc9
|
||||
# via -r requirements.in
|
||||
platformdirs==2.6.2 \
|
||||
--hash=sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490 \
|
||||
--hash=sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2
|
||||
platformdirs==4.3.6 \
|
||||
--hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
|
||||
--hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
|
||||
# via
|
||||
# black
|
||||
# mkdocs-get-deps
|
||||
|
@ -1232,9 +1228,9 @@ ppft==1.7.6.9 \
|
|||
--hash=sha256:73161c67474ea9d81d04bcdad166d399cff3f084d5d2dc21ebdd46c075bbc265 \
|
||||
--hash=sha256:dab36548db5ca3055067fbe6b1a17db5fee29f3c366c579a9a27cebb52ed96f0
|
||||
# via pathos
|
||||
pre-commit==4.0.1 \
|
||||
--hash=sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2 \
|
||||
--hash=sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878
|
||||
pre-commit==4.1.0 \
|
||||
--hash=sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4 \
|
||||
--hash=sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b
|
||||
# via -r requirements.in
|
||||
propcache==0.2.0 \
|
||||
--hash=sha256:00181262b17e517df2cd85656fcd6b4e70946fe62cd625b9d74ac9977b64d8d9 \
|
||||
|
@ -1366,49 +1362,49 @@ protobuf==4.21.12 \
|
|||
# googleapis-common-protos
|
||||
# grpcio-status
|
||||
# proto-plus
|
||||
pyarrow==18.1.0 \
|
||||
--hash=sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe \
|
||||
--hash=sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e \
|
||||
--hash=sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54 \
|
||||
--hash=sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99 \
|
||||
--hash=sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e \
|
||||
--hash=sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9 \
|
||||
--hash=sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181 \
|
||||
--hash=sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76 \
|
||||
--hash=sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c \
|
||||
--hash=sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c \
|
||||
--hash=sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56 \
|
||||
--hash=sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754 \
|
||||
--hash=sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b \
|
||||
--hash=sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9 \
|
||||
--hash=sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992 \
|
||||
--hash=sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc \
|
||||
--hash=sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7 \
|
||||
--hash=sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa \
|
||||
--hash=sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b \
|
||||
--hash=sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73 \
|
||||
--hash=sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812 \
|
||||
--hash=sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d \
|
||||
--hash=sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052 \
|
||||
--hash=sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191 \
|
||||
--hash=sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386 \
|
||||
--hash=sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324 \
|
||||
--hash=sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4 \
|
||||
--hash=sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba \
|
||||
--hash=sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470 \
|
||||
--hash=sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71 \
|
||||
--hash=sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30 \
|
||||
--hash=sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33 \
|
||||
--hash=sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a \
|
||||
--hash=sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8 \
|
||||
--hash=sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee \
|
||||
--hash=sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c \
|
||||
--hash=sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6 \
|
||||
--hash=sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854 \
|
||||
--hash=sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0 \
|
||||
--hash=sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21 \
|
||||
--hash=sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2 \
|
||||
--hash=sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c
|
||||
pyarrow==19.0.0 \
|
||||
--hash=sha256:239ca66d9a05844bdf5af128861af525e14df3c9591bcc05bac25918e650d3a2 \
|
||||
--hash=sha256:2795064647add0f16563e57e3d294dbfc067b723f0fd82ecd80af56dad15f503 \
|
||||
--hash=sha256:29cd86c8001a94f768f79440bf83fee23963af5e7bc68ce3a7e5f120e17edf89 \
|
||||
--hash=sha256:2a0144a712d990d60f7f42b7a31f0acaccf4c1e43e957f7b1ad58150d6f639c1 \
|
||||
--hash=sha256:2a1a109dfda558eb011e5f6385837daffd920d54ca00669f7a11132d0b1e6042 \
|
||||
--hash=sha256:2b6d3ce4288793350dc2d08d1e184fd70631ea22a4ff9ea5c4ff182130249d9b \
|
||||
--hash=sha256:2f672f5364b2d7829ef7c94be199bb88bf5661dd485e21d2d37de12ccb78a136 \
|
||||
--hash=sha256:3c1c162c4660e0978411a4761f91113dde8da3433683efa473501254563dcbe8 \
|
||||
--hash=sha256:450a7d27e840e4d9a384b5c77199d489b401529e75a3b7a3799d4cd7957f2f9c \
|
||||
--hash=sha256:4624c89d6f777c580e8732c27bb8e77fd1433b89707f17c04af7635dd9638351 \
|
||||
--hash=sha256:4d8b0c0de0a73df1f1bf439af1b60f273d719d70648e898bc077547649bb8352 \
|
||||
--hash=sha256:5418d4d0fab3a0ed497bad21d17a7973aad336d66ad4932a3f5f7480d4ca0c04 \
|
||||
--hash=sha256:597360ffc71fc8cceea1aec1fb60cb510571a744fffc87db33d551d5de919bec \
|
||||
--hash=sha256:5e8a28b918e2e878c918f6d89137386c06fe577cd08d73a6be8dafb317dc2d73 \
|
||||
--hash=sha256:62ef8360ff256e960f57ce0299090fb86423afed5e46f18f1225f960e05aae3d \
|
||||
--hash=sha256:66732e39eaa2247996a6b04c8aa33e3503d351831424cdf8d2e9a0582ac54b34 \
|
||||
--hash=sha256:718947fb6d82409013a74b176bf93e0f49ef952d8a2ecd068fecd192a97885b7 \
|
||||
--hash=sha256:8d47c691765cf497aaeed4954d226568563f1b3b74ff61139f2d77876717084b \
|
||||
--hash=sha256:8e3a839bf36ec03b4315dc924d36dcde5444a50066f1c10f8290293c0427b46a \
|
||||
--hash=sha256:9348a0137568c45601b031a8d118275069435f151cbb77e6a08a27e8125f59d4 \
|
||||
--hash=sha256:a08e2a8a039a3f72afb67a6668180f09fddaa38fe0d21f13212b4aba4b5d2451 \
|
||||
--hash=sha256:a218670b26fb1bc74796458d97bcab072765f9b524f95b2fccad70158feb8b17 \
|
||||
--hash=sha256:a22a4bc0937856263df8b94f2f2781b33dd7f876f787ed746608e06902d691a5 \
|
||||
--hash=sha256:a7bbe7109ab6198688b7079cbad5a8c22de4d47c4880d8e4847520a83b0d1b68 \
|
||||
--hash=sha256:a92aff08e23d281c69835e4a47b80569242a504095ef6a6223c1f6bb8883431d \
|
||||
--hash=sha256:b34d3bde38eba66190b215bae441646330f8e9da05c29e4b5dd3e41bde701098 \
|
||||
--hash=sha256:b903afaa5df66d50fc38672ad095806443b05f202c792694f3a604ead7c6ea6e \
|
||||
--hash=sha256:be686bf625aa7b9bada18defb3a3ea3981c1099697239788ff111d87f04cd263 \
|
||||
--hash=sha256:c0423393e4a07ff6fea08feb44153302dd261d0551cc3b538ea7a5dc853af43a \
|
||||
--hash=sha256:c318eda14f6627966997a7d8c374a87d084a94e4e38e9abbe97395c215830e0c \
|
||||
--hash=sha256:c3b78eff5968a1889a0f3bc81ca57e1e19b75f664d9c61a42a604bf9d8402aae \
|
||||
--hash=sha256:c73268cf557e688efb60f1ccbc7376f7e18cd8e2acae9e663e98b194c40c1a2d \
|
||||
--hash=sha256:c751c1c93955b7a84c06794df46f1cec93e18610dcd5ab7d08e89a81df70a849 \
|
||||
--hash=sha256:ce42275097512d9e4e4a39aade58ef2b3798a93aa3026566b7892177c266f735 \
|
||||
--hash=sha256:cf3bf0ce511b833f7bc5f5bb3127ba731e97222023a444b7359f3a22e2a3b463 \
|
||||
--hash=sha256:da410b70a7ab8eb524112f037a7a35da7128b33d484f7671a264a4c224ac131d \
|
||||
--hash=sha256:e675a3ad4732b92d72e4d24009707e923cab76b0d088e5054914f11a797ebe44 \
|
||||
--hash=sha256:e82c3d5e44e969c217827b780ed8faf7ac4c53f934ae9238872e749fa531f7c9 \
|
||||
--hash=sha256:edfe6d3916e915ada9acc4e48f6dafca7efdbad2e6283db6fd9385a1b23055f1 \
|
||||
--hash=sha256:f094742275586cdd6b1a03655ccff3b24b2610c3af76f810356c4c71d24a2a6c \
|
||||
--hash=sha256:f208c3b58a6df3b239e0bb130e13bc7487ed14f39a9ff357b6415e3f6339b560 \
|
||||
--hash=sha256:f43f5aef2a13d4d56adadae5720d1fed4c1356c993eda8b59dace4b5983843c1
|
||||
# via -r requirements.in
|
||||
pyasn1==0.4.8 \
|
||||
--hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \
|
||||
|
@ -1943,9 +1939,9 @@ requests-oauthlib==1.3.1 \
|
|||
--hash=sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5 \
|
||||
--hash=sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a
|
||||
# via google-auth-oauthlib
|
||||
responses==0.25.3 \
|
||||
--hash=sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb \
|
||||
--hash=sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba
|
||||
responses==0.25.6 \
|
||||
--hash=sha256:9cac8f21e1193bb150ec557875377e41ed56248aed94e4567ed644db564bacf1 \
|
||||
--hash=sha256:eae7ce61a9603004e76c05691e7c389e59652d91e94b419623c12bbfb8e331d8
|
||||
# via -r requirements.in
|
||||
rich==12.6.0 \
|
||||
--hash=sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e \
|
||||
|
@ -2179,12 +2175,12 @@ stripe==6.4.0 \
|
|||
--hash=sha256:d1ab2739aa6511f34ba77f66fca9f86cffe2dbb17df84bd224a38e7254efe61b \
|
||||
--hash=sha256:d99e97e6d3f6dbaf9361ebd07b3024ec6f9d7b1f77724c8127338e0754314e9d
|
||||
# via -r requirements.in
|
||||
symbolic==12.12.4 \
|
||||
--hash=sha256:358961063d90d368485b75df178722f984424401b92b6e56e9587da0814387e8 \
|
||||
--hash=sha256:39c8bebeedd497cf4c6ab4844a382687cb95fe9915613b37ddc053cd5c45dd42 \
|
||||
--hash=sha256:afcc276195be07045ec7f5f1a32f5542120833fac5c39d067dfe500387f5643a \
|
||||
--hash=sha256:d71ad7e681d69a193ef8e71553356844eeb88abeab06e48a3b20ad2c1cf38d2c \
|
||||
--hash=sha256:e9951ad949e235089571497e732b1bf04f93605d43e06cb352a4dce45b7a8b7c
|
||||
symbolic==12.13.3 \
|
||||
--hash=sha256:0517fbb15b8d1b4e261ecbc35fecdc16bd35b563a7230d70d7e0091c073ecae1 \
|
||||
--hash=sha256:2cb76190abb134e030487d93d75a1011a873fe70162492aa048f383d5e4eb914 \
|
||||
--hash=sha256:39475cbc0ed1dfb0dd183428b7d7cb4ee85b59fac7b101a0a04bd54bbde010e9 \
|
||||
--hash=sha256:3f0d83d7260e88bfe05f359e3a972833b1490f7b20a65e628e8df8c61c133d70 \
|
||||
--hash=sha256:79806c75a854447ab770ff84e43daae4f73445963b13df043cbb871ce664b3be
|
||||
# via -r requirements.in
|
||||
toml==0.10.2 \
|
||||
--hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
|
||||
|
@ -2234,13 +2230,13 @@ types-python-dateutil==2.9.0.20241206 \
|
|||
--hash=sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb \
|
||||
--hash=sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53
|
||||
# via -r requirements.in
|
||||
types-pytz==2024.2.0.20241003 \
|
||||
--hash=sha256:3e22df1336c0c6ad1d29163c8fda82736909eb977281cb823c57f8bae07118b7 \
|
||||
--hash=sha256:575dc38f385a922a212bac00a7d6d2e16e141132a3c955078f4a4fd13ed6cb44
|
||||
types-pytz==2024.2.0.20241221 \
|
||||
--hash=sha256:06d7cde9613e9f7504766a0554a270c369434b50e00975b3a4a0f6eed0f2c1a9 \
|
||||
--hash=sha256:8fc03195329c43637ed4f593663df721fef919b60a969066e22606edf0b53ad5
|
||||
# via -r requirements.in
|
||||
types-pyyaml==6.0.12.20240917 \
|
||||
--hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \
|
||||
--hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587
|
||||
types-pyyaml==6.0.12.20241230 \
|
||||
--hash=sha256:7f07622dbd34bb9c8b264fe860a17e0efcad00d50b5f27e93984909d9363498c \
|
||||
--hash=sha256:fa4d32565219b68e6dee5f67534c722e53c00d1cfc09c435ef04d7353e1e96e6
|
||||
# via
|
||||
# -r requirements.in
|
||||
# bigeye-sdk
|
||||
|
@ -2360,9 +2356,9 @@ urllib3==2.2.2 \
|
|||
# requests
|
||||
# responses
|
||||
# types-requests
|
||||
virtualenv==20.17.1 \
|
||||
--hash=sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4 \
|
||||
--hash=sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058
|
||||
virtualenv==20.26.6 \
|
||||
--hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \
|
||||
--hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2
|
||||
# via pre-commit
|
||||
watchdog==2.2.1 \
|
||||
--hash=sha256:102a60093090fc3ff76c983367b19849b7cc24ec414a43c0333680106e62aae1 \
|
||||
|
@ -2572,9 +2568,9 @@ pip==23.3.2 \
|
|||
--hash=sha256:5052d7889c1f9d05224cd41741acb7c5d6fa735ab34e339624a614eaaa7e7d76 \
|
||||
--hash=sha256:7fd9972f96db22c8077a1ee2691b172c8089b17a5652a44494a9ecb0d78f9149
|
||||
# via pip-tools
|
||||
setuptools==75.6.0 \
|
||||
--hash=sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6 \
|
||||
--hash=sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d
|
||||
setuptools==75.8.0 \
|
||||
--hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \
|
||||
--hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3
|
||||
# via
|
||||
# bigeye-sdk
|
||||
# nodeenv
|
||||
|
|
|
@ -8,9 +8,9 @@ saved_metric_definitions:
|
|||
threshold:
|
||||
type: CONSTANT
|
||||
upper_bound: 0
|
||||
metric_schedule:
|
||||
named_schedule:
|
||||
name: default
|
||||
schedule_frequency:
|
||||
interval_type: MINUTES
|
||||
interval_value: 0
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
|
@ -24,9 +24,9 @@ saved_metric_definitions:
|
|||
threshold:
|
||||
type: CONSTANT
|
||||
upper_bound: 0
|
||||
metric_schedule:
|
||||
named_schedule:
|
||||
name: default
|
||||
schedule_frequency:
|
||||
interval_type: MINUTES
|
||||
interval_value: 0
|
||||
rct_overrides:
|
||||
- submission_date
|
||||
- saved_metric_id: freshness_fail
|
||||
|
@ -71,9 +71,9 @@ saved_metric_definitions:
|
|||
type: CONSTANT
|
||||
lower_bound: 1
|
||||
upper_bound: 1
|
||||
metric_schedule:
|
||||
named_schedule:
|
||||
name: default
|
||||
schedule_frequency:
|
||||
interval_type: MINUTES
|
||||
interval_value: 0
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
|
@ -88,9 +88,9 @@ saved_metric_definitions:
|
|||
type: CONSTANT
|
||||
lower_bound: 2
|
||||
upper_bound: 2
|
||||
metric_schedule:
|
||||
named_schedule:
|
||||
name: default
|
||||
schedule_frequency:
|
||||
interval_type: MINUTES
|
||||
interval_value: 0
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
|
@ -105,9 +105,9 @@ saved_metric_definitions:
|
|||
type: CONSTANT
|
||||
lower_bound: 1
|
||||
upper_bound: 1
|
||||
metric_schedule:
|
||||
named_schedule:
|
||||
name: default
|
||||
schedule_frequency:
|
||||
interval_type: MINUTES
|
||||
interval_value: 0
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
friendly_name: Addon Events (client) View
|
||||
description: |
|
||||
A view on top of the events table pulling out addon relevant columns
|
||||
to act as a central gate for downstream addon models.
|
||||
|
||||
See events documentation for more information about the fields:
|
||||
https://firefox-source-docs.mozilla.org/toolkit/components/telemetry/collection/events.html
|
||||
|
||||
Addon event specific data documentation:
|
||||
https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#174-336
|
||||
owners:
|
||||
- kik@mozilla.com
|
||||
labels:
|
||||
application: amo
|
|
@ -0,0 +1,36 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.addons.addon_events_clients`
|
||||
AS
|
||||
SELECT
|
||||
submission_date,
|
||||
`timestamp`,
|
||||
COALESCE(
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addonId'),
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addon_id')
|
||||
) AS addon_id,
|
||||
COALESCE(
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addonVersion'),
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'addon_version')
|
||||
) AS addon_version,
|
||||
event_string_value,
|
||||
client_id,
|
||||
sample_id,
|
||||
normalized_channel,
|
||||
country,
|
||||
locale,
|
||||
app_name,
|
||||
app_version,
|
||||
os AS os_name,
|
||||
os_version,
|
||||
event_timestamp,
|
||||
event_category,
|
||||
event_method AS method,
|
||||
event_object AS `object`,
|
||||
event_string_value AS `value`,
|
||||
event_map_values,
|
||||
event_process,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.telemetry.events`
|
||||
WHERE
|
||||
submission_date >= '2021-08-09'
|
||||
AND event_category IN ("addonsManager", "addonsSearchDetection")
|
|
@ -0,0 +1,11 @@
|
|||
friendly_name: Addons
|
||||
description: |-
|
||||
Dataset for accessing Addons data.
|
||||
dataset_base_acl: view
|
||||
user_facing: true
|
||||
labels: {}
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
- workgroup:amo/prod
|
|
@ -0,0 +1,11 @@
|
|||
---
|
||||
friendly_name: Addon - Search Detection
|
||||
description: |
|
||||
A view for accessing the addons search detection aggregate.
|
||||
|
||||
Addon event specific data documentation:
|
||||
https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#174-336
|
||||
owners:
|
||||
- kik@mozilla.com
|
||||
labels:
|
||||
application: amo
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.addons.search_detection`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.addons_derived.search_detection_v1`
|
|
@ -0,0 +1,17 @@
|
|||
friendly_name: Addons Derived
|
||||
description: |-
|
||||
Derived data used for Addons related analysis.
|
||||
dataset_base_acl: derived
|
||||
user_facing: false
|
||||
labels: {}
|
||||
default_table_workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
- workgroup:amo/prod
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
- workgroup:amo/prod
|
||||
syndication: {}
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
friendly_name: Addons - Search Detection
|
||||
description: |
|
||||
Used for addons search detection and based on event_category: addonsSearchDetection.
|
||||
|
||||
This event category is recorded when a request for a search query is redirected to another
|
||||
domain. This can be via the webRequest API, or through other means.
|
||||
|
||||
More information about this event_category:
|
||||
https://searchfox.org/mozilla-central/source/toolkit/components/telemetry/Events.yaml#304
|
||||
owners:
|
||||
- kik@mozilla.com
|
||||
labels:
|
||||
application: amo
|
||||
incremental: true
|
||||
schedule: daily
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_amo_stats
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: false
|
||||
clustering:
|
||||
fields:
|
||||
- normalized_channel
|
||||
- app_name
|
||||
- os_name
|
|
@ -0,0 +1,37 @@
|
|||
SELECT
|
||||
submission_date,
|
||||
normalized_channel,
|
||||
country,
|
||||
app_name,
|
||||
app_version,
|
||||
os_name,
|
||||
os_version,
|
||||
addon_id,
|
||||
addon_version,
|
||||
method,
|
||||
`object`,
|
||||
`value`,
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'from') AS from_etld,
|
||||
`moz-fx-data-shared-prod.udf.get_key`(event_map_values, 'to') AS to_etld,
|
||||
COUNT(*) AS event_count,
|
||||
COUNT(DISTINCT client_id) AS unique_clients_cnt,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.addons.addon_events_clients`
|
||||
WHERE
|
||||
submission_date = @submission_date
|
||||
AND event_category = "addonsSearchDetection"
|
||||
GROUP BY
|
||||
submission_date,
|
||||
normalized_channel,
|
||||
country,
|
||||
app_name,
|
||||
app_version,
|
||||
os_name,
|
||||
os_version,
|
||||
addon_id,
|
||||
addon_version,
|
||||
method,
|
||||
`object`,
|
||||
`value`,
|
||||
from_etld,
|
||||
to_etld
|
|
@ -0,0 +1,97 @@
|
|||
# Based on telemetry.events: https://docs.telemetry.mozilla.org/concepts/pipeline/event_pipeline
|
||||
fields:
|
||||
- mode: NULLABLE
|
||||
name: submission_date
|
||||
type: DATE
|
||||
description: |
|
||||
Processing date.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: normalized_channel
|
||||
type: STRING
|
||||
description: |
|
||||
Normalized channel name of the Firefox app (for example: releaes, beta, nightly).
|
||||
|
||||
- mode: NULLABLE
|
||||
name: country
|
||||
type: STRING
|
||||
description: |
|
||||
An ISO 3166-1 alpha-2 country code.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: app_name
|
||||
type: STRING
|
||||
description: |
|
||||
Name of the Firefox app.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: app_version
|
||||
type: STRING
|
||||
description: |
|
||||
Version of the Firefox app.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: os_name
|
||||
type: STRING
|
||||
description: |
|
||||
Name of the operating system used by the client.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: os_version
|
||||
type: STRING
|
||||
description: |
|
||||
Version of the operating system used by the client.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: addon_id
|
||||
type: STRING
|
||||
description:
|
||||
The id of the addon, if available.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: addon_version
|
||||
type: STRING
|
||||
description: |
|
||||
The version of the addon.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: method
|
||||
type: STRING
|
||||
description: |
|
||||
This describes the type of event that occurred.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: object
|
||||
type: STRING
|
||||
description: |
|
||||
The object the event occurred.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: value
|
||||
type: STRING
|
||||
description: |
|
||||
This is a user defined value, providing context for the event (Optional).
|
||||
|
||||
- mode: NULLABLE
|
||||
name: from_etld
|
||||
type: STRING
|
||||
description: |
|
||||
The original domain (eTLD+1) of the request that matched a search endpoint.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: to_etld
|
||||
type: STRING
|
||||
description: |
|
||||
The domain (eTLD+1) of the final redirect target.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: event_count
|
||||
type: INTEGER
|
||||
description: |
|
||||
Number of events in this grouping.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: unique_clients_cnt
|
||||
type: INTEGER
|
||||
description: |
|
||||
Number of unique clients in this grouping.
|
|
@ -7,6 +7,9 @@ description: >
|
|||
pings have limitations on length of properties attached to events
|
||||
and addon_id values are sometimes too long. The AMO stats application
|
||||
looks up records in this table based on the hashed_addon_id.
|
||||
|
||||
The hashed_addon_id is only valid for install_stats:
|
||||
https://searchfox.org/mozilla-central/rev/5b061cdc4d40d44988dc61aa941cfbd98e31791f/toolkit/components/telemetry/Events.yaml#233-234
|
||||
owners:
|
||||
- kik@mozilla.com
|
||||
labels:
|
||||
|
|
|
@ -1,62 +1,61 @@
|
|||
type: BIGCONFIG_FILE
|
||||
|
||||
tag_deployments:
|
||||
- collection:
|
||||
name: Operational Checks
|
||||
notification_channels:
|
||||
- slack: '#de-bigeye-triage'
|
||||
deployments:
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.date
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_id
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_name
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_id
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.impressions
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.clicks
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.downloads
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.new_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.activated_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.repeat_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.week_4_retained_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.spend
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.lifetime_value
|
||||
metrics:
|
||||
- saved_metric_id: is_not_null
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code
|
||||
metrics:
|
||||
- saved_metric_id: is_2_char_len
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.*
|
||||
metrics:
|
||||
- saved_metric_id: volume
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- saved_metric_id: freshness
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- collection:
|
||||
name: Operational Checks
|
||||
notification_channels:
|
||||
- slack: '#de-bigeye-triage'
|
||||
deployments:
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.date
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_id
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_name
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.ad_group_id
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.impressions
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.clicks
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.downloads
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.new_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.activated_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.repeat_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.week_4_retained_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.spend
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.lifetime_value
|
||||
metrics:
|
||||
- saved_metric_id: is_not_null
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.campaign_country_code
|
||||
metrics:
|
||||
- saved_metric_id: is_2_char_len
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.apple_ads_external.ios_app_campaign_stats_v1.*
|
||||
metrics:
|
||||
- saved_metric_id: volume
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- saved_metric_id: freshness
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
friendly_name: Default Browser Agent
|
||||
description: |-
|
||||
Data related to default browser agent
|
||||
dataset_base_acl: view
|
||||
user_facing: true
|
||||
labels: {}
|
||||
default_table_workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
default_table_expiration_ms: null
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
syndication: {}
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.default_browser_agent.default_browser_agg`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.default_browser_agent_derived.default_browser_agg_v1`
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.default_browser_agent.default_browser_agg_by_os`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.default_browser_agent_derived.default_browser_agg_by_os_v1`
|
|
@ -0,0 +1,16 @@
|
|||
friendly_name: Default Browser Agent Derived
|
||||
description: |-
|
||||
Data related to default browser agent
|
||||
dataset_base_acl: derived
|
||||
user_facing: false
|
||||
labels: {}
|
||||
default_table_workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
default_table_expiration_ms: null
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
syndication: {}
|
|
@ -0,0 +1,9 @@
|
|||
2024-12-27:
|
||||
start_date: 2023-01-01
|
||||
end_date: 2023-12-31
|
||||
reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
|
||||
watchers:
|
||||
- wichan@mozilla.com
|
||||
status: Complete
|
||||
shredder_mitigation: false
|
||||
override_retention_limit: false
|
|
@ -0,0 +1,23 @@
|
|||
friendly_name: Default Browser Aggregates Derived By OS version
|
||||
description: |-
|
||||
Aggregate table for default browser setting by os version
|
||||
owners:
|
||||
- wichan@mozilla.com
|
||||
labels:
|
||||
incremental: true
|
||||
owner1: wichan@mozilla.com
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_default_browser_aggregates
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
||||
range_partitioning: null
|
||||
clustering:
|
||||
fields:
|
||||
- os_version
|
||||
references: {}
|
|
@ -0,0 +1,19 @@
|
|||
SELECT
|
||||
DATE(submission_timestamp) AS submission_date,
|
||||
os_version,
|
||||
mozfun.norm.get_windows_info(`os_version`).name AS windows_name,
|
||||
default_browser,
|
||||
previous_default_browser,
|
||||
mozfun.norm.truncate_version(build_version, 'major') AS build_version_major,
|
||||
COUNT(*) AS row_count
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.default_browser_agent.default_browser`
|
||||
WHERE
|
||||
DATE(submission_timestamp) = @submission_date
|
||||
GROUP BY
|
||||
DATE(submission_timestamp),
|
||||
os_version,
|
||||
mozfun.norm.get_windows_info(`os_version`).name,
|
||||
default_browser,
|
||||
previous_default_browser,
|
||||
mozfun.norm.truncate_version(build_version, 'major')
|
|
@ -0,0 +1,29 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Date when the ingestion edge server accepted this message.
|
||||
- name: os_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Windows version number in major.minor.build.UBR format (UBR is optional, only available on Win10).
|
||||
- name: windows_name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Name of the window os version.
|
||||
- name: default_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: The current default browser.
|
||||
- name: previous_default_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: What the default browser was before it was changed to the current one.
|
||||
- name: build_version_major
|
||||
type: NUMERIC
|
||||
mode: NULLABLE
|
||||
description: Major build version.
|
||||
- name: row_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: The total number of rows aggregated.
|
|
@ -0,0 +1,9 @@
|
|||
2024-12-27:
|
||||
start_date: 2023-01-01
|
||||
end_date: 2023-12-31
|
||||
reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
|
||||
watchers:
|
||||
- wichan@mozilla.com
|
||||
status: Complete
|
||||
shredder_mitigation: false
|
||||
override_retention_limit: false
|
|
@ -0,0 +1,23 @@
|
|||
friendly_name: Default Browser Aggregates Derived
|
||||
description: |-
|
||||
Aggregate table for default browser setting
|
||||
owners:
|
||||
- wichan@mozilla.com
|
||||
labels:
|
||||
incremental: true
|
||||
owner1: wichan@mozilla.com
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_default_browser_aggregates
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
||||
range_partitioning: null
|
||||
clustering:
|
||||
fields:
|
||||
- normalized_country_code
|
||||
references: {}
|
|
@ -0,0 +1,27 @@
|
|||
SELECT
|
||||
DATE(submission_timestamp) AS submission_date,
|
||||
mozfun.norm.truncate_version(build_version, 'major') AS build_version_major,
|
||||
mozfun.norm.get_windows_info(`os_version`).name AS windows_name,
|
||||
normalized_country_code,
|
||||
build_channel,
|
||||
default_browser,
|
||||
previous_default_browser,
|
||||
notification_shown,
|
||||
notification_type,
|
||||
notification_action,
|
||||
COUNT(*) AS row_count
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.default_browser_agent.default_browser`
|
||||
WHERE
|
||||
DATE(submission_timestamp) = @submission_date
|
||||
GROUP BY
|
||||
DATE(submission_timestamp),
|
||||
mozfun.norm.truncate_version(build_version, 'major'),
|
||||
mozfun.norm.get_windows_info(`os_version`).name,
|
||||
normalized_country_code,
|
||||
build_channel,
|
||||
default_browser,
|
||||
previous_default_browser,
|
||||
notification_shown,
|
||||
notification_type,
|
||||
notification_action
|
|
@ -0,0 +1,46 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Date when the ingestion edge server accepted this message.
|
||||
- name: build_version_major
|
||||
type: NUMERIC
|
||||
mode: NULLABLE
|
||||
description: Major build version.
|
||||
- name: windows_name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Name of the window os version.
|
||||
- name: normalized_country_code
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: An ISO 3166-1 alpha-2 country code.
|
||||
- name: build_channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: build channel.
|
||||
- name: default_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: The current default browser.
|
||||
- name: previous_default_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: What the default browser was before it was changed to the current one.
|
||||
- name: notification_shown
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Whether the notification was shown, not shown, or resulted in an error.
|
||||
- name: notification_type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Whether the notification shown was the initial notification, or the
|
||||
followup.
|
||||
- name: notification_action
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: What action was taken by the user then the toast notification was displayed.
|
||||
- name: row_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: The total number of rows aggregated.
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.fenix.locale_aggregates`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix_derived.locale_aggregates_v1`
|
|
@ -0,0 +1,20 @@
|
|||
friendly_name: Feature Usage Metrics (Android - Fenix)
|
||||
description: |-
|
||||
Metrics from metrics pings for mobile feature usage dashboards (Fenix)
|
||||
owners:
|
||||
- rzhao@mozilla.com
|
||||
- vsabino@mozilla.com
|
||||
labels:
|
||||
application: fenix
|
||||
incremental: true
|
||||
owner1: rzhao
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_mobile_feature_usage
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
|
@ -0,0 +1,331 @@
|
|||
WITH baseline_clients AS (
|
||||
SELECT
|
||||
submission_date AS dau_date,
|
||||
client_id
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix.baseline_clients_daily`
|
||||
WHERE
|
||||
submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
|
||||
AND durations > 0
|
||||
AND LOWER(COALESCE(isp, "")) <> "browserstack"
|
||||
AND LOWER(COALESCE(distribution_id, "")) <> "mozillaonline"
|
||||
),
|
||||
client_attribution AS (
|
||||
SELECT
|
||||
client_id,
|
||||
adjust_network,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix.attribution_clients`
|
||||
),
|
||||
metrics_dau AS (
|
||||
-- assign a DAU date for each metric ping while keeping it de-duplicated
|
||||
SELECT
|
||||
m.*,
|
||||
MAX(dau_date) OVER (PARTITION BY document_id) AS dau_date
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix.metrics` m
|
||||
JOIN
|
||||
baseline_clients
|
||||
ON client_info.client_id = client_id
|
||||
-- offset by at least one to reflect metrics ping design considerations
|
||||
AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
|
||||
BETWEEN 1
|
||||
AND 4
|
||||
WHERE
|
||||
DATE(submission_timestamp)
|
||||
BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
|
||||
AND @submission_date
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY submission_timestamp) = 1
|
||||
),
|
||||
metric_ping_clients_feature_usage AS (
|
||||
SELECT
|
||||
dau_date,
|
||||
client_info.client_id,
|
||||
ARRAY_AGG(normalized_channel ORDER BY submission_timestamp DESC)[SAFE_OFFSET(0)] AS channel,
|
||||
ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp DESC)[
|
||||
SAFE_OFFSET(0)
|
||||
] AS country,
|
||||
LOGICAL_OR(COALESCE(metrics.boolean.metrics_default_browser, FALSE)) AS is_default_browser,
|
||||
--Credential Management: Logins
|
||||
SUM(COALESCE(metrics.counter.logins_deleted, 0)) AS logins_deleted,
|
||||
SUM(COALESCE(metrics.counter.logins_modified, 0)) AS logins_modified,
|
||||
MAX(COALESCE(metrics.quantity.logins_saved_all, 0)) AS currently_stored_logins,
|
||||
--Credential Management: Credit Cards
|
||||
SUM(COALESCE(metrics.counter.credit_cards_deleted, 0)) AS credit_cards_deleted,
|
||||
MAX(COALESCE(metrics.quantity.credit_cards_saved_all, 0)) AS currently_stored_credit_cards,
|
||||
--Credential Management: Addresses
|
||||
SUM(COALESCE(metrics.counter.addresses_deleted, 0)) AS addresses_deleted,
|
||||
SUM(COALESCE(metrics.counter.addresses_updated, 0)) AS addresses_modified,
|
||||
MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS currently_stored_addresses,
|
||||
--Bookmark
|
||||
SUM(COALESCE(metrics_bookmarks_add_table.value, 0)) AS bookmarks_add,
|
||||
SUM(COALESCE(metrics_bookmarks_delete_table.value, 0)) AS bookmarks_delete,
|
||||
SUM(COALESCE(metrics_bookmarks_edit_table.value, 0)) AS bookmarks_edit,
|
||||
SUM(COALESCE(metrics_bookmarks_open_table.value, 0)) AS bookmarks_open,
|
||||
MAX(
|
||||
COALESCE(metrics.counter.metrics_desktop_bookmarks_count, 0)
|
||||
) AS metrics_desktop_bookmarks_count,
|
||||
MAX(
|
||||
COALESCE(metrics.counter.metrics_mobile_bookmarks_count, 0)
|
||||
) AS metrics_mobile_bookmarks_count,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.metrics_has_desktop_bookmarks, FALSE)) AS INT64
|
||||
) AS metrics_has_desktop_bookmarks,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.metrics_has_mobile_bookmarks, FALSE)) AS INT64
|
||||
) AS metrics_has_mobile_bookmarks,
|
||||
--Privacy
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "standard", FALSE)
|
||||
) AS INT64
|
||||
) AS etp_standard,
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "strict", FALSE)
|
||||
) AS INT64
|
||||
) AS etp_strict,
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(LOWER(metrics.string.preferences_enhanced_tracking_protection) = "custom", FALSE)
|
||||
) AS INT64
|
||||
) AS etp_custom,
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(
|
||||
LOWER(metrics.string.preferences_enhanced_tracking_protection) NOT IN (
|
||||
'custom',
|
||||
'standard',
|
||||
'strict'
|
||||
),
|
||||
FALSE
|
||||
)
|
||||
) AS INT64
|
||||
) AS etp_disabled,
|
||||
--Tab count
|
||||
MAX(
|
||||
COALESCE(metrics.counter.metrics_private_tabs_open_count, 0)
|
||||
) AS metrics_private_tabs_open_count,
|
||||
MAX(COALESCE(metrics.counter.metrics_tabs_open_count, 0)) AS metrics_tabs_open_count,
|
||||
--Default browser
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.metrics_default_browser, FALSE)) AS INT64
|
||||
) AS metrics_default_browser,
|
||||
--Awesomebar Location
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(metrics.string.preferences_toolbar_position_setting IN ('top', 'fixed_top'), FALSE)
|
||||
) AS INT64
|
||||
) AS awesomebar_top,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.string.preferences_toolbar_position_setting = 'bottom', FALSE)) AS INT64
|
||||
) AS awesomebar_bottom,
|
||||
CAST(
|
||||
MAX(
|
||||
COALESCE(
|
||||
LOWER(metrics.string.preferences_toolbar_position_setting) NOT IN (
|
||||
'top',
|
||||
'fixed_top',
|
||||
'bottom'
|
||||
),
|
||||
FALSE
|
||||
)
|
||||
) AS INT64
|
||||
) AS awesomebar_null,
|
||||
--Notification
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.metrics_notifications_allowed, FALSE)) AS INT64
|
||||
) AS metrics_notifications_allowed,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.events_marketing_notification_allowed, FALSE)) AS INT64
|
||||
) AS events_marketing_notification_allowed,
|
||||
--Customize Homepage
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_contile, FALSE)) AS INT64
|
||||
) AS customize_home_contile,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_jump_back_in, FALSE)) AS INT64
|
||||
) AS customize_home_jump_back_in,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_most_visited_sites, FALSE)) AS INT64
|
||||
) AS customize_home_most_visited_sites,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_pocket, FALSE)) AS INT64
|
||||
) AS customize_home_pocket,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_recently_saved, FALSE)) AS INT64
|
||||
) AS customize_home_recently_saved,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.customize_home_recently_visited, FALSE)) AS INT64
|
||||
) AS customize_home_recently_visited
|
||||
FROM
|
||||
metrics_dau
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.metrics_bookmarks_add) AS metrics_bookmarks_add_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.metrics_bookmarks_delete) AS metrics_bookmarks_delete_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.metrics_bookmarks_edit) AS metrics_bookmarks_edit_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.metrics_bookmarks_open) AS metrics_bookmarks_open_table
|
||||
GROUP BY
|
||||
dau_date,
|
||||
client_id
|
||||
)
|
||||
-- Aggregated feature usage
|
||||
SELECT
|
||||
@submission_date AS submission_date,
|
||||
dau_date AS metric_date,
|
||||
COUNT(DISTINCT client_id) AS clients,
|
||||
channel,
|
||||
country,
|
||||
adjust_network,
|
||||
is_default_browser,
|
||||
/*Logins*/
|
||||
--logins deleted
|
||||
COUNT(DISTINCT IF(logins_deleted > 0, client_id, NULL)) AS logins_deleted_users,
|
||||
SUM(logins_deleted) AS logins_deleted,
|
||||
--logins modified
|
||||
COUNT(DISTINCT IF(logins_modified > 0, client_id, NULL)) AS logins_modified_users,
|
||||
SUM(logins_modified) AS logins_modified,
|
||||
--logins currently stored
|
||||
COUNT(DISTINCT IF(currently_stored_logins > 0, client_id, NULL)) AS currently_stored_logins_users,
|
||||
SUM(currently_stored_logins) AS currently_stored_logins,
|
||||
/*Credit Card*/
|
||||
--credit card deleted
|
||||
COUNT(DISTINCT IF(credit_cards_deleted > 0, client_id, NULL)) AS credit_cards_deleted_users,
|
||||
SUM(credit_cards_deleted) AS credit_cards_deleted,
|
||||
--credit card currently stored
|
||||
COUNT(
|
||||
DISTINCT IF(currently_stored_credit_cards > 0, client_id, NULL)
|
||||
) AS currently_stored_credit_cards_users,
|
||||
SUM(currently_stored_credit_cards) AS currently_stored_credit_cards,
|
||||
/*Address*/
|
||||
--address deleted
|
||||
COUNT(DISTINCT IF(addresses_deleted > 0, client_id, NULL)) AS addresses_deleted_users,
|
||||
SUM(addresses_deleted) AS addresses_deleted,
|
||||
--address modified
|
||||
COUNT(DISTINCT IF(addresses_modified > 0, client_id, NULL)) AS addresses_modified_users,
|
||||
SUM(addresses_modified) AS addresses_modified,
|
||||
-- addresses currently stored
|
||||
COUNT(
|
||||
DISTINCT IF(currently_stored_addresses > 0, client_id, NULL)
|
||||
) AS currently_stored_addresses_users,
|
||||
SUM(currently_stored_addresses) AS currently_stored_addresses,
|
||||
/*Bookmark*/
|
||||
--bookmarks_add
|
||||
COUNT(DISTINCT IF(bookmarks_add > 0, client_id, NULL)) AS bookmarks_add_users,
|
||||
SUM(bookmarks_add) AS bookmarks_add,
|
||||
--bookmarks_delete
|
||||
COUNT(DISTINCT IF(bookmarks_delete > 0, client_id, NULL)) AS bookmarks_delete_users,
|
||||
SUM(bookmarks_delete) AS bookmarks_delete,
|
||||
--bookmarks_edit
|
||||
COUNT(DISTINCT IF(bookmarks_edit > 0, client_id, NULL)) AS bookmarks_edit_users,
|
||||
SUM(bookmarks_edit) AS bookmarks_edit,
|
||||
--bookmarks_open
|
||||
COUNT(DISTINCT IF(bookmarks_open > 0, client_id, NULL)) AS bookmarks_open_users,
|
||||
SUM(bookmarks_open) AS bookmarks_open,
|
||||
--metrics_desktop_bookmarks_count
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_desktop_bookmarks_count > 0, client_id, NULL)
|
||||
) AS metrics_desktop_bookmarks_count_users,
|
||||
SUM(metrics_desktop_bookmarks_count) AS metrics_desktop_bookmarks_count,
|
||||
--metrics_mobile_bookmarks_count
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_mobile_bookmarks_count > 0, client_id, NULL)
|
||||
) AS metrics_mobile_bookmarks_count_users,
|
||||
SUM(metrics_mobile_bookmarks_count) AS metrics_mobile_bookmarks_count,
|
||||
--metrics_has_desktop_bookmarks
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_has_desktop_bookmarks > 0, client_id, NULL)
|
||||
) AS metrics_has_desktop_bookmarks_users,
|
||||
SUM(metrics_has_desktop_bookmarks) AS metrics_has_desktop_bookmarks,
|
||||
--metrics_has_mobile_bookmarks
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_has_mobile_bookmarks > 0, client_id, NULL)
|
||||
) AS metrics_has_mobile_bookmarks_users,
|
||||
SUM(metrics_has_mobile_bookmarks) AS metrics_has_mobile_bookmarks,
|
||||
/*Privacy*/
|
||||
--etp_standard
|
||||
COUNT(DISTINCT IF(etp_standard > 0, client_id, NULL)) AS etp_standard_users,
|
||||
SUM(etp_standard) AS etp_standard,
|
||||
--etp_strict
|
||||
COUNT(DISTINCT IF(etp_strict > 0, client_id, NULL)) AS etp_strict_users,
|
||||
SUM(etp_strict) AS etp_strict,
|
||||
--etp_custom
|
||||
COUNT(DISTINCT IF(etp_custom > 0, client_id, NULL)) AS etp_custom_users,
|
||||
SUM(etp_custom) AS etp_custom,
|
||||
--etp_disabled
|
||||
COUNT(DISTINCT IF(etp_disabled > 0, client_id, NULL)) AS etp_disabled_users,
|
||||
SUM(etp_disabled) AS etp_disabled,
|
||||
/*Tab Count*/
|
||||
--metrics_private_tabs_open_count
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_private_tabs_open_count > 0, client_id, NULL)
|
||||
) AS metrics_private_tabs_open_count_users,
|
||||
SUM(metrics_private_tabs_open_count) AS metrics_private_tabs_open_count,
|
||||
--metrics_tabs_open_count
|
||||
COUNT(DISTINCT IF(metrics_tabs_open_count > 0, client_id, NULL)) AS metrics_tabs_open_count_users,
|
||||
SUM(metrics_tabs_open_count) AS metrics_tabs_open_count,
|
||||
/*Default Browser*/
|
||||
--metrics_default_browser
|
||||
COUNT(DISTINCT IF(metrics_default_browser > 0, client_id, NULL)) AS metrics_default_browser_users,
|
||||
SUM(metrics_default_browser) AS metrics_default_browser,
|
||||
/*Awesomebar Location*/
|
||||
--awesomebar_top
|
||||
COUNT(DISTINCT IF(awesomebar_top > 0, client_id, NULL)) AS awesomebar_top_users,
|
||||
--awesomebar_bottom
|
||||
COUNT(DISTINCT IF(awesomebar_bottom > 0, client_id, NULL)) AS awesomebar_bottom_users,
|
||||
--awesomebar_null
|
||||
COUNT(DISTINCT IF(awesomebar_null > 0, client_id, NULL)) AS awesomebar_null_users,
|
||||
/*Notificaion*/
|
||||
--metrics_notifications_allowed
|
||||
COUNT(
|
||||
DISTINCT IF(metrics_notifications_allowed > 0, client_id, NULL)
|
||||
) AS metrics_notifications_allowed_users,
|
||||
SUM(metrics_notifications_allowed) AS metrics_notifications_allowed,
|
||||
--events_marketing_notification_allowed
|
||||
COUNT(
|
||||
DISTINCT IF(events_marketing_notification_allowed > 0, client_id, NULL)
|
||||
) AS events_marketing_notification_allowed_users,
|
||||
SUM(events_marketing_notification_allowed) AS events_marketing_notification_allowed,
|
||||
/*Customize Homepage*/
|
||||
--customize_home_contile
|
||||
COUNT(DISTINCT IF(customize_home_contile > 0, client_id, NULL)) AS customize_home_contile_users,
|
||||
SUM(customize_home_contile) AS customize_home_contile,
|
||||
--customize_home_jump_back_in_users
|
||||
COUNT(
|
||||
DISTINCT IF(customize_home_jump_back_in > 0, client_id, NULL)
|
||||
) AS customize_home_jump_back_in_users,
|
||||
SUM(customize_home_jump_back_in) AS customize_home_jump_back_in,
|
||||
--customize_home_most_visited_sites_users
|
||||
COUNT(
|
||||
DISTINCT IF(customize_home_most_visited_sites > 0, client_id, NULL)
|
||||
) AS customize_home_most_visited_sites_users,
|
||||
SUM(customize_home_most_visited_sites) AS customize_home_most_visited_sites,
|
||||
--customize_home_pocket_users
|
||||
COUNT(DISTINCT IF(customize_home_pocket > 0, client_id, NULL)) AS customize_home_pocket_users,
|
||||
SUM(customize_home_pocket) AS customize_home_pocket,
|
||||
--customize_home_recently_saved_users
|
||||
COUNT(
|
||||
DISTINCT IF(customize_home_recently_saved > 0, client_id, NULL)
|
||||
) AS customize_home_recently_saved_users,
|
||||
SUM(customize_home_recently_saved) AS customize_home_recently_saved,
|
||||
--customize_home_recently_visited_users
|
||||
COUNT(
|
||||
DISTINCT IF(customize_home_recently_visited > 0, client_id, NULL)
|
||||
) AS customize_home_recently_visited_users,
|
||||
SUM(customize_home_recently_visited) AS customize_home_recently_visited
|
||||
FROM
|
||||
metric_ping_clients_feature_usage
|
||||
LEFT JOIN
|
||||
client_attribution
|
||||
USING (client_id)
|
||||
GROUP BY
|
||||
submission_date,
|
||||
metric_date,
|
||||
channel,
|
||||
country,
|
||||
adjust_network,
|
||||
is_default_browser
|
|
@ -0,0 +1,289 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Airflow's internal execution_date.
|
||||
- name: metric_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: reporting date for feature usage, aligned with DAU.
|
||||
- name: clients
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: number of clients accounted for in the aggregation.
|
||||
- name: channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Channel
|
||||
- name: country
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Country
|
||||
- name: adjust_network
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Adjust Network
|
||||
- name: is_default_browser
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
description: Is Default Browser
|
||||
- name: logins_deleted_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Deleted Users
|
||||
- name: logins_deleted
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Deleted
|
||||
- name: logins_modified_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Modified Users
|
||||
- name: logins_modified
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Modified
|
||||
- name: currently_stored_logins_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Logins Users
|
||||
- name: currently_stored_logins
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Logins
|
||||
- name: credit_cards_deleted_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Cards Deleted Users
|
||||
- name: credit_cards_deleted
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Cards Deleted
|
||||
- name: currently_stored_credit_cards_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Credit Cards Users
|
||||
- name: currently_stored_credit_cards
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Credit Cards
|
||||
- name: addresses_deleted_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Deleted Users
|
||||
- name: addresses_deleted
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Deleted
|
||||
- name: addresses_modified_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Modified Users
|
||||
- name: addresses_modified
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Modified
|
||||
- name: currently_stored_addresses_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Addresses Users
|
||||
- name: currently_stored_addresses
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Currently Stored Addresses
|
||||
- name: bookmarks_add_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Add Users
|
||||
- name: bookmarks_add
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Add
|
||||
- name: bookmarks_delete_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Delete Users
|
||||
- name: bookmarks_delete
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Delete
|
||||
- name: bookmarks_edit_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Edit Users
|
||||
- name: bookmarks_edit
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Edit
|
||||
- name: bookmarks_open_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Open Users
|
||||
- name: bookmarks_open
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Open
|
||||
- name: metrics_desktop_bookmarks_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_desktop_bookmarks_count_users
|
||||
- name: metrics_desktop_bookmarks_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_desktop_bookmarks_count
|
||||
- name: metrics_mobile_bookmarks_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_mobile_bookmarks_count_users
|
||||
- name: metrics_mobile_bookmarks_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_mobile_bookmarks_count
|
||||
- name: metrics_has_desktop_bookmarks_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_has_desktop_bookmarks_users
|
||||
- name: metrics_has_desktop_bookmarks
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Metric Has Desktop Bookmarks
|
||||
- name: metrics_has_mobile_bookmarks_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Metrics Has Mobile Bookmarks Users
|
||||
- name: metrics_has_mobile_bookmarks
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Metrics Has Mobile Bookmarks
|
||||
- name: etp_standard_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Standard Users
|
||||
- name: etp_standard
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Standard
|
||||
- name: etp_strict_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Strict Users
|
||||
- name: etp_strict
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Strict
|
||||
- name: etp_custom_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Custom Users
|
||||
- name: etp_custom
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Custom
|
||||
- name: etp_disabled_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Disabled Users
|
||||
- name: etp_disabled
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: ETP Disabled
|
||||
- name: metrics_private_tabs_open_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_private_tabs_open_count_users
|
||||
- name: metrics_private_tabs_open_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_private_tabs_open_count
|
||||
- name: metrics_tabs_open_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_tabs_open_count_users
|
||||
- name: metrics_tabs_open_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_tabs_open_count
|
||||
- name: metrics_default_browser_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_default_browser_users
|
||||
- name: metrics_default_browser
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_default_browser
|
||||
- name: awesomebar_top_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: awesomebar_top_users
|
||||
- name: awesomebar_bottom_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: awesomebar_bottom_users
|
||||
- name: awesomebar_null_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: awesomebar_null_users
|
||||
- name: metrics_notifications_allowed_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_notifications_allowed_users
|
||||
- name: metrics_notifications_allowed
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: metrics_notifications_allowed
|
||||
- name: events_marketing_notification_allowed_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: events_marketing_notification_allowed_users
|
||||
- name: events_marketing_notification_allowed
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Events Marketing Notification Allowed
|
||||
- name: customize_home_contile_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_contile_users
|
||||
- name: customize_home_contile
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_contile
|
||||
- name: customize_home_jump_back_in_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_jump_back_in_users
|
||||
- name: customize_home_jump_back_in
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_jump_back_in
|
||||
- name: customize_home_most_visited_sites_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_most_visited_sites_users
|
||||
- name: customize_home_most_visited_sites
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_most_visited_sites
|
||||
- name: customize_home_pocket_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_pocket_users
|
||||
- name: customize_home_pocket
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_pocket
|
||||
- name: customize_home_recently_saved_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_recently_saved_users
|
||||
- name: customize_home_recently_saved
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: customize_home_recently_saved
|
||||
- name: customize_home_recently_visited_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Customize Home Recently Visited Users
|
||||
- name: customize_home_recently_visited
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Customize Home Recently Visited
|
|
@ -0,0 +1,23 @@
|
|||
friendly_name: Locale Aggregates
|
||||
description: |-
|
||||
Fenix DAU, WAU, MAU and client counts by locale and core dimensions of analysis.
|
||||
owners:
|
||||
- lvargas@mozilla.com
|
||||
labels:
|
||||
incremental: true
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_analytics_aggregations
|
||||
date_partition_offset: -1
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
||||
clustering:
|
||||
fields:
|
||||
- country
|
||||
- channel
|
||||
- locale
|
||||
references: {}
|
|
@ -0,0 +1,98 @@
|
|||
-- Query for fenix_derived.locale_aggregates_v1
|
||||
WITH metrics AS (
|
||||
-- Metrics ping may arrive in the same or next day as the baseline ping.
|
||||
SELECT
|
||||
client_id,
|
||||
ARRAY_AGG(normalized_channel IGNORE NULLS ORDER BY submission_date ASC)[
|
||||
SAFE_OFFSET(0)
|
||||
] AS normalized_channel,
|
||||
ARRAY_AGG(is_default_browser IGNORE NULLS ORDER BY submission_date ASC)[
|
||||
SAFE_OFFSET(0)
|
||||
] AS is_default_browser
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix.metrics_clients_last_seen`
|
||||
WHERE
|
||||
DATE(submission_date)
|
||||
BETWEEN @submission_date
|
||||
AND DATE_ADD(@submission_date, INTERVAL 1 DAY)
|
||||
GROUP BY
|
||||
client_id
|
||||
),
|
||||
baseline AS (
|
||||
SELECT
|
||||
client_id,
|
||||
submission_date,
|
||||
app_name,
|
||||
app_display_version,
|
||||
normalized_channel,
|
||||
country,
|
||||
city,
|
||||
locale,
|
||||
normalized_os,
|
||||
normalized_os_version,
|
||||
is_daily_user,
|
||||
is_weekly_user,
|
||||
is_monthly_user,
|
||||
is_dau,
|
||||
is_wau,
|
||||
is_mau
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.fenix.active_users`
|
||||
WHERE
|
||||
submission_date = @submission_date
|
||||
),
|
||||
unioned AS (
|
||||
SELECT
|
||||
baseline.client_id,
|
||||
baseline.submission_date,
|
||||
baseline.app_name,
|
||||
baseline.app_display_version AS app_version,
|
||||
baseline.normalized_channel AS channel,
|
||||
IFNULL(baseline.country, '??') country,
|
||||
baseline.city,
|
||||
baseline.locale,
|
||||
baseline.normalized_os AS os,
|
||||
baseline.normalized_os_version AS os_version,
|
||||
metrics.is_default_browser,
|
||||
is_daily_user,
|
||||
is_weekly_user,
|
||||
is_monthly_user,
|
||||
is_dau,
|
||||
is_wau,
|
||||
is_mau
|
||||
FROM
|
||||
baseline
|
||||
LEFT JOIN
|
||||
metrics
|
||||
ON baseline.client_id = metrics.client_id
|
||||
AND baseline.normalized_channel IS NOT DISTINCT FROM metrics.normalized_channel
|
||||
)
|
||||
SELECT
|
||||
unioned.* EXCEPT (
|
||||
client_id,
|
||||
is_daily_user,
|
||||
is_weekly_user,
|
||||
is_monthly_user,
|
||||
is_dau,
|
||||
is_wau,
|
||||
is_mau
|
||||
),
|
||||
COUNTIF(is_daily_user) AS daily_users,
|
||||
COUNTIF(is_weekly_user) AS weekly_users,
|
||||
COUNTIF(is_monthly_user) AS monthly_users,
|
||||
COUNTIF(is_dau) AS dau,
|
||||
COUNTIF(is_wau) AS wau,
|
||||
COUNTIF(is_mau) AS mau
|
||||
FROM
|
||||
unioned
|
||||
GROUP BY
|
||||
submission_date,
|
||||
app_name,
|
||||
app_version,
|
||||
channel,
|
||||
country,
|
||||
city,
|
||||
locale,
|
||||
os,
|
||||
os_version,
|
||||
is_default_browser
|
|
@ -0,0 +1,65 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Date when ping is received on the server side.
|
||||
- name: app_name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser name.
|
||||
- name: app_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser version installed on the client.
|
||||
- name: channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser installation channel installed on the client.
|
||||
- name: country
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Country reported by the client.
|
||||
- name: city
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: City reported by the client.
|
||||
- name: locale
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Locale reported by the client, which is a combination of language and regional settings.
|
||||
- name: os
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Operating System reported by the client.
|
||||
- name: os_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: OS version with windows_build_number as reported by the client.
|
||||
- name: is_default_browser
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
description: Whether the browser is set as the default browser on the client side.
|
||||
- name: daily_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who report a ping in a day.
|
||||
- name: weekly_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 7 days.
|
||||
- name: monthly_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 28 days.
|
||||
- name: dau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who reported a ping on the submission_date that qualify as active.
|
||||
- name: wau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 7 days and qualify as active.
|
||||
- name: mau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 28 days and qualify as active.
|
|
@ -35,9 +35,9 @@ client_search AS (
|
|||
USING (client_id)
|
||||
WHERE
|
||||
(submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
|
||||
AND normalized_app_name = 'Fenix'
|
||||
AND normalized_app_name_os = "Firefox Android"
|
||||
GROUP BY
|
||||
1
|
||||
client_id
|
||||
),
|
||||
dou AS (
|
||||
SELECT
|
||||
|
@ -71,7 +71,7 @@ adjust_client AS (
|
|||
AND metrics.string.first_session_network IS NOT NULL
|
||||
AND metrics.string.first_session_network <> ''
|
||||
GROUP BY
|
||||
1
|
||||
client_id
|
||||
)
|
||||
SELECT
|
||||
client_id,
|
||||
|
@ -93,7 +93,7 @@ SELECT
|
|||
first_seen_date,
|
||||
submission_date,
|
||||
1 AS new_profile,
|
||||
CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS integer) AS activated
|
||||
CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS INTEGER) AS activated
|
||||
FROM
|
||||
dou
|
||||
INNER JOIN
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.firefox_accounts.funnel_events_source`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_accounts_derived.funnel_events_source_v1`
|
|
@ -1,21 +0,0 @@
|
|||
friendly_name: Funnel Events Source
|
||||
description: |-
|
||||
The materialized events used to create the events_daily rollup.
|
||||
Not exposed as a view since it's not meant for consumption;
|
||||
use firefox_accounts.fxa_all_events
|
||||
instead. This is materialized solely because
|
||||
the events_daily queries are too complex for BigQuery
|
||||
otherwise.
|
||||
owners:
|
||||
- wlachance@mozilla.com
|
||||
labels:
|
||||
table_type: client_level
|
||||
scheduling:
|
||||
dag_name: bqetl_event_rollup
|
||||
arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION']
|
||||
task_name: funnel_events_source__v1
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: false
|
|
@ -1,58 +0,0 @@
|
|||
SELECT
|
||||
DATE(`timestamp`) AS submission_date,
|
||||
user_id AS client_id,
|
||||
`moz-fx-data-shared-prod`.udf.safe_sample_id(user_id) AS sample_id,
|
||||
SPLIT(event_type, ' - ')[OFFSET(0)] AS category,
|
||||
SPLIT(event_type, ' - ')[OFFSET(1)] AS event,
|
||||
[
|
||||
STRUCT('service' AS key, service AS value),
|
||||
STRUCT('email_type' AS key, email_type AS value),
|
||||
STRUCT('oauth_client_id' AS key, oauth_client_id AS value),
|
||||
STRUCT('connect_device_flow' AS key, connect_device_flow AS value),
|
||||
STRUCT('connect_device_os' AS key, connect_device_os AS value),
|
||||
STRUCT('sync_device_count' AS key, sync_device_count AS value),
|
||||
STRUCT('email_sender' AS key, email_sender AS value),
|
||||
STRUCT('email_service' AS key, email_service AS value),
|
||||
STRUCT('email_template' AS key, email_template AS value),
|
||||
STRUCT('email_version' AS key, email_version AS value)
|
||||
] AS extra,
|
||||
CAST([] AS ARRAY<STRUCT<key STRING, value STRING>>) AS experiments,
|
||||
logger,
|
||||
event_type,
|
||||
app_version,
|
||||
os_name,
|
||||
os_version,
|
||||
country,
|
||||
`language`,
|
||||
user_id,
|
||||
device_id,
|
||||
`timestamp`,
|
||||
receiveTimestamp,
|
||||
utm_term,
|
||||
utm_source,
|
||||
utm_medium,
|
||||
utm_campaign,
|
||||
utm_content,
|
||||
ua_version,
|
||||
ua_browser,
|
||||
entrypoint,
|
||||
flow_id,
|
||||
service,
|
||||
email_type,
|
||||
email_provider,
|
||||
oauth_client_id,
|
||||
connect_device_flow,
|
||||
connect_device_os,
|
||||
sync_device_count,
|
||||
sync_active_devices_day,
|
||||
sync_active_devices_week,
|
||||
sync_active_devices_month,
|
||||
email_sender,
|
||||
email_service,
|
||||
email_template,
|
||||
email_version,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_accounts.fxa_all_events`
|
||||
WHERE
|
||||
DATE(`timestamp`) = @submission_date
|
||||
AND fxa_log IN ('content', 'auth', 'oauth')
|
|
@ -1,138 +0,0 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
- name: client_id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sample_id
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
- name: category
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: event
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: extra
|
||||
type: RECORD
|
||||
mode: REPEATED
|
||||
fields:
|
||||
- name: key
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: value
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: experiments
|
||||
type: RECORD
|
||||
mode: REPEATED
|
||||
fields:
|
||||
- name: key
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: value
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: logger
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: event_type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: app_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: os_name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: os_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: country
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: language
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: user_id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: device_id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: timestamp
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: receiveTimestamp
|
||||
type: TIMESTAMP
|
||||
mode: NULLABLE
|
||||
- name: utm_term
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: utm_source
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: utm_medium
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: utm_campaign
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: utm_content
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: ua_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: ua_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: entrypoint
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: flow_id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: service
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_provider
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: oauth_client_id
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: connect_device_flow
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: connect_device_os
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sync_device_count
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sync_active_devices_day
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sync_active_devices_week
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: sync_active_devices_month
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_sender
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_service
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_template
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health Counts
|
||||
description: >
|
||||
Counts calculated daily regarding the health of the Accounts service. E.g. # of issues open.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Counts
|
|
@ -0,0 +1,13 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: date
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: type
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: state
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: count
|
||||
type: INTEGER
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health Issues
|
||||
description: >
|
||||
Mirrors Jira Issue data for Firefox Accounts.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Issue Data
|
|
@ -0,0 +1,61 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: key
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: permalink
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: id
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: assignee
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: created
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: found_in
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: issuetype
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: reporter
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: resolution
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: resolutiondate
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: status
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: storypoints
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: summary
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: updated
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: epic
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: originalstorypoints
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: timetoresolve
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: severity
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: priority
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: labels
|
||||
type: STRING
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health Lighthouse
|
||||
description: >
|
||||
Front-end health metrics for Firefox Accounts, calculated from Lighthouse reports.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Lighthouse
|
|
@ -0,0 +1,22 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: fetchTime
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: url
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: category
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: audit
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: score
|
||||
type: NUMERIC
|
||||
- mode: NULLABLE
|
||||
name: numVal
|
||||
type: NUMERIC
|
||||
- mode: NULLABLE
|
||||
name: formfactor
|
||||
type: STRING
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health Pingdom
|
||||
description: >
|
||||
Records pingdom values for the Accounts service.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Pingdom Data
|
|
@ -0,0 +1,25 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: avgresponse
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: downtime
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: starttime
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: unmonitored
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: uptime
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: project
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: fuzzy_uptime_percent
|
||||
type: FLOAT
|
||||
- mode: NULLABLE
|
||||
name: unmonitored_percent
|
||||
type: FLOAT
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health PRs
|
||||
description: >
|
||||
Mirrors data about Account's GitHub PRs.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Pull Request Data
|
|
@ -0,0 +1,46 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: id
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: number
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: url
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: title
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: state
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: author
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: created_at
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: updated_at
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: closed_at
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: merged_at
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: draft
|
||||
type: BOOLEAN
|
||||
- mode: NULLABLE
|
||||
name: time_to_close
|
||||
type: INTEGER
|
||||
- mode: NULLABLE
|
||||
name: reviewers
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: has_an_issue
|
||||
type: BOOLEAN
|
||||
- mode: NULLABLE
|
||||
name: time_to_merge
|
||||
type: INTEGER
|
|
@ -0,0 +1,12 @@
|
|||
friendly_name: Accounts Health Tags
|
||||
description: >
|
||||
Tracks push timings for Firefox Accounts.
|
||||
owners:
|
||||
- wclouser@mozilla.com
|
||||
external_data:
|
||||
format: google_sheets
|
||||
source_uris:
|
||||
- https://docs.google.com/spreadsheets/d/1uGHOS-cY_OUqicbBkS2Z20EK0hLq3PDw28TZcxP5RHI/edit
|
||||
options:
|
||||
skip_leading_rows: 1
|
||||
range: Tag Data
|
|
@ -0,0 +1,22 @@
|
|||
fields:
|
||||
- mode: NULLABLE
|
||||
name: timestamp
|
||||
type: TIMESTAMP
|
||||
- mode: NULLABLE
|
||||
name: server
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: version
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: commit
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: l10n
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: tospp
|
||||
type: STRING
|
||||
- mode: NULLABLE
|
||||
name: since_last_push
|
||||
type: FLOAT
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.firefox_desktop.locale_aggregates`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_desktop_derived.locale_aggregates_v1`
|
|
@ -0,0 +1,16 @@
|
|||
friendly_name: Firefox Desktop Background Defaultagent
|
||||
description: |-
|
||||
Data related to firefox desktop background default agent
|
||||
dataset_base_acl: view
|
||||
user_facing: true
|
||||
labels: {}
|
||||
default_table_workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
default_table_expiration_ms: null
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
syndication: {}
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.firefox_desktop_background_defaultagent.default_agent_agg`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_desktop_background_defaultagent_derived.default_agent_agg_v1`
|
|
@ -0,0 +1,16 @@
|
|||
friendly_name: Firefox Desktop Background Defaultagent Derived
|
||||
description: |-
|
||||
Data related to firefox desktop background default agent
|
||||
dataset_base_acl: derived
|
||||
user_facing: false
|
||||
labels: {}
|
||||
default_table_workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
default_table_expiration_ms: null
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
||||
syndication: {}
|
|
@ -0,0 +1,9 @@
|
|||
2024-12-27:
|
||||
start_date: 2023-01-01
|
||||
end_date: 2023-12-31
|
||||
reason: https://mozilla-hub.atlassian.net/browse/DENG-6889
|
||||
watchers:
|
||||
- wichan@mozilla.com
|
||||
status: Complete
|
||||
shredder_mitigation: false
|
||||
override_retention_limit: false
|
|
@ -0,0 +1,20 @@
|
|||
friendly_name: Firefox Desktop Background Default Agent Derived
|
||||
description: |-
|
||||
Aggregate table for firefox desktop background default agent
|
||||
owners:
|
||||
- wichan@mozilla.com
|
||||
labels:
|
||||
incremental: true
|
||||
owner1: wichan@mozilla.com
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_default_browser_aggregates
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: false
|
||||
expiration_days: null
|
||||
range_partitioning: null
|
||||
references: {}
|
|
@ -0,0 +1,21 @@
|
|||
SELECT
|
||||
DATE(submission_timestamp) AS submission_date,
|
||||
mozfun.norm.truncate_version(client_info.app_display_version, 'major') AS build_version_major,
|
||||
normalized_channel,
|
||||
metrics.string.system_default_browser,
|
||||
metrics.string.system_default_previous_browser,
|
||||
metrics.boolean.notification_show_success,
|
||||
metrics.string.notification_action,
|
||||
COUNT(*) AS row_count
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_desktop_background_defaultagent.default_agent`
|
||||
WHERE
|
||||
DATE(submission_timestamp) = @submission_date
|
||||
GROUP BY
|
||||
DATE(submission_timestamp),
|
||||
mozfun.norm.truncate_version(client_info.app_display_version, 'major'),
|
||||
normalized_channel,
|
||||
metrics.string.system_default_browser,
|
||||
metrics.string.system_default_previous_browser,
|
||||
metrics.boolean.notification_show_success,
|
||||
metrics.string.notification_action
|
|
@ -0,0 +1,33 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Date when the ingestion edge server accepted this message.
|
||||
- name: build_version_major
|
||||
type: NUMERIC
|
||||
mode: NULLABLE
|
||||
description: Major build version.
|
||||
- name: normalized_channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: normalized channel.
|
||||
- name: system_default_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Which browser is currently set as the system default web browser.
|
||||
- name: system_default_previous_browser
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Which browser was set as the system default before it was changed to the current setting.
|
||||
- name: notification_show_success
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
description: Whether a notification was shown or not. Possible value include "shown" and "error".
|
||||
- name: notification_action
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: The action that the user took in response to the notification.
|
||||
- name: row_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: The total number of rows aggregated.
|
|
@ -6,6 +6,8 @@ owners:
|
|||
labels:
|
||||
incremental: true
|
||||
owner1: example
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_desktop_installs_v1
|
||||
bigquery:
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
friendly_name: Locale Aggregates
|
||||
description: |-
|
||||
Firefox Desktop DAU, WAU, MAU and client counts by locale and core dimensions of analysis.
|
||||
owners:
|
||||
- lvargas@mozilla.com
|
||||
labels:
|
||||
incremental: true
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_analytics_aggregations
|
||||
date_partition_offset: -1
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
||||
clustering:
|
||||
fields:
|
||||
- country
|
||||
- channel
|
||||
- locale
|
||||
references: {}
|
|
@ -0,0 +1,57 @@
|
|||
-- Query for firefox_desktop_derived.locale_aggregates_v1
|
||||
WITH todays_metrics AS (
|
||||
SELECT
|
||||
client_id,
|
||||
submission_date,
|
||||
app_name,
|
||||
app_version,
|
||||
normalized_channel AS channel,
|
||||
IFNULL(country, '??') AS country,
|
||||
city,
|
||||
locale,
|
||||
os,
|
||||
COALESCE(
|
||||
`mozfun.norm.windows_version_info`(os, normalized_os_version, windows_build_number),
|
||||
normalized_os_version
|
||||
) AS os_version,
|
||||
is_default_browser,
|
||||
is_daily_user,
|
||||
is_weekly_user,
|
||||
is_monthly_user,
|
||||
is_dau,
|
||||
is_wau,
|
||||
is_mau
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.telemetry.desktop_active_users`
|
||||
WHERE
|
||||
submission_date = @submission_date
|
||||
)
|
||||
SELECT
|
||||
todays_metrics.* EXCEPT (
|
||||
client_id,
|
||||
is_daily_user,
|
||||
is_weekly_user,
|
||||
is_monthly_user,
|
||||
is_dau,
|
||||
is_wau,
|
||||
is_mau
|
||||
),
|
||||
COUNTIF(is_daily_user) AS daily_users,
|
||||
COUNTIF(is_weekly_user) AS weekly_users,
|
||||
COUNTIF(is_monthly_user) AS monthly_users,
|
||||
COUNTIF(is_dau) AS dau,
|
||||
COUNTIF(is_wau) AS wau,
|
||||
COUNTIF(is_mau) AS mau
|
||||
FROM
|
||||
todays_metrics
|
||||
GROUP BY
|
||||
submission_date,
|
||||
app_name,
|
||||
app_version,
|
||||
channel,
|
||||
country,
|
||||
city,
|
||||
locale,
|
||||
os,
|
||||
os_version,
|
||||
is_default_browser
|
|
@ -0,0 +1,65 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Date when ping is received on the server side.
|
||||
- name: app_name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser name.
|
||||
- name: app_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser version installed on the client.
|
||||
- name: channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Browser installation channel installed on the client.
|
||||
- name: country
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Country reported by the client.
|
||||
- name: city
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: City reported by the client.
|
||||
- name: locale
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Locale reported by the client, which is a combination of language and regional settings.
|
||||
- name: os
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Operating System reported by the client.
|
||||
- name: os_version
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: OS version with windows_build_number as reported by the client.
|
||||
- name: is_default_browser
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
description: Whether the browser is set as the default browser on the client side.
|
||||
- name: daily_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who report a ping in a day.
|
||||
- name: weekly_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 7 days.
|
||||
- name: monthly_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 28 days.
|
||||
- name: dau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who reported a ping on the submission_date that qualify as active.
|
||||
- name: wau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 7 days and qualify as active.
|
||||
- name: mau
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Count of users who have reported a ping over the last 28 days and qualify as active.
|
|
@ -0,0 +1,19 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.firefox_ios.app_store_choice_screen_selection`
|
||||
AS
|
||||
SELECT
|
||||
* EXCEPT (logical_date) REPLACE(DATE(`date`) AS `date`),
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_ios_derived.app_store_choice_screen_selection_v1`
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY
|
||||
`date`,
|
||||
territory,
|
||||
build,
|
||||
device,
|
||||
platform,
|
||||
release_type
|
||||
ORDER BY
|
||||
logical_date DESC
|
||||
) = 1
|
|
@ -0,0 +1,41 @@
|
|||
friendly_name: Apple App Store Choice Screen Selection (post-iOS 18.2 only)
|
||||
description: |
|
||||
Apple App Store Choice Screen Engagement (post-iOS 18.2 only)
|
||||
This report includes rates around user selection/egagement with our browser
|
||||
via the App Store Choice Screen.
|
||||
|
||||
Official docs: https://developer.apple.com/documentation/analytics-reports/browser-choice-screen-selection
|
||||
|
||||
owners:
|
||||
- kik@mozilla.com
|
||||
labels:
|
||||
schedule: daily
|
||||
incremental: true
|
||||
table_type: aggregate
|
||||
shredder_mitigation: false
|
||||
scheduling:
|
||||
dag_name: bqetl_firefox_ios
|
||||
depends_on_past: false
|
||||
arguments:
|
||||
- --date={{macros.ds_add(ds, -5)}}
|
||||
- --connect_app_id=989804926
|
||||
- --partition_field=logical_date
|
||||
|
||||
secrets:
|
||||
- deploy_target: CONNECT_ISSUER_ID
|
||||
key: bqetl_firefox_ios__app_store_connect_issuer_id
|
||||
- deploy_target: CONNECT_KEY_ID
|
||||
key: bqetl_firefox_ios__app_store_connect_key_id
|
||||
- deploy_target: CONNECT_KEY
|
||||
key: bqetl_firefox_ios__app_store_connect_key
|
||||
|
||||
date_partition_offset: -5
|
||||
retry_delay: 30m
|
||||
retries: 2
|
||||
email_on_retry: false
|
||||
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: logical_date
|
||||
require_partition_filter: false
|
|
@ -0,0 +1,260 @@
|
|||
"""Fetch Analytics Report for Choice Screen Engagement in the App store via App Store Connect."""
|
||||
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
from sys import exit
|
||||
|
||||
import jwt
|
||||
import pandas as pd
|
||||
import requests
|
||||
from google.cloud import bigquery
|
||||
|
||||
from bigquery_etl.schema import Schema
|
||||
|
||||
#
|
||||
# https://developer.apple.com/documentation/AppStoreConnectAPI/downloading-analytics-reports
|
||||
CONNECT_ISSUER_ID = os.environ.get("CONNECT_ISSUER_ID")
|
||||
CONNECT_KEY_ID = os.environ.get("CONNECT_KEY_ID")
|
||||
CONNECT_KEY = os.environ.get("CONNECT_KEY")
|
||||
|
||||
HOST = "https://api.appstoreconnect.apple.com"
|
||||
REPORT_TITLE = "Browser Choice Screen Selection"
|
||||
REPORT_DATA_DELIMITER = ","
|
||||
|
||||
SCHEMA_FILE = Path(__file__).parent / "schema.yaml"
|
||||
SCHEMA = Schema.from_schema_file(SCHEMA_FILE).to_bigquery_schema()
|
||||
|
||||
|
||||
def generate_jwt_token(issuer_id, key_id, private_key):
|
||||
"""Generate jtw token to be used for API authentication."""
|
||||
payload = {
|
||||
"iss": issuer_id,
|
||||
"iat": int(time.time()),
|
||||
"exp": int(time.time()) + 20 * 60, # Token valid for 20 minutes
|
||||
"aud": "appstoreconnect-v1",
|
||||
}
|
||||
algorithm = "ES256"
|
||||
|
||||
return jwt.encode(
|
||||
payload,
|
||||
private_key.replace("\\n", "\n"),
|
||||
algorithm=algorithm,
|
||||
headers={"kid": key_id, "alg": algorithm, "typ": "JWT"},
|
||||
)
|
||||
|
||||
|
||||
def download_gz_file(url, target_file_path):
|
||||
"""Download and decompress a gz file."""
|
||||
with requests.get(url, stream=True) as response:
|
||||
response.raise_for_status()
|
||||
file_content = io.BytesIO()
|
||||
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
if chunk: # Filter out keep-alive new chunks
|
||||
file_content.write(chunk)
|
||||
|
||||
# # Move the file content pointer to the beginning for reading
|
||||
file_content.seek(0)
|
||||
|
||||
# # Decompress from in-memory buffer to file
|
||||
with gzip.GzipFile(fileobj=file_content, mode="rb") as f_in:
|
||||
with open(target_file_path, "wb") as f_out:
|
||||
f_out.write(f_in.read())
|
||||
|
||||
|
||||
def api_call(url, jwt_token):
|
||||
"""Handle making API requests to App Store Connect."""
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
response = requests.get(url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
return response.json()
|
||||
|
||||
|
||||
def get_paginated_data(url, jwt_token):
|
||||
"""Get data from all result pages."""
|
||||
data = []
|
||||
while True:
|
||||
result = api_call(url, jwt_token)
|
||||
data.extend(result["data"])
|
||||
if next := result["links"].get("next"):
|
||||
url = next
|
||||
else:
|
||||
return data
|
||||
|
||||
|
||||
def fetch_app_reports(jwt_token, app_id):
|
||||
"""Retrieve the report requests for the given app."""
|
||||
endpoint = f"apps/{app_id}/analyticsReportRequests"
|
||||
response_data = api_call(f"{HOST}/v1/{endpoint}", jwt_token)
|
||||
return response_data.get("data")
|
||||
|
||||
|
||||
def fetch_report_instances(jwt_token, report_id):
|
||||
"""Retrieve a list of report instances from the App Store."""
|
||||
endpoint = f"v1/analyticsReportInstances/{report_id}/segments"
|
||||
response_data = api_call(f"{HOST}/{endpoint}", jwt_token)
|
||||
return response_data.get("data")
|
||||
|
||||
|
||||
def fetch_report_data(app_id, date, jwt_token, target_file_path):
|
||||
"""Fetch and prints the report data."""
|
||||
# "ONGOING" provides current data and generates reports daily, weekly and monthly.
|
||||
# assumed there is only 1 ongoing report request at any given time
|
||||
ongoing_analytics_report_request = list(
|
||||
filter(
|
||||
lambda x: x["attributes"]["accessType"] == "ONGOING",
|
||||
fetch_app_reports(jwt_token, app_id),
|
||||
)
|
||||
)[0]
|
||||
|
||||
analytics_reports = get_paginated_data(
|
||||
ongoing_analytics_report_request["relationships"]["reports"]["links"][
|
||||
"related"
|
||||
],
|
||||
jwt_token,
|
||||
)
|
||||
|
||||
# for now we just assume there is only one ongoing browser choice screen report
|
||||
browser_choice_screen_analytics_report = list(
|
||||
filter(
|
||||
lambda x: x["attributes"]["name"] == REPORT_TITLE,
|
||||
analytics_reports,
|
||||
)
|
||||
)[0]
|
||||
|
||||
report_instances = api_call(
|
||||
browser_choice_screen_analytics_report["relationships"]["instances"]["links"][
|
||||
"related"
|
||||
],
|
||||
jwt_token,
|
||||
)["data"]
|
||||
|
||||
daily_report_instances = list(
|
||||
filter(lambda x: x["attributes"]["granularity"] == "DAILY", report_instances)
|
||||
)
|
||||
|
||||
try:
|
||||
specific_date_report_instance = list(
|
||||
filter(
|
||||
lambda x: x["attributes"]["processingDate"] == date,
|
||||
daily_report_instances,
|
||||
)
|
||||
)[0]
|
||||
except IndexError:
|
||||
logging.error(
|
||||
f"It appears that report: `{REPORT_TITLE}` for date: `{date}` does not exist, potentially it has not yet been made available."
|
||||
)
|
||||
exit(1)
|
||||
|
||||
analytics_report_segments = api_call(
|
||||
specific_date_report_instance["relationships"]["segments"]["links"]["related"],
|
||||
jwt_token,
|
||||
)["data"][0]
|
||||
|
||||
checksum = analytics_report_segments["attributes"]["checksum"]
|
||||
|
||||
response = requests.get(analytics_report_segments["attributes"]["url"])
|
||||
response.raise_for_status()
|
||||
|
||||
download_gz_file(
|
||||
analytics_report_segments["attributes"]["url"],
|
||||
target_file_path=target_file_path,
|
||||
)
|
||||
|
||||
return target_file_path, checksum
|
||||
|
||||
|
||||
def upload_to_bigquery(
|
||||
local_file_path, project, dataset, table_name, date, partition_field
|
||||
):
|
||||
"""Upload the data to bigquery."""
|
||||
df = pd.read_csv(local_file_path, delimiter=REPORT_DATA_DELIMITER)
|
||||
df[partition_field] = date
|
||||
df.columns = [x.lower().replace(" ", "_") for x in df.columns]
|
||||
|
||||
df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d")
|
||||
df[partition_field] = pd.to_datetime(df["logical_date"], format="%Y-%m-%d")
|
||||
|
||||
job_config = bigquery.LoadJobConfig(
|
||||
create_disposition=bigquery.CreateDisposition.CREATE_IF_NEEDED,
|
||||
write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
|
||||
schema=SCHEMA,
|
||||
)
|
||||
|
||||
client = bigquery.Client(project)
|
||||
destination = f"{project}.{dataset}.{table_name}${date.replace('-', '')}"
|
||||
|
||||
job = client.load_table_from_dataframe(df, destination, job_config=job_config)
|
||||
|
||||
while client.get_job(job.job_id, location=job.location).state != "DONE":
|
||||
logging.info("Waiting for the bq load job to be done, job_id: %s." % job.job_id)
|
||||
time.sleep(5)
|
||||
|
||||
return job.result(), destination
|
||||
|
||||
|
||||
def main():
|
||||
"""Input data, call functions, get stuff done."""
|
||||
parser = ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--date", required=True)
|
||||
parser.add_argument(
|
||||
"--connect_app_id", required=True, help="App Store Connect App ID"
|
||||
)
|
||||
parser.add_argument("--project", default="moz-fx-data-shared-prod")
|
||||
parser.add_argument("--dataset", default="firefox_ios_derived")
|
||||
parser.add_argument("--table", default="app_store_choice_screen_selection_v1")
|
||||
parser.add_argument("--partition_field", default="logical_date")
|
||||
parser.add_argument(
|
||||
"--connect_issuer_id",
|
||||
default=CONNECT_ISSUER_ID,
|
||||
help="App Store Connect Issuer ID",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--connect_key_id", default=CONNECT_KEY_ID, help="App Store Connect Key ID"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--connect_private_key",
|
||||
default=CONNECT_KEY,
|
||||
help="Private key",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
jwt_token = generate_jwt_token(
|
||||
args.connect_issuer_id, args.connect_key_id, args.connect_private_key
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile() as temp_file:
|
||||
report_file, _ = fetch_report_data(
|
||||
app_id=args.connect_app_id,
|
||||
date=args.date,
|
||||
jwt_token=jwt_token,
|
||||
target_file_path=temp_file.name,
|
||||
)
|
||||
|
||||
logging.info("Report file downloaded: %s" % report_file)
|
||||
|
||||
_, destination_table = upload_to_bigquery(
|
||||
temp_file.name,
|
||||
args.project,
|
||||
args.dataset,
|
||||
args.table,
|
||||
args.date,
|
||||
args.partition_field,
|
||||
)
|
||||
|
||||
logging.info("BigQuery table has been created: %s" % destination_table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
|
@ -0,0 +1,92 @@
|
|||
fields:
|
||||
|
||||
- mode: NULLABLE
|
||||
name: date
|
||||
type: DATETIME
|
||||
description: |
|
||||
Date on which the event occurred. For weekly and monthly instances,
|
||||
this column represents the first day of the week and month, respectively.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: territory
|
||||
type: STRING
|
||||
description: |
|
||||
The user’s region code as set in Settings > General > Language & Region. \
|
||||
This may not correspond to the user’s Apple Account or App Store storefront.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: build
|
||||
type: STRING
|
||||
description: |
|
||||
Build of device on which event occurred.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: device
|
||||
type: STRING
|
||||
description: |
|
||||
The device on which the event occurred.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: platform
|
||||
type: STRING
|
||||
description: |
|
||||
OS version on the device on which the event occurred
|
||||
|
||||
- mode: NULLABLE
|
||||
name: release_type
|
||||
type: STRING
|
||||
description: |
|
||||
Type of software release.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: selection_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app \
|
||||
as default among all unique devices that chose a default.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: new_install_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app \
|
||||
as default without having it installed already, among \
|
||||
all unique devices that chose your browser app as a default.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: existing_install_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app \
|
||||
as default and already had it installed, among \
|
||||
all unique devices that chose your browser app as a default.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: product_page_conversion_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app as default \
|
||||
after viewing its product page, \
|
||||
among all unique devices that viewed your browser app’s product page.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: informed_selection_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app \
|
||||
as default that had previously viewed its product page, \
|
||||
among all unique devices that chose your browser app as a default.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: direct_selection_rate
|
||||
type: FLOAT
|
||||
description: |
|
||||
Percentage of unique devices that set your browser app \
|
||||
as default without viewing its product page, \
|
||||
among all unique devices that chose your browser app as a default.
|
||||
|
||||
- mode: NULLABLE
|
||||
name: logical_date
|
||||
type: DATETIME
|
||||
description: |
|
||||
Logical date used by the date for processing.
|
|
@ -0,0 +1,20 @@
|
|||
friendly_name: Feature Usage Metrics (Firefox iOS)
|
||||
description: |-
|
||||
Metrics from metrics pings for mobile feature usage dashboards (iOS)
|
||||
owners:
|
||||
- rzhao@mozilla.com
|
||||
- vsabino@mozilla.com
|
||||
labels:
|
||||
application: firefox_ios
|
||||
incremental: true
|
||||
owner1: rzhao
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_mobile_feature_usage
|
||||
bigquery:
|
||||
time_partitioning:
|
||||
type: day
|
||||
field: submission_date
|
||||
require_partition_filter: true
|
||||
expiration_days: null
|
|
@ -0,0 +1,316 @@
|
|||
WITH baseline_clients AS (
|
||||
SELECT
|
||||
submission_date AS dau_date,
|
||||
client_id
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_ios.baseline_clients_daily`
|
||||
WHERE
|
||||
submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
|
||||
AND durations > 0
|
||||
AND LOWER(COALESCE(isp, "")) <> "browserstack"
|
||||
),
|
||||
client_attribution AS (
|
||||
SELECT
|
||||
client_id,
|
||||
adjust_network,
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_ios.attribution_clients`
|
||||
),
|
||||
metrics_dau AS (
|
||||
-- assign a DAU date for each metric ping while keeping it de-duplicated
|
||||
SELECT
|
||||
m.*,
|
||||
MAX(dau_date) OVER (PARTITION BY document_id) AS dau_date
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.firefox_ios.metrics` m
|
||||
JOIN
|
||||
baseline_clients
|
||||
ON client_info.client_id = client_id
|
||||
-- offset by at least one to reflect metrics ping design considerations
|
||||
AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
|
||||
BETWEEN 1
|
||||
AND 4
|
||||
WHERE
|
||||
DATE(submission_timestamp)
|
||||
BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
|
||||
AND @submission_date
|
||||
QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY submission_timestamp) = 1
|
||||
),
|
||||
metric_ping_clients_feature_usage AS (
|
||||
SELECT
|
||||
dau_date,
|
||||
client_info.client_id,
|
||||
ARRAY_AGG(normalized_channel ORDER BY submission_timestamp DESC)[SAFE_OFFSET(0)] AS channel,
|
||||
ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp DESC)[
|
||||
SAFE_OFFSET(0)
|
||||
] AS country,
|
||||
IF(SUM(metrics.counter.app_opened_as_default_browser) > 0, TRUE, FALSE) AS is_default_browser,
|
||||
--Credential Management: Logins
|
||||
SUM(COALESCE(metrics.counter.logins_deleted, 0)) AS logins_deleted,
|
||||
SUM(COALESCE(metrics.counter.logins_modified, 0)) AS logins_modified,
|
||||
SUM(COALESCE(metrics.counter.logins_saved, 0)) AS logins_saved,
|
||||
MAX(COALESCE(metrics.quantity.logins_saved_all, 0)) AS logins_saved_all,
|
||||
--Credential Management: Credit Cards
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.credit_card_autofill_enabled, FALSE)) AS INT64
|
||||
) AS credit_card_autofill_enabled,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.credit_card_sync_enabled, FALSE)) AS INT64
|
||||
) AS credit_card_sync_enabled,
|
||||
SUM(COALESCE(metrics.counter.credit_card_deleted, 0)) AS credit_card_deleted,
|
||||
SUM(COALESCE(metrics.counter.credit_card_modified, 0)) AS credit_card_modified,
|
||||
SUM(COALESCE(metrics.counter.credit_card_saved, 0)) AS credit_card_saved,
|
||||
MAX(COALESCE(metrics.quantity.credit_card_saved_all, 0)) AS credit_card_saved_all,
|
||||
--Bookmark
|
||||
SUM(COALESCE(bookmarks_add_table.value, 0)) AS bookmarks_add,
|
||||
SUM(COALESCE(bookmarks_delete_table.value, 0)) AS bookmarks_delete,
|
||||
SUM(COALESCE(bookmarks_edit_table.value, 0)) AS bookmarks_edit,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.bookmarks_has_mobile_bookmarks, FALSE)) AS INT64
|
||||
) AS has_mobile_bookmarks,
|
||||
MAX(COALESCE(metrics.quantity.bookmarks_mobile_bookmarks_count, 0)) AS mobile_bookmarks_count,
|
||||
SUM(COALESCE(bookmarks_open_table.value, 0)) AS bookmarks_open,
|
||||
SUM(COALESCE(bookmarks_view_list_table.value, 0)) AS bookmarks_view_list,
|
||||
--FxA
|
||||
SUM(COALESCE(metrics.counter.sync_create_account_pressed, 0)) AS sync_create_account_pressed,
|
||||
SUM(COALESCE(metrics.counter.sync_open_tab, 0)) AS sync_open_tab,
|
||||
SUM(COALESCE(metrics.counter.sync_sign_in_sync_pressed, 0)) AS sync_sign_in_sync_pressed,
|
||||
--Privacy
|
||||
MAX(COALESCE(metrics.quantity.tabs_private_tabs_quantity, 0)) AS tabs_private_tabs_quantity,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_close_private_tabs, FALSE)) AS INT64
|
||||
) AS preferences_close_private_tabs,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.tracking_protection_enabled, FALSE)) AS INT64
|
||||
) AS tracking_protection_enabled,
|
||||
CAST(
|
||||
MAX(COALESCE(LOWER(metrics.string.tracking_protection_strength) = "strict", FALSE)) AS INT64
|
||||
) AS tracking_protection_strict_enabled,
|
||||
--Tab Count
|
||||
MAX(COALESCE(metrics.quantity.tabs_normal_tabs_quantity, 0)) AS tabs_normal_tabs_quantity,
|
||||
MAX(COALESCE(metrics.quantity.tabs_inactive_tabs_count, 0)) AS tabs_inactive_tabs_count,
|
||||
--Default Browser
|
||||
SUM(
|
||||
COALESCE(metrics.counter.app_opened_as_default_browser, 0)
|
||||
) AS app_opened_as_default_browser,
|
||||
SUM(
|
||||
COALESCE(metrics.counter.settings_menu_set_as_default_browser_pressed, 0)
|
||||
) AS settings_menu_set_as_default_browser_pressed,
|
||||
--Notification
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_sync_notifs, FALSE)) AS INT64
|
||||
) AS preferences_sync_notifs,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_tips_and_features_notifs, FALSE)) AS INT64
|
||||
) AS preferences_tips_and_features_notifs,
|
||||
--Customize Home
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_jump_back_in, FALSE)) AS INT64
|
||||
) AS preferences_jump_back_in,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_recently_visited, FALSE)) AS INT64
|
||||
) AS preferences_recently_visited,
|
||||
CAST(
|
||||
MAX(COALESCE(metrics.boolean.preferences_recently_saved, FALSE)) AS INT64
|
||||
) AS preferences_recently_saved,
|
||||
CAST(MAX(COALESCE(metrics.boolean.preferences_pocket, FALSE)) AS INT64) AS preferences_pocket,
|
||||
SUM(COALESCE(metrics.counter.app_menu_customize_homepage, 0)) AS app_menu_customize_homepage,
|
||||
SUM(
|
||||
COALESCE(metrics.counter.firefox_home_page_customize_homepage_button, 0)
|
||||
) AS firefox_home_page_customize_homepage_button,
|
||||
--Address
|
||||
MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS addresses_saved_all
|
||||
FROM
|
||||
metrics_dau
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.bookmarks_add) AS bookmarks_add_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.bookmarks_delete) AS bookmarks_delete_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.bookmarks_edit) AS bookmarks_edit_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.bookmarks_open) AS bookmarks_open_table
|
||||
LEFT JOIN
|
||||
UNNEST(metrics.labeled_counter.bookmarks_view_list) AS bookmarks_view_list_table
|
||||
GROUP BY
|
||||
dau_date,
|
||||
client_id
|
||||
)
|
||||
-- Aggregated feature usage
|
||||
SELECT
|
||||
@submission_date AS submission_date,
|
||||
dau_date AS metric_date,
|
||||
COUNT(DISTINCT client_id) AS clients,
|
||||
channel,
|
||||
country,
|
||||
adjust_network,
|
||||
is_default_browser,
|
||||
/*Logins*/
|
||||
-- logins_deleted
|
||||
COUNT(DISTINCT IF(logins_deleted > 0, client_id, NULL)) AS logins_deleted_users,
|
||||
SUM(logins_deleted) AS logins_deleted,
|
||||
-- logins_modified
|
||||
COUNT(DISTINCT IF(logins_modified > 0, client_id, NULL)) AS logins_modified_users,
|
||||
SUM(logins_modified) AS logins_modified,
|
||||
-- logins_saved
|
||||
COUNT(DISTINCT IF(logins_saved > 0, client_id, NULL)) AS logins_saved_users,
|
||||
SUM(logins_saved) AS logins_saved,
|
||||
-- logins_saved_all
|
||||
COUNT(DISTINCT IF(logins_saved_all > 0, client_id, NULL)) AS logins_saved_all_users,
|
||||
SUM(logins_saved_all) AS logins_saved_all,
|
||||
/*Credit Card*/
|
||||
--credit card autofill enabled
|
||||
COUNT(
|
||||
DISTINCT IF(credit_card_autofill_enabled > 0, client_id, NULL)
|
||||
) AS credit_card_autofill_enabled_users,
|
||||
SUM(credit_card_autofill_enabled) AS credit_card_autofill_enabled,
|
||||
--credit_card_sync_enabled
|
||||
COUNT(
|
||||
DISTINCT IF(credit_card_sync_enabled > 0, client_id, NULL)
|
||||
) AS credit_card_sync_enabled_users,
|
||||
SUM(credit_card_sync_enabled) AS credit_card_sync_enabled,
|
||||
--credit_card_deleted
|
||||
COUNT(DISTINCT IF(credit_card_deleted > 0, client_id, NULL)) AS credit_card_deleted_users,
|
||||
SUM(credit_card_deleted) AS credit_card_deleted,
|
||||
--credit_card_modified
|
||||
COUNT(DISTINCT IF(credit_card_modified > 0, client_id, NULL)) AS credit_card_modified_users,
|
||||
SUM(credit_card_modified) AS credit_card_modified,
|
||||
--credit_card_saved
|
||||
COUNT(DISTINCT IF(credit_card_saved > 0, client_id, NULL)) AS credit_card_saved_users,
|
||||
SUM(credit_card_saved) AS credit_card_saved,
|
||||
--credit_card_saved_all
|
||||
COUNT(DISTINCT IF(credit_card_saved_all > 0, client_id, NULL)) AS credit_card_saved_all_users,
|
||||
SUM(credit_card_saved_all) AS credit_card_saved_all,
|
||||
/*Bookmark*/
|
||||
--bookmarks_add
|
||||
COUNT(DISTINCT IF(bookmarks_add > 0, client_id, NULL)) AS bookmarks_add_users,
|
||||
SUM(bookmarks_add) AS bookmarks_add,
|
||||
-- Bookmarks Delete
|
||||
COUNT(DISTINCT IF(bookmarks_delete > 0, client_id, NULL)) AS bookmarks_delete_users,
|
||||
SUM(bookmarks_delete) AS bookmarks_delete,
|
||||
-- Bookmarks Edit
|
||||
COUNT(DISTINCT IF(bookmarks_edit > 0, client_id, NULL)) AS bookmarks_edit_users,
|
||||
SUM(bookmarks_edit) AS bookmarks_edit,
|
||||
-- Has Mobile Bookmarks
|
||||
COUNT(DISTINCT IF(has_mobile_bookmarks > 0, client_id, NULL)) AS has_mobile_bookmarks_users,
|
||||
SUM(has_mobile_bookmarks) AS has_mobile_bookmarks,
|
||||
-- Mobile Bookmarks Count
|
||||
COUNT(DISTINCT IF(mobile_bookmarks_count > 0, client_id, NULL)) AS mobile_bookmarks_count_users,
|
||||
SUM(mobile_bookmarks_count) AS mobile_bookmarks_count,
|
||||
-- Bookmarks Open
|
||||
COUNT(DISTINCT IF(bookmarks_open > 0, client_id, NULL)) AS bookmarks_open_users,
|
||||
SUM(bookmarks_open) AS bookmarks_open,
|
||||
-- Bookmarks View List
|
||||
COUNT(DISTINCT IF(bookmarks_view_list > 0, client_id, NULL)) AS bookmarks_view_list_users,
|
||||
SUM(bookmarks_view_list) AS bookmarks_view_list,
|
||||
/*FxA*/
|
||||
--sync_create_account_pressed
|
||||
COUNT(
|
||||
DISTINCT IF(sync_create_account_pressed > 0, client_id, NULL)
|
||||
) AS sync_create_account_pressed_users,
|
||||
SUM(sync_create_account_pressed) AS sync_create_account_pressed,
|
||||
--sync_open_tab
|
||||
COUNT(DISTINCT IF(sync_open_tab > 0, client_id, NULL)) AS sync_open_tab_users,
|
||||
SUM(sync_open_tab) AS sync_open_tab,
|
||||
--sync_sign_in_sync_pressed
|
||||
COUNT(
|
||||
DISTINCT IF(sync_sign_in_sync_pressed > 0, client_id, NULL)
|
||||
) AS sync_sign_in_sync_pressed_users,
|
||||
SUM(sync_sign_in_sync_pressed) AS sync_sign_in_sync_pressed,
|
||||
/*Privacy*/
|
||||
--tabs_private_tabs_quantity
|
||||
COUNT(
|
||||
DISTINCT IF(tabs_private_tabs_quantity > 0, client_id, NULL)
|
||||
) AS tabs_private_tabs_quantity_users,
|
||||
SUM(tabs_private_tabs_quantity) AS tabs_private_tabs_quantity,
|
||||
-- Preferences Close Private Tabs
|
||||
COUNT(
|
||||
DISTINCT IF(preferences_close_private_tabs > 0, client_id, NULL)
|
||||
) AS preferences_close_private_tabs_users,
|
||||
SUM(preferences_close_private_tabs) AS preferences_close_private_tabs,
|
||||
-- Tracking Protection Enabled
|
||||
COUNT(
|
||||
DISTINCT IF(tracking_protection_enabled > 0, client_id, NULL)
|
||||
) AS tracking_protection_enabled_users,
|
||||
SUM(tracking_protection_enabled) AS tracking_protection_enabled,
|
||||
-- Tracking Protection Strict
|
||||
COUNT(
|
||||
DISTINCT IF(tracking_protection_strict_enabled > 0, client_id, NULL)
|
||||
) AS tracking_protection_strict_users,
|
||||
SUM(tracking_protection_strict_enabled) AS tracking_protection_strict,
|
||||
/*Tab Count*/
|
||||
--tabs_normal_tabs_quantity
|
||||
COUNT(
|
||||
DISTINCT IF(tabs_normal_tabs_quantity > 0, client_id, NULL)
|
||||
) AS tabs_normal_tabs_quantity_users,
|
||||
SUM(tabs_normal_tabs_quantity) AS tabs_normal_tabs_quantity,
|
||||
--tabs_inactive_tabs_count
|
||||
COUNT(
|
||||
DISTINCT IF(tabs_inactive_tabs_count > 0, client_id, NULL)
|
||||
) AS tabs_inactive_tabs_count_users,
|
||||
SUM(tabs_inactive_tabs_count) AS tabs_inactive_tabs_count,
|
||||
/*Default Browser*/
|
||||
--app_opened_as_default_browser
|
||||
COUNT(
|
||||
DISTINCT IF(app_opened_as_default_browser > 0, client_id, NULL)
|
||||
) AS app_opened_as_default_browser_users,
|
||||
SUM(app_opened_as_default_browser) AS app_opened_as_default_browser,
|
||||
-- settings_menu_set_as_default_browser_pressed
|
||||
COUNT(
|
||||
DISTINCT IF(settings_menu_set_as_default_browser_pressed > 0, client_id, NULL)
|
||||
) AS settings_menu_set_as_default_browser_pressed_users,
|
||||
SUM(settings_menu_set_as_default_browser_pressed) AS settings_menu_set_as_default_browser_pressed,
|
||||
/*Notification*/
|
||||
--preferences_sync_notifs
|
||||
COUNT(DISTINCT IF(preferences_sync_notifs > 0, client_id, NULL)) AS preferences_sync_notifs_users,
|
||||
SUM(preferences_sync_notifs) AS preferences_sync_notifs,
|
||||
-- preferences_tips_and_features_notifs
|
||||
COUNT(
|
||||
DISTINCT IF(preferences_tips_and_features_notifs > 0, client_id, NULL)
|
||||
) AS preferences_tips_and_features_notifs_users,
|
||||
SUM(preferences_tips_and_features_notifs) AS preferences_tips_and_features_notifs,
|
||||
/*Customize Home*/
|
||||
--preferences_jump_back_in
|
||||
COUNT(
|
||||
DISTINCT IF(preferences_jump_back_in > 0, client_id, NULL)
|
||||
) AS preferences_jump_back_in_users,
|
||||
SUM(preferences_jump_back_in) AS preferences_jump_back_in,
|
||||
-- Preferences Recently Visited
|
||||
COUNT(
|
||||
DISTINCT IF(preferences_recently_visited > 0, client_id, NULL)
|
||||
) AS preferences_recently_visited_users,
|
||||
SUM(preferences_recently_visited) AS preferences_recently_visited,
|
||||
-- Preferences Recently Saved
|
||||
COUNT(
|
||||
DISTINCT IF(preferences_recently_saved > 0, client_id, NULL)
|
||||
) AS preferences_recently_saved_users,
|
||||
SUM(preferences_recently_saved) AS preferences_recently_saved,
|
||||
-- Preferences Pocket
|
||||
COUNT(DISTINCT IF(preferences_pocket > 0, client_id, NULL)) AS preferences_pocket_users,
|
||||
SUM(preferences_pocket) AS preferences_pocket,
|
||||
-- App Menu Customize Homepage
|
||||
COUNT(
|
||||
DISTINCT IF(app_menu_customize_homepage > 0, client_id, NULL)
|
||||
) AS app_menu_customize_homepage_users,
|
||||
SUM(app_menu_customize_homepage) AS app_menu_customize_homepage,
|
||||
-- Firefox Home Page Customize Homepage Button
|
||||
COUNT(
|
||||
DISTINCT IF(firefox_home_page_customize_homepage_button > 0, client_id, NULL)
|
||||
) AS firefox_home_page_customize_homepage_button_users,
|
||||
SUM(firefox_home_page_customize_homepage_button) AS firefox_home_page_customize_homepage_button,
|
||||
-- addresses_saved_all
|
||||
COUNT(DISTINCT IF(addresses_saved_all > 0, client_id, NULL)) AS addresses_saved_all_users,
|
||||
SUM(addresses_saved_all) AS addresses_saved_all
|
||||
FROM
|
||||
metric_ping_clients_feature_usage
|
||||
LEFT JOIN
|
||||
client_attribution
|
||||
USING (client_id)
|
||||
GROUP BY
|
||||
submission_date,
|
||||
metric_date,
|
||||
channel,
|
||||
country,
|
||||
adjust_network,
|
||||
is_default_browser
|
|
@ -0,0 +1,325 @@
|
|||
fields:
|
||||
- name: submission_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: Airflow's internal execution_date.
|
||||
- name: metric_date
|
||||
type: DATE
|
||||
mode: NULLABLE
|
||||
description: reporting date for feature usage, aligned with DAU.
|
||||
- name: clients
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: number of clients accounted for in the aggregation.
|
||||
- name: channel
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Channel
|
||||
- name: country
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Country
|
||||
- name: adjust_network
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
description: Adjust Network
|
||||
- name: is_default_browser
|
||||
type: BOOLEAN
|
||||
mode: NULLABLE
|
||||
description: Is Default Browser
|
||||
- name: logins_deleted_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Deleted Users
|
||||
- name: logins_deleted
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Deleted
|
||||
- name: logins_modified_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Modified Users
|
||||
- name: logins_modified
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Modified
|
||||
- name: logins_saved_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Saved Users
|
||||
- name: logins_saved
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Saved
|
||||
- name: logins_saved_all_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Saved All Users
|
||||
- name: logins_saved_all
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Logins Saved All
|
||||
- name: credit_card_autofill_enabled_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Autofill Enabled Users
|
||||
- name: credit_card_autofill_enabled
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Autofill Enabled
|
||||
- name: credit_card_sync_enabled_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Sync Enabled Users
|
||||
- name: credit_card_sync_enabled
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Sync Enabled
|
||||
- name: credit_card_deleted_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Deleted Users
|
||||
- name: credit_card_deleted
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Deleted
|
||||
- name: credit_card_modified_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Modified Users
|
||||
- name: credit_card_modified
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Modified
|
||||
- name: credit_card_saved_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Saved Users
|
||||
- name: credit_card_saved
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Saved
|
||||
- name: credit_card_saved_all_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Saved All Users
|
||||
- name: credit_card_saved_all
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Credit Card Saved All
|
||||
- name: bookmarks_add_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Add Users
|
||||
- name: bookmarks_add
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Add
|
||||
- name: bookmarks_delete_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Delete Users
|
||||
- name: bookmarks_delete
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Delete
|
||||
- name: bookmarks_edit_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Edit Users
|
||||
- name: bookmarks_edit
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Edit
|
||||
- name: has_mobile_bookmarks_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Has Mobile Bookmarks Users
|
||||
- name: has_mobile_bookmarks
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Has Mobile Bookmarks
|
||||
- name: mobile_bookmarks_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Mobile Bookmarks Count Users
|
||||
- name: mobile_bookmarks_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Mobile Bookmarks Count
|
||||
- name: bookmarks_open_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Open Users
|
||||
- name: bookmarks_open
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks Open
|
||||
- name: bookmarks_view_list_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks View List Users
|
||||
- name: bookmarks_view_list
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Bookmarks View List
|
||||
- name: sync_create_account_pressed_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Create Account Pressed Users
|
||||
- name: sync_create_account_pressed
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Create Account Pressed
|
||||
- name: sync_open_tab_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Open Tab Users
|
||||
- name: sync_open_tab
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Open Tab
|
||||
- name: sync_sign_in_sync_pressed_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Sign In Sync Pressed Users
|
||||
- name: sync_sign_in_sync_pressed
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Sync Sign In Sync Pressed
|
||||
- name: tabs_private_tabs_quantity_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Private Tabs Quantity Users
|
||||
- name: tabs_private_tabs_quantity
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Private Tabs Quantity
|
||||
- name: preferences_close_private_tabs_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Close Private Tabs Users
|
||||
- name: preferences_close_private_tabs
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Close Private Tabs
|
||||
- name: tracking_protection_enabled_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tracking Protection Enabled Users
|
||||
- name: tracking_protection_enabled
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tracking Protection Enabled
|
||||
- name: tracking_protection_strict_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tracking Protection Strict Users
|
||||
- name: tracking_protection_strict
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tracking Protection Strict
|
||||
- name: tabs_normal_tabs_quantity_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Normal Tabs Quantity Users
|
||||
- name: tabs_normal_tabs_quantity
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Normal Tabs Quantity
|
||||
- name: tabs_inactive_tabs_count_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Inactive Tabs Count Users
|
||||
- name: tabs_inactive_tabs_count
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Tabs Inactive Tabs Count
|
||||
- name: app_opened_as_default_browser_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: App Opened as Default Browser Users
|
||||
- name: app_opened_as_default_browser
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: App Opened as Default Browser
|
||||
- name: settings_menu_set_as_default_browser_pressed_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Settings Menu Set as Default Browser Pressed Users
|
||||
- name: settings_menu_set_as_default_browser_pressed
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Settings Menu Set as Default Browser Pressed
|
||||
- name: preferences_sync_notifs_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Sync Notifs Users
|
||||
- name: preferences_sync_notifs
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: preferences_sync_notifs
|
||||
- name: preferences_tips_and_features_notifs_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Tips & Features Notifs Users
|
||||
- name: preferences_tips_and_features_notifs
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Tips & Features Notifs
|
||||
- name: preferences_jump_back_in_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Jump Back in Users
|
||||
- name: preferences_jump_back_in
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Jump Back In
|
||||
- name: preferences_recently_visited_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Recently Visited Users
|
||||
- name: preferences_recently_visited
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Recently Visited
|
||||
- name: preferences_recently_saved_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Recently Saved Users
|
||||
- name: preferences_recently_saved
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Recently Saved
|
||||
- name: preferences_pocket_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Pocket Users
|
||||
- name: preferences_pocket
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Preferences Pocket
|
||||
- name: app_menu_customize_homepage_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: App Menu Customize Homepage Users
|
||||
- name: app_menu_customize_homepage
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: App Menu Customize Homepage
|
||||
- name: firefox_home_page_customize_homepage_button_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Firefox Home Page Customize Homepage Button Users
|
||||
- name: firefox_home_page_customize_homepage_button
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Firefox Home Page Customize Homepage Button
|
||||
- name: addresses_saved_all_users
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Saved All Users
|
||||
- name: addresses_saved_all
|
||||
type: INTEGER
|
||||
mode: NULLABLE
|
||||
description: Addresses Saved All
|
|
@ -35,10 +35,9 @@ client_search AS (
|
|||
USING (client_id)
|
||||
WHERE
|
||||
(submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
|
||||
AND normalized_app_name = 'Fennec'
|
||||
AND os = 'iOS'
|
||||
AND normalized_app_name_os = "Firefox iOS"
|
||||
GROUP BY
|
||||
1
|
||||
client_id
|
||||
),
|
||||
dou AS (
|
||||
SELECT
|
||||
|
@ -70,7 +69,7 @@ SELECT
|
|||
first_seen_date,
|
||||
submission_date,
|
||||
1 AS new_profile,
|
||||
CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS integer) AS activated
|
||||
CAST(days_2_7 > 1 AND COALESCE(search_count, 0) > 0 AS INTEGER) AS activated
|
||||
FROM
|
||||
dou
|
||||
INNER JOIN
|
||||
|
|
|
@ -27,8 +27,7 @@ client_search AS (
|
|||
`moz-fx-data-shared-prod.search.mobile_search_clients_daily`
|
||||
WHERE
|
||||
(submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
|
||||
AND os = 'iOS'
|
||||
AND normalized_app_name = 'Fennec'
|
||||
AND normalized_app_name_os = 'Firefox iOS'
|
||||
GROUP BY
|
||||
client_id
|
||||
),
|
||||
|
|
|
@ -1,61 +1,60 @@
|
|||
type: BIGCONFIG_FILE
|
||||
|
||||
tag_deployments:
|
||||
- collection:
|
||||
name: Operational Checks
|
||||
notification_channels:
|
||||
- slack: '#de-bigeye-triage'
|
||||
deployments:
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.date
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_region
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_language
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.ad_group
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.impressions
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.clicks
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.new_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.activated_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.repeat_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.week_4_retained_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.spend
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.lifetime_value
|
||||
metrics:
|
||||
- saved_metric_id: is_not_null
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
|
||||
metrics:
|
||||
- saved_metric_id: is_2_char_len
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.*
|
||||
metrics:
|
||||
- saved_metric_id: volume
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- saved_metric_id: freshness
|
||||
lookback:
|
||||
lookback_type: DATA_TIME
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
rct_overrides:
|
||||
- date
|
||||
- collection:
|
||||
name: Operational Checks
|
||||
notification_channels:
|
||||
- slack: '#de-bigeye-triage'
|
||||
deployments:
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.date
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_region
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_language
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.ad_group
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.impressions
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.clicks
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.new_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.activated_profiles
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.repeat_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.week_4_retained_users
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.spend
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.lifetime_value
|
||||
metrics:
|
||||
- saved_metric_id: is_not_null
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.campaign_country_code
|
||||
metrics:
|
||||
- saved_metric_id: is_2_char_len
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- column_selectors:
|
||||
- name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.google_ads_derived.android_app_campaign_stats_v1.*
|
||||
metrics:
|
||||
- saved_metric_id: volume
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
- saved_metric_id: freshness
|
||||
lookback:
|
||||
lookback_window:
|
||||
interval_type: DAYS
|
||||
interval_value: 28
|
||||
lookback_type: DATA_TIME
|
||||
rct_overrides:
|
||||
- date
|
||||
|
|
|
@ -12,6 +12,8 @@ owners:
|
|||
labels:
|
||||
incremental: true
|
||||
owner1: kwindau@mozilla.com
|
||||
table_type: aggregate
|
||||
shredder_mitigation: true
|
||||
scheduling:
|
||||
dag_name: bqetl_fivetran_google_ads
|
||||
depends_on_past: false
|
||||
|
|
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
friendly_name: Jira Service Desk Field Options
|
||||
description: |-
|
||||
Jira field options for Service Desk synced via Fivetran.
|
|
@ -0,0 +1,7 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.jira_service_desk.field_option`
|
||||
AS
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.field_option`
|
|
@ -1,7 +1,69 @@
|
|||
CREATE OR REPLACE VIEW
|
||||
`moz-fx-data-shared-prod.jira_service_desk.issue`
|
||||
AS
|
||||
WITH requests AS (
|
||||
-- Join requests related data
|
||||
SELECT
|
||||
* EXCEPT (id, description),
|
||||
id AS request_id,
|
||||
description AS request_description
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
* EXCEPT (_fivetran_synced)
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.request`
|
||||
) AS request
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
* EXCEPT (_fivetran_synced, service_desk_id)
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.request_type`
|
||||
) AS request_type
|
||||
ON request.request_type_id = request_type.id
|
||||
),
|
||||
fields AS (
|
||||
-- Join fields related data
|
||||
SELECT
|
||||
* EXCEPT (id, field_id, description),
|
||||
id AS field_id,
|
||||
description AS field_description
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
* EXCEPT (_fivetran_synced, _fivetran_deleted, name),
|
||||
name AS field_name
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.field`
|
||||
) AS field
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
* EXCEPT (_fivetran_synced)
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.issue_field_history`
|
||||
) AS field_history
|
||||
ON field_history.field_id = field.id
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT
|
||||
* EXCEPT (_fivetran_synced, id, name),
|
||||
id AS field_option_id,
|
||||
name AS field_option_name
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.field_option`
|
||||
) AS field_option
|
||||
ON field_history.value = SAFE_CAST(field_option.field_option_id AS STRING)
|
||||
)
|
||||
-- Join requests and fields data with issues
|
||||
SELECT
|
||||
*
|
||||
* EXCEPT (issue_id, parent_id)
|
||||
FROM
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.issue`
|
||||
`moz-fx-data-shared-prod.jira_service_desk_syndicate.issue` AS issue
|
||||
LEFT JOIN
|
||||
requests
|
||||
ON issue.id = requests.issue_id
|
||||
LEFT JOIN
|
||||
fields
|
||||
ON issue.id = fields.issue_id
|
||||
|
|
|
@ -32,6 +32,10 @@ class BigQueryAPI:
|
|||
bigquery.SchemaField("account_id", "STRING"),
|
||||
bigquery.SchemaField("account_status", "STRING"),
|
||||
bigquery.SchemaField("name", "STRING"),
|
||||
bigquery.SchemaField("time_zone", "STRING"),
|
||||
bigquery.SchemaField("email_address", "STRING"),
|
||||
bigquery.SchemaField("account_type", "STRING"),
|
||||
bigquery.SchemaField("locale", "STRING"),
|
||||
],
|
||||
autodetect=False,
|
||||
write_disposition=write_disposition,
|
||||
|
@ -44,6 +48,7 @@ class BigQueryAPI:
|
|||
job.result()
|
||||
|
||||
|
||||
|
||||
class JiraAPI:
|
||||
def __init__(self, args) -> None:
|
||||
self.logger = logging.getLogger(self.__class__.__name__)
|
||||
|
@ -56,7 +61,7 @@ class JiraAPI:
|
|||
self.base_jira_url = args.base_jira_url
|
||||
self.auth = HTTPBasicAuth(self.secrets_dict.get('jira_username'), self.secrets_dict.get('jira_token'))
|
||||
|
||||
def get_users_paged(self, max_results=300):
|
||||
def get_users_paged(self, max_results=500):
|
||||
startAt = 0
|
||||
headers = {"Accept": "application/json"}
|
||||
|
||||
|
@ -87,6 +92,10 @@ class JiraAPI:
|
|||
"active" if user.get("active", "") else "inactive"
|
||||
),
|
||||
"name": user.get("displayName", ""),
|
||||
"time_zone": user.get("timeZone", ""),
|
||||
"email_address": user.get("emailAddress", ""),
|
||||
"account_type": user.get("accountType", ""),
|
||||
"locale": user.get("locale", ""),
|
||||
}
|
||||
for user in users
|
||||
]
|
||||
|
|
|
@ -8,3 +8,15 @@ fields:
|
|||
- name: name
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: time_zone
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: email_address
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: account_type
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
- name: locale
|
||||
type: STRING
|
||||
mode: NULLABLE
|
||||
|
|
|
@ -15,6 +15,7 @@ syndication:
|
|||
- status
|
||||
- resolution
|
||||
- field
|
||||
- field_option
|
||||
- issue_field_history
|
||||
- issue_multiselect_history
|
||||
- status_category
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
friendly_name: Legal tables
|
||||
description: |-
|
||||
Reference data
|
||||
dataset_base_acl: derived
|
||||
user_facing: false
|
||||
labels: {}
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mozilla-confidential
|
|
@ -1,11 +0,0 @@
|
|||
friendly_name: mlhackweek-search
|
||||
# yamllint disable rule:line-length
|
||||
description: |-
|
||||
User-facing views related to document namespace mlhackweek-search; see https://github.com/mozilla-services/mozilla-pipeline-schemas/tree/generated-schemas/schemas/mlhackweek-search
|
||||
dataset_base_acl: view_restricted
|
||||
user_facing: true
|
||||
labels: {}
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mlhackweek-search
|
|
@ -1,11 +0,0 @@
|
|||
friendly_name: mlhackweek-search Derived
|
||||
# yamllint disable rule:line-length
|
||||
description: |-
|
||||
Derived tables related to document namespace mlhackweek-search, usually populated via queries defined in https://github.com/mozilla/bigquery-etl and managed by Airflow
|
||||
dataset_base_acl: derived_restricted
|
||||
user_facing: false
|
||||
labels: {}
|
||||
workgroup_access:
|
||||
- role: roles/bigquery.dataViewer
|
||||
members:
|
||||
- workgroup:mlhackweek-search
|
|
@ -8,6 +8,11 @@ SELECT
|
|||
reference_table_id,
|
||||
creation_date,
|
||||
task_duration,
|
||||
EXTRACT(HOUR FROM task_duration) * 3600 + EXTRACT(MINUTE FROM task_duration) * 60 + EXTRACT(
|
||||
SECOND
|
||||
FROM
|
||||
task_duration
|
||||
) AS task_duration_seconds,
|
||||
total_terabytes_processed,
|
||||
total_terabytes_billed,
|
||||
total_slot_ms,
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
friendly_name: Event Counts Glean
|
||||
description: |-
|
||||
Daily event counts for all Glean apps, derived from monitoring_derived.event_monitoring_aggregates_v1
|
||||
owners:
|
||||
- bewu@mozilla.com
|
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче