* "Ran YAMLlint on all yaml files" * "Moved product info metadata table to README file" * "Reformatted yaml lists" * "Updated line breaks so script runs" * "Updated line breaks so script runs" * "Undid line breaks" * "Created custom config file" * "Removed base document id" * "Undid line breaks" * "Reformatted code" * "Trimmed whitespace" * "Undid line break" * "Introduced newline" * "Trimmed whitespace" * "Added yamillint to config file" * "Added yamllint to config file" * "Moved up yamllint test" * "Trimmed whitespace" * "Trimmed whitespace" * "Trimmed whitespace" * "Trimmed whitespace" * "Removing hyphen to fix CI error" * "Indentation to remove CI error" * "Included yamllint install in build run" * "Added yamllint in requirements.txt and .in file" * "Moved install yamllint step to its own stage" * "Updated yamllint test" * "Updated circleci step" * "Reformatted code" * "Added yamllint to circleci steps" * "Added checkout block to yamllint step" * "Trimmed whitespace" * "Undid yamllint step" * "Specified directory name for yamllint test" * "Fixed yamlint errors" * "Fixed yamllint errors" * "Fixed yamllint errors" * "Fixed yamllint errors" * "Ignore pathway in linting" * "Added ignore venv pathway during linting" * "Updated ignore block" * "Updated ignore block" * "Removed ignore block" * "Updated ignore block" * "Indented base as a list" * "Indented base item" * Update tests/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_last_seen_v1/test_day_bit_shifting/expect.yaml Co-authored-by: Anthony Miyaguchi <acmiyaguchi@gmail.com> * "Resolved linting errors" * "Referenced tables put back on same line" * "Fixed linting error" * Update sql/moz-fx-data-shared-prod/account_ecosystem_derived/fxa_logging_users_daily_v1/metadata.yaml Co-authored-by: Anthony Miyaguchi <acmiyaguchi@gmail.com> * "Fixed linting error" Co-authored-by: Anthony Miyaguchi <acmiyaguchi@gmail.com>
This commit is contained in:
Rhys
GitHub
Родитель
415ee2fb62
Коммит
1ace0fe2b7
|
|
@ -1,32 +1,38 @@
|
||||||
|
---
|
||||||
version: 2
|
version: 2
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
docker: &docker
|
docker: &docker
|
||||||
- image: python:3.8
|
- image: python:3.8
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- restore_cache:
|
- restore_cache:
|
||||||
keys:
|
keys:
|
||||||
# when lock files change, use increasingly general patterns to restore cache
|
# when lock files change, use increasingly general
|
||||||
- &cache_key
|
# patterns to restore cache
|
||||||
python-packages-v1-{{ .Branch }}-{{ checksum "requirements.in" }}-{{ checksum "requirements.txt" }}
|
- &cache_key
|
||||||
- python-packages-v1-{{ .Branch }}-{{ checksum "requirements.in" }}-
|
# yamllint disable-line rule:line-length
|
||||||
- python-packages-v1-{{ .Branch }}-
|
python-packages-v1-{{ .Branch }}-{{ checksum "requirements.in" }}-{{ checksum "requirements.txt" }}
|
||||||
- python-packages-v1-
|
- python-packages-v1-{{ .Branch }}-{{ checksum "requirements.in" }}-
|
||||||
- &build
|
- python-packages-v1-{{ .Branch }}-
|
||||||
run:
|
- python-packages-v1-
|
||||||
name: Build
|
- &build
|
||||||
command: |
|
run:
|
||||||
python3.8 -m venv venv/
|
name: Build
|
||||||
venv/bin/pip install pip-tools
|
command: |
|
||||||
venv/bin/pip-sync
|
python3.8 -m venv venv/
|
||||||
- run:
|
venv/bin/pip install pip-tools
|
||||||
name: PyTest with linters
|
venv/bin/pip-sync
|
||||||
command: PATH="venv/bin:$PATH" script/entrypoint
|
- run:
|
||||||
- save_cache:
|
name: Yamllint Test
|
||||||
paths:
|
command: PATH="venv/bin:$PATH" yamllint -c .yamllint.yaml .
|
||||||
- venv/
|
- run:
|
||||||
key: *cache_key
|
name: PyTest with linters
|
||||||
|
command: PATH="venv/bin:$PATH" script/entrypoint
|
||||||
|
- save_cache:
|
||||||
|
paths:
|
||||||
|
- venv/
|
||||||
|
key: *cache_key
|
||||||
verify-format-sql:
|
verify-format-sql:
|
||||||
docker: *docker
|
docker: *docker
|
||||||
steps:
|
steps:
|
||||||
|
|
@ -39,7 +45,8 @@ jobs:
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- run:
|
- run:
|
||||||
name: Verify that requirements.txt contains the right dependencies for this python version
|
name: Verify that requirements.txt contains the right dependencies
|
||||||
|
for this python version
|
||||||
command: |
|
command: |
|
||||||
pip install pip-tools
|
pip install pip-tools
|
||||||
pip-compile --quiet --generate-hashes requirements.in
|
pip-compile --quiet --generate-hashes requirements.in
|
||||||
|
|
@ -62,28 +69,31 @@ jobs:
|
||||||
integration:
|
integration:
|
||||||
docker: *docker
|
docker: *docker
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- &skip_forked_pr
|
- &skip_forked_pr
|
||||||
run:
|
run:
|
||||||
name: Early return if this build is from a forked PR
|
name: Early return if this build is from a forked PR
|
||||||
command: |
|
command: |
|
||||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then
|
if [ -n "$CIRCLE_PR_NUMBER" ]; then
|
||||||
echo "Cannot pass creds to forked PRs, so marking this step successful"
|
echo "Cannot pass creds to forked PRs,
|
||||||
circleci step halt
|
so marking this step successful"
|
||||||
fi
|
circleci step halt
|
||||||
- *build
|
fi
|
||||||
- &pytest_integration_test
|
- *build
|
||||||
run:
|
- &pytest_integration_test
|
||||||
name: PyTest Integration Test
|
run:
|
||||||
# Google's client libraries will check for GOOGLE_APPLICATION_CREDENTIALS
|
name: PyTest Integration Test
|
||||||
# and use a file in that location for credentials if present;
|
# Google's client libraries will check for
|
||||||
# See https://cloud.google.com/docs/authentication/production
|
# GOOGLE_APPLICATION_CREDENTIALS
|
||||||
command: |
|
# and use a file in that location for credentials if present;
|
||||||
export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcp.json"
|
# See https://cloud.google.com/docs/authentication/production
|
||||||
echo "$GCLOUD_SERVICE_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS"
|
command: |
|
||||||
PATH="venv/bin:$PATH" script/entrypoint -m integration
|
export GOOGLE_APPLICATION_CREDENTIALS="/tmp/gcp.json"
|
||||||
|
echo "$GCLOUD_SERVICE_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS"
|
||||||
|
PATH="venv/bin:$PATH" script/entrypoint -m integration
|
||||||
validate-dags:
|
validate-dags:
|
||||||
# based on https://github.com/mozilla/telemetry-airflow/blob/master/.circleci/config.yml
|
# based on
|
||||||
|
# https://github.com/mozilla/telemetry-airflow/blob/master/.circleci/config.yml
|
||||||
machine:
|
machine:
|
||||||
image: ubuntu-1604:201903-01
|
image: ubuntu-1604:201903-01
|
||||||
docker_layer_caching: true
|
docker_layer_caching: true
|
||||||
|
|
@ -109,95 +119,101 @@ jobs:
|
||||||
verify-dags-up-to-date:
|
verify-dags-up-to-date:
|
||||||
docker: *docker
|
docker: *docker
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- *build
|
- *build
|
||||||
- run:
|
- run:
|
||||||
name: Generate DAGs
|
name: Generate DAGs
|
||||||
command: PATH="venv/bin:$PATH" script/generate_airflow_dags
|
command: PATH="venv/bin:$PATH" script/generate_airflow_dags
|
||||||
- run:
|
- run:
|
||||||
name: Verify that DAGs were correctly generated and are up-to-date
|
name: Verify that DAGs were correctly generated and are up-to-date
|
||||||
command: git diff --exit-code
|
command: git diff --exit-code
|
||||||
validate-docs:
|
validate-docs:
|
||||||
docker: *docker
|
docker: *docker
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- *build
|
- *build
|
||||||
- run:
|
- run:
|
||||||
name: Validate doc examples
|
name: Validate doc examples
|
||||||
command: PATH="venv/bin:$PATH" script/validate_docs
|
command: PATH="venv/bin:$PATH" script/validate_docs
|
||||||
docs:
|
docs:
|
||||||
docker: *docker
|
docker: *docker
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- &skip_forked_pr
|
- &skip_forked_pr
|
||||||
run:
|
run:
|
||||||
name: Early return if this build is from a forked PR
|
name: Early return if this build is from a forked PR
|
||||||
command: |
|
command: |
|
||||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then
|
if [ -n "$CIRCLE_PR_NUMBER" ]; then
|
||||||
echo "Cannot pass creds to forked PRs, so marking this step successful"
|
echo "Cannot pass creds to forked PRs,
|
||||||
circleci step halt
|
so marking this step successful"
|
||||||
fi
|
circleci step halt
|
||||||
- *build
|
fi
|
||||||
- run:
|
- *build
|
||||||
name: Install dependencies
|
- run:
|
||||||
command: pip install mkdocs markdown-include
|
name: Install dependencies
|
||||||
- add_ssh_keys:
|
command: pip install mkdocs markdown-include
|
||||||
fingerprints:
|
- add_ssh_keys:
|
||||||
"ab:b5:f7:55:92:0a:72:c4:63:0e:57:be:cd:66:32:53"
|
fingerprints: "ab:b5:f7:55:92:0a:72:c4:63:0e:57:be:cd:66:32:53"
|
||||||
- run:
|
- run:
|
||||||
name: Build and deploy docs
|
name: Build and deploy docs
|
||||||
command: |
|
command: |
|
||||||
PATH="venv/bin:$PATH" script/generate_docs --output_dir=generated_docs/
|
PATH="venv/bin:$PATH" script/generate_docs \
|
||||||
cd generated_docs/
|
--output_dir=generated_docs/
|
||||||
mkdocs gh-deploy -m "[ci skip] Deployed {sha} with MkDocs version: {version}"
|
cd generated_docs/
|
||||||
|
mkdocs gh-deploy -m "[ci skip] Deployed
|
||||||
|
{sha} with MkDocs version: {version}"
|
||||||
deploy:
|
deploy:
|
||||||
parameters:
|
parameters:
|
||||||
image:
|
image:
|
||||||
type: string
|
type: string
|
||||||
docker:
|
docker:
|
||||||
# bash required for step: Determine docker image name
|
# bash required for step: Determine docker image name
|
||||||
- image: relud/docker-bash
|
- image: relud/docker-bash
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
- checkout
|
||||||
- setup_remote_docker:
|
- setup_remote_docker:
|
||||||
docker_layer_caching: true
|
docker_layer_caching: true
|
||||||
- run:
|
- run:
|
||||||
name: Determine docker image name
|
name: Determine docker image name
|
||||||
command: echo 'IMAGE="${CIRCLE_PROJECT_USERNAME+$CIRCLE_PROJECT_USERNAME/}${CIRCLE_PROJECT_REPONAME:-bigquery-etl}:${CIRCLE_TAG:-latest}"' > $BASH_ENV
|
command:
|
||||||
- run:
|
# yamllint disable-line rule:line-length
|
||||||
name: Build docker image
|
echo 'IMAGE="${CIRCLE_PROJECT_USERNAME+$CIRCLE_PROJECT_USERNAME/} ${CIRCLE_PROJECT_REPONAME:-bigquery-etl}:${CIRCLE_TAG:-latest}"' > $BASH_ENV
|
||||||
command: docker build . --pull --tag "$IMAGE"
|
|
||||||
- run:
|
- run:
|
||||||
name: Deploy to Dockerhub
|
name: Build docker image
|
||||||
command: |
|
command: docker build . --pull --tag "$IMAGE"
|
||||||
echo "${DOCKER_PASS:?}" | docker login -u "${DOCKER_USER:?}" --password-stdin
|
- run:
|
||||||
docker push "$IMAGE"
|
name: Deploy to Dockerhub
|
||||||
|
command: |
|
||||||
|
echo "${DOCKER_PASS:?}" | \
|
||||||
|
docker login -u "${DOCKER_USER:?}" --password-stdin
|
||||||
|
docker push "$IMAGE"
|
||||||
|
|
||||||
workflows:
|
workflows:
|
||||||
version: 2
|
version: 2
|
||||||
build:
|
build:
|
||||||
jobs:
|
jobs:
|
||||||
- build:
|
- build:
|
||||||
context: data-eng-circleci-tests
|
context: data-eng-circleci-tests
|
||||||
- verify-format-sql
|
- verify-format-sql
|
||||||
- verify-requirements
|
- verify-requirements
|
||||||
- dry-run-sql
|
- dry-run-sql
|
||||||
- validate-metadata
|
- validate-metadata
|
||||||
- integration
|
- integration
|
||||||
- validate-dags
|
- validate-dags
|
||||||
- verify-dags-up-to-date
|
- verify-dags-up-to-date
|
||||||
- validate-docs
|
- validate-docs
|
||||||
- docs:
|
- docs:
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only: master
|
only: master
|
||||||
- deploy:
|
- deploy:
|
||||||
context: data-eng-bigquery-etl-dockerhub
|
context: data-eng-bigquery-etl-dockerhub
|
||||||
requires:
|
requires:
|
||||||
# can't run in parallel because CIRCLE_BUILD_NUM is same
|
# can't run in parallel because CIRCLE_BUILD_NUM is same
|
||||||
- build
|
- build
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only: master
|
only: master
|
||||||
tags:
|
tags:
|
||||||
only: /.*/
|
only: /.*/
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
env:
|
env:
|
||||||
browser: true
|
browser: true
|
||||||
commonjs: true
|
commonjs: true
|
||||||
|
|
@ -9,4 +10,4 @@ globals:
|
||||||
parserOptions:
|
parserOptions:
|
||||||
ecmaVersion: 2018
|
ecmaVersion: 2018
|
||||||
rules:
|
rules:
|
||||||
no-console: off
|
no-console: 0
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
|
---
|
||||||
version: 2
|
version: 2
|
||||||
updates:
|
updates:
|
||||||
- package-ecosystem: pip
|
- package-ecosystem: pip
|
||||||
directory: /
|
directory: /
|
||||||
schedule:
|
schedule:
|
||||||
interval: daily
|
interval: daily
|
||||||
reviewers:
|
reviewers:
|
||||||
- relud
|
- relud
|
||||||
- jklukas
|
- jklukas
|
||||||
labels:
|
labels:
|
||||||
- dependencies
|
- dependencies
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
---
|
||||||
|
rules:
|
||||||
|
line-length:
|
||||||
|
allow-non-breakable-words: true
|
||||||
|
allow-non-breakable-inline-mappings: true
|
||||||
|
|
||||||
|
ignore: |
|
||||||
|
venv/
|
||||||
138
dags.yaml
138
dags.yaml
|
|
@ -1,9 +1,15 @@
|
||||||
|
---
|
||||||
bqetl_error_aggregates:
|
bqetl_error_aggregates:
|
||||||
schedule_interval: 3h
|
schedule_interval: 3h
|
||||||
default_args:
|
default_args:
|
||||||
owner: bewu@mozilla.com
|
owner: bewu@mozilla.com
|
||||||
email: ['telemetry-alerts@mozilla.com', 'bewu@mozilla.com', 'wlachance@mozilla.com']
|
email:
|
||||||
start_date: '2019-11-01'
|
[
|
||||||
|
"telemetry-alerts@mozilla.com",
|
||||||
|
"bewu@mozilla.com",
|
||||||
|
"wlachance@mozilla.com",
|
||||||
|
]
|
||||||
|
start_date: "2019-11-01"
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 20m
|
retry_delay: 20m
|
||||||
depends_on_past: false
|
depends_on_past: false
|
||||||
|
|
@ -11,27 +17,32 @@ bqetl_error_aggregates:
|
||||||
bqetl_ssl_ratios:
|
bqetl_ssl_ratios:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: chutten@mozilla.com
|
owner: chutten@mozilla.com
|
||||||
start_date: '2019-07-20'
|
start_date: "2019-07-20"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'chutten@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "chutten@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
bqetl_amo_stats:
|
bqetl_amo_stats:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2020-06-01'
|
start_date: "2020-06-01"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
bqetl_vrbrowser:
|
bqetl_vrbrowser:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com', 'ascholtz@mozilla.com']
|
email:
|
||||||
|
[
|
||||||
|
"telemetry-alerts@mozilla.com",
|
||||||
|
"jklukas@mozilla.com",
|
||||||
|
"ascholtz@mozilla.com",
|
||||||
|
]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -39,8 +50,8 @@ bqetl_core:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -48,8 +59,8 @@ bqetl_nondesktop:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -57,8 +68,8 @@ bqetl_mobile_search:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: bewu@mozilla.com
|
owner: bewu@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'bewu@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "bewu@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -66,8 +77,8 @@ bqetl_fxa_events:
|
||||||
schedule_interval: 30 1 * * *
|
schedule_interval: 30 1 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-03-01'
|
start_date: "2019-03-01"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 10m
|
retry_delay: 10m
|
||||||
|
|
||||||
|
|
@ -75,8 +86,8 @@ bqetl_gud:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -84,8 +95,8 @@ bqetl_messaging_system:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: najiang@mozilla.com
|
owner: najiang@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'najiang@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "najiang@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -93,8 +104,8 @@ bqetl_activity_stream:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2019-07-25'
|
start_date: "2019-07-25"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 1
|
retries: 1
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -102,8 +113,9 @@ bqetl_search:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: bewu@mozilla.com
|
owner: bewu@mozilla.com
|
||||||
start_date: '2018-11-27'
|
start_date: "2018-11-27"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'bewu@mozilla.com', 'frank@mozilla.com']
|
email:
|
||||||
|
["telemetry-alerts@mozilla.com", "bewu@mozilla.com", "frank@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -111,8 +123,8 @@ bqetl_addons:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: bmiroglio@mozilla.com
|
owner: bmiroglio@mozilla.com
|
||||||
start_date: '2018-11-27'
|
start_date: "2018-11-27"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'bmiroglio@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "bmiroglio@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -120,8 +132,8 @@ bqetl_devtools:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2018-11-27'
|
start_date: "2018-11-27"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'jklukas@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "jklukas@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -129,8 +141,14 @@ bqetl_main_summary:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: dthorn@mozilla.com
|
owner: dthorn@mozilla.com
|
||||||
start_date: '2018-11-27'
|
start_date: "2018-11-27"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com', 'jklukas@mozilla.com', 'frank@mozilla.com']
|
email:
|
||||||
|
[
|
||||||
|
"telemetry-alerts@mozilla.com",
|
||||||
|
"dthorn@mozilla.com",
|
||||||
|
"jklukas@mozilla.com",
|
||||||
|
"frank@mozilla.com",
|
||||||
|
]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -138,8 +156,8 @@ bqetl_experiments_daily:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: ssuh@mozilla.com
|
owner: ssuh@mozilla.com
|
||||||
start_date: '2018-11-27'
|
start_date: "2018-11-27"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'ssuh@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "ssuh@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -147,8 +165,8 @@ bqetl_document_sample:
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
default_args:
|
default_args:
|
||||||
owner: amiyaguchi@mozilla.com
|
owner: amiyaguchi@mozilla.com
|
||||||
start_date: '2020-02-17'
|
start_date: "2020-02-17"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'amiyaguchi@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "amiyaguchi@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -156,19 +174,19 @@ bqetl_asn_aggregates:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: ascholtz@mozilla.com
|
owner: ascholtz@mozilla.com
|
||||||
start_date: '2020-04-05'
|
start_date: "2020-04-05"
|
||||||
email: ['ascholtz@mozilla.com', 'tdsmith@mozilla.com']
|
email: ["ascholtz@mozilla.com", "tdsmith@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
# DAG for exporting query data marked as public to GCS
|
# DAG for exporting query data marked as public to GCS
|
||||||
# queries should not be explicitly assigned to this DAG (it's done automatically)
|
# queries should not be explicitly assigned to this DAG (done automatically)
|
||||||
bqetl_public_data_json:
|
bqetl_public_data_json:
|
||||||
schedule_interval: 0 4 * * *
|
schedule_interval: 0 4 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: ascholtz@mozilla.com
|
owner: ascholtz@mozilla.com
|
||||||
start_date: '2020-04-14'
|
start_date: "2020-04-14"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'ascholtz@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "ascholtz@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -177,8 +195,8 @@ bqetl_internet_outages:
|
||||||
schedule_interval: 0 3 * * *
|
schedule_interval: 0 3 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: aplacitelli@mozilla.com
|
owner: aplacitelli@mozilla.com
|
||||||
start_date: '2020-01-01'
|
start_date: "2020-01-01"
|
||||||
email: ['aplacitelli@mozilla.com', 'sguha@mozilla.com']
|
email: ["aplacitelli@mozilla.com", "sguha@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -186,8 +204,8 @@ bqetl_deletion_request_volume:
|
||||||
schedule_interval: 0 1 * * *
|
schedule_interval: 0 1 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: dthorn@mozilla.com
|
owner: dthorn@mozilla.com
|
||||||
start_date: '2020-06-29'
|
start_date: "2020-06-29"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -195,8 +213,8 @@ bqetl_fenix_event_rollup:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: frank@mozilla.com
|
owner: frank@mozilla.com
|
||||||
start_date: '2020-09-09'
|
start_date: "2020-09-09"
|
||||||
email: ['frank@mozilla.com']
|
email: ["frank@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -204,8 +222,8 @@ bqetl_account_ecosystem:
|
||||||
schedule_interval: 0 2 * * *
|
schedule_interval: 0 2 * * *
|
||||||
default_args:
|
default_args:
|
||||||
owner: jklukas@mozilla.com
|
owner: jklukas@mozilla.com
|
||||||
start_date: '2020-09-17'
|
start_date: "2020-09-17"
|
||||||
email: ['jklukas@mozilla.com']
|
email: ["jklukas@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -213,8 +231,8 @@ bqetl_stripe:
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
default_args:
|
default_args:
|
||||||
owner: dthorn@mozilla.com
|
owner: dthorn@mozilla.com
|
||||||
start_date: '2020-10-05'
|
start_date: "2020-10-05"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 5m
|
retry_delay: 5m
|
||||||
|
|
||||||
|
|
@ -222,8 +240,8 @@ bqetl_mozilla_vpn:
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
default_args:
|
default_args:
|
||||||
owner: dthorn@mozilla.com
|
owner: dthorn@mozilla.com
|
||||||
start_date: '2020-10-08'
|
start_date: "2020-10-08"
|
||||||
email: ['telemetry-alerts@mozilla.com', 'dthorn@mozilla.com']
|
email: ["telemetry-alerts@mozilla.com", "dthorn@mozilla.com"]
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
|
|
||||||
|
|
@ -231,14 +249,14 @@ bqetl_org_mozilla_fenix_derived:
|
||||||
default_args:
|
default_args:
|
||||||
depends_on_past: false
|
depends_on_past: false
|
||||||
email:
|
email:
|
||||||
- amiyaguchi@mozilla.com
|
- amiyaguchi@mozilla.com
|
||||||
- telemetry-alerts@mozilla.com
|
- telemetry-alerts@mozilla.com
|
||||||
email_on_failure: true
|
email_on_failure: true
|
||||||
email_on_retry: true
|
email_on_retry: true
|
||||||
owner: amiyaguchi@mozilla.com
|
owner: amiyaguchi@mozilla.com
|
||||||
retries: 2
|
retries: 2
|
||||||
retry_delay: 30m
|
retry_delay: 30m
|
||||||
start_date: '2020-10-18'
|
start_date: "2020-10-18"
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
|
|
||||||
bqetl_google_analytics_derived:
|
bqetl_google_analytics_derived:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
site_name: BigQuery ETL
|
site_name: BigQuery ETL
|
||||||
site_description: Mozilla BigQuery ETL
|
site_description: Mozilla BigQuery ETL
|
||||||
site_author: Mozilla Data Platform Team
|
site_author: Mozilla Data Platform Team
|
||||||
|
|
|
||||||
|
|
@ -21,3 +21,4 @@ stripe==2.55.0
|
||||||
ujson==4.0.1
|
ujson==4.0.1
|
||||||
pandas==1.1.3
|
pandas==1.1.3
|
||||||
jsonschema==3.2.0
|
jsonschema==3.2.0
|
||||||
|
yamllint==1.25.0
|
||||||
|
|
|
||||||
|
|
@ -366,7 +366,7 @@ pandas==1.1.3 \
|
||||||
pathspec==0.8.0 \
|
pathspec==0.8.0 \
|
||||||
--hash=sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0 \
|
--hash=sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0 \
|
||||||
--hash=sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061 \
|
--hash=sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061 \
|
||||||
# via black
|
# via black, yamllint
|
||||||
pluggy==0.13.1 \
|
pluggy==0.13.1 \
|
||||||
--hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \
|
--hash=sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0 \
|
||||||
--hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d \
|
--hash=sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d \
|
||||||
|
|
@ -475,7 +475,7 @@ pyyaml==5.3.1 \
|
||||||
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
|
--hash=sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d \
|
||||||
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
|
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
|
||||||
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
|
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
|
||||||
# via -r requirements.in, mozilla-schema-generator
|
# via -r requirements.in, mozilla-schema-generator, yamllint
|
||||||
regex==2020.9.27 \
|
regex==2020.9.27 \
|
||||||
--hash=sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef \
|
--hash=sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef \
|
||||||
--hash=sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c \
|
--hash=sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c \
|
||||||
|
|
@ -593,6 +593,10 @@ urllib3==1.25.10 \
|
||||||
--hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \
|
--hash=sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a \
|
||||||
--hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461 \
|
--hash=sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461 \
|
||||||
# via requests
|
# via requests
|
||||||
|
yamllint==1.25.0 \
|
||||||
|
--hash=sha256:b1549cbe5b47b6ba67bdeea31720f5c51431a4d0c076c1557952d841f7223519 \
|
||||||
|
--hash=sha256:c7be4d0d2584a1b561498fa9acb77ad22eb434a109725c7781373ae496d823b3 \
|
||||||
|
# via -r requirements.in
|
||||||
yarl==1.6.0 \
|
yarl==1.6.0 \
|
||||||
--hash=sha256:04a54f126a0732af75e5edc9addeaa2113e2ca7c6fce8974a63549a70a25e50e \
|
--hash=sha256:04a54f126a0732af75e5edc9addeaa2113e2ca7c6fce8974a63549a70a25e50e \
|
||||||
--hash=sha256:3cc860d72ed989f3b1f3abbd6ecf38e412de722fb38b8f1b1a086315cf0d69c5 \
|
--hash=sha256:3cc860d72ed989f3b1f3abbd6ecf38e412de722fb38b8f1b1a086315cf0d69c5 \
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
|
---
|
||||||
friendly_name: Experimenter experiments
|
friendly_name: Experimenter experiments
|
||||||
description: Experimenter experiments periodically imported via the Experimenter API.
|
description: Experimenter experiments periodically imported via the
|
||||||
|
Experimenter API.
|
||||||
labels:
|
labels:
|
||||||
incremental: false
|
incremental: false
|
||||||
schedule: ten-minute
|
schedule: ten-minute
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
description: Daily summary Google analytics data for blog.mozilla.org landing page
|
description: Daily summary Google analytics data for blog.mozilla.org
|
||||||
|
landing page
|
||||||
friendly_name: Blogs Landing Page Summary
|
friendly_name: Blogs Landing Page Summary
|
||||||
labels:
|
labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
|
|
@ -8,5 +9,5 @@ owners:
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_google_analytics_derived
|
dag_name: bqetl_google_analytics_derived
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_goals_v1']
|
- ["moz-fx-data-marketing-prod", "ga_derived", "blogs_goals_v1"]
|
||||||
- ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_sessions_v1']
|
- ["moz-fx-data-marketing-prod", "ga_derived", "blogs_sessions_v1"]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
description: Intermediate table containing normalized sessions for blog.mozilla.org
|
description: Intermediate table containing normalized sessions
|
||||||
|
for blog.mozilla.org
|
||||||
friendly_name: Blogs Sessions
|
friendly_name: Blogs Sessions
|
||||||
labels:
|
labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
|
|
@ -8,4 +9,4 @@ owners:
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_google_analytics_derived
|
dag_name: bqetl_google_analytics_derived
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ['moz-fx-data-marketing-prod', 'ga_derived', 'blogs_empty_check_v1']
|
- ["moz-fx-data-marketing-prod", "ga_derived", "blogs_empty_check_v1"]
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,16 @@
|
||||||
|
---
|
||||||
friendly_name: AET Clients Daily
|
friendly_name: AET Clients Daily
|
||||||
description: >
|
description: >
|
||||||
One row per user per service per day, showing metrics across services.
|
One row per user per service per day, showing metrics across services.
|
||||||
|
|
||||||
The `user_id` and `client_id` are directly related to the `ecosystem_user_id`
|
The `user_id` and `client_id` are directly related to the
|
||||||
and `ecosystem_client_id` primitive identifiers for Account Ecosystem Telemetry,
|
`ecosystem_user_id` and `ecosystem_client_id` primitive
|
||||||
but are abstracted to prevent fingerprinting and to provide continuity across
|
identifiers for Account Ecosystem Telemetry, but are abstracted
|
||||||
user password reset events. In this view, we are guaranteed that a logical user
|
to prevent fingerprinting and to provide continuity across
|
||||||
is represented by a consistent `user_id` over time.
|
user password reset events. In this view, we are guaranteed that
|
||||||
|
a logical user is represented by a consistent `user_id` over time.
|
||||||
|
|
||||||
For rows representing client telemetry, this view looks up `user_id` at runtime
|
For rows representing client telemetry, this view looks up `user_id`
|
||||||
based on `client_id` so that we can have `user_id` values present for any client
|
at runtime based on `client_id` so that we can have `user_id` values
|
||||||
that has ever logged in to FxA, even for older rows before the first login.
|
present for any client that has ever logged in to FxA, even for older
|
||||||
|
rows before the first login.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: AET Desktop Clients Daily
|
friendly_name: AET Desktop Clients Daily
|
||||||
description: >
|
description: >
|
||||||
One row per desktop client per day aggregating all AET pings received for that client.
|
One row per desktop client per day aggregating all AET pings received
|
||||||
|
for that client.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Ecosystem Client ID Lookup
|
friendly_name: Ecosystem Client ID Lookup
|
||||||
description: >
|
description: >
|
||||||
Lookup table of ecosystem_client_id_hash to canonical_id.
|
Lookup table of ecosystem_client_id_hash to canonical_id.
|
||||||
|
|
@ -9,9 +10,11 @@ labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_account_ecosystem
|
dag_name: bqetl_account_ecosystem
|
||||||
depends_on_past: True
|
depends_on_past: true
|
||||||
# We access a restricted table for getting an HMAC key, so cannot dry run
|
# We access a restricted table for getting an HMAC key, so cannot dry run
|
||||||
# and must explicitly list referenced tables.
|
# and must explicitly list referenced tables.
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ['moz-fx-data-shared-prod', 'telemetry_stable', 'account_ecosystem_v4']
|
- ['moz-fx-data-shared-prod', 'telemetry_stable',
|
||||||
- ['moz-fx-data-shared-prod', 'account_ecosystem_derived', 'ecosystem_user_id_lookup_v1']
|
'account_ecosystem_v4']
|
||||||
|
- ['moz-fx-data-shared-prod', 'account_ecosystem_derived',
|
||||||
|
'ecosystem_user_id_lookup_v1']
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Ecosystem User ID Lookup
|
friendly_name: Ecosystem User ID Lookup
|
||||||
description: >
|
description: >
|
||||||
Lookup table of ecosystem_user_id to canonical_id.
|
Lookup table of ecosystem_user_id to canonical_id.
|
||||||
|
|
@ -14,9 +15,14 @@ labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_account_ecosystem
|
dag_name: bqetl_account_ecosystem
|
||||||
depends_on_past: True
|
depends_on_past: true
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'firefox_accounts_stable', 'account_ecosystem_v1']]
|
referenced_tables:
|
||||||
|
- [
|
||||||
|
"moz-fx-data-shared-prod",
|
||||||
|
"firefox_accounts_stable",
|
||||||
|
"account_ecosystem_v1",
|
||||||
|
]
|
||||||
# This is an unpartitioned table where the script adds rows via INSERT INTO,
|
# This is an unpartitioned table where the script adds rows via INSERT INTO,
|
||||||
# thus the custom settings below.
|
# thus the custom settings below.$
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
parameters: ["submission_date:DATE:{{ds}}"]
|
parameters: ["submission_date:DATE:{{ds}}"]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: AET Logging Users Daily
|
friendly_name: AET Logging Users Daily
|
||||||
description: >
|
description: >
|
||||||
One row per canonical_id per oauth service per day aggregating all AET events received for that user.
|
One row per canonical_id per oauth service per day aggregating
|
||||||
|
all AET events received for that user.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -12,5 +14,13 @@ scheduling:
|
||||||
# We access a restricted table for getting an HMAC key, so cannot dry run
|
# We access a restricted table for getting an HMAC key, so cannot dry run
|
||||||
# and must explicitly list referenced tables.
|
# and must explicitly list referenced tables.
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ['moz-fx-data-shared-prod', 'firefox_accounts_stable', 'account_ecosystem_v1']
|
- [
|
||||||
- ['moz-fx-data-shared-prod', 'account_ecosystem_derived', 'ecosystem_user_id_lookup_v1']
|
"moz-fx-data-shared-prod",
|
||||||
|
"firefox_accounts_stable",
|
||||||
|
"account_ecosystem_v1",
|
||||||
|
]
|
||||||
|
- [
|
||||||
|
"moz-fx-data-shared-prod",
|
||||||
|
"account_ecosystem_derived",
|
||||||
|
"ecosystem_user_id_lookup_v1",
|
||||||
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Ecosystem Client ID Deletion
|
friendly_name: Ecosystem Client ID Deletion
|
||||||
description: >
|
description: >
|
||||||
Provides ecosystem_client_id values that have appeared in a deletion-request ping.
|
Provides ecosystem_client_id values that have appeared in a
|
||||||
|
deletion-request ping.
|
||||||
|
|
||||||
Also includes the hashed version of the ID to enable deletion of rows in derived
|
Also includes the hashed version of the ID to enable deletion
|
||||||
tables containing the hash.
|
of rows in derived tables containing the hash.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: Impression Stats By Experiment
|
friendly_name: Impression Stats By Experiment
|
||||||
description: Representation of tile impression statistics, clustered by experiment_id to allow efficient analysis of individual experiments
|
description: Representation of tile impression statistics,
|
||||||
|
clustered by experiment_id to allow efficient analysis of
|
||||||
|
individual experiments
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Impression Stats Flat
|
friendly_name: Impression Stats Flat
|
||||||
description: Unnested representation of tile impression statistics
|
description: Unnested representation of tile impression statistics
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: AMO Stats DAU dev/stage
|
friendly_name: AMO Stats DAU dev/stage
|
||||||
description: >-
|
description: >-
|
||||||
Reduced stats table for dev and stage versions of the AMO service.
|
Reduced stats table for dev and stage versions of the AMO service.
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: AMO Installs dev/stage
|
friendly_name: AMO Installs dev/stage
|
||||||
description: >
|
description: >
|
||||||
Reduced daily installs table for dev and stage versions of the AMO service.
|
Reduced daily installs table for dev and stage versions of the AMO service.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: AMO Stats DAU
|
friendly_name: AMO Stats DAU
|
||||||
description: >-
|
description: >-
|
||||||
Daily user statistics to power addons.mozilla.org stats pages. See bug 1572873.
|
Daily user statistics to power addons.mozilla.org stats pages.
|
||||||
|
See bug 1572873.
|
||||||
|
|
||||||
Each row in this table represents a particular addon on a particular day
|
Each row in this table represents a particular addon on a particular day
|
||||||
and provides all the information needed to populate the various
|
and provides all the information needed to populate the various
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,12 @@
|
||||||
|
---
|
||||||
friendly_name: AMO Stats DAU
|
friendly_name: AMO Stats DAU
|
||||||
description: >
|
description: >
|
||||||
Daily install statistics to power addons.mozilla.org stats pages. See bug 1654330.
|
Daily install statistics to power addons.mozilla.org stats pages.
|
||||||
Note that this table uses a hashed_addon_id defined as `TO_HEX(SHA256(addon_id))`
|
See bug 1654330. Note that this table uses a hashed_addon_id
|
||||||
because the underlying event pings have limitations on length of properties attached
|
defined as `TO_HEX(SHA256(addon_id))` because the underlying event
|
||||||
to events and addon_id values are sometimes too long. The AMO stats application looks
|
pings have limitations on length of properties attached to events
|
||||||
up records in this table based on the hashed_addon_id.
|
and addon_id values are sometimes too long. The AMO stats application
|
||||||
|
looks up records in this table based on the hashed_addon_id.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Desktop addons by client
|
friendly_name: Desktop addons by client
|
||||||
description: >-
|
description: >-
|
||||||
Clients_daily-like table that records only the dimensions and addon info
|
Clients_daily-like table that records only the dimensions and addon info
|
||||||
|
|
@ -12,4 +13,5 @@ scheduling:
|
||||||
dag_name: bqetl_amo_stats
|
dag_name: bqetl_amo_stats
|
||||||
# provide this value so that DAG generation does not have to dry run the
|
# provide this value so that DAG generation does not have to dry run the
|
||||||
# query to get it, and that would be slow because main_v4 is referenced
|
# query to get it, and that would be slow because main_v4 is referenced
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable',
|
||||||
|
'main_v4']]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Fenix addons by client
|
friendly_name: Fenix addons by client
|
||||||
description: >-
|
description: >-
|
||||||
Clients_daily-like table on top of the various Firefox for Android channels
|
Clients_daily-like table on top of the various Firefox for Android channels
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Firefox Accounts Exact Mau 28
|
friendly_name: Firefox Accounts Exact Mau 28
|
||||||
description: Base table for exact FxA MAU by dimensions
|
description: Base table for exact FxA MAU by dimensions
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Amplitude Export
|
friendly_name: FxA Amplitude Export
|
||||||
description: >
|
description: >
|
||||||
Derived from FxA logs, this table contains active events and user property
|
Derived from FxA logs, this table contains active events and user property
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Hashed User IDs
|
friendly_name: FxA Hashed User IDs
|
||||||
description: >
|
description: >
|
||||||
Contains exactly one entry for each FxA user, giving the 64-character hashed
|
Contains exactly one entry for each FxA user, giving the 64-character hashed
|
||||||
user ID used for all exports to Amplitude. This table is useful as a lookup
|
user ID used for all exports to Amplitude. This table is useful as a lookup
|
||||||
for sync pings which contain a truncated 32-character version of the same hash.
|
for sync pings which contain a truncated 32-character version of the same
|
||||||
|
hash.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Auth Bounce Events
|
friendly_name: FxA Auth Bounce Events
|
||||||
description: Selected Amplitude events extracted from FxA auth_bounce server logs
|
description: Selected Amplitude events extracted from FxA auth_bounce
|
||||||
|
server logs
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Auth Events
|
friendly_name: FxA Auth Events
|
||||||
description: Selected Amplitude events extracted from FxA auth server logs
|
description: Selected Amplitude events extracted from FxA auth server logs
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Content Events
|
friendly_name: FxA Content Events
|
||||||
description: Selected Amplitude events extracted from FxA content server logs
|
description: Selected Amplitude events extracted from FxA content server logs
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Delete Events
|
friendly_name: FxA Delete Events
|
||||||
description: Deletion events extracted from FxA auth server logs used as signal for Mozilla to delete analysis data associated with the user
|
description: Deletion events extracted from FxA auth server logs
|
||||||
|
used as signal for Mozilla to delete analysis data associated with
|
||||||
|
the user
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Log Auth Events
|
friendly_name: FxA Log Auth Events
|
||||||
description: >-
|
description: >-
|
||||||
A subset of FxA auth logs that is sometimes useful for ad hoc longitudinal analysis.
|
A subset of FxA auth logs that is sometimes useful
|
||||||
|
for ad hoc longitudinal analysis.
|
||||||
|
|
||||||
See https://bugzilla.mozilla.org/show_bug.cgi?id=1628708
|
See https://bugzilla.mozilla.org/show_bug.cgi?id=1628708
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Log Content Events
|
friendly_name: FxA Log Content Events
|
||||||
description: >-
|
description: >-
|
||||||
A subset of FxA content server logs that is sometimes useful for ad hoc longitudinal analysis.
|
A subset of FxA content server logs that is sometimes useful
|
||||||
|
for ad hoc longitudinal analysis.
|
||||||
|
|
||||||
See https://bugzilla.mozilla.org/show_bug.cgi?id=1628708
|
See https://bugzilla.mozilla.org/show_bug.cgi?id=1628708
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Log Device Command Events
|
friendly_name: FxA Log Device Command Events
|
||||||
description: >-
|
description: >-
|
||||||
A subset of FxA auth server logs related to "send tab" activity.
|
A subset of FxA auth server logs related to "send tab" activity.
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Users Daily
|
friendly_name: FxA Users Daily
|
||||||
description: Usage aggregations per FxA user per day
|
description: Usage aggregations per FxA user per day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Users Last Seen
|
friendly_name: FxA Users Last Seen
|
||||||
description: Usage aggregations per FxA user per day over a 28-day window
|
description: Usage aggregations per FxA user per day over a 28-day window
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Users Services Daily
|
friendly_name: FxA Users Services Daily
|
||||||
description: Usage aggregations per FxA user per FxA service per day
|
description: Usage aggregations per FxA user per FxA service per day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Users Services First Seen
|
friendly_name: FxA Users Services First Seen
|
||||||
description: Usage aggregations describing when each FxA user was first seen
|
description: Usage aggregations describing when each FxA user was first seen
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
|
---
|
||||||
friendly_name: FxA Users Services Last Seen
|
friendly_name: FxA Users Services Last Seen
|
||||||
description: Usage aggregations per FxA user per FxA service per day over a 28-day window
|
description: Usage aggregations per FxA user per FxA service
|
||||||
|
per day over a 28-day window
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,30 +1,60 @@
|
||||||
|
---
|
||||||
friendly_name: Internet Outages
|
friendly_name: Internet Outages
|
||||||
description: |-
|
description: |-
|
||||||
This contains a set aggregated metrics that correlate to internet outages for different countries in the world.
|
This contains a set aggregated metrics that correlate to internet
|
||||||
|
outages for different countries in the world.
|
||||||
|
|
||||||
The dataset contains the following fields:
|
The dataset contains the following fields:
|
||||||
- `country`: the Country code of the client.
|
- `country`: the Country code of the client.
|
||||||
- `city`: the City name (only for cities with a population >= 15000, 'unknown' otherwise).
|
- `city`: the City name (only for cities with a population >= 15000,
|
||||||
- `datetime`: the date and the time (truncated to hour) the data was submitted by the client.
|
'unknown' otherwise).
|
||||||
- `proportion_undefined`: the proportion of users who failed to send telemetry for a reason that was not listed in the other cases.
|
- `datetime`: the date and the time (truncated to hour) the data was
|
||||||
- `proportion_timeout`: the proportion of users that had their connection timeout while uploading telemetry ([after 90s, in Firefox Desktop](https://searchfox.org/mozilla-central/rev/fa2df28a49883612bd7af4dacd80cdfedcccd2f6/toolkit/components/telemetry/app/TelemetrySend.jsm#81)).
|
submitted by the client.
|
||||||
- `proportion_abort`: the proportion of users that had their connection terminated by the client (for example, terminating open connections before shutting down).
|
- `proportion_undefined`: the proportion of users who failed to send
|
||||||
- `proportion_unreachable`: the proportion of users that failed to upload telemetry because the server was not reachable (e.g. because the host was not reachable, proxy problems or OS waking up after a suspension).
|
telemetry for a reason that was not listed in the other cases.
|
||||||
- `proportion_terminated`: the proportion of users that had their connection terminated internally by the networking code.
|
- `proportion_timeout`: the proportion of users that had their connection
|
||||||
- `proportion_channel_open`: the proportion of users for which the upload request was terminated immediately, by the client, because of a Necko internal error.
|
timeout while uploading telemetry
|
||||||
- `avg_dns_success_time`: the average time it takes for a successful DNS resolution, in milliseconds.
|
([after 90s, in Firefox Desktop](
|
||||||
- `missing_dns_success`: counts how many sessions did not report the `DNS_LOOKUP_TIME` histogram.
|
https://searchfox.org/mozilla-central/rev/fa2df28a49883612bd7af4dacd80cdfedcccd2f6/toolkit/components/telemetry/app/TelemetrySend.jsm#81)).
|
||||||
- `avg_dns_failure_time`: the average time it takes for an unsuccessful DNS resolution, in milliseconds.
|
- `proportion_abort`: the proportion of users that had their connection
|
||||||
- `missing_dns_failure`: counts how many sessions did not report the `DNS_FAILED_LOOKUP_TIME` histogram.
|
terminated by the client (for example, terminating open connections before
|
||||||
- `count_dns_failure`: the average count of unsuccessful DNS resolutions reported.
|
shutting down).
|
||||||
- `ssl_error_prop`: the proportion of users that reported an error through the `SSL_CERT_VERIFICATION_ERRORS` histogram.
|
- `proportion_unreachable`: the proportion of users that failed to upload
|
||||||
- `avg_tls_handshake_time`: the average time after the TCP SYN to ready for HTTP, in milliseconds.
|
telemetry because the server was not reachable (e.g. because the host was
|
||||||
|
not reachable, proxy problems or OS waking up after a suspension).
|
||||||
|
- `proportion_terminated`: the proportion of users that had their connection
|
||||||
|
terminated internally by the networking code.
|
||||||
|
- `proportion_channel_open`: the proportion of users for which the upload
|
||||||
|
request was terminated immediately, by the client, because of a Necko
|
||||||
|
internal error.
|
||||||
|
- `avg_dns_success_time`: the average time it takes for a successful DNS
|
||||||
|
resolution, in milliseconds.
|
||||||
|
- `missing_dns_success`: counts how many sessions did not report the
|
||||||
|
`DNS_LOOKUP_TIME` histogram.
|
||||||
|
- `avg_dns_failure_time`: the average time it takes for an unsuccessful DNS
|
||||||
|
resolution, in milliseconds.
|
||||||
|
- `missing_dns_failure`: counts how many sessions did not report the
|
||||||
|
`DNS_FAILED_LOOKUP_TIME` histogram.
|
||||||
|
- `count_dns_failure`: the average count of unsuccessful DNS resolutions
|
||||||
|
reported.
|
||||||
|
- `ssl_error_prop`: the proportion of users that reported an error through
|
||||||
|
the `SSL_CERT_VERIFICATION_ERRORS` histogram.
|
||||||
|
- `avg_tls_handshake_time`: the average time after the TCP SYN to ready
|
||||||
|
for HTTP, in milliseconds.
|
||||||
|
|
||||||
Caveats with the data:
|
Caveats with the data:
|
||||||
As with any observational data, there are many caveats and interpretation must be done carefully. Below is a list of issues we have considered, but it is not exhaustive.
|
As with any observational data, there are many caveats and interpretation must
|
||||||
- Firefox users are not representative of the general population in their region.
|
be done carefully. Below is a list of issues we have considered, but it is not
|
||||||
- Users can experience multiple types of failures and so the proportions are not summable. For example, if 2.4% of clients had a timeout and 2.6% of clients had eUnreachable that doesn't necessarily mean that 5.0% of clients had a timeout or a eUnreachable
|
exhaustive.
|
||||||
- Geographical data is based on IPGeo databases. These databases are imperfect, so some activity may be attributed to the wrong location. Further, proxy and VPN usage can create geo-attribution errors.
|
- Firefox users are not representative of the general population in their
|
||||||
|
region.
|
||||||
|
- Users can experience multiple types of failures and so the proportions
|
||||||
|
are not summable. For example, if 2.4% of clients had a timeout and 2.6% of
|
||||||
|
clients had eUnreachable that doesn't necessarily mean that 5.0% of clients
|
||||||
|
had a timeout or a eUnreachable
|
||||||
|
- Geographical data is based on IPGeo databases. These databases are
|
||||||
|
imperfect, so some activity may be attributed to the wrong location.
|
||||||
|
Further, proxy and VPN usage can create geo-attribution errors.
|
||||||
|
|
||||||
owners:
|
owners:
|
||||||
- aplacitelli@mozilla.com
|
- aplacitelli@mozilla.com
|
||||||
|
|
@ -33,7 +63,7 @@ labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
public_json: false
|
public_json: false
|
||||||
public_bigquery: false
|
public_bigquery: false
|
||||||
review_bugs:
|
review_bugs:
|
||||||
- 1640204
|
- 1640204
|
||||||
incremental_export: false
|
incremental_export: false
|
||||||
scheduling:
|
scheduling:
|
||||||
|
|
@ -41,6 +71,6 @@ scheduling:
|
||||||
# provide this value so that DAG generation does not have to dry run the
|
# provide this value so that DAG generation does not have to dry run the
|
||||||
# query to get it, and that would be slow because main_v4 is referenced
|
# query to get it, and that would be slow because main_v4 is referenced
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']
|
- ["moz-fx-data-shared-prod", "telemetry_stable", "main_v4"]
|
||||||
- ['moz-fx-data-shared-prod', 'telemetry_derived', 'clients_daily_v6']
|
- ["moz-fx-data-shared-prod", "telemetry_derived", "clients_daily_v6"]
|
||||||
- ['moz-fx-data-shared-prod', 'telemetry_stable', 'health_v4']
|
- ["moz-fx-data-shared-prod", "telemetry_stable", "health_v4"]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System CFR Exact MAU by Dimensions
|
friendly_name: Messaging System CFR Exact MAU by Dimensions
|
||||||
description: >
|
description: >
|
||||||
Monthly active users using CFR aggregated across unique sets of dimensions.
|
Monthly active users using CFR aggregated across unique sets of dimensions.
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System CFR Users Daily
|
friendly_name: Messaging System CFR Users Daily
|
||||||
description: Daily users of CFR, partitioned by day
|
description: Daily users of CFR, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System CFR Users Last Seen
|
friendly_name: Messaging System CFR Users Last Seen
|
||||||
description: >
|
description: >
|
||||||
Captures history of activity of each client using CFR in 28 day
|
Captures history of activity of each client using CFR in 28 day
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Onboarding Exact MAU by Dimensions
|
friendly_name: Messaging System Onboarding Exact MAU by Dimensions
|
||||||
description: >
|
description: >
|
||||||
Monthly active users using Onboarding aggregated across unique sets of dimensions.
|
Monthly active users using Onboarding aggregated across unique
|
||||||
|
sets of dimensions.
|
||||||
owners:
|
owners:
|
||||||
- najiang@mozilla.com
|
- najiang@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Onboarding Users Daily
|
friendly_name: Messaging System Onboarding Users Daily
|
||||||
description: Daily users of Onboarding, partitioned by day
|
description: Daily users of Onboarding, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Onboarding Users Last Seen
|
friendly_name: Messaging System Onboarding Users Last Seen
|
||||||
description: >
|
description: >
|
||||||
Captures history of activity of each client using Onboarding in 28 day
|
Captures history of activity of each client using Onboarding in 28 day
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Snippets Exact MAU by Dimensions
|
friendly_name: Messaging System Snippets Exact MAU by Dimensions
|
||||||
description: >
|
description: >
|
||||||
Monthly active users using Snippets aggregated across unique sets of dimensions.
|
Monthly active users using Snippets aggregated across unique sets
|
||||||
|
of dimensions.
|
||||||
owners:
|
owners:
|
||||||
- najiang@mozilla.com
|
- najiang@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Snippets Users Daily
|
friendly_name: Messaging System Snippets Users Daily
|
||||||
description: Daily users of Snippets, partitioned by day
|
description: Daily users of Snippets, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Messaging System Snippets Users Last Seen
|
friendly_name: Messaging System Snippets Users Last Seen
|
||||||
description: >
|
description: >
|
||||||
Captures history of activity of each client using Snippets in 28 day
|
Captures history of activity of each client using Snippets in 28 day
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Bigquery-etl Scheduled Queries Cost
|
friendly_name: Bigquery-etl Scheduled Queries Cost
|
||||||
description: Cost of scheduled bigquery-etl queries, partitioned by day.
|
description: Cost of scheduled bigquery-etl queries, partitioned by day.
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,9 @@
|
||||||
description: Number of accesses to destination tables of scheduled bigquery-etl queries, partitioned by day.
|
---
|
||||||
|
description: Number of accesses to destination tables of scheduled
|
||||||
|
bigquery-etl queries, partitioned by day.
|
||||||
friendly_name: Bigquery-etl Scheduled Query Usage
|
friendly_name: Bigquery-etl Scheduled Query Usage
|
||||||
labels:
|
labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
schedule: daily
|
schedule: daily
|
||||||
owners:
|
owners:
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
description: Determines column sizes of specific tables and partitions via dry runs.
|
description: Determines column sizes of specific tables and partitions
|
||||||
|
via dry runs.
|
||||||
friendly_name: Column Size
|
friendly_name: Column Size
|
||||||
labels:
|
labels:
|
||||||
incremental: true
|
incremental: true
|
||||||
owners:
|
owners:
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Deletion Request Volume
|
friendly_name: Deletion Request Volume
|
||||||
description: >
|
description: >
|
||||||
A daily count of deletion request pings by document namespace
|
A daily count of deletion request pings by document namespace
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Document Sample
|
friendly_name: Document Sample
|
||||||
description: Document samples non-prod.
|
description: Document samples non-prod.
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stable Table Sizes
|
friendly_name: Stable Table Sizes
|
||||||
description: >
|
description: >
|
||||||
Table sizes of stable tables, partitioned by day.
|
Table sizes of stable tables, partitioned by day.
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
friendly_name: Mozilla VPN FxA Login Flows
|
friendly_name: Mozilla VPN FxA Login Flows
|
||||||
description: >
|
description: >
|
||||||
A list of users that attempted to login to FxA, to sign up for or login to Mozilla VPN.
|
A list of users that attempted to login to FxA, to sign up for or
|
||||||
|
login to Mozilla VPN.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -8,10 +9,19 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
parameters:
|
parameters:
|
||||||
- "date:DATE:{{ds}}"
|
- "date:DATE:{{ds}}"
|
||||||
referenced_tables:
|
referenced_tables:
|
||||||
- ["moz-fx-data-shared-prod", "firefox_accounts_derived", "fxa_auth_events_v1"]
|
- [
|
||||||
- ["moz-fx-data-shared-prod", "firefox_accounts_derived", "fxa_content_events_v1"]
|
"moz-fx-data-shared-prod",
|
||||||
|
"firefox_accounts_derived",
|
||||||
|
"fxa_auth_events_v1",
|
||||||
|
]
|
||||||
|
- [
|
||||||
|
"moz-fx-data-shared-prod",
|
||||||
|
"firefox_accounts_derived",
|
||||||
|
"fxa_content_events_v1",
|
||||||
|
]
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,9 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
parameters:
|
parameters:
|
||||||
- "date:DATE:{{ds}}"
|
- "date:DATE:{{ds}}"
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
friendly_name: Mozilla VPN Users
|
friendly_name: Mozilla VPN Users
|
||||||
description: >
|
description: >
|
||||||
A subset of Mozilla VPN users columns that are accessible to a broader audience.
|
A subset of Mozilla VPN users columns that are accessible to a
|
||||||
|
broader audience.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -8,6 +9,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
referenced_tables: [["moz-fx-data-shared-prod", "mozilla_vpn_external","users_v1"]]
|
referenced_tables:
|
||||||
|
[["moz-fx-data-shared-prod", "mozilla_vpn_external", "users_v1"]]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: Mozilla VPN Waitlist
|
friendly_name: Mozilla VPN Waitlist
|
||||||
description: >
|
description: >
|
||||||
A subset of Mozilla VPN waitlist columns and values that are accessible to a broader audience.
|
A subset of Mozilla VPN waitlist columns and values that are
|
||||||
|
accessible to a broader audience.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -8,6 +10,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'mozilla_vpn_external','waitlist_v1']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'mozilla_vpn_external',
|
||||||
|
'waitlist_v1']]
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Mozilla VPN Devices
|
friendly_name: Mozilla VPN Devices
|
||||||
description: >
|
description: >
|
||||||
A mirror of the devices table from the Mozilla VPN (Guardian) CloudSQL database, updated daily to
|
A mirror of the devices table from the Mozilla VPN (Guardian)
|
||||||
match the current state of the table. The table history is not needed, because changes made are not
|
CloudSQL database, updated daily to match the current state of
|
||||||
destructive, except in the case of self-serve data deletion.
|
the table. The table history is not needed, because changes
|
||||||
|
made are not destructive, except in the case of self-serve
|
||||||
|
data deletion.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -10,12 +13,16 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
depends_on_past: true
|
depends_on_past: true
|
||||||
parameters:
|
parameters:
|
||||||
# The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
|
# The external_database_query argument in EXTERNAL_QUERY must be
|
||||||
# parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
|
# a literal string or query parameter, and cannot be generated
|
||||||
# so the entire value must be provided as a STRING query parameter to handle specific dates:
|
# at runtime using function calls like CONCAT or FORMAT, so the
|
||||||
- "external_database_query:STRING:SELECT * FROM devices WHERE DATE(updated_at) = DATE '{{ds}}'"
|
# entire value must be provided as a STRING query parameter to
|
||||||
|
# handle specific dates:
|
||||||
|
- "external_database_query:STRING:SELECT * FROM devices
|
||||||
|
WHERE DATE(updated_at) = DATE '{{ds}}'"
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Mozilla VPN Subscriptions
|
friendly_name: Mozilla VPN Subscriptions
|
||||||
description: >
|
description: >
|
||||||
A mirror of the subscriptions table from the Mozilla VPN (Guardian) CloudSQL database, updated daily to
|
A mirror of the subscriptions table from the Mozilla VPN
|
||||||
match the current state of the table. The table history is not needed, because changes made are not
|
(Guardian) CloudSQL database, updated daily to
|
||||||
destructive, except in the case of self-serve data deletion.
|
match the current state of the table. The table history
|
||||||
|
is not needed, because changes made are not destructive,
|
||||||
|
except in the case of self-serve data deletion.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -10,12 +13,16 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
depends_on_past: true
|
depends_on_past: true
|
||||||
parameters:
|
parameters:
|
||||||
# The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
|
# The external_database_query argument in EXTERNAL_QUERY must
|
||||||
# parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
|
# be a literal string or query parameter, and cannot be generated
|
||||||
# so the entire value must be provided as a STRING query parameter to handle specific dates:
|
# at runtime using function calls like CONCAT or FORMAT, so the
|
||||||
- "external_database_query:STRING:SELECT * FROM subscriptions WHERE DATE(updated_at) = DATE '{{ds}}'"
|
# entire value must be provided as a STRING query parameter to
|
||||||
|
# handle specific dates:
|
||||||
|
- "external_database_query:STRING:SELECT * FROM subscriptions
|
||||||
|
WHERE DATE(updated_at) = DATE '{{ds}}'"
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Mozilla VPN Users
|
friendly_name: Mozilla VPN Users
|
||||||
description: >
|
description: >
|
||||||
A mirror of the users table from the Mozilla VPN (Guardian) CloudSQL database, updated daily to
|
A mirror of the users table from the Mozilla VPN (Guardian)
|
||||||
match the current state of the table. The table history is not needed, because changes made are not
|
CloudSQL database, updated daily to match the current state
|
||||||
destructive, except in the case of self-serve data deletion.
|
of the table. The table history is not needed, because changes
|
||||||
|
made are not destructive, except in the case of self-serve
|
||||||
|
data deletion.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -10,12 +13,16 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
depends_on_past: true
|
depends_on_past: true
|
||||||
parameters:
|
parameters:
|
||||||
# The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
|
# The external_database_query argument in EXTERNAL_QUERY must
|
||||||
# parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
|
# be a literal string or query parameter, and cannot be generated
|
||||||
# so the entire value must be provided as a STRING query parameter to handle specific dates:
|
# at runtime using function calls like CONCAT or FORMAT, so
|
||||||
- "external_database_query:STRING:SELECT * FROM users WHERE DATE(updated_at) = DATE '{{ds}}'"
|
# the entire value must be provided as a STRING query parameter
|
||||||
|
# to handle specific dates:
|
||||||
|
- "external_database_query:STRING:SELECT * FROM users
|
||||||
|
WHERE DATE(updated_at) = DATE '{{ds}}'"
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Mozilla VPN Waitlist
|
friendly_name: Mozilla VPN Waitlist
|
||||||
description: >
|
description: >
|
||||||
A mirror of the waitlist table from the Mozilla VPN (Guardian) CloudSQL database, updated daily to
|
A mirror of the waitlist table from the Mozilla VPN (Guardian)
|
||||||
match the current state of the table. The table history is not needed, because changes made are not
|
CloudSQL database, updated daily to match the current state of
|
||||||
destructive, except in the case of self-serve data deletion.
|
the table. The table history is not needed, because changes made
|
||||||
|
are not destructive, except in the case of self-serve
|
||||||
|
data deletion.
|
||||||
owners:
|
owners:
|
||||||
- dthorn@mozilla.com
|
- dthorn@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
@ -10,12 +13,16 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_mozilla_vpn
|
dag_name: bqetl_mozilla_vpn
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter: null
|
date_partition_parameter: null
|
||||||
depends_on_past: true
|
depends_on_past: true
|
||||||
parameters:
|
parameters:
|
||||||
# The external_database_query argument in EXTERNAL_QUERY must be a literal string or query
|
# The external_database_query argument in EXTERNAL_QUERY must
|
||||||
# parameter, and cannot be generated at runtime using function calls like CONCAT or FORMAT,
|
# be a literal string or query parameter, and cannot be generated
|
||||||
# so the entire value must be provided as a STRING query parameter to handle specific dates:
|
# at runtime using function calls like CONCAT or FORMAT, so the
|
||||||
- "external_database_query:STRING:SELECT * FROM vpn_waitlist WHERE DATE(updated_at) = DATE '{{ds}}'"
|
# entire value must be provided as a STRING query parameter to
|
||||||
|
# handle specific dates:
|
||||||
|
- "external_database_query:STRING:SELECT * FROM vpn_waitlist
|
||||||
|
WHERE DATE(updated_at) = DATE '{{ds}}'"
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Geckoview versions corresponding to build hours
|
friendly_name: Geckoview versions corresponding to build hours
|
||||||
description: |
|
description: |
|
||||||
Enumerate the most common GeckoView version for each build hour for nightly"
|
Enumerate the most common GeckoView version for each build hour for nightly"
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Firefox Android Release Event Types
|
friendly_name: Firefox Android Release Event Types
|
||||||
description: Recreate a view on the most recent data from event_types_v1
|
description: Recreate a view on the most recent data from event_types_v1
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -9,6 +10,8 @@ labels:
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_fenix_event_rollup
|
dag_name: bqetl_fenix_event_rollup
|
||||||
destination_table: null
|
destination_table: null
|
||||||
sql_file_path: 'sql/moz-fx-data-shared-prod/org_mozilla_firefox/event_types_v1/query.sql'
|
sql_file_path: 'sql/moz-fx-data-shared-prod/org_mozilla_firefox
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'org_mozilla_firefox_derived', 'event_types_v1']]
|
/event_types_v1/query.sql'
|
||||||
|
referenced_tables: [['moz-fx-data-shared-prod', 'org_mozilla_firefox_derived',
|
||||||
|
'event_types_v1']]
|
||||||
parameters: ['submission_date:DATE:{{ds}}']
|
parameters: ['submission_date:DATE:{{ds}}']
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
|
---
|
||||||
friendly_name: Firefox Android Release Event Types
|
friendly_name: Firefox Android Release Event Types
|
||||||
description: Retrieve the set of [events, event_properties] and record them in a table.
|
description: Retrieve the set of [events, event_properties]
|
||||||
|
and record them in a table.
|
||||||
owners:
|
owners:
|
||||||
- frank@mozilla.com
|
- frank@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Events Daily
|
friendly_name: Events Daily
|
||||||
description: Packed event representation with one-row per-client
|
description: Packed event representation with one-row per-client
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -9,6 +10,6 @@ labels:
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_fenix_event_rollup
|
dag_name: bqetl_fenix_event_rollup
|
||||||
referenced_tables: [
|
referenced_tables: [
|
||||||
['moz-fx-data-shared-prod', 'org_mozilla_firefox', 'event_types_v1'],
|
['moz-fx-data-shared-prod', 'org_mozilla_firefox', 'event_types_v1'],
|
||||||
['moz-fx-data-shared-prod', 'org_mozilla_firefox_stable', 'events_v1']
|
['moz-fx-data-shared-prod', 'org_mozilla_firefox_stable', 'events_v1']
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
|
---
|
||||||
friendly_name: Migrated Clients
|
friendly_name: Migrated Clients
|
||||||
description: >
|
description: >
|
||||||
Contains the Fennec Client ID and Fenix Client ID for any clients
|
Contains the Fennec Client ID and Fenix Client ID for any clients
|
||||||
who have migrated from Fennec to Fenix. This is across all release
|
who have migrated from Fennec to Fenix. This is across all release
|
||||||
channels. Every client will have at most one entry, and if a
|
channels. Every client will have at most one entry, and if a
|
||||||
Fennec client is not present then it indicates that they have
|
Fennec client is not present then it indicates that they have
|
||||||
not migrated. Fennec Client ID, channel, and submission_date
|
not migrated. Fennec Client ID, channel, and submission_date
|
||||||
are taken from the _last_ value seen for that client.
|
are taken from the _last_ value seen for that client.
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
|
---
|
||||||
friendly_name: VR Browser Baseline Daily
|
friendly_name: VR Browser Baseline Daily
|
||||||
description: >
|
description: >
|
||||||
A daily aggregate of baseline pings from each Firefox Reality client,
|
A daily aggregate of baseline pings from each Firefox Reality client,
|
||||||
partitioned by day
|
partitioned by day
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
application: firefox-reality
|
application: firefox-reality
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
|
---
|
||||||
friendly_name: VR Browser Clients Daily
|
friendly_name: VR Browser Clients Daily
|
||||||
description: >
|
description: >
|
||||||
A daily aggregate of baseline and metrics pings from each Firefox Reality client,
|
A daily aggregate of baseline and metrics pings from each
|
||||||
partitioned by day
|
Firefox Reality client, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
application: firefox-reality
|
application: firefox-reality
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
|
---
|
||||||
friendly_name: VR Browser Clients Last Seen
|
friendly_name: VR Browser Clients Last Seen
|
||||||
description: >
|
description: >
|
||||||
Captures history of activity of each Firefox Reality client in 28 day
|
Captures history of activity of each Firefox Reality client in 28 day
|
||||||
windows for each submission date.
|
windows for each submission date.
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
application: firefox-reality
|
application: firefox-reality
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,10 @@
|
||||||
|
---
|
||||||
friendly_name: VR Browser Metrics Daily
|
friendly_name: VR Browser Metrics Daily
|
||||||
description: >
|
description: >
|
||||||
A daily aggregate of metrics pings from each Firefox Reality client,
|
A daily aggregate of metrics pings from each Firefox Reality client,
|
||||||
partitioned by day
|
partitioned by day
|
||||||
owners:
|
owners:
|
||||||
- jklukas@mozilla.com
|
- jklukas@mozilla.com
|
||||||
- ascholtz@mozilla.com
|
- ascholtz@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
application: firefox-reality
|
application: firefox-reality
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
|
---
|
||||||
friendly_name: Mobile Search Clients Daily
|
friendly_name: Mobile Search Clients Daily
|
||||||
description: >
|
description: >
|
||||||
Daily mobile clients sending pings that have searches,
|
Daily mobile clients sending pings that have searches,
|
||||||
aggregated across unique sets of dimensions and partitioned by day
|
aggregated across unique sets of dimensions and partitioned by day
|
||||||
owners:
|
owners:
|
||||||
- bewu@mozilla.com
|
- bewu@mozilla.com
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
|
---
|
||||||
friendly_name: Mobile Search Clients Daily
|
friendly_name: Mobile Search Clients Daily
|
||||||
description: >
|
description: >
|
||||||
A daily aggregate of baseline and metrics pings that have searches from each mobile client,
|
A daily aggregate of baseline and metrics pings that have
|
||||||
partitioned by day
|
searches from each mobile client, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
- bewu@mozilla.com
|
- bewu@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Mobile Search Clients Last Seen
|
friendly_name: Mobile Search Clients Last Seen
|
||||||
description: >-
|
description: >-
|
||||||
Captures search activity of a single mobile client in the past 365 days
|
Captures search activity of a single mobile client in the past 365 days
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Search Aggregates
|
friendly_name: Search Aggregates
|
||||||
description: >-
|
description: >-
|
||||||
Daily search clients, aggregated across unique sets of dimensions
|
Daily search clients, aggregated across unique sets of dimensions
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Search Clients Daily
|
friendly_name: Search Clients Daily
|
||||||
description: A daily aggregate of search clients, partitioned by day
|
description: A daily aggregate of search clients, partitioned by day
|
||||||
owners:
|
owners:
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Search Clients Last Seen
|
friendly_name: Search Clients Last Seen
|
||||||
description: >-
|
description: >-
|
||||||
Captures history of activity of each search client in 28 day
|
Captures history of activity of each search client in 28 day
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,11 @@
|
||||||
|
---
|
||||||
friendly_name: Search Metric Contribution
|
friendly_name: Search Metric Contribution
|
||||||
description: >
|
description: >
|
||||||
A table helps us look at question in 28 day
|
A table helps us look at question in 28 day
|
||||||
windows for each submission date: "What percent of one search metirc (ad clicks, search with ads, sap, active hours) are contributed by users in the top [10, 20, 30] percent of the (ad clicks, search with ads, sap, active hours) distribution?"
|
windows for each submission date: "What percent of one search
|
||||||
|
metirc (ad clicks, search with ads, sap, active hours) are contributed
|
||||||
|
by users in the top [10, 20, 30] percent of the (ad clicks, search with
|
||||||
|
ads, sap, active hours) distribution?"
|
||||||
owners:
|
owners:
|
||||||
- bmiroglio@mozilla.com
|
- bmiroglio@mozilla.com
|
||||||
labels:
|
labels:
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external', 'customers_v1']]
|
referenced_tables:
|
||||||
|
[["moz-fx-data-shared-prod", "stripe_external", "customers_v1"]]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Plans
|
friendly_name: Stripe Plans
|
||||||
description: Stripe Plans as of UTC midnight. See bug 1667889.
|
description: Stripe Plans as of UTC midnight. See bug 1667889.
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,6 +8,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external', 'plans_v1']]
|
referenced_tables:
|
||||||
|
[["moz-fx-data-shared-prod", "stripe_external", "plans_v1"]]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Products
|
friendly_name: Stripe Products
|
||||||
description: Stripe Products as of UTC midnight. See bug 1667889.
|
description: Stripe Products as of UTC midnight. See bug 1667889.
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,6 +8,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external', 'products_v1']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external',
|
||||||
|
'products_v1']]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Subscriptions
|
friendly_name: Stripe Subscriptions
|
||||||
description: Stripe Subscriptions as of UTC midnight. See bug 1667889.
|
description: Stripe Subscriptions as of UTC midnight. See bug 1667889.
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,6 +8,8 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external', 'subscriptions_v1']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'stripe_external',
|
||||||
|
'subscriptions_v1']]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Charges
|
friendly_name: Stripe Charges
|
||||||
description: Stripe Charges as of UTC midnight
|
description: Stripe Charges as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Credit Notes
|
friendly_name: Stripe Credit Notes
|
||||||
description: Stripe Credit Notes as of UTC midnight
|
description: Stripe Credit Notes as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Customers
|
friendly_name: Stripe Customers
|
||||||
description: Stripe Customers as of UTC midnight
|
description: Stripe Customers as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Disputes
|
friendly_name: Stripe Disputes
|
||||||
description: Stripe Disputes as of UTC midnight
|
description: Stripe Disputes as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Invoices
|
friendly_name: Stripe Invoices
|
||||||
description: Stripe Invoices as of UTC midnight
|
description: Stripe Invoices as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Payment Intents
|
friendly_name: Stripe Payment Intents
|
||||||
description: Stripe Payment Intents as of UTC midnight
|
description: Stripe Payment Intents as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Payouts
|
friendly_name: Stripe Payouts
|
||||||
description: Stripe Payouts as of UTC midnight
|
description: Stripe Payouts as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Plans
|
friendly_name: Stripe Plans
|
||||||
description: Stripe Plans as of UTC midnight
|
description: Stripe Plans as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Prices
|
friendly_name: Stripe Prices
|
||||||
description: Stripe Prices as of UTC midnight
|
description: Stripe Prices as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Products
|
friendly_name: Stripe Products
|
||||||
description: Stripe Products as of UTC midnight
|
description: Stripe Products as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Setup Intents
|
friendly_name: Stripe Setup Intents
|
||||||
description: Stripe Setup Intents as of UTC midnight
|
description: Stripe Setup Intents as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Stripe Subscriptions
|
friendly_name: Stripe Subscriptions
|
||||||
description: Stripe Subscriptions as of UTC midnight
|
description: Stripe Subscriptions as of UTC midnight
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -7,12 +8,13 @@ labels:
|
||||||
schedule: daily
|
schedule: daily
|
||||||
scheduling:
|
scheduling:
|
||||||
dag_name: bqetl_stripe
|
dag_name: bqetl_stripe
|
||||||
# destination is the whole table, not a single partition, so don't use date_partition_parameter
|
# destination is the whole table, not a single partition,
|
||||||
|
# so don't use date_partition_parameter
|
||||||
date_partition_parameter:
|
date_partition_parameter:
|
||||||
parameters:
|
parameters:
|
||||||
- 'date:DATE:{{ds}}'
|
- "date:DATE:{{ds}}"
|
||||||
depends_on:
|
depends_on:
|
||||||
- dag_name: stripe
|
- dag_name: stripe
|
||||||
task_id: stripe_import_events
|
task_id: stripe_import_events
|
||||||
schedule_interval: daily
|
schedule_interval: daily
|
||||||
referenced_tables: []
|
referenced_tables: []
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Clients first seen
|
friendly_name: Clients first seen
|
||||||
description: >-
|
description: >-
|
||||||
Identical in schema to clients_daily except that each client appears only
|
Identical in schema to clients_daily except that each client appears only
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Desktop 1-Week Retention
|
friendly_name: Desktop 1-Week Retention
|
||||||
description: >
|
description: >
|
||||||
Among profiles that were active at least once in the week starting on the
|
Among profiles that were active at least once in the week starting on the
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Addon Aggregates
|
friendly_name: Addon Aggregates
|
||||||
description: >-
|
description: >-
|
||||||
Addon usages by clients, aggregated across unique sets of dimensions
|
Addon usages by clients, aggregated across unique sets of dimensions
|
||||||
|
|
@ -12,4 +13,5 @@ scheduling:
|
||||||
dag_name: bqetl_addons
|
dag_name: bqetl_addons
|
||||||
# provide this value so that DAG generation does not have to dry run the
|
# provide this value so that DAG generation does not have to dry run the
|
||||||
# query to get it, and that would be slow because main_v4 is referenced
|
# query to get it, and that would be slow because main_v4 is referenced
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable',
|
||||||
|
'main_v4']]
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
friendly_name: Addon Names
|
friendly_name: Addon Names
|
||||||
description: Addon IDs, names and number of occurences.
|
description: Addon IDs, names and number of occurences.
|
||||||
owners:
|
owners:
|
||||||
|
|
@ -15,4 +16,5 @@ scheduling:
|
||||||
parameters: ["submission_date:DATE:{{ds}}"]
|
parameters: ["submission_date:DATE:{{ds}}"]
|
||||||
# provide this value so that DAG generation does not have to dry run the
|
# provide this value so that DAG generation does not have to dry run the
|
||||||
# query to get it, and that would be slow because main_v4 is referenced
|
# query to get it, and that would be slow because main_v4 is referenced
|
||||||
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable', 'main_v4']]
|
referenced_tables: [['moz-fx-data-shared-prod', 'telemetry_stable',
|
||||||
|
'main_v4']]
|
||||||
|
|
|
||||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче