зеркало из https://github.com/mozilla/opmon.git
Load configs from metric-hub
This commit is contained in:
Anna Scholtz
Родитель
3291261a49
Коммит
64eaf95515
|
|
@ -26,10 +26,13 @@ jobs:
|
|||
venv/bin/pip install --progress-bar off --upgrade -r requirements.txt
|
||||
- run:
|
||||
name: PyTest
|
||||
command: venv/bin/pytest --black --flake8 --pydocstyle --ignore=opmon/tests/integration/
|
||||
command: venv/bin/pytest --black --pydocstyle --ignore=opmon/tests/integration/
|
||||
- run:
|
||||
name: isort
|
||||
command: venv/bin/isort --check opmon
|
||||
- run:
|
||||
name: flake8
|
||||
command: venv/bin/flake8 opmon
|
||||
- run:
|
||||
name: Mypy
|
||||
command: venv/bin/mypy opmon
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
The diagram above shows the relationship between different parts of the Operational Monitoring system. At a high level, data flows through the system in the following way:
|
||||
|
||||
1. Users create project config files as described on [dtmo](https://docs.telemetry.mozilla.org/cookbooks/operational_monitoring.html) in [opmon-config](https://github.com/mozilla/opmon-config)
|
||||
1. Users create project config files as described on [dtmo](https://docs.telemetry.mozilla.org/cookbooks/operational_monitoring.html) in [opmon/ in metric-hub](https://github.com/mozilla/metric-hub/tree/main/opmon)
|
||||
2. Two daily jobs run in Airflow to process the config files: one to generate + run the ETL and one to generate LookML for views/explores/dashboards
|
||||
3. Updated LookML dashboards and explores are available once per day and loading them runs aggregates on the fly by referencing relevant BigQuery tables.
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ This can have a wide range of causes. First check if the tables are in the `oper
|
|||
|
||||
* Check the logs for the latest run of [OpMon on Airflow](https://workflow.telemetry.mozilla.org/tree?dag_id=operational_monitoring)
|
||||
* Check the logs in `operational_monitoring_derived.logs_v1` table
|
||||
* Check the [project configuration](https://github.com/mozilla/opmon-config)
|
||||
* Check the [project configuration](https://github.com/mozilla/metric-hub/tree/main/opmon)
|
||||
* Check if the `start_date` has been set to a date in the future
|
||||
* Check if configuration values are pulled in from Experimenter (this is the case for experiments and rollouts)
|
||||
* Install the CLI tool and run locally
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ class BigQueryClient:
|
|||
|
||||
for part in query:
|
||||
config = bigquery.job.QueryJobConfig(default_dataset=bq_dataset, **base_kwargs)
|
||||
job = self.client.query(query, config)
|
||||
job = self.client.query(part, config)
|
||||
# block on result
|
||||
job.result()
|
||||
parts.append(job)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from metric_config_parser.experiment import Experiment
|
|||
from metric_config_parser.monitoring import MonitoringSpec
|
||||
from pytz import UTC
|
||||
|
||||
DEFAULT_CONFIG_REPO = "https://github.com/mozilla/opmon-config"
|
||||
DEFAULT_CONFIG_REPO = "https://github.com/mozilla/metric-hub/tree/main/opmon"
|
||||
METRIC_HUB_REPO = "https://github.com/mozilla/metric-hub"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -202,7 +202,9 @@ class ExperimentCollection:
|
|||
USER_AGENT = "opmon"
|
||||
|
||||
@classmethod
|
||||
def from_experimenter(cls, session: requests.Session = None) -> "ExperimentCollection":
|
||||
def from_experimenter(
|
||||
cls, session: Optional[requests.Session] = None
|
||||
) -> "ExperimentCollection":
|
||||
"""Fetch all experiments from Experimenter."""
|
||||
session = session or requests.Session()
|
||||
legacy_experiments_json = retry_get(
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ class TestConfigLoader:
|
|||
|
||||
def test_configs_from(self):
|
||||
configs_collection = ConfigLoader.with_configs_from(
|
||||
["https://github.com/mozilla/jetstream-config"]
|
||||
["https://github.com/mozilla/metric-hub/tree/main/opmon"]
|
||||
)
|
||||
assert configs_collection.configs is not None
|
||||
assert len(configs_collection.configs.configs) == len(ConfigLoader.configs.configs)
|
||||
|
|
@ -29,6 +29,7 @@ class TestConfigLoader:
|
|||
def test_get_data_source(self):
|
||||
metric = list(ConfigLoader.configs.definitions[0].spec.metrics.definitions.values())[0]
|
||||
platform = ConfigLoader.configs.definitions[0].platform
|
||||
|
||||
assert (
|
||||
ConfigLoader.configs.get_data_source_definition(metric.data_source.name, platform)
|
||||
is not None
|
||||
|
|
|
|||
143
requirements.in
143
requirements.in
|
|
@ -4,127 +4,137 @@
|
|||
#
|
||||
# update_deps
|
||||
#
|
||||
attrs==21.4.0
|
||||
attrs==22.2.0
|
||||
# via
|
||||
# cattrs
|
||||
# jsonschema
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
# pytest
|
||||
black==22.1.0
|
||||
black==22.12.0
|
||||
# via pytest-black
|
||||
cachetools==5.0.0
|
||||
cachetools==5.3.0
|
||||
# via google-auth
|
||||
cattrs==1.10.0
|
||||
# via mozilla-opmon
|
||||
certifi==2021.10.8
|
||||
cattrs==22.2.0
|
||||
# via
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
certifi==2022.12.7
|
||||
# via requests
|
||||
charset-normalizer==2.0.12
|
||||
charset-normalizer==3.0.1
|
||||
# via requests
|
||||
click==8.0.4
|
||||
click==8.1.3
|
||||
# via
|
||||
# black
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
coverage[toml]==6.3.2
|
||||
coverage[toml]==7.0.5
|
||||
# via
|
||||
# mozilla-opmon
|
||||
# pytest-cov
|
||||
flake8==4.0.1
|
||||
exceptiongroup==1.1.0
|
||||
# via
|
||||
# cattrs
|
||||
# pytest
|
||||
flake8==6.0.0
|
||||
# via pytest-flake8
|
||||
gitdb==4.0.9
|
||||
gitdb==4.0.10
|
||||
# via gitpython
|
||||
gitpython==3.1.27
|
||||
# via mozilla-opmon
|
||||
google-api-core[grpc]==2.7.1
|
||||
gitpython==3.1.30
|
||||
# via
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
google-api-core[grpc]==2.11.0
|
||||
# via
|
||||
# google-cloud-bigquery
|
||||
# google-cloud-core
|
||||
google-auth==2.6.2
|
||||
google-auth==2.16.0
|
||||
# via
|
||||
# google-api-core
|
||||
# google-cloud-core
|
||||
google-cloud-bigquery==2.34.2
|
||||
google-cloud-bigquery==3.4.2
|
||||
# via mozilla-opmon
|
||||
google-cloud-core==2.2.3
|
||||
google-cloud-core==2.3.2
|
||||
# via google-cloud-bigquery
|
||||
google-crc32c==1.3.0
|
||||
google-crc32c==1.5.0
|
||||
# via google-resumable-media
|
||||
google-resumable-media==2.3.2
|
||||
google-resumable-media==2.4.1
|
||||
# via google-cloud-bigquery
|
||||
googleapis-common-protos==1.56.0
|
||||
googleapis-common-protos==1.58.0
|
||||
# via
|
||||
# google-api-core
|
||||
# grpcio-status
|
||||
grpcio==1.45.0
|
||||
grpcio==1.51.1
|
||||
# via
|
||||
# google-api-core
|
||||
# google-cloud-bigquery
|
||||
# grpcio-status
|
||||
# mozilla-opmon
|
||||
grpcio-status==1.45.0
|
||||
grpcio-status==1.51.1
|
||||
# via google-api-core
|
||||
idna==3.3
|
||||
idna==3.4
|
||||
# via requests
|
||||
importlib-resources==5.4.0
|
||||
importlib-resources==5.10.2
|
||||
# via jsonschema
|
||||
iniconfig==1.1.1
|
||||
iniconfig==2.0.0
|
||||
# via pytest
|
||||
isort==5.10.1
|
||||
isort==5.11.4
|
||||
# via mozilla-opmon
|
||||
mozilla-metric-config-parser==2022.11.2
|
||||
jinja2==3.1.2
|
||||
# via
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
jsonschema==4.17.3
|
||||
# via mozilla-opmon
|
||||
jinja2==3.1.0
|
||||
# via mozilla-opmon
|
||||
jsonschema==4.4.0
|
||||
# via mozilla-opmon
|
||||
markupsafe==2.1.1
|
||||
markupsafe==2.1.2
|
||||
# via jinja2
|
||||
mccabe==0.6.1
|
||||
mccabe==0.7.0
|
||||
# via flake8
|
||||
mozilla-metric-config-parser==2023.1.2
|
||||
# via mozilla-opmon
|
||||
# via -r -
|
||||
mypy==0.942
|
||||
mypy==0.991
|
||||
# via mozilla-opmon
|
||||
mypy-extensions==0.4.3
|
||||
# via
|
||||
# black
|
||||
# mypy
|
||||
packaging==21.3
|
||||
packaging==23.0
|
||||
# via
|
||||
# google-cloud-bigquery
|
||||
# pytest
|
||||
pathspec==0.9.0
|
||||
pathspec==0.10.3
|
||||
# via black
|
||||
platformdirs==2.5.1
|
||||
pkgutil-resolve-name==1.3.10
|
||||
# via jsonschema
|
||||
platformdirs==2.6.2
|
||||
# via black
|
||||
pluggy==1.0.0
|
||||
# via pytest
|
||||
proto-plus==1.20.3
|
||||
proto-plus==1.22.2
|
||||
# via google-cloud-bigquery
|
||||
protobuf==3.19.5
|
||||
protobuf==4.21.12
|
||||
# via
|
||||
# google-api-core
|
||||
# google-cloud-bigquery
|
||||
# googleapis-common-protos
|
||||
# grpcio-status
|
||||
# proto-plus
|
||||
py==1.11.0
|
||||
# via pytest
|
||||
pyasn1==0.4.8
|
||||
# via
|
||||
# pyasn1-modules
|
||||
# rsa
|
||||
pyasn1-modules==0.2.8
|
||||
# via google-auth
|
||||
pycodestyle==2.8.0
|
||||
pycodestyle==2.10.0
|
||||
# via flake8
|
||||
pydocstyle==6.1.1
|
||||
pydocstyle==6.3.0
|
||||
# via pytest-pydocstyle
|
||||
pyflakes==2.4.0
|
||||
pyflakes==3.0.1
|
||||
# via flake8
|
||||
pyparsing==3.0.7
|
||||
# via packaging
|
||||
pyrsistent==0.18.1
|
||||
pyrsistent==0.19.3
|
||||
# via jsonschema
|
||||
pytest==7.1.1
|
||||
pytest==7.2.1
|
||||
# via
|
||||
# mozilla-opmon
|
||||
# pytest-black
|
||||
|
|
@ -133,27 +143,29 @@ pytest==7.1.1
|
|||
# pytest-pydocstyle
|
||||
pytest-black==0.3.12
|
||||
# via mozilla-opmon
|
||||
pytest-cov==3.0.0
|
||||
pytest-cov==4.0.0
|
||||
# via mozilla-opmon
|
||||
pytest-flake8==1.1.1
|
||||
# via mozilla-opmon
|
||||
pytest-pydocstyle==2.3.0
|
||||
pytest-pydocstyle==2.3.2
|
||||
# via mozilla-opmon
|
||||
python-dateutil==2.8.2
|
||||
# via google-cloud-bigquery
|
||||
pytz==2022.1
|
||||
# via mozilla-opmon
|
||||
requests==2.27.1
|
||||
pytz==2022.7.1
|
||||
# via
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
requests==2.28.2
|
||||
# via
|
||||
# google-api-core
|
||||
# google-cloud-bigquery
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
rsa==4.8
|
||||
rsa==4.9
|
||||
# via google-auth
|
||||
six==1.16.0
|
||||
# via
|
||||
# google-auth
|
||||
# grpcio
|
||||
# python-dateutil
|
||||
smmap==5.0.0
|
||||
# via gitdb
|
||||
|
|
@ -161,6 +173,7 @@ snowballstemmer==2.2.0
|
|||
# via pydocstyle
|
||||
toml==0.10.2
|
||||
# via
|
||||
# mozilla-jetstream-config-parser
|
||||
# mozilla-opmon
|
||||
# pytest-black
|
||||
tomli==2.0.1
|
||||
|
|
@ -173,25 +186,25 @@ types-futures==3.3.8
|
|||
# via mozilla-opmon
|
||||
types-pkg-resources==0.1.3
|
||||
# via mozilla-opmon
|
||||
types-protobuf==3.19.15
|
||||
types-protobuf==4.21.0.3
|
||||
# via mozilla-opmon
|
||||
types-pytz==2021.3.6
|
||||
types-pytz==2022.7.1.0
|
||||
# via mozilla-opmon
|
||||
types-pyyaml==6.0.5
|
||||
types-pyyaml==6.0.12.3
|
||||
# via mozilla-opmon
|
||||
types-requests==2.27.14
|
||||
types-requests==2.28.11.8
|
||||
# via mozilla-opmon
|
||||
types-six==1.16.12
|
||||
types-six==1.16.21.4
|
||||
# via mozilla-opmon
|
||||
types-toml==0.10.4
|
||||
types-toml==0.10.8.1
|
||||
# via mozilla-opmon
|
||||
types-urllib3==1.26.11
|
||||
types-urllib3==1.26.25.4
|
||||
# via types-requests
|
||||
typing-extensions==4.1.1
|
||||
typing-extensions==4.4.0
|
||||
# via
|
||||
# black
|
||||
# mypy
|
||||
urllib3==1.26.9
|
||||
urllib3==1.26.14
|
||||
# via requests
|
||||
zipp==3.7.0
|
||||
zipp==3.11.0
|
||||
# via importlib-resources
|
||||
|
|
|
|||
976
requirements.txt
976
requirements.txt
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Загрузка…
Ссылка в новой задаче