diff --git a/.gitignore b/.gitignore index b375d9d..4d439c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ - default.profraw stash/ diff --git a/slow_regressions/data/gen_test_data_query.py b/slow_regressions/data/gen_test_data_query.py index 8b5b568..cbbdf0b 100644 --- a/slow_regressions/data/gen_test_data_query.py +++ b/slow_regressions/data/gen_test_data_query.py @@ -23,13 +23,13 @@ def subdate_diff(subdate, **diff_kw): @click.option("--start_date", default=None) @click.option("--end_date", default=None) def fmt_test_data_query( - fill_yesterday=True, + fill_yesterday=False, backfill=False, end_date=None, bh_start_date=None, start_date=None, ): - if not backfill or fill_yesterday or start_date: + if not (backfill or fill_yesterday or start_date): raise ValueError( "At least specify `backfill` or `fill_yesterday`" ) diff --git a/slow_regressions/data/load_test_data.sh b/slow_regressions/data/load_test_data.sh new file mode 100644 index 0000000..105afaf --- /dev/null +++ b/slow_regressions/data/load_test_data.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -o xtrace + +DEST_PROJ="moz-fx-data-bq-data-science" +DEST_TABLE="$DEST_PROJ:wbeard.test_slow_regression_test_data" + +python gen_test_data_query.py \ + --fill_yesterday=True | \ + bq query --use_legacy_sql=false \ + --destination_table="$DEST_TABLE" \ + --replace=true \ + --project_id=moz-fx-data-derived-datasets \ No newline at end of file diff --git a/slow_regressions/data/test_data.sql b/slow_regressions/data/test_data.sql index c30ede7..3980cf8 100644 --- a/slow_regressions/data/test_data.sql +++ b/slow_regressions/data/test_data.sql @@ -1,6 +1,3 @@ -DECLARE BH_START_DATE, START_DATE, END_DATE date; -SET (BH_START_DATE, START_DATE, END_DATE) = ('{BH_START_DATE}', '{START_DATE}', '{END_DATE}'); --- SET (BH_START_DATE, START_DATE, END_DATE) = ('2019-06-01', '2019-06-01', CURRENT_DATE()); /* - deduped test version @@ -16,6 +13,15 @@ Common frameworks: 'job_resource_usage', 'vcs', 'build_metrics', 'browsertime', 'devtools', 'js-bench' */ +CREATE TEMP FUNCTION bh_start_date() returns date AS ( + '{BH_START_DATE}' +); + +CREATE TEMP FUNCTION major_vers(st string) AS ( + -- '10.0' => 10 + cast(regexp_extract(st, '(\\d+)\\.?') as int64) +); + -- Talos tests that measure release criteria CREATE TEMP FUNCTION isDesiredTalos(name STRING) AS ( name IN ('tabpaint', 'tabswitch', 'startup_about_home_paint', 'sessionrestore') @@ -104,7 +110,7 @@ with bh_base as and b.build.target.os = 'win' and b.build.target.locale = 'en-US' and build.source.product = 'firefox' - and date(build.build.date) >= START_DATE + and date(build.build.date) >= bh_start_date() -- START_DATE -- don't want dot-versions, like 72.0.2 and ndots(build.target.version) < 2 ) @@ -126,7 +132,7 @@ gbv1 as , gbv as ( select g1.*, - coalesce(next_release_date, END_DATE) as next_release_date + coalesce(next_release_date, '{END_DATE}') as next_release_date from gbv1 g1 left join ( select @@ -145,7 +151,7 @@ left join ( nth_version desc) as vers_ord, from (select day from - unnest( GENERATE_DATE_ARRAY(BH_START_DATE, END_DATE, interval 1 day) ) as day + unnest( GENERATE_DATE_ARRAY(bh_start_date(), '{END_DATE}', interval 1 day) ) as day ) days cross join gbv where @@ -224,14 +230,15 @@ left join ( ) as def on def.taskId = perf.taskId WHERE date(perf.time) > "2019-07-01" -- min date of db - and date(perf.time) between START_DATE and END_DATE + and date(perf.time) between '{START_DATE}' and '{END_DATE}' and isReleaseCriteria_notime(perf.framework, project, perf.platform, s.name) ) select b.*, - vers.version as bvers + vers.version as bvers, + major_vers(vers.version) as mvers from base b left join day_vers vers on date(b.time) = vers.day