From 6e15d70cce8ea97a2ce11b950937c2b6f6e42557 Mon Sep 17 00:00:00 2001
From: wcbeard <wcbeard10@gmail.com>
Date: Thu, 9 Jul 2020 09:45:26 -0400
Subject: [PATCH] dl versions and scripts

---
 .gitignore                                   |  1 -
 slow_regressions/data/gen_test_data_query.py |  4 ++--
 slow_regressions/data/load_test_data.sh      | 12 ++++++++++
 slow_regressions/data/test_data.sql          | 23 +++++++++++++-------
 4 files changed, 29 insertions(+), 11 deletions(-)
 create mode 100644 slow_regressions/data/load_test_data.sh

diff --git a/.gitignore b/.gitignore
index b375d9d..4d439c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,2 @@
-
 default.profraw
 stash/
diff --git a/slow_regressions/data/gen_test_data_query.py b/slow_regressions/data/gen_test_data_query.py
index 8b5b568..cbbdf0b 100644
--- a/slow_regressions/data/gen_test_data_query.py
+++ b/slow_regressions/data/gen_test_data_query.py
@@ -23,13 +23,13 @@ def subdate_diff(subdate, **diff_kw):
 @click.option("--start_date", default=None)
 @click.option("--end_date", default=None)
 def fmt_test_data_query(
-    fill_yesterday=True,
+    fill_yesterday=False,
     backfill=False,
     end_date=None,
     bh_start_date=None,
     start_date=None,
 ):
-    if not backfill or fill_yesterday or start_date:
+    if not (backfill or fill_yesterday or start_date):
         raise ValueError(
             "At least specify `backfill` or `fill_yesterday`"
         )
diff --git a/slow_regressions/data/load_test_data.sh b/slow_regressions/data/load_test_data.sh
new file mode 100644
index 0000000..105afaf
--- /dev/null
+++ b/slow_regressions/data/load_test_data.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -o xtrace
+
+DEST_PROJ="moz-fx-data-bq-data-science"
+DEST_TABLE="$DEST_PROJ:wbeard.test_slow_regression_test_data"
+
+python gen_test_data_query.py \
+    --fill_yesterday=True | \
+    bq query --use_legacy_sql=false \
+    --destination_table="$DEST_TABLE" \
+    --replace=true \
+    --project_id=moz-fx-data-derived-datasets
\ No newline at end of file
diff --git a/slow_regressions/data/test_data.sql b/slow_regressions/data/test_data.sql
index c30ede7..3980cf8 100644
--- a/slow_regressions/data/test_data.sql
+++ b/slow_regressions/data/test_data.sql
@@ -1,6 +1,3 @@
-DECLARE BH_START_DATE, START_DATE, END_DATE date;
-SET (BH_START_DATE, START_DATE, END_DATE) = ('{BH_START_DATE}', '{START_DATE}', '{END_DATE}');
--- SET (BH_START_DATE, START_DATE, END_DATE) = ('2019-06-01', '2019-06-01', CURRENT_DATE());
 /*
 - deduped test version
 
@@ -16,6 +13,15 @@ Common frameworks: 'job_resource_usage', 'vcs', 'build_metrics',
   'browsertime', 'devtools', 'js-bench'
 */
 
+CREATE TEMP FUNCTION bh_start_date() returns date AS (
+  '{BH_START_DATE}'
+);
+
+CREATE TEMP FUNCTION major_vers(st string) AS (
+  -- '10.0' => 10
+  cast(regexp_extract(st, '(\\d+)\\.?') as int64)
+);
+
 -- Talos tests that measure release criteria
 CREATE TEMP FUNCTION isDesiredTalos(name STRING) AS (
     name IN ('tabpaint', 'tabswitch', 'startup_about_home_paint', 'sessionrestore')
@@ -104,7 +110,7 @@ with bh_base as
     and b.build.target.os = 'win' 
     and b.build.target.locale = 'en-US' 
     and build.source.product = 'firefox' 
-    and date(build.build.date) >= START_DATE
+    and date(build.build.date) >= bh_start_date() -- START_DATE
     -- don't want dot-versions, like 72.0.2
     and ndots(build.target.version) < 2
 )
@@ -126,7 +132,7 @@ gbv1 as
 , gbv as (
 select
   g1.*,
-  coalesce(next_release_date, END_DATE) as next_release_date
+  coalesce(next_release_date, '{END_DATE}') as next_release_date
 from gbv1 g1
 left join (
   select
@@ -145,7 +151,7 @@ left join (
     nth_version desc) as vers_ord,
   from (select day 
         from
-            unnest( GENERATE_DATE_ARRAY(BH_START_DATE, END_DATE, interval 1 day) ) as day 
+            unnest( GENERATE_DATE_ARRAY(bh_start_date(), '{END_DATE}', interval 1 day) ) as day 
     )
     days cross join gbv 
   where
@@ -224,14 +230,15 @@ left join (
   ) as def on def.taskId = perf.taskId
 WHERE
   date(perf.time) > "2019-07-01" -- min date of db
-  and date(perf.time) between START_DATE and END_DATE
+  and date(perf.time) between '{START_DATE}' and '{END_DATE}'
   and isReleaseCriteria_notime(perf.framework, project, perf.platform, s.name)
 )
 
 
 select
   b.*,
-  vers.version as bvers
+  vers.version as bvers,
+  major_vers(vers.version) as mvers
 from base b
 left join day_vers vers
   on date(b.time) = vers.day