Add DETERMINISTIC modifier to JS functions

This recently released feature allows query results using JS UDFs to be cached.

See https://issuetracker.google.com/issues/138310623

bigquery_etl/glam/templates/extract_probe_counts_v1.sql

@@ -2,7 +2,7 @@
 
 CREATE TEMP FUNCTION udf_js_flatten(histogram ARRAY<STRUCT<key STRING, value FLOAT64>>)
 RETURNS STRING
-LANGUAGE js
+DETERMINISTIC LANGUAGE js
 AS
   '''
     let obj = {};

bigquery_etl/glam/templates/probe_counts_v1.udf.histogram.sql

@@ -1,6 +1,6 @@
 CREATE TEMP FUNCTION udf_exponential_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
 RETURNS ARRAY<FLOAT64>
-LANGUAGE js
+DETERMINISTIC LANGUAGE js
 AS
   '''
   let logMax = Math.log(max);
@@ -22,7 +22,7 @@ AS
 
 CREATE TEMP FUNCTION udf_linear_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
 RETURNS ARRAY<FLOAT64>
-LANGUAGE js
+DETERMINISTIC LANGUAGE js
 AS
   '''
   let result = [0];
@@ -39,7 +39,7 @@ CREATE TEMP FUNCTION udf_functional_buckets(
   range_max INT64
 )
 RETURNS ARRAY<FLOAT64>
-LANGUAGE js
+DETERMINISTIC LANGUAGE js
 AS
   '''
   function sample_to_bucket_index(sample) {

sql/amo_prod/desktop_addons_by_client_v1/query.sql

@@ -1,7 +1,7 @@
 -- See https://stackoverflow.com/a/44885334/1260237
 -- and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent
 CREATE TEMPORARY FUNCTION decode_uri_component(path STRING)
-RETURNS STRING
+RETURNS STRING DETERMINISTIC
 LANGUAGE js
 AS
   """

sql/glam_etl/org_mozilla_fenix_glam_release__extract_probe_counts_v1/query.sql

@@ -1,6 +1,6 @@
 -- query for org_mozilla_fenix_glam_release__extract_probe_counts_v1;
 CREATE TEMP FUNCTION udf_js_flatten(histogram ARRAY<STRUCT<key STRING, value FLOAT64>>)
-RETURNS STRING
+RETURNS STRING DETERMINISTIC
 LANGUAGE js
 AS
   '''

sql/glam_etl/org_mozilla_fenix_glam_release__histogram_probe_counts_v1/query.sql

@@ -1,6 +1,6 @@
 -- query for org_mozilla_fenix_glam_release__histogram_probe_counts_v1;
 CREATE TEMP FUNCTION udf_exponential_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
-RETURNS ARRAY<FLOAT64>
+RETURNS ARRAY<FLOAT64> DETERMINISTIC
 LANGUAGE js
 AS
   '''
@@ -22,7 +22,7 @@ AS
 ''';
 
 CREATE TEMP FUNCTION udf_linear_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
-RETURNS ARRAY<FLOAT64>
+RETURNS ARRAY<FLOAT64> DETERMINISTIC
 LANGUAGE js
 AS
   '''
@@ -39,7 +39,7 @@ CREATE TEMP FUNCTION udf_functional_buckets(
   buckets_per_magnitude INT64,
   range_max INT64
 )
-RETURNS ARRAY<FLOAT64>
+RETURNS ARRAY<FLOAT64> DETERMINISTIC
 LANGUAGE js
 AS
   '''

sql/telemetry_derived/asn_aggregates_v1/query.sql

@@ -3,7 +3,7 @@
 -- FROM `static.geoip2_isp_blocks_ipv4`
 -- WHERE NET.IP_TRUNC(NET.SAFE_IP_FROM_STRING("100.1.0.255"), CAST(SPLIT(network, "/")[OFFSET(1)] AS INT64)) = NET.SAFE_IP_FROM_STRING(SPLIT(network, "/")[OFFSET(0)])
 CREATE TEMPORARY FUNCTION get_client_ip(xff STRING, remote_address STRING, pipeline_proxy STRING)
-RETURNS STRING
+RETURNS STRING DETERMINISTIC
 LANGUAGE js
 AS
   """

sql/telemetry_derived/clients_daily_histogram_aggregates_content_v1/query.sql

@@ -1,6 +1,6 @@
 -- Query generated by: templates/clients_daily_histogram_aggregates.sql.py
 CREATE TEMPORARY FUNCTION get_keyval_pairs(y STRING)
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   '''
@@ -14,7 +14,7 @@ AS
 
         -- convert a string like '[5, 6, 7]' to an array struct
 CREATE TEMPORARY FUNCTION string_to_arr(y STRING)
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   '''

sql/telemetry_derived/clients_daily_histogram_aggregates_parent_v1/query.sql

@@ -1,6 +1,6 @@
 -- Query generated by: templates/clients_daily_histogram_aggregates.sql.py
 CREATE TEMPORARY FUNCTION get_keyval_pairs(y STRING)
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   '''
@@ -14,7 +14,7 @@ AS
 
         -- convert a string like '[5, 6, 7]' to an array struct
 CREATE TEMPORARY FUNCTION string_to_arr(y STRING)
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   '''

sql/telemetry_derived/clients_histogram_probe_counts_v1/query.sql

@@ -1,6 +1,6 @@
 CREATE TEMP FUNCTION udf_exponential_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
 RETURNS ARRAY<FLOAT64>
-LANGUAGE js AS
+DETERMINISTIC LANGUAGE js AS
 '''
   let logMax = Math.log(max);
   let current = min;
@@ -21,7 +21,7 @@ LANGUAGE js AS
 
 CREATE TEMP FUNCTION udf_linear_buckets(min FLOAT64, max FLOAT64, nBuckets FLOAT64)
 RETURNS ARRAY<FLOAT64>
-LANGUAGE js AS
+DETERMINISTIC LANGUAGE js AS
 '''
   let result = [0];
   for (let i = 1; i < Math.min(nBuckets, max, 10000); i++) {

sql/telemetry_derived/clients_scalar_probe_counts_v1/query.sql

@@ -21,7 +21,7 @@ RETURNS ARRAY<STRUCT<key STRING, value FLOAT64>> AS (
 
 CREATE TEMPORARY FUNCTION udf_generate_buckets(min_bucket FLOAT64, max_bucket FLOAT64, num_buckets INT64)
 RETURNS ARRAY<STRING>
-LANGUAGE js
+DETERMINISTIC LANGUAGE js
 AS
 '''
   let bucket_size = (max_bucket - min_bucket) / num_buckets;

sql/telemetry_derived/glam_client_probe_counts_extract_v1/query.sql

@@ -1,5 +1,5 @@
 CREATE TEMP FUNCTION udf_js_flatten(histogram ARRAY<STRUCT<key STRING, value FLOAT64>>)
-RETURNS STRING
+RETURNS STRING DETERMINISTIC
 LANGUAGE js
 AS
   '''

udf_js/crc32/udf.sql

@@ -12,7 +12,7 @@ See https://en.wikipedia.org/wiki/Cyclic_redundancy_check
 
 */
 CREATE OR REPLACE FUNCTION udf_js.crc32(data STRING)
-RETURNS INT64
+RETURNS INT64 DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/glean_percentile/udf.sql

@@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION udf_js.glean_percentile(
   histogram ARRAY<STRUCT<key STRING, value FLOAT64>>,
   type STRING
 )
-RETURNS FLOAT64
+RETURNS FLOAT64 DETERMINISTIC
 LANGUAGE js
 AS
   '''

udf_js/gunzip/udf.sql

@@ -7,7 +7,7 @@ the atob function for decoding base64.
 
 */
 CREATE OR REPLACE FUNCTION udf_js.gunzip(input BYTES)
-RETURNS STRING
+RETURNS STRING DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/jackknife_mean_ci/udf.sql

@@ -16,7 +16,7 @@ CREATE OR REPLACE FUNCTION udf_js.jackknife_mean_ci(
   n_buckets INT64,
   values_per_bucket ARRAY<FLOAT64>
 )
-RETURNS STRUCT<low FLOAT64, high FLOAT64, pm FLOAT64>
+RETURNS STRUCT<low FLOAT64, high FLOAT64, pm FLOAT64> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/jackknife_ratio_ci/udf.sql

@@ -37,7 +37,7 @@ CREATE OR REPLACE FUNCTION udf_js.jackknife_ratio_ci(
   n_buckets INT64,
   values_per_bucket ARRAY<STRUCT<numerator FLOAT64, denominator FLOAT64>>
 )
-RETURNS STRUCT<mean FLOAT64, low FLOAT64, high FLOAT64, pm FLOAT64>
+RETURNS STRUCT<mean FLOAT64, low FLOAT64, high FLOAT64, pm FLOAT64> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/jackknife_sum_ci/udf.sql

@@ -32,7 +32,7 @@ GROUP BY
 
 */
 CREATE OR REPLACE FUNCTION udf_js.jackknife_sum_ci(n_buckets INT64, counts_per_bucket ARRAY<INT64>)
-RETURNS STRUCT<total INT64, low INT64, high INT64, pm INT64>
+RETURNS STRUCT<total INT64, low INT64, high INT64, pm INT64> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/json_extract_events/udf.sql

@@ -9,7 +9,7 @@ RETURNS ARRAY<
     event_string_value STRING,
     event_map_values ARRAY<STRUCT<key STRING, value STRING>>
   >
->
+> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/json_extract_histogram/udf.sql

@@ -14,7 +14,7 @@ RETURNS STRUCT<
   `sum` INT64,
   `range` ARRAY<INT64>,
   `values` ARRAY<STRUCT<key INT64, value INT64>>
->
+> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/json_extract_keyed_histogram/udf.sql

@@ -19,7 +19,7 @@ RETURNS ARRAY<
     `range` ARRAY<INT64>,
     `values` ARRAY<STRUCT<key INT64, value INT64>>
   >
->
+> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/json_extract_missing_cols/udf.sql

@@ -24,7 +24,7 @@ CREATE OR REPLACE FUNCTION udf_js.json_extract_missing_cols(
   indicates_node ARRAY<STRING>,
   known_nodes ARRAY<STRING>
 )
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/main_summary_active_addons/udf.sql

@@ -59,6 +59,7 @@ RETURNS ARRAY<
     multiprocess_compatible BOOL
   >
 >
+DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/main_summary_addon_scalars/udf.sql

@@ -20,7 +20,7 @@ RETURNS STRUCT<
   >,
   uint_addon_scalars ARRAY<STRUCT<key STRING, value INT64>>,
   boolean_addon_scalars ARRAY<STRUCT<key STRING, value BOOL>>
->
+> DETERMINISTIC
 LANGUAGE js
 AS
   """

udf_js/main_summary_disabled_addons/udf.sql

@@ -11,7 +11,7 @@ CREATE OR REPLACE FUNCTION udf_js.main_summary_disabled_addons(
   active_addon_ids ARRAY<STRING>,
   addon_details_json STRING
 )
-RETURNS ARRAY<STRING>
+RETURNS ARRAY<STRING> DETERMINISTIC
 LANGUAGE js
 AS
   """