fix(glam): add a more robust way to fetch glean tables (#5787)
* fix(glam): add a more robust way to fetch glean tables * Add jq to image * Remove useless space
This commit is contained in:
Родитель
044702503f
Коммит
e71fa15800
|
@ -22,7 +22,7 @@ FROM google/cloud-sdk:${GOOGLE_CLOUD_SDK_VERSION}-alpine AS google-cloud-sdk
|
|||
|
||||
FROM base
|
||||
# add bash for entrypoint
|
||||
RUN mkdir -p /usr/share/man/man1 && apt-get update -qqy && apt-get install -qqy bash git
|
||||
RUN mkdir -p /usr/share/man/man1 && apt-get update -qqy && apt-get install -qqy bash git jq
|
||||
COPY --from=google-cloud-sdk /google-cloud-sdk /google-cloud-sdk
|
||||
ENV PATH /google-cloud-sdk/bin:$PATH
|
||||
COPY --from=python-deps /usr/local /usr/local
|
||||
|
|
|
@ -63,13 +63,12 @@ function write_clients_daily_aggregates {
|
|||
# GLAM only supports tables with glean_ping_* schema.
|
||||
# Also excluding use_counters because they are not supported
|
||||
# and their generated query is too big to run
|
||||
tables=$(
|
||||
bq ls "$qualified" \
|
||||
| grep TABLE \
|
||||
| grep 'schema_id:glean_ping_[0-9]\+' \
|
||||
| grep -v use_counters \
|
||||
| awk '{print $1}'
|
||||
)
|
||||
tables=$(bq ls --format=json "$qualified" | \
|
||||
jq -r '.[] |
|
||||
select(.labels.schema_id | test("^glean_ping_[0-9]+")) |
|
||||
select(.type == "TABLE") |
|
||||
select(.tableReference.tableId | test("^use_counters.*") | not) |
|
||||
"\(.tableReference.tableId)"')
|
||||
# generate all of the schemas in parallel
|
||||
for table in $tables; do
|
||||
write_scalars "$product" "$dataset" "$table" "$dst_project" "$sql_dir" &
|
||||
|
|
Загрузка…
Ссылка в новой задаче