49 строки
2.5 KiB
Bash
Executable File
49 строки
2.5 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
|
|
# Use credentials from environment
|
|
if [ -n "$GCLOUD_SERVICE_KEY" ]; then
|
|
echo "DEBUG: SETTING UP CREDS BLOCK HIT" # TODO: once the issue is solved this can be removed.
|
|
# Google's client libraries will check for GOOGLE_APPLICATION_CREDENTIALS
|
|
# and use a file in that location for credentials if present;
|
|
# See https://cloud.google.com/docs/authentication/production
|
|
export GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-/tmp/gcp.json}"
|
|
# Only write the credentials file if it doesn't exist yet or doesn't match the current credentials.
|
|
if [ ! -e "$GOOGLE_APPLICATION_CREDENTIALS" ] || [ "$(cat "$GOOGLE_APPLICATION_CREDENTIALS" | md5sum)" != "$(echo "$GCLOUD_SERVICE_KEY" | md5sum)" ]; then
|
|
echo "$GCLOUD_SERVICE_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS"
|
|
echo "DEBUG: Wrote $(echo "$GCLOUD_SERVICE_KEY" | wc -c) bytes to '$GOOGLE_APPLICATION_CREDENTIALS'."
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ] && which gcloud > /dev/null; then
|
|
echo "DEBUG: Authenticating with GCP key file '$GOOGLE_APPLICATION_CREDENTIALS' containing $(cat $GOOGLE_APPLICATION_CREDENTIALS | wc -c) bytes."
|
|
gcloud --verbosity=debug --quiet auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
|
|
python -W ignore -c 'import google.auth; print("project = ", google.auth.default()[1])' >> ~/.config/gcloud/configurations/config_default
|
|
sed "1i--project_id=$(gcloud config get-value project)" -i ~/.bigqueryrc
|
|
fi
|
|
|
|
if [ "$#" = 0 ]; then
|
|
# Default to 8 workers because most of the tests are io-bound
|
|
# When using pytest-xdist, total worker initialization time scales
|
|
# linearly to the number of workers. 8 workers seems to bring the best
|
|
# performance. See https://github.com/pytest-dev/pytest-xdist/issues/346
|
|
exec pytest --black --flake8 --isort --mypy-ignore-missing-imports --pydocstyle -n 8
|
|
elif [ "${1:0:1}" = - ]; then
|
|
# First argument is a flag, assume intended executable is pytest
|
|
exec pytest "$@"
|
|
elif [ "$1" = "query" ]; then
|
|
# For an invocation like:
|
|
# query [options] FILE
|
|
# we dispatch to a script that inspects metadata and emulates the following call:
|
|
# bq query [options] < FILE
|
|
exec script/bqetl query run "${@: -1}" "${@:1:$#-1}"
|
|
elif [ "$XCOM_PUSH" = "true" ]; then
|
|
# KubernetesPodOperator will extract the contents of /airflow/xcom/return.json as an xcom
|
|
# if the xcom_push parameter is true
|
|
mkdir -p /airflow/xcom/ && touch /airflow/xcom/return.json
|
|
exec "$@" | tee /airflow/xcom/return.json
|
|
else
|
|
exec "$@"
|
|
fi
|