Optimizes image verification steps. (#14780)
So far we had matrix of builds that verified images - each image was verified by separate matrix-based job and those verifications were run after all images were alredy available. This step optimizes it. Those steps are run in the same job as "waiting for image", also they run in parallel which will make them a bit faster. This verification is fast and it can be run on any machine in parallel without any problems.
This commit is contained in:
Jarek Potiuk
GitHub
Родитель
3bbf9aea0b
Коммит
c59ab1ddcd
|
|
@ -272,38 +272,9 @@ jobs:
|
|||
# The job will set the output "githubRegistry" - result of auto-detect which registry has
|
||||
# been used by checking where the image can be downloaded from.
|
||||
#
|
||||
run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
|
||||
run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
|
||||
|
||||
|
||||
verify-ci-images:
|
||||
timeout-minutes: 20
|
||||
name: "Verify CI Image Py${{matrix.python-version}}"
|
||||
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
|
||||
needs: [build-info, ci-images]
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
|
||||
env:
|
||||
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
|
||||
BACKEND: sqlite
|
||||
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
|
||||
if: needs.build-info.outputs.image-build == 'true'
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
if: needs.build-info.outputs.waitForImage == 'true'
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: "Free space"
|
||||
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
|
||||
if: |
|
||||
needs.build-info.outputs.waitForImage == 'true'
|
||||
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
|
||||
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
|
||||
- name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
|
||||
run: ./scripts/ci/images/ci_verify_ci_image.sh
|
||||
if: needs.build-info.outputs.waitForImage == 'true'
|
||||
|
||||
static-checks:
|
||||
timeout-minutes: 30
|
||||
name: "Static checks"
|
||||
|
|
@ -948,34 +919,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
|
|||
env:
|
||||
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
|
||||
${{needs.build-info.outputs.pythonVersionsListAsString}}
|
||||
run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
|
||||
|
||||
verify-prod-images:
|
||||
timeout-minutes: 20
|
||||
name: "Verify Prod Image Py${{matrix.python-version}}"
|
||||
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
|
||||
needs: [build-info, prod-images]
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
|
||||
env:
|
||||
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
|
||||
BACKEND: sqlite
|
||||
GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
|
||||
steps:
|
||||
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
|
||||
uses: actions/checkout@v2
|
||||
if: needs.build-info.outputs.waitForImage == 'true'
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: "Free space"
|
||||
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
|
||||
if: needs.build-info.outputs.waitForImage == 'true'
|
||||
- name: "Prepare PROD Image"
|
||||
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
|
||||
- name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
|
||||
run: ./scripts/ci/images/ci_verify_prod_image.sh
|
||||
if: needs.build-info.outputs.waitForImage == 'true'
|
||||
run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
|
||||
|
||||
tests-kubernetes:
|
||||
timeout-minutes: 50
|
||||
|
|
@ -1068,7 +1012,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
|
|||
- prepare-provider-packages
|
||||
- test-provider-packages-released-airflow
|
||||
- prod-images
|
||||
- verify-prod-images
|
||||
- docs
|
||||
if: >
|
||||
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ||
|
||||
|
|
@ -1116,7 +1059,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
|
|||
- tests-mysql
|
||||
- tests-kubernetes
|
||||
- ci-images
|
||||
- verify-ci-images
|
||||
- docs
|
||||
if: >
|
||||
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ||
|
||||
|
|
@ -1204,8 +1146,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
|
|||
needs:
|
||||
- build-info
|
||||
- constraints
|
||||
- verify-ci-images
|
||||
- verify-prod-images
|
||||
- ci-images
|
||||
- prod-images
|
||||
- static-checks
|
||||
- static-checks-pylint
|
||||
- tests-sqlite
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# shellcheck source=scripts/ci/libraries/_script_init.sh
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh"
|
||||
|
||||
function pull_ci_image() {
|
||||
local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
start_end::group_start "Pulling ${image_name_with_tag} image"
|
||||
|
||||
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
|
||||
start_end::group_end
|
||||
|
||||
}
|
||||
|
||||
|
||||
build_images::prepare_ci_build
|
||||
|
||||
pull_ci_image
|
||||
|
||||
verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
# shellcheck source=scripts/ci/libraries/_script_init.sh
|
||||
. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh"
|
||||
|
||||
function pull_prod_image() {
|
||||
local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
|
||||
|
||||
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
|
||||
start_end::group_end
|
||||
}
|
||||
|
||||
build_images::prepare_prod_build
|
||||
|
||||
pull_prod_image
|
||||
|
||||
verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
|
||||
|
|
@ -15,12 +15,27 @@
|
|||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
set -euo pipefail
|
||||
|
||||
# We cannot perform full initialization because it will be done later in the "single run" scripts
|
||||
# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed
|
||||
LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd)
|
||||
# shellcheck source=scripts/ci/libraries/_all_libs.sh
|
||||
source "${LIBRARIES_DIR}/_all_libs.sh"
|
||||
|
||||
initialization::set_output_color_variables
|
||||
|
||||
parallel::make_sure_gnu_parallel_is_installed
|
||||
|
||||
echo
|
||||
echo "Waiting for all CI images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
|
||||
echo
|
||||
|
||||
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
|
||||
do
|
||||
export PYTHON_MAJOR_MINOR_VERSION
|
||||
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_ci_image.sh"
|
||||
done
|
||||
parallel::initialize_monitoring
|
||||
|
||||
parallel::monitor_progress
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
parallel --results "${PARALLEL_MONITORED_DIR}" \
|
||||
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_ci_image.sh" ::: \
|
||||
${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
|
||||
|
|
@ -15,13 +15,27 @@
|
|||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
set -euo pipefail
|
||||
|
||||
# We cannot perform full initialization because it will be done later in the "single run" scripts
|
||||
# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed
|
||||
LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd)
|
||||
# shellcheck source=scripts/ci/libraries/_all_libs.sh
|
||||
source "${LIBRARIES_DIR}/_all_libs.sh"
|
||||
|
||||
initialization::set_output_color_variables
|
||||
|
||||
parallel::make_sure_gnu_parallel_is_installed
|
||||
|
||||
echo
|
||||
echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
|
||||
echo
|
||||
|
||||
for PYTHON_MAJOR_MINOR_VERSION in ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
|
||||
do
|
||||
export PYTHON_MAJOR_MINOR_VERSION
|
||||
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_prod_image.sh"
|
||||
done
|
||||
parallel::initialize_monitoring
|
||||
|
||||
parallel::monitor_progress
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
parallel --results "${PARALLEL_MONITORED_DIR}" \
|
||||
"$( dirname "${BASH_SOURCE[0]}" )/ci_wait_for_and_verify_prod_image.sh" ::: \
|
||||
${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}
|
||||
|
|
@ -15,9 +15,28 @@
|
|||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
if [[ $1 == "" ]]; then
|
||||
>&2 echo "Requires python MAJOR/MINOR version as first parameter"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PYTHON_MAJOR_MINOR_VERSION=$1
|
||||
shift
|
||||
|
||||
|
||||
# shellcheck source=scripts/ci/libraries/_script_init.sh
|
||||
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
|
||||
|
||||
function pull_ci_image() {
|
||||
local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
start_end::group_start "Pulling ${image_name_with_tag} image"
|
||||
|
||||
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
|
||||
start_end::group_end
|
||||
|
||||
}
|
||||
|
||||
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
|
||||
|
||||
build_image::configure_docker_registry
|
||||
|
|
@ -29,4 +48,10 @@ start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear"
|
|||
push_pull_remove_images::wait_for_github_registry_image \
|
||||
"${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
|
||||
build_images::prepare_ci_build
|
||||
|
||||
pull_ci_image
|
||||
|
||||
verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
|
||||
|
||||
start_end::group_end
|
||||
|
|
@ -15,17 +15,41 @@
|
|||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
if [[ $1 == "" ]]; then
|
||||
>&2 echo "Requires python MAJOR/MINOR version as first parameter"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PYTHON_MAJOR_MINOR_VERSION=$1
|
||||
shift
|
||||
|
||||
|
||||
# shellcheck source=scripts/ci/libraries/_script_init.sh
|
||||
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
|
||||
|
||||
function pull_prod_image() {
|
||||
local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
|
||||
|
||||
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
|
||||
start_end::group_end
|
||||
}
|
||||
|
||||
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
|
||||
|
||||
build_image::configure_docker_registry
|
||||
|
||||
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
|
||||
|
||||
start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE_NAME} image to appear"
|
||||
|
||||
push_pull_remove_images::wait_for_github_registry_image \
|
||||
"${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
|
||||
|
||||
start_end::group_end
|
||||
|
||||
build_images::prepare_prod_build
|
||||
|
||||
pull_prod_image
|
||||
|
||||
verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
|
||||
|
|
@ -71,6 +71,7 @@ function parallel::monitor_loop() {
|
|||
for directory in "${PARALLEL_MONITORED_DIR}"/*/*
|
||||
do
|
||||
parallel_process=$(basename "${directory}")
|
||||
|
||||
echo "${COLOR_BLUE}### The last lines for ${parallel_process} process ###${COLOR_RESET}"
|
||||
echo
|
||||
tail -2 "${directory}/stdout" || true
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче