diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml index 5a26c0f0c4..e831a2830b 100644 --- a/.github/workflows/cancel.yml +++ b/.github/workflows/cancel.yml @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +--- name: Cancel on: [push, pull_request] jobs: diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 0000000000..915c40e06b --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: OpenAPI Build +on: + push: + branches: ['master'] + pull_request: + branches: ['master'] + +jobs: + test-openapi-client-generation: + name: "Test OpenAPI client generation" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: 0 + - name: "Generate client codegen diff" + run: ./scripts/ci/openapi/client_codegen_diff.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e5c36584a6..e0bd20161b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: rev: v1.1.7 hooks: - id: forbid-tabs - exclude: ^docs/Makefile$ + exclude: ^docs/Makefile$|^clients/gen/go.sh - id: insert-license name: Add license for all SQL files files: \.sql$ diff --git a/chart/values.yaml b/chart/values.yaml index 8fa7ab6329..f0d5fa4cb9 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -223,12 +223,12 @@ webserver: extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi # Create initial user. defaultUser: @@ -265,12 +265,12 @@ flower: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: type: ClusterIP @@ -281,12 +281,12 @@ statsd: # Additional network policies as needed extraNetworkPolicies: [] resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi service: extraAnnotations: {} diff --git a/clients/README.md b/clients/README.md new file mode 100644 index 0000000000..668a98a04c --- /dev/null +++ b/clients/README.md @@ -0,0 +1,30 @@ + +Airflow OpenAPI clients +======================= + +Supported languages: + +* [Golang](https://github.com/apache/airflow-client-go) generated through `./gen/go.sh`. + + +## Dependencies + +All client generation scripts use [pre-commit](https://pre-commit.com/#install) +to prepend license header to generated code. diff --git a/clients/gen/common.sh b/clients/gen/common.sh new file mode 100755 index 0000000000..46fc645518 --- /dev/null +++ b/clients/gen/common.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OPENAPI_GENERATOR_CLI_VER=4.3.1 +GIT_USER=${GIT_USER:-apache} + +function gen_client { + lang=$1 + shift + docker run --rm \ + -v "${SPEC_PATH}:/spec" \ + -v "${OUTPUT_DIR}:/output" \ + openapitools/openapi-generator-cli:v${OPENAPI_GENERATOR_CLI_VER} \ + generate \ + --input-spec "/spec" \ + --generator-name "${lang}" \ + --git-user-id "${GIT_USER}" \ + --output "/output" "$@" +} diff --git a/clients/gen/go.sh b/clients/gen/go.sh new file mode 100755 index 0000000000..8c58efae84 --- /dev/null +++ b/clients/gen/go.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ "$#" -ne 2 ]; then + echo "USAGE: $0 SPEC_PATH OUTPUT_DIR" + exit 1 +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# shellcheck source=./clients/gen/common.sh +source "${SCRIPT_DIR}/common.sh" + +VERSION=1.0.0 +go_config=( + "packageVersion=${VERSION}" + "enumClassPrefix=true" +) + +SPEC_PATH=$(realpath "$1") +if [ ! -d "$2" ]; then + echo "$2 is not a valid directory or does not exist." + exit 1 +fi +OUTPUT_DIR=$(realpath "$2") + +# create openapi ignore file to keep generated code clean +cat < "${OUTPUT_DIR}/.openapi-generator-ignore" +.travis.yml +git_push.sh +EOF + +set -ex +IFS=',' + +SPEC_PATH="${SPEC_PATH}" \ +OUTPUT_DIR="${OUTPUT_DIR}" \ + gen_client go \ + --package-name airflow \ + --git-repo-id airflow/clients/go/airflow \ + --additional-properties "${go_config[*]}" + +# patch generated client to support problem HTTP API +# this patch can be removed after following upstream patch gets merged: +# https://github.com/OpenAPITools/openapi-generator/pull/6793 +cd "${OUTPUT_DIR}" && patch -b <<'EOF' +--- client.go ++++ client.go +@@ -37,7 +37,7 @@ import ( + ) + + var ( +- jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`) ++ jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?(?:problem\+)?json)`) + xmlCheck = regexp.MustCompile(`(?i:(?:application|text)/xml)`) + ) +EOF + +pushd "${OUTPUT_DIR}" + # prepend license headers + pre-commit run --all-files || true +popd diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index 0e24db7c2d..7ba83a518c 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -18,8 +18,8 @@ version: "2.2" services: airflow: - stdin_open: true # docker run -i - tty: true # docker run -t + stdin_open: true # docker run -i + tty: true # docker run -t # We need to mount files an directories individually because some files # such apache_airflow.egg-info should not be mounted from host # we only mount those files that it makes sense to edit while developing diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh new file mode 100755 index 0000000000..6b846fe3e2 --- /dev/null +++ b/scripts/ci/openapi/client_codegen_diff.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}") + +# shellcheck source=scripts/ci/libraries/_initialization.sh +. "${SCRIPTS_CI_DIR}"/../libraries/_initialization.sh +get_environment_for_builds_on_ci + +git remote add target "https://github.com/${CI_TARGET_REPO}" +git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1 + +echo "Diffing openapi spec against ${CI_TARGET_BRANCH}..." + +SPEC_FILE=airflow/api_connexion/openapi/v1.yaml +if ! git diff --name-only "${CI_TARGET_BRANCH}" HEAD | grep "${SPEC_FILE}" ; then + echo "no openapi spec change detected, going to skip client code gen validation." + exit 0 +fi + +echo "openapi spec change detected. comparing codegen diff..." + +mkdir -p ./clients/go/airflow +./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow +mkdir -p ./clients/go_target_branch/airflow +git checkout "${CI_TARGET_BRANCH}" ./airflow/api_connexion/openapi/v1.yaml +./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go_target_branch/airflow +diff ./clients/go_target_branch/airflow ./clients/go/airflow || true