generate go client from openapi spec (#9502)

* generate go client from openapi spec

* move openapi codegen to seperate workflow

.github/workflows/cancel.yml

@@ -14,7 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+---
 name: Cancel
 on: [push, pull_request]
 jobs:

.github/workflows/openapi.yml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: OpenAPI Build
+on:
+  push:
+    branches: ['master']
+  pull_request:
+    branches: ['master']
+
+jobs:
+  test-openapi-client-generation:
+    name: "Test OpenAPI client generation"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+      - name: "Generate client codegen diff"
+        run: ./scripts/ci/openapi/client_codegen_diff.sh

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     rev: v1.1.7
     hooks:
       - id: forbid-tabs
-        exclude: ^docs/Makefile$
+        exclude: ^docs/Makefile$|^clients/gen/go.sh
       - id: insert-license
         name: Add license for all SQL files
         files: \.sql$

chart/values.yaml

@@ -223,12 +223,12 @@ webserver:
   extraNetworkPolicies: []
 
   resources: {}
-    # limits:
-    #   cpu: 100m
-    #   memory: 128Mi
-    # requests:
-    #   cpu: 100m
-    #   memory: 128Mi
+  #   limits:
+  #     cpu: 100m
+  #     memory: 128Mi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 128Mi
 
   # Create initial user.
   defaultUser:
@@ -265,12 +265,12 @@ flower:
   # Additional network policies as needed
   extraNetworkPolicies: []
   resources: {}
-    # limits:
-    #   cpu: 100m
-    #   memory: 128Mi
-    # requests:
-    #   cpu: 100m
-    #   memory: 128Mi
+  #   limits:
+  #     cpu: 100m
+  #     memory: 128Mi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 128Mi
 
   service:
     type: ClusterIP
@@ -281,12 +281,12 @@ statsd:
   # Additional network policies as needed
   extraNetworkPolicies: []
   resources: {}
-    # limits:
-    #   cpu: 100m
-    #   memory: 128Mi
-    # requests:
-    #   cpu: 100m
-    #   memory: 128Mi
+  #   limits:
+  #     cpu: 100m
+  #     memory: 128Mi
+  #   requests:
+  #     cpu: 100m
+  #     memory: 128Mi
 
   service:
     extraAnnotations: {}

clients/README.md

@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+Airflow OpenAPI clients
+=======================
+
+Supported languages:
+
+* [Golang](https://github.com/apache/airflow-client-go) generated through `./gen/go.sh`.
+
+
+## Dependencies
+
+All client generation scripts use [pre-commit](https://pre-commit.com/#install)
+to prepend license header to generated code.

clients/gen/common.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+OPENAPI_GENERATOR_CLI_VER=4.3.1
+GIT_USER=${GIT_USER:-apache}
+
+function gen_client {
+    lang=$1
+    shift
+    docker run --rm \
+        -v "${SPEC_PATH}:/spec" \
+        -v "${OUTPUT_DIR}:/output" \
+        openapitools/openapi-generator-cli:v${OPENAPI_GENERATOR_CLI_VER} \
+        generate \
+        --input-spec "/spec" \
+        --generator-name "${lang}" \
+        --git-user-id "${GIT_USER}" \
+        --output "/output" "$@"
+}

clients/gen/go.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ "$#" -ne 2 ]; then
+    echo "USAGE: $0 SPEC_PATH OUTPUT_DIR"
+    exit 1
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+# shellcheck source=./clients/gen/common.sh
+source "${SCRIPT_DIR}/common.sh"
+
+VERSION=1.0.0
+go_config=(
+    "packageVersion=${VERSION}"
+    "enumClassPrefix=true"
+)
+
+SPEC_PATH=$(realpath "$1")
+if [ ! -d "$2" ]; then
+    echo "$2 is not a valid directory or does not exist."
+    exit 1
+fi
+OUTPUT_DIR=$(realpath "$2")
+
+# create openapi ignore file to keep generated code clean
+cat <<EOF > "${OUTPUT_DIR}/.openapi-generator-ignore"
+.travis.yml
+git_push.sh
+EOF
+
+set -ex
+IFS=','
+
+SPEC_PATH="${SPEC_PATH}" \
+OUTPUT_DIR="${OUTPUT_DIR}" \
+    gen_client go \
+    --package-name airflow \
+    --git-repo-id airflow/clients/go/airflow \
+    --additional-properties "${go_config[*]}"
+
+# patch generated client to support problem HTTP API
+# this patch can be removed after following upstream patch gets merged:
+# https://github.com/OpenAPITools/openapi-generator/pull/6793
+cd "${OUTPUT_DIR}" && patch -b <<'EOF'
+--- client.go
++++ client.go
+@@ -37,7 +37,7 @@ import (
+ )
+
+ var (
+-	jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?json)`)
++	jsonCheck = regexp.MustCompile(`(?i:(?:application|text)/(?:vnd\.[^;]+\+)?(?:problem\+)?json)`)
+	xmlCheck  = regexp.MustCompile(`(?i:(?:application|text)/xml)`)
+ )
+EOF
+
+pushd "${OUTPUT_DIR}"
+    # prepend license headers
+    pre-commit run --all-files || true
+popd

scripts/ci/docker-compose/local.yml

@@ -18,8 +18,8 @@
 version: "2.2"
 services:
   airflow:
-    stdin_open: true # docker run -i
-    tty: true        # docker run -t
+    stdin_open: true  # docker run -i
+    tty: true  # docker run -t
     # We need to mount files an directories individually because some files
     # such apache_airflow.egg-info should not be mounted from host
     # we only mount those files that it makes sense to edit while developing

scripts/ci/openapi/client_codegen_diff.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SCRIPTS_CI_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=scripts/ci/libraries/_initialization.sh
+. "${SCRIPTS_CI_DIR}"/../libraries/_initialization.sh
+get_environment_for_builds_on_ci
+
+git remote add target "https://github.com/${CI_TARGET_REPO}"
+git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
+
+echo "Diffing openapi spec against ${CI_TARGET_BRANCH}..."
+
+SPEC_FILE=airflow/api_connexion/openapi/v1.yaml
+if ! git diff --name-only "${CI_TARGET_BRANCH}" HEAD | grep "${SPEC_FILE}" ; then
+    echo "no openapi spec change detected, going to skip client code gen validation."
+    exit 0
+fi
+
+echo "openapi spec change detected. comparing codegen diff..."
+
+mkdir -p ./clients/go/airflow
+./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go/airflow
+mkdir -p ./clients/go_target_branch/airflow
+git checkout "${CI_TARGET_BRANCH}" ./airflow/api_connexion/openapi/v1.yaml
+./clients/gen/go.sh ./airflow/api_connexion/openapi/v1.yaml ./clients/go_target_branch/airflow
+diff ./clients/go_target_branch/airflow ./clients/go/airflow || true