test: e2e test (#143)

Test:
- added e2e test for Linux, windows and macOS
- includes 14 test cases: imported pem/pfx certs chain, unordered
pem/pfx certs chain, azure key vault csr pem/pfx certs chain,
self-signed pem/pfx certs, partial pem/pfx certs chain with local cert
bundle and corresponding invalid cases.
- the E2E automatically run for PRs from `Azure/notation-azure-kv` repo
and will be skipped for folked repo

Additional files:
- dockerfile for build distribution registry on windows
- zot configuration for local registry on macOS
- generate-certs.sh script for create certificate on azure key vault for
future re-generating certificates.

---------

Signed-off-by: Junjie Gao <junjiegao@microsoft.com>

.github/workflows/test.yml

@@ -32,14 +32,11 @@ jobs:
           DEFAULT_BRANCH: main
           DEFAULT_WORKSPACE: ./Notation.Plugin.AzureKeyVault
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          FILTER_REGEX_EXCLUDE: .*Tests/.*
+          FILTER_REGEX_EXCLUDE: '.*Tests/.*|.*.yml|.*/scripts/generate-certs.sh'
           VALIDATE_MARKDOWN: false
-  build:
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    name: "Build"
-    runs-on: ${{ matrix.os }}
+  test:
+    name: Unit Testing and Build
+    runs-on: ubuntu-latest
     timeout-minutes: 5
     permissions:
       contents: read
@@ -52,7 +49,147 @@ jobs:
         uses: actions/checkout@v3
       - name: Run unit tests
         run: make test
-      - name: Build testing
-        run: make build
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
+      - name: Build bianry for E2E testing
+        run: |
+          ./scripts/build.sh v0.0.1 linux-x64
+          ./scripts/build.sh v0.0.1 win-x64
+          ./scripts/build.sh v0.0.1 osx-x64
+      - name: Upload Linux artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: linux-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          retention-days: 1
+      - name: Upload macOS artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: darwin-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_darwin_amd64.tar.gz
+          retention-days: 1
+      - name: Upload Windows artifact
+        if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+        uses: actions/upload-artifact@v2
+        with:
+          name: win-amd64-binary
+          path: ./bin/artifacts/notation-azure-kv_0.0.1_windows_amd64.zip
+          retention-days: 1
+  e2e-linux:
+    name: E2E testing on Linux
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          nohup python3 -m http.server --directory ./bin/artifacts/ &
+
+          # prepare the environment variables for E2E
+          artifactName=notation-azure-kv_0.0.1_linux_amd64.tar.gz
+          checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
+          echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
+          echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
+      - name: Prepare container registry
+        run: |
+          docker run  --name registry --rm -d -p 5000:5000 registry:2
+          docker pull hello-world:latest
+          docker tag hello-world:latest localhost:5000/hello-world:v1
+          docker push localhost:5000/hello-world:v1
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}
+  e2e-windows:
+    name: E2E testing on Windows
+    runs-on: windows-2022
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: win-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          # wsl bash
+          bash -c 'nohup python3 -m http.server --directory ./bin/artifacts/ &'
+
+          # Prepare the environment variables for E2E
+          $artifactName = "notation-azure-kv_0.0.1_windows_amd64.zip"
+          $checksum = (Get-FileHash ".\bin\artifacts\$artifactName" -Algorithm SHA256).Hash
+          "pluginChecksum=$checksum" | Out-File -Append -FilePath $Env:GITHUB_ENV
+          "pluginDownloadURL=http://localhost:8000/$artifactName" | Out-File -Append -FilePath $Env:GITHUB_ENV
+        shell: pwsh
+      - name: Prepare container registry
+        run: |
+          docker run  --name registry --rm -d -p 5000:5000 junjiegaomsft/registry:v2.8.2-ltsc2022
+          docker pull hello-world:latest
+          docker tag hello-world:latest localhost:5000/hello-world:v1
+          docker push localhost:5000/hello-world:v1
+        shell: pwsh
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}
+  e2e-macos:
+    name: E2E testing on macOS
+    runs-on: macos-13
+    needs: test
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push'
+    steps:
+      - name: Check out code into the project directory
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: darwin-amd64-binary
+          path: ./bin/artifacts
+      - name: Run download server locally
+        run: |
+          nohup python3 -m http.server --directory ./bin/artifacts/ &
+
+          # prepare the environment variables for E2E
+          artifactName=notation-azure-kv_0.0.1_darwin_amd64.tar.gz
+          checksum=$(shasum -a 256 "./bin/artifacts/$artifactName" | awk '{print $1}')
+          echo "pluginChecksum=$checksum" >> "$GITHUB_ENV"
+          echo "pluginDownloadURL=http://localhost:8000/$artifactName" >> "$GITHUB_ENV"
+      - name: Prepare container registry
+        run: |
+          # start zot registry
+          wget -O zot https://github.com/project-zot/zot/releases/download/v2.0.0-rc7/zot-darwin-amd64-minimal
+          chmod +x zot
+          nohup ./zot serve ./test/e2e/zot/config.json &
+
+          # install oras
+          wget -O oras.tar.gz https://github.com/oras-project/oras/releases/download/v1.1.0/oras_1.1.0_darwin_amd64.tar.gz
+          tar -zxf oras.tar.gz
+          ./oras push localhost:5000/hello-world:v1 --artifact-type  application/octet-stream ./LICENSE
+      - name: Azure login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - name: E2E testing
+        uses: ./test/e2e
+        with:
+          pluginDownloadURL: ${{ env.pluginDownloadURL }}
+          pluginChecksum: ${{ env.pluginChecksum }}

scripts/generate-certs.sh

@@ -0,0 +1,384 @@
+#!/bin/bash
+# This script generates various certificates on Azure Key Vault for E2E testing
+# It may be used when initial setup is needed or when certificates need to 
+# be regenerated.
+#
+# Prerequisites:
+#   - Azure CLI
+#   - OpenSSL
+#   - an existing Azure Key Vault
+#   - env AKV_SUB: Azure Key Vault subscription
+#   - env AKV_NAME: Azure Key Vault name
+
+set -ex
+
+# check Azure Key Vault subscription
+if [ -z "$AKV_SUB" ]; then
+  echo "AKV_SUB is not set"
+  exit 1
+fi
+
+# check Azure Key Vault name
+if [ -z "$AKV_NAME" ]; then
+  echo "AKV_NAME is not set"
+  exit 1
+fi
+
+if ! command -v az &> /dev/null
+then
+  echo "az command could not be found"
+  exit 1
+fi
+
+if ! command -v openssl &> /dev/null
+then
+  echo "openssl command could not be found"
+  exit 1
+fi
+
+
+# Generate self-signed PKCS12 certificate
+function createSelfSignedPKCS12() {
+  local certName
+  certName="self-signed-pkcs12"
+  local policy
+  policy=$(cat << EOF
+{
+  "issuerParameters": {
+    "name": "Self"
+  },
+  "keyProperties": {
+    "exportable": false,
+    "keySize": 2048,
+    "keyType": "RSA",
+    "reuseKey": true
+  },
+  "secretProperties": {
+    "contentType": "application/x-pkcs12"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ],
+    "keyUsage": [
+      "digitalSignature"
+    ],
+    "subject": "CN=Test-Signer,C=US,ST=WA,O=notation",
+    "validityInMonths": 1200
+  }
+}
+EOF
+)
+  az keyvault certificate create -n "$certName" --vault-name "$AKV_NAME" -p "$policy"
+  echo "created ${certName}"
+}
+
+function createSelfSignedPEM() {
+  local certName
+  certName="self-signed-pem"
+  local policy
+  policy=$(cat << EOF
+{
+  "issuerParameters": {
+    "name": "Self"
+  },
+  "keyProperties": {
+    "exportable": false,
+    "keySize": 2048,
+    "keyType": "RSA",
+    "reuseKey": true
+  },
+  "secretProperties": {
+    "contentType": "application/x-pem-file"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ],
+    "keyUsage": [
+      "digitalSignature"
+    ],
+    "subject": "CN=Test-Signer,C=US,ST=WA,O=notation",
+    "validityInMonths": 1200
+  }
+}
+EOF
+)
+  az keyvault certificate create -n "$certName" --vault-name "$AKV_NAME" -p "$policy"
+  echo "created ${certName}"
+}
+
+# Generate CA issued PKCS12 certificate chain
+# leaf.crt -> inter2.crt -> inter1.crt -> ca.crt
+function generateCertChain() {
+  # generate CA key and certificate
+  echo "Generating CA key and certificate..."
+  openssl genrsa -out ca.key 2048
+  openssl req -new -x509 -days 36500 -key ca.key -subj "/O=Notation/CN=Notation Root CA" -out ca.crt -addext "keyUsage=critical,keyCertSign"
+
+  # generate intermediate ca 1
+  echo "Gneerating intermediate CA 1"
+  openssl req -newkey rsa:2048 -nodes -keyout inter1.key -subj "/CN=Notation.inter1" -out inter1.csr
+  openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:TRUE\nkeyUsage=critical,keyCertSign") -days 36500 -in inter1.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out inter1.crt
+
+  # generate intermediate ca 2
+  echo "Gneerating intermediate CA 2"
+  openssl req -newkey rsa:2048 -nodes -keyout inter2.key -subj "/CN=Notation.inter2" -out inter2.csr
+  openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:TRUE\nkeyUsage=critical,keyCertSign") -days 36500 -in inter2.csr -CA inter1.crt -CAkey inter1.key -CAcreateserial -out inter2.crt
+
+  # generate leaf key and certificate
+  echo "Generating leaf key and certificate..."
+  # openssl genrsa -out leaf.key 2048
+  openssl req -newkey rsa:2048 -nodes -keyout leaf.key -subj "/CN=Test-Signer/C=US/ST=WA/O=notation" -out leaf.csr
+  openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature") -days 36500 -in leaf.csr -CA inter2.crt -CAkey inter2.key -CAcreateserial -out leaf.crt
+
+  # generate PEM certificate chain
+  cat leaf.key leaf.crt inter2.crt inter1.crt ca.crt > cert-chain.pem
+  cat leaf.key leaf.crt ca.crt inter1.crt inter2.crt  > unordered-cert-chain.pem
+
+  # generate PKCS12 certificate chain
+  openssl pkcs12 -export -out cert-chain.pfx -inkey leaf.key -in cert-chain.pem
+  openssl pkcs12 -export -out unordered-cert-chain.pfx -inkey leaf.key -in unordered-cert-chain.pem
+
+  # generate partial PEM certificate chain
+  cat leaf.key leaf.crt > partial-cert-chain.pem
+  # generate partial PKCS12 certificate chain
+  openssl pkcs12 -export -out partial-cert-chain.pfx -inkey leaf.key -in partial-cert-chain.pem
+  # create cert bundle for e2e
+  cat inter2.crt inter1.crt ca.crt > cert-bundle.pem
+}
+
+function importCAIssuedPKCS12CertificateChain() {
+  local certName
+  certName="imported-ca-issued-pkcs12"
+  local unorderedCertName
+  unorderedCertName="imported-ca-issued-pkcs12-unordered"
+  local policy
+  policy=$(cat << EOF
+{
+  "keyProperties": {
+    "exportable": false
+  },
+  "secretProperties": {
+    "contentType": "application/x-pkcs12"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ]
+  }
+}
+EOF
+)
+  az keyvault certificate import --file ./cert-chain.pfx -n "$certName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${certName}"
+  az keyvault certificate import --file ./unordered-cert-chain.pfx -n "$unorderedCertName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${unorderedCertName}"
+}
+
+function importCAIssuedPEMCertificateChain() {
+  local certName
+  certName="imported-ca-issued-pem"
+  local unorderedCertName
+  unorderedCertName="imported-ca-issued-pem-unordered"
+  local policy
+  policy=$(cat << EOF
+{
+  "keyProperties": {
+    "exportable": false
+  },
+  "secretProperties": {
+    "contentType": "application/x-pem-file"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ]
+  }
+}
+EOF
+)
+  az keyvault certificate import --file ./cert-chain.pem -n "$certName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${certName}"
+  az keyvault certificate import --file ./unordered-cert-chain.pem -n "$unorderedCertName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${unorderedCertName}"
+}
+
+function createPKCS12CertificateChainWithCSR() {
+  local certName
+  certName="csr-ca-issued-pkcs12-chain"
+  local policy
+  policy=$(cat << EOF
+{
+  "issuerParameters": {
+    "name": "Unknown"
+  },
+  "keyProperties": {
+    "exportable": false,
+    "keySize": 2048,
+    "keyType": "RSA",
+    "reuseKey": true
+  },
+  "secretProperties": {
+    "contentType": "application/x-pkcs12"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ],
+    "keyUsage": [
+      "digitalSignature"
+    ],
+    "subject": "CN=Test-Signer,C=US,ST=WA,O=notation",
+    "validityInMonths": 1200
+  }
+}
+EOF
+)
+  az keyvault certificate create -n "$certName" --vault-name "$AKV_NAME" -p "$policy"
+  echo "created ${certName} with CSR"
+
+  # download CSR
+  local csr
+  csr=$(az keyvault certificate pending show --vault-name "$AKV_NAME" --name "$certName" --query "csr" -o tsv)
+  local csrPath
+  csrPath=${certName}.csr
+  printf -- "-----BEGIN CERTIFICATE REQUEST-----\n%s\n-----END CERTIFICATE REQUEST-----\n" "$csr" > ${csrPath}
+  local signedCertPath
+  signedCertPath="${certName}.crt"
+
+  # issue certificate chain with CSR
+  openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature") -days 36500 -in $csrPath -CA inter2.crt -CAkey inter2.key -CAcreateserial -out $signedCertPath
+  certChainPath="${certName}-chain.crt"
+  cat "$signedCertPath" inter2.crt inter1.crt ca.crt > "$certChainPath"
+
+  # merge certificate chain
+  az keyvault certificate pending merge --vault-name "$AKV_NAME" --name "$certName" --file "$certChainPath"
+  echo "merged ${certName}"
+}
+
+function createPEMCertificateChainWithCSR() {
+  local certName
+  certName="csr-ca-issued-pem-chain"
+  local policy
+  policy=$(cat << EOF
+{
+  "issuerParameters": {
+    "name": "Unknown"
+  },
+  "keyProperties": {
+    "exportable": false,
+    "keySize": 2048,
+    "keyType": "RSA",
+    "reuseKey": true
+  },
+  "secretProperties": {
+    "contentType": "application/x-pem-file"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ],
+    "keyUsage": [
+      "digitalSignature"
+    ],
+    "subject": "CN=Test-Signer,C=US,ST=WA,O=notation",
+    "validityInMonths": 1200
+  }
+}
+EOF
+)
+  az keyvault certificate create -n "$certName" --vault-name "$AKV_NAME" -p "$policy"
+  echo "created ${certName} with CSR"
+
+  # download CSR
+  local csr
+  csr=$(az keyvault certificate pending show --vault-name "$AKV_NAME" --name "$certName" --query "csr" -o tsv)
+  local csrPath
+  csrPath=${certName}.csr
+  printf -- "-----BEGIN CERTIFICATE REQUEST-----\n%s\n-----END CERTIFICATE REQUEST-----\n" "$csr" > ${csrPath}
+  local signedCertPath
+  signedCertPath="${certName}.crt"
+
+  # issue certificate chain with CSR
+  openssl x509 -req -extfile <(printf "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature") -days 36500 -in "$csrPath" -CA inter2.crt -CAkey inter2.key -CAcreateserial -out "$signedCertPath"
+  certChainPath="${certName}-chain.crt"
+  cat "$signedCertPath" inter2.crt inter1.crt ca.crt > "$certChainPath"
+
+  # merge certificate chain
+  az keyvault certificate pending merge --vault-name "$AKV_NAME" --name "$certName" --file "$certChainPath"
+  echo "merged ${certName}"
+}
+
+function createPartialPKCS12CertificateChain() {
+  local certName
+  certName="partial-pkcs12-cert-chain"
+  local policy
+  policy=$(cat << EOF
+{
+  "keyProperties": {
+    "exportable": false
+  },
+  "secretProperties": {
+    "contentType": "application/x-pkcs12"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ]
+  }
+}
+EOF
+)
+  az keyvault certificate import --file ./partial-cert-chain.pfx -n "$certName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${certName}"
+}
+
+function createPartialPEMCertificateChain() {
+  local certName
+  certName="partial-pem-cert-chain"
+  local policy
+  policy=$(cat << EOF
+{
+  "keyProperties": {
+    "exportable": false
+  },
+  "secretProperties": {
+    "contentType": "application/x-pem-file"
+  },
+  "x509CertificateProperties": {
+    "ekus": [
+        "1.3.6.1.5.5.7.3.3"
+    ]
+  }
+}
+EOF
+)
+  az keyvault certificate import --file ./partial-cert-chain.pem -n "$certName" --vault-name "$AKV_NAME" --policy "$policy"
+  echo "imported ${certName}"
+}
+
+function main() {
+  az account set --subscription "$AKV_SUB"
+
+  # self signed certificate
+  createSelfSignedPKCS12
+  createSelfSignedPEM
+
+  # certificate chain
+  mkdir -p ./bin/certs && cd ./bin/certs
+  generateCertChain
+  # import local certificate chain to Azure Key Vault
+  importCAIssuedPKCS12CertificateChain
+  importCAIssuedPEMCertificateChain
+
+  # # issue certificate chain with Azure Key Vualt CSR
+  createPKCS12CertificateChainWithCSR
+  createPEMCertificateChainWithCSR
+
+  # partial certificate chain
+  createPartialPKCS12CertificateChain
+  createPartialPEMCertificateChain
+}
+
+main

test/e2e/action.yml

@@ -0,0 +1,184 @@
+---
+name: 'notation-azure-kv E2E testing'
+description: 'This is a E2E test action for testing the notation-azure-kv plugin on mltiple platforms.'
+inputs:
+  pluginDownloadURL:
+    description: 'Where to download the plugin'
+    required: true
+  pluginChecksum:
+    description: 'The checksum of the plugin'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: setup Notation CLI
+      uses: notaryproject/notation-action/setup@v1
+      with:
+        version: 1.0.0
+    - name: self-signed pkcs12 certificate 
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/self-signed-pkcs12/70747b2064c0488e936eba7a29acc4c6
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'self_signed=true'
+    - name: self-signed pem certificate 
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/self-signed-pem/a2c329545a934f0aaf434afe64bb392d
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'self_signed=true'
+    - name: imported ca-issued pem
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pem/5a768b6209564c3cb30ecc30d800dc43
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: imported ca-issued pem unordered
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pem-unordered/c0dcfcda9a454880aec242c70dcb1e2a
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: imported ca-issued pkcs12
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pkcs12/20548a2bcaba42308f609df2d79682b5
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: imported ca-issued pkcs12 unordered
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pkcs12-unordered/b4fdf86062e44839b666ce8ff3f3a470
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: csr ca-issued pem chain
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/csr-ca-issued-pem-chain/09cd1aeaaa894e60b0ef83f062604863
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: csr ca-issued pem chain
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/csr-ca-issued-pkcs12-chain/aad06a96a2684d6ab79a4ad84cbe917e
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: partial pem cert chain with local cert bundle
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/partial-pem-cert-chain/bf6299c95b96492894be0230935bdab8
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'ca_certs=./test/e2e/certs/cert-bundle.pem'
+    - name: pkcs12 cert chain with local cert bundle
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/partial-pkcs12-cert-chain/c90493832b4148ee80e2aa10ada67a0b
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'ca_certs=./test/e2e/certs/cert-bundle.pem'
+    - name: partial pem cert chain with local invalid local cert bundle
+      continue-on-error: true
+      id: partial-pem-cert-chain-invalid-local-cert-bundle
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/partial-pem-cert-chain/bf6299c95b96492894be0230935bdab8
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'ca_certs=./test/e2e/certs/root.pem'
+    - name: 'Should Fail: partial pem cert chain with local invalid local cert bundle'
+      if: steps.partial-pem-cert-chain-invalid-local-cert-bundle.outcome != 'failure'
+      run: |
+          echo "partial pem certificate chain with invalid local cert bundle should failed, but succeeded."
+          exit 1
+      shell: bash
+    - name: partial pem cert chain without local cert bundle
+      continue-on-error: true
+      id: partial-pem-cert-chain-without-local-cert-bundle
+      uses: notaryproject/notation-action/sign@v1
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/partial-pem-cert-chain/bf6299c95b96492894be0230935bdab8
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+    - name: 'Should Fail: partial pem cert chain without local cert bundle'
+      if: steps.partial-pem-cert-chain-without-local-cert-bundle.outcome != 'failure'
+      run: |
+          echo "partial pem certificate chain without local cert bundle should failed, but succeeded."
+          exit 1
+      shell: bash
+    - name: certificate chain with self signed plugin config
+      uses: notaryproject/notation-action/sign@v1
+      continue-on-error: true
+      id: certificate-chain-with-self-signed-plugin-config
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pem/5a768b6209564c3cb30ecc30d800dc43
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: 'self_signed=true'
+    - name: 'Should Fail: certificate chain with self signed plugin config'
+      if: steps.certificate-chain-with-self-signed-plugin-config.outcome != 'failure'
+      run: |
+          echo "certificate chain with self signed plugin config should failed, but succeeded."
+          exit 1
+      shell: bash
+    - name: both self signed and ca certs plugin config exist
+      uses: notaryproject/notation-action/sign@v1
+      continue-on-error: true
+      id: both-self-signed-and-ca-certs-plugin-config-exist
+      with:
+        plugin_name: azure-kv
+        plugin_url: ${{ inputs.pluginDownloadURL }}
+        plugin_checksum: ${{ inputs.pluginChecksum }}
+        key_id: https://acrci-test-kv.vault.azure.net/keys/imported-ca-issued-pem/5a768b6209564c3cb30ecc30d800dc43
+        target_artifact_reference: localhost:5000/hello-world:v1
+        signature_format: cose
+        plugin_config: |
+          self_signed=true
+          ca_certs=./test/e2e/certs/cert-bundle.pem
+    - name: 'Should Fail: both self signed and ca certs plugin config exist'
+      if: steps.both-self-signed-and-ca-certs-plugin-config-exist.outcome != 'failure'
+      run: |
+          echo "both self signed and ca certs plugin config exist should failed, but succeeded."
+          exit 1
+      shell: bash

test/e2e/certs/cert-bundle.pem

@@ -0,0 +1,59 @@
+-----BEGIN CERTIFICATE-----
+MIIDJzCCAg+gAwIBAgIUDoxyb/j0D4lEHhZSkAa+sxactx8wDQYJKoZIhvcNAQEL
+BQAwGjEYMBYGA1UEAwwPTm90YXRpb24uaW50ZXIxMCAXDTIzMTEyMDA4MTEzMloY
+DzIxMjMxMDI3MDgxMTMyWjAaMRgwFgYDVQQDDA9Ob3RhdGlvbi5pbnRlcjIwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDN9zD7llQSAqpfln26ruKcaPbd
+jVFHGXvl0xMMWSuYrldVpnqnhuDxMOu6boDrpL+hhdE9nUzKQRyIqYi2E3Xvd9cC
+AMOObTWdJAJTIGMFCE8tBXOjOT1qsxdIPEm6msXj/JFjKOqMFx7+GDVWJKIc1bFP
+BnL2QFRAQewIn8lweMMOwXdFCPQkDwiIkC2RfuYH+/dOOKpOMRKyhhUv6VzPdpkR
+bN7a2xx7jwKZftXY4wYvmlDIiliF6+ZRa2UlDIRVab94e8bl+KVo4c/B6V+18xEP
+XUClrhFdpQ+CEEhyHHO2s1AliGPbd+MTcZcNKHFMnoAdMeUbMbqiCgekyB/VAgMB
+AAGjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgIEMB0GA1UdDgQW
+BBQIqhNrwB25To/R1fxTJ4pZqFbFJDAfBgNVHSMEGDAWgBTTgEZPu1hCSJi5GL9t
+DDV4aGcCszANBgkqhkiG9w0BAQsFAAOCAQEAYIPbfIVNiSg7cJZ1erFBafhRNQLu
+YLObNX4monpoNRzjoAQ0yP5TsBHRw5m1CK0bYQIVO8pSwjP//lj5vS+eOsQU0aEh
+mqkjnS+By++i+QkJuHcqe53uOUlSriQEKurlgoNtXbdnf9RJj1A8Sk2gT1i1KYvV
+uRoDc4Yy4vpUskU0zj1fDfab+Frk/VUnj8S0lJHPTfmEIrqmXkBEq92oF+ojR5MV
+QAOWgiGoW1j9G5pGgRDeU3QNfeuv/ZySlLWo10/hOjATbuAlgriczCAJQHuLA0xl
+uxmrOpF6u+hTLelK0HvbXIpGwJaF+X5I1M1t3Do+4fpnbekvX+gwMfz+aw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDOzCCAiOgAwIBAgIUazQlRBbQ5en3CDA77cWaLe9TtVswDQYJKoZIhvcNAQEL
+BQAwLjERMA8GA1UECgwITm90YXRpb24xGTAXBgNVBAMMEE5vdGF0aW9uIFJvb3Qg
+Q0EwIBcNMjMxMTIwMDgxMTMyWhgPMjEyMzEwMjcwODExMzJaMBoxGDAWBgNVBAMM
+D05vdGF0aW9uLmludGVyMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AKLv7nFaX4/U/mfWhAFl0gx81AOkoDHDNMRegqy972PDX7Pa1DiCenqKTkCNcZHr
+IpLU53esQOpt76O7xvdvbaLotBEWHfUTZDyeob0L9pRcswfxdnJ1QECS3O8e4vrl
+PqcpR/FLSr0t5dD9/g4z+qAZMAhGuSOsid0yKIv5j20ZjmesBMzVw8/0mfDyVpPr
+FSuGP2F1WVgbBkARy+zc1JvmAbEbJyvB6aheesE/gvXTU2tbaSLJi8DpyfVM/6LQ
+DEBt7en+gPXTvZZcO2ozyly2LRyK6tXPMY5QPNvrWEkHlIDrh65YBu8lbUSc8/FK
+Qr6TQhyXNYA2ZcSn2IAL+ecCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNV
+HQ8BAf8EBAMCAgQwHQYDVR0OBBYEFNOARk+7WEJImLkYv20MNXhoZwKzMB8GA1Ud
+IwQYMBaAFJNKU40LYCpUiCOxuos1wKYPCBgbMA0GCSqGSIb3DQEBCwUAA4IBAQAc
+xpnyX9a5u+CyrltvODHm7aPvmYAz64Rl4uX794cr2FJc7I1ANyez1rPn3wOBhedt
+DqZdOOKEkWO5w5w0vEmI1Xl9e3+9vjC7r3QMqiKwOnj9JoqOTr8Ae9b/muHToJsy
+O6a4VXjHjPZ94ByGa4GD/zDfE0aSCpv8Qpn0YSwdAw5PToh5Dl1yOEBmIqnw+RpH
+jFnh09T+vfM6qDwwMebkV8PhC3fopv8Hs+5McEEL4nqVSDBiKmqhBH96dHyZdIvT
+khTvEYxJTI5KJcy5krfcijf/2eGH+eu72loc5QdR+GZO+N4zngrxQ2IvP1PstUvI
+jOKkEIdbsp1y8qBzBVwQ
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIDTzCCAjegAwIBAgIUIFgur+/BkZlNg8uRAaBxr8/dBmcwDQYJKoZIhvcNAQEL
+BQAwLjERMA8GA1UECgwITm90YXRpb24xGTAXBgNVBAMMEE5vdGF0aW9uIFJvb3Qg
+Q0EwIBcNMjMxMTIwMDgxMTMxWhgPMjEyMzEwMjcwODExMzFaMC4xETAPBgNVBAoM
+CE5vdGF0aW9uMRkwFwYDVQQDDBBOb3RhdGlvbiBSb290IENBMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsm8WZ9bXtieEbP2hqt4ws+v0tNhXfmyd32i5
+KV+vtYZQH5ASr0lDDtGHZPs0tayoKk7FnbGUIBl2aeAPqefKYSwIe5UCMDIxGCbu
+Ql76fX0/1EMDja+SJyJ4641ey3HOhyskIptf8GaHQLsYCgekuB0IPtQC7tSesO7O
+j0RpSZiN1oKTperHyeWbLtmHKGi+IY+I+BcMMmJkCub/g98N5UDB+SMyLd6LCVCx
++3JvcJDwZTREEUMfVrJpySsGlmy0fU3OSZy2xtglODPwba75AKVWsASBsEjJKIkz
+Tn7fSLrLkFlmOpdj2KdZWdJhWJaWlqJU4rtWJ9ApKMYKCoB9PQIDAQABo2MwYTAd
+BgNVHQ4EFgQUk0pTjQtgKlSII7G6izXApg8IGBswHwYDVR0jBBgwFoAUk0pTjQtg
+KlSII7G6izXApg8IGBswDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAgQw
+DQYJKoZIhvcNAQELBQADggEBAGGZ6feRKfEtxgQwFdLIELn8dxae6dO+2EX+EvLt
+9kFQ+73PKkSI7nwog+hk6zxIP36+aDROb7ySxfEK05o8ONSE5SeSmbWFoFRha4tQ
+HWE7YabU7AieQPLDD7nGttCgqMSwTlrI0+1UiKcKwR99pQphLFhJIWHyB0mISpbc
+HgjO1ACMm2AEYCM9yDD001VKKOW+I2rcZl1VHlyPeCrJHTSHrYLdIfEyPzyWwVdn
+Nbl9LRH7qF+tgWvKdAQSA9DWQINRPh8nHEa+Au+AUqmMSuvBhTNcf0LygPP+O59p
+jXGfwAc5XGXCLjgzuTnxquvia2dc1qGJZEa40iqwh1O6HwM=
+-----END CERTIFICATE-----

test/e2e/certs/root.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDTzCCAjegAwIBAgIUIFgur+/BkZlNg8uRAaBxr8/dBmcwDQYJKoZIhvcNAQEL
+BQAwLjERMA8GA1UECgwITm90YXRpb24xGTAXBgNVBAMMEE5vdGF0aW9uIFJvb3Qg
+Q0EwIBcNMjMxMTIwMDgxMTMxWhgPMjEyMzEwMjcwODExMzFaMC4xETAPBgNVBAoM
+CE5vdGF0aW9uMRkwFwYDVQQDDBBOb3RhdGlvbiBSb290IENBMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsm8WZ9bXtieEbP2hqt4ws+v0tNhXfmyd32i5
+KV+vtYZQH5ASr0lDDtGHZPs0tayoKk7FnbGUIBl2aeAPqefKYSwIe5UCMDIxGCbu
+Ql76fX0/1EMDja+SJyJ4641ey3HOhyskIptf8GaHQLsYCgekuB0IPtQC7tSesO7O
+j0RpSZiN1oKTperHyeWbLtmHKGi+IY+I+BcMMmJkCub/g98N5UDB+SMyLd6LCVCx
++3JvcJDwZTREEUMfVrJpySsGlmy0fU3OSZy2xtglODPwba75AKVWsASBsEjJKIkz
+Tn7fSLrLkFlmOpdj2KdZWdJhWJaWlqJU4rtWJ9ApKMYKCoB9PQIDAQABo2MwYTAd
+BgNVHQ4EFgQUk0pTjQtgKlSII7G6izXApg8IGBswHwYDVR0jBBgwFoAUk0pTjQtg
+KlSII7G6izXApg8IGBswDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAgQw
+DQYJKoZIhvcNAQELBQADggEBAGGZ6feRKfEtxgQwFdLIELn8dxae6dO+2EX+EvLt
+9kFQ+73PKkSI7nwog+hk6zxIP36+aDROb7ySxfEK05o8ONSE5SeSmbWFoFRha4tQ
+HWE7YabU7AieQPLDD7nGttCgqMSwTlrI0+1UiKcKwR99pQphLFhJIWHyB0mISpbc
+HgjO1ACMm2AEYCM9yDD001VKKOW+I2rcZl1VHlyPeCrJHTSHrYLdIfEyPzyWwVdn
+Nbl9LRH7qF+tgWvKdAQSA9DWQINRPh8nHEa+Au+AUqmMSuvBhTNcf0LygPP+O59p
+jXGfwAc5XGXCLjgzuTnxquvia2dc1qGJZEa40iqwh1O6HwM=
+-----END CERTIFICATE-----

test/e2e/distribution/dockerfile.windows

@@ -0,0 +1,31 @@
+ARG IMAGE_VERSION
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21-nanoserver-$IMAGE_VERSION AS builder
+
+ARG DISTRIBUTION_VERSION="2.8.2"
+ARG SOURCE_CODE_URL="https://github.com/distribution/distribution/archive/refs/tags/v${DISTRIBUTION_VERSION}.zip"
+
+WORKDIR "C:\\distribution"
+
+ENV GO111MODULE="auto" \
+    GOPATH="C:\\distribution"
+
+# Download distribution source code and build registry
+RUN curl.exe -o distribution.zip -L %SOURCE_CODE_URL% && \
+    mkdir .\src\github.com\docker\ && \
+    tar.exe -C .\src\github.com\docker\ -x -p -f .\distribution.zip && \
+    ren .\src\github.com\docker\distribution-%DISTRIBUTION_VERSION% distribution && \
+    go build -o ".\\registry.exe" ".\\src\\github.com\\docker\\distribution\\cmd\\registry\\"
+
+# Runtime image
+FROM mcr.microsoft.com/windows/nanoserver:$IMAGE_VERSION
+
+WORKDIR "C:\\distribution"
+
+# Copy the built registry binary from the builder image
+COPY --from=builder "C:\\distribution\\registry.exe" ".\\"
+
+# Copy the configuration file
+COPY ".\\registry.conf" ".\\"
+
+# Start the registry
+CMD ["C:\\distribution\\registry.exe", "serve", "C:\\distribution\\registry.conf"]

test/e2e/distribution/registry.conf

@@ -0,0 +1,18 @@
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+  filesystem:
+    rootdirectory: ./tmp/registry
+http:
+  addr: :5000
+  headers:
+    X-Content-Type-Options: [nosniff]
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3

test/e2e/zot/config.json

@@ -0,0 +1,12 @@
+    {
+      "storage":{
+          "rootDirectory":"."
+      },
+      "http":{
+          "address":"0.0.0.0",
+          "port":"5000"
+      },
+      "log":{
+          "level":"debug"
+      }
+    }