Add tests to easily make sure the output looks like something expected

.github/workflows/tos.yaml

@@ -42,14 +42,18 @@ jobs:
         id: set-output
         run: |
           accepted=$(echo '${{ steps.parse.outputs.data }}' | jq -r '.["terms-of-services"].list[0].checked' )
-          echo $accepted
-          if [[ accepted == '' || accepted == 'null' ]]; then
+          echo "accepted=$accepted" >> $GITHUB_OUTPUT
+
+      - name: Validate tos
+        if: steps.set-output.outputs.accepted == 'null'
+        run: |
+          # Check if $GITHUB_STEP_SUMMARY exist o avoid the following lines to error when running with act
+          if [[ -w "$GITHUB_STEP_SUMMARY" ]]; then
             echo "No checkbox for the tos found in the body of the issue ${{ inputs.issue_number }}" >> $GITHUB_STEP_SUMMARY
             echo "Is the \"terms-of-services\" field present?" >> $GITHUB_STEP_SUMMARY
             echo ${{ steps.parse.outputs.data }} >> $GITHUB_STEP_SUMMARY
-            exit 1
           fi
-          echo "accepted=$accepted" >> $GITHUB_OUTPUT
+          exit 1
 
   tos-not-accepted:
     runs-on: ubuntu-latest
@@ -114,7 +118,7 @@ jobs:
   tos-not-found:
     runs-on: ubuntu-latest
     needs: [ parse-issue ]
-    if: needs.parse-issue.outputs.accepted == 'null'
+    if: failure()
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/validation.yaml

@@ -11,32 +11,33 @@ jobs:
   parse-issue:
     runs-on: ubuntu-latest
     outputs:
-      repository: ${{ steps.parse.outputs.repository }}
+      repository: ${{ steps.set-output.outputs.repository }}
     steps:
       - name: Parse issue body
         id: parse
         uses: zentered/issue-forms-body-parser@v1.5.1
 
       - name: Find extension repository
-        id: parse-repository
+        id: set-output
         run: |
           repository=$(echo '${{ steps.parse.outputs.data }}' | jq -r '.["docker-hub-repository-name"].text' )
           echo "repository=$repository" >> $GITHUB_OUTPUT
-          
-          if [[ $repository == null ]]; then
-            # Check if $GITHUB_STEP_SUMMARY exist o avoid the following lines to error when running with act
-            if [[ -w "$GITHUB_STEP_SUMMARY" ]]; then
-              echo "No repository found in the body of the issue ${{ inputs.issue_number }}" >> $GITHUB_STEP_SUMMARY
-              echo "Is the \"docker-hub-repository-name\" field present?" >> $GITHUB_STEP_SUMMARY
-              echo ${{ steps.parse.outputs.data }} >> $GITHUB_STEP_SUMMARY
-            fi
-            exit 1
+
+      - name: Validate repository
+        if: steps.set-output.outputs.repository == 'null'
+        run: |
+          # Check if $GITHUB_STEP_SUMMARY exist o avoid the following lines to error when running with act
+          if [[ -w "$GITHUB_STEP_SUMMARY" ]]; then
+            echo "No repository found in the body of the issue ${{ inputs.issue_number }}" >> $GITHUB_STEP_SUMMARY
+            echo "Is the \"docker-hub-repository-name\" field present?" >> $GITHUB_STEP_SUMMARY
+            echo ${{ steps.parse.outputs.data }} >> $GITHUB_STEP_SUMMARY
           fi
+          exit 1
 
   error:
     runs-on: ubuntu-latest
     needs: parse-issue
-    if: failure() && needs.parse-issue.outputs.repository == null
+    if: failure()
     steps:
       - uses: actions/checkout@v3
 
@@ -59,17 +60,26 @@ jobs:
         run: exit 1
 
   noop-validate:
-    if: env.ACT
+    if: github.event.act == true
     runs-on: ubuntu-latest
     needs: parse-issue
     outputs:
       validation_output: ${{ steps.set-output.outputs.validation_output }}
     steps:
       - id: set-output
-        run: echo "validation_output=No errors" >> $GITHUB_OUTPUT
+        run: |
+          if [[ ${TEST_VALIDATION_FAILED} ]]; then
+            echo "validation_output=There are errors" >> $GITHUB_OUTPUT
+            exit 1
+          fi
+          if [[ ${TEST_VALIDATION_ERRORED} ]]; then
+            # Do not set the validation_output variable on failure
+            exit 1
+          fi
+          echo "validation_output=No errors" >> $GITHUB_OUTPUT
 
   validate:
-    if: !env.ACT
+    if: github.event.act == false
     name: Validate extension
     runs-on: macOS-latest
     needs: parse-issue
@@ -183,11 +193,8 @@ jobs:
 
   validation-succeeded:
     runs-on: ubuntu-latest
-    needs: [parse-issue, validate, noop-validate]
-    # always() make sure the job is run even if the noop-validate or validate jobs are skipped
-    if: |
-      always()
-      && (needs.validate.result == 'success' || needs.noop-validate.result== 'success')
+    needs: [parse-issue, noop-validate, validate]
+    if: always() && (needs.validate.result == 'success' || needs.noop-validate.result == 'success')
     steps:
       - uses: actions/checkout@v3
 
@@ -216,8 +223,11 @@ jobs:
 
   validation-failed:
     runs-on: ubuntu-latest
-    needs: [parse-issue, validate, noop-validate]
-    if: needs.validate.result != 'success' ||  needs.noop-validate.result != 'success'
+    needs: [parse-issue, noop-validate, validate]
+    if: |
+      always() &&
+      (needs.validate.result == 'failure' && needs.validate.outputs.validation_output != '') ||
+      (needs.noop-validate.result == 'failure' && needs.noop-validate.outputs.validation_output != '')
     steps:
       - uses: actions/checkout@v3
 
@@ -248,8 +258,11 @@ jobs:
 
   validation-errored:
     runs-on: ubuntu-latest
-    needs: [validate, noop-validate]
-    if: needs.validate.result == 'failure' || needs.noop-validate.result == 'failure'
+    needs: [noop-validate, validate]
+    if: |
+      always() &&
+      (needs.validate.result == 'failure' && needs.validate.outputs.validation_output == '') ||
+      (needs.noop-validate.result == 'failure' && needs.noop-validate.outputs.validation_output == '')
     steps:
       - uses: actions/checkout@v3
 
@@ -275,4 +288,4 @@ jobs:
           gh issue edit ${{ github.event.issue.number }} --add-label "docker/validation-errored"
 
       - name: Mark job as failed
-        run: exit 1
+        run: exit 1

.gitignore

@@ -0,0 +1 @@
+node_modules/

package.json

@@ -0,0 +1,11 @@
+{
+  "license": "Apache-2.0",
+  "scripts": {
+    "test": "bats tests.bats"
+  },
+  "devDependencies": {
+    "bats": "github:bats-core/bats-core#master",
+    "bats-assert": "https://github.com/bats-core/bats-assert",
+    "bats-support": "https://github.com/bats-core/bats-support"
+  }
+}

test/README.md

@@ -0,0 +1,23 @@
+# How to test the workflows locally?
+
+## Prerequisites
+
+You must install [act](https://github.com/nektos/act)
+
+## Run the workflows locally
+
+The test folder contains json files for each scenario you want to test. 
+You can run any of the scenarios by running the following command:
+
+```bash
+act <event> -e test/<scenario>.json
+```
+
+## Use tests.bats
+
+You can also run tests using [bats](https://github.com/bats-core/bats-core).
+
+```node
+npm install
+npm run test
+```

test/issue_comment/with_command.json

@@ -0,0 +1,12 @@
+{
+  "act": true,
+  "action": "created",
+  "comment": {
+    "body": "/validate"
+  },
+  "issue":{
+    "state": "open",
+    "number": 1,
+    "body": "### Docker Hub repository name\n\nbenjaming/extension-builder-extension\n\n### Terms of services\n\n- [X] I accept the [term of services](https://www.docker.com/legal/extensions_marketplace_developer_agreement/)"
+  }
+}

test/issue_comment/with_command_on_closed_issue.json

@@ -0,0 +1,12 @@
+{
+  "act": true,
+  "action": "created",
+  "comment": {
+    "body": "/validate"
+  },
+  "issue":{
+    "state": "closed",
+    "number": 1,
+    "body": "### Docker Hub repository name\n\nbenjaming/extension-builder-extension\n\n### Terms of services\n\n- [X] I accept the [term of services](https://www.docker.com/legal/extensions_marketplace_developer_agreement/)"
+  }
+}

test/issues/no_repository.json

@@ -0,0 +1,10 @@
+{
+  "act": true,
+  "issue":{
+    "state": "open",
+    "body": "### Terms of services\n\n- [x] I accept the [term of services](https://www.docker.com/legal/extensions_marketplace_developer_agreement/)",
+    "user": {
+      "login": "benjaming"
+    }
+  }
+}

test/issues/no_tos.json

@@ -0,0 +1,10 @@
+{
+  "act": true,
+  "issue":{
+    "state": "open",
+    "body": "### Docker Hub repository name\n\nbenjaming/extension-builder-extension",
+    "user": {
+      "login": "benjaming"
+    }
+  }
+}

test/issues/tos_accepted.json

@@ -0,0 +1,11 @@
+{
+  "act": true,
+  "issue":{
+    "state": "open",
+    "number": 1,
+    "body": "### Docker Hub repository name\n\nbenjaming/extension-builder-extension\n\n### Terms of services\n\n- [X] I accept the [term of services](https://www.docker.com/legal/extensions_marketplace_developer_agreement/)",
+    "user": {
+      "login": "benjaming"
+    }
+  }
+}

test/issues/tos_not_accepted.json

@@ -0,0 +1,10 @@
+{
+  "act": true,
+  "issue":{
+    "state": "open",
+    "body": "### Docker Hub repository name\n\nbenjaming/extension-builder-extension\n\n### Terms of services\n\n- [ ] I accept the [term of services](https://www.docker.com/legal/extensions_marketplace_developer_agreement/)",
+    "user": {
+      "login": "benjaming"
+    }
+  }
+}

tests.bats

@@ -0,0 +1,81 @@
+setup() {
+    load 'node_modules/bats-support/load'
+    load 'node_modules/bats-assert/load'
+}
+
+@test "validation succeeded" {
+    run act issues -e test/issues/tos_accepted.json \
+        --container-architecture linux/amd64
+
+    assert_line '[validate/Extension validation/validation-succeeded] 🏁  Job succeeded'
+    refute_line --partial 'validation-failed'
+    refute_line --partial 'validation-error'
+    assert_success
+}
+
+@test "validation failed" {
+    run act issues -e test/issues/tos_accepted.json \
+        --env TEST_VALIDATION_FAILED=true \
+        --container-architecture linux/amd64
+
+    assert_line '[validate/Extension validation/validation-failed   ]   ❌  Failure - Main Mark job as failed'
+    refute_line --partial 'validation-succeeded'
+    refute_line --partial 'validation-error'
+    assert_failure
+}
+
+@test "validation errored" {
+    run act issues -e test/issues/tos_accepted.json \
+        --env TEST_VALIDATION_ERRORED=true \
+        --container-architecture linux/amd64
+
+    assert_line '[validate/Extension validation/validation-errored  ]   ❌  Failure - Main Mark job as failed'
+    refute_line --partial 'validation-succeeded'
+    refute_line --partial 'validation-failed'
+    assert_failure
+}
+
+@test "do not validate extension when tos are not accepted" {
+    run act issues -e test/issues/tos_not_accepted.json \
+        --container-architecture linux/amd64
+
+    assert_line '[TOS Check/tos-not-accepted                    ] 🏁  Job succeeded'
+    assert_success
+}
+
+@test "do not validate extension when tos are missing" {
+    run act issues -e test/issues/no_tos.json \
+        --container-architecture linux/amd64
+
+    assert_line '[TOS Check/Ensure Terms of Service are accepted]   ❌  Failure - Main Validate tos'
+    assert_line '[TOS Check/tos-not-found                       ]   ❌  Failure - Main Mark job as failed'
+    assert_failure
+}
+
+@test "do not validate extension when repository is missing" {
+    run act issues -e test/issues/no_repository.json \
+        --container-architecture linux/amd64
+
+    assert_line '[validate/Extension validation/parse-issue]   ❌  Failure - Main Validate repository'
+    assert_line '[validate/Extension validation/error             ]   ❌  Failure - Main Mark job as failed'
+    assert_failure
+}
+
+@test "validate extension when comment contains /validate" {
+    run act issue_comment -e test/issue_comment/with_command.json \
+        --container-architecture linux/amd64
+
+    assert_line '[TOS Check/Ensure Terms of Service are accepted]   ✅  Success - Main Run /validate command'
+    assert_line '[validate/Extension validation/validation-succeeded] 🏁  Job succeeded'
+    refute_line --partial 'validation-failed'
+    refute_line --partial 'validation-error'
+    assert_success
+}
+
+@test "do not validate extension when comment contains /validate on a closed issue" {
+    run act issue_comment -e test/issue_comment/with_command_on_closed_issue.json \
+        --container-architecture linux/amd64
+
+    refute_output
+    assert_success
+}