Merge remote-tracking branch 'private/enterprise-3.10-release' into 3.10.3-patch

2023-10-02 19:44:00 +00:00 · 2023-10-02 19:44:00 +00:00 · 0f9a5a34e1
--- a/.github/linters/.yaml-lint.yml
+++ b/.github/linters/.yaml-lint.yml
@ -0,0 +1,53 @@
+---
+###########################################
+# These are the rules used for            #
+# linting all the yaml files in the stack #
+# NOTE:                                   #
+# You can disable line with:              #
+# # yamllint disable-line                 #
+###########################################
+rules:
+  braces:
+    level: warning
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: 1
+    max-spaces-inside-empty: 5
+  brackets:
+    level: warning
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+    min-spaces-inside-empty: 1
+    max-spaces-inside-empty: 5
+  colons:
+    level: warning
+    max-spaces-before: 0
+    max-spaces-after: 1
+  commas:
+    level: warning
+    max-spaces-before: 0
+    min-spaces-after: 1
+    max-spaces-after: 1
+  comments: disable
+  comments-indentation: disable
+  document-end: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+    max: 2
+    max-start: 0
+    max-end: 0
+  hyphens:
+    level: warning
+    max-spaces-after: 1
+  indentation:
+    level: warning
+    spaces: consistent
+    indent-sequences: true
+    check-multi-line-strings: false
+  key-duplicates: enable
+  line-length: disable
+  new-line-at-end-of-file: disable
+  new-lines:
+    type: unix
+  trailing-spaces: disable
--- a/.github/workflows/build-and-release.yml
+++ b/.github/workflows/build-and-release.yml
@ -0,0 +1,139 @@
+---
+name: Build and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      gh-token:
+        description: 'GitHub Token - used to create a commit in the backup-utils repo'
+        required: true
+        type: string
+      version:
+        description: 'Version - patch version of the release (e.g. x.y.z)'
+        required: true
+        type: string
+      draft:
+        description: 'Draft - true if the release should be a draft'
+        required: true
+        type: boolean
+        default: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # resulting token still gets denied by the backup-utils repo
+      # see: https://github.com/actions/create-github-app-token/pull/46
+      # - uses: timreimherr/create-github-app-token@main
+      #   id: app-token
+      #   with:
+      #     # required
+      #     app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
+      #     private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
+      #     owner: ${{ github.repository_owner }}
+      #     repositories: backup-utils,backup-utils-private
+      - name: Checkout backup-utils-private
+        uses: actions/checkout@v4
+        with:
+          token: ${{ github.event.inputs.gh-token }}
+          repository: github/backup-utils-private
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y moreutils debhelper help2man devscripts gzip
+      - name: Create tag # this is required for the build scripts
+        run: |
+          git config user.name "${{ github.actor }}"
+          git config user.email "ghes-releases-team@github.com"
+          git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}"
+          git push origin "v${{ github.event.inputs.version }}"
+      - name: Package deb
+        run: |
+          ./script/package-deb
+      # many need to remove this once release-notes compilation is automated
+      - name: Rename deb artifact
+        run: |
+            for file in dist/github-backup-utils_*_all.deb; do
+              if [[ -f "$file" ]]; then
+                mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb"
+              fi
+            done
+      - name: Upload deb artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: github-backup-utils_${{ github.event.inputs.version }}_all.deb
+          path: |
+            dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb
+      - name: Package tarball
+        run: |
+          ./script/package-tarball
+      - name: Upload tarball artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
+          path: |
+            dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
+  release:
+    needs: build
+    runs-on: ubuntu-latest
+    outputs:
+      commit_hash: ${{ steps.empty-commit.outputs.commit_hash }}
+    steps:
+      # resulting token still gets denied by the backup-utils repo
+      # see: https://github.com/actions/create-github-app-token/pull/46
+      # - uses: timreimherr/create-github-app-token@main
+      #   id: app-token
+      #   with:
+      #     app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }}
+      #     private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }}
+      #     owner: ${{ github.repository_owner }}
+      #     repositories: backup-utils,backup-utils-private
+      - name: Checkout backup-utils
+        uses: actions/checkout@v4
+        with:
+          token: ${{ github.event.inputs.gh-token }}
+          repository: github/backup-utils
+          ref: master
+      - name: Create empty commit
+        uses: stefanzweifel/git-auto-commit-action@v4
+        id: empty-commit
+        with:
+          branch: master
+          commit_message: "${{ github.event.inputs.version }} release"
+          commit_user_name: "${{ github.actor }}"
+          commit_user_email: "ghes-releases-team@github.com"
+          commit_options: "--allow-empty"
+          skip_dirty_check: true
+      - name: Checkout backup-utils
+        uses: actions/checkout@v4
+        with:
+          token: ${{ github.event.inputs.gh-token }}
+          repository: github/backup-utils-private
+      - name: Download deb artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: github-backup-utils_${{ github.event.inputs.version }}_all.deb
+      - name: Download tarball artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          token: ${{ github.event.inputs.gh-token }}
+          repo: backup-utils
+          name: |
+            GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}
+          artifacts: |
+            github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \
+            github-backup-utils_${{ github.event.inputs.version }}_all.deb
+          tag: v${{ github.event.inputs.version }}
+          commit: ${{ steps.empty-commit.outputs.commit_hash }}
+          bodyFile: release-notes/${{ github.event.inputs.version }}.md
+          draft: ${{ github.event.inputs.draft }}
+          allowUpdates: true
+          artifactContentType: "raw"
+
+
+
+
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -1,3 +1,4 @@
+---
 name: Lint Code Base

 on:
@ -19,3 +20,4 @@ jobs:
        env:
          VALIDATE_ALL_CODEBASE: false
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          FILTER_REGEX_EXCLUDE: .*release-notes/.*
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -1,14 +1,11 @@
 name: Test and build

-on: [pull_request]
+on: [pull_request, workflow_dispatch]

 jobs:
  build:
    strategy:
      matrix:
-        # macos-latest references are kept here for historical purposes. removed macos-latest from the 
-        #matrix as it is not a typical case for users and causes a lot of friction with other linux-based 
-        # installs. Recommend developing on codespaces or using an ubuntu container. 
        os: ['ubuntu-22.04', 'ubuntu-20.04']
      fail-fast: false
    runs-on: ${{ matrix.os }}
@ -20,12 +17,6 @@ jobs:
        wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz"
        tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz"
        sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck
-      if: matrix.os != 'macos-latest'
-    - name: Install Dependencies (macOS)
-      run: |
-        brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget
-        brew install moreutils gawk
-      if: matrix.os == 'macos-latest'
    - name: Get Sources
      uses: actions/checkout@v3
    - name: Test
@ -35,4 +26,3 @@ jobs:
      shell: bash
    - name: Build (Linux)
      run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc
-      if: matrix.os != 'macos-latest'
--- a/.releaseignore
+++ b/.releaseignore
@ -0,0 +1,2 @@
+ownership.yaml
+.github
--- a/bin/ghe-backup
+++ b/bin/ghe-backup
@ -163,11 +163,11 @@ echo "$GHE_REMOTE_VERSION" > version
 # Setup progress tracking
 init-progress
 export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
-echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
 export PROGRESS_TYPE="Backup"
-echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
 export PROGRESS=0 # Used to track progress of backup
-echo "$PROGRESS" > /tmp/backup-utils-progress
+echo "$PROGRESS" > /tmp/backup-utils-progress/progress

 OPTIONAL_STEPS=0
 # Backup actions+mssql
@ -191,7 +191,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then
 fi

 PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
-echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total

 # check that incremental settings are valid if set
 is_inc=$(is_incremental_backup_feature_on)
--- a/bin/ghe-backup-progress
+++ b/bin/ghe-backup-progress
@ -31,7 +31,7 @@ while true; do
 done

 check_for_progress_file() {
-    if [ ! -f /tmp/backup-utils-progress-info ]; then
+    if [ ! -f /tmp/backup-utils-progress/info ]; then
        echo "No progress file found. Has a backup or restore been started?"
        exit 1
    fi
@ -39,18 +39,18 @@ check_for_progress_file() {

 if [ -n "$ONCE" ]; then
    check_for_progress_file
-    cat /tmp/backup-utils-progress-info
+    cat /tmp/backup-utils-progress/info
 else
    check_for_progress_file
    clear
-    cat /tmp/backup-utils-progress-info
+    cat /tmp/backup-utils-progress/info
    while true; do
    if read -r -t 1 -n 1; then
 	clear
        exit ;
    else
 	clear
-	cat /tmp/backup-utils-progress-info
+	cat /tmp/backup-utils-progress/info
    fi
    done
 fi
--- a/bin/ghe-restore
+++ b/bin/ghe-restore
@ -315,11 +315,11 @@ fi
 export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))

 init-progress
-echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
 export PROGRESS_TYPE="Restore"
-echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
+echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type
 export PROGRESS=0 # Used to track progress of restore
-echo "$PROGRESS" > /tmp/backup-utils-progress
+echo "$PROGRESS" > /tmp/backup-utils-progress/progress

 # Log restore start message locally and in /var/log/syslog on remote instance
 START_TIME=$(date +%s)
--- a/script/cibuild
+++ b/script/cibuild
@ -5,7 +5,7 @@ set -e
 # Enable verbose logging of ssh commands
 export GHE_VERBOSE_SSH=true

-if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then
+if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then
  exit 1
 fi

--- a/script/package-deb
+++ b/script/package-deb
@ -8,6 +8,9 @@ set -e
 # Change into project root
 cd "$(dirname "$0")"/..

+# Fetch tags from remote repository
+git fetch --tags
+
 # Basic package name and version.
 PKG_BASE="github-backup-utils"
 PKG_VERS="$(git describe --tags)"
@ -22,6 +25,14 @@ mkdir -p dist/debuild
 distdir="$(pwd)/dist/debuild/$PKG_NAME"
 git clone -q . "$distdir"
 cd "$distdir"
+
+echo "Removing files listed in .releaseignore ..."
+while IFS= read -r line; do
+    rm -rf "$line"
+done < .releaseignore
+
+echo "Removing .releaseignore ..."
+rm -f .releaseignore
 git checkout -q "$PKG_HEAD"

 debuild -uc -us 1>&2
--- a/script/package-tarball
+++ b/script/package-tarball
@ -8,11 +8,24 @@ set -e
 # Change into project root
 cd "$(dirname "$0")"/..

+# Fetch tags from remote repository
+git fetch --tags
+
 # Basic package name and version.
 PKG_BASE="github-backup-utils"
 PKG_VERS="$(git describe --tags)"
 PKG_NAME="${PKG_BASE}-${PKG_VERS}"

+# Remove all files or directories listed in .releaseignore
+echo "Removing files listed in .releaseignore ..."
+while IFS= read -r line; do
+    rm -rf "$line"
+done < .releaseignore
+
+# Remove the .releaseignore file itself
+echo "Removing .releaseignore ..."
+rm -f .releaseignore
+
 # Run git-archive to generate tarball
 echo "Creating ${PKG_NAME}.tar.gz ..."
 mkdir -p dist
--- a/share/github-backup-utils/ghe-backup-config
+++ b/share/github-backup-utils/ghe-backup-config
@ -217,8 +217,10 @@ ghe_parallel_check() {
  GHE_PARALLEL_COMMAND="parallel"
  local x
  for x in \
+      /usr/bin/parallel-moreutils \
      /usr/bin/parallel.moreutils \
      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils-parallel \
      /usr/bin/moreutils.parallel \
      /usr/bin/moreutils_parallel \
      ; do
@ -650,12 +652,22 @@ restore-secret() {

 #initialize progress tracking by clearing out the temp files used to track
 init-progress() {
-  rm -f /tmp/backup-utils-progress*
+  if [ -d /tmp/backup-utils-progress ]; then
+    rm -rf /tmp/backup-utils-progress/*
+  fi
+
+  mkdir -p /tmp/backup-utils-progress
+  chmod -R 777 /tmp/backup-utils-progress
+  
+  touch /tmp/backup-utils-progress/total
+  touch /tmp/backup-utils-progress/type
+  touch /tmp/backup-utils-progress/progress
+  touch /tmp/backup-utils-progress/info
 }


 #increase total count of progress
 increment-progress-total-count() {
  ((PROGRESS_TOTAL += $1))
-  echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+  echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total
 }
--- a/share/github-backup-utils/track-progress
+++ b/share/github-backup-utils/track-progress
@ -1,13 +1,12 @@
 #!/usr/bin/env bash
 #/ track-progress: track progress of backup or restore tasks

-# Current version is working solely with backups
 progress(){
    
-    PROGRESS=$(cat /tmp/backup-utils-progress)
-    PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total)
-    PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type)
+    PROGRESS=$(cat /tmp/backup-utils-progress/progress)
+    PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total)
+    PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type)
    PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc)
-    echo $((PROGRESS + 1)) > /tmp/backup-utils-progress
-    echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info
+    echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress
+    echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info
 }
--- a/test/test-ghe-backup.sh
+++ b/test/test-ghe-backup.sh
@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot"
  [ "$first_snapshot" != "$this_snapshot" ]

  verify_all_backedup_data
+
+  verify_progress_cleanup_process
 )
 end_test

--- a/test/test-ghe-host-check.sh
+++ b/test/test-ghe-host-check.sh
@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
  read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION)
  bu_major_minor="$bu_version_major.$bu_version_minor"
  releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json)
-  supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")')
+  latest_value=$(echo "$releases" | jq -r '.latest')
+  latest_major_version=$(echo $latest_value | cut -d "." -f 1-2)
+  # Replace "latest" with the derived major version in the releases string
+  releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g')
+  # Use the modified releases string as needed
+  supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")')
  # shellcheck disable=SC2207 # Command required as alternatives fail
-  keys=($(echo $releases | jq -r 'keys[]'))
+  keys=($(echo "$releases_with_replacement" | jq -r 'keys[]'))

  if [ -z "$supported" ]
  then
--- a/test/testlib.sh
+++ b/test/testlib.sh
@ -494,6 +494,12 @@ verify_all_backedup_data() {
  verify_common_data
 }

+# A unified method to make sure post backup, the cleanup process works
+verify_progress_cleanup_process() {
+  set -e
+  sudo -u nobody rm -rf /tmp/backup-utils-progress/*
+}
+
 # A unified method to check everything restored when performing a full restore
 # during testing.
 verify_all_restored_data() {
@ -600,8 +606,10 @@ setup_moreutils_parallel() {
  # We need moreutils parallel
  local x
  for x in \
+      /usr/bin/parallel-moreutils \
      /usr/bin/parallel.moreutils \
      /usr/bin/parallel_moreutils \
+      /usr/bin/moreutils-parallel \
      /usr/bin/moreutils.parallel \
      /usr/bin/moreutils_parallel \
      ; do