From 0a4001f3470caf16e11a986186e429711ec62a6b Mon Sep 17 00:00:00 2001 From: Krayon Date: Wed, 24 May 2023 13:40:57 +1000 Subject: [PATCH 001/410] Add git minimum version --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index a7dc45ebe..a66863d8b 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From 64905db77228de10dedc7459086ff56f2f57e6c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 20:59:50 +0000 Subject: [PATCH 002/410] Bump peter-evans/create-pull-request from 3 to 5 Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 3 to 5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v3...v5) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/rsync-docker-bump.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index c59e4ede1..6012e3986 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -22,7 +22,7 @@ jobs: sed -i -E "s/RSYNC_TAG=[0-9\.]+/RSYNC_TAG=${{ steps.latest_tag.outputs.latest_tag }}/g" Dockerfile - name: Create Pull Request for tag update - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "Update rsync tag in Dockerfile" From 79d487591def1c48e7affe2d314a73a70c6e4bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 13 Jul 2023 15:51:23 +0200 Subject: [PATCH 003/410] Avoid redundant rsync operations In noncluster environments in a high-availability configuration, there is only one target server to restore repository data onto. However, the current implementation performs one rsync task per node in the replication network. In a setup with one primary instance and two passive replicas, this would amount to three rsync tasks with identical or almost identical file lists. Aside from the rsync task for transferring repository data from the backup snapshot onto the target server, the other rsync operations per replica are unnecessary. Avoiding these redundant rsync tasks reduces the runtime of ghe-restore-repositories by about 12 % for a customer with roughly 1 TB of repository data. With GHE_PARALLEL_ENABLED=yes configured, this change also prevents a race condition, which could occur when multiple rsync processes try to write to the same file on the target server at the same time. --- .../ghe-restore-repositories | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index d59864196..e8e1dcdbe 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -102,42 +102,49 @@ done > $tmp_list IFS=$OLDIFS bm_end "$(basename $0) - Building network list" -# The server returns a list of routes: -# -# a/nw/a8/3f/02/100000855 dgit-node1 dgit-node2 dgit-node3 -# a/nw/a8/bc/8d/100000880 dgit-node1 dgit-node2 dgit-node4 -# a/nw/a5/06/81/100000659 dgit-node3 dgit-node2 dgit-node4 -# ... -# -# One route per line. -# -# NOTE: The route generation is performed on the appliance as it is considerably -# more performant than performing over an SSH pipe. -# -bm_start "$(basename $0) - Transferring network list" -cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" -- sponge $remote_tmp_list -cat $tmp_list | ghe_debug -bm_end "$(basename $0) - Transferring network list" - -bm_start "$(basename $0) - Generating routes" -restore_routes_script="github-env ./bin/dgit-cluster-restore-routes" -if ghe-ssh "$GHE_HOSTNAME" test -e /usr/local/share/enterprise/ghe-restore-network-routes; then - restore_routes_script="/usr/local/share/enterprise/ghe-restore-network-routes" +if $CLUSTER; then + # The server returns a list of routes: + # + # a/nw/a8/3f/02/100000855 dgit-node1 dgit-node2 dgit-node3 + # a/nw/a8/bc/8d/100000880 dgit-node1 dgit-node2 dgit-node4 + # a/nw/a5/06/81/100000659 dgit-node3 dgit-node2 dgit-node4 + # ... + # + # One route per line. + # + # NOTE: The route generation is performed on the appliance as it is considerably + # more performant than performing over an SSH pipe. + # + bm_start "$(basename $0) - Transferring network list" + cat $tmp_list | ghe-ssh "$GHE_HOSTNAME" -- sponge $remote_tmp_list + cat $tmp_list | ghe_debug + bm_end "$(basename $0) - Transferring network list" + + bm_start "$(basename $0) - Generating routes" + restore_routes_script="github-env ./bin/dgit-cluster-restore-routes" + if ghe-ssh "$GHE_HOSTNAME" test -e /usr/local/share/enterprise/ghe-restore-network-routes; then + restore_routes_script="/usr/local/share/enterprise/ghe-restore-network-routes" + fi + echo "cat $remote_tmp_list | $restore_routes_script | grep 'git-server-' > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash + ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug + bm_end "$(basename $0) - Generating routes" + + bm_start "$(basename $0) - Fetching routes" + ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list + cat $routes_list | ghe_debug + bm_end "$(basename $0) - Fetching routes" + + bm_start "$(basename $0) - Processing routes" + + cat $routes_list | awk -v tempdir="$tempdir" '{ for(i=2;i<=NF;i++){ print $1 > (tempdir"/"$i".rsync") }}' + cat $routes_list | awk '{ n = split($1, p, "/"); printf p[n] " /data/repositories/" $1; $1=""; print $0}' > $to_restore + ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" + bm_end "$(basename $0) - Processing routes" +else + # In noncluster setups, the primary instance owns all repository networks, so all network paths + # are to be synchronized to the primary instance. + cp "$tmp_list" "$tempdir/git-server-primary.rsync" fi -echo "cat $remote_tmp_list | $restore_routes_script | grep 'git-server-' > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash -ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug -bm_end "$(basename $0) - Generating routes" - -bm_start "$(basename $0) - Fetching routes" -ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list -cat $routes_list | ghe_debug -bm_end "$(basename $0) - Fetching routes" - -bm_start "$(basename $0) - Processing routes" -cat $routes_list | awk -v tempdir="$tempdir" '{ for(i=2;i<=NF;i++){ print $1 > (tempdir"/"$i".rsync") }}' -cat $routes_list | awk '{ n = split($1, p, "/"); printf p[n] " /data/repositories/" $1; $1=""; print $0}' > $to_restore -ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" -bm_end "$(basename $0) - Processing routes" if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then log_warn "Warning: no routes found, skipping repositories restore ..." From 91b21fe085105f34ea4b03690fb3ec7fa2dcf7a3 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 27 Jul 2023 16:22:52 +0000 Subject: [PATCH 004/410] backup container prefix for actions --- share/github-backup-utils/ghe-backup-config | 14 +++++++------- share/github-backup-utils/ghe-backup-settings | 1 + share/github-backup-utils/ghe-restore-actions | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 31e9aff0a..00bcf716e 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -372,13 +372,13 @@ export GHE_DATA_DIR : "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" # Check that utils are not being run directly on GHE appliance. -if [ -f "$GHE_RELEASE_FILE" ]; then - echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 - echo " The backup utilities should be run on a host dedicated to" 1>&2 - echo " long-term permanent storage and must have network connectivity" 1>&2 - echo " with the GitHub Enterprise appliance." 1>&2 - exit 1 -fi +# if [ -f "$GHE_RELEASE_FILE" ]; then +# echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 +# echo " The backup utilities should be run on a host dedicated to" 1>&2 +# echo " long-term permanent storage and must have network connectivity" 1>&2 +# echo " with the GitHub Enterprise appliance." 1>&2 +# exit 1 +# fi GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 43bd41029..ad1c4e49c 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -122,6 +122,7 @@ if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 93596033a..af33c60d4 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -68,6 +68,7 @@ restore-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secr restore-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" restore-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" restore-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" +restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" restore-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" restore-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" From 591416ccd98ddd9065362ac74efaa488aba18b06 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 27 Jul 2023 18:47:17 +0000 Subject: [PATCH 005/410] fix track-progress --- share/github-backup-utils/track-progress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index 3f67ca9e8..1381cbbef 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -7,7 +7,7 @@ progress(){ PROGRESS=$(cat /tmp/backup-utils-progress) PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) - PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) + PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100") echo $((PROGRESS +1)) > /tmp/backup-utils-progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info } From d83e0085ff44d355167ec066b418a45244941cec Mon Sep 17 00:00:00 2001 From: David Daly Date: Fri, 28 Jul 2023 18:31:21 +0000 Subject: [PATCH 006/410] update to only restore prefix when using -c --- share/github-backup-utils/ghe-backup-config | 14 +++++++------- share/github-backup-utils/ghe-restore-actions | 8 +++++++- share/github-backup-utils/track-progress | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 00bcf716e..31e9aff0a 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -372,13 +372,13 @@ export GHE_DATA_DIR : "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" # Check that utils are not being run directly on GHE appliance. -# if [ -f "$GHE_RELEASE_FILE" ]; then -# echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 -# echo " The backup utilities should be run on a host dedicated to" 1>&2 -# echo " long-term permanent storage and must have network connectivity" 1>&2 -# echo " with the GitHub Enterprise appliance." 1>&2 -# exit 1 -# fi +if [ -f "$GHE_RELEASE_FILE" ]; then + echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 + echo " The backup utilities should be run on a host dedicated to" 1>&2 + echo " long-term permanent storage and must have network connectivity" 1>&2 + echo " with the GitHub Enterprise appliance." 1>&2 + exit 1 +fi GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index af33c60d4..3db864c6f 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -68,7 +68,6 @@ restore-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secr restore-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" restore-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" restore-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" -restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" restore-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" restore-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" @@ -86,6 +85,13 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" +# Restore storage container prefix, but only if the `-c` option is used with ghe-restore +# `-c` should be used if restoring to an unconfigured appliance or when sif restoring to an unconfigured appliance or when +# specified manually. +# This is to avoid a staging instance using the same bucket prefix settings as production in the case of a staging instance restored from production +if $RESTORE_SETTINGS; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" +fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index 1381cbbef..3f67ca9e8 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -7,7 +7,7 @@ progress(){ PROGRESS=$(cat /tmp/backup-utils-progress) PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) - PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100") + PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) echo $((PROGRESS +1)) > /tmp/backup-utils-progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info } From 4210881c233efaf4dc9cdd71e6649e5eb8413647 Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 2 Aug 2023 21:12:52 +0000 Subject: [PATCH 007/410] allow backups on gheboot for testing --- share/github-backup-utils/ghe-backup-config | 14 +++++++------- share/github-backup-utils/ghe-restore-actions | 10 ++++++++++ share/github-backup-utils/track-progress | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 31e9aff0a..00bcf716e 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -372,13 +372,13 @@ export GHE_DATA_DIR : "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" # Check that utils are not being run directly on GHE appliance. -if [ -f "$GHE_RELEASE_FILE" ]; then - echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 - echo " The backup utilities should be run on a host dedicated to" 1>&2 - echo " long-term permanent storage and must have network connectivity" 1>&2 - echo " with the GitHub Enterprise appliance." 1>&2 - exit 1 -fi +# if [ -f "$GHE_RELEASE_FILE" ]; then +# echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 +# echo " The backup utilities should be run on a host dedicated to" 1>&2 +# echo " long-term permanent storage and must have network connectivity" 1>&2 +# echo " with the GitHub Enterprise appliance." 1>&2 +# exit 1 +# fi GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 3db864c6f..a24997453 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -92,6 +92,16 @@ restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert if $RESTORE_SETTINGS; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi +# Restore storage container prefix, but only if the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +# This value will only be present in backups from versions >= 3.10 so needs to be wrapped in version checks. +# if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then +# snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) +# # shellcheck disable=SC2046 # Word splitting is required to populate the variables +# read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) +# if [ "$(version $snapshot_version_major.$snapshot_version_minor.0)" -a $RESTORE_SETTINGS ]; then +# restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" +# fi +# fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index 3f67ca9e8..1381cbbef 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -7,7 +7,7 @@ progress(){ PROGRESS=$(cat /tmp/backup-utils-progress) PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) - PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) + PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100") echo $((PROGRESS +1)) > /tmp/backup-utils-progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info } From 54b7224298a660dbec688f5af7c15d7edac7d222 Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 2 Aug 2023 21:49:04 +0000 Subject: [PATCH 008/410] add check for greater than 3.9 --- share/github-backup-utils/ghe-restore-actions | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index a24997453..4eb846f7d 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -85,23 +85,16 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" -# Restore storage container prefix, but only if the `-c` option is used with ghe-restore -# `-c` should be used if restoring to an unconfigured appliance or when sif restoring to an unconfigured appliance or when -# specified manually. -# This is to avoid a staging instance using the same bucket prefix settings as production in the case of a staging instance restored from production -if $RESTORE_SETTINGS; then - restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" -fi # Restore storage container prefix, but only if the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings # This value will only be present in backups from versions >= 3.10 so needs to be wrapped in version checks. -# if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then -# snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) -# # shellcheck disable=SC2046 # Word splitting is required to populate the variables -# read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) -# if [ "$(version $snapshot_version_major.$snapshot_version_minor.0)" -a $RESTORE_SETTINGS ]; then -# restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" -# fi -# fi +if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then + snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) + # shellcheck disable=SC2046 # Word splitting is required to populate the variables + read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) + if [ "$snapshot_version_major" -eq "3" -a "$snapshot_version_minor" -ge "9" -a $RESTORE_SETTINGS ]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + fi +fi # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." From dc95625544e4956c5898c1b8fde38a96e17b7e87 Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 2 Aug 2023 22:16:57 +0000 Subject: [PATCH 009/410] update to check for 3.10 --- share/github-backup-utils/ghe-backup-config | 14 +++++++------- share/github-backup-utils/ghe-restore-actions | 2 +- share/github-backup-utils/track-progress | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 1203e346c..02a8df435 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -374,13 +374,13 @@ export GHE_DATA_DIR : "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" # Check that utils are not being run directly on GHE appliance. -# if [ -f "$GHE_RELEASE_FILE" ]; then -# echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 -# echo " The backup utilities should be run on a host dedicated to" 1>&2 -# echo " long-term permanent storage and must have network connectivity" 1>&2 -# echo " with the GitHub Enterprise appliance." 1>&2 -# exit 1 -# fi +if [ -f "$GHE_RELEASE_FILE" ]; then + echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 + echo " The backup utilities should be run on a host dedicated to" 1>&2 + echo " long-term permanent storage and must have network connectivity" 1>&2 + echo " with the GitHub Enterprise appliance." 1>&2 + exit 1 +fi GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 4eb846f7d..d5b493e3c 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -91,7 +91,7 @@ if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) # shellcheck disable=SC2046 # Word splitting is required to populate the variables read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) - if [ "$snapshot_version_major" -eq "3" -a "$snapshot_version_minor" -ge "9" -a $RESTORE_SETTINGS ]; then + if [ "$snapshot_version_major" -eq "3" -a "$snapshot_version_minor" -ge "10" -a $RESTORE_SETTINGS ]; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi fi diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index 00a9d2c65..bf50c42aa 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -11,4 +11,4 @@ progress(){ PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) echo $((PROGRESS + 1)) > /tmp/backup-utils-progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info -} \ No newline at end of file +} From e05b21ff9fd315f2cc2bf3babcc220b934b255b5 Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 2 Aug 2023 22:31:46 +0000 Subject: [PATCH 010/410] fix linting --- share/github-backup-utils/ghe-restore-actions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index d5b493e3c..78f75ab31 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -88,10 +88,10 @@ restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert # Restore storage container prefix, but only if the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings # This value will only be present in backups from versions >= 3.10 so needs to be wrapped in version checks. if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) + snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) - if [ "$snapshot_version_major" -eq "3" -a "$snapshot_version_minor" -ge "10" -a $RESTORE_SETTINGS ]; then + read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") + if [[ "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" && $RESTORE_SETTINGS ]]; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi fi From 116350d6880f04fc5d345ab60a4869e3e3b1c523 Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 2 Aug 2023 23:26:26 +0000 Subject: [PATCH 011/410] cleanup version check --- share/github-backup-utils/ghe-restore-actions | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 78f75ab31..8a67da02f 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -87,13 +87,11 @@ restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert # Restore storage container prefix, but only if the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings # This value will only be present in backups from versions >= 3.10 so needs to be wrapped in version checks. -if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") - # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") - if [[ "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" && $RESTORE_SETTINGS ]]; then - restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" - fi +snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") +# shellcheck disable=SC2046 # Word splitting is required to populate the variables +read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") +if [[ $RESTORE_SETTINGS && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10"]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi # Setup the database logins. From 40dbc40df5b0d159ac8b977fa3260c1812603454 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 3 Aug 2023 00:39:05 +0000 Subject: [PATCH 012/410] fix test --- share/github-backup-utils/ghe-restore-actions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 8a67da02f..cdc5480b5 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -90,7 +90,7 @@ restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") # shellcheck disable=SC2046 # Word splitting is required to populate the variables read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") -if [[ $RESTORE_SETTINGS && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10"]]; then +if [[ $RESTORE_SETTINGS && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" ]]; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi From acca07136c1ac90aefce6615ccafd81567ca0065 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 3 Aug 2023 00:54:14 +0000 Subject: [PATCH 013/410] fix restore settings check --- share/github-backup-utils/ghe-restore-actions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index cdc5480b5..d30d54488 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -90,7 +90,7 @@ restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") # shellcheck disable=SC2046 # Word splitting is required to populate the variables read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") -if [[ $RESTORE_SETTINGS && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" ]]; then +if [[ $RESTORE_SETTINGS == "true" && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" ]]; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" fi From a0a1a9fbec22aa02e25084c49dcebbad60420278 Mon Sep 17 00:00:00 2001 From: Terrell Broomer Date: Fri, 4 Aug 2023 15:30:59 +0900 Subject: [PATCH 014/410] update mssl scripts to exit 1 on failure --- share/github-backup-utils/ghe-backup-mssql | 2 +- share/github-backup-utils/ghe-restore-mssql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index 37b09ca9d..b22a4c2eb 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -62,7 +62,7 @@ fi if ! export_tool_available ; then log_error "ghe-export-mssql is not available" 1>&2 - exit + exit 1 fi add_minute() { diff --git a/share/github-backup-utils/ghe-restore-mssql b/share/github-backup-utils/ghe-restore-mssql index 3b62ce2c8..e7d48c61f 100755 --- a/share/github-backup-utils/ghe-restore-mssql +++ b/share/github-backup-utils/ghe-restore-mssql @@ -57,7 +57,7 @@ fi if ! import_tool_available; then ghe_verbose "ghe-import-mssql is not available" - exit + exit 1 fi # Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. From d4c7dd95570df6a9b712e9b995b4ebdd7e9fd573 Mon Sep 17 00:00:00 2001 From: Terrell Broomer Date: Mon, 7 Aug 2023 15:40:27 +0900 Subject: [PATCH 015/410] make super-linter happy --- .shellcheckrc | 1 + share/github-backup-utils/ghe-restore-mssql | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .shellcheckrc diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 000000000..1135b65a4 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +disable=SC2002 #useless cat \ No newline at end of file diff --git a/share/github-backup-utils/ghe-restore-mssql b/share/github-backup-utils/ghe-restore-mssql index e7d48c61f..e6fbb58e8 100755 --- a/share/github-backup-utils/ghe-restore-mssql +++ b/share/github-backup-utils/ghe-restore-mssql @@ -22,11 +22,11 @@ import_tool_available() { } ghe_ssh_mssql() { - ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" + ghe-ssh "$opts" "$ssh_config_file_opt" "$GHE_MSSQL_PRIMARY_HOST" "$@" } cleanup() { - rm -rf $tempdir + rm -rf "$tempdir" } trap 'cleanup' EXIT INT @@ -65,7 +65,7 @@ ghe_remote_version_required "$GHE_HOSTNAME" # The snapshot to restore should be set by the ghe-restore command but this lets # us run this script directly. -: ${GHE_RESTORE_SNAPSHOT:=current} +: "${GHE_RESTORE_SNAPSHOT:=current}" # The directory holding the snapshot to restore snapshot_dir_mssql="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql" @@ -79,13 +79,13 @@ do filename="${b##*/}" ghe_verbose "Transferring $filename to appliance host" - cat $snapshot_dir_mssql/$filename | ghe_ssh_mssql "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1" + cat "$snapshot_dir_mssql/$filename" | ghe_ssh_mssql "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1" done # Change owner to mssql:mssql to ready for restore ghe_ssh_mssql "sudo chown -R mssql:mssql $appliance_dir" # Invoke restore command -bm_start "$(basename $0)" +bm_start "$(basename "$0")" ghe_ssh_mssql -- "ghe-import-mssql" < "/dev/null" 1>&3 -bm_end "$(basename $0)" +bm_end "$(basename "$0")" From 4df4edf4b9894145f6d1ae8bea485c12105e6819 Mon Sep 17 00:00:00 2001 From: ddivad195 Date: Thu, 10 Aug 2023 19:09:50 +0000 Subject: [PATCH 016/410] update restore to check data exists vs check version --- share/github-backup-utils/ghe-restore-actions | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index d30d54488..14cfd1007 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -85,13 +85,13 @@ restore-secret "Actions Launch service private key" "actions-launch-app-app-priv restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" -# Restore storage container prefix, but only if the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings -# This value will only be present in backups from versions >= 3.10 so needs to be wrapped in version checks. -snapshot_version=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/version") -# shellcheck disable=SC2046 # Word splitting is required to populate the variables -read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version "$snapshot_version") -if [[ $RESTORE_SETTINGS == "true" && "$snapshot_version_major" -eq "3" && "$snapshot_version_minor" -ge "10" ]]; then - restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" +# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then + if [[ $RESTORE_SETTINGS == "true" ]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + fi +else + log_warn "Actions storage container prefix not present in backup. Skipping ..." fi # Setup the database logins. From ad23aec7627e24b137763a52251c3a3d3d7a6744 Mon Sep 17 00:00:00 2001 From: David Daly Date: Thu, 10 Aug 2023 20:56:41 +0100 Subject: [PATCH 017/410] update logic to check if '-c' is set first Co-authored-by: boxofyellow <54955040+boxofyellow@users.noreply.github.com> --- share/github-backup-utils/ghe-restore-actions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 14cfd1007..303a2abd2 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -86,12 +86,12 @@ restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" # Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings -if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then - if [[ $RESTORE_SETTINGS == "true" ]]; then +if [[ $RESTORE_SETTINGS == "true" ]]; then + if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + else + log_warn "Actions storage container prefix not present in backup. Skipping ..." fi -else - log_warn "Actions storage container prefix not present in backup. Skipping ..." fi # Setup the database logins. From f4fa25f3f8e0c66ee87dfeecee973c1b9761f7d9 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Thu, 10 Aug 2023 22:15:48 +0000 Subject: [PATCH 018/410] remove on-exit mode --- share/github-backup-utils/track-progress | 1 - 1 file changed, 1 deletion(-) diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index bf50c42aa..d88b3704f 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,6 +1,5 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -set -e # Current version is working solely with backups progress(){ From 6b6e1bfea8ca0b5494628190649fade68a70e34e Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Thu, 10 Aug 2023 23:04:46 +0000 Subject: [PATCH 019/410] add script to be excluded --- test/test-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 92ac538cc..231d78edb 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -49,7 +49,7 @@ begin_test "shellopts: set -e set on all scripts" # Check all executable scripts checked into the repo, except bm.sh, ghe-backup-config, ghe-rsync and the dummy test scripts set +x cd $BASE_PATH - git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do + git ls-tree -r HEAD | grep -Ev 'bm.sh|ghe-backup-config|ghe-rsync|track-progress|test/bin' | grep -E '^1007|.*\..*sh$' | awk '{print $4}' | while read -r script; do if head -n1 "$script" | grep -E -w "sh|bash" >/dev/null 2>&1; then grep -q "set -e" $script || echo $script >> $results || true fi From a44aa3d24cd15810021fc140fbf5a8d64289c614 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Sat, 12 Aug 2023 07:58:10 -0400 Subject: [PATCH 020/410] Bringing in a fix from the public repo Bringing in a fix from https://github.com/github/backup-utils/pull/1094 --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index f3f7d67f6..1320eb583 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](#april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer, and [bc][12] v1.07 or newer. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From 5ace2f2b931ac26e8daf5e11be3338ac2c4e5659 Mon Sep 17 00:00:00 2001 From: boxofyellow <54955040+boxofyellow@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:11:43 -0400 Subject: [PATCH 021/410] Get any backups that were created off the box even if we have a failure (#442) --- share/github-backup-utils/ghe-backup-mssql | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql index 37b09ca9d..f168b2cfe 100755 --- a/share/github-backup-utils/ghe-backup-mssql +++ b/share/github-backup-utils/ghe-backup-mssql @@ -335,9 +335,12 @@ if [ -n "$backup_type" ]; then elif [ "$backup_type" = "transaction" ]; then backup_command='ghe-export-mssql -t' fi - + + backup_failed= + bm_start "$(basename "$0")" - ghe_ssh_mssql -- "$backup_command" + # record if generating the backup failed, this will allow us to collect any backups that may have been produced, even if they are not complete they are better than nothing + ghe_ssh_mssql -- "$backup_command" || backup_failed='true' bm_end "$(basename "$0")" # Configure the backup cadence on the appliance, which is used for diagnostics @@ -352,4 +355,9 @@ if [ -n "$backup_type" ]; then ghe_verbose "Transferring to backup host $b" ghe_ssh_mssql "sudo cat $appliance_dir/$b" > "$backup_dir"/"$b" done + + if [ -n "$backup_failed" ]; then + log_error 'ghe-export-mssql failed to backup at least one mssql database' 1>&2 + exit 1 + fi fi From dff57d9267d48b4116c6cdaa0c04dbeb8fc5db3f Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 14 Aug 2023 16:27:46 -0700 Subject: [PATCH 022/410] Leave release in draft state by default --- script/release | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/script/release b/script/release index b7b61615e..42f2d0f46 100755 --- a/script/release +++ b/script/release @@ -34,6 +34,9 @@ DEB_PKG_NAME = 'github-backup-utils' GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? GH_STABLE_BRANCH = "" +# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI ++PUBLISH = ENV['PUBLISH'] == 'true' || false + CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium <%- changes.each do |ch| -%> @@ -480,8 +483,10 @@ if $PROGRAM_NAME == __FILE__ attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - puts 'Publishing release...' - publish_release res['id'] + if PUBLISH do + puts 'Publishing release...' + publish_release res['id'] + end puts 'Cleaning up...' clean_up version @@ -489,6 +494,10 @@ if $PROGRAM_NAME == __FILE__ puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch + if !PUBLISH do + puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' + end + puts 'Released!' rescue RuntimeError => e $stderr.puts "Error: #{e}" From 105ebb6b5a15c882fa28f79b1d291ebd5cdc6ef5 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 14 Aug 2023 16:42:19 -0700 Subject: [PATCH 023/410] Fix conditional statement --- script/release | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/release b/script/release index 42f2d0f46..d8b1ee9b2 100755 --- a/script/release +++ b/script/release @@ -483,7 +483,7 @@ if $PROGRAM_NAME == __FILE__ attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - if PUBLISH do + if PUBLISH puts 'Publishing release...' publish_release res['id'] end @@ -494,7 +494,7 @@ if $PROGRAM_NAME == __FILE__ puts "Updating #{GH_STABLE_BRANCH} branch..." update_stable_branch - if !PUBLISH do + if !PUBLISH puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' end From 4b434c20a0d6d7de6870ba73b8675a585e549c95 Mon Sep 17 00:00:00 2001 From: Devin Dooley Date: Mon, 14 Aug 2023 16:46:59 -0700 Subject: [PATCH 024/410] Remove accidental + sign --- script/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release b/script/release index d8b1ee9b2..891cd9f60 100755 --- a/script/release +++ b/script/release @@ -35,7 +35,7 @@ GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow GH_STABLE_BRANCH = "" # If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI -+PUBLISH = ENV['PUBLISH'] == 'true' || false +PUBLISH = ENV['PUBLISH'] == 'true' || false CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium From 550f3032f88cf16d31ddd421ebc79ba08917285f Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 15 Aug 2023 21:16:18 +0200 Subject: [PATCH 025/410] transfer_size are estimated data transfer sizes (#488) * Update bin/ghe-host-check --- bin/ghe-host-check | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9f6ca2603..1c3252645 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -184,7 +184,8 @@ SKIP_MSG echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2 echo "" 1>&2 - printf '### Data Transfer Sizes + printf '### Estimated Data Transfer Sizes + - repositories: %d MB - pages: %d MB - elasticsearch: %d MB @@ -192,7 +193,8 @@ SKIP_MSG - minio: %d MB - mysql: %d MB - actions: %d MB - - mssql: %d MB\n' \ + - mssql: %d MB +\n' \ "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2 if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then From 826215c15fa25ef5af9d1432f5173241d3e2b69d Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Wed, 16 Aug 2023 12:31:06 -0400 Subject: [PATCH 026/410] Change shellcheck to use stable, not latest Shelcheck is using latest and a recent change broke all of our tests. Changing this so we use the stable version as we don't need the newest version of shellcheck for tests. --- .github/workflows/main.yml | 6 +++--- test/test-shellcheck.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 809e9df50..da7695ad1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,9 +17,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck + wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" + tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" + sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck if: matrix.os != 'macos-latest' - name: Install Dependencies (macOS) run: | diff --git a/test/test-shellcheck.sh b/test/test-shellcheck.sh index 231d78edb..82c0fbb4f 100755 --- a/test/test-shellcheck.sh +++ b/test/test-shellcheck.sh @@ -11,8 +11,8 @@ begin_test "shellcheck: reports no errors or warnings" set -e # We manually install the latest Shellcheck on Linux builds as other options # are too old. - if [ -x "$BASE_PATH/shellcheck-latest/shellcheck" ]; then - shellcheck() { "$BASE_PATH/shellcheck-latest/shellcheck" "$@"; } + if [ -x "$BASE_PATH/shellcheck-stable/shellcheck" ]; then + shellcheck() { "$BASE_PATH/shellcheck-stable/shellcheck" "$@"; } fi if ! type shellcheck 1>/dev/null 2>&1; then From 3825fcbc25dadc67b37834f629320b5c49a38fd3 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Thu, 17 Aug 2023 11:19:09 +0200 Subject: [PATCH 027/410] Adding host online check (#492) * adding host online check * fix test --------- Co-authored-by: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> --- bin/ghe-host-check | 12 +++++++++--- test/bin/ghe-cluster-host-check | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100755 test/bin/ghe-cluster-host-check diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 1c3252645..104885114 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,13 +91,19 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -# ensure all nodes in the cluster are running the same version +# ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then + online_status=$(ghe-ssh "$host" ghe-cluster-host-check) + if [ "$online_status" != "Cluster is ready to configure." ]; then + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + exit 1 + fi + node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&3 + echo "Version mismatch: $node_version_list" 1>&2 + echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi diff --git a/test/bin/ghe-cluster-host-check b/test/bin/ghe-cluster-host-check new file mode 100755 index 000000000..3120d85de --- /dev/null +++ b/test/bin/ghe-cluster-host-check @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Usage: ghe-cluster-host-check +# Emulates a cluster reachability check +set -e +echo "Cluster is ready to configure." From 6c8648da655d472e99390d786d99d79e17a6cd63 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Thu, 17 Aug 2023 15:03:00 +0200 Subject: [PATCH 028/410] fix typo (#508) --- bin/ghe-restore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 92ae72443..19603ae47 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -469,13 +469,13 @@ fi # Always restore column encryption keys if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then - log_info "Always restore encrypted column encryption keys on GHES verions 3.7.0+" + log_info "Always restore encrypted column encryption keys on GHES versions 3.7.0+" fi ghe-restore-column-encryption-keys "$GHE_HOSTNAME" # Always restore secret scanning encryption keys if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+" + log_info "Always restore secret scanning encryption keys on GHES versions 3.8.0+" increment-progress-total-count 1 ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME" fi From 07162bd5882220d03d5d4edc73e4a2c97ab760cc Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 17 Aug 2023 17:40:09 -0400 Subject: [PATCH 029/410] Host key fix (#513) --- bin/ghe-backup | 66 ++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 2043e2fad..86ee33dd7 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -54,38 +54,7 @@ export CALLING_SCRIPT="ghe-backup" # shellcheck source=share/github-backup-utils/ghe-backup-config . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" -# Setup progress tracking -init-progress -export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total -export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type -export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress - -OPTIONAL_STEPS=0 -# Backup actions+mssql -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) -fi - -# Backup fsck -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -# Backup minio -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Backup pages -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total # Check to make sure moreutils parallel is installed and working properly ghe_parallel_check @@ -186,9 +155,44 @@ fi # Perform a host connection check and establish the remote appliance version. # The version is available in the GHE_REMOTE_VERSION variable and also written # to a version file in the snapshot directory itself. +# ghe_remote_version_required should be run before any other instances of ghe-ssh +# to ensure that there are no problems with host key verification. ghe_remote_version_required echo "$GHE_REMOTE_VERSION" > version +# Setup progress tracking +init-progress +export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +export PROGRESS_TYPE="Backup" +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +export PROGRESS=0 # Used to track progress of backup +echo "$PROGRESS" > /tmp/backup-utils-progress + +OPTIONAL_STEPS=0 +# Backup actions+mssql +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) +fi + +# Backup fsck +if [ "$GHE_BACKUP_FSCK" = "yes" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup minio +if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +# Backup pages +if [ "$GHE_BACKUP_PAGES" != "no" ]; then + OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) +fi + +PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) From 82b2e2a0a7149b8652ffa93b98e594714cccb1d4 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Fri, 18 Aug 2023 18:33:15 +0000 Subject: [PATCH 030/410] fix for cluster transfer sizes --- share/github-backup-utils/ghe-rsync-size | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) mode change 100755 => 100644 share/github-backup-utils/ghe-rsync-size diff --git a/share/github-backup-utils/ghe-rsync-size b/share/github-backup-utils/ghe-rsync-size old mode 100755 new mode 100644 index c61a80b3b..317c2f6cc --- a/share/github-backup-utils/ghe-rsync-size +++ b/share/github-backup-utils/ghe-rsync-size @@ -21,6 +21,7 @@ fi transfer_size() { + local host=$GHE_HOSTNAME local backup_data=$1 if [[ "$1" == "mssql" ]]; then data_user_dir="/data/user/$1/backups" @@ -58,20 +59,36 @@ transfer_size() ;; esac + ghe-ssh "$host" "[ -f /etc/github/cluster ]" + cluster_status=$? + + if [[ $cluster_status -eq 0 ]]; then + cluster_conf_out=$(ghe-ssh "$host" "cat /data/user/common/cluster.conf") + cluster_nodes_output=$(ghe-ssh "$host" "ghe-cluster-nodes -i") + if [[ ( "$1" == "elasticsearch" || "$1" == "storage" || "$1" == "pages" ) ]]; then + host_server=$(echo "$cluster_conf_out" | awk -v srv="$backup_data-server = true" '/cluster/ { prevA = $0 } $0 ~ srv { print prevA }' | head -1 | awk -F '"' '{print $2}') + elif [[ "$1" == "mysql" ]]; then + host_server=$(ghe-ssh "$host" "ghe-config cluster.mysql-master") + elif [[ "$1" == "repositories" ]]; then + host_server=$(echo "$cluster_conf_out" | awk '/git-server = true/ { print prevA } /cluster/ { prevA = $0 }' | head -1 | awk -F '"' '{print $2}') + fi + host=$(echo "$cluster_nodes_output" | grep "$host_server" | awk '{print $2}' | head -1) + fi + if [ -d "${GHE_DATA_DIR}/current/$1" ]; then total_file_size=$(ghe-rsync -arn --stats \ -e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \ --rsync-path="sudo -u $user rsync" \ "$link_dest"/"$1" \ --ignore-missing-args \ - "$GHE_HOSTNAME:$data_user_dir/" \ + "$host:$data_user_dir/" \ "$dest_dir/" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g') else total_file_size=$(ghe-rsync -arn --stats \ -e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \ --rsync-path="sudo -u $user rsync" \ --ignore-missing-args \ - "$GHE_HOSTNAME:$data_user_dir/" \ + "$host:$data_user_dir/" \ "$dest_dir/" | grep "Total transferred file size" | sed 's/.*size: //; s/,//g') fi From fb5aa593c1282939c0fdc22b9a1efedca79b3a1c Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Mon, 21 Aug 2023 13:49:08 -0500 Subject: [PATCH 031/410] Restore content scanning keys --- .../ghe-restore-secret-scanning-encryption-keys | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys index aa225bc07..4785a6d77 100755 --- a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -36,4 +36,8 @@ log_info "Restoring secret scanning encrypted secrets transit keys" restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +# Restore secret scanning content scanning keys if present +log_info "Restoring secret scanning content scanning keys" +restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + bm_end "$(basename $0)" From 5a64931986a310c3634b0e461d5966e3fd2b4e7f Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Mon, 21 Aug 2023 15:11:18 -0700 Subject: [PATCH 032/410] Added test of backup --- test/test-ghe-backup.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 745538264..9184746e3 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -802,6 +802,30 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt ) end_test +begin_test "ghe-backup takes backup of secret scanning encrypted content encryption keys" +( + set -e + + required_secrets=( + "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + ghe-backup + + required_files=( + "secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "foo" ] + done +) +end_test + begin_test "ghe-backup takes backup of Actions settings" ( set -e From 8dad53ef128fa622c8f4216a3f12bc275b64a1bd Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Mon, 21 Aug 2023 15:26:36 -0700 Subject: [PATCH 033/410] Added test for restoring encrypted content keys --- test/test-ghe-restore.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index ef13b7697..39d2bced6 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -454,6 +454,32 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test +begin_test "ghe-restore with secret scanning encrypted content encryption keys" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.11.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 + done +) +end_test + # Setup Actions data for the subsequent tests setup_actions_test_data "$GHE_DATA_DIR/1" From 5596eb8280263b9acc5b1a7420c54119d034b31e Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Mon, 21 Aug 2023 15:27:44 -0700 Subject: [PATCH 034/410] Added encrypted content key to backup-settings --- share/github-backup-utils/ghe-backup-settings | 1 + 1 file changed, 1 insertion(+) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 3a05feba7..362e230cf 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -90,6 +90,7 @@ backup-secret "secret scanning encrypted secrets current storage key" "secret-sc backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then From dd8e655bd24b5a510598073e2f522f545e327204 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Mon, 21 Aug 2023 15:54:15 -0700 Subject: [PATCH 035/410] Fixed restore tests --- test/test-ghe-restore.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 39d2bced6..cf93baa96 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -454,7 +454,33 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test -begin_test "ghe-restore with secret scanning encrypted content encryption keys" +begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0+" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.10.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.11.0 + done +) +end_test + +begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions 3.11.0+" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -475,7 +501,7 @@ begin_test "ghe-restore with secret scanning encrypted content encryption keys" ) for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting this to have been restored successfully for versions 3.11.0+ done ) end_test From 8bd786e011d6df7caea55ee652e8a072099b5b6a Mon Sep 17 00:00:00 2001 From: Robert Bolender Date: Mon, 21 Aug 2023 22:22:18 -0700 Subject: [PATCH 036/410] Only backup secret scanning secrets on GHES versions 3.8.0+ --- share/github-backup-utils/ghe-backup-settings | 11 +++--- test/test-ghe-backup.sh | 34 +++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 3a05feba7..aac02dde7 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -86,10 +86,13 @@ if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" fi -backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" -backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" -backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" -backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +# secret scanning encrypted secrets keys were added in GHES 3.8.0 +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 745538264..fb3f9db6a 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -772,7 +772,7 @@ begin_test "ghe-backup takes backup of encrypted column encryption keying materi ) end_test -begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys" +begin_test "ghe-backup does not take backups of secret scanning encrypted secrets encryption keys on versions below 3.8.0" ( set -e @@ -787,7 +787,37 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done - ghe-backup + GHE_REMOTE_VERSION=3.7.0 ghe-backup -v | grep -q "secret scanning encrypted secrets" && exit 1 + + required_files=( + "secret-scanning-encrypted-secrets-current-storage-key" + "secret-scanning-encrypted-secrets-delimited-storage-keys" + "secret-scanning-encrypted-secrets-current-shared-transit-key" + "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted secrets encryption keys on versions 3.8.0+" +( + set -e + + required_secrets=( + "secrets.secret-scanning.encrypted-secrets-current-storage-key" + "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.8.0 ghe-backup required_files=( "secret-scanning-encrypted-secrets-current-storage-key" From a5e9b0e5c64c4baa38c7c226fb5a515907f64156 Mon Sep 17 00:00:00 2001 From: "github-service-catalog[bot]" <66641770+github-service-catalog[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:30:02 -0400 Subject: [PATCH 037/410] Update service ownership data (#519) * Update service ownership data Co-authored-by: gamefiend <410344+gamefiend@users.noreply.github.com> --------- Co-authored-by: github-service-catalog[bot] <66641770+github-service-catalog[bot]@users.noreply.github.com> Co-authored-by: gamefiend <410344+gamefiend@users.noreply.github.com> Co-authored-by: Quinn Murphy --- ownership.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 ownership.yaml diff --git a/ownership.yaml b/ownership.yaml new file mode 100644 index 000000000..bde619edc --- /dev/null +++ b/ownership.yaml @@ -0,0 +1,29 @@ +--- +version: 1 +ownership: +- name: ghes-backup-utilities + long_name: GHES Backup Utilities + description: GitHub Enterprise Disaster Recover Solution + kind: logical + repo: https://github.com/github/backup-utils-private + qos: best_effort + team_slack: ghes-lifecycle-aor + team: github/ghes-lifecycle + maintainer: whitneyimura + exec_sponsor: jakuboleksy + tier: 3 + product_manager: davidjarzebowski + sev1: + slack: ghes-on-call + alert_slack: ghes-backup-utils + pagerduty: https://github.pagerduty.com/escalation_policies#PBQWK20 + tta: 30 minutes + sev2: + issue: https://github.com/github/ghes/issues/new + tta: 1 business day + sev3: + issue: https://github.com/github/ghes/issues + tta: 1 week + support_squad: + slack: support-squad-infrastructure + issue: https://github.com/github/support-squad-infrastructure/issues From b66db2af6b90f800da9c176867395f32305499de Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 22 Aug 2023 08:30:55 -0700 Subject: [PATCH 038/410] Only backup 3.11 and up --- share/github-backup-utils/ghe-backup-settings | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 362e230cf..28a251cec 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -90,7 +90,10 @@ backup-secret "secret scanning encrypted secrets current storage key" "secret-sc backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" -backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then + backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" +fi # Backup argon secrets for multiuser from ghes version 3.8 onwards if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then From 6437469aca30ee4616e1f53bdf765871bae79ca7 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Tue, 22 Aug 2023 16:11:42 -0400 Subject: [PATCH 039/410] change super-linter repo and Bash linting level (#529) * change super-linter repo Changed super-linter link to conform with change: > NOTICE: If your use of the super-linter action failed around April 26th, 2023, we changed the organization name from github to super-linter so you will need to update your references to this action from github/super-linter to super-linter/super-linter. * Update .github/workflows/lint.yml add BASH_SEVERITY settings --- .github/workflows/lint.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d15f123bb..0c6af658b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,8 @@ jobs: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 - name: Lint Code Base - uses: github/super-linter@v5 + uses: super-linter/super-linter@v5 env: VALIDATE_ALL_CODEBASE: false + BASH_SEVERITY: error GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 534ff5e9d232303bcf833eb5f6881b476f6cb21c Mon Sep 17 00:00:00 2001 From: Terrell Broomer Date: Wed, 23 Aug 2023 09:55:38 +0900 Subject: [PATCH 040/410] revert unccessary changes --- share/github-backup-utils/ghe-restore-mssql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-mssql b/share/github-backup-utils/ghe-restore-mssql index e6fbb58e8..e7d48c61f 100755 --- a/share/github-backup-utils/ghe-restore-mssql +++ b/share/github-backup-utils/ghe-restore-mssql @@ -22,11 +22,11 @@ import_tool_available() { } ghe_ssh_mssql() { - ghe-ssh "$opts" "$ssh_config_file_opt" "$GHE_MSSQL_PRIMARY_HOST" "$@" + ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" } cleanup() { - rm -rf "$tempdir" + rm -rf $tempdir } trap 'cleanup' EXIT INT @@ -65,7 +65,7 @@ ghe_remote_version_required "$GHE_HOSTNAME" # The snapshot to restore should be set by the ghe-restore command but this lets # us run this script directly. -: "${GHE_RESTORE_SNAPSHOT:=current}" +: ${GHE_RESTORE_SNAPSHOT:=current} # The directory holding the snapshot to restore snapshot_dir_mssql="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql" @@ -79,13 +79,13 @@ do filename="${b##*/}" ghe_verbose "Transferring $filename to appliance host" - cat "$snapshot_dir_mssql/$filename" | ghe_ssh_mssql "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1" + cat $snapshot_dir_mssql/$filename | ghe_ssh_mssql "sudo tee -a $appliance_dir/$filename >/dev/null 2>&1" done # Change owner to mssql:mssql to ready for restore ghe_ssh_mssql "sudo chown -R mssql:mssql $appliance_dir" # Invoke restore command -bm_start "$(basename "$0")" +bm_start "$(basename $0)" ghe_ssh_mssql -- "ghe-import-mssql" < "/dev/null" 1>&3 -bm_end "$(basename "$0")" +bm_end "$(basename $0)" From 4cc448a88c8030b9e48eb8b249be6ac92116ee6b Mon Sep 17 00:00:00 2001 From: Terrell Broomer Date: Wed, 23 Aug 2023 10:33:13 +0900 Subject: [PATCH 041/410] remove shellcheckrc --- .shellcheckrc | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .shellcheckrc diff --git a/.shellcheckrc b/.shellcheckrc deleted file mode 100644 index 1135b65a4..000000000 --- a/.shellcheckrc +++ /dev/null @@ -1 +0,0 @@ -disable=SC2002 #useless cat \ No newline at end of file From f14f8dd62116841dcf3c6129a571e05e3d19fe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 23 Aug 2023 20:56:36 +0200 Subject: [PATCH 042/410] Clarify special handling for cluster environments --- share/github-backup-utils/ghe-restore-repositories | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index e8e1dcdbe..67bbf8e34 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -102,6 +102,11 @@ done > $tmp_list IFS=$OLDIFS bm_end "$(basename $0) - Building network list" +# In cluster environments, we need to ensure that all repository networks are replicated back to the +# same Spokes nodes that they were present on when the backup was taken. For this, the list of +# routes of each repository network is first obtained. Afterward, an rsync file list is created for +# each Spokes node including only those repository networks for which there was a route to the +# respective Spokes node. if $CLUSTER; then # The server returns a list of routes: # @@ -140,9 +145,11 @@ if $CLUSTER; then cat $routes_list | awk '{ n = split($1, p, "/"); printf p[n] " /data/repositories/" $1; $1=""; print $0}' > $to_restore ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" bm_end "$(basename $0) - Processing routes" +# There is no need to collect routes and split them by Spokes server in noncluster setups because +# we need to transfer all repository networks to the primary instance unconditionally, regardless of +# the Spokes route list captured during the backup. As we already have the list of all repository +# network paths, we can simply use that as the rsync file list in noncluster environments. else - # In noncluster setups, the primary instance owns all repository networks, so all network paths - # are to be synchronized to the primary instance. cp "$tmp_list" "$tempdir/git-server-primary.rsync" fi From 4a6e0aed7adf86f9a9415bf8eb6d686d161a4d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 23 Aug 2023 22:02:56 +0200 Subject: [PATCH 043/410] Allow manually triggering testing workflow This is to facilitate testing while developing new features, as the test suite is not trivial to run on a local machine or even a fork and as opening pull requests for all experimental features would create a lot of noise. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da7695ad1..3e4445915 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: Test and build -on: [pull_request] +on: [pull_request, workflow_dispatch] jobs: build: From 3bee49085de900d8f2f0920cb5c9263d72c784f6 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 24 Aug 2023 14:52:11 -0400 Subject: [PATCH 044/410] Placing initial CODEOWNERS file (#532) still need to discuss the best users to set as codeowners of certain sections, will uncomment after ascertaining best teams --- .github/CODEOWNERS | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..fecb57ee9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,7 @@ +# Backup-Utils owned by lifecycle AOR +* @github/ghes-lifecycle +# Actions related backups and restores +# /share/github-backup-utils/*-actions @github/ghes-lifecycle @github/ +# Git related backups and restores +# /share/github-backup-utils/*-repositories @github/ghes-lifecycle @github/ +# /share/github-backup-utils/*-git-hooks @github/ghes-lifecycle @github/ \ No newline at end of file From 4abe738f06a649aef3d91575053593af6bd39c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Tue, 22 Aug 2023 02:19:25 +0200 Subject: [PATCH 045/410] Fix script permissions In order to satisfy the linter when making changes to this script, it should be executable. --- test/testlib.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 test/testlib.sh diff --git a/test/testlib.sh b/test/testlib.sh old mode 100644 new mode 100755 From fb9754530b6f9881e26cd4f12399930bab8c2c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Sat, 19 Aug 2023 03:22:57 +0200 Subject: [PATCH 046/410] Find parallel in more locations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some distributions append or prepend “moreutils” to the parallel command with a hyphen to distinguish it from the GNU version of parallel, such as Arch Linux [1]. To facilitate testing on such platforms (for example to test backup-utils with upcoming versions of rsync and newer Linux kernel versions), this adds support for finding parallel in those locations. [1] https://archlinux.org/packages/extra/x86_64/moreutils/ --- share/github-backup-utils/ghe-backup-config | 2 ++ test/testlib.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 2e176b510..5b7f1c3d3 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -217,8 +217,10 @@ ghe_parallel_check() { GHE_PARALLEL_COMMAND="parallel" local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do diff --git a/test/testlib.sh b/test/testlib.sh index 5dd5ac451..8e130eb75 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -600,8 +600,10 @@ setup_moreutils_parallel() { # We need moreutils parallel local x for x in \ + /usr/bin/parallel-moreutils \ /usr/bin/parallel.moreutils \ /usr/bin/parallel_moreutils \ + /usr/bin/moreutils-parallel \ /usr/bin/moreutils.parallel \ /usr/bin/moreutils_parallel \ ; do From 97102ce9e3d23725b8dfbec2a5125b5ba8e4b3d7 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Mon, 28 Aug 2023 19:50:27 +0000 Subject: [PATCH 047/410] switch to case --- share/github-backup-utils/ghe-rsync-size | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-rsync-size b/share/github-backup-utils/ghe-rsync-size index 317c2f6cc..3e454f6bf 100644 --- a/share/github-backup-utils/ghe-rsync-size +++ b/share/github-backup-utils/ghe-rsync-size @@ -65,14 +65,22 @@ transfer_size() if [[ $cluster_status -eq 0 ]]; then cluster_conf_out=$(ghe-ssh "$host" "cat /data/user/common/cluster.conf") cluster_nodes_output=$(ghe-ssh "$host" "ghe-cluster-nodes -i") - if [[ ( "$1" == "elasticsearch" || "$1" == "storage" || "$1" == "pages" ) ]]; then - host_server=$(echo "$cluster_conf_out" | awk -v srv="$backup_data-server = true" '/cluster/ { prevA = $0 } $0 ~ srv { print prevA }' | head -1 | awk -F '"' '{print $2}') - elif [[ "$1" == "mysql" ]]; then - host_server=$(ghe-ssh "$host" "ghe-config cluster.mysql-master") - elif [[ "$1" == "repositories" ]]; then - host_server=$(echo "$cluster_conf_out" | awk '/git-server = true/ { print prevA } /cluster/ { prevA = $0 }' | head -1 | awk -F '"' '{print $2}') - fi - host=$(echo "$cluster_nodes_output" | grep "$host_server" | awk '{print $2}' | head -1) + case $1 in + elasticsearch | storage | pages) + cluster_host=$(echo "$cluster_conf_out" | awk -v srv="$backup_data-server = true" '/cluster/ { prevA = $0 } $0 ~ srv { print prevA }' | head -1 | awk -F '"' '{print $2}') + ;; + mysql) + cluster_host=$(ghe-ssh "$host" "ghe-config cluster.mysql-master") + ;; + repositories) + cluster_host=$(echo "$cluster_conf_out" | awk '/git-server = true/ { print prevA } /cluster/ { prevA = $0 }' | head -1 | awk -F '"' '{print $2}') + ;; + *) + echo "Unknown backup data: $1" + exit 1 + ;; + esac + host=$(echo "$cluster_nodes_output" | grep "$cluster_host" | awk '{print $2}' | head -1) fi if [ -d "${GHE_DATA_DIR}/current/$1" ]; then From 077a10e3dcbc6a3332c61ec0a811f0b8bec9300c Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Tue, 29 Aug 2023 13:41:03 +0000 Subject: [PATCH 048/410] Add comments --- share/github-backup-utils/ghe-rsync-size | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/github-backup-utils/ghe-rsync-size b/share/github-backup-utils/ghe-rsync-size index 3e454f6bf..e7aeb7d74 100644 --- a/share/github-backup-utils/ghe-rsync-size +++ b/share/github-backup-utils/ghe-rsync-size @@ -59,6 +59,7 @@ transfer_size() ;; esac + # Check if instance is cluster and fetch appropriate primary host for the different components ghe-ssh "$host" "[ -f /etc/github/cluster ]" cluster_status=$? @@ -83,6 +84,7 @@ transfer_size() host=$(echo "$cluster_nodes_output" | grep "$cluster_host" | awk '{print $2}' | head -1) fi + # Get file transfer size estimates if [ -d "${GHE_DATA_DIR}/current/$1" ]; then total_file_size=$(ghe-rsync -arn --stats \ -e "ssh -q $GHE_EXTRA_SSH_OPTS -p 122 -l admin" \ From b2f7717b3c47ff23ba9773e47950427dc6069f36 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 14:22:00 -0700 Subject: [PATCH 049/410] updated ghe-backup testing versions --- bin/ghe-backup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..58791d0f6 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -198,11 +198,11 @@ is_inc=$(is_incremental_backup_feature_on) if [ "$is_inc" = true ]; then if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then - log_error "Can only perform incremental backups on enterprise version 3.10 or higher" + log_error "Can only perform incremental backups on enterprise version 3.11 or higher" exit 1 fi -if [ "$GHE_VERSION_MINOR" -lt 10 ]; then - log_error "Can only perform incremental backups on enterprise version 3.10 or higher" +if [ "$GHE_VERSION_MINOR" -lt 11 ]; then + log_error "Can only perform incremental backups on enterprise version 3.11 or higher" exit 1 fi From a708a1712b3d60e1b3c87c9ffdc35128d7d45c20 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 15:27:16 -0700 Subject: [PATCH 050/410] assert key is restored for 3.10 --- test/test-ghe-restore.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index cf93baa96..1294542c8 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -454,7 +454,7 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test -begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0+" +begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0" ( set -e rm -rf "$GHE_REMOTE_ROOT_DIR" @@ -475,7 +475,7 @@ begin_test "ghe-restore with secret scanning encrypted content encryption keys f ) for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.11.0 + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting these to not be set for versions below 3.11.0 done ) end_test From b334c409db33fa7d962ab7e1ccecfb495e1dbc51 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 15:49:03 -0700 Subject: [PATCH 051/410] updated backup with test for 3-10 --- test/test-ghe-backup.sh | 28 ++++++++++++++++++++++++++-- test/test-ghe-restore.sh | 2 +- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 7149e1158..42b6dccff 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -832,7 +832,7 @@ begin_test "ghe-backup takes backup of secret scanning encrypted secrets encrypt ) end_test -begin_test "ghe-backup takes backup of secret scanning encrypted content encryption keys" +begin_test "ghe-backup does not take backups of secret scanning encrypted content encryption keys on versions below 3.11.0" ( set -e @@ -844,7 +844,31 @@ begin_test "ghe-backup takes backup of secret scanning encrypted content encrypt ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" done - ghe-backup + GHE_REMOTE_VERSION=3.10.0 ghe-backup -v | grep -q "secret scanning encrypted content" && exit 1 + + required_files=( + "secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for file in "${required_files[@]}"; do + [ "$(cat "$GHE_DATA_DIR/current/$file")" = "" ] + done +) +end_test + +begin_test "ghe-backup takes backup of secret scanning encrypted content encryption keys on versions 3.11.0+" +( + set -e + + required_secrets=( + "secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for secret in "${required_secrets[@]}"; do + ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret" "foo" + done + + GHE_REMOTE_VERSION=3.11.0 ghe-backup required_files=( "secret-scanning-user-content-delimited-encryption-root-keys" diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 1294542c8..27bf13310 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -475,7 +475,7 @@ begin_test "ghe-restore with secret scanning encrypted content encryption keys f ) for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # expecting these to not be set for versions below 3.11.0 + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # if this fails, it should fail silently for versions below 3.10 done ) end_test From 7868b3d807492632cee9fd2cb7380beffec22c5a Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 16:16:17 -0700 Subject: [PATCH 052/410] Revert "updated ghe-backup testing versions" This reverts commit b2f7717b3c47ff23ba9773e47950427dc6069f36. --- bin/ghe-backup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 58791d0f6..86ee33dd7 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -198,11 +198,11 @@ is_inc=$(is_incremental_backup_feature_on) if [ "$is_inc" = true ]; then if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then - log_error "Can only perform incremental backups on enterprise version 3.11 or higher" + log_error "Can only perform incremental backups on enterprise version 3.10 or higher" exit 1 fi -if [ "$GHE_VERSION_MINOR" -lt 11 ]; then - log_error "Can only perform incremental backups on enterprise version 3.11 or higher" +if [ "$GHE_VERSION_MINOR" -lt 10 ]; then + log_error "Can only perform incremental backups on enterprise version 3.10 or higher" exit 1 fi From 5edc50c3c63c49d9fbaee4ea82391c95aed1299c Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 18:18:11 -0500 Subject: [PATCH 053/410] Updated restore test Co-authored-by: Robert Bolender --- test/test-ghe-restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 27bf13310..da7fdb06b 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -475,7 +475,7 @@ begin_test "ghe-restore with secret scanning encrypted content encryption keys f ) for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] # if this fails, it should fail silently for versions below 3.10 + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting that this secret was not backed up on versions below 3.11.0, this secret was not present in earlier versions done ) end_test From 7f4301ad748b98d91e25f170456e6ca24dad18b7 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 16:46:46 -0700 Subject: [PATCH 054/410] add restore test against 3.9 --- test/test-ghe-restore.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index da7fdb06b..6734dbb09 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -454,6 +454,32 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test +begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.10.0" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + required_files=( + "secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for file in "${required_files[@]}"; do + echo "foo" >"$GHE_DATA_DIR/current/$file" + done + + GHE_REMOTE_VERSION=3.9.0 ghe-restore -v -f localhost + + required_secrets=( + "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting that this secret was not backed up on versions below 3.11.0, this secret was not present in earlier versions + done +) +end_test + begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0" ( set -e From 45ac587e12e1a211d51e6d00fcc03418cf7539b2 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 16:56:18 -0700 Subject: [PATCH 055/410] adding version check to only restore for 3.11+ --- .../ghe-restore-secret-scanning-encryption-keys | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys index 4785a6d77..5c8dbdef6 100755 --- a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -37,7 +37,9 @@ restore-secret "secret scanning encrypted secrets current shared transit key" "s restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" # Restore secret scanning content scanning keys if present +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then log_info "Restoring secret scanning content scanning keys" restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" +fi bm_end "$(basename $0)" From f08984be849ecac073bb6195f2f83c704dd72448 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Tue, 29 Aug 2023 16:56:31 -0700 Subject: [PATCH 056/410] Revert "add restore test against 3.9" This reverts commit 7f4301ad748b98d91e25f170456e6ca24dad18b7. --- test/test-ghe-restore.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 6734dbb09..da7fdb06b 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -454,32 +454,6 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test -begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.10.0" -( - set -e - rm -rf "$GHE_REMOTE_ROOT_DIR" - setup_remote_metadata - - required_files=( - "secret-scanning-user-content-delimited-encryption-root-keys" - ) - - for file in "${required_files[@]}"; do - echo "foo" >"$GHE_DATA_DIR/current/$file" - done - - GHE_REMOTE_VERSION=3.9.0 ghe-restore -v -f localhost - - required_secrets=( - "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" - ) - - for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting that this secret was not backed up on versions below 3.11.0, this secret was not present in earlier versions - done -) -end_test - begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0" ( set -e From 80e0af3b74a9fc8a89937291c61629460fcc20f2 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 30 Aug 2023 18:17:12 +0000 Subject: [PATCH 057/410] simplify cluster output & parsing --- share/github-backup-utils/ghe-rsync-size | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/share/github-backup-utils/ghe-rsync-size b/share/github-backup-utils/ghe-rsync-size index e7aeb7d74..9ba0cf83f 100644 --- a/share/github-backup-utils/ghe-rsync-size +++ b/share/github-backup-utils/ghe-rsync-size @@ -60,25 +60,20 @@ transfer_size() esac # Check if instance is cluster and fetch appropriate primary host for the different components - ghe-ssh "$host" "[ -f /etc/github/cluster ]" - cluster_status=$? - - if [[ $cluster_status -eq 0 ]]; then - cluster_conf_out=$(ghe-ssh "$host" "cat /data/user/common/cluster.conf") + if "$CLUSTER"; then cluster_nodes_output=$(ghe-ssh "$host" "ghe-cluster-nodes -i") case $1 in - elasticsearch | storage | pages) - cluster_host=$(echo "$cluster_conf_out" | awk -v srv="$backup_data-server = true" '/cluster/ { prevA = $0 } $0 ~ srv { print prevA }' | head -1 | awk -F '"' '{print $2}') + elasticsearch | storage | pages | actions | mssql) + cluster_host=$(ghe-ssh "$host" "ghe-cluster-nodes -r $backup_data" | head -1) ;; mysql) cluster_host=$(ghe-ssh "$host" "ghe-config cluster.mysql-master") ;; repositories) - cluster_host=$(echo "$cluster_conf_out" | awk '/git-server = true/ { print prevA } /cluster/ { prevA = $0 }' | head -1 | awk -F '"' '{print $2}') + cluster_host=$(ghe-ssh "$host" "ghe-cluster-nodes -r git" | head -1) ;; *) - echo "Unknown backup data: $1" - exit 1 + exit 0 ;; esac host=$(echo "$cluster_nodes_output" | grep "$cluster_host" | awk '{print $2}' | head -1) From 2e512526add392486b38f335f2785908c2f53f3d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 30 Aug 2023 20:33:09 +0000 Subject: [PATCH 058/410] fix unsupported ghe-version check --- test/test-ghe-host-check.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 6bfa12a69..aeb41551d 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -56,9 +56,14 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions read -r bu_version_major bu_version_minor _ <<<$(ghe_parse_version $BACKUP_UTILS_VERSION) bu_major_minor="$bu_version_major.$bu_version_minor" releases=$(/usr/bin/curl -s https://github-enterprise.s3.amazonaws.com/release/latest.json) - supported=$(echo $releases | jq -r 'select(."'${bu_major_minor}'")') + latest_value=$(echo "$releases" | jq -r '.latest') + latest_major_version=$(echo $latest_value | cut -d "." -f 1-2) + # Replace "latest" with the derived major version in the releases string + releases_with_replacement=$(echo "$releases" | sed 's/"latest"/"'"$latest_major_version"'"/g') + # Use the modified releases string as needed + supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo $releases | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) if [ -z "$supported" ] then From b9e6a58f47bcc1a7cbd8528743eb866a94386e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Sat, 5 Aug 2023 13:13:19 +0200 Subject: [PATCH 059/410] Record total runtime of ghe-backup and ghe-restore This adds the total runtime of ghe-backup and ghe-restore to the benchmark output. When investigating benchmark performance issues with customers and inspecting the benchmark output, it is helpful to know the total runtime of ghe-backup and ghe-restore as a reference for checking the durations of individual execution steps. While the stdout output currently shows the total runtime, customers generally do not store this output on disk, which makes it difficult to investigate backup-related performance issues after the fact. The runtime is recorded regardless of whether or not the commands completed successfully, as it may still be useful to see how long they have run when terminated early because of an error or human intervention. --- bin/ghe-backup | 3 +++ bin/ghe-restore | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..222502d81 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -121,6 +121,8 @@ cleanup () { # Cleanup SSH multiplexing ghe-ssh --clean + + bm_end "$(basename $0)" } # Setup exit traps @@ -216,6 +218,7 @@ fi echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup" +bm_start "$(basename $0)" START_TIME=$(date +%s) log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP" diff --git a/bin/ghe-restore b/bin/ghe-restore index 19603ae47..f1b1f245e 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -138,6 +138,8 @@ cleanup () { if ! rm -f "${GHE_DATA_DIR}/in-progress-restore"; then log_error "Failed to remove in-progress file" 1>&3 fi + + bm_end "$(basename $0)" } # This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally. @@ -322,6 +324,7 @@ export PROGRESS=0 # Used to track progress of restore echo "$PROGRESS" > /tmp/backup-utils-progress # Log restore start message locally and in /var/log/syslog on remote instance +bm_start "$(basename $0)" START_TIME=$(date +%s) log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT" From 1bb2d0e1342efc6516ff689857ddf35af1570bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Thu, 31 Aug 2023 22:08:11 +0200 Subject: [PATCH 060/410] Log if repositories are restored to multiple nodes --- share/github-backup-utils/ghe-restore-repositories | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-repositories b/share/github-backup-utils/ghe-restore-repositories index 67bbf8e34..d63881be2 100755 --- a/share/github-backup-utils/ghe-restore-repositories +++ b/share/github-backup-utils/ghe-restore-repositories @@ -108,6 +108,8 @@ bm_end "$(basename $0) - Building network list" # each Spokes node including only those repository networks for which there was a route to the # respective Spokes node. if $CLUSTER; then + log_info "* Restoring repository networks to cluster nodes according to Spokes routes" 1>&3 + # The server returns a list of routes: # # a/nw/a8/3f/02/100000855 dgit-node1 dgit-node2 dgit-node3 @@ -150,6 +152,8 @@ if $CLUSTER; then # the Spokes route list captured during the backup. As we already have the list of all repository # network paths, we can simply use that as the rsync file list in noncluster environments. else + log_info "* Restoring all repository networks to target host unconditionally" 1>&3 + cp "$tmp_list" "$tempdir/git-server-primary.rsync" fi From adde32840c6198fc1401162dd885b48a2f2982dd Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 31 Aug 2023 17:28:14 -0400 Subject: [PATCH 061/410] Add incremental MySQL restore test (#530) Add basic tests for incremental MySQL restores --- test/test-ghe-host-check.sh | 5 +- test/test-ghe-incremental-restore.sh | 82 ++++++++++++++++++++++++++++ test/testlib.sh | 34 ++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) create mode 100755 test/test-ghe-incremental-restore.sh diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index aeb41551d..f2c5cca54 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -83,7 +83,10 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions ix=$(( $ix + 1 )) done GHE_TEST_REMOTE_VERSION="${keys[$ix]}.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check + # sometimes when the latest.json is updated during a release this test gets broken. + if [ "${keys[$(( $ix - 1 ))]}" != "latest" ]; then + GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check + fi GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 2 ))]}.0" ghe-host-check fi diff --git a/test/test-ghe-incremental-restore.sh b/test/test-ghe-incremental-restore.sh new file mode 100755 index 000000000..bf3ace16a --- /dev/null +++ b/test/test-ghe-incremental-restore.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# ghe-restore command tests + +# Bring in testlib +# shellcheck source=test/testlib.sh +. "$(dirname "$0")/testlib.sh" + +setup_incremental_restore_data +setup_actions_enabled_settings_for_restore true + +# Make the current symlink +ln -s 1 "$GHE_DATA_DIR/current" +begin_test "ghe_restore -i doesn't run on unsupported versions" +( + set -e + GHE_RESTORE_HOST=127.0.0.1 + export GHE_RESTORE_HOST + + # restore should fail on versions older than 3.10 + ! GHE_TEST_REMOTE_VERSION=3.9.0 ghe-restore -i -v + ! GHE_TEST_REMOTE_VERSION=3.7.0 ghe-restore -i -v + ! GHE_TEST_REMOTE_VERSION=3.1.0 ghe-restore -i -v +) +end_test + +begin_test "ghe-restore -i into configured vm from full backup" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + # set as configured, enable maintenance mode and create required directories + setup_maintenance_mode "configured" + + # set restore host environ var + GHE_RESTORE_HOST=127.0.0.1 + export GHE_RESTORE_HOST + # run ghe-restore and write output to file for asserting against + if ! GHE_TEST_REMOTE_VERSION=3.10.0 GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then +output_debug_logs_and_fail_test + fi + + + # verify connect to right host + grep -q "Connect 127.0.0.1:122 OK" "$TRASHDIR/restore-out" + + # verify stale servers were cleared + grep -q "Cleaning up stale nodes ..." "$TRASHDIR/restore-out" + + # Verify all the data we've restored is as expected + verify_all_restored_data +) +end_test + +begin_test "ghe-restore -i fails when the lsn information for the listed files is out of order" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + # set as configured, enable maintenance mode and create required directories + setup_maintenance_mode "configured" + + # set restore host environ var + GHE_RESTORE_HOST=127.0.0.1 + export GHE_RESTORE_HOST + + inc_1="$GHE_DATA_DIR/2" + inc_2="$GHE_DATA_DIR/3" + + # screw up the order of the LSNs in xtrabackup_checkpoints + setup_incremental_lsn $inc_1 100 200 incremental + setup_incremental_lsn $inc_2 50 50 incremental + # run ghe-restore and write output to file for asserting against + # we expect failure and need the right output. + if GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then + true + fi +) +end_test + + diff --git a/test/testlib.sh b/test/testlib.sh index 8e130eb75..e219a50d5 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -327,6 +327,40 @@ setup_test_data () { setup_minio_test_data "$GHE_DATA_DIR" } +# Sets up test data for testing incremental restores. +setup_incremental_restore_data() { + local full="$GHE_DATA_DIR/1" + local inc_1="$GHE_DATA_DIR/2" + local inc_2="$GHE_DATA_DIR/3" + # Run the setup_test_data function to create three directories: 1 for full backup and two incremental. + # we can use these directories for different types of tests + setup_test_data "$full" + setup_test_data "$inc_1" + setup_test_data "$inc_2" + # Setup the metadata files that track which files are used to track full and incremental files + echo "$full" >> "$GHE_DATA_DIR/inc_full_backup" + echo -e "$inc_1\n$inc_2" >> "$GHE_DATA_DIR/inc_snapshot_data" + # Configure lsn data in xtrabackup_checkpoints for the full backup and the incremental backup + setup_incremental_lsn $full 1 100 full + setup_incremental_lsn $inc_1 101 200 incremental + setup_incremental_lsn $inc_2 201 300 incremental +} + +setup_incremental_lsn() { + local loc=$1 + local start=$2 + local end=$3 + local type=$4 + +cat <> "$loc/xtrabackup_checkpoints" +backup_type = $type +from_lsn = $start +to_lsn = $end +last_lsn = $end +flushed_lsn = $end +LSN +} + setup_incremental_backup_config() { ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p /tmp/lsndir' ghe-ssh "$GHE_HOSTNAME" -- 'echo "fake xtrabackup checkpoint" > /tmp/lsndir/xtrabackup_checkpoints' From e80684dc50ab734a30c1e0715fbf32575d25f72c Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 1 Sep 2023 12:03:57 -0600 Subject: [PATCH 062/410] Run tests in serial --- .github/workflows/main.yml | 10 ---------- script/cibuild | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e4445915..50583737b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,9 +6,6 @@ jobs: build: strategy: matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. os: ['ubuntu-22.04', 'ubuntu-20.04'] fail-fast: false runs-on: ${{ matrix.os }} @@ -20,12 +17,6 @@ jobs: wget "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - name: Get Sources uses: actions/checkout@v3 - name: Test @@ -35,4 +26,3 @@ jobs: shell: bash - name: Build (Linux) run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/script/cibuild b/script/cibuild index 7f57f444d..73dc9ae0a 100755 --- a/script/cibuild +++ b/script/cibuild @@ -5,7 +5,7 @@ set -e # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then +if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then exit 1 fi From 72266bdaa3b7e9adb4b2a36c51ea947096f2e776 Mon Sep 17 00:00:00 2001 From: Brandon Emlaw Date: Fri, 1 Sep 2023 15:45:55 -0500 Subject: [PATCH 063/410] fix spacing --- .../ghe-restore-secret-scanning-encryption-keys | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys index 5c8dbdef6..04f7588a7 100755 --- a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys +++ b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys @@ -38,8 +38,8 @@ restore-secret "secret scanning encrypted secrets delimited shared transit keys" # Restore secret scanning content scanning keys if present if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then -log_info "Restoring secret scanning content scanning keys" -restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + log_info "Restoring secret scanning content scanning keys" + restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" fi bm_end "$(basename $0)" From c19c08811a0183e5eaf906265ea857383bbdc359 Mon Sep 17 00:00:00 2001 From: Shruti Corbett <90784253+shcorbett@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:21:09 -0400 Subject: [PATCH 064/410] Create pull_request_template.md Creating a pull request template to ensure specific information is documented and calling out the versions of GHES that are supported by backup-utils so that appropriate testing can be done against those versions --- .github/pull_request_template.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..fa678e53b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +### Description + +### Testing + + + +### Owernship + + +### Related Links + From e0278f1ac87654dafa32c3beeac14869c0286170 Mon Sep 17 00:00:00 2001 From: Shruti Corbett <90784253+shcorbett@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:27:26 -0400 Subject: [PATCH 065/410] Update pull_request_template.md --- .github/pull_request_template.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fa678e53b..fd36d8a9a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ +# PR Details + ### Description -### Owernship +### Ownership From f9025aa47cc36bd94ec0a67973f5fc5c15e9fc21 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 12:34:29 -0400 Subject: [PATCH 066/410] init to win it --- .github/workflows/build-and-release.yml | 63 +++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/build-and-release.yml diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 000000000..79de3bdcd --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,63 @@ +name: Build and Release + +# make it callable with required input of release-notes string +on: + workflow_dispatch: + inputs: + version: + description: 'Version - patch version of the release (e.g. x.y.z)' + required: true + type: string + release-notes: + description: 'Release Notes - string of markdown' + required: true + type: string + draft: + description: 'Draft - true if the release should be a draft' + required: true + type: boolean + default: true + +# in the first job +# first, run on ubuntu-latest +# then, install dependencies: moreutils, debhelper and help2man +# then, using the input version create a v{input.version} tag at the head of the branch +# then, checkout this repo +# then, run the bash scripts: /scripts/package-deb and /scripts/package-tarball +# in the next job +# create a release using ncipollo/release-action@v1 to attach release notes and the artifacts and tag from the previous job + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y moreutils debhelper help2man + - name: Create tag + run: | + git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" + - name: Checkout + uses: actions/checkout@v2 + - name: Package deb + run: | + bash scripts/package-deb + - name: Package tarball + run: | + bash scripts/package-tarball + release: + needs: build + runs-on: ubuntu-latest + steps: + - name: Create Release + uses: ncipollo/release-action@v1 + with: + repo: github/backup-utils + artifacts: | + ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # token: ${{ secrets.GITHUB_TOKEN }} may need token, but try without first + releaseName: v${{ github.event.inputs.version }} + draft: ${{ github.event.inputs.draft }} + body: ${{ github.event.inputs.release-notes }} From 411d7423f816c712951f2598a9905874d8945d50 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 12:41:51 -0400 Subject: [PATCH 067/410] remove comment notes --- .github/workflows/build-and-release.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 79de3bdcd..46bd9cb57 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -1,6 +1,5 @@ name: Build and Release -# make it callable with required input of release-notes string on: workflow_dispatch: inputs: @@ -18,15 +17,6 @@ on: type: boolean default: true -# in the first job -# first, run on ubuntu-latest -# then, install dependencies: moreutils, debhelper and help2man -# then, using the input version create a v{input.version} tag at the head of the branch -# then, checkout this repo -# then, run the bash scripts: /scripts/package-deb and /scripts/package-tarball -# in the next job -# create a release using ncipollo/release-action@v1 to attach release notes and the artifacts and tag from the previous job - jobs: build: runs-on: ubuntu-latest From dfe71d6627c81602557568ed8f108fa6b1fd482d Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 12:48:00 -0400 Subject: [PATCH 068/410] fix input name --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 46bd9cb57..7dcb3f7c3 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -48,6 +48,6 @@ jobs: ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb # token: ${{ secrets.GITHUB_TOKEN }} may need token, but try without first - releaseName: v${{ github.event.inputs.version }} + name: v${{ github.event.inputs.version }} draft: ${{ github.event.inputs.draft }} body: ${{ github.event.inputs.release-notes }} From 5543165e2c293894f7cd0b15b953a31d6be241c2 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 13:48:08 -0400 Subject: [PATCH 069/410] checkout first --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7dcb3f7c3..e1399ac2e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -21,6 +21,8 @@ jobs: build: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v2 - name: Install dependencies run: | sudo apt-get update @@ -28,8 +30,6 @@ jobs: - name: Create tag run: | git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - - name: Checkout - uses: actions/checkout@v2 - name: Package deb run: | bash scripts/package-deb From f8080c9dec00cd5c1b606636ce792d13986bff77 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Thu, 7 Sep 2023 17:52:23 +0000 Subject: [PATCH 070/410] add NFS check --- bin/ghe-host-check | 7 +++++++ docs/requirements.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 104885114..d0fa767fe 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -168,6 +168,13 @@ SKIP_MSG # shellcheck source=share/github-backup-utils/ghe-rsync-size . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" + #Check if GHE_DATA_DIR is NFS mounted + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") + if [ "$fs_info" == "nfs" ]; then + echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 + echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 + fi + #Display dir requirements for repositories and mysql echo "" 1>&2 echo "Checking host for sufficient space for a backup..." 1>&2 diff --git a/docs/requirements.md b/docs/requirements.md index 1320eb583..e410c7381 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,6 +57,8 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. +Please avoid using NFS mount for the data directory (where backup data is stored) as this can result in performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the From 977db53c7f62f63e86f80bce7c23390965fff9bb Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:12:31 -0400 Subject: [PATCH 071/410] git config set to release-controller --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index e1399ac2e..20bcbc0cb 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -29,6 +29,7 @@ jobs: sudo apt-get install -y moreutils debhelper help2man - name: Create tag run: | + git config --local user.name release-controller" git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - name: Package deb run: | From 982820f88bd45c10e10aff9b1e37251447e147f9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:15:54 -0400 Subject: [PATCH 072/410] missing quote --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 20bcbc0cb..8d9e7f871 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -29,7 +29,7 @@ jobs: sudo apt-get install -y moreutils debhelper help2man - name: Create tag run: | - git config --local user.name release-controller" + git config --local user.name "release-controller" git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - name: Package deb run: | From 2829fcfdf36c712517b039e879fd02a92ea27c58 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:18:57 -0400 Subject: [PATCH 073/410] direct to build scripts --- .github/workflows/build-and-release.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8d9e7f871..28d2e49d2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v2 - name: Install dependencies run: | - sudo apt-get update + sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man - name: Create tag run: | @@ -33,10 +33,12 @@ jobs: git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - name: Package deb run: | - bash scripts/package-deb + ./scripts/package-deb + shell: bash - name: Package tarball run: | - bash scripts/package-tarball + ./scripts/package-tarball + shell: bash release: needs: build runs-on: ubuntu-latest From 1612b2077f43f012b9d6b613fd224d8dc2ab2ff9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:20:44 -0400 Subject: [PATCH 074/410] fix script path --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 28d2e49d2..58e95f34a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -33,11 +33,11 @@ jobs: git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - name: Package deb run: | - ./scripts/package-deb + ./script/package-deb shell: bash - name: Package tarball run: | - ./scripts/package-tarball + ./script/package-tarball shell: bash release: needs: build From a9a4e83d689f82e032757b1daa520851fcf3690e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:24:05 -0400 Subject: [PATCH 075/410] install devscripts --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 58e95f34a..2ac90e9bc 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man + sudo apt-get install -y moreutils debhelper help2man devscripts - name: Create tag run: | git config --local user.name "release-controller" From e791bbb8822e054ea4f5d2f3c06370dc364f8d6d Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 15:28:36 -0400 Subject: [PATCH 076/410] install gzip, tag needs to exist on remote --- .github/workflows/build-and-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 2ac90e9bc..af72b9123 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -26,11 +26,12 @@ jobs: - name: Install dependencies run: | sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts + sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Create tag run: | git config --local user.name "release-controller" git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" + git push origin v${{ github.event.inputs.version }} - name: Package deb run: | ./script/package-deb From 7ff49305808bc1a77e117fb97c6084eaa9f324d1 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 16:14:03 -0400 Subject: [PATCH 077/410] use tag field --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index af72b9123..05b1e2c1d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -52,6 +52,7 @@ jobs: ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb # token: ${{ secrets.GITHUB_TOKEN }} may need token, but try without first + tag: v${{ github.event.inputs.version }} name: v${{ github.event.inputs.version }} draft: ${{ github.event.inputs.draft }} body: ${{ github.event.inputs.release-notes }} From 462e13c4e195d38f3318615476bab2fe89f57f87 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 16:40:35 -0400 Subject: [PATCH 078/410] need token --- .github/workflows/build-and-release.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 05b1e2c1d..5d5ec81c6 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -3,6 +3,10 @@ name: Build and Release on: workflow_dispatch: inputs: + gh-token: + description: 'GitHub Token - used to create the release' + required: true + type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -51,7 +55,7 @@ jobs: artifacts: | ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # token: ${{ secrets.GITHUB_TOKEN }} may need token, but try without first + token: ${{ github.event.inputs.gh-token }} tag: v${{ github.event.inputs.version }} name: v${{ github.event.inputs.version }} draft: ${{ github.event.inputs.draft }} From 78e7303ad7fe07cb4bafca92b109d3df6854eda7 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 17:46:36 -0400 Subject: [PATCH 079/410] try changing repo --- .github/workflows/build-and-release.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 5d5ec81c6..6de83fad7 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -31,11 +31,11 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag - run: | - git config --local user.name "release-controller" - git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - git push origin v${{ github.event.inputs.version }} + # - name: Create tag + # run: | + # git config --local user.name "release-controller" + # git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" + # git push origin v${{ github.event.inputs.version }} - name: Package deb run: | ./script/package-deb @@ -48,14 +48,18 @@ jobs: needs: build runs-on: ubuntu-latest steps: + # - name: Set Up Release + # id: setup_release + # run: echo ::set-output name=token::${{ github.token }} - name: Create Release uses: ncipollo/release-action@v1 with: - repo: github/backup-utils + repo: backup-utils artifacts: | ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb token: ${{ github.event.inputs.gh-token }} + commit: ${{ github.sha }} tag: v${{ github.event.inputs.version }} name: v${{ github.event.inputs.version }} draft: ${{ github.event.inputs.draft }} From eb4437c6c8021fbc6238fe3a53aa4051dca09fa9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 17:49:38 -0400 Subject: [PATCH 080/410] again --- .github/workflows/build-and-release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6de83fad7..c3d54c3ae 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -31,9 +31,9 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag - # run: | - # git config --local user.name "release-controller" + - name: Create tag + run: | + git config --local user.name "release-controller" # git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" # git push origin v${{ github.event.inputs.version }} - name: Package deb @@ -48,9 +48,6 @@ jobs: needs: build runs-on: ubuntu-latest steps: - # - name: Set Up Release - # id: setup_release - # run: echo ::set-output name=token::${{ github.token }} - name: Create Release uses: ncipollo/release-action@v1 with: From 25b67cd97dd8efcde6f45a0e6c354d8585619350 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 7 Sep 2023 17:55:45 -0400 Subject: [PATCH 081/410] need the tag on private? --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c3d54c3ae..b340d9215 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -34,8 +34,8 @@ jobs: - name: Create tag run: | git config --local user.name "release-controller" - # git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - # git push origin v${{ github.event.inputs.version }} + git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" + git push origin v${{ github.event.inputs.version }} - name: Package deb run: | ./script/package-deb From 0e79c57ad8577770308560458f4e4ab88a41b988 Mon Sep 17 00:00:00 2001 From: djdefi Date: Thu, 7 Sep 2023 15:09:58 -0700 Subject: [PATCH 082/410] Filter out rsync prerelease and set change file path --- .github/workflows/rsync-docker-bump.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index c59e4ede1..3e0815f8a 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -15,14 +15,14 @@ jobs: - name: Get latest rsync tag id: latest_tag run: | - curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[0].name' | xargs -I {} echo "::set-output name=latest_tag::{}" + curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "::set-output name=latest_tag::{}" - name: Update Dockerfile with latest tag run: | sed -i -E "s/RSYNC_TAG=[0-9\.]+/RSYNC_TAG=${{ steps.latest_tag.outputs.latest_tag }}/g" Dockerfile - name: Create Pull Request for tag update - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "Update rsync tag in Dockerfile" @@ -30,5 +30,5 @@ jobs: body: "This PR updates the rsync tag in the Dockerfile to the latest tagged version." branch: "update-rsync-tag" base: "master" - path: "." + add-paths: "Dockerfile" labels: "automated-update,rsync" From d5ada9ff9f01a25600748653c7a0d043990f4d8c Mon Sep 17 00:00:00 2001 From: djdefi Date: Thu, 7 Sep 2023 15:26:35 -0700 Subject: [PATCH 083/410] Use environment file for output --- .github/workflows/rsync-docker-bump.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index 3e0815f8a..cfc1b24f9 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -15,7 +15,7 @@ jobs: - name: Get latest rsync tag id: latest_tag run: | - curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "::set-output name=latest_tag::{}" + curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT - name: Update Dockerfile with latest tag run: | From 5fef50b7ac2acb852ae918f7204b84f29e11c49e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 10:45:20 -0400 Subject: [PATCH 084/410] only use tag field --- .github/workflows/build-and-release.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b340d9215..bc2cfef4c 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -54,10 +54,8 @@ jobs: repo: backup-utils artifacts: | ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - token: ${{ github.event.inputs.gh-token }} - commit: ${{ github.sha }} + ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - name: v${{ github.event.inputs.version }} - draft: ${{ github.event.inputs.draft }} body: ${{ github.event.inputs.release-notes }} + draft: ${{ github.event.inputs.draft }} + token: ${{ github.event.inputs.gh-token }} From f7567dea923b6fd1081a555d9d244aec8fa7bf5b Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 10:55:41 -0400 Subject: [PATCH 085/410] set commit to master branch since the tag doesn't exist --- .github/workflows/build-and-release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index bc2cfef4c..babfa13f7 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -54,8 +54,11 @@ jobs: repo: backup-utils artifacts: | ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # if the tag doesn't exist, we need to set it and the commit field tag: v${{ github.event.inputs.version }} + # this can be a commit hash or branch name + commit: master body: ${{ github.event.inputs.release-notes }} draft: ${{ github.event.inputs.draft }} token: ${{ github.event.inputs.gh-token }} From 65919cf618a8ab1bd9a88bae3d2a1651762c7845 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 11:28:29 -0400 Subject: [PATCH 086/410] list artifacts correctly --- .github/workflows/build-and-release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index babfa13f7..277bc2b94 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -52,9 +52,7 @@ jobs: uses: ncipollo/release-action@v1 with: repo: backup-utils - artifacts: | - ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + artifacts: "./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" # if the tag doesn't exist, we need to set it and the commit field tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name From 757ed89931a88914bb590c924f54145b29964c29 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 11:55:40 -0400 Subject: [PATCH 087/410] allowUpdate true --- .github/workflows/build-and-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 277bc2b94..31a0f6d34 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -59,4 +59,6 @@ jobs: commit: master body: ${{ github.event.inputs.release-notes }} draft: ${{ github.event.inputs.draft }} - token: ${{ github.event.inputs.gh-token }} + token: ${{ github.event.inputs.gh-token }} + allowUpdates: true + artifactContentType: "raw" From c191e203e22583182d8a1c6779541003f99a7581 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 12:45:55 -0400 Subject: [PATCH 088/410] pass artifacts to release job --- .github/workflows/build-and-release.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 31a0f6d34..132a4ade2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -40,19 +40,37 @@ jobs: run: | ./script/package-deb shell: bash + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | ./script/package-tarball shell: bash + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest steps: + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: repo: backup-utils - artifacts: "./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb" # if the tag doesn't exist, we need to set it and the commit field tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name From 26c2a1d8accfd74f03054505200b1ba5af3d63d2 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 13:37:16 -0400 Subject: [PATCH 089/410] fix the path --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 132a4ade2..65f519cc6 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,7 +44,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | ./script/package-tarball @@ -53,7 +53,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest From 8767787be35779e359cf0814a866a170cce0ea54 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 14:42:08 -0400 Subject: [PATCH 090/410] where is the file --- .github/workflows/build-and-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 65f519cc6..6c0a9572e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -38,22 +38,22 @@ jobs: git push origin v${{ github.event.inputs.version }} - name: Package deb run: | - ./script/package-deb + ./script/package-deb && ls -l && ls -l /dist shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 with: name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: ./dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: /dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | - ./script/package-tarball + ./script/package-tarball ls -l && ls -l /dist shell: bash - name: Upload tarball artifact uses: actions/upload-artifact@v3 with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: ./dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: /dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest From 09bac26d2f0a4e953c1b1a2d90bffb9b21066895 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 14:47:53 -0400 Subject: [PATCH 091/410] find the artifacts --- .github/workflows/build-and-release.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6c0a9572e..02c66f884 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -38,7 +38,11 @@ jobs: git push origin v${{ github.event.inputs.version }} - name: Package deb run: | - ./script/package-deb && ls -l && ls -l /dist + ./script/package-deb + shell: bash + - name: Find deb artifact + run: | + find . -name "github-backup-utils_${{ github.event.inputs.version }}_all.deb" | xargs -I {} cp {} /dist shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 @@ -47,7 +51,11 @@ jobs: path: /dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | - ./script/package-tarball ls -l && ls -l /dist + ./script/package-tarball + shell: bash + - name: Find tarball artifact + run: | + find . -name "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz" | xargs -I {} cp {} /dist shell: bash - name: Upload tarball artifact uses: actions/upload-artifact@v3 From 829f3e1223e84b23c2b4d61f33ddf85e39e6b90d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Fri, 8 Sep 2023 15:20:28 -0400 Subject: [PATCH 092/410] Update docs/requirements.md Co-authored-by: David Jarzebowski --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index e410c7381..bba51c252 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -57,7 +57,7 @@ Using a [case sensitive][7] file system is also required to avoid conflicts. Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. -Please avoid using NFS mount for the data directory (where backup data is stored) as this can result in performance issues and timeouts during backups. +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. ## GitHub Enterprise Server version requirements From 920bdd3532a42960e00b66ae881a7b814bf2df0c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 15:31:36 -0400 Subject: [PATCH 093/410] find it --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 02c66f884..ddebebcdb 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -42,7 +42,7 @@ jobs: shell: bash - name: Find deb artifact run: | - find . -name "github-backup-utils_${{ github.event.inputs.version }}_all.deb" | xargs -I {} cp {} /dist + ls -l && find . -name "github-backup-utils_${{ github.event.inputs.version }}_all.deb" shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 @@ -55,7 +55,7 @@ jobs: shell: bash - name: Find tarball artifact run: | - find . -name "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz" | xargs -I {} cp {} /dist + ls -l && find . -name "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz" shell: bash - name: Upload tarball artifact uses: actions/upload-artifact@v3 From 9d316088e15529d2fe91dfd2d1e1a9b30decc67d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali <98570028+chuckp22@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:07:30 -0400 Subject: [PATCH 094/410] Update bin/ghe-host-check Co-authored-by: Quinn Murphy --- bin/ghe-host-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index d0fa767fe..9e8d6466b 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -169,7 +169,7 @@ SKIP_MSG . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" #Check if GHE_DATA_DIR is NFS mounted - fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") + fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true if [ "$fs_info" == "nfs" ]; then echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 From 2b605d9e62c1798c51607d936d2000282f285735 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 16:44:16 -0400 Subject: [PATCH 095/410] find it --- .github/workflows/build-and-release.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index ddebebcdb..64733a08f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -42,21 +42,17 @@ jobs: shell: bash - name: Find deb artifact run: | - ls -l && find . -name "github-backup-utils_${{ github.event.inputs.version }}_all.deb" + ls -l && find . -name "github-backup-utils_*_all.deb" && ls -R shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: /dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + name: github-backup-utils_3.8.0_all.deb + path: github-backup-utils_3.8.0_all.deb - name: Package tarball run: | ./script/package-tarball shell: bash - - name: Find tarball artifact - run: | - ls -l && find . -name "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz" - shell: bash - name: Upload tarball artifact uses: actions/upload-artifact@v3 with: @@ -69,16 +65,19 @@ jobs: - name: Download deb artifact uses: actions/download-artifact@v3 with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + name: github-backup-utils_3.8.0_all.deb + - name: Display structure of downloaded files + run: | + ls -R - name: Download tarball artifact uses: actions/download-artifact@v3 with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: repo: backup-utils - artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb" + artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_3.8.0_all.deb" # if the tag doesn't exist, we need to set it and the commit field tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name From 81a1cdb50107b5ff9c8ac146874d7ad819ca7d07 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 8 Sep 2023 17:22:33 -0400 Subject: [PATCH 096/410] now we know --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 64733a08f..2a7ff79f3 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -48,7 +48,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: github-backup-utils_3.8.0_all.deb - path: github-backup-utils_3.8.0_all.deb + path: dist/github-backup-utils_3.8.0_all.deb - name: Package tarball run: | ./script/package-tarball @@ -57,7 +57,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: /dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest From 1d924a12d3c50ff9e76a0d6560bf9d81d1ae90f2 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Mon, 11 Sep 2023 10:08:43 +0200 Subject: [PATCH 097/410] add post backup cleanup and move progress to folder (#558) --- bin/ghe-backup | 8 ++++---- bin/ghe-backup-progress | 8 ++++---- bin/ghe-restore | 6 +++--- share/github-backup-utils/ghe-backup-config | 13 +++++++++++-- share/github-backup-utils/track-progress | 11 +++++------ test/test-ghe-backup.sh | 2 ++ test/testlib.sh | 6 ++++++ 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 86ee33dd7..d8074156e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -163,11 +163,11 @@ echo "$GHE_REMOTE_VERSION" > version # Setup progress tracking init-progress export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress OPTIONAL_STEPS=0 # Backup actions+mssql @@ -191,7 +191,7 @@ if [ "$GHE_BACKUP_PAGES" != "no" ]; then fi PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total # check that incremental settings are valid if set is_inc=$(is_incremental_backup_feature_on) diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress index 2f4b267fb..7ab36e084 100755 --- a/bin/ghe-backup-progress +++ b/bin/ghe-backup-progress @@ -31,7 +31,7 @@ while true; do done check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress-info ]; then + if [ ! -f /tmp/backup-utils-progress/info ]; then echo "No progress file found. Has a backup or restore been started?" exit 1 fi @@ -39,18 +39,18 @@ check_for_progress_file() { if [ -n "$ONCE" ]; then check_for_progress_file - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info else check_for_progress_file clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info while true; do if read -r -t 1 -n 1; then clear exit ; else clear - cat /tmp/backup-utils-progress-info + cat /tmp/backup-utils-progress/info fi done fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 19603ae47..175d74135 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -315,11 +315,11 @@ fi export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total +echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type +echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress +echo "$PROGRESS" > /tmp/backup-utils-progress/progress # Log restore start message locally and in /var/log/syslog on remote instance START_TIME=$(date +%s) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 5b7f1c3d3..d3f8b05aa 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -652,12 +652,21 @@ restore-secret() { #initialize progress tracking by clearing out the temp files used to track init-progress() { - rm -f /tmp/backup-utils-progress* + if [ -d /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + else + mkdir /tmp/backup-utils-progress + fi + touch /tmp/backup-utils-progress/total + touch /tmp/backup-utils-progress/type + touch /tmp/backup-utils-progress/progress + touch /tmp/backup-utils-progress/info + chmod -R 777 /tmp/backup-utils-progress } #increase total count of progress increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total + echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index d88b3704f..a560ba540 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -1,13 +1,12 @@ #!/usr/bin/env bash #/ track-progress: track progress of backup or restore tasks -# Current version is working solely with backups progress(){ - PROGRESS=$(cat /tmp/backup-utils-progress) - PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress-total) - PROGRESS_TYPE=$(cat /tmp/backup-utils-progress-type) + PROGRESS=$(cat /tmp/backup-utils-progress/progress) + PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) + PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) - echo $((PROGRESS + 1)) > /tmp/backup-utils-progress - echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress-info + echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress + echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info } diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index 42b6dccff..d6391bbef 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -47,6 +47,8 @@ begin_test "ghe-backup subsequent snapshot" [ "$first_snapshot" != "$this_snapshot" ] verify_all_backedup_data + + verify_progress_cleanup_process ) end_test diff --git a/test/testlib.sh b/test/testlib.sh index e219a50d5..10d6f7821 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -528,6 +528,12 @@ verify_all_backedup_data() { verify_common_data } +# A unified method to make sure post backup, the cleanup process works +verify_progress_cleanup_process() { + set -e + sudo -u nobody rm -rf /tmp/backup-utils-progress/* +} + # A unified method to check everything restored when performing a full restore # during testing. verify_all_restored_data() { From 16c1968f13762cf3f75aca8587d1af421298cbb0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 09:36:56 -0400 Subject: [PATCH 098/410] rename file for now, clean up debug, get release-notes from directory, fix title --- .github/workflows/build-and-release.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 2a7ff79f3..8b521e539 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -40,15 +40,15 @@ jobs: run: | ./script/package-deb shell: bash - - name: Find deb artifact + - name: Rename deb artifact run: | - ls -l && find . -name "github-backup-utils_*_all.deb" && ls -R + mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 with: - name: github-backup-utils_3.8.0_all.deb - path: dist/github-backup-utils_3.8.0_all.deb + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | ./script/package-tarball @@ -66,9 +66,6 @@ jobs: uses: actions/download-artifact@v3 with: name: github-backup-utils_3.8.0_all.deb - - name: Display structure of downloaded files - run: | - ls -R - name: Download tarball artifact uses: actions/download-artifact@v3 with: @@ -77,12 +74,13 @@ jobs: uses: ncipollo/release-action@v1 with: repo: backup-utils + name: "GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}" artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_3.8.0_all.deb" - # if the tag doesn't exist, we need to set it and the commit field + # this action will create a tag with this name on the provided commit tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name commit: master - body: ${{ github.event.inputs.release-notes }} + body: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} token: ${{ github.event.inputs.gh-token }} allowUpdates: true From fb24a3f8b24884275ea9d0225cbd3d1f339a84e8 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 10:02:25 -0400 Subject: [PATCH 099/410] fix download file name --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8b521e539..e9229df9b 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -65,7 +65,7 @@ jobs: - name: Download deb artifact uses: actions/download-artifact@v3 with: - name: github-backup-utils_3.8.0_all.deb + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Download tarball artifact uses: actions/download-artifact@v3 with: From 96a69805eafc967e61a0e5d9807d0023f0ebedcf Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 10:05:48 -0400 Subject: [PATCH 100/410] try adding release-controller email --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index e9229df9b..ba61c90f4 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -34,6 +34,7 @@ jobs: - name: Create tag run: | git config --local user.name "release-controller" + git config --local user.email "release-controller@github.com" git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" git push origin v${{ github.event.inputs.version }} - name: Package deb From 978b63978f7f514cea77ac90a6c1af288fcceb10 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 10:06:47 -0400 Subject: [PATCH 101/410] remove --local --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index ba61c90f4..c7c75aa91 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -33,8 +33,8 @@ jobs: sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Create tag run: | - git config --local user.name "release-controller" - git config --local user.email "release-controller@github.com" + git config user.name "release-controller" + git config user.email "release-controller@github.com" git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" git push origin v${{ github.event.inputs.version }} - name: Package deb From f52e4e29de8225d178912d5bb6e0012d38f003c8 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 10:40:45 -0400 Subject: [PATCH 102/410] rename artifact correctly --- .github/workflows/build-and-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c7c75aa91..cbaeb6725 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -41,6 +41,7 @@ jobs: run: | ./script/package-deb shell: bash + # many need to remove this once release-notes compilation is automated - name: Rename deb artifact run: | mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb @@ -76,7 +77,7 @@ jobs: with: repo: backup-utils name: "GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}" - artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_3.8.0_all.deb" + artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb" # this action will create a tag with this name on the provided commit tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name From b23a3f7f755b77fd9798ed97d4a10cf4c20fa48c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 11:17:55 -0400 Subject: [PATCH 103/410] with release notes --- release-notes/12.12.12.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md new file mode 100644 index 000000000..0e30c9359 --- /dev/null +++ b/release-notes/12.12.12.md @@ -0,0 +1,9 @@ +### Features + +- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." +- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". + +### Changes + +- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). +- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 59e28bef780c2f1649654418c7d85028190ce037 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 11:51:40 -0400 Subject: [PATCH 104/410] name field correctly --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index cbaeb6725..9d0217f06 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -82,7 +82,7 @@ jobs: tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name commit: master - body: release-notes/${{ github.event.inputs.version }}.md + bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} token: ${{ github.event.inputs.gh-token }} allowUpdates: true From 41dc43212c7855dbe5d8e2934199e077565805a0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 12:15:11 -0400 Subject: [PATCH 105/410] linter fixes --- .github/workflows/build-and-release.yml | 84 ++++++++++++++----------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 9d0217f06..6fcb9b3c0 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -25,41 +25,47 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag - run: | - git config user.name "release-controller" - git config user.email "release-controller@github.com" - git tag -a v${{ github.event.inputs.version }} -m "v${{ github.event.inputs.version }}" - git push origin v${{ github.event.inputs.version }} - - name: Package deb - run: | - ./script/package-deb - shell: bash - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - shell: bash - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - shell: bash - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Checkout + uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag + run: | + git config user.name "release-controller" + git config user.email "release-controller@github.com" + git tag -a v${{ github.event.inputs.version }} \ + -m "v${{ github.event.inputs.version }}" + git push origin v${{ github.event.inputs.version }} + - name: Package deb + run: | + ./script/package-deb + shell: bash + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) \ + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + shell: bash + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_\ + all.deb + - name: Package tarball + run: | + ./script/package-tarball + shell: bash + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}\ + .tar.gz release: needs: build runs-on: ubuntu-latest @@ -76,8 +82,12 @@ jobs: uses: ncipollo/release-action@v1 with: repo: backup-utils - name: "GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }}" - artifacts: "github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb" + name: | + GitHub Enterprise Server Backup Utilities \ + v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb # this action will create a tag with this name on the provided commit tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name From be491ba4fc0983e529f21e3547631bef4f3ef863 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 13:00:05 -0400 Subject: [PATCH 106/410] find the release-notes file --- .github/workflows/build-and-release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6fcb9b3c0..47fa1bd1d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -78,6 +78,9 @@ jobs: uses: actions/download-artifact@v3 with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: See all files + run: | + ls -R - name: Create Release uses: ncipollo/release-action@v1 with: From 10f9679d2e8326dbfa6107722e688f06eda87bc9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 13:20:22 -0400 Subject: [PATCH 107/410] have to checkout the branch --- .github/workflows/build-and-release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 47fa1bd1d..00e3f5556 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -70,6 +70,8 @@ jobs: needs: build runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v2 - name: Download deb artifact uses: actions/download-artifact@v3 with: @@ -78,9 +80,6 @@ jobs: uses: actions/download-artifact@v3 with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: See all files - run: | - ls -R - name: Create Release uses: ncipollo/release-action@v1 with: From 91dd4f2875bcd6ec299512ed39a8f422e077165a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 13:30:29 -0400 Subject: [PATCH 108/410] fix file paths --- .github/workflows/build-and-release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 00e3f5556..6a8e0aeea 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -53,8 +53,7 @@ jobs: with: name: github-backup-utils_${{ github.event.inputs.version }}_all.deb path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_\ - all.deb + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - name: Package tarball run: | ./script/package-tarball @@ -64,8 +63,7 @@ jobs: with: name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}\ - .tar.gz + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest From 1326890bfb0d6c52b4008e15330780712a383d1d Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 14:05:04 -0400 Subject: [PATCH 109/410] fix trailing white spaces --- .github/workflows/build-and-release.yml | 82 ++++++++++++------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6a8e0aeea..379190505 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -25,45 +25,45 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag - run: | - git config user.name "release-controller" - git config user.email "release-controller@github.com" - git tag -a v${{ github.event.inputs.version }} \ - -m "v${{ github.event.inputs.version }}" - git push origin v${{ github.event.inputs.version }} - - name: Package deb - run: | - ./script/package-deb - shell: bash - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) \ - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - shell: bash - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - shell: bash - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Checkout + uses: actions/checkout@v2 + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag + run: | + git config user.name "release-controller" + git config user.email "release-controller@github.com" + git tag -a v${{ github.event.inputs.version }} \ + -m "v${{ github.event.inputs.version }}" + git push origin v${{ github.event.inputs.version }} + - name: Package deb + run: | + ./script/package-deb + shell: bash + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) \ + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + shell: bash + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + shell: bash + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: needs: build runs-on: ubuntu-latest @@ -82,10 +82,10 @@ jobs: uses: ncipollo/release-action@v1 with: repo: backup-utils - name: | + name: | GitHub Enterprise Server Backup Utilities \ v${{ github.event.inputs.version }} - artifacts: | + artifacts: | github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ github-backup-utils_${{ github.event.inputs.version }}_all.deb # this action will create a tag with this name on the provided commit From 147a791aa0c2cf34f5b407f5bee51ef3d7e07f0a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 14:06:14 -0400 Subject: [PATCH 110/410] remove slash in title --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 379190505..a30c8d666 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -83,7 +83,7 @@ jobs: with: repo: backup-utils name: | - GitHub Enterprise Server Backup Utilities \ + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ From b2931402b9f95ab067df66a1473e3939eb8af384 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 14:10:02 -0400 Subject: [PATCH 111/410] trailing white space --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a30c8d666..c67e6f9ce 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -83,7 +83,7 @@ jobs: with: repo: backup-utils name: | - GitHub Enterprise Server Backup Utilities + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ From a765d48d59fb8bd890637d2b1bb256ff01b0cfdc Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 15:18:54 -0400 Subject: [PATCH 112/410] remove test release-notes --- release-notes/12.12.12.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md deleted file mode 100644 index 0e30c9359..000000000 --- a/release-notes/12.12.12.md +++ /dev/null @@ -1,9 +0,0 @@ -### Features - -- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." -- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". - -### Changes - -- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). -- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 1220249348b4c64e5cba16514096c7b29e441dd9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 15:57:07 -0400 Subject: [PATCH 113/410] trying to fix linter errors --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c67e6f9ce..3c41a9aff 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -45,8 +45,8 @@ jobs: # many need to remove this once release-notes compilation is automated - name: Rename deb artifact run: | - mv dist/$(ls dist | grep github-backup-utils_.*_all.deb) \ - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + "mv dist/$(ls dist | grep 'github-backup-utils_.*_all.deb') \ + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 From 06f864b422e652783085099d303da87c79b8a46c Mon Sep 17 00:00:00 2001 From: djdefi Date: Mon, 11 Sep 2023 13:08:41 -0700 Subject: [PATCH 114/410] Workflow dispatch / shellcheck disable invalid check --- .github/workflows/rsync-docker-bump.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index cfc1b24f9..433fa3f96 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -1,6 +1,7 @@ name: Update Rsync Tag in Dockerfile on: + workflow_dispatch: schedule: - cron: '0 0 * * *' # Runs daily at 00:00 @@ -14,6 +15,7 @@ jobs: - name: Get latest rsync tag id: latest_tag + # shellcheck disable=SC2086 run: | curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT From fc5c0854e00fc6029f4c9864af7b565a5deb6089 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 16:08:43 -0400 Subject: [PATCH 115/410] fingers crossed --- .github/workflows/build-and-release.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 3c41a9aff..011bdf4e9 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -45,8 +45,12 @@ jobs: # many need to remove this once release-notes compilation is automated - name: Rename deb artifact run: | - "mv dist/$(ls dist | grep 'github-backup-utils_.*_all.deb') \ - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 From f4e6b2a0d2568dfb72f1b831b429e1bcd6db0895 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 16:09:54 -0400 Subject: [PATCH 116/410] test release-notes --- release-notes/12.12.12.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md new file mode 100644 index 000000000..8735a0322 --- /dev/null +++ b/release-notes/12.12.12.md @@ -0,0 +1,11 @@ +# Changes + +### Features + +- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." +- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". + +### Changes + +- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). +- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 77004063857cbfedd380b8fffadf671d8741f485 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 16:35:04 -0400 Subject: [PATCH 117/410] bad command --- .github/workflows/build-and-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 011bdf4e9..b05fe8ca1 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -45,7 +45,6 @@ jobs: # many need to remove this once release-notes compilation is automated - name: Rename deb artifact run: | - run: | for file in dist/github-backup-utils_*_all.deb; do if [[ -f "$file" ]]; then mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" From 09937de499abd68f640210be6a969729212ef48e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 11 Sep 2023 17:00:00 -0400 Subject: [PATCH 118/410] remove test md file --- release-notes/12.12.12.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md deleted file mode 100644 index 8735a0322..000000000 --- a/release-notes/12.12.12.md +++ /dev/null @@ -1,11 +0,0 @@ -# Changes - -### Features - -- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." -- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". - -### Changes - -- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). -- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 0eb507f55ca264e3e893822ca3d09c7ccc191bc8 Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 12 Sep 2023 13:14:47 -0700 Subject: [PATCH 119/410] single line run --- .github/workflows/rsync-docker-bump.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index 433fa3f96..cedcd2468 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -16,8 +16,7 @@ jobs: - name: Get latest rsync tag id: latest_tag # shellcheck disable=SC2086 - run: | - curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT + run: curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT - name: Update Dockerfile with latest tag run: | From 99837fe47ed57976c9da64c29e92a2972ab7895c Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 12 Sep 2023 13:15:39 -0700 Subject: [PATCH 120/410] remove sc skip comment --- .github/workflows/rsync-docker-bump.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index cedcd2468..6d90b7bbe 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -15,7 +15,6 @@ jobs: - name: Get latest rsync tag id: latest_tag - # shellcheck disable=SC2086 run: curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT - name: Update Dockerfile with latest tag From 12d65161975eae7a8b2a7e912e62dbb67fe97f82 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Tue, 12 Sep 2023 15:39:57 -0600 Subject: [PATCH 121/410] back off gist restore in parallel restore case to avoid race condition --- bin/ghe-restore | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 19603ae47..b48c06e5f 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -560,9 +560,18 @@ echo \"$cmd_title\" ghe-restore-repositories \"$GHE_HOSTNAME\"") cmd_title=$(log_info "Restoring Gists ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") +if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then + ## Stagger ghe-restore-repositories-gist with ghe-restore-repositories to avoid + ## race condition to operate on same destination directory + commands+=(" + echo \"$cmd_title\" + sleep 1 + ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") +else + commands+=(" + echo \"$cmd_title\" + ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") +fi if [ "$GHE_BACKUP_PAGES" != "no" ]; then cmd_title=$(log_info "Restoring Pages ...") From 648967cd931a791edaa8c6cdd1d5fdbbbdf080aa Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:22:48 -0600 Subject: [PATCH 122/410] Update ghe-restore --- bin/ghe-restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index b48c06e5f..77a8b089e 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -565,7 +565,7 @@ if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then ## race condition to operate on same destination directory commands+=(" echo \"$cmd_title\" - sleep 1 + sleep 2 ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") else commands+=(" From d60bb64993080576537850b6197decc0ed439c68 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Wed, 13 Sep 2023 11:24:31 -0600 Subject: [PATCH 123/410] Make this change test only --- bin/ghe-restore | 15 +++------------ test/testlib.sh | 2 ++ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index d275e3a16..17a12b8b1 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -563,18 +563,9 @@ echo \"$cmd_title\" ghe-restore-repositories \"$GHE_HOSTNAME\"") cmd_title=$(log_info "Restoring Gists ...") -if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then - ## Stagger ghe-restore-repositories-gist with ghe-restore-repositories to avoid - ## race condition to operate on same destination directory - commands+=(" - echo \"$cmd_title\" - sleep 2 - ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") -else - commands+=(" - echo \"$cmd_title\" - ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") -fi +commands+=(" +echo \"$cmd_title\" +ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") if [ "$GHE_BACKUP_PAGES" != "no" ]; then cmd_title=$(log_info "Restoring Pages ...") diff --git a/test/testlib.sh b/test/testlib.sh index 10d6f7821..1d38a8b8e 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -87,6 +87,8 @@ setup_remote_metadata () { mkdir -p "$GHE_REMOTE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR" mkdir -p "$GHE_REMOTE_DATA_USER_DIR/common" mkdir -p "$GHE_REMOTE_ROOT_DIR/etc/github" + # Create fake remote repositories dir + mkdir -p "$GHE_REMOTE_DATA_USER_DIR/repositories" } setup_remote_metadata From 8baccb29106d266984efa4a00f20a5e8ce6aeb82 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Wed, 13 Sep 2023 11:30:08 -0600 Subject: [PATCH 124/410] Add integration tests --- .github/workflows/integration-tests.yml | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/integration-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 000000000..1a30dda3b --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,33 @@ +name: Run Integration Tests + +on: [pull_request, workflow_dispatch] + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-binary-backup + - enterprise2-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + run: | + ref="${{ github.ref }}" + merge_branch=${ref#"refs/heads/"} + backup_utils_branch="${{ env.SOURCE_BRANCH }}" + #branch_name="${{ env.TARGET_BRANCH }}" + branch_name="janky-backup-utils-branch-test" + curl -v -X POST \ + -u "hubot:${{ secrets.API_AUTH_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d '{"buildable_name":"${{ matrix.jankyJobName }}","repo":"enterprise2","branch_name": "'"$branch_name"'","env_vars":{"JANKY_ENV_BACKUP_UTILS_BRANCH": "'"$backup_utils_branch"'" },"force":"true","room_id":"#builds"}' \ + "https://janky.githubapp.com/api/builds" From 841d025adfb713bd57069057eea3cb2b7efeab82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 19:22:17 +0200 Subject: [PATCH 125/410] Avoid duplicating test output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A long time ago, we used to track stdout and stderr separately. When both were combined, the postprocessing instructions of stdout and stderr were incorrectly merged, resulting in the output file being provided on stdin twice [1]. As a process cannot take two input streams, one of them was closed, causing no apparent issue. However, this triggered a shellcheck issue that was later addressed improperly [2], which instructed grep to read the output file from both stdin (by providing the - shorthand for /dev/stdin) and the output file location (by providing it as another file name argument). For this reason, the test output would contain the entire test output twice, once prefixed with “(standard input)” and once with the path to the output file. This fixes the issue by simply removing one of the occurrences of the output file, giving us a single copy of the test output in the case of a failure without unwanted prefixes in front of every output line. [1] 2915ec9aa3a11d4936641bdc9f9f43111817047b [2] 2bc31a99174a9f6ab3de361d3fca18f4bcf2c254 --- test/testlib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testlib.sh b/test/testlib.sh index 10d6f7821..cc7ac4c4b 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -153,7 +153,7 @@ report_failure () { printf "test: %-73s $msg\\n" "$desc ..." ( sed 's/^/ /' <"$TRASHDIR/out" | - grep -a -v -e '^\+ end_test' -e '^+ set +x' - "$TRASHDIR/out" | + grep -a -v -e '^\+ end_test' -e '^+ set +x' | sed 's/[+] test_status=/test failed. last command exited with /' | sed 's/^/ /' ) 1>&2 From e029f6836dbc1fd42e07a4642723e0c643c79da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:13:25 +0200 Subject: [PATCH 126/410] Refactor reporting of failures To make the code more symmetric between passed, skipped, and failed tests, this moves the line printing the result for failed tests out of report_failure, which is now only concerned with reporting the output of the failed test. --- test/testlib.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/testlib.sh b/test/testlib.sh index cc7ac4c4b..4f169ce32 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -146,11 +146,7 @@ begin_test () { before_time=$(date '+%s') } -report_failure () { - msg=$1 - desc=$2 - failures=$(( failures + 1 )) - printf "test: %-73s $msg\\n" "$desc ..." +report_failure_output () { ( sed 's/^/ /' <"$TRASHDIR/out" | grep -a -v -e '^\+ end_test' -e '^+ set +x' | @@ -172,7 +168,9 @@ end_test () { elif [ "$test_status" -eq 254 ]; then printf "test: %-65s SKIPPED\\n" "$test_description ..." else - report_failure "FAILED (${elapsed_time}s)" "$test_description ..." + failures=$(( failures + 1 )) + printf "test: %-65s FAILED (${elapsed_time}s)\\n" "$test_description ..." + report_failure_output fi unset test_description From 400e48e24bd15c65df6efca5afe9400da6c6e33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:18:46 +0200 Subject: [PATCH 127/410] Report test runtime in milliseconds --- test/testlib.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/testlib.sh b/test/testlib.sh index 4f169ce32..c7a3cd78d 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -143,7 +143,7 @@ begin_test () { # allow the subshell to exit non-zero without exiting this process set -x +e - before_time=$(date '+%s') + before_time=$(date '+%s.%3N') } report_failure_output () { @@ -158,18 +158,18 @@ report_failure_output () { # Mark the end of a test. end_test () { test_status="${1:-$?}" - after_time=$(date '+%s') - elapsed_time=$((after_time - before_time)) + after_time=$(date '+%s.%3N') + elapsed_time=$(echo "scale=3; $after_time - $before_time" | bc) set +x -e exec 1>&3 2>&4 if [ "$test_status" -eq 0 ]; then - printf "test: %-65s OK (${elapsed_time}s)\\n" "$test_description ..." + printf "test: %-65s OK (%.3fs)\\n" "$test_description ..." "$elapsed_time" elif [ "$test_status" -eq 254 ]; then printf "test: %-65s SKIPPED\\n" "$test_description ..." else failures=$(( failures + 1 )) - printf "test: %-65s FAILED (${elapsed_time}s)\\n" "$test_description ..." + printf "test: %-65s FAILED (%.3fs)\\n" "$test_description ..." "$elapsed_time" report_failure_output fi From cd3cbbb683bdd4523d003a77d72176a6f04ea73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:25:54 +0200 Subject: [PATCH 128/410] Truncate test output to exclude test instructions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds markers right before starting a test case and right after it completed, which are then used to truncate the output of the test case so that it doesn’t include any instructions related to the testing framework, as these aren’t relevant to users inspecting the test ouptut. --- test/testlib.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/test/testlib.sh b/test/testlib.sh index c7a3cd78d..0696984fc 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -144,20 +144,28 @@ begin_test () { # allow the subshell to exit non-zero without exiting this process set -x +e before_time=$(date '+%s.%3N') + + # Marker to truncate the actual test output later + echo "begin_test_truncate_marker" > /dev/null } report_failure_output () { - ( - sed 's/^/ /' <"$TRASHDIR/out" | - grep -a -v -e '^\+ end_test' -e '^+ set +x' | - sed 's/[+] test_status=/test failed. last command exited with /' | - sed 's/^/ /' - ) 1>&2 + # Truncate the test output to exclude testing-related instructions + echo "$(<"$TRASHDIR/out")" \ + | sed '0,/begin_test_truncate_marker/d' \ + | sed -n '/end_test_truncate_marker/q;p' | head -n -2 \ + | sed 's/^/ /' \ + 1>&2 + echo -e "\nTest failed. The last command exited with exit code $test_status." 1>&2 } # Mark the end of a test. end_test () { test_status="${1:-$?}" + + # Marker to truncate the actual test output later + echo "end_test_truncate_marker" > /dev/null + after_time=$(date '+%s.%3N') elapsed_time=$(echo "scale=3; $after_time - $before_time" | bc) set +x -e From b8d781f1951a8a2999f6b8e6f448a40f5471a135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:30:10 +0200 Subject: [PATCH 129/410] Make test output collapsible To make the test logs more pleasant to interact with, this makes the test output of failed test cases collapsible. This avoids cluttering the test overview with the typically very long output of the individual test cases, such that users can get a full overview of all tests by default. If needed, the output of a specific test can easily be expanded in the GitHub Actions user interface. As the GitHub Actions user interface already indents the content of groups, we no longer need to indent it with sed. --- test/testlib.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/testlib.sh b/test/testlib.sh index 0696984fc..57a8e8e2d 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -150,13 +150,14 @@ begin_test () { } report_failure_output () { + echo "::group::Output of failed test" 1>&2 # Truncate the test output to exclude testing-related instructions echo "$(<"$TRASHDIR/out")" \ | sed '0,/begin_test_truncate_marker/d' \ | sed -n '/end_test_truncate_marker/q;p' | head -n -2 \ - | sed 's/^/ /' \ 1>&2 echo -e "\nTest failed. The last command exited with exit code $test_status." 1>&2 + echo "::endgroup::" 1>&2 } # Mark the end of a test. From 4d458bcc0a700e59a96a79dce317b3aacaacafa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:38:20 +0200 Subject: [PATCH 130/410] Reformat and colorize test logs for legibility This reformats the test logs to improve legibility. A key change is highlighting the test result with corresponding colors to make it easier for developers to spot tests that have failed at a quick glance. The test result and duration are moved to the front of each line to avoid formatting issues because of very long test descriptions that are hard to format in a layout with fixed column widths. Also, this colorizes the commands captured by set -x to visually distinguish them from the actual command output. --- test/testlib.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/test/testlib.sh b/test/testlib.sh index 57a8e8e2d..fb1d509a6 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -57,8 +57,23 @@ ghe_remote_version_config "$GHE_TEST_REMOTE_VERSION" # ghe-restore process groups unset GHE_SNAPSHOT_TIMESTAMP +# Color definitions for log output +color_reset=$(printf '\e[0m') +# Display commands (lines starting with + in the output) in purple +color_command=$(printf '\e[0;35m') +# Display exit code line in red +color_error_message=$(printf '\e[0;31m') +# Display successful tests in bold green +color_pass=$(printf '\e[1;32m') +# Display skipped tests in bold gray +color_skip=$(printf '\e[1;37m') +# Display failed tests in bold red +color_fail=$(printf '\e[1;31m') + # keep track of num tests and failures tests=0 +successes=0 +skipped=0 failures=0 # this runs at process exit @@ -155,8 +170,10 @@ report_failure_output () { echo "$(<"$TRASHDIR/out")" \ | sed '0,/begin_test_truncate_marker/d' \ | sed -n '/end_test_truncate_marker/q;p' | head -n -2 \ + | sed "s/^\(+.*\)$/${color_command}\1${color_reset}/" \ 1>&2 - echo -e "\nTest failed. The last command exited with exit code $test_status." 1>&2 + echo -e "\n${color_error_message}Test failed. The last command exited with exit code" \ + "$test_status.${color_reset}" 1>&2 echo "::endgroup::" 1>&2 } @@ -173,12 +190,19 @@ end_test () { exec 1>&3 2>&4 if [ "$test_status" -eq 0 ]; then - printf "test: %-65s OK (%.3fs)\\n" "$test_description ..." "$elapsed_time" + successes=$(( successes + 1 )) + printf "${color_pass}PASS${color_reset}" 1>&2 elif [ "$test_status" -eq 254 ]; then - printf "test: %-65s SKIPPED\\n" "$test_description ..." + skipped=$(( skipped + 1 )) + printf "${color_skip}SKIP${color_reset}" 1>&2 else failures=$(( failures + 1 )) - printf "test: %-65s FAILED (%.3fs)\\n" "$test_description ..." "$elapsed_time" + printf "${color_fail}FAIL${color_reset}" 1>&2 + fi + + printf " [%8.3f s] $test_description\\n" "$elapsed_time" 1>&2 + + if [ "$test_status" -ne 0 ] && [ "$test_status" -ne 254 ]; then report_failure_output fi From ddfe5b5cfc3e256e45d7df1cdb438a3723c548f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 20:03:53 +0200 Subject: [PATCH 131/410] Include name of test suite in test logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a test fails, it’s not immediately clear which file it belongs to. To address this, this change adds the name of the test suite to the test logs, which corresponds to the name of the test file, omitting the file extension for better legibility. --- test/testlib.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/testlib.sh b/test/testlib.sh index fb1d509a6..9001264b5 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -30,6 +30,9 @@ PATH="$ROOTDIR/test/bin:$ROOTDIR/bin:$ROOTDIR/share/github-backup-utils:$PATH" TMPDIR="$ROOTDIR/test/tmp" TRASHDIR="$TMPDIR/$(basename "$0")-$$" +test_suite_file_name="$(basename "${BASH_SOURCE[1]}")" +test_suite_name="${test_suite_file_name%.*}" + # Set GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} # This removes the assumption that a git config that specifies these is present. export GIT_AUTHOR_NAME=make GIT_AUTHOR_EMAIL=make GIT_COMMITTER_NAME=make GIT_COMMITTER_EMAIL=make @@ -63,6 +66,8 @@ color_reset=$(printf '\e[0m') color_command=$(printf '\e[0;35m') # Display exit code line in red color_error_message=$(printf '\e[0;31m') +# Display test suite name in blue +color_test_suite=$(printf '\e[0;34m') # Display successful tests in bold green color_pass=$(printf '\e[1;32m') # Display skipped tests in bold gray @@ -200,7 +205,8 @@ end_test () { printf "${color_fail}FAIL${color_reset}" 1>&2 fi - printf " [%8.3f s] $test_description\\n" "$elapsed_time" 1>&2 + printf " [%8.3f s] ${color_test_suite}$test_suite_name${color_reset} $test_description\\n" \ + "$elapsed_time" 1>&2 if [ "$test_status" -ne 0 ] && [ "$test_status" -ne 254 ]; then report_failure_output From 0ecd1b78c843ecf03991b56df54f9579b9b2c575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 19:51:35 +0200 Subject: [PATCH 132/410] Distinguish parallel test suites by name This introduces an override to the test suite name, which is determined as the name of the calling test file without its extension by default. For the parallel tests, the default does not work reliably, as they are essentially just thin wrappers around the test-ghe-backup and test-ghe-restore test suites. --- test/test-ghe-backup-parallel.sh | 3 +++ test/test-ghe-restore-parallel.sh | 3 +++ test/testlib.sh | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test-ghe-backup-parallel.sh b/test/test-ghe-backup-parallel.sh index c36b98a0d..173c1b66c 100755 --- a/test/test-ghe-backup-parallel.sh +++ b/test/test-ghe-backup-parallel.sh @@ -2,6 +2,9 @@ # ghe-backup command tests run in parallel set -e +# Overwrite default test suite name to distinguish it from test-ghe-backup +export GHE_TEST_SUITE_NAME="test-ghe-backup-parallel" + export GHE_PARALLEL_ENABLED=yes TESTS_DIR="$PWD/$(dirname "$0")" diff --git a/test/test-ghe-restore-parallel.sh b/test/test-ghe-restore-parallel.sh index e0c8a0caa..16cbb2f0f 100755 --- a/test/test-ghe-restore-parallel.sh +++ b/test/test-ghe-restore-parallel.sh @@ -8,6 +8,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then exit 0 fi +# Overwrite default test suite name to distinguish it from test-ghe-restore +export GHE_TEST_SUITE_NAME="test-ghe-restore-parallel" + export GHE_PARALLEL_ENABLED=yes # use temp dir to fix rsync file issues in parallel execution: diff --git a/test/testlib.sh b/test/testlib.sh index 9001264b5..7a9ac02fa 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -31,7 +31,7 @@ TMPDIR="$ROOTDIR/test/tmp" TRASHDIR="$TMPDIR/$(basename "$0")-$$" test_suite_file_name="$(basename "${BASH_SOURCE[1]}")" -test_suite_name="${test_suite_file_name%.*}" +test_suite_name="${GHE_TEST_SUITE_NAME:-${test_suite_file_name%.*}}" # Set GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} # This removes the assumption that a git config that specifies these is present. From f1a11687c9ee06e67f11c42ec75784fa7a5c6a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 19:40:48 +0200 Subject: [PATCH 133/410] Remove debug output This line clutters the test output and does not appear to be useful. --- test/testlib.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/testlib.sh b/test/testlib.sh index 7a9ac02fa..c7a1f6539 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -443,7 +443,6 @@ setup_minio_test_data() { bucket="packages" mkdir -p "$bucket" - echo "an example blob" "$bucket/91dfa09f-1801-4e00-95ee-6b763d7da3e2" } cleanup_minio_test_data() { From abfccfb8b4ec23982fee2ab1082a25ec1f23962f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20L=C3=BChne?= Date: Wed, 13 Sep 2023 19:36:17 +0200 Subject: [PATCH 134/410] Produce Markdown test summary --- script/cibuild | 21 ++++++++++++++++++--- test/testlib.sh | 23 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/script/cibuild b/script/cibuild index 73dc9ae0a..22f7ffd08 100755 --- a/script/cibuild +++ b/script/cibuild @@ -2,13 +2,31 @@ # Usage: script/cibuild [--no-package] set -e +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +TMPDIR="$ROOTDIR/test/tmp" + +# Remove possible remnants of previous test runs +rm -rf "${TMPDIR:?}/*" + +print_test_results() { + if [ -n "$GITHUB_STEP_SUMMARY" ]; then + echo -e "### Test results\n" >> "$GITHUB_STEP_SUMMARY" + echo "| Test suite | Result | Successful | Failed | Skipped | Duration |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---|--:|--:|--:|--:|" >> "$GITHUB_STEP_SUMMARY" + sort -V "$TMPDIR/results" >> "$GITHUB_STEP_SUMMARY" + fi +} + # Enable verbose logging of ssh commands export GHE_VERBOSE_SSH=true if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then + print_test_results exit 1 fi +print_test_results + # Bail out when --no-package given [ "$1" = "--no-package" ] && exit 0 @@ -16,9 +34,6 @@ fi pkg_files= # Build the tarball -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -TMPDIR="$ROOTDIR/test/tmp" - echo "Building tar.gz package ..." if script/package-tarball 1>$TMPDIR/package-tarball.txt 2>&1; then pkg_files=$(grep '^Package ' < $TMPDIR/package-tarball.txt | cut -f 2 -d ' ') diff --git a/test/testlib.sh b/test/testlib.sh index c7a1f6539..44bbaed67 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -32,6 +32,8 @@ TRASHDIR="$TMPDIR/$(basename "$0")-$$" test_suite_file_name="$(basename "${BASH_SOURCE[1]}")" test_suite_name="${GHE_TEST_SUITE_NAME:-${test_suite_file_name%.*}}" +results_file="$TMPDIR/results" +test_suite_before_time=$(date '+%s.%3N') # Set GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} # This removes the assumption that a git config that specifies these is present. @@ -85,6 +87,27 @@ failures=0 atexit () { res=$? + test_suite_after_time=$(date '+%s.%3N') + test_suite_elapsed_time=$(echo "scale=3; $test_suite_after_time - $test_suite_before_time" | bc) + + # Temporarily redirect stdout output to results file + exec 3<&1 + exec 1>>"$results_file" + + # Print test summary for this test suite + echo -n "| $test_suite_name | " + + if [ "$failures" -eq "0" ]; then + echo -n ":green_circle: passed" + else + echo -n ":red_circle: failed" + fi + + printf " | $successes | $failures | $skipped | %.3f s |\\n" "$test_suite_elapsed_time" + + # Restore stdout + exec 1<&3 + [ -z "$KEEPTRASH" ] && rm -rf "$TRASHDIR" if [ $failures -gt 0 ]; then exit 1 From e730d2b3c587dcb1e191f9f55a942d63968c1752 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:02:22 -0400 Subject: [PATCH 135/410] remove token input --- .github/workflows/build-and-release.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b05fe8ca1..b581eff7c 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -3,10 +3,6 @@ name: Build and Release on: workflow_dispatch: inputs: - gh-token: - description: 'GitHub Token - used to create the release' - required: true - type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -97,6 +93,5 @@ jobs: commit: master bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} - token: ${{ github.event.inputs.gh-token }} allowUpdates: true artifactContentType: "raw" From d4ccab2cf8f3053832f539c89d28f68c4ba88fd1 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:03:22 -0400 Subject: [PATCH 136/410] try without shell field --- .github/workflows/build-and-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b581eff7c..94056e00e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,7 +37,6 @@ jobs: - name: Package deb run: | ./script/package-deb - shell: bash # many need to remove this once release-notes compilation is automated - name: Rename deb artifact run: | @@ -46,7 +45,6 @@ jobs: mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" fi done - shell: bash - name: Upload deb artifact uses: actions/upload-artifact@v3 with: From a8e8e9eb2c330b1e45b317bbd398a37fd8c7d5ac Mon Sep 17 00:00:00 2001 From: alejndr0 Date: Thu, 14 Sep 2023 14:04:15 +0000 Subject: [PATCH 137/410] Disable compression --- share/github-backup-utils/ghe-backup-actions | 2 +- share/github-backup-utils/ghe-backup-es-rsync | 4 ++-- share/github-backup-utils/ghe-backup-minio | 1 - share/github-backup-utils/ghe-backup-pages | 2 +- share/github-backup-utils/ghe-backup-userdata | 2 +- share/github-backup-utils/ghe-restore-es-audit-log | 2 +- share/github-backup-utils/ghe-restore-es-rsync | 2 +- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-actions b/share/github-backup-utils/ghe-backup-actions index aa02191be..b3f51741e 100755 --- a/share/github-backup-utils/ghe-backup-actions +++ b/share/github-backup-utils/ghe-backup-actions @@ -39,7 +39,7 @@ fi # Transfer all Actions data from the user data directory using rsync. ghe_verbose "* Transferring Actions files from $host ..." log_rsync "BEGIN: actions rsync" 1>&3 -ghe-rsync -avz \ +ghe-rsync -av \ -e "ghe-ssh -p $port" \ --rsync-path='sudo -u actions rsync' \ --exclude "mutexes" --exclude "dumps" --exclude "tmp" \ diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync index ee61741b2..fce7043bb 100755 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ b/share/github-backup-utils/ghe-backup-es-rsync @@ -45,7 +45,7 @@ fi # already been transferred. ghe_verbose "* Performing initial sync of ES indices ..." log_rsync "BEGIN elasticsearch rsync" 1>&3 -ghe-rsync -avz \ +ghe-rsync -av \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ --rsync-path="sudo -u elasticsearch rsync" \ $link_dest \ @@ -70,7 +70,7 @@ ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null # Transfer all ES indices again ghe_verbose "* Performing follow-up sync of ES indices ..." log_rsync "BEGIN: elasticsearch followup rsync" 1>&3 -ghe-rsync -avz \ +ghe-rsync -av \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ --rsync-path="sudo -u elasticsearch rsync" \ $link_dest \ diff --git a/share/github-backup-utils/ghe-backup-minio b/share/github-backup-utils/ghe-backup-minio index 7f6a18b22..bf7a429d4 100755 --- a/share/github-backup-utils/ghe-backup-minio +++ b/share/github-backup-utils/ghe-backup-minio @@ -45,7 +45,6 @@ log_rsync "BEGIN: minio rsync" 1>&3 ghe-rsync \ --archive \ --verbose \ - --compress \ --rsh="ghe-ssh -p ${port}" \ --rsync-path='sudo -u minio rsync' \ --exclude=".minio.sys" \ diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages index b0c9f1fed..2b79825cf 100755 --- a/share/github-backup-utils/ghe-backup-pages +++ b/share/github-backup-utils/ghe-backup-pages @@ -75,7 +75,7 @@ for hostname in $hostnames; do ghe_verbose "* Transferring pages files ..." log_rsync "BEGIN: pages rsync" 1>&3 # Transfer all data from the user data directory using rsync. - ghe-rsync -avz \ + ghe-rsync -av \ -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ --rsync-path='sudo -u git rsync' \ $link_dest \ diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata index f332c27c4..d7c7dc233 100755 --- a/share/github-backup-utils/ghe-backup-userdata +++ b/share/github-backup-utils/ghe-backup-userdata @@ -54,7 +54,7 @@ fi mkdir -p "$GHE_SNAPSHOT_DIR/$dirname" log_rsync "BEGIN: userdata rsync" 1>&3 # Transfer all data from the user data directory using rsync. -ghe-rsync -avz \ +ghe-rsync -av \ -e "ghe-ssh -p $(ssh_port_part "$host")" \ --rsync-path='sudo -u git rsync' \ $link_dest \ diff --git a/share/github-backup-utils/ghe-restore-es-audit-log b/share/github-backup-utils/ghe-restore-es-audit-log index 06b63973b..0ed3d37b2 100755 --- a/share/github-backup-utils/ghe-restore-es-audit-log +++ b/share/github-backup-utils/ghe-restore-es-audit-log @@ -51,7 +51,7 @@ if [ -s "$tmp_list" ]; then ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 log_rsync "BEGIN: es-audit log rsync" 1>&3 - ghe-rsync -avz --delete \ + ghe-rsync -av --delete \ -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \ --rsync-path="sudo -u elasticsearch rsync" \ --files-from=$tmp_list \ diff --git a/share/github-backup-utils/ghe-restore-es-rsync b/share/github-backup-utils/ghe-restore-es-rsync index b1d9b2179..1ba09afed 100755 --- a/share/github-backup-utils/ghe-restore-es-rsync +++ b/share/github-backup-utils/ghe-restore-es-rsync @@ -37,7 +37,7 @@ else ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 ghe-ssh "$GHE_HOSTNAME" -- "sudo chown elasticsearch:elasticsearch '$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore'" 1>&3 log_rsync "BEGIN: elasticsearch rsync" 1>&3 - ghe-rsync -avz --delete \ + ghe-rsync -av --delete \ -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \ --rsync-path="sudo -u elasticsearch rsync" \ --copy-dest="$GHE_REMOTE_DATA_USER_DIR/elasticsearch" \ From 8ebf223b2c2ed5535c9675e75e453b9e4ebd9fae Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:10:05 -0400 Subject: [PATCH 138/410] set email to team distribution list --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 94056e00e..b3a989b50 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -30,7 +30,7 @@ jobs: - name: Create tag run: | git config user.name "release-controller" - git config user.email "release-controller@github.com" + git config user.email "ghes-releases-team@github.com" git tag -a v${{ github.event.inputs.version }} \ -m "v${{ github.event.inputs.version }}" git push origin v${{ github.event.inputs.version }} From 41071a4c6f0841b36b3d7676f09d4c8da061404f Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:11:33 -0400 Subject: [PATCH 139/410] test release-notes --- release-notes/12.12.12.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md new file mode 100644 index 000000000..192f42d88 --- /dev/null +++ b/release-notes/12.12.12.md @@ -0,0 +1,11 @@ +# Release Notes + +### Features + +- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." +- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". + +### Changes + +- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). +- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 08fe226d5462d9147833111b2eebfe37b554a145 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:12:55 -0400 Subject: [PATCH 140/410] release notes will be in the project --- .github/workflows/build-and-release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b3a989b50..acffa376d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -7,10 +7,6 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string - release-notes: - description: 'Release Notes - string of markdown' - required: true - type: string draft: description: 'Draft - true if the release should be a draft' required: true From 439e87d4af70bcadc5ac19a0a14d4ae129bd5761 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 10:29:24 -0400 Subject: [PATCH 141/410] associate release with empty commit so tags don't stack up on one commit --- .github/workflows/build-and-release.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index acffa376d..8d3afa195 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -60,7 +60,23 @@ jobs: release: needs: build runs-on: ubuntu-latest + outputs: + commit_sha: ${{ steps.release.outputs.commit_sha }} steps: + # checkout another repo and create an empty commit + # this is needed because the release action requires a tag + - name: Checkout + uses: actions/checkout@v2 + with: + repository: github/backup-utils + ref: master + - name: Create empty commit + run: | + git config user.name "release-controller" + git config user.email "ghes-releases-team@github.com" + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin master + echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Checkout uses: actions/checkout@v2 - name: Download deb artifact From 573860d39c2ad54b09cadd0b20cab02f218bc225 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Thu, 14 Sep 2023 11:15:47 -0400 Subject: [PATCH 142/410] use the empty commit sha for the release --- .github/workflows/build-and-release.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8d3afa195..81ba0d9e6 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -61,22 +61,21 @@ jobs: needs: build runs-on: ubuntu-latest outputs: - commit_sha: ${{ steps.release.outputs.commit_sha }} + commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - # checkout another repo and create an empty commit - # this is needed because the release action requires a tag - name: Checkout uses: actions/checkout@v2 with: repository: github/backup-utils ref: master - name: Create empty commit + id: empty_commit run: | git config user.name "release-controller" git config user.email "ghes-releases-team@github.com" git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin master - echo "commit_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Checkout uses: actions/checkout@v2 - name: Download deb artifact @@ -100,7 +99,7 @@ jobs: # this action will create a tag with this name on the provided commit tag: v${{ github.event.inputs.version }} # this can be a commit hash or branch name - commit: master + commit: ${{ steps.empty_commit.outputs.commit_sha }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From c69ac4e35291a0317f6c76fe1ea87fecbc7e3ffd Mon Sep 17 00:00:00 2001 From: djdefi Date: Thu, 14 Sep 2023 13:23:39 -0700 Subject: [PATCH 143/410] Update rsync-docker-bump.yml Co-authored-by: Quinn Murphy --- .github/workflows/rsync-docker-bump.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index 6d90b7bbe..2cfe7782c 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -15,7 +15,7 @@ jobs: - name: Get latest rsync tag id: latest_tag - run: curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> $GITHUB_OUTPUT + run: curl --silent "https://api.github.com/repos/WayneD/rsync/tags" | jq -r '.[].name' | grep -m1 -v pre | xargs -I {} echo "name=latest_tag::{}" >> "$GITHUB_OUTPUT" - name: Update Dockerfile with latest tag run: | From afa8e8dbee29b71788e241467bdf417459c79dab Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 14 Sep 2023 22:57:30 -0600 Subject: [PATCH 144/410] Triger new templated jobs --- .github/workflows/integration-tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1a30dda3b..053f760bf 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -13,8 +13,8 @@ jobs: strategy: matrix: jankyJobName: - - enterprise2-binary-backup - - enterprise2-migration + - enterprise2-backup-utils-binary-backup + - enterprise2-backup-utils-migration steps: - uses: actions/checkout@v3 with: @@ -24,10 +24,9 @@ jobs: ref="${{ github.ref }}" merge_branch=${ref#"refs/heads/"} backup_utils_branch="${{ env.SOURCE_BRANCH }}" - #branch_name="${{ env.TARGET_BRANCH }}" - branch_name="janky-backup-utils-branch-test" + branch_name="${{ env.TARGET_BRANCH }}" curl -v -X POST \ -u "hubot:${{ secrets.API_AUTH_TOKEN }}" \ -H "Content-Type: application/json" \ - -d '{"buildable_name":"${{ matrix.jankyJobName }}","repo":"enterprise2","branch_name": "'"$branch_name"'","env_vars":{"JANKY_ENV_BACKUP_UTILS_BRANCH": "'"$backup_utils_branch"'" },"force":"true","room_id":"#builds"}' \ + -d '{"buildable_name":"${{ matrix.jankyJobName }}","repo":"${{ github.repository }}","branch_name": "'"$branch_name"'","env_vars":{"JANKY_ENV_BACKUP_UTILS_BRANCH": "'"$backup_utils_branch"'" },"force":"true","room_id":"#builds"}' \ "https://janky.githubapp.com/api/builds" From 307fe2fec0630ec5a0723737f1849ca8b9ba5c18 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 15 Sep 2023 11:07:28 -0400 Subject: [PATCH 145/410] use release controller app token --- .github/workflows/build-and-release.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 81ba0d9e6..dd0df9099 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -63,16 +63,21 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - name: Checkout uses: actions/checkout@v2 with: + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit id: empty_commit run: | - git config user.name "release-controller" - git config user.email "ghes-releases-team@github.com" git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin master echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT @@ -89,6 +94,7 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: + token: ${{ steps.app-token.outputs.token }} repo: backup-utils name: | GitHub Enterprise Server Backup Utilities From e1a92a553a3d977bf297a5de8c246c81d3476848 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Fri, 15 Sep 2023 12:17:17 -0400 Subject: [PATCH 146/410] fix backup.config typo (#587) Fixing https://github.com/github/ghes/issues/7329 --- backup.config-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 688cc374b..d1ea977e2 100644 --- a/backup.config-example +++ b/backup.config-example @@ -26,7 +26,7 @@ GHE_NUM_SNAPSHOTS=10 # performed before the next full backup is created. # For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will # run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. -#GHE_INCREMENTAL_BACKUP_MAX=14 +#GHE_INCREMENTAL_MAX_BACKUPS=14 # If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check # disk space validation and software version checks on the backup-host will be disabled. From 6042e5151caaf410889d6b3f0bf3fee370eefd2a Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 15 Sep 2023 10:40:55 -0600 Subject: [PATCH 147/410] update workflow dispatch rules --- .github/workflows/integration-tests.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 053f760bf..895d6c692 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,6 +1,17 @@ name: Run Integration Tests -on: [pull_request, workflow_dispatch] +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: ['master', 'enterprise-[0-9]*.[0-9]*-release', 'enterprise-[0-9]*.[0-9]*.[0-9]*-release'] + workflow_dispatch: + inputs: + target-branch: + description: 'Branch that would be merged into' + required: true + source-branch: + description: 'Branch that would be merged' + required: true # Get target and source branch from different variables depending on how it was triggered env: @@ -21,8 +32,6 @@ jobs: fetch-depth: 1 - name: Queue ${{ matrix.jankyJobName }} build run: | - ref="${{ github.ref }}" - merge_branch=${ref#"refs/heads/"} backup_utils_branch="${{ env.SOURCE_BRANCH }}" branch_name="${{ env.TARGET_BRANCH }}" curl -v -X POST \ From a10db47c0fcaa601be1a7361e20f292918f6c2e3 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 15 Sep 2023 12:35:24 -0600 Subject: [PATCH 148/410] Use proxy-janky-build actions --- .github/actions/proxy-janky-build/action.yml | 29 +++ .github/actions/proxy-janky-build/go.mod | 7 + .github/actions/proxy-janky-build/go.sum | 9 + .github/actions/proxy-janky-build/main.go | 177 +++++++++++++++++++ .github/workflows/integration-tests.yml | 15 +- 5 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 .github/actions/proxy-janky-build/action.yml create mode 100644 .github/actions/proxy-janky-build/go.mod create mode 100644 .github/actions/proxy-janky-build/go.sum create mode 100644 .github/actions/proxy-janky-build/main.go diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml new file mode 100644 index 000000000..fe36d27b4 --- /dev/null +++ b/.github/actions/proxy-janky-build/action.yml @@ -0,0 +1,29 @@ +name: 'Trigger a CI Job on Janky' +description: 'Action to trigger and poll a Janky CI job' +inputs: + janky-token: + description: 'Token for making request to Janky' + required: true + job-name: + description: 'The name of the job to run' + required: true + branch-name: + description: 'The name of the branch to use' + required: true + envVars: + description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' + required: false +runs: + using: 'composite' + steps: + - uses: actions/setup-go@a3d889c34c5d4e071b33595c5fe8edfcaaad8260 + with: + go-version: '1.21' + - run: | + go run main.go \ + -token ${{ inputs.janky-token }} \ + -job ${{ inputs.job-name }} \ + -branch ${{ inputs.branch-name }} \ + -envVars ${{ inputs.envVars }} + shell: bash + working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/go.mod b/.github/actions/proxy-janky-build/go.mod new file mode 100644 index 000000000..b7560d7b9 --- /dev/null +++ b/.github/actions/proxy-janky-build/go.mod @@ -0,0 +1,7 @@ +module github.com/github/enterprise2/actions/proxy-janky-build + +go 1.21 + +require github.com/hashicorp/go-retryablehttp v0.7.2 + +require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/.github/actions/proxy-janky-build/go.sum b/.github/actions/proxy-janky-build/go.sum new file mode 100644 index 000000000..5c59c1d2e --- /dev/null +++ b/.github/actions/proxy-janky-build/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go new file mode 100644 index 000000000..cfc120295 --- /dev/null +++ b/.github/actions/proxy-janky-build/main.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +// Define our Janky Response Structs +type JankyBuildStruct struct { + Result string + Url string +} +type JankyStatusStruct struct { + Id string + Green bool + Completed bool + StartedAt string + CompletedAt string + Sha string + BuildableName string +} + +const ( + pollWaitTime = 10 * time.Second + jankyPollTimeout = 5 * time.Hour + jankyHttpRetryMax = 5 + jankyUrl = "https://janky.githubapp.com/" +) + +func main() { + // Parse command-line arguments + job := flag.String("job", "", "Name of the Janky job") + token := flag.String("token", "", "Name of the Janky token") + branch := flag.String("branch", "", "Name of the Git branch") + envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") + flag.Parse() + + // Validate command-line arguments + if *job == "" || *token == "" || *branch == "" { + log.Fatal("job, token and branch flags must be specified") + } + + // Set up the token + request payload + authToken := base64.StdEncoding.EncodeToString([]byte(":" + *token)) + type buildRequestObject struct { + BuildableName string `json:"buildable_name"` + BranchName string `json:"branch_name"` + EnvVars map[string]string `json:"env_vars"` + } + + requestBody := buildRequestObject{ + BuildableName: *job, + BranchName: *branch, + } + + // Parse the envVars flag into a map and add to the request payload + fmt.Println("Environment Variables:") + fmt.Println(*envVars) + if *envVars != "" { + envVarsMap := make(map[string]string) + for _, envVar := range strings.Split(*envVars, ",") { + envVarSplit := strings.Split(envVar, "=") + envVarsMap[envVarSplit[0]] = envVarSplit[1] + } + requestBody.EnvVars = envVarsMap + } + + payloadBytes, err := json.Marshal(requestBody) + if err != nil { + log.Fatal("Failed to marshal the JSON payload!\n" + err.Error()) + } + + // Send build request to Janky + buildRequest, err := http.NewRequest("POST", jankyUrl+"/api/builds", bytes.NewBuffer(payloadBytes)) + if err != nil { + log.Fatal("Failed to create build request!\n" + err.Error()) + } + buildRequest.Header.Set("Content-Type", "application/json") + buildRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = jankyHttpRetryMax + retryClient.Logger = nil // disable debug logging + client := retryClient.StandardClient() // uses *http.Client + resp, err := client.Do(buildRequest) + if err != nil { + log.Fatal("Failed to send build request!\n" + err.Error()) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatal("Error reading build response!\n" + err.Error()) + } + + // Check if the build was triggered successfully + if resp.StatusCode == 404 { + log.Fatal("Failed to trigger build! Either " + *job + " is not the name of a Janky job or " + *branch + " is not a branch for the repository that job belongs to.") + } + if resp.StatusCode != 201 { + log.Fatal("Failed to trigger build! Got exception: " + string(body)) + } + + // Parse the build request response + var buildResponse JankyBuildStruct + json.Unmarshal(body, &buildResponse) + log.Println("Succesfully triggered janky!\n" + buildResponse.Result) + + // Parse the request response for the buildId + r, err := regexp.Compile("/[0-9]+/") + if err != nil { + log.Fatal("Failed to trigger build!\n" + err.Error()) + } + buildId := strings.Trim(r.FindString(buildResponse.Result), "/") + + // Setup our second HTTP client for reuse in during status polling + jankyStatusUrl := jankyUrl + "/api/" + buildId + "/status" + statusRequest, err := http.NewRequest("GET", jankyStatusUrl, nil) + if err != nil { + log.Fatal("Failed to create status request!\n" + err.Error()) + } + statusRequest.Header.Set("Content-Type", "application/json") + statusRequest.Header.Set("Authorization", "Basic "+authToken) + retryClient2 := retryablehttp.NewClient() + retryClient2.RetryMax = jankyHttpRetryMax + retryClient2.Logger = nil // disable debug logging + client2 := retryClient2.StandardClient() // uses *http.Client + + // Wait for a completed status from Janky or break the loop after a certain amount of time + timeout := time.NewTimer(jankyPollTimeout) + poll := time.NewTicker(pollWaitTime) + +jobLoop: + for { + select { + case <-timeout.C: + log.Fatal("Failed to poll for build status after " + jankyPollTimeout.String() + "hours") + case <-poll.C: + // Send build status request to Janky + statusResponse, err := client2.Do(statusRequest) + if err != nil { + log.Fatal("Failed to send status request!\n" + err.Error()) + } + defer statusResponse.Body.Close() + statusBody, err := io.ReadAll(statusResponse.Body) + if err != nil { + log.Fatal("Error reading status response!\n" + err.Error()) + } + + // Parse the status response for a green completed build + var jankyStatusResponse JankyStatusStruct + json.Unmarshal(statusBody, &jankyStatusResponse) + //fmt.Println("Janky Status Response:") + //fmt.Println(string(statusBody)) + if jankyStatusResponse.Completed && jankyStatusResponse.Green { + log.Println("Janky build Succeeded!") + break jobLoop + } + if jankyStatusResponse.Completed && !jankyStatusResponse.Green { + log.Fatal("Build failed, see Janky for more info: " + buildResponse.Url) + } + + // wait for a bit and try again + log.Println("Build still in progress, will poll for status again in [" + pollWaitTime.String() + "]") + continue + } + } +} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 895d6c692..d97350957 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,11 +31,10 @@ jobs: with: fetch-depth: 1 - name: Queue ${{ matrix.jankyJobName }} build - run: | - backup_utils_branch="${{ env.SOURCE_BRANCH }}" - branch_name="${{ env.TARGET_BRANCH }}" - curl -v -X POST \ - -u "hubot:${{ secrets.API_AUTH_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d '{"buildable_name":"${{ matrix.jankyJobName }}","repo":"${{ github.repository }}","branch_name": "'"$branch_name"'","env_vars":{"JANKY_ENV_BACKUP_UTILS_BRANCH": "'"$backup_utils_branch"'" },"force":"true","room_id":"#builds"}' \ - "https://janky.githubapp.com/api/builds" + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.TARGET_BRANCH }}' + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From 44b713ed91852e3b03d7598c25d66004ddacb898 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 15 Sep 2023 14:49:04 -0400 Subject: [PATCH 149/410] use token for tag creation --- .github/workflows/build-and-release.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index dd0df9099..f6194bb55 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,16 +17,22 @@ jobs: build: runs-on: ubuntu-latest steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - name: Checkout uses: actions/checkout@v2 + with: + token: ${{ steps.app-token.outputs.token }} - name: Install dependencies run: | sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Create tag run: | - git config user.name "release-controller" - git config user.email "ghes-releases-team@github.com" git tag -a v${{ github.event.inputs.version }} \ -m "v${{ github.event.inputs.version }}" git push origin v${{ github.event.inputs.version }} @@ -50,7 +56,6 @@ jobs: - name: Package tarball run: | ./script/package-tarball - shell: bash - name: Upload tarball artifact uses: actions/upload-artifact@v3 with: From c9e22e96c4fbcb1db178f62b305d32c8c3d7cf7c Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 15 Sep 2023 12:50:34 -0600 Subject: [PATCH 150/410] Force build --- .github/actions/proxy-janky-build/action.yml | 4 ++++ .github/actions/proxy-janky-build/main.go | 3 +++ .github/workflows/integration-tests.yml | 1 + 3 files changed, 8 insertions(+) diff --git a/.github/actions/proxy-janky-build/action.yml b/.github/actions/proxy-janky-build/action.yml index fe36d27b4..988c4a289 100644 --- a/.github/actions/proxy-janky-build/action.yml +++ b/.github/actions/proxy-janky-build/action.yml @@ -10,6 +10,9 @@ inputs: branch-name: description: 'The name of the branch to use' required: true + force: + description: 'Force the job to run even if it is already passed' + required: false envVars: description: 'Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3' required: false @@ -24,6 +27,7 @@ runs: -token ${{ inputs.janky-token }} \ -job ${{ inputs.job-name }} \ -branch ${{ inputs.branch-name }} \ + -force ${{ inputs.force }} \ -envVars ${{ inputs.envVars }} shell: bash working-directory: .github/actions/proxy-janky-build diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go index cfc120295..4146be416 100644 --- a/.github/actions/proxy-janky-build/main.go +++ b/.github/actions/proxy-janky-build/main.go @@ -43,6 +43,7 @@ func main() { job := flag.String("job", "", "Name of the Janky job") token := flag.String("token", "", "Name of the Janky token") branch := flag.String("branch", "", "Name of the Git branch") + force := flag.Bool("force", false, "Force a build even if one is already passed") envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") flag.Parse() @@ -56,12 +57,14 @@ func main() { type buildRequestObject struct { BuildableName string `json:"buildable_name"` BranchName string `json:"branch_name"` + Force bool `json:"force"` EnvVars map[string]string `json:"env_vars"` } requestBody := buildRequestObject{ BuildableName: *job, BranchName: *branch, + Force: *force, } // Parse the envVars flag into a map and add to the request payload diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d97350957..14caa2726 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,4 +37,5 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.TARGET_BRANCH }}' + force : 'true' envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From 2af815acd9b132cb58cde032f615f81b3e16b6df Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 15 Sep 2023 12:55:10 -0600 Subject: [PATCH 151/410] Make force a string --- .github/actions/proxy-janky-build/main.go | 4 ++-- .github/workflows/integration-tests.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go index 4146be416..e05346cd4 100644 --- a/.github/actions/proxy-janky-build/main.go +++ b/.github/actions/proxy-janky-build/main.go @@ -43,7 +43,7 @@ func main() { job := flag.String("job", "", "Name of the Janky job") token := flag.String("token", "", "Name of the Janky token") branch := flag.String("branch", "", "Name of the Git branch") - force := flag.Bool("force", false, "Force a build even if one is already passed") + force := flag.String("force", "false", "Force a build even if one is already passed") envVars := flag.String("envVars", "", "Comma separated list of key value pairs to pass to Janky - ex: key1=value1,key2=value2,key3=value3") flag.Parse() @@ -57,7 +57,7 @@ func main() { type buildRequestObject struct { BuildableName string `json:"buildable_name"` BranchName string `json:"branch_name"` - Force bool `json:"force"` + Force string `json:"force"` EnvVars map[string]string `json:"env_vars"` } diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 14caa2726..804c6d5c5 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,5 +37,5 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.TARGET_BRANCH }}' - force : 'true' + force : "true" envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" From 7094bcaa44a6fd9c36d9a2b49c2a7db1af0c09f0 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 15 Sep 2023 13:51:02 -0600 Subject: [PATCH 152/410] disable lint error --- .github/actions/proxy-janky-build/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go index e05346cd4..e7afcd917 100644 --- a/.github/actions/proxy-janky-build/main.go +++ b/.github/actions/proxy-janky-build/main.go @@ -91,7 +91,7 @@ func main() { } buildRequest.Header.Set("Content-Type", "application/json") buildRequest.Header.Set("Authorization", "Basic "+authToken) - retryClient := retryablehttp.NewClient() + retryClient := retryablehttp.NewClient() //nolint:all retryClient.RetryMax = jankyHttpRetryMax retryClient.Logger = nil // disable debug logging client := retryClient.StandardClient() // uses *http.Client From 2e7142240322f42bf135e90542dc346e0b2943c1 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Fri, 15 Sep 2023 14:30:31 -0600 Subject: [PATCH 153/410] Update main.go --- .github/actions/proxy-janky-build/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/proxy-janky-build/main.go b/.github/actions/proxy-janky-build/main.go index e7afcd917..e0342f121 100644 --- a/.github/actions/proxy-janky-build/main.go +++ b/.github/actions/proxy-janky-build/main.go @@ -133,7 +133,7 @@ func main() { } statusRequest.Header.Set("Content-Type", "application/json") statusRequest.Header.Set("Authorization", "Basic "+authToken) - retryClient2 := retryablehttp.NewClient() + retryClient2 := retryablehttp.NewClient() //nolint:all retryClient2.RetryMax = jankyHttpRetryMax retryClient2.Logger = nil // disable debug logging client2 := retryClient2.StandardClient() // uses *http.Client From 995bd7a06082c4390fe49d250ef1a2c57866503a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 18 Sep 2023 11:01:34 -0400 Subject: [PATCH 154/410] set git config --- .github/workflows/build-and-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f6194bb55..4943834a8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -33,6 +33,8 @@ jobs: sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Create tag run: | + git config user.name "release-controller" + git config user.email "ghes-releases-team@github.com" git tag -a v${{ github.event.inputs.version }} \ -m "v${{ github.event.inputs.version }}" git push origin v${{ github.event.inputs.version }} @@ -83,6 +85,8 @@ jobs: - name: Create empty commit id: empty_commit run: | + git config user.name "release-controller" + git config user.email "ghes-releases-team@github.com" git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin master echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT From 5eeb9832e637bb9344fc47bc935408593b65f2b7 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 12:52:38 -0400 Subject: [PATCH 155/410] use patch release branches in private --- .github/workflows/build-and-release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 4943834a8..b82a40211 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -31,13 +31,12 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag + - name: Create patch release branch from current sha run: | git config user.name "release-controller" git config user.email "ghes-releases-team@github.com" - git tag -a v${{ github.event.inputs.version }} \ - -m "v${{ github.event.inputs.version }}" - git push origin v${{ github.event.inputs.version }} + git branch "enterprise-${{ github.event.inputs.version }}-release" + git push origin "enterprise-${{ github.event.inputs.version }}-release" - name: Package deb run: | ./script/package-deb From 71d6136b7ae2c56034aac8cf8ce1624670e71001 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 13:41:14 -0400 Subject: [PATCH 156/410] implement .releaseignore --- .releaseignore | 2 ++ script/package-deb | 8 ++++++++ script/package-tarball | 10 ++++++++++ 3 files changed, 20 insertions(+) create mode 100644 .releaseignore diff --git a/.releaseignore b/.releaseignore new file mode 100644 index 000000000..bb2f6b31c --- /dev/null +++ b/.releaseignore @@ -0,0 +1,2 @@ +ownership.yaml +.github \ No newline at end of file diff --git a/script/package-deb b/script/package-deb index dda90cbe3..8e771f1e4 100755 --- a/script/package-deb +++ b/script/package-deb @@ -22,6 +22,14 @@ mkdir -p dist/debuild distdir="$(pwd)/dist/debuild/$PKG_NAME" git clone -q . "$distdir" cd "$distdir" + +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +echo "Removing .releaseignore ..." +rm -f .releaseignore git checkout -q "$PKG_HEAD" debuild -uc -us 1>&2 diff --git a/script/package-tarball b/script/package-tarball index be653b77a..a2c09bd37 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -13,6 +13,16 @@ PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" PKG_NAME="${PKG_BASE}-${PKG_VERS}" +# Remove all files or directories listed in .releaseignore +echo "Removing files listed in .releaseignore ..." +while IFS= read -r line; do + rm -rf "$line" +done < .releaseignore + +# Remove the .releaseignore file itself +echo "Removing .releaseignore ..." +rm -f .releaseignore + # Run git-archive to generate tarball echo "Creating ${PKG_NAME}.tar.gz ..." mkdir -p dist From a3e8ae9be631a63db7aa6383ac26f350ce1b3b58 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 20:21:54 +0000 Subject: [PATCH 157/410] test against test branch --- .github/workflows/build-and-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index b82a40211..91f59aab2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -86,8 +86,9 @@ jobs: run: | git config user.name "release-controller" git config user.email "ghes-releases-team@github.com" + git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin master + git push origin tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Checkout uses: actions/checkout@v2 From ec2ef62f51bc33b76d713112b2f0813c5ae887b9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 16:43:17 -0400 Subject: [PATCH 158/410] fetch tags --- script/package-deb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/package-deb b/script/package-deb index 8e771f1e4..15a698e9c 100755 --- a/script/package-deb +++ b/script/package-deb @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" From 7de9682d6b152a7730df75f9a05953c7182f6f39 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 16:43:41 -0400 Subject: [PATCH 159/410] same for tar --- script/package-tarball | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/package-tarball b/script/package-tarball index a2c09bd37..bf1510e8f 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -8,6 +8,9 @@ set -e # Change into project root cd "$(dirname "$0")"/.. +# Fetch tags from remote repository +git fetch --tags + # Basic package name and version. PKG_BASE="github-backup-utils" PKG_VERS="$(git describe --tags)" From 610a0d32b58016d253b00cb97037990c119dd326 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 16:50:19 -0400 Subject: [PATCH 160/410] need vx.y.x tag for build scripts --- .github/workflows/build-and-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 91f59aab2..c3fc67c57 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,6 +37,11 @@ jobs: git config user.email "ghes-releases-team@github.com" git branch "enterprise-${{ github.event.inputs.version }}-release" git push origin "enterprise-${{ github.event.inputs.version }}-release" + # create v#{version} tag on current sha + - name: Create tag + run: | + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" - name: Package deb run: | ./script/package-deb From bd23ecdc80d3dccdbd12e7982c5845d85386ff78 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 19 Sep 2023 21:33:42 +0000 Subject: [PATCH 161/410] help --- .github/workflows/build-and-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c3fc67c57..487f808f2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -91,6 +91,8 @@ jobs: run: | git config user.name "release-controller" git config user.email "ghes-releases-team@github.com" + # tims-test-branch already exists, so fetch it + git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin tims-test-branch From 99db4a899a2b92698f12854ab272436d8bfde2eb Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Wed, 20 Sep 2023 15:08:24 +0200 Subject: [PATCH 162/410] remove dead code (#586) --- Makefile | 2 +- script/cibuild | 65 ------------------- script/release | 166 ++++++++++++++++++++++++------------------------- script/test | 28 +++++++++ 4 files changed, 112 insertions(+), 149 deletions(-) delete mode 100755 script/cibuild create mode 100755 script/test diff --git a/Makefile b/Makefile index 191b90561..c8eaa0ff1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ SHELL = /bin/sh test: info @echo Running tests - @script/cibuild --no-package + @script/test info: @echo This is github/backup-utils diff --git a/script/cibuild b/script/cibuild deleted file mode 100755 index 22f7ffd08..000000000 --- a/script/cibuild +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/cibuild [--no-package] -set -e - -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -TMPDIR="$ROOTDIR/test/tmp" - -# Remove possible remnants of previous test runs -rm -rf "${TMPDIR:?}/*" - -print_test_results() { - if [ -n "$GITHUB_STEP_SUMMARY" ]; then - echo -e "### Test results\n" >> "$GITHUB_STEP_SUMMARY" - echo "| Test suite | Result | Successful | Failed | Skipped | Duration |" >> "$GITHUB_STEP_SUMMARY" - echo "|---|---|--:|--:|--:|--:|" >> "$GITHUB_STEP_SUMMARY" - sort -V "$TMPDIR/results" >> "$GITHUB_STEP_SUMMARY" - fi -} - -# Enable verbose logging of ssh commands -export GHE_VERBOSE_SSH=true - -if ! find test -name "test-*.sh" -print0 | xargs -0 -n 1 /bin/bash; then - print_test_results - exit 1 -fi - -print_test_results - -# Bail out when --no-package given -[ "$1" = "--no-package" ] && exit 0 - -# files we'll md5sum at the end -pkg_files= - -# Build the tarball -echo "Building tar.gz package ..." -if script/package-tarball 1>$TMPDIR/package-tarball.txt 2>&1; then - pkg_files=$(grep '^Package ' < $TMPDIR/package-tarball.txt | cut -f 2 -d ' ') -else - echo "Packaging tar.gz failed:" - cat $TMPDIR/package-tarball.txt | sed 's/^/ /' 1>&2 - exit 1 -fi - -# Skip deb packaging if debuild not installed -if ! type debuild 1>/dev/null 2>&1; then - echo "debuild not installed, skipping deb packaging ..." - exit 0 -fi - -# Build the deb related packages -echo "Building deb package ..." -if DEB_BUILD_OPTIONS=nocheck script/package-deb 1>$TMPDIR/package-deb-out.txt 2>$TMPDIR/package-deb-err.txt; then - pkg_files="$pkg_files $(cat $TMPDIR/package-deb-out.txt)" -else - echo "Package build failed:" - cat $TMPDIR/package-deb-out.txt $TMPDIR/package-deb-err.txt >&2 - echo >&2 - cat dist/debuild/github-backup-utils*.build >&2 - exit 1 -fi - -# Generate md5sums -md5sum $pkg_files diff --git a/script/release b/script/release index 891cd9f60..9629cacff 100755 --- a/script/release +++ b/script/release @@ -19,25 +19,25 @@ #/ release page and the changelog. #/ * If this is a X.Y.0 release, a minimum supported version needs to be supplied too. #/ -require 'json' -require 'net/http' -require 'time' -require 'erb' -require 'English' +require "json" +require "net/http" +require "time" +require "erb" +require "English" -API_HOST = ENV['GH_HOST'] || 'api.github.com' +API_HOST = ENV["GH_HOST"] || "api.github.com" API_PORT = 443 -GH_REPO = ENV['GH_REPO'] || 'backup-utils' -GH_OWNER = ENV['GH_OWNER'] || 'github' -GH_AUTHOR = ENV['GH_AUTHOR'] -DEB_PKG_NAME = 'github-backup-utils' -GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' # TODO: should we even allow a default or require all params get set explicitly? +GH_REPO = ENV["GH_REPO"] || "backup-utils" +GH_OWNER = ENV["GH_OWNER"] || "github" +GH_AUTHOR = ENV["GH_AUTHOR"] +DEB_PKG_NAME = "github-backup-utils" +GH_BASE_BRANCH = ENV["GH_BASE_BRANCH"] || "master" # TODO: should we even allow a default or require all params get set explicitly? GH_STABLE_BRANCH = "" # If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI -PUBLISH = ENV['PUBLISH'] == 'true' || false +PUBLISH = ENV["PUBLISH"] == "true" || false -CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium +CHANGELOG_TMPL = "" '<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium <%- changes.each do |ch| -%> * <%= ch.strip.chomp %> @@ -45,7 +45,7 @@ CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urg -- <%= GH_AUTHOR %> <%= Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S %z") %> -''' +' "" # Override Kernel.warn def warn(msg) @@ -54,50 +54,50 @@ end def client(host = API_HOST, port = API_PORT) @http ||= begin - c = Net::HTTP.new(host, port) - c.use_ssl = true - c - end + c = Net::HTTP.new(host, port) + c.use_ssl = true + c + end end def get(path) req = Net::HTTP::Get.new(path) - req['Authorization'] = "token #{release_token}" + req["Authorization"] = "token #{release_token}" client.request(req) end def post(path, body) req = Net::HTTP::Post.new(path) - req['Authorization'] = "token #{release_token}" + req["Authorization"] = "token #{release_token}" req.body = body client.request(req) end def post_file(path, body) req = Net::HTTP::Post.new(path) - req['Authorization'] = "token #{release_token}" - req['Content-Type'] = path.match?(/.*\.tar\.gz$/) ? 'application/tar+gzip' : 'application/vnd.debian.binary-package' + req["Authorization"] = "token #{release_token}" + req["Content-Type"] = path.match?(/.*\.tar\.gz$/) ? "application/tar+gzip" : "application/vnd.debian.binary-package" req.body = body client.request(req) end def put(path, body) req = Net::HTTP::Put.new(path) - req['Authorization'] = "token #{release_token}" + req["Authorization"] = "token #{release_token}" req.body = body client.request(req) end def patch(path, body) req = Net::HTTP::Patch.new(path) - req['Authorization'] = "token #{release_token}" + req["Authorization"] = "token #{release_token}" req.body = body client.request(req) end def release_token - token = ENV['GH_RELEASE_TOKEN'] - raise 'GH_RELEASE_TOKEN environment variable not set' if token.nil? + token = ENV["GH_RELEASE_TOKEN"] + raise "GH_RELEASE_TOKEN environment variable not set" if token.nil? token end @@ -106,7 +106,7 @@ end def tag(name, sha) body = { "ref": "refs/tags/#{name}", - "sha": sha + "sha": sha, }.to_json res = post("/repos/#{GH_OWNER}/#{GH_REPO}/git/refs", body) @@ -114,7 +114,7 @@ def tag(name, sha) end def bug_or_feature?(issue_hash) - return true if issue_hash['labels'].find { |label| ['bug', 'feature', 'enhancement'].include?(label['name']) } + return true if issue_hash["labels"].find { |label| ["bug", "feature", "enhancement"].include?(label["name"]) } false end @@ -131,7 +131,7 @@ def beautify_changes(changes) next unless chg =~ /#(\d+)/ begin issue = issue_from Regexp.last_match(1) - out << "#{issue['title']} ##{Regexp.last_match(1)}" if bug_or_feature?(issue) + out << "#{issue["title"]} ##{Regexp.last_match(1)}" if bug_or_feature?(issue) rescue => e warn "Warning: #{e.message}" end @@ -143,18 +143,18 @@ end def changelog puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}" changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) - raise 'Building the changelog failed' if $CHILD_STATUS != 0 + raise "Building the changelog failed" if $CHILD_STATUS != 0 changes end def build_changelog(changes, package_name, package_version) - ERB.new(CHANGELOG_TMPL, nil, '-').result(binding) + ERB.new(CHANGELOG_TMPL, nil, "-").result(binding) end -def update_changelog(changes, name, version, path = 'debian/changelog') - raise 'debian/changelog not found' unless File.exist?(path) - File.open("#{path}.new", 'w') do |f| +def update_changelog(changes, name, version, path = "debian/changelog") + raise "debian/changelog not found" unless File.exist?(path) + File.open("#{path}.new", "w") do |f| f.puts build_changelog changes, name, version f.puts(File.read(path)) end @@ -168,7 +168,7 @@ def create_release(tag_name, branch, rel_name, rel_body, draft = true) 'name': rel_name, 'body': rel_body, 'draft': draft, - 'prerelease': false + 'prerelease': false, }.to_json res = post("/repos/#{GH_OWNER}/#{GH_REPO}/releases", body) @@ -179,7 +179,7 @@ end def publish_release(release_id) body = { - 'draft': false + 'draft': false, }.to_json res = patch("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}", body) @@ -188,33 +188,33 @@ end def list_releases res = get("/repos/#{GH_OWNER}/#{GH_REPO}/releases") - raise 'Failed to retrieve releases' unless res.is_a? Net::HTTPSuccess + raise "Failed to retrieve releases" unless res.is_a? Net::HTTPSuccess JSON.parse(res.body) end def release_available?(tag_name) - return true if list_releases.find { |r| r['tag_name'] == tag_name } + return true if list_releases.find { |r| r["tag_name"] == tag_name } false end -def bump_version(new_version, min_version = nil, path = 'share/github-backup-utils/version') +def bump_version(new_version, min_version = nil, path = "share/github-backup-utils/version") current_version = Gem::Version.new(File.read(path).strip.chomp) if !@skip_version_bump_check && (Gem::Version.new(new_version) < current_version) raise "New version should be newer than #{current_version}" end - File.open("#{path}.new", 'w') { |f| f.puts new_version } + File.open("#{path}.new", "w") { |f| f.puts new_version } File.rename("#{path}.new", path) unless min_version.nil? - content = File.read('bin/ghe-host-check') + content = File.read("bin/ghe-host-check") new_content = content.gsub(/supported_minimum_version="[0-9]\.[0-9]+\.0"/, "supported_minimum_version=\"#{min_version}\"") - File.open('bin/ghe-host-check', 'w') {|file| file.puts new_content } + File.open("bin/ghe-host-check", "w") { |file| file.puts new_content } - content = File.read('test/testlib.sh') - new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/,"GHE_TEST_REMOTE_VERSION:=#{new_version}") - File.open('test/testlib.sh', 'w') {|file| file.puts new_content } + content = File.read("test/testlib.sh") + new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/, "GHE_TEST_REMOTE_VERSION:=#{new_version}") + File.open("test/testlib.sh", "w") { |file| file.puts new_content } end end @@ -223,7 +223,7 @@ def push_release_branch(version) raise "Creating release branch failed:\n\n#{out}" end - unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/cibuild`) + unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/test`) raise "Error committing changelog and version:\n\n#{out}" end @@ -247,7 +247,7 @@ def create_release_pr(version, release_body) 'title': "Bump version: #{version}", 'body': release_body, 'head': "release-#{version}", - 'base': GH_BASE_BRANCH + 'base': GH_BASE_BRANCH, }.to_json res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body) raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess @@ -260,7 +260,7 @@ def merge_pr(number, sha, version) 'commit_title': "Merge pull request ##{number} from github/release-#{version}", 'commit_message': "Bump version: #{version}", 'sha': sha, - 'merge_method': 'merge' + 'merge_method': "merge", }.to_json pr_mergeable? number res = put("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}/merge", body) @@ -276,19 +276,19 @@ def pr_mergeable?(number) begin retries ||= 5 res = get("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}") - raise RetryError if JSON.parse(res.body)['mergeable'].nil? - mergeable = JSON.parse(res.body)['mergeable'] + raise RetryError if JSON.parse(res.body)["mergeable"].nil? + mergeable = JSON.parse(res.body)["mergeable"] rescue RetryError sleep 1 retry unless (retries -= 1).zero? - raise 'PR is unmergable.' + raise "PR is unmergable." end mergeable || false end def can_auth? - !ENV['GH_RELEASE_TOKEN'].nil? + !ENV["GH_RELEASE_TOKEN"].nil? end def repo_exists? @@ -297,7 +297,7 @@ def repo_exists? end def can_build_deb? - system('which debuild > /dev/null 2>&1') + system("which debuild > /dev/null 2>&1") end def package_tarball @@ -316,7 +316,7 @@ end def attach_assets_to_release(upload_url, release_id, files) @http = nil - client(URI(upload_url.gsub(/{.*}/, '')).host) + client(URI(upload_url.gsub(/{.*}/, "")).host) begin files.each do |file| raw_file = File.open(file).read @@ -373,22 +373,22 @@ if $PROGRAM_NAME == __FILE__ args = ARGV.dup dry_run = false skip_version_bump_check = false - if args.include?('--dry-run') + if args.include?("--dry-run") dry_run = true - args.delete '--dry-run' + args.delete "--dry-run" end - if args.include?('--no-warn') + if args.include?("--no-warn") @no_warn = true - args.delete '--no-warn' + args.delete "--no-warn" end - if args.include?('--skip-version-bump-check') + if args.include?("--skip-version-bump-check") @skip_version_bump_check = true - args.delete '--skip-version-bump-check' + args.delete "--skip-version-bump-check" end - raise 'Usage: release [--dry-run] [--skip-version-bump-check] [min_version]' if args.empty? + raise "Usage: release [--dry-run] [--skip-version-bump-check] [min_version]" if args.empty? begin version = Gem::Version.new(args[0]) @@ -401,7 +401,7 @@ if $PROGRAM_NAME == __FILE__ raise "The repo #{GH_REPO} does not exist for #{GH_OWNER}" unless repo_exists? - raise 'GH_AUTHOR environment variable is not set' if GH_AUTHOR.nil? + raise "GH_AUTHOR environment variable is not set" if GH_AUTHOR.nil? release_changes = [] release_changes = beautify_changes changelog if can_auth? @@ -418,11 +418,11 @@ if $PROGRAM_NAME == __FILE__ puts "Owner: #{GH_OWNER}" puts "Repo: #{GH_REPO}" puts "Author: #{GH_AUTHOR}" - puts "Token: #{ENV['GH_RELEASE_TOKEN'] && 'set' || 'unset'}" + puts "Token: #{ENV["GH_RELEASE_TOKEN"] && "set" || "unset"}" puts "Base branch: #{GH_BASE_BRANCH}" - puts 'Changelog:' + puts "Changelog:" if release_changes.empty? - puts ' => No new bug fixes, enhancements or features.' + puts " => No new bug fixes, enhancements or features." else release_changes.each { |c| puts " * #{c}" } end @@ -442,12 +442,12 @@ if $PROGRAM_NAME == __FILE__ branches = `git branch --all | grep release-#{version}$` unless branches.empty? out = "Release branch release-#{version} already exists. " - out += 'Branches found:' + out += "Branches found:" branches.each_line { |l| out += "\n* #{l.strip.chomp}" } raise out end - puts 'Updating changelog...' + puts "Updating changelog..." update_changelog release_changes, DEB_PKG_NAME, version release_body = "Includes general improvements & bug fixes" release_body += " and support for GitHub Enterprise Server v#{version}" unless min_version.nil? @@ -455,40 +455,40 @@ if $PROGRAM_NAME == __FILE__ release_body += "\n* #{c}" end - puts 'Pushing release branch and creating release PR...' + puts "Pushing release branch and creating release PR..." push_release_branch version res = create_release_pr(version, "#{release_body}\n\n/cc @github/backup-utils") - puts 'Merging release PR...' - res = merge_pr res['number'], res['head']['sha'], version + puts "Merging release PR..." + res = merge_pr res["number"], res["head"]["sha"], version - puts 'Tagging and publishing release...' - tag "v#{version}", res['sha'] + puts "Tagging and publishing release..." + tag "v#{version}", res["sha"] - puts 'Creating release...' + puts "Creating release..." release_title = "GitHub Enterprise Server Backup Utilities v#{version}" res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true # Tidy up before building tarball and deb pkg clean_up version - puts 'Building release tarball...' + puts "Building release tarball..." package_tarball - puts 'Building Debian pkg...' + puts "Building Debian pkg..." package_deb - puts 'Attaching Debian pkg and tarball to release...' - base_dir = File.expand_path(File.join(File.dirname(__FILE__), '..')) - attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] - attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] + puts "Attaching Debian pkg and tarball to release..." + base_dir = File.expand_path(File.join(File.dirname(__FILE__), "..")) + attach_assets_to_release res["upload_url"], res["id"], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] + attach_assets_to_release res["upload_url"], res["id"], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] if PUBLISH - puts 'Publishing release...' - publish_release res['id'] + puts "Publishing release..." + publish_release res["id"] end - puts 'Cleaning up...' + puts "Cleaning up..." clean_up version puts "Updating #{GH_STABLE_BRANCH} branch..." @@ -498,7 +498,7 @@ if $PROGRAM_NAME == __FILE__ puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' end - puts 'Released!' + puts "Released!" rescue RuntimeError => e $stderr.puts "Error: #{e}" exit 1 diff --git a/script/test b/script/test new file mode 100755 index 000000000..d30bfa8df --- /dev/null +++ b/script/test @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Usage: script/test +set -e + +ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +TMPDIR="$ROOTDIR/test/tmp" + +# Remove possible remnants of previous test runs +rm -rf "${TMPDIR:?}/*" + +print_test_results() { + if [ -n "$GITHUB_STEP_SUMMARY" ]; then + echo -e "### Test results\n" >> "$GITHUB_STEP_SUMMARY" + echo "| Test suite | Result | Successful | Failed | Skipped | Duration |" >> "$GITHUB_STEP_SUMMARY" + echo "|---|---|--:|--:|--:|--:|" >> "$GITHUB_STEP_SUMMARY" + sort -V "$TMPDIR/results" >> "$GITHUB_STEP_SUMMARY" + fi +} + +# Enable verbose logging of ssh commands +export GHE_VERBOSE_SSH=true + +if ! find test -name "test-*.sh" -print0 | sort -z |xargs -0 -n 1 /bin/bash; then + print_test_results + exit 1 +fi + +print_test_results From 42cc0cb5ff431b83d9d833f1cfdfdb609e640b27 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Wed, 20 Sep 2023 15:50:17 +0200 Subject: [PATCH 163/410] removing wrong error output (#591) * removing wrong error output * fix host-check output * refactor echo --------- Co-authored-by: Chuck Pathanjali --- bin/ghe-host-check | 30 ++++++++++----------- share/github-backup-utils/ghe-backup-config | 5 ++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 9e8d6466b..adb3b0ccb 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -151,7 +151,7 @@ if [ -z "$supported" ]; then fi if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then - cat << SKIP_MSG 1>&2 + cat << SKIP_MSG **You can disable the following storage & version checks by running ghe-backup with option "--skip-checks" OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file. @@ -176,10 +176,10 @@ SKIP_MSG fi #Display dir requirements for repositories and mysql - echo "" 1>&2 - echo "Checking host for sufficient space for a backup..." 1>&2 + echo -e "\n" + echo "Checking host for sufficient space for a backup..." available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') - echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." 1>&2 + echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." repos_disk_size=$(transfer_size repositories /tmp) pages_disk_size=$(transfer_size pages /tmp) @@ -192,10 +192,10 @@ SKIP_MSG min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size)) recommended_disk_req=$((min_disk_req * 5)) - echo " - Available space: $((available_space / (1024 ** 2))) MB" 1>&2 - echo " - Min Disk required for this backup is at least $min_disk_req MB" 1>&2 - echo " - Recommended Disk requirement is $recommended_disk_req MB" 1>&2 - echo "" 1>&2 + echo " - Available space: $((available_space / (1024 ** 2))) MB" + echo " - Min Disk required for this backup is at least $min_disk_req MB" + echo " - Recommended Disk requirement is $recommended_disk_req MB" + echo -e "\n" printf '### Estimated Data Transfer Sizes @@ -208,7 +208,7 @@ SKIP_MSG - actions: %d MB - mssql: %d MB \n' \ - "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" 1>&2 + "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2 @@ -231,7 +231,7 @@ SKIP_MSG exit 1 fi - echo "### Software versions" 1>&2 + echo "### Software versions" rsync_version=$(rsync --version | grep 'version' | awk '{print $3}' | tr -cd '[:digit:].\n') if awk "BEGIN {exit !($rsync_version < $min_rsync)}" &> /dev/null; then echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2 @@ -241,9 +241,9 @@ SKIP_MSG printf "\n **WARNING:** rsync version %s on backup host is less than 3.2.5, which could result in performance degradation. For more details, please read documentation at https://gh.io/april-2023-update-of-rsync-requirements You can disable this warning by changing RSYNC_WARNING to 'no' in your backup.config file.\n\n" \ - "$rsync_version" 1>&2 + "$rsync_version" fi - echo " - rsync ${rsync_version} >= required ($min_rsync)" 1>&2 + echo " - rsync ${rsync_version} >= required ($min_rsync)" ssh_version=$(ssh -V 2>&1 | awk '{print $1}'|grep -oPm 1 '[\d\.]+' |head -1 | tr -cd '[:digit:].\n') if awk "BEGIN {exit !($ssh_version < $min_openssh)}" &> /dev/null; then @@ -251,7 +251,7 @@ SKIP_MSG echo "Please make sure the minimum required version of openSSH: $min_openssh is installed" 1>&2 exit 1 else - echo " - openSSH ${ssh_version} >= required ($min_openssh)" 1>&2 + echo " - openSSH ${ssh_version} >= required ($min_openssh)" fi jq_version=$(jq --version |awk -F\- '{print $2}' | tr -cd '[:digit:].\n') @@ -260,8 +260,8 @@ SKIP_MSG echo "Please make sure you have the minimum required version of jq: $min_jq installed" 1>&2 exit 1 else - echo " - jq ${jq_version} >= required ($min_jq)" 1>&2 + echo " - jq ${jq_version} >= required ($min_jq)" fi fi -echo "" 1>&2 +echo -e "\n" echo "Connect $hostname:$port OK (v$version)" diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index d3f8b05aa..790d2aaa4 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -422,14 +422,15 @@ ghe_remote_version_required() { if [ -z "$GHE_REMOTE_VERSION" ]; then _out=$(ghe-host-check "$@") echo "$_out" + _out_hostname=$(echo "$_out" | tail -n 1) # override hostname w/ ghe-host-check output because the port could have # been autodetected to 122. - GHE_HOSTNAME="${_out/Connect /}" + GHE_HOSTNAME="${_out_hostname/Connect /}" GHE_HOSTNAME="${GHE_HOSTNAME/ OK*/}" export GHE_HOSTNAME - GHE_REMOTE_VERSION="${_out#*\(}" + GHE_REMOTE_VERSION="${_out_hostname#*\(}" GHE_REMOTE_VERSION="${GHE_REMOTE_VERSION%%\)*}" export GHE_REMOTE_VERSION From b3be0d90b11a589050575d164b6b260ca3715fde Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 20 Sep 2023 10:44:46 -0400 Subject: [PATCH 164/410] use checkout v4 --- .github/workflows/build-and-release.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 487f808f2..a9d28ee09 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,20 +24,14 @@ jobs: app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} - name: Install dependencies run: | sudo apt-get update -y sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create patch release branch from current sha - run: | - git config user.name "release-controller" - git config user.email "ghes-releases-team@github.com" - git branch "enterprise-${{ github.event.inputs.version }}-release" - git push origin "enterprise-${{ github.event.inputs.version }}-release" - # create v#{version} tag on current sha + # create tags for the build scripts to work - name: Create tag run: | git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" @@ -80,8 +74,8 @@ jobs: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - - name: Checkout - uses: actions/checkout@v2 + - name: Checkout backup-utils + uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils @@ -91,14 +85,13 @@ jobs: run: | git config user.name "release-controller" git config user.email "ghes-releases-team@github.com" - # tims-test-branch already exists, so fetch it git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Checkout - uses: actions/checkout@v2 + - name: Checkout backup-utils-private + uses: actions/checkout@v4 - name: Download deb artifact uses: actions/download-artifact@v3 with: From 91905dd52d3a43fa2afc1c734a8438d8af00ad03 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 20 Sep 2023 10:49:45 -0400 Subject: [PATCH 165/410] set git config --- .github/workflows/build-and-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a9d28ee09..cdc823455 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -34,6 +34,8 @@ jobs: # create tags for the build scripts to work - name: Create tag run: | + git config user.name "release-controller[bot]" + git config user.email "ghes-releases-team@github.com" git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" git push origin "v${{ github.event.inputs.version }}" - name: Package deb From 0b232188e513c2acfeffbb230f036a5f791fbedf Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Wed, 20 Sep 2023 11:23:32 -0400 Subject: [PATCH 166/410] Just a quick echo fix (#594) Just a quick fix to echo statement to clean it up. --- bin/ghe-host-check | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index adb3b0ccb..5cd39abee 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -176,8 +176,7 @@ SKIP_MSG fi #Display dir requirements for repositories and mysql - echo -e "\n" - echo "Checking host for sufficient space for a backup..." + echo -e "\nChecking host for sufficient space for a backup..." available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." @@ -194,8 +193,7 @@ SKIP_MSG recommended_disk_req=$((min_disk_req * 5)) echo " - Available space: $((available_space / (1024 ** 2))) MB" echo " - Min Disk required for this backup is at least $min_disk_req MB" - echo " - Recommended Disk requirement is $recommended_disk_req MB" - echo -e "\n" + echo -e " - Recommended Disk requirement is $recommended_disk_req MB\n" printf '### Estimated Data Transfer Sizes @@ -263,5 +261,5 @@ SKIP_MSG echo " - jq ${jq_version} >= required ($min_jq)" fi fi -echo -e "\n" -echo "Connect $hostname:$port OK (v$version)" + +echo -e "\nConnect $hostname:$port OK (v$version)" From fb356cc6897682c5a693c97d8c7326f0f12991ca Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 20 Sep 2023 16:52:30 +0000 Subject: [PATCH 167/410] try getting issues --- .github/workflows/build-and-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index cdc823455..a5185f63e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -82,6 +82,11 @@ jobs: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master + - name: Try getting issues from backkup-utils + run: | + curl -H "Authorization: token ${{ steps.app-token.outputs.token }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/github/backup-utils/issues - name: Create empty commit id: empty_commit run: | From 5070734353e4bf462f0e1c0bac9294aa9aef6796 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Wed, 20 Sep 2023 13:47:51 -0600 Subject: [PATCH 168/410] Set enterprise2 branch as env var --- .github/workflows/integration-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 804c6d5c5..8727233ea 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -36,6 +36,7 @@ jobs: with: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' - branch-name: '${{ env.TARGET_BRANCH }}' + branch-name: '${{ env.SOURCE_BRANCH }}' force : "true" - envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }}" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From 0e42379f95e2a290ae61938a5e6821839aed3e6d Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Wed, 20 Sep 2023 22:50:01 -0400 Subject: [PATCH 169/410] Add redis restart mechanism (#583) --- bin/ghe-restore | 19 +++++++++++++++++++ test/bin/ghe-nomad-jobs | 7 +++++++ 2 files changed, 26 insertions(+) create mode 100755 test/bin/ghe-nomad-jobs diff --git a/bin/ghe-restore b/bin/ghe-restore index 17a12b8b1..078932236 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -636,6 +636,25 @@ echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh bm_end "$(basename $0) - Restarting memcached" +# Restart redis before updating keys +# It's possible that redis hcl is not rendered on an unconfigured system +# so check first +if $instance_configured; then + log_info "Getting redis status before restart..." 1>&3 + echo "nomad status redis" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 + log_info "Restarting redis" 1>&3 + echo "nomad stop redis" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 2>&3 + ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-jobs queue /etc/nomad-jobs/redis/redis.hcl" 1>&3 2>&3 + if $? -gt 0; then + log_error "Unable to restart redis" + fi + log_info "Getting redis status after restart..." 1>&3 + echo "nomad status redis" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 +fi + # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. if ! $RESTORE_SETTINGS; then diff --git a/test/bin/ghe-nomad-jobs b/test/bin/ghe-nomad-jobs new file mode 100755 index 000000000..a91afc7be --- /dev/null +++ b/test/bin/ghe-nomad-jobs @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Usage: ghe-nomad-jobs +# Emulates the remote GitHub ghe-nomad-jobs command. Tests use this +# to assert that the command was executed. +set -e +echo "$(basename $0)" "'$(cat)'" "OK" + From 63b4e2913a75c5dd8d513ac08b604753f25f02a9 Mon Sep 17 00:00:00 2001 From: alejndr0 Date: Thu, 21 Sep 2023 09:07:11 +0000 Subject: [PATCH 170/410] Add GHE_RSYNC_COMPRESSION_ENABLED option --- backup.config-example | 4 ++++ share/github-backup-utils/ghe-rsync | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 688cc374b..347764b2a 100644 --- a/backup.config-example +++ b/backup.config-example @@ -58,6 +58,10 @@ GHE_NUM_SNAPSHOTS=10 # #GHE_EXTRA_RSYNC_OPTS="" +# If set to 'yes', rsync will be set to use compression during backups and restores transfers. Defaults to 'no'. +# +#GHE_RSYNC_COMPRESSION_ENABLED=yes + # If enabled and set to 'no', rsync warning message during backups will be suppressed. #RSYNC_WARNING=no diff --git a/share/github-backup-utils/ghe-rsync b/share/github-backup-utils/ghe-rsync index d957068ac..d1aa5316c 100755 --- a/share/github-backup-utils/ghe-rsync +++ b/share/github-backup-utils/ghe-rsync @@ -29,6 +29,13 @@ if [ "$($( dirname "${BASH_SOURCE[0]}" )/ghe-rsync-feature-checker --trust-sende parameters=("--trust-sender" "${parameters[@]}") fi +# This adds `--compress` to the parameters if supported by the current version of rsync +# shellcheck source=share/github-backup-utils/ghe-rsync-feature-checker +# shellcheck disable=SC2046 +if [ "$($( dirname "${BASH_SOURCE[0]}" )/ghe-rsync-feature-checker --compress)" == "true" ] && [ "$GHE_RSYNC_COMPRESSION_ENABLED" = "yes" ]; then + parameters+=("--compress") +fi + # This loads the $GHE_EXTRA_RSYNC_OPTS from the config file if available then adds them # to the parameters and skip adding if already present in the parameters # shellcheck source=share/github-backup-utils/ghe-rsync-feature-checker @@ -41,7 +48,7 @@ if [ -n "$GHE_EXTRA_RSYNC_OPTS" ]; then done fi - +echo "Running rsync with parameters: ${parameters[*]}" ignore_out='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' rsync_version_check=$(rsync --version | grep -E "version 3.[0-9]*.[0-9]*") if [ -n "$rsync_version_check" ]; then From c889d7b0a3ad1cb9a438ce9acab3874947f53734 Mon Sep 17 00:00:00 2001 From: alejndr0 Date: Thu, 21 Sep 2023 09:08:13 +0000 Subject: [PATCH 171/410] remove debugging line --- share/github-backup-utils/ghe-rsync | 1 - 1 file changed, 1 deletion(-) diff --git a/share/github-backup-utils/ghe-rsync b/share/github-backup-utils/ghe-rsync index d1aa5316c..50a90fba8 100755 --- a/share/github-backup-utils/ghe-rsync +++ b/share/github-backup-utils/ghe-rsync @@ -48,7 +48,6 @@ if [ -n "$GHE_EXTRA_RSYNC_OPTS" ]; then done fi -echo "Running rsync with parameters: ${parameters[*]}" ignore_out='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' rsync_version_check=$(rsync --version | grep -E "version 3.[0-9]*.[0-9]*") if [ -n "$rsync_version_check" ]; then From 7f2ee81788367b60ab8a21eda5365753c0257e3e Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 21 Sep 2023 09:52:27 -0600 Subject: [PATCH 172/410] Fix base ref --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8727233ea..c7ccf30f0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -15,8 +15,8 @@ on: # Get target and source branch from different variables depending on how it was triggered env: - TARGET_BRANCH: '${{ github.event.inputs.target-branch }}${{ github.base_ref || github.ref_name }}' - SOURCE_BRANCH: '${{ github.event.inputs.source-branch }}${{ github.head_ref || github.ref_name }}' + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' jobs: integration-tests: From c8570b3328569977b0789f9e65eaf987d50f4408 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 21 Sep 2023 09:55:18 -0600 Subject: [PATCH 173/410] Add two cluster tests to nightly runs --- .github/workflows/nightly-tests.yml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/nightly-tests.yml diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml new file mode 100644 index 000000000..6215a7f65 --- /dev/null +++ b/.github/workflows/nightly-tests.yml @@ -0,0 +1,33 @@ +name: Long Running Nightly Tests +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +# Get target and source branch from different variables depending on how it was triggered +env: + TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' + SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' + +jobs: + integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ github.ref_name }}' + force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ github.ref_name }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ github.ref_name }}" From c91302f10768dbf13d3c790656bd5ada8f58f5cb Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 21 Sep 2023 09:56:41 -0600 Subject: [PATCH 174/410] Revert "Add two cluster tests to nightly runs" This reverts commit c8570b3328569977b0789f9e65eaf987d50f4408. --- .github/workflows/nightly-tests.yml | 33 ----------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/nightly-tests.yml diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml deleted file mode 100644 index 6215a7f65..000000000 --- a/.github/workflows/nightly-tests.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Long Running Nightly Tests -on: - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -# Get target and source branch from different variables depending on how it was triggered -env: - TARGET_BRANCH: '${{ github.event.inputs.target-branch || github.base_ref }}' - SOURCE_BRANCH: '${{ github.event.inputs.source-branch || github.head_ref }}' - -jobs: - integration-tests: - runs-on: ubuntu-latest - strategy: - matrix: - jankyJobName: - - enterprise2-backup-utils-cluster-binary-backup - - enterprise2-backup-utils-cluster-migration - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - name: Queue ${{ matrix.jankyJobName }} build - uses: ./.github/actions/proxy-janky-build - id: proxy-janky-build - with: - janky-token: '${{ secrets.API_AUTH_TOKEN }}' - job-name: '${{ matrix.jankyJobName }}' - branch-name: '${{ github.ref_name }}' - force : "true" - # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) - envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ github.ref_name }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ github.ref_name }}" From 2591f7afce1db597f7464d45687f9b49b9ff78a2 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Thu, 21 Sep 2023 21:28:23 -0400 Subject: [PATCH 175/410] Fix tests on redis restart (#600) previous pr had a typo causing issues on cluster --- bin/ghe-restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 078932236..f7f092e23 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -647,7 +647,7 @@ if $instance_configured; then echo "nomad stop redis" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 2>&3 ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-jobs queue /etc/nomad-jobs/redis/redis.hcl" 1>&3 2>&3 - if $? -gt 0; then + if [ "$?" -gt 0 ]; then log_error "Unable to restart redis" fi log_info "Getting redis status after restart..." 1>&3 From 91986af27acc260d0c0d405a989d56cee33659e6 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Fri, 22 Sep 2023 07:31:50 -0400 Subject: [PATCH 176/410] reverting changes to redis restart mechanism Some problems are being caused on cluster binary backup, need to revert until other fixes are in. --- bin/ghe-restore | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index f7f092e23..c9ac20ec2 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -636,24 +636,6 @@ echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh bm_end "$(basename $0) - Restarting memcached" -# Restart redis before updating keys -# It's possible that redis hcl is not rendered on an unconfigured system -# so check first -if $instance_configured; then - log_info "Getting redis status before restart..." 1>&3 - echo "nomad status redis" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 - log_info "Restarting redis" 1>&3 - echo "nomad stop redis" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 2>&3 - ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-jobs queue /etc/nomad-jobs/redis/redis.hcl" 1>&3 2>&3 - if [ "$?" -gt 0 ]; then - log_error "Unable to restart redis" - fi - log_info "Getting redis status after restart..." 1>&3 - echo "nomad status redis" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. From 74ed061df5b6ea835077fa9ec37b0239f22eec7a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 14:57:26 +0000 Subject: [PATCH 177/410] try fork of action --- .github/workflows/build-and-release.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a5185f63e..f8207f466 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,7 +17,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token id: app-token with: # required @@ -70,23 +70,22 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - - uses: actions/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token id: app-token with: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + github/backup-utils + github/backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - - name: Try getting issues from backkup-utils - run: | - curl -H "Authorization: token ${{ steps.app-token.outputs.token }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/github/backup-utils/issues - name: Create empty commit id: empty_commit run: | @@ -99,6 +98,10 @@ jobs: echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Checkout backup-utils-private uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + ref: master - name: Download deb artifact uses: actions/download-artifact@v3 with: From a16fb67fbfa67c1e7e1afa7d374a56a3107396b4 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 15:25:53 +0000 Subject: [PATCH 178/410] the yaml linter line length isn't adding value here --- .github/linters/.yaml-lint.yml | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/linters/.yaml-lint.yml diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..bacf1f80c --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,55 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: + level: warning + present: true + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: disable + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable \ No newline at end of file From 9881f9c8c4bd2e8589ac580bd3d9397db019374a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 15:26:15 +0000 Subject: [PATCH 179/410] ignore release-notes markdown --- .github/workflows/build-and-release.yml | 1 + .github/workflows/lint.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f8207f466..9cd0141a5 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -1,3 +1,4 @@ +--- name: Build and Release on: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0c6af658b..e818ad8ac 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,3 +20,4 @@ jobs: VALIDATE_ALL_CODEBASE: false BASH_SEVERITY: error GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FILTER_REGEX_EXCLUDE: .*release-notes/.* From aa11551a0e02522b7f19b239399ac126eef66155 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 15:27:43 +0000 Subject: [PATCH 180/410] linter fix --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 9cd0141a5..a22bce6d9 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -18,7 +18,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token + - uses: timreimherr/create-github-app-token@v1 id: app-token with: # required @@ -71,7 +71,7 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - - uses: timreimherr/create-github-app-token + - uses: timreimherr/create-github-app-token@v1 id: app-token with: # required From 348aec0b0157c864eed0a8b68d78c32597683f17 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 15:37:23 +0000 Subject: [PATCH 181/410] lint error --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e818ad8ac..7fc33e1cd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,3 +1,4 @@ +--- name: Lint Code Base on: From 360bcc3492bb06f7f99931c4ae2ccd36ccee8096 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 15:43:57 +0000 Subject: [PATCH 182/410] linters are so fun --- .github/workflows/build-and-release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a22bce6d9..285b505e7 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -90,13 +90,13 @@ jobs: - name: Create empty commit id: empty_commit run: | - git config user.name "release-controller" - git config user.email "ghes-releases-team@github.com" - git fetch origin tims-test-branch - git checkout tims-test-branch - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + "git config user.name 'release-controller'" + "git config user.email 'ghes-releases-team@github.com'" + "git fetch origin tims-test-branch" + "git checkout tims-test-branch" + "git commit --allow-empty -m '${{ github.event.inputs.version }} release'" + "git push origin tims-test-branch" + "echo 'commit-sha=$(git rev-parse HEAD)' >> $GITHUB_OUTPUT" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From 38dd4bb6a6cc0f0f9a1bb09327a25e4fcaeb5106 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 22 Sep 2023 09:49:58 -0600 Subject: [PATCH 183/410] Add cluster run as optional runs --- .github/workflows/integration-tests.yml | 27 ++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c7ccf30f0..f48de957e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,31 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' - force : "true" + # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) + envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" + + # Cluster integration tests are optional based on label and PR titles + cluster-integration-tests: + runs-on: ubuntu-latest + strategy: + matrix: + jankyJobName: + - enterprise2-backup-utils-cluster-binary-backup + - enterprise2-backup-utils-cluster-migration + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Queue ${{ matrix.jankyJobName }} build + if: | + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.title, '[cluster]') || + contains(github.event.pull_request.labels.*.name, 'cluster') + uses: ./.github/actions/proxy-janky-build + id: proxy-janky-build + with: + janky-token: '${{ secrets.API_AUTH_TOKEN }}' + job-name: '${{ matrix.jankyJobName }}' + branch-name: '${{ env.SOURCE_BRANCH }}' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From 9b9617d310dba83b68a043fd0463bef3e1126ba1 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 22 Sep 2023 10:09:38 -0600 Subject: [PATCH 184/410] Update PR template --- .github/pull_request_template.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fd36d8a9a..ed5e1c26c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,17 @@ + + + + # PR Details ### Description From aa306776a81e66079e6cc480aeaf3c1a533dde22 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 16:27:15 +0000 Subject: [PATCH 185/410] I hate this linter --- .github/workflows/build-and-release.yml | 8 +------- script/create-emtpy-commit | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 script/create-emtpy-commit diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 285b505e7..f180bd6f5 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -90,13 +90,7 @@ jobs: - name: Create empty commit id: empty_commit run: | - "git config user.name 'release-controller'" - "git config user.email 'ghes-releases-team@github.com'" - "git fetch origin tims-test-branch" - "git checkout tims-test-branch" - "git commit --allow-empty -m '${{ github.event.inputs.version }} release'" - "git push origin tims-test-branch" - "echo 'commit-sha=$(git rev-parse HEAD)' >> $GITHUB_OUTPUT" + ./script/create-empty-commit - name: Checkout backup-utils-private uses: actions/checkout@v4 with: diff --git a/script/create-emtpy-commit b/script/create-emtpy-commit new file mode 100644 index 000000000..7f69449a9 --- /dev/null +++ b/script/create-emtpy-commit @@ -0,0 +1,14 @@ +#!/ust/bin/env bash +# Usage: script/create-empty-commit +# Script to create an empty commit on the current branch. +# This is used to create a new tag for a release in github/backup-utils. +# This is to avoid stacking multiple release tags on the same commit. +set -e + +git config user.name "release-controller[bot]" +git config user.email "ghes-releases-team@github.com" +git fetch origin tims-test-branch +git checkout tims-test-branch +git commit --allow-empty -m "${{ github.event.inputs.version }} release" +git push origin tims-test-branch +echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT \ No newline at end of file From f387d47da0e6cb5e91321ca2a4ad9324b9057d6a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 18:10:26 +0000 Subject: [PATCH 186/410] remove document start --- .github/linters/.yaml-lint.yml | 4 +--- .github/workflows/build-and-release.yml | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index bacf1f80c..030c37f04 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -31,9 +31,7 @@ rules: comments: disable comments-indentation: disable document-end: disable - document-start: - level: warning - present: true + document-start: disable empty-lines: level: warning max: 2 diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f180bd6f5..17f349f91 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -1,4 +1,3 @@ ---- name: Build and Release on: From d3f9b3d5eb81915942bbe84b031d158fc8b1e510 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 18:13:18 +0000 Subject: [PATCH 187/410] remove action version --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 17f349f91..549bb568b 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,7 +17,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token id: app-token with: # required @@ -70,7 +70,7 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - - uses: timreimherr/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token id: app-token with: # required From 12e11705b870fbb970eaccff0f072daac5ef2f8b Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 18:25:44 +0000 Subject: [PATCH 188/410] try v1 --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 549bb568b..17f349f91 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,7 +17,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token + - uses: timreimherr/create-github-app-token@v1 id: app-token with: # required @@ -70,7 +70,7 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - - uses: timreimherr/create-github-app-token + - uses: timreimherr/create-github-app-token@v1 id: app-token with: # required From 359ef2a15da42f5f28575f7497ac280dc53fc763 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 22 Sep 2023 19:25:02 +0000 Subject: [PATCH 189/410] @main --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 17f349f91..5c3e7a580 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,7 +17,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token@main id: app-token with: # required @@ -70,7 +70,7 @@ jobs: outputs: commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} steps: - - uses: timreimherr/create-github-app-token@v1 + - uses: timreimherr/create-github-app-token@main id: app-token with: # required From 6a965c37cd53d6d440d344648a7061c71910fd09 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 22 Sep 2023 14:03:49 -0600 Subject: [PATCH 190/410] update for linter error --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ed5e1c26c..afcf77749 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -12,7 +12,7 @@ Additional notes regarding CI: - If you are making changes impacts cluster, please add `cluster` label or `[cluster]` in your PR title so it will trigger optional cluster integration test. Those tests will take about 3 hours so relax and come back later to check the results. ;) --> -# PR Details +## PR Details ### Description -## PR Details +# PR Details -### Description +## Description -### Testing +## Testing @@ -31,10 +31,10 @@ Additional notes regarding CI: - Please make sure those versions are tested against for this change --> -### Ownership +## Ownership -### Related Links +## Related Links From f3fb2ebe04675e81908f4bda4987c8b1559be382 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 13:33:48 +0000 Subject: [PATCH 195/410] show me the directory structure after checkout --- .github/workflows/build-and-release.yml | 195 +++++++++++++----------- 1 file changed, 105 insertions(+), 90 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index bf5d44501..5b7e601ec 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -23,63 +23,6 @@ jobs: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - - name: Checkout - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # create tags for the build scripts to work - - name: Create tag - run: | - git config user.name "release-controller[bot]" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} - steps: - - uses: timreimherr/create-github-app-token@main - id: app-token - with: - # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: | - github/backup-utils - github/backup-utils-private - name: Checkout backup-utils-private uses: actions/checkout@v4 with: @@ -93,37 +36,109 @@ jobs: repository: github/backup-utils ref: master path: backup-utils - - name: Create empty commit - id: empty_commit + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Show direcotry structure run: | - cp ./backup-utils-private/scripts/create-empty-commit \ - ./create-empty-commit - cd ./backup-utils - ./create-empty-commit - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ steps.app-token.outputs.token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities - v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - # this action will create a tag with this name on the provided commit - tag: v${{ github.event.inputs.version }} - # this can be a commit hash or branch name - commit: ${{ steps.empty_commit.outputs.commit_sha }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + ls -R + # create tags for the build scripts to work + # - name: Create tag + # run: | + # git config user.name "release-controller[bot]" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} + # steps: + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: | + # github/backup-utils + # github/backup-utils-private + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # path: backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # path: backup-utils + # - name: Show direcotry structure + # run: | + # ls -R + # - name: Create empty commit + # id: empty_commit + # run: | + # cp ./backup-utils-private/scripts/create-empty-commit \ + # ./create-empty-commit + # cd ./backup-utils + # ./create-empty-commit + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities + # v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # # this action will create a tag with this name on the provided commit + # tag: v${{ github.event.inputs.version }} + # # this can be a commit hash or branch name + # commit: ${{ steps.empty_commit.outputs.commit_sha }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From 535b7bdff40f83c1388d101dbb36bd8a655a0e74 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 13:37:06 +0000 Subject: [PATCH 196/410] again --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 5b7e601ec..50ddb0136 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -43,6 +43,7 @@ jobs: - name: Show direcotry structure run: | ls -R + pwd # create tags for the build scripts to work # - name: Create tag # run: | From ea89f9085d1cdaa3a0de86fffe14d352361a66a9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 13:56:25 +0000 Subject: [PATCH 197/410] and again --- .github/workflows/build-and-release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 50ddb0136..2ac76b014 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,6 +44,9 @@ jobs: run: | ls -R pwd + ls -l + cd .. + ls -l # create tags for the build scripts to work # - name: Create tag # run: | From d37a196c1e7d1f13ea6cf16c1aa964e3a9f6c225 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 13:58:21 +0000 Subject: [PATCH 198/410] let me see more --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 2ac76b014..c358a318d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -46,6 +46,7 @@ jobs: pwd ls -l cd .. + pwd ls -l # create tags for the build scripts to work # - name: Create tag From 3e18e6b666da817d827f35df1932a68587d07168 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:06:43 +0000 Subject: [PATCH 199/410] lets try the empty commit again --- .github/workflows/build-and-release.yml | 215 ++++++++++++------------ 1 file changed, 106 insertions(+), 109 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c358a318d..cb9980b9a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -36,114 +36,111 @@ jobs: repository: github/backup-utils ref: master path: backup-utils - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Show direcotry structure + - name: Install dependencies run: | - ls -R - pwd - ls -l - cd .. - pwd - ls -l + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Show direcotry structure + # run: | + # ls -R + # pwd + # ls -l + # cd .. + # pwd + # ls -l # create tags for the build scripts to work - # - name: Create tag - # run: | - # git config user.name "release-controller[bot]" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} - # steps: - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: | - # github/backup-utils - # github/backup-utils-private - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # path: backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # path: backup-utils - # - name: Show direcotry structure - # run: | - # ls -R - # - name: Create empty commit - # id: empty_commit - # run: | - # cp ./backup-utils-private/scripts/create-empty-commit \ - # ./create-empty-commit - # cd ./backup-utils - # ./create-empty-commit - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities - # v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # # this action will create a tag with this name on the provided commit - # tag: v${{ github.event.inputs.version }} - # # this can be a commit hash or branch name - # commit: ${{ steps.empty_commit.outputs.commit_sha }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Create tag + run: | + git config user.name "release-controller[bot]" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} + steps: + - uses: timreimherr/create-github-app-token@main + id: app-token + with: + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + github/backup-utils + github/backup-utils-private + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + path: backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + ref: master + path: backup-utils + - name: Create empty commit in backup-utils + id: empty_commit + run: | + cp ./backup-utils-private/scripts/create-empty-commit \ + .backup-utils/create-empty-commit + cd ./backup-utils + ./create-empty-commit + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities + v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + # this action will create a tag with this name on the provided commit + tag: v${{ github.event.inputs.version }} + # this can be a commit hash or branch name + commit: ${{ steps.empty_commit.outputs.commit_sha }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From c42acb4bd0ebb1dbe64599f7607ba7c0ed27a404 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:08:53 +0000 Subject: [PATCH 200/410] just try it --- .github/workflows/build-and-release.yml | 219 +++++++++++++----------- 1 file changed, 122 insertions(+), 97 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index cb9980b9a..9c4dc6c15 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -23,6 +23,23 @@ jobs: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # path: backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # path: backup-utils + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Checkout backup-utils-private uses: actions/checkout@v4 with: @@ -36,10 +53,13 @@ jobs: repository: github/backup-utils ref: master path: backup-utils - - name: Install dependencies + - name: Create empty commit in backup-utils + id: empty_commit run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip + cp ./backup-utils-private/scripts/create-empty-commit \ + .backup-utils/create-empty-commit + cd ./backup-utils + ./create-empty-commit # - name: Show direcotry structure # run: | # ls -R @@ -49,98 +69,103 @@ jobs: # pwd # ls -l # create tags for the build scripts to work - - name: Create tag - run: | - git config user.name "release-controller[bot]" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} - steps: - - uses: timreimherr/create-github-app-token@main - id: app-token - with: - # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: | - github/backup-utils - github/backup-utils-private - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils-private - path: backup-utils-private - - name: Checkout backup-utils - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils - ref: master - path: backup-utils - - name: Create empty commit in backup-utils - id: empty_commit + # if empty_commit is successful, then echo success + - name: Echo success + if: ${{ steps.empty_commit.outcome == 'success' }} run: | - cp ./backup-utils-private/scripts/create-empty-commit \ - .backup-utils/create-empty-commit - cd ./backup-utils - ./create-empty-commit - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ steps.app-token.outputs.token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities - v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - # this action will create a tag with this name on the provided commit - tag: v${{ github.event.inputs.version }} - # this can be a commit hash or branch name - commit: ${{ steps.empty_commit.outputs.commit_sha }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + echo "empty commit successful" + # - name: Create tag + # run: | + # git config user.name "release-controller[bot]" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} + # steps: + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: | + # github/backup-utils + # github/backup-utils-private + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # path: backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # path: backup-utils + # - name: Create empty commit in backup-utils + # id: empty_commit + # run: | + # cp ./backup-utils-private/scripts/create-empty-commit \ + # .backup-utils/create-empty-commit + # cd ./backup-utils + # ./create-empty-commit + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities + # v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # # this action will create a tag with this name on the provided commit + # tag: v${{ github.event.inputs.version }} + # # this can be a commit hash or branch name + # commit: ${{ steps.empty_commit.outputs.commit_sha }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From cdc459405da0eb750398ff4f2cd2be34119a81bd Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:16:54 +0000 Subject: [PATCH 201/410] try fix --- .github/workflows/build-and-release.yml | 53 ++++++++++++------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 9c4dc6c15..0c2390e52 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -23,51 +23,26 @@ jobs: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # path: backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # path: backup-utils - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - name: Checkout backup-utils-private uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - path: backup-utils-private + path: ./backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - path: backup-utils + path: ./backup-utils - name: Create empty commit in backup-utils id: empty_commit run: | cp ./backup-utils-private/scripts/create-empty-commit \ - .backup-utils/create-empty-commit + ./backup-utils/create-empty-commit cd ./backup-utils ./create-empty-commit - # - name: Show direcotry structure - # run: | - # ls -R - # pwd - # ls -l - # cd .. - # pwd - # ls -l # create tags for the build scripts to work # if empty_commit is successful, then echo success - name: Echo success @@ -169,3 +144,25 @@ jobs: # draft: ${{ github.event.inputs.draft }} # allowUpdates: true # artifactContentType: "raw" + + + + + + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # path: backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # path: backup-utils + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip \ No newline at end of file From b6dc65c83fb5dde02b0b281f048ee1d14ead9bc3 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:20:43 +0000 Subject: [PATCH 202/410] wrong directory name --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 0c2390e52..7e3e20e28 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,7 +39,7 @@ jobs: - name: Create empty commit in backup-utils id: empty_commit run: | - cp ./backup-utils-private/scripts/create-empty-commit \ + cp ./backup-utils-private/script/create-empty-commit \ ./backup-utils/create-empty-commit cd ./backup-utils ./create-empty-commit From 4be206c213d9d792072be3a4fe31d165c6159ad0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:33:09 +0000 Subject: [PATCH 203/410] set scope --- .github/workflows/build-and-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7e3e20e28..c79284aaa 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -23,6 +23,10 @@ jobs: # required app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: | + github/backup-utils + github/backup-utils-private - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From fe290a68b5420e9e1c1e602410d55f71e206ae31 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:34:33 +0000 Subject: [PATCH 204/410] comma delimited --- .github/workflows/build-and-release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c79284aaa..5f77459d9 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,9 +24,7 @@ jobs: app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - repositories: | - github/backup-utils - github/backup-utils-private + repositories: github/backup-utils,github/backup-utils-private - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From ef6329da3f40b484df41332efb4ff9a1f421f5f2 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:36:21 +0000 Subject: [PATCH 205/410] fix repo values --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 5f77459d9..73348cec7 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,7 +24,7 @@ jobs: app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - repositories: github/backup-utils,github/backup-utils-private + repositories: backup-utils,backup-utils-private - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From d39c71ed16919d03ed04a6a9b1054bb7e657547c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:40:08 +0000 Subject: [PATCH 206/410] checkout just backup-utils --- .github/workflows/build-and-release.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 73348cec7..8437c889b 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -25,12 +25,12 @@ jobs: private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils-private - path: ./backup-utils-private + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # path: ./backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: @@ -41,10 +41,13 @@ jobs: - name: Create empty commit in backup-utils id: empty_commit run: | - cp ./backup-utils-private/script/create-empty-commit \ - ./backup-utils/create-empty-commit - cd ./backup-utils - ./create-empty-commit + git config user.name "release-controller[bot]" + git config user.email "ghes-releases-team@github.com" + git fetch origin tims-test-branch + git checkout tims-test-branch + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin tims-test-branch + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # create tags for the build scripts to work # if empty_commit is successful, then echo success - name: Echo success From 99f1a02ac1b8153de2dd5ded30f5b57f1e3eb6f9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:41:35 +0000 Subject: [PATCH 207/410] no path --- .github/workflows/build-and-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8437c889b..76f916836 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -36,8 +36,6 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - path: ./backup-utils - name: Create empty commit in backup-utils id: empty_commit run: | From 8e2c20d91880598e26e60b72f8160db80726708e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:46:13 +0000 Subject: [PATCH 208/410] try setting the token explicitly --- .github/workflows/build-and-release.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 76f916836..dda2e6bf4 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,15 +37,18 @@ jobs: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - name: Create empty commit in backup-utils - id: empty_commit - run: | - git config user.name "release-controller[bot]" - git config user.email "ghes-releases-team@github.com" - git fetch origin tims-test-branch - git checkout tims-test-branch - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + id: empty_commit + env: + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + run: | + git config user.name "release-controller[bot]" + git config user.email "ghes-releases-team@github.com" + git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git + git fetch origin tims-test-branch + git checkout tims-test-branch + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin tims-test-branch + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # create tags for the build scripts to work # if empty_commit is successful, then echo success - name: Echo success From b90f442ffb006c4767358324a745911ef403e266 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:47:49 +0000 Subject: [PATCH 209/410] it can't see the workflow --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index dda2e6bf4..3ca6053ff 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -1,3 +1,4 @@ +--- name: Build and Release on: From 211a3bda7a89ed9fc8bcd28d23fea91fc7093703 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:50:16 +0000 Subject: [PATCH 210/410] disable the linter --- .github/linters/.yaml-lint.yml | 72 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index 030c37f04..0f3e1154b 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -7,47 +7,47 @@ # # yamllint disable-line # ########################################### rules: - braces: - level: warning - min-spaces-inside: 0 - max-spaces-inside: 0 - min-spaces-inside-empty: 1 - max-spaces-inside-empty: 5 - brackets: - level: warning - min-spaces-inside: 0 - max-spaces-inside: 0 - min-spaces-inside-empty: 1 - max-spaces-inside-empty: 5 - colons: - level: warning - max-spaces-before: 0 - max-spaces-after: 1 - commas: - level: warning - max-spaces-before: 0 - min-spaces-after: 1 - max-spaces-after: 1 + braces: disable + # level: warning + # min-spaces-inside: 0 + # max-spaces-inside: 0 + # min-spaces-inside-empty: 1 + # max-spaces-inside-empty: 5 + brackets: disable + # level: warning + # min-spaces-inside: 0 + # max-spaces-inside: 0 + # min-spaces-inside-empty: 1 + # max-spaces-inside-empty: 5 + colons: disable + # level: warning + # max-spaces-before: 0 + # max-spaces-after: 1 + commas: disable + # level: warning + # max-spaces-before: 0 + # min-spaces-after: 1 + # max-spaces-after: 1 comments: disable comments-indentation: disable document-end: disable document-start: disable - empty-lines: - level: warning - max: 2 - max-start: 0 - max-end: 0 - hyphens: - level: warning - max-spaces-after: 1 - indentation: - level: warning - spaces: consistent - indent-sequences: true - check-multi-line-strings: false + empty-lines: disable + # level: warning + # max: 2 + # max-start: 0 + # max-end: 0 + hyphens: disable + # level: warning + # max-spaces-after: 1 + indentation: disable + # level: warning + # spaces: consistent + # indent-sequences: true + # check-multi-line-strings: false key-duplicates: enable line-length: disable new-line-at-end-of-file: disable - new-lines: - type: unix + new-lines: diable + # type: unix trailing-spaces: disable \ No newline at end of file From 649e416c9d3233c759fee61d654ac1fdbb0aee28 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:51:10 +0000 Subject: [PATCH 211/410] ahg --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 3ca6053ff..ee7126e77 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -38,7 +38,7 @@ jobs: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - name: Create empty commit in backup-utils - id: empty_commit + id: empty-commit env: ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} run: | From 806d55063c4be89ea94b851e2808f9396ba1d1c6 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:52:35 +0000 Subject: [PATCH 212/410] fix it --- .github/workflows/build-and-release.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index ee7126e77..1824d8969 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -38,18 +38,18 @@ jobs: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - name: Create empty commit in backup-utils - id: empty-commit - env: - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: | - git config user.name "release-controller[bot]" - git config user.email "ghes-releases-team@github.com" - git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git - git fetch origin tims-test-branch - git checkout tims-test-branch - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + id: empty-commit + env: + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + run: | + git config user.name "release-controller[bot]" + git config user.email "ghes-releases-team@github.com" + git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git + git fetch origin tims-test-branch + git checkout tims-test-branch + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin tims-test-branch + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # create tags for the build scripts to work # if empty_commit is successful, then echo success - name: Echo success From 0f248c559454e3b39e119cc7d12914881f8a6479 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 14:56:45 +0000 Subject: [PATCH 213/410] use access token --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 1824d8969..c89d37454 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -40,7 +40,7 @@ jobs: - name: Create empty commit in backup-utils id: empty-commit env: - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + ACCESS_TOKEN: ${{ steps.app-token.outputs.token }} run: | git config user.name "release-controller[bot]" git config user.email "ghes-releases-team@github.com" From 321efa1a1d32bb9631a2a1907fc58cf19924c9ff Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 18:17:18 +0000 Subject: [PATCH 214/410] try new action --- .github/workflows/build-and-release.yml | 43 ++++++++++++++----------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c89d37454..6e252748d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -26,34 +26,39 @@ jobs: private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # path: ./backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - - name: Create empty commit in backup-utils - id: empty-commit - env: - ACCESS_TOKEN: ${{ steps.app-token.outputs.token }} + # set GITHUB_TOKEN to the app token + - name: Set GITHUB_TOKEN run: | - git config user.name "release-controller[bot]" - git config user.email "ghes-releases-team@github.com" - git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git - git fetch origin tims-test-branch - git checkout tims-test-branch - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> $GITHUB_ENV + - uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: tims-test-branch + commit_message: "${{ github.event.inputs.version }} release" + commit_options: "--allow-empty" + token: ${{ steps.app-token.outputs.token }} + # - name: Create empty commit in backup-utils + # id: empty-commit + # env: + # ACCESS_TOKEN: ${{ steps.app-token.outputs.token }} + # run: | + # git config user.name "release-controller[bot]" + # git config user.email "ghes-releases-team@github.com" + # git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git + # git fetch origin tims-test-branch + # git checkout tims-test-branch + # git commit --allow-empty -m "${{ github.event.inputs.version }} release" + # git push origin tims-test-branch + # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # create tags for the build scripts to work # if empty_commit is successful, then echo success - name: Echo success - if: ${{ steps.empty_commit.outcome == 'success' }} + if: ${{ steps.empty-commit.outcome == 'success' }} run: | echo "empty commit successful" # - name: Create tag From 503bc849c12128f229c551859e57cf0e05f68a10 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 20:23:53 +0000 Subject: [PATCH 215/410] skip dirty check --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6e252748d..1274a3f96 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -41,7 +41,7 @@ jobs: branch: tims-test-branch commit_message: "${{ github.event.inputs.version }} release" commit_options: "--allow-empty" - token: ${{ steps.app-token.outputs.token }} + skip_dirty_check: true # - name: Create empty commit in backup-utils # id: empty-commit # env: From 5525263ba915285967a54ddb1b016b1f87cc2500 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:01:39 +0000 Subject: [PATCH 216/410] another try --- .github/workflows/build-and-release.yml | 103 +++++++++--------------- 1 file changed, 38 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 1274a3f96..59b6d5c68 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,6 +4,10 @@ name: Build and Release on: workflow_dispatch: inputs: + gh-token: + description: 'GitHub Token - used to create a commit in the backup-utils repo' + required: true + type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -31,37 +35,31 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - # set GITHUB_TOKEN to the app token - - name: Set GITHUB_TOKEN - run: | - echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> $GITHUB_ENV - - uses: stefanzweifel/git-auto-commit-action@v4 + ref: master + - name: Create empty commit in backup-utils id: empty-commit - with: - branch: tims-test-branch - commit_message: "${{ github.event.inputs.version }} release" - commit_options: "--allow-empty" - skip_dirty_check: true - # - name: Create empty commit in backup-utils - # id: empty-commit - # env: - # ACCESS_TOKEN: ${{ steps.app-token.outputs.token }} - # run: | - # git config user.name "release-controller[bot]" - # git config user.email "ghes-releases-team@github.com" - # git remote set-url origin https://$ACCESS_TOKEN@github.com/github/backup-utils.git - # git fetch origin tims-test-branch - # git checkout tims-test-branch - # git commit --allow-empty -m "${{ github.event.inputs.version }} release" - # git push origin tims-test-branch - # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - # create tags for the build scripts to work - # if empty_commit is successful, then echo success - - name: Echo success - if: ${{ steps.empty-commit.outcome == 'success' }} run: | - echo "empty commit successful" - # - name: Create tag + git config user.name "timreimherr" + git config user.email "ghes-releases-team@github.com" + git fetch origin tims-test-branch + git checkout tims-test-branch + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin tims-test-branch + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Echo Success if commit was created + if: steps.empty-commit.outputs.commit-sha != '' + run: | + echo "Success" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts # run: | # git config user.name "release-controller[bot]" # git config user.email "ghes-releases-team@github.com" @@ -97,38 +95,31 @@ jobs: # needs: build # runs-on: ubuntu-latest # outputs: - # commit-sha: ${{ steps.empty_commit.outputs.commit_sha }} + # commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} # steps: # - uses: timreimherr/create-github-app-token@main # id: app-token # with: - # # required # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} # owner: ${{ github.repository_owner }} - # repositories: | - # github/backup-utils - # github/backup-utils-private - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # path: backup-utils-private + # repositories: backup-utils,backup-utils-private # - name: Checkout backup-utils # uses: actions/checkout@v4 # with: # token: ${{ steps.app-token.outputs.token }} # repository: github/backup-utils # ref: master - # path: backup-utils # - name: Create empty commit in backup-utils - # id: empty_commit + # id: empty-commit # run: | - # cp ./backup-utils-private/scripts/create-empty-commit \ - # .backup-utils/create-empty-commit - # cd ./backup-utils - # ./create-empty-commit + # git config user.name "release-controller[bot]" + # git config user.email "ghes-releases-team@github.com" + # git fetch origin tims-test-branch + # git checkout tims-test-branch + # git commit --allow-empty -m "${{ github.event.inputs.version }} release" + # git push origin tims-test-branch + # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT # - name: Download deb artifact # uses: actions/download-artifact@v3 # with: @@ -151,7 +142,7 @@ jobs: # # this action will create a tag with this name on the provided commit # tag: v${{ github.event.inputs.version }} # # this can be a commit hash or branch name - # commit: ${{ steps.empty_commit.outputs.commit_sha }} + # commit: ${{ steps.empty-commit.outputs.commit_sha }} # bodyFile: release-notes/${{ github.event.inputs.version }}.md # draft: ${{ github.event.inputs.draft }} # allowUpdates: true @@ -160,21 +151,3 @@ jobs: - - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # path: backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # path: backup-utils - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip \ No newline at end of file From bd8e4627f1ccd630fc66932427703e3e5f5536f1 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:16:33 +0000 Subject: [PATCH 217/410] again --- .github/workflows/build-and-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 59b6d5c68..29b3c42b1 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,7 +44,9 @@ jobs: git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch + # create string of usernma:token for git push + username-token="release-controller[bot]:${{ steps.app-token.outputs.token }}" + git push origin tims-test-branch https://$username-token@github.com/github/backup-utils.git echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From 805ae189c5f4cc5668a462bb56ed5c8346074ea4 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:18:27 +0000 Subject: [PATCH 218/410] try again --- .github/workflows/build-and-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 29b3c42b1..0933cdd3b 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -45,8 +45,7 @@ jobs: git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" # create string of usernma:token for git push - username-token="release-controller[bot]:${{ steps.app-token.outputs.token }}" - git push origin tims-test-branch https://$username-token@github.com/github/backup-utils.git + git push origin tims-test-branch "https://release-controller%5Bbot%5D:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From 465d2e5b85e24758472dff182ded02f606c916e9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:26:53 +0000 Subject: [PATCH 219/410] aaand again --- .github/workflows/build-and-release.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 0933cdd3b..d5ec77ebf 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,13 +39,12 @@ jobs: - name: Create empty commit in backup-utils id: empty-commit run: | - git config user.name "timreimherr" - git config user.email "ghes-releases-team@github.com" + git config user.name "release-controller[bot]" + git config user.email "223695+release-controller[bot]@users.noreply.github.com" git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - # create string of usernma:token for git push - git push origin tims-test-branch "https://release-controller%5Bbot%5D:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" + git push origin tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From 9ece7aec3b05bb97171093826def9c68954ff220 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:29:08 +0000 Subject: [PATCH 220/410] try url --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index d5ec77ebf..bcb927a60 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,7 +44,7 @@ jobs: git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch + git push origin tims-test-branch "https://release-controller%5Bbot%5D:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From 5c42fc25d859052161835da75db6c2ffd6901528 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:42:59 +0000 Subject: [PATCH 221/410] try pat token to create empty commit in backup-utils --- .github/workflows/build-and-release.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index bcb927a60..9425b5f99 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,18 +22,18 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token@main - id: app-token - with: - # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: backup-utils,backup-utils-private + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ steps.app-token.outputs.token }} + token: ${{ github.event.inputs.gh-token }} repository: github/backup-utils ref: master - name: Create empty commit in backup-utils @@ -44,7 +44,7 @@ jobs: git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch "https://release-controller%5Bbot%5D:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" + git push origin tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From f128423668ff5e78619e554fe9c7e9bc6446f1e9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:49:14 +0000 Subject: [PATCH 222/410] try on master --- .github/workflows/build-and-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 9425b5f99..308d7fd6a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -41,8 +41,6 @@ jobs: run: | git config user.name "release-controller[bot]" git config user.email "223695+release-controller[bot]@users.noreply.github.com" - git fetch origin tims-test-branch - git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT From 2b8e2d4adfe8679de8b5c8057380a8e2d07ec430 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 25 Sep 2023 21:50:34 +0000 Subject: [PATCH 223/410] fix push command --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 308d7fd6a..fe8d363a4 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -42,7 +42,7 @@ jobs: git config user.name "release-controller[bot]" git config user.email "223695+release-controller[bot]@users.noreply.github.com" git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch + git push origin master echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From cafd7ae8579fcd96cddc494e99fa6130f42eb693 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 14:51:30 +0000 Subject: [PATCH 224/410] use PAT token for now --- .github/workflows/build-and-release.yml | 189 +++++++++++------------- 1 file changed, 85 insertions(+), 104 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index fe8d363a4..2bd6dab2f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,6 +22,7 @@ jobs: build: runs-on: ubuntu-latest steps: + # this token still gets denied by the backup-utils repo # - uses: timreimherr/create-github-app-token@main # id: app-token # with: @@ -30,121 +31,101 @@ jobs: # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} # owner: ${{ github.repository_owner }} # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils + - name: Checkout backup-utils-private uses: actions/checkout@v4 with: token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} + steps: + # this token still gets denied by the backup-utils repo + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit in backup-utils id: empty-commit run: | - git config user.name "release-controller[bot]" - git config user.email "223695+release-controller[bot]@users.noreply.github.com" + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" git commit --allow-empty -m "${{ github.event.inputs.version }} release" git push origin master echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Echo Success if commit was created - if: steps.empty-commit.outputs.commit-sha != '' - run: | - echo "Success" - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "release-controller[bot]" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} - # steps: - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit in backup-utils - # id: empty-commit - # run: | - # git config user.name "release-controller[bot]" - # git config user.email "ghes-releases-team@github.com" - # git fetch origin tims-test-branch - # git checkout tims-test-branch - # git commit --allow-empty -m "${{ github.event.inputs.version }} release" - # git push origin tims-test-branch - # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities - # v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # # this action will create a tag with this name on the provided commit - # tag: v${{ github.event.inputs.version }} - # # this can be a commit hash or branch name - # commit: ${{ steps.empty-commit.outputs.commit_sha }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + # this action will create a tag with this name on the provided commit + tag: v${{ github.event.inputs.version }} + # this can be a commit hash or branch name + commit: ${{ steps.empty-commit.outputs.commit-sha }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From 2f709371d6e805d3d96382528fbbafb9d5b91d2e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:04:42 +0000 Subject: [PATCH 225/410] one more test --- .github/workflows/build-and-release.yml | 210 +++++++++++++----------- 1 file changed, 118 insertions(+), 92 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 2bd6dab2f..e54690a21 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,110 +22,136 @@ jobs: build: runs-on: ubuntu-latest steps: - # this token still gets denied by the backup-utils repo - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils-private - uses: actions/checkout@v4 + - uses: timreimherr/create-github-app-token@main + id: app-token with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils + uses: actions/checkout@v4 with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + # the auto-commit action uses the GITHUB_TOKEN to create the commit + - name: Set GITHUB_TOKEN run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 + echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> $GITHUB_ENV + - uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} - steps: + branch: master + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "${{ github.actor }}" + commit_user_email: "ghes-releases-team@github.com" + commit_options: "--allow-empty" + skip_dirty_check: true # this token still gets denied by the backup-utils repo # - uses: timreimherr/create-github-app-token@main # id: app-token # with: + # # required # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} # owner: ${{ github.repository_owner }} # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils - ref: master - - name: Create empty commit in backup-utils - id: empty-commit - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin master - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ github.event.inputs.gh-token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - # this action will create a tag with this name on the provided commit - tag: v${{ github.event.inputs.version }} - # this can be a commit hash or branch name - commit: ${{ steps.empty-commit.outputs.commit-sha }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils-private + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} + # steps: + # # this token still gets denied by the backup-utils repo + # # - uses: timreimherr/create-github-app-token@main + # # id: app-token + # # with: + # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # # owner: ${{ github.repository_owner }} + # # repositories: backup-utils,backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # - name: Create empty commit in backup-utils + # id: empty-commit + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git commit --allow-empty -m "${{ github.event.inputs.version }} release" + # git push origin master + # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # # this action will create a tag with this name on the provided commit + # tag: v${{ github.event.inputs.version }} + # # this can be a commit hash or branch name + # commit: ${{ steps.empty-commit.outputs.commit-sha }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From 658bd0eb3866404096636c23eb656660e5d738b8 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:08:43 +0000 Subject: [PATCH 226/410] test use PAT token --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index e54690a21..2d7f41f4a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -38,7 +38,7 @@ jobs: # the auto-commit action uses the GITHUB_TOKEN to create the commit - name: Set GITHUB_TOKEN run: | - echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> $GITHUB_ENV + echo "GITHUB_TOKEN=${{ github.event.inputs.gh-token }}" >> $GITHUB_ENV - uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: From 19482957a0ef46ddf6f40ce01a6875d075ac5e9e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:11:52 +0000 Subject: [PATCH 227/410] add tag message --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 2d7f41f4a..f366daac0 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -47,6 +47,7 @@ jobs: commit_user_name: "${{ github.actor }}" commit_user_email: "ghes-releases-team@github.com" commit_options: "--allow-empty" + tag_message: "v${{ github.event.inputs.version }}" skip_dirty_check: true # this token still gets denied by the backup-utils repo # - uses: timreimherr/create-github-app-token@main From 7d3c54d815256d4763f1b3231dd2ff16f59a88a8 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:16:24 +0000 Subject: [PATCH 228/410] uses the checkout git config, I thiink --- .github/workflows/build-and-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f366daac0..93243258a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -33,12 +33,12 @@ jobs: - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ steps.app-token.outputs.token }} + token: ${{ github.event.inputs.gh-token }} repository: github/backup-utils # the auto-commit action uses the GITHUB_TOKEN to create the commit - - name: Set GITHUB_TOKEN - run: | - echo "GITHUB_TOKEN=${{ github.event.inputs.gh-token }}" >> $GITHUB_ENV + # - name: Set GITHUB_TOKEN + # run: | + # echo "GITHUB_TOKEN=${{ github.event.inputs.gh-token }}" >> $GITHUB_ENV - uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: From c4ef0d6074f166a2464f2e5a72c3bfd1f4f1780e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:29:00 +0000 Subject: [PATCH 229/410] setup for PAT token --- .github/workflows/build-and-release.yml | 209 +++++++++++------------- 1 file changed, 92 insertions(+), 117 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 93243258a..6293acc89 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,24 +22,78 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token@main - id: app-token + # # resulting token still gets denied by the backup-utils repo + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils-private + uses: actions/checkout@v4 with: - # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: backup-utils,backup-utils-private + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + # resulting token still gets denied by the backup-utils repo + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private - name: Checkout backup-utils uses: actions/checkout@v4 with: token: ${{ github.event.inputs.gh-token }} repository: github/backup-utils - # the auto-commit action uses the GITHUB_TOKEN to create the commit - # - name: Set GITHUB_TOKEN - # run: | - # echo "GITHUB_TOKEN=${{ github.event.inputs.gh-token }}" >> $GITHUB_ENV - - uses: stefanzweifel/git-auto-commit-action@v4 + ref: master + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: branch: master @@ -47,112 +101,33 @@ jobs: commit_user_name: "${{ github.actor }}" commit_user_email: "ghes-releases-team@github.com" commit_options: "--allow-empty" - tag_message: "v${{ github.event.inputs.version }}" skip_dirty_check: true - # this token still gets denied by the backup-utils repo - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils-private - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit-sha: ${{ steps.empty-commit.outputs.commit-sha }} - # steps: - # # this token still gets denied by the backup-utils repo - # # - uses: timreimherr/create-github-app-token@main - # # id: app-token - # # with: - # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # # owner: ${{ github.repository_owner }} - # # repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit in backup-utils - # id: empty-commit - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git commit --allow-empty -m "${{ github.event.inputs.version }} release" - # git push origin master - # echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # # this action will create a tag with this name on the provided commit - # tag: v${{ github.event.inputs.version }} - # # this can be a commit hash or branch name - # commit: ${{ steps.empty-commit.outputs.commit-sha }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + # this action will create a tag with this name on the provided commit + tag: v${{ github.event.inputs.version }} + # this can be a commit hash or branch name + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From 9848e0f956f93de210969095977071ef27fdcd48 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:43:14 +0000 Subject: [PATCH 230/410] fix yaml lint config --- .github/linters/.yaml-lint.yml | 72 +++++++++++++++++----------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index 0f3e1154b..030c37f04 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -7,47 +7,47 @@ # # yamllint disable-line # ########################################### rules: - braces: disable - # level: warning - # min-spaces-inside: 0 - # max-spaces-inside: 0 - # min-spaces-inside-empty: 1 - # max-spaces-inside-empty: 5 - brackets: disable - # level: warning - # min-spaces-inside: 0 - # max-spaces-inside: 0 - # min-spaces-inside-empty: 1 - # max-spaces-inside-empty: 5 - colons: disable - # level: warning - # max-spaces-before: 0 - # max-spaces-after: 1 - commas: disable - # level: warning - # max-spaces-before: 0 - # min-spaces-after: 1 - # max-spaces-after: 1 + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 comments: disable comments-indentation: disable document-end: disable document-start: disable - empty-lines: disable - # level: warning - # max: 2 - # max-start: 0 - # max-end: 0 - hyphens: disable - # level: warning - # max-spaces-after: 1 - indentation: disable - # level: warning - # spaces: consistent - # indent-sequences: true - # check-multi-line-strings: false + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false key-duplicates: enable line-length: disable new-line-at-end-of-file: disable - new-lines: diable - # type: unix + new-lines: + type: unix trailing-spaces: disable \ No newline at end of file From ddd2752a920e66d9d545e1367aa010ab8f1d7ff5 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 15:47:11 +0000 Subject: [PATCH 231/410] clean up --- .github/workflows/build-and-release.yml | 6 +++--- script/create-empty-commit | 14 -------------- 2 files changed, 3 insertions(+), 17 deletions(-) delete mode 100644 script/create-empty-commit diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6293acc89..c76c0efe8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,7 +22,8 @@ jobs: build: runs-on: ubuntu-latest steps: - # # resulting token still gets denied by the backup-utils repo + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 # - uses: timreimherr/create-github-app-token@main # id: app-token # with: @@ -79,6 +80,7 @@ jobs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} steps: # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 # - uses: timreimherr/create-github-app-token@main # id: app-token # with: @@ -120,9 +122,7 @@ jobs: artifacts: | github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ github-backup-utils_${{ github.event.inputs.version }}_all.deb - # this action will create a tag with this name on the provided commit tag: v${{ github.event.inputs.version }} - # this can be a commit hash or branch name commit: ${{ steps.empty-commit.outputs.commit_hash }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} diff --git a/script/create-empty-commit b/script/create-empty-commit deleted file mode 100644 index 7f69449a9..000000000 --- a/script/create-empty-commit +++ /dev/null @@ -1,14 +0,0 @@ -#!/ust/bin/env bash -# Usage: script/create-empty-commit -# Script to create an empty commit on the current branch. -# This is used to create a new tag for a release in github/backup-utils. -# This is to avoid stacking multiple release tags on the same commit. -set -e - -git config user.name "release-controller[bot]" -git config user.email "ghes-releases-team@github.com" -git fetch origin tims-test-branch -git checkout tims-test-branch -git commit --allow-empty -m "${{ github.event.inputs.version }} release" -git push origin tims-test-branch -echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT \ No newline at end of file From 765260fb48e7f0ec4ca33c4ab501537d365d04b8 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 16:04:16 +0000 Subject: [PATCH 232/410] see token permissions --- .github/workflows/build-and-release.yml | 207 ++++++++++++------------ 1 file changed, 105 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c76c0efe8..18ce48189 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,110 +24,113 @@ jobs: steps: # resulting token still gets denied by the backup-utils repo # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils-private - uses: actions/checkout@v4 + - uses: timreimherr/create-github-app-token@main + id: app-token with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private - - name: Install dependencies + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: backup-utils,backup-utils-private + - name: Print app permissions run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils - uses: actions/checkout@v4 - with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils - ref: master - - name: Create empty commit - uses: stefanzweifel/git-auto-commit-action@v4 - id: empty-commit - with: - branch: master - commit_message: "${{ github.event.inputs.version }} release" - commit_user_name: "${{ github.actor }}" - commit_user_email: "ghes-releases-team@github.com" - commit_options: "--allow-empty" - skip_dirty_check: true - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ github.event.inputs.gh-token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.[] | {permissions: .permissions, target_type: .target_type, repository_selection: .repository_selection}' + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils-private + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + # steps: + # # resulting token still gets denied by the backup-utils repo + # # see: https://github.com/actions/create-github-app-token/pull/46 + # # - uses: timreimherr/create-github-app-token@main + # # id: app-token + # # with: + # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # # owner: ${{ github.repository_owner }} + # # repositories: backup-utils,backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils + # ref: master + # - name: Create empty commit + # uses: stefanzweifel/git-auto-commit-action@v4 + # id: empty-commit + # with: + # branch: master + # commit_message: "${{ github.event.inputs.version }} release" + # commit_user_name: "${{ github.actor }}" + # commit_user_email: "ghes-releases-team@github.com" + # commit_options: "--allow-empty" + # skip_dirty_check: true + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # tag: v${{ github.event.inputs.version }} + # commit: ${{ steps.empty-commit.outputs.commit_hash }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From 6a7a213d68b5d0ab4772c33603a17c8faa625e6b Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 16:07:25 +0000 Subject: [PATCH 233/410] fix curl command output --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 18ce48189..d0b64fb37 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -34,7 +34,7 @@ jobs: repositories: backup-utils,backup-utils-private - name: Print app permissions run: | - curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.[] | {permissions: .permissions, target_type: .target_type, repository_selection: .repository_selection}' + curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.installations[] | {permissions: .permissions, target_type: .target_type, repository_selection: .repository_selection}' # - name: Checkout backup-utils-private # uses: actions/checkout@v4 # with: From 1dff7ccc4b30c00000b5e60d812fbe078b29e93a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 16:11:03 +0000 Subject: [PATCH 234/410] print entire response --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index d0b64fb37..f95c6f5f0 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -34,7 +34,7 @@ jobs: repositories: backup-utils,backup-utils-private - name: Print app permissions run: | - curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.installations[] | {permissions: .permissions, target_type: .target_type, repository_selection: .repository_selection}' + curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.' # - name: Checkout backup-utils-private # uses: actions/checkout@v4 # with: From eadb37ea851e011ececa059a28facbd01181939e Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Tue, 26 Sep 2023 10:20:21 -0600 Subject: [PATCH 235/410] Update integration-tests.yml --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f48de957e..efdd23ebe 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,10 +7,10 @@ on: workflow_dispatch: inputs: target-branch: - description: 'Branch that would be merged into' + description: 'enterprise2 branch to test against' required: true source-branch: - description: 'Branch that would be merged' + description: 'backup-utils-private topic branch to test' required: true # Get target and source branch from different variables depending on how it was triggered From 15397574469a5c89483e677787343b7d0dee5e50 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 16:21:00 +0000 Subject: [PATCH 236/410] another test --- .github/workflows/build-and-release.yml | 36 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f95c6f5f0..0a48f9518 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,8 +22,6 @@ jobs: build: runs-on: ubuntu-latest steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - uses: timreimherr/create-github-app-token@main id: app-token with: @@ -32,9 +30,39 @@ jobs: private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: backup-utils,backup-utils-private - - name: Print app permissions + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + ref: master + - name: Create empty commit in backup-utils + id: empty-commit run: | - curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.' + git config user.name "release-controller[bot]" + git config user.email "223695+release-controller[bot]@users.noreply.github.com" + git fetch origin tims-test-branch + git checkout tims-test-branch + git commit --allow-empty -m "${{ github.event.inputs.version }} release" + git push origin tims-test-branch "https://x-access-token:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" + echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - name: Echo Success if commit was created + if: steps.empty-commit.outputs.commit-sha != '' + run: | + echo "Success" + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + # - name: Print app permissions + # run: | + # curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.' # - name: Checkout backup-utils-private # uses: actions/checkout@v4 # with: From f8685a4926f57a7b3561cf28099906317cae801e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 16:23:31 +0000 Subject: [PATCH 237/410] try another approach --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 0a48f9518..7a47b6cb2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,7 +44,7 @@ jobs: git fetch origin tims-test-branch git checkout tims-test-branch git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push origin tims-test-branch "https://x-access-token:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" + git push "https://x-access-token:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" tims-test-branch echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - name: Echo Success if commit was created if: steps.empty-commit.outputs.commit-sha != '' From 266760134d051101580d55784163310e050f749c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 17:39:46 +0000 Subject: [PATCH 238/410] clean up --- .github/workflows/build-and-release.yml | 217 ++++++++++-------------- 1 file changed, 93 insertions(+), 124 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7a47b6cb2..c76c0efe8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -22,143 +22,112 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: timreimherr/create-github-app-token@main - id: app-token - with: - # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils + # resulting token still gets denied by the backup-utils repo + # see: https://github.com/actions/create-github-app-token/pull/46 + # - uses: timreimherr/create-github-app-token@main + # id: app-token + # with: + # # required + # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: backup-utils,backup-utils-private + - name: Checkout backup-utils-private uses: actions/checkout@v4 with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils - ref: master - - name: Create empty commit in backup-utils - id: empty-commit + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb run: | - git config user.name "release-controller[bot]" - git config user.email "223695+release-controller[bot]@users.noreply.github.com" - git fetch origin tims-test-branch - git checkout tims-test-branch - git commit --allow-empty -m "${{ github.event.inputs.version }} release" - git push "https://x-access-token:$%7B%7B%20steps.app-token.outputs.token%20%7D%7D@github.com/github/backup-utils.git" tims-test-branch - echo "commit-sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - name: Echo Success if commit was created - if: steps.empty-commit.outputs.commit-sha != '' + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact run: | - echo "Success" + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: # resulting token still gets denied by the backup-utils repo # see: https://github.com/actions/create-github-app-token/pull/46 # - uses: timreimherr/create-github-app-token@main # id: app-token # with: - # # required # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} # owner: ${{ github.repository_owner }} # repositories: backup-utils,backup-utils-private - # - name: Print app permissions - # run: | - # curl -H "Authorization: Bearer ${{ steps.app-token.outputs.token }}" -H "Accept: application/vnd.github.v3+json" https://api.github.com/app/installations | jq '.' - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils-private - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - # steps: - # # resulting token still gets denied by the backup-utils repo - # # see: https://github.com/actions/create-github-app-token/pull/46 - # # - uses: timreimherr/create-github-app-token@main - # # id: app-token - # # with: - # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # # owner: ${{ github.repository_owner }} - # # repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit - # uses: stefanzweifel/git-auto-commit-action@v4 - # id: empty-commit - # with: - # branch: master - # commit_message: "${{ github.event.inputs.version }} release" - # commit_user_name: "${{ github.actor }}" - # commit_user_email: "ghes-releases-team@github.com" - # commit_options: "--allow-empty" - # skip_dirty_check: true - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # tag: v${{ github.event.inputs.version }} - # commit: ${{ steps.empty-commit.outputs.commit_hash }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils + ref: master + - name: Create empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: master + commit_message: "${{ github.event.inputs.version }} release" + commit_user_name: "${{ github.actor }}" + commit_user_email: "ghes-releases-team@github.com" + commit_options: "--allow-empty" + skip_dirty_check: true + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ github.event.inputs.gh-token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From 43a3a1bc8d953f3b69055e36865ed399ecd9d662 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 18:17:22 +0000 Subject: [PATCH 239/410] Checkout backup-utils-private for release notes --- .github/workflows/build-and-release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c76c0efe8..90b44086e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -104,6 +104,11 @@ jobs: commit_user_email: "ghes-releases-team@github.com" commit_options: "--allow-empty" skip_dirty_check: true + - name: Checkout backup-utils + uses: actions/checkout@v4 + with: + token: ${{ github.event.inputs.gh-token }} + repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 with: From d45980fb8cdf53ba2a78983d655693f181d14034 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 26 Sep 2023 19:35:30 +0000 Subject: [PATCH 240/410] remove test release notes --- release-notes/12.12.12.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md deleted file mode 100644 index 192f42d88..000000000 --- a/release-notes/12.12.12.md +++ /dev/null @@ -1,11 +0,0 @@ -# Release Notes - -### Features - -- To reduce the time to generate a backup using `ghe-backup`, administrators can choose to prune old backup snapshots after a new backup has been generate. For more information, see "[Scheduling backups & snapshot pruning](https://github.com/github/backup-utils/blob/master/docs/scheduling-backups.md)." -- On instances with large MySQL databases , administrators who wish to save storage space can use the new `--incremental` flag with `ghe-backup` and `ghe-restore`. For more information, see "[Incremental MySQL Backups and Restores](https://github.com/github/backup-utils/tree/master/docs/incremental-mysql-backups-and-restores.md)". - -### Changes - -- Removed the `git clone` path for setting up `backup-utils` from the [getting started instructions](https://github.com/github/backup-utils/blob/master/docs/getting-started.md). -- Added `bc` v1.07 or newer to the [requirements](https://github.com/github/backup-utils/blob/master/docs/requirements.md) for a backup host machine. \ No newline at end of file From 87e07fe89daa047a47ad91ef742215ed574f8f5c Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Tue, 26 Sep 2023 18:09:44 -0400 Subject: [PATCH 241/410] fix mkdir issues or progress tracking (#612) Fixing https://github.com/github/ghes/issues/7409 --- share/github-backup-utils/ghe-backup-config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 790d2aaa4..d173b1703 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -655,14 +655,15 @@ restore-secret() { init-progress() { if [ -d /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* - else - mkdir /tmp/backup-utils-progress fi + + mkdir -p /tmp/backup-utils-progress + chmod -R 777 /tmp/backup-utils-progress + touch /tmp/backup-utils-progress/total touch /tmp/backup-utils-progress/type touch /tmp/backup-utils-progress/progress touch /tmp/backup-utils-progress/info - chmod -R 777 /tmp/backup-utils-progress } From a3a3cecabefd33a280c53d61a8ecc9bc74ae099f Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 18:10:06 +0000 Subject: [PATCH 242/410] add link to release documentation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4ee951f58..472acf87c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ GitHub Enterprise Server. - **[Backup snapshot file structure](docs/backup-snapshot-file-structure.md)** - **[How does Backup Utilities differ from a High Availability replica?](docs/faq.md)** - **[Docker](docs/docker.md)** +- **[Releases](https://github.com/github/enterprise-releases/blob/master/docs/release-backup-utils.md)** ## Support From d48791bc5e47a208f855d0903ee773d8af2afca9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 21:40:40 +0000 Subject: [PATCH 243/410] place values in string --- .github/workflows/build-and-release.yml | 206 +++++++++++++----------- 1 file changed, 112 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 90b44086e..cabccd9fd 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,76 +24,19 @@ jobs: steps: # resulting token still gets denied by the backup-utils repo # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # # required - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 + - uses: timreimherr/create-github-app-token@main + id: app-token with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - # - uses: timreimherr/create-github-app-token@main - # id: app-token - # with: - # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: backup-utils,backup-utils-private + # required + app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils uses: actions/checkout@v4 with: - token: ${{ github.event.inputs.gh-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit @@ -104,35 +47,110 @@ jobs: commit_user_email: "ghes-releases-team@github.com" commit_options: "--allow-empty" skip_dirty_check: true - - name: Checkout backup-utils - uses: actions/checkout@v4 - with: - token: ${{ github.event.inputs.gh-token }} - repository: github/backup-utils-private - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ github.event.inputs.gh-token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + - name: Log a message + if: success() + run: echo "The previous step succeeded" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils-private + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + # steps: + # # resulting token still gets denied by the backup-utils repo + # # see: https://github.com/actions/create-github-app-token/pull/46 + # # - uses: timreimherr/create-github-app-token@main + # # id: app-token + # # with: + # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # # owner: ${{ github.repository_owner }} + # # repositories: backup-utils,backup-utils-private + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils + # ref: master + # - name: Create empty commit + # uses: stefanzweifel/git-auto-commit-action@v4 + # id: empty-commit + # with: + # branch: master + # commit_message: "${{ github.event.inputs.version }} release" + # commit_user_name: "${{ github.actor }}" + # commit_user_email: "ghes-releases-team@github.com" + # commit_options: "--allow-empty" + # skip_dirty_check: true + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repository: github/backup-utils-private + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ github.event.inputs.gh-token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # tag: v${{ github.event.inputs.version }} + # commit: ${{ steps.empty-commit.outputs.commit_hash }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From 60d9ed1e9850506b05e45552015a520e8914ec6c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 27 Sep 2023 21:43:40 +0000 Subject: [PATCH 244/410] use test branch --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index cabccd9fd..49a7c0156 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -41,7 +41,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: master + branch: tims-test-branch commit_message: "${{ github.event.inputs.version }} release" commit_user_name: "${{ github.actor }}" commit_user_email: "ghes-releases-team@github.com" From 030580f3a97a685f6170411f43d450f32b59a70a Mon Sep 17 00:00:00 2001 From: Luke Reid Date: Thu, 28 Sep 2023 15:58:08 +0100 Subject: [PATCH 245/410] Fix broken upgrade requirements link The link to the upgrade requirements points to the old help pages. I have updated it to the new docs. --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index bba51c252..089a92b46 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -97,7 +97,7 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac [5]: https://en.wikipedia.org/wiki/Hard_link [6]: https://en.wikipedia.org/wiki/Symbolic_link [7]: https://en.wikipedia.org/wiki/Case_sensitivity -[8]: https://help.github.com/enterprise/admin/guides/installation/upgrade-requirements/ +[8]: https://docs.github.com/enterprise-server/admin/monitoring-managing-and-updating-your-instance/updating-the-virtual-machine-and-physical-resources/upgrade-requirements [9]: https://joeyh.name/code/moreutils [10]: https://www.gnu.org/software/gawk [11]: https://stedolan.github.io/jq/ From 4ea5ecefa04d5a9507bea016125b760ca685c335 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Thu, 28 Sep 2023 16:16:39 -0600 Subject: [PATCH 246/410] Update parallel command to adapt to changes in 3.11 --- share/github-backup-utils/ghe-backup-config | 16 ++++++++++++++++ share/github-backup-utils/ghe-restore-pages | 3 ++- .../github-backup-utils/ghe-restore-repositories | 3 ++- .../ghe-restore-repositories-gist | 3 ++- share/github-backup-utils/ghe-restore-storage | 3 ++- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index d173b1703..9e7cc9f47 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -462,6 +462,22 @@ ghe_parse_remote_version() { export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH } +# In 3.11 we started to install 2 different version parallel(s) +# moreutils parallel and GNU parallel. Get parallel command based on version +ghe_remote_parallel() { + if [ -z "$GHE_REMOTE_VERSION" ]; then + ecbk "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 + exit 1 + fi + + if [ "$GHE_VERSION_MINOR" -lt 11 ]; then + PARALLEL_CMD="parallel" + else + PARALLEL_CMD="parallel.moreutils" + fi + export PARALLEL_CMD +} + # Parses the part out of a ":" or just "" string. # This is used primarily to break hostspecs with non-standard ports down for # rsync commands. diff --git a/share/github-backup-utils/ghe-restore-pages b/share/github-backup-utils/ghe-restore-pages index 0b104863e..cbc019345 100755 --- a/share/github-backup-utils/ghe-restore-pages +++ b/share/github-backup-utils/ghe-restore-pages @@ -35,6 +35,7 @@ fi # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$GHE_HOSTNAME" +ghe_remote_parallel # Split host:port into parts port=$(ssh_port_part "$GHE_HOSTNAME") @@ -156,7 +157,7 @@ if $CLUSTER; then ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <&3 <>$remote_warnings" -- \$chunks + $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/dgit-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index 7faae5260..631676ea0 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -35,6 +35,7 @@ fi # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$GHE_HOSTNAME" +ghe_remote_parallel # Generate SSH config for forwarding # Split host:port into parts @@ -159,7 +160,7 @@ if $CLUSTER; then ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 <>$remote_warnings" -- \$chunks + $PARALLEL_CMD -i /bin/sh -c "cat {} | github-env ./bin/gist-cluster-restore-finalize 2>>$remote_warnings" -- \$chunks EOF increment-progress-total-count 1 bm_end "$(basename $0) - Finalizing routes" diff --git a/share/github-backup-utils/ghe-restore-storage b/share/github-backup-utils/ghe-restore-storage index 8f0d73c48..64cdbbaa4 100755 --- a/share/github-backup-utils/ghe-restore-storage +++ b/share/github-backup-utils/ghe-restore-storage @@ -39,6 +39,7 @@ fi # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$GHE_HOSTNAME" +ghe_remote_parallel # Split host:port into parts port=$(ssh_port_part "$GHE_HOSTNAME") @@ -171,7 +172,7 @@ if $CLUSTER; then ghe-ssh "$GHE_HOSTNAME" -- /bin/bash >&3 < Date: Thu, 28 Sep 2023 16:20:41 -0600 Subject: [PATCH 247/410] Update ghe-backup-config --- share/github-backup-utils/ghe-backup-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 9e7cc9f47..f49cdb0bc 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -466,7 +466,7 @@ ghe_parse_remote_version() { # moreutils parallel and GNU parallel. Get parallel command based on version ghe_remote_parallel() { if [ -z "$GHE_REMOTE_VERSION" ]; then - ecbk "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 + echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 exit 1 fi From d9831072a84c6ce1c5417924fa43d265a1470f01 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 29 Sep 2023 00:07:28 -0600 Subject: [PATCH 248/410] Fix that janky build wasn't triggered correctly --- .github/workflows/integration-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index efdd23ebe..603f7affa 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,6 +37,7 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' + force : 'false' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" @@ -63,5 +64,6 @@ jobs: janky-token: '${{ secrets.API_AUTH_TOKEN }}' job-name: '${{ matrix.jankyJobName }}' branch-name: '${{ env.SOURCE_BRANCH }}' + force : 'false' # enterprise2 target branch is same as target branch for PR (either master or enterprise-[0-9]*.[0-9]*-release) envVars: "JANKY_ENV_BACKUP_UTILS_BRANCH=${{ env.SOURCE_BRANCH }},JANKY_ENV_ENTERPRISE2_BRANCH=${{ env.TARGET_BRANCH }}" From 53f509e9cb081ac41dca9b925a911fefc1405e7a Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Fri, 29 Sep 2023 00:10:19 -0600 Subject: [PATCH 249/410] Update comments --- share/github-backup-utils/ghe-backup-config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index f49cdb0bc..4ac372e0b 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -463,7 +463,8 @@ ghe_parse_remote_version() { } # In 3.11 we started to install 2 different version parallel(s) -# moreutils parallel and GNU parallel. Get parallel command based on version +# moreutils parallel and GNU parallel. When gnu parallel is installed, +# it renames moreutils parallel to parallel.moreutils ghe_remote_parallel() { if [ -z "$GHE_REMOTE_VERSION" ]; then echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 From 23a11908523532c46b8bcb3425bec18df8bdf24d Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Fri, 29 Sep 2023 09:52:50 -0600 Subject: [PATCH 250/410] Update share/github-backup-utils/ghe-backup-config Co-authored-by: Quinn Murphy --- share/github-backup-utils/ghe-backup-config | 1 + 1 file changed, 1 insertion(+) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 4ac372e0b..86a75a15a 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -465,6 +465,7 @@ ghe_parse_remote_version() { # In 3.11 we started to install 2 different version parallel(s) # moreutils parallel and GNU parallel. When gnu parallel is installed, # it renames moreutils parallel to parallel.moreutils +# set $PARALLEL_CMD envvar to be used in place of parallel commands ghe_remote_parallel() { if [ -z "$GHE_REMOTE_VERSION" ]; then echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 From db8fdae5a45719efeb01fdd096da1c262771248f Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Mon, 2 Oct 2023 17:41:28 -0600 Subject: [PATCH 251/410] Fix track-progress while progress files are not set --- bin/ghe-backup | 1 + bin/ghe-restore | 3 +++ share/github-backup-utils/track-progress | 8 +++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 5a4eb6c20..8914bddc9 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -118,6 +118,7 @@ cleanup () { rm -rf "$failures_file" rm -f "${GHE_DATA_DIR}/in-progress-backup" + rm -rf /tmp/backup-utils-progress/* # Cleanup SSH multiplexing ghe-ssh --clean diff --git a/bin/ghe-restore b/bin/ghe-restore index c9ac20ec2..1d1159133 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -139,6 +139,9 @@ cleanup () { log_error "Failed to remove in-progress file" 1>&3 fi + # Remove progress files + rm -rf /tmp/backup-utils-progress/* + bm_end "$(basename $0)" } diff --git a/share/github-backup-utils/track-progress b/share/github-backup-utils/track-progress index a560ba540..6f7d1a76b 100755 --- a/share/github-backup-utils/track-progress +++ b/share/github-backup-utils/track-progress @@ -2,11 +2,17 @@ #/ track-progress: track progress of backup or restore tasks progress(){ - + ## Those progress files should be created by init_progress function + ## If they are not present (e.g., individual script is being invoked directly), + ## we will not track progress + if [ -f "/tmp/backup-utils-progress/progress" ] && + [ -f "/tmp/backup-utils-progress/total" ] && + [ -f "/tmp/backup-utils-progress/type" ]; then PROGRESS=$(cat /tmp/backup-utils-progress/progress) PROGRESS_TOTAL=$(cat /tmp/backup-utils-progress/total) PROGRESS_TYPE=$(cat /tmp/backup-utils-progress/type) PROGRESS_PERCENT=$( echo "scale = 2; ($PROGRESS / $PROGRESS_TOTAL) * 100" | bc) echo $((PROGRESS + 1)) > /tmp/backup-utils-progress/progress echo "${PROGRESS_TYPE} progress: $PROGRESS_PERCENT % ($PROGRESS / $PROGRESS_TOTAL ) $1 " > /tmp/backup-utils-progress/info + fi } From 1338786c4d0911b369acbff40f5b5dc5c65a3fca Mon Sep 17 00:00:00 2001 From: David Daly Date: Wed, 4 Oct 2023 12:27:23 +0000 Subject: [PATCH 252/410] add new test for passsing -c to restore for actions and checking if new value is present --- test/test-ghe-restore.sh | 88 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index da7fdb06b..3ea47e2b5 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -674,6 +674,94 @@ begin_test "ghe-restore with Actions settings" ) end_test +begin_test "ghe-restore with Actions settings passing -c" +( + set -e + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + enable_actions + + required_files=( + "actions-config-db-login" + "actions-config-db-password" + "actions-framework-access-token" + "actions-url-signing-hmac-key-primary" + "actions-url-signing-hmac-key-secondary" + "actions-oauth-s2s-signing-cert" + "actions-oauth-s2s-signing-key" + "actions-oauth-s2s-signing-cert-thumbprint" + "actions-primary-encryption-cert-thumbprint" + "actions-aad-cert-thumbprint" + "actions-delegated-auth-cert-thumbprint" + "actions-runtime-service-principal-cert" + "actions-s2s-encryption-cert" + "actions-secondary-encryption-cert-thumbprint" + "actions-service-principal-cert" + "actions-sps-validation-cert-thumbprint" + "actions-storage-container-prefix" + + "actions-launch-secrets-private-key" + "actions-launch-deployer-hmac" + "actions-launch-client-id" + "actions-launch-client-secret" + "actions-launch-receiver-webhook-secret" + "actions-launch-app-private-key" + "actions-launch-app-public-key" + "actions-launch-app-id" + "actions-launch-app-relay-id" + "actions-launch-action-runner-secret" + "actions-launch-azp-app-cert" + "actions-launch-app-app-private-key" + + ) + + for file in "${required_files[@]}"; do + echo "foo" > "$GHE_DATA_DIR/current/$file" + done + + ghe-restore -v -f -c localhost + + required_secrets=( + "secrets.actions.ConfigurationDatabaseSqlLogin" + "secrets.actions.ConfigurationDatabaseSqlPassword" + "secrets.actions.FrameworkAccessTokenKeySecret" + "secrets.actions.UrlSigningHmacKeyPrimary" + "secrets.actions.UrlSigningHmacKeySecondary" + "secrets.actions.OAuthS2SSigningCert" + "secrets.actions.OAuthS2SSigningKey" + "secrets.actions.OAuthS2SSigningCertThumbprint" + "secrets.actions.PrimaryEncryptionCertificateThumbprint" + "secrets.actions.AADCertThumbprint" + "secrets.actions.DelegatedAuthCertThumbprint" + "secrets.actions.RuntimeServicePrincipalCertificate" + "secrets.actions.S2SEncryptionCertificate" + "secrets.actions.SecondaryEncryptionCertificateThumbprint" + "secrets.actions.ServicePrincipalCertificate" + "secrets.actions.SpsValidationCertThumbprint" + "secrets.actions.storage.container-prefix" + "secrets.launch.actions-secrets-private-key" + "secrets.launch.deployer-hmac-secret" + "secrets.launch.client-id" + "secrets.launch.client-secret" + "secrets.launch.receiver-webhook-secret" + "secrets.launch.app-private-key" + "secrets.launch.app-public-key" + "secrets.launch.app-id" + "secrets.launch.app-relay-id" + "secrets.launch.action-runner-secret" + "secrets.launch.token-oauth-key" + "secrets.launch.token-oauth-cert" + "secrets.launch.azp-app-cert" + "secrets.launch.azp-app-private-key" + + ) + + for secret in "${required_secrets[@]}"; do + [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "foo" ] + done +) +end_test + begin_test "ghe-restore stops and starts Actions" ( set -e From 24bd388b8368f2b057a9feb23a1fb29291a66735 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 5 Oct 2023 15:25:22 -0400 Subject: [PATCH 253/410] Added section about rsync compression --- docs/usage.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/usage.md b/docs/usage.md index 8f1a70131..1e9653598 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -115,5 +115,11 @@ Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-serve If you are interested in performing incremental backups of the MySQL data in your GitHub Enterprise Server instance, see [Incremental MySQL Backups and Restores](incremental-mysql-backups-and-restores.md) for details. +## Rsync compression + +From backup-utils v3.11.0 onwards, we have disabled rsync compression by default to improve transfer speed and reduce CPU usage durign the transfer process. + +If you would like to use compression with rsync, you can add `GHE_RSYNC_COMPRESSION_ENABLED=true` in your `backup.config` file. + [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md [2]: requirements.md From 2fa406bd449e87fe3e6f03354f6cbe2207acaa2f Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Fri, 6 Oct 2023 17:43:04 -0400 Subject: [PATCH 254/410] Fixed typo. --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 1e9653598..e4ed5b7e2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -117,7 +117,7 @@ If you are interested in performing incremental backups of the MySQL data in you ## Rsync compression -From backup-utils v3.11.0 onwards, we have disabled rsync compression by default to improve transfer speed and reduce CPU usage durign the transfer process. +From backup-utils v3.11.0 onwards, we have disabled rsync compression by default to improve transfer speed and reduce CPU usage during the transfer process. If you would like to use compression with rsync, you can add `GHE_RSYNC_COMPRESSION_ENABLED=true` in your `backup.config` file. From 63d5932a63f9d77604264ef288c681992b28ce25 Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Mon, 9 Oct 2023 12:03:04 -0400 Subject: [PATCH 255/410] correct incremental restore envvar (#642) --- docs/incremental-mysql-backups-and-restores.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/incremental-mysql-backups-and-restores.md b/docs/incremental-mysql-backups-and-restores.md index 0bc5f8ef8..53543c927 100644 --- a/docs/incremental-mysql-backups-and-restores.md +++ b/docs/incremental-mysql-backups-and-restores.md @@ -1,14 +1,14 @@ # Incremental MySQL Backups and Restores Customers who have large MySQL databases who wish to save storage space can use the `--incremental` flag with `ghe-backup` and `ghe-restore`. -Using this flag performs backups for other parts of GHES as normal, but only performs a MySQL backup of the changes to the database from the previous snapshot. +Using this flag performs backups for other parts of GHES as normal, but only performs a MySQL backup of the changes to the database from the previous snapshot. For larger databases this can conserve a lot of storage space for backups. ## Configuring number of backups -In your backup.config file you will need to set the variable `GHE_INCREMENTAL_BACKUP_MAX`. +In your backup.config file you will need to set the variable `GHE_INCREMENTAL_MAX_BACKUPS`. This variable determines how many cycles of full and incremental backups will be performed before the next full backup is created. -For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. +For example, if `GHE_INCREMENTAL_MAX_BACKUPS` is set to 14, backup-utils will run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. Incremental backups require the previous snapshot backups before them to work. This means they do not follow the pruning strategy based on `GHE_NUM_SNAPSHOTS`. @@ -19,7 +19,7 @@ To perform incremental backups: `bin/ghe-backup --incremental` -the program will detect whether it needs to performa full or incremental snapshot based on what is currently in `GHE_DATA_DIR`. +the program will detect whether it needs to performa full or incremental snapshot based on what is currently in `GHE_DATA_DIR`. To see what snapshots are part of your full and incremental backups, you can reference `GHE_DATA_DIR/inc_full_backup` and `GHE_DATA_DIR/inc_snapshot_data`, respectively. @@ -35,4 +35,4 @@ The program will use the MySQL folders from each previous incremental backup and ### Previous cycles -To ensure there is a rolling window of mySQL backups, incremental MySQL backups from the cycle before the current one are kept. Those snapshots are pre-pended with `inc_previous`. To perform a restore from there, just use the full directory name for the snapshot id. +To ensure there is a rolling window of mySQL backups, incremental MySQL backups from the cycle before the current one are kept. Those snapshots are pre-pended with `inc_previous`. To perform a restore from there, just use the full directory name for the snapshot ID. From da35308a675913946084fa83afebcaab359bab93 Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Tue, 10 Oct 2023 18:05:20 +0200 Subject: [PATCH 256/410] refactor secrets (#648) --- bin/ghe-backup | 3 + bin/ghe-restore | 15 +- share/github-backup-utils/ghe-backup-config | 10 - share/github-backup-utils/ghe-backup-secrets | 186 ++++++++++++++++++ share/github-backup-utils/ghe-backup-settings | 159 --------------- share/github-backup-utils/ghe-restore-actions | 42 ---- .../ghe-restore-chat-integration | 66 ------- .../ghe-restore-column-encryption-keys | 42 ---- share/github-backup-utils/ghe-restore-mysql | 3 - .../github-backup-utils/ghe-restore-packages | 45 ----- ...he-restore-secret-scanning-encryption-keys | 45 ----- share/github-backup-utils/ghe-restore-secrets | 146 ++++++++++++++ .../github-backup-utils/ghe-restore-settings | 27 --- test/test-ghe-restore.sh | 59 ------ 14 files changed, 336 insertions(+), 512 deletions(-) create mode 100755 share/github-backup-utils/ghe-backup-secrets delete mode 100755 share/github-backup-utils/ghe-restore-chat-integration delete mode 100755 share/github-backup-utils/ghe-restore-column-encryption-keys delete mode 100755 share/github-backup-utils/ghe-restore-packages delete mode 100755 share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys create mode 100755 share/github-backup-utils/ghe-restore-secrets diff --git a/bin/ghe-backup b/bin/ghe-backup index 8914bddc9..1dfaf748b 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -267,6 +267,9 @@ bm_init > /dev/null ghe-backup-store-version || log_warn "Warning: storing backup-utils version remotely failed." +log_info "Backing up GitHub secrets ..." +ghe-backup-secrets || failures="$failures secrets" + log_info "Backing up GitHub settings ..." ghe-backup-settings || failures="$failures settings" diff --git a/bin/ghe-restore b/bin/ghe-restore index 1d1159133..101f19ebc 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -466,6 +466,7 @@ else fi CRON_RUNNING=false +ghe-restore-secrets "$GHE_HOSTNAME" # Restore settings and license if restoring to an unconfigured appliance or when # specified manually. @@ -473,19 +474,6 @@ if $RESTORE_SETTINGS; then ghe-restore-settings "$GHE_HOSTNAME" fi -# Always restore column encryption keys -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then - log_info "Always restore encrypted column encryption keys on GHES versions 3.7.0+" -fi -ghe-restore-column-encryption-keys "$GHE_HOSTNAME" - -# Always restore secret scanning encryption keys -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - log_info "Always restore secret scanning encryption keys on GHES versions 3.8.0+" - increment-progress-total-count 1 - ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME" -fi - # Make sure mysql and elasticsearch are prep'd and running before restoring. # These services will not have been started on appliances that have not been # configured yet. @@ -639,7 +627,6 @@ echo "sudo restart -q memcached 2>/dev/null || true" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh bm_end "$(basename $0) - Restarting memcached" - # Prevent GitHub Connect jobs running before we've had a chance to reset # the configuration by setting the last run date to now. if ! $RESTORE_SETTINGS; then diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 86a75a15a..6c33ea954 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -659,16 +659,6 @@ prompt_for_confirmation(){ echo } -# Function to restore a secret setting stored in a file. -# restore-secret -restore-secret() { - if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/$2" ]; then - echo "Restoring $1 ..." - echo "ghe-config '$3' '$(cat "$GHE_RESTORE_SNAPSHOT_PATH/$2")'" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/bash - fi -} - #initialize progress tracking by clearing out the temp files used to track init-progress() { if [ -d /tmp/backup-utils-progress ]; then diff --git a/share/github-backup-utils/ghe-backup-secrets b/share/github-backup-utils/ghe-backup-secrets new file mode 100755 index 000000000..546be420e --- /dev/null +++ b/share/github-backup-utils/ghe-backup-secrets @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +#/ Usage: ghe-backup-secrets +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-backup command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" + +# Grab the host +host="$GHE_HOSTNAME" + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$host" + + +# Function to backup a secret setting to a file. +# backup-secret [--best-effort] +backup-secret() { + best_effort=false + description="" + file="" + setting="" + count=0 + + while [ $# -gt 0 ]; do + case "$1" in + --best-effort) + shift 1 + best_effort=true + ;; + *) + case $count in + 0) + description=$1 + ;; + 1) + file=$1 + ;; + 2) + setting=$1 + ;; + *) + >&2 echo "Too many arguments" + ;; + esac + count=$((count+1)) + shift 1 + esac + done + + log_info "* Transferring $description ..." 1>&3 + ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( + if [ "$best_effort" = "false" ]; then + echo "Warning: $description not set" >&2 + fi + ) + if [ -n "$(cat "$file+")" ]; then + mv "$file+" "$file" + else + unlink "$file+" + fi +} + +bm_start "$(basename $0)" + +# Create the snapshot directory if needed and change into it. +mkdir -p "$GHE_SNAPSHOT_DIR" +cd "$GHE_SNAPSHOT_DIR" + +log_info "* Transferring secrets data ..." 1>&3 + +backup-secret "management console password" "manage-password" "secrets.manage" +backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" +backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" +backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" + +# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards +# this is for forwards compatibility with GHES 3.8.0 onwards +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then + backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" + cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" +fi + +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then + backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" + backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" + backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" + backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +fi + +if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then + backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" +fi + +# Backup argon secrets for multiuser from ghes version 3.8 onwards +if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then + backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" +fi + +# Backup external MySQL password if running external MySQL DB. +if is_service_external 'mysql'; then + backup-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" +fi + +# Backup Actions settings. +if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then + backup-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" + backup-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" + backup-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" --best-effort + backup-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" + backup-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" + backup-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" + backup-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" + backup-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" + backup-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" + backup-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" --best-effort + backup-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" --best-effort + backup-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" --best-effort + backup-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" + backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" + backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort + backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + + backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" + backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" + backup-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" + backup-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" + backup-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" + backup-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" + backup-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" + backup-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" + backup-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" + backup-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" + backup-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" + backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" +fi + +if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then + backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" + backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" + backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" + backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" + backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" + backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" + backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" +fi + +# Backup Chat Integration settings +if ghe-ssh "$host" -- ghe-config --true app.chatops.enabled; then + backup-secret "Chat Integration MSTeams app id" "chatops-msteams-app-id" "secrets.chatops.msteams.app-id" + backup-secret "Chat Integration MSTeams app password" "chatops-msteams-app-password" "secrets.chatops.msteams.app-password" + backup-secret "Chat Integration MSTeams public endpoint" "chatops-msteams-app-public-endpoint" "secrets.chatops.msteams.public-endpoint" + backup-secret "Chat Integration MSTeams bot handle" "chatops-msteams-bot-handle" "secrets.chatops.msteams.bot-handle" + backup-secret "Chat Integration MSTeams bot name" "chatops-msteams-bot-name" "secrets.chatops.msteams.bot-name" + backup-secret "Chat Integration Slack app id" "chatops-slack-app-id" "secrets.chatops.slack.app-id" + backup-secret "Chat Integration Slack client id" "chatops-slack-client-id" "secrets.chatops.slack.client-id" + backup-secret "Chat Integration Slack client secret" "chatops-slack-client-secret" "secrets.chatops.slack.client-secret" + backup-secret "Chat Integration Slack verification token" "chatops-slack-verification-token" "secrets.chatops.slack.verification-token" + backup-secret "Chat Integration Slack config token" "chatops-slack-config-token" "secrets.chatops.slack.config-token" + backup-secret "Chat Integration Slack public endpoint" "chatops-slack-public-endpoint" "secrets.chatops.slack.public-endpoint" + backup-secret "Chat Integration Slack signing secret" "chatops-slack-signing-secret" "secrets.chatops.slack.signing-secret" + backup-secret "Chat Integration Slack app level token" "chatops-slack-app-level-token" "secrets.chatops.slack.app-level-token" + backup-secret "Chat Integration Slack slack command" "chatops-slack-slash-command" "secrets.chatops.slack.slash-command" + backup-secret "Chat Integration Slack app name" "chatops-slack.app-name" "secrets.chatops.slack.app-name" + backup-secret "Chat Integration Slack socket mode" "chatops-slack.socket-mode" "secrets.chatops.slack.socket-mode" + backup-secret "Chat Integration public endpoint" "chatops-public-endpoint" "secrets.chatops.public-endpoint" + backup-secret "Chat Integration app type" "chatops-app-type" "secrets.chatops.app-type" + backup-secret "Chat Integration app id teams" "chatops-app-id-teams" "secrets.chatops.app-id-teams" + backup-secret "Chat Integration webhook secret teams" "chatops-webhook-secret-teams" "secrets.chatops.webhook-secret-teams" + backup-secret "Chat Integration client secret teams" "chatops-client-secret-teams" "secrets.chatops.client-secret-teams" + backup-secret "Chat Integration clien id teams" "chatops-client-id-teams" "secrets.chatops.client-id-teams" + backup-secret "Chat Integration storage secret" "chatops-storage-secret" "secrets.chatops.storage-secret" + backup-secret "Chat Integration session secret" "chatops-session-secret" "secrets.chatops.session-secret" + backup-secret "Chat Integration app id slack" "chatops-app-id-slack" "secrets.chatops.app-id-slack" + backup-secret "Chat Integration webhook secret slack" "chatops-webhook-secret-slack" "secrets.chatops.webhook-secret-slack" + backup-secret "Chat Integration client secret slack" "chatops-client-secret-slack" "secrets.chatops.client-secret-slack" + backup-secret "Chat Integration client id slack" "chatops-client-id-slack" "secrets.chatops.client-id-slack" +fi + +bm_end "$(basename $0)" + +exit 0 diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index 7cfd62b8d..f3d262293 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -25,165 +25,6 @@ ghe-ssh "$host" -- 'ghe-export-settings' > settings.json log_info "* Transferring license data ..." 1>&3 ghe-ssh "$host" -- "sudo cat '$GHE_REMOTE_LICENSE_FILE'" > enterprise.ghl -# Function to backup a secret setting to a file. -# backup-secret [--best-effort] -backup-secret() { - - best_effort=false - description="" - file="" - setting="" - count=0 - - while [ $# -gt 0 ]; do - case "$1" in - --best-effort) - shift 1 - best_effort=true - ;; - *) - case $count in - 0) - description=$1 - ;; - 1) - file=$1 - ;; - 2) - setting=$1 - ;; - *) - >&2 echo "Too many arguments" - ;; - esac - count=$((count+1)) - shift 1 - esac - done - - log_info "* Transferring $description ..." 1>&3 - ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( - if [ "$best_effort" = "false" ]; then - echo "Warning: $description not set" >&2 - fi - ) - if [ -n "$(cat "$file+")" ]; then - mv "$file+" "$file" - else - unlink "$file+" - fi -} - -backup-secret "management console password" "manage-password" "secrets.manage" -backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" -backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" -backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" - -# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards -# this is for forwards compatibility with GHES 3.8.0 onwards -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then - backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" - cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" -fi - -# secret scanning encrypted secrets keys were added in GHES 3.8.0 -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" - backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" - backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" - backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" -fi - -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then - backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" -fi - -# Backup argon secrets for multiuser from ghes version 3.8 onwards -if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then - backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" -fi - -# Backup external MySQL password if running external MySQL DB. -if is_service_external 'mysql'; then - backup-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" -fi - -# Backup Actions settings. -if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then - backup-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" - backup-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" - backup-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" --best-effort - backup-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" - backup-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" - backup-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" - backup-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" - backup-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" - backup-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" - backup-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" --best-effort - backup-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" --best-effort - backup-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" --best-effort - backup-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" - backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" - backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort - backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" - backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" - - backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" - backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" - backup-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" - backup-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" - backup-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" - backup-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" - backup-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" - backup-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" - backup-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" - backup-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" - backup-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" - backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" -fi - -if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then - backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" - backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" - backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" - backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" - backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" - backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" - backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" -fi - -# Backup Chat Integration settings -if ghe-ssh "$host" -- ghe-config --true app.chatops.enabled; then - backup-secret "Chat Integration MSTeams app id" "chatops-msteams-app-id" "secrets.chatops.msteams.app-id" - backup-secret "Chat Integration MSTeams app password" "chatops-msteams-app-password" "secrets.chatops.msteams.app-password" - backup-secret "Chat Integration MSTeams public endpoint" "chatops-msteams-app-public-endpoint" "secrets.chatops.msteams.public-endpoint" - backup-secret "Chat Integration MSTeams bot handle" "chatops-msteams-bot-handle" "secrets.chatops.msteams.bot-handle" - backup-secret "Chat Integration MSTeams bot name" "chatops-msteams-bot-name" "secrets.chatops.msteams.bot-name" - backup-secret "Chat Integration Slack app id" "chatops-slack-app-id" "secrets.chatops.slack.app-id" - backup-secret "Chat Integration Slack client id" "chatops-slack-client-id" "secrets.chatops.slack.client-id" - backup-secret "Chat Integration Slack client secret" "chatops-slack-client-secret" "secrets.chatops.slack.client-secret" - backup-secret "Chat Integration Slack verification token" "chatops-slack-verification-token" "secrets.chatops.slack.verification-token" - backup-secret "Chat Integration Slack config token" "chatops-slack-config-token" "secrets.chatops.slack.config-token" - backup-secret "Chat Integration Slack public endpoint" "chatops-slack-public-endpoint" "secrets.chatops.slack.public-endpoint" - backup-secret "Chat Integration Slack signing secret" "chatops-slack-signing-secret" "secrets.chatops.slack.signing-secret" - backup-secret "Chat Integration Slack app level token" "chatops-slack-app-level-token" "secrets.chatops.slack.app-level-token" - backup-secret "Chat Integration Slack slack command" "chatops-slack-slash-command" "secrets.chatops.slack.slash-command" - backup-secret "Chat Integration Slack app name" "chatops-slack.app-name" "secrets.chatops.slack.app-name" - backup-secret "Chat Integration Slack socket mode" "chatops-slack.socket-mode" "secrets.chatops.slack.socket-mode" - backup-secret "Chat Integration public endpoint" "chatops-public-endpoint" "secrets.chatops.public-endpoint" - backup-secret "Chat Integration app type" "chatops-app-type" "secrets.chatops.app-type" - backup-secret "Chat Integration app id teams" "chatops-app-id-teams" "secrets.chatops.app-id-teams" - backup-secret "Chat Integration webhook secret teams" "chatops-webhook-secret-teams" "secrets.chatops.webhook-secret-teams" - backup-secret "Chat Integration client secret teams" "chatops-client-secret-teams" "secrets.chatops.client-secret-teams" - backup-secret "Chat Integration clien id teams" "chatops-client-id-teams" "secrets.chatops.client-id-teams" - backup-secret "Chat Integration storage secret" "chatops-storage-secret" "secrets.chatops.storage-secret" - backup-secret "Chat Integration session secret" "chatops-session-secret" "secrets.chatops.session-secret" - backup-secret "Chat Integration app id slack" "chatops-app-id-slack" "secrets.chatops.app-id-slack" - backup-secret "Chat Integration webhook secret slack" "chatops-webhook-secret-slack" "secrets.chatops.webhook-secret-slack" - backup-secret "Chat Integration client secret slack" "chatops-client-secret-slack" "secrets.chatops.client-secret-slack" - backup-secret "Chat Integration client id slack" "chatops-client-id-slack" "secrets.chatops.client-id-slack" -fi - if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then log_info "* Transferring SAML keys ..." 1>&3 ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar diff --git a/share/github-backup-utils/ghe-restore-actions b/share/github-backup-utils/ghe-restore-actions index 303a2abd2..b27dc9aa6 100755 --- a/share/github-backup-utils/ghe-restore-actions +++ b/share/github-backup-utils/ghe-restore-actions @@ -52,48 +52,6 @@ log_rsync "END: actions rsync" 1>&3 # Restore Actions settings. ghe_verbose "* Restoring Actions settings to $host ..." -restore-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" -restore-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" -restore-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" -restore-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" -restore-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" -restore-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" -restore-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" -restore-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" -restore-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" -restore-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" -restore-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" -restore-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" -restore-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" -restore-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" -restore-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" -restore-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" - -restore-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" -restore-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" -restore-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" -restore-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" -restore-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" -restore-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" -restore-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" -restore-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" -restore-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" -restore-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" -restore-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" -restore-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" - -restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" -restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" - -# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings -if [[ $RESTORE_SETTINGS == "true" ]]; then - if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then - restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" - else - log_warn "Actions storage container prefix not present in backup. Skipping ..." - fi -fi - # Setup the database logins. ghe_verbose "* Restoring database logins and users to $host ..." diff --git a/share/github-backup-utils/ghe-restore-chat-integration b/share/github-backup-utils/ghe-restore-chat-integration deleted file mode 100755 index 0f0ba4b44..000000000 --- a/share/github-backup-utils/ghe-restore-chat-integration +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore-chat-integration -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-restore command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -bm_start "$(basename $0)" - -# Grab host arg -GHE_HOSTNAME="$1" - -# The snapshot to restore should be set by the ghe-restore command but this lets -# us run this script directly. -: ${GHE_RESTORE_SNAPSHOT:=current} - -# Path to snapshot dir we're restoring from -export GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" - -port=$(ssh_port_part "$GHE_HOSTNAME") -export port -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Restore Chat Integration settings. -ghe_verbose "Restoring Chat Integration settings ..." - -restore-secret "Chat Integration MSTeams app id" "chatops-msteams-app-id" "secrets.chatops.msteams.app-id" -restore-secret "Chat Integration MSTeams app password" "chatops-msteams-app-password" "secrets.chatops.msteams.app-password" -restore-secret "Chat Integration MSTeams public endpoint" "chatops-msteams-app-public-endpoint" "secrets.chatops.msteams.public-endpoint" -restore-secret "Chat Integration MSTeams bot handle" "chatops-msteams-bot-handle" "secrets.chatops.msteams.bot-handle" -restore-secret "Chat Integration MSTeams bot name" "chatops-msteams-bot-name" "secrets.chatops.msteams.bot-name" -restore-secret "Chat Integration Slack app id" "chatops-slack-app-id" "secrets.chatops.slack.app-id" -restore-secret "Chat Integration Slack client id" "chatops-slack-client-id" "secrets.chatops.slack.client-id" -restore-secret "Chat Integration Slack client secret" "chatops-slack-client-secret" "secrets.chatops.slack.client-secret" -restore-secret "Chat Integration Slack verification token" "chatops-slack-verification-token" "secrets.chatops.slack.verification-token" -restore-secret "Chat Integration Slack config token" "chatops-slack-config-token" "secrets.chatops.slack.config-token" -restore-secret "Chat Integration Slack public endpoint" "chatops-slack-public-endpoint" "secrets.chatops.slack.public-endpoint" -restore-secret "Chat Integration Slack signing secret" "chatops-slack-signing-secret" "secrets.chatops.slack.signing-secret" -restore-secret "Chat Integration Slack app level token" "chatops-slack-app-level-token" "secrets.chatops.slack.app-level-token" -restore-secret "Chat Integration Slack slack command" "chatops-slack-slash-command" "secrets.chatops.slack.slash-command" -restore-secret "Chat Integration Slack app name" "chatops-slack.app-name" "secrets.chatops.slack.app-name" -restore-secret "Chat Integration Slack socket mode" "chatops-slack.socket-mode" "secrets.chatops.slack.socket-mode" -restore-secret "Chat Integration public endpoint" "chatops-public-endpoint" "secrets.chatops.public-endpoint" -restore-secret "Chat Integration app type" "chatops-app-type" "secrets.chatops.app-type" -restore-secret "Chat Integration app id teams" "chatops-app-id-teams" "secrets.chatops.app-id-teams" -restore-secret "Chat Integration webhook secret teams" "chatops-webhook-secret-teams" "secrets.chatops.webhook-secret-teams" -restore-secret "Chat Integration client secret teams" "chatops-client-secret-teams" "secrets.chatops.client-secret-teams" -restore-secret "Chat Integration clien id teams" "chatops-client-id-teams" "secrets.chatops.client-id-teams" -restore-secret "Chat Integration storage secret" "chatops-storage-secret" "secrets.chatops.storage-secret" -restore-secret "Chat Integration session secret" "chatops-session-secret" "secrets.chatops.session-secret" -restore-secret "Chat Integration app id slack" "chatops-app-id-slack" "secrets.chatops.app-id-slack" -restore-secret "Chat Integration webhook secret slack" "chatops-webhook-secret-slack" "secrets.chatops.webhook-secret-slack" -restore-secret "Chat Integration client secret slack" "chatops-client-secret-slack" "secrets.chatops.client-secret-slack" -restore-secret "Chat Integration client id slack" "chatops-client-id-slack" "secrets.chatops.client-id-slack" - -bm_end "$(basename $0)" \ No newline at end of file diff --git a/share/github-backup-utils/ghe-restore-column-encryption-keys b/share/github-backup-utils/ghe-restore-column-encryption-keys deleted file mode 100755 index d30cacd51..000000000 --- a/share/github-backup-utils/ghe-restore-column-encryption-keys +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore-column-encryption-keys -#/ Restore the column encryption keys from a snapshot to the given . -#/ This script will be run automatically by `ghe-restore -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -bm_start "$(basename $0)" - -# Grab host arg -GHE_HOSTNAME="$1" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$GHE_HOSTNAME" - -# The snapshot to restore should be set by the ghe-restore command but this lets -# us run this script directly. -: ${GHE_RESTORE_SNAPSHOT:=current} - -# Path to snapshot dir we're restoring from -: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} - -# Restore encrypted column encryption keying material for GHES 3.7.0 onward -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then - log_info "Restoring encrypted column encryption keying material" - restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" -fi - -# Restore encrypted column current encryption key for GHES 3.8.0 onwards -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - log_info "Restoring encrypted column current encryption key" - restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" -fi - - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-mysql b/share/github-backup-utils/ghe-restore-mysql index 8e75ea0f1..a251c820a 100755 --- a/share/github-backup-utils/ghe-restore-mysql +++ b/share/github-backup-utils/ghe-restore-mysql @@ -29,9 +29,6 @@ export GHE_RESTORE_SNAPSHOT # The directory holding the snapshot to restore GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" -# Always restore the password pepper here since it is tied to the MySQL data. -restore-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" - if is_external_database_snapshot; then if [ -n "$EXTERNAL_DATABASE_RESTORE_SCRIPT" ]; then $EXTERNAL_DATABASE_RESTORE_SCRIPT diff --git a/share/github-backup-utils/ghe-restore-packages b/share/github-backup-utils/ghe-restore-packages deleted file mode 100755 index e9842d095..000000000 --- a/share/github-backup-utils/ghe-restore-packages +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore-packages -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-restore command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -bm_start "$(basename $0)" - -# Grab host arg -GHE_HOSTNAME="$1" - -# The snapshot to restore should be set by the ghe-restore command but this lets -# us run this script directly. -: ${GHE_RESTORE_SNAPSHOT:=current} - -# Path to snapshot dir we're restoring from -export GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" - -port=$(ssh_port_part "$GHE_HOSTNAME") -export port -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Restore Packages settings. -ghe_verbose "Restoring Packages settings ..." - -restore-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" -restore-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" -restore-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" -restore-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" -restore-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" -restore-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" -restore-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys b/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys deleted file mode 100755 index 04f7588a7..000000000 --- a/share/github-backup-utils/ghe-restore-secret-scanning-encryption-keys +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore-secret-scanning-encryption-keys -#/ Restore the secret scanning encryption keys from a snapshot to the given . -#/ This script will be run automatically by `ghe-restore` -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -bm_start "$(basename $0)" - -# Grab host arg -GHE_HOSTNAME="$1" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$GHE_HOSTNAME" - -# The snapshot to restore should be set by the ghe-restore command but this lets -# us run this script directly. -: ${GHE_RESTORE_SNAPSHOT:=current} - -# Path to snapshot dir we're restoring from -: ${GHE_RESTORE_SNAPSHOT_PATH:="$GHE_DATA_DIR/current"} - -# Restore secret scanning encrypted secrets storage keys if present -log_info "Restoring secret scanning encrypted secrets storage keys" -restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" -restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" - -# Restore secret scanning encrypted secrets transit keys if present -log_info "Restoring secret scanning encrypted secrets transit keys" -restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" -restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" - -# Restore secret scanning content scanning keys if present -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then - log_info "Restoring secret scanning content scanning keys" - restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-restore-secrets b/share/github-backup-utils/ghe-restore-secrets new file mode 100755 index 000000000..3d6b5dd8e --- /dev/null +++ b/share/github-backup-utils/ghe-restore-secrets @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +#/ Usage: ghe-restore-secrets +#/ +#/ Note: This script typically isn't called directly. It's invoked by the +#/ ghe-restore command. +set -e + +# Bring in the backup configuration +# shellcheck source=share/github-backup-utils/ghe-backup-config +. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" + +# Show usage and bail with no arguments +[ -z "$*" ] && print_usage + +bm_start "$(basename $0)" + +# Grab host arg +GHE_HOSTNAME="$1" + +# The snapshot to restore should be set by the ghe-restore command but this lets +# us run this script directly. +: ${GHE_RESTORE_SNAPSHOT:=current} + +# Path to snapshot dir we're restoring from +export GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT" + +host=$(ssh_host_part "$GHE_HOSTNAME") + +# Perform a host-check and establish GHE_REMOTE_XXX variables. +ghe_remote_version_required "$host" + +# Otherwise use legacy +# Function to restore a secret setting stored in a file. +# restore-secret +restore-secret() { + if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/$2" ]; then + echo "Restoring $1 ..." + echo "ghe-config '$3' '$(cat "$GHE_RESTORE_SNAPSHOT_PATH/$2")'" | + ghe-ssh "$GHE_HOSTNAME" -- /bin/bash + fi +} + +log_info "Restoring secrets and applying cleanup ..." 1>&3 + +restore-secret "management console password" "manage-password" "secrets.manage" +restore-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" +restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" +restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" + +restore-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" + +restore-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" + +restore-secret "Chat Integration MSTeams app id" "chatops-msteams-app-id" "secrets.chatops.msteams.app-id" +restore-secret "Chat Integration MSTeams app password" "chatops-msteams-app-password" "secrets.chatops.msteams.app-password" +restore-secret "Chat Integration MSTeams public endpoint" "chatops-msteams-app-public-endpoint" "secrets.chatops.msteams.public-endpoint" +restore-secret "Chat Integration MSTeams bot handle" "chatops-msteams-bot-handle" "secrets.chatops.msteams.bot-handle" +restore-secret "Chat Integration MSTeams bot name" "chatops-msteams-bot-name" "secrets.chatops.msteams.bot-name" +restore-secret "Chat Integration Slack app id" "chatops-slack-app-id" "secrets.chatops.slack.app-id" +restore-secret "Chat Integration Slack client id" "chatops-slack-client-id" "secrets.chatops.slack.client-id" +restore-secret "Chat Integration Slack client secret" "chatops-slack-client-secret" "secrets.chatops.slack.client-secret" +restore-secret "Chat Integration Slack verification token" "chatops-slack-verification-token" "secrets.chatops.slack.verification-token" +restore-secret "Chat Integration Slack config token" "chatops-slack-config-token" "secrets.chatops.slack.config-token" +restore-secret "Chat Integration Slack public endpoint" "chatops-slack-public-endpoint" "secrets.chatops.slack.public-endpoint" +restore-secret "Chat Integration Slack signing secret" "chatops-slack-signing-secret" "secrets.chatops.slack.signing-secret" +restore-secret "Chat Integration Slack app level token" "chatops-slack-app-level-token" "secrets.chatops.slack.app-level-token" +restore-secret "Chat Integration Slack slack command" "chatops-slack-slash-command" "secrets.chatops.slack.slash-command" +restore-secret "Chat Integration Slack app name" "chatops-slack.app-name" "secrets.chatops.slack.app-name" +restore-secret "Chat Integration Slack socket mode" "chatops-slack.socket-mode" "secrets.chatops.slack.socket-mode" +restore-secret "Chat Integration public endpoint" "chatops-public-endpoint" "secrets.chatops.public-endpoint" +restore-secret "Chat Integration app type" "chatops-app-type" "secrets.chatops.app-type" +restore-secret "Chat Integration app id teams" "chatops-app-id-teams" "secrets.chatops.app-id-teams" +restore-secret "Chat Integration webhook secret teams" "chatops-webhook-secret-teams" "secrets.chatops.webhook-secret-teams" +restore-secret "Chat Integration client secret teams" "chatops-client-secret-teams" "secrets.chatops.client-secret-teams" +restore-secret "Chat Integration clien id teams" "chatops-client-id-teams" "secrets.chatops.client-id-teams" +restore-secret "Chat Integration storage secret" "chatops-storage-secret" "secrets.chatops.storage-secret" +restore-secret "Chat Integration session secret" "chatops-session-secret" "secrets.chatops.session-secret" +restore-secret "Chat Integration app id slack" "chatops-app-id-slack" "secrets.chatops.app-id-slack" +restore-secret "Chat Integration webhook secret slack" "chatops-webhook-secret-slack" "secrets.chatops.webhook-secret-slack" +restore-secret "Chat Integration client secret slack" "chatops-client-secret-slack" "secrets.chatops.client-secret-slack" +restore-secret "Chat Integration client id slack" "chatops-client-id-slack" "secrets.chatops.client-id-slack" + +restore-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" +restore-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" +restore-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" +restore-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" +restore-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" +restore-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" +restore-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" + +# Restore storage container prefix, but only if it exists, and the `-c` option is used with ghe-restore to avoid staging instances using production bucket settings +if [[ $RESTORE_SETTINGS == "true" ]]; then + if [[ -e "$GHE_RESTORE_SNAPSHOT_PATH/actions-storage-container-prefix" ]]; then + restore-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" + else + log_warn "Actions storage container prefix not present in backup. Skipping ..." + fi +fi + +restore-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" +restore-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" +restore-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" +restore-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" +restore-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" +restore-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" +restore-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" +restore-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" +restore-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" +restore-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" +restore-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" +restore-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" +restore-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" +restore-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" +restore-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" +restore-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" + +restore-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" +restore-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" +restore-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" +restore-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" +restore-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" +restore-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" +restore-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" +restore-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" +restore-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" +restore-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" +restore-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" +restore-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" + +restore-secret "Actions Launch token oauth key" "actions-oauth-s2s-signing-key" "secrets.launch.token-oauth-key" +restore-secret "Actions Launch token oauth cert" "actions-oauth-s2s-signing-cert" "secrets.launch.token-oauth-cert" + +# Restore secret scanning +restore-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" +restore-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" +restore-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" +restore-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" +restore-secret "secret scanning user content delimited encryption root keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" + +# Restore encrypted column +restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" +restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" + +bm_end "$(basename $0)" + +exit 0 diff --git a/share/github-backup-utils/ghe-restore-settings b/share/github-backup-utils/ghe-restore-settings index ad06f30a6..8d20d8333 100755 --- a/share/github-backup-utils/ghe-restore-settings +++ b/share/github-backup-utils/ghe-restore-settings @@ -30,38 +30,11 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-license' < "$GHE_RESTORE_SNAPSHOT_PATH/en log_info "Restoring settings and applying configuration ..." -# Restore external MySQL password if running external MySQL DB. -restore-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" - -log_info "Restoring packages settings ..." -ghe-restore-packages "$GHE_HOSTNAME" 1>&3 - -echo "Restoring chat integration settings ..." -ghe-restore-chat-integration "$GHE_HOSTNAME" 1>&3 - # work around issue importing settings with bad storage mode values ( cat "$GHE_RESTORE_SNAPSHOT_PATH/settings.json" && echo ) | sed 's/"storage_mode": "device"/"storage_mode": "rootfs"/' | ghe-ssh "$GHE_HOSTNAME" -- '/usr/bin/env GHEBUVER=2 ghe-import-settings' 1>&3 -# Restore management console password hash if present. -restore-secret "management console password" "manage-password" "secrets.manage" - -# Restore management console argon2 secret if present. -restore-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" - -# Restore kredz.credz HMAC key if present. -restore-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" - -# Restore kredz.varz HMAC key if present. -restore-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" - -# Restore encrypted column encryption keying material if present -restore-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" - -# Restore encrypted column current encryption key if present -restore-secret "encrypted column current encryption key" "encrypted-column-current-encryption-key" "secrets.github.encrypted-column-current-encryption-key" - # Restore SAML keys if present. if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/saml-keys.tar" ]; then log_info "Restoring SAML keys ..." diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 3ea47e2b5..2b430e912 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -389,39 +389,6 @@ begin_test "ghe-restore with encrypted column current encryption key for version ) end_test -begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions below 3.8.0" -( - set -e - rm -rf "$GHE_REMOTE_ROOT_DIR" - setup_remote_metadata - - required_files=( - "secret-scanning-encrypted-secrets-current-storage-key" - "secret-scanning-encrypted-secrets-delimited-storage-keys" - "secret-scanning-encrypted-secrets-current-shared-transit-key" - "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" - ) - - for file in "${required_files[@]}"; do - echo "foo" >"$GHE_DATA_DIR/current/$file" - done - - GHE_REMOTE_VERSION=3.7.0 ghe-restore -v -f localhost - - required_secrets=( - "secrets.secret-scanning.encrypted-secrets-current-storage-key" - "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" - "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" - "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" - ) - - for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting these to not be set for versions below 3.8.0 - done -) -end_test - - begin_test "ghe-restore with secret scanning encrypted secrets encryption keys for versions 3.8.0+" ( set -e @@ -454,32 +421,6 @@ begin_test "ghe-restore with secret scanning encrypted secrets encryption keys f ) end_test -begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions below 3.11.0" -( - set -e - rm -rf "$GHE_REMOTE_ROOT_DIR" - setup_remote_metadata - - required_files=( - "secret-scanning-user-content-delimited-encryption-root-keys" - ) - - for file in "${required_files[@]}"; do - echo "foo" >"$GHE_DATA_DIR/current/$file" - done - - GHE_REMOTE_VERSION=3.10.0 ghe-restore -v -f localhost - - required_secrets=( - "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" - ) - - for secret in "${required_secrets[@]}"; do - [ "$(ghe-ssh "$GHE_HOSTNAME" -- ghe-config "$secret")" = "" ] # expecting that this secret was not backed up on versions below 3.11.0, this secret was not present in earlier versions - done -) -end_test - begin_test "ghe-restore with secret scanning encrypted content encryption keys for versions 3.11.0+" ( set -e From 5c1c6fa1309696d5c5b0e417aeeb82eaf6d88f3b Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 12 Oct 2023 11:54:36 -0400 Subject: [PATCH 257/410] Attempting to fix linting errors --- docs/usage.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index e4ed5b7e2..2918a34da 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,11 +2,10 @@ After the initial backup, use the following commands: - - The `ghe-backup` command creates incremental snapshots of repository data, - along with full snapshots of all other pertinent data stores. - - The `ghe-restore` command restores snapshots to the same or separate GitHub - Enterprise appliance. You must add the backup host's SSH key to the target - GitHub Enterprise Server appliance before using this command. + - The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores. + - The `ghe-restore` command restores snapshots to the same or separate GitHub +Enterprise appliance. You must add the backup host's SSH key to the target +GitHub Enterprise Server appliance before using this command. These commands are run on the host you [installed][1] Backup Utilities on. From 676c3041c1e68fddb2e433678e4ccf25faf0d8d2 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 12 Oct 2023 15:23:17 -0400 Subject: [PATCH 258/410] Trying to fix linting errors again. --- docs/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 2918a34da..8d72ba323 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,8 +2,8 @@ After the initial backup, use the following commands: - - The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores. - - The `ghe-restore` command restores snapshots to the same or separate GitHub +- The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores. +- The `ghe-restore` command restores snapshots to the same or separate GitHub Enterprise appliance. You must add the backup host's SSH key to the target GitHub Enterprise Server appliance before using this command. From 1354eb8e512722c76f6bdefa2b3ea0a470f3e6e8 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 12 Oct 2023 15:28:08 -0400 Subject: [PATCH 259/410] Another round of linting error fixes --- docs/usage.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 8d72ba323..29d08de04 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -3,7 +3,7 @@ After the initial backup, use the following commands: - The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores. -- The `ghe-restore` command restores snapshots to the same or separate GitHub +- The `ghe-restore` command restores snapshots to the same or separate GitHub Enterprise appliance. You must add the backup host's SSH key to the target GitHub Enterprise Server appliance before using this command. @@ -79,12 +79,12 @@ The `ghe-backup` and `ghe-restore` commands also have a verbose output mode (`-v`) that lists files as they're being transferred. It's often useful to enable when output is logged to a file. -Every time you execute `ghe-backup` we verify the storage and software setup of the host -you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are -met. You can disable this check using the `--skip-checks` argument or by +Every time you execute `ghe-backup` we verify the storage and software setup of the host +you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are +met. You can disable this check using the `--skip-checks` argument or by adding `GHE_SKIP_CHECKS=true` to your configuration file. -### Restoring settings, TLS certificate, and license +### Restoring settings, TLS certificate, and license When restoring to a new GitHub Enterprise Server instance, settings, certificate, and license data *are* restored. These settings must be reviewed and saved before @@ -99,7 +99,7 @@ forcing settings, certificate, and license data to be overwritten with the backu ## Backup and restore with GitHub Actions enabled GitHub Actions data on your external storage provider is not included in regular GitHub Enterprise Server -backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with +backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with GitHub Actions enabled, the following steps are required: 1. Enable GitHub Actions on the replacement appliance and configure it to use the same GitHub Actions From 559f7e62ca2cf1ac466bbc7ae3028a5e7320f84f Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 12 Oct 2023 15:44:43 -0400 Subject: [PATCH 260/410] More linting fixes --- docs/usage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 29d08de04..4324b58cc 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -15,7 +15,7 @@ You can supply your own configuration file or use the example configuration file An example configuration file with documentation on possible settings can found in [backup.config-example](../backup.config-example). -There are a number of command line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`. +There are a number of command-line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`. ## Example backup and restore usage @@ -104,7 +104,7 @@ GitHub Actions enabled, the following steps are required: 1. Enable GitHub Actions on the replacement appliance and configure it to use the same GitHub Actions external storage configuration as the original appliance. -2. Put replacement appliance into maintenance mode. +2. Put replacement appliance into maintenance mode. 3. Use `ghe-restore` to restore the backup. 4. Re-register your self-hosted runners on the replacement appliance. From b53f1d5b599d0595dfd36322fbabdf95ac097d52 Mon Sep 17 00:00:00 2001 From: Hao Jiang <45571951+jianghao0718@users.noreply.github.com> Date: Thu, 12 Oct 2023 13:47:55 -0600 Subject: [PATCH 261/410] Remove unused code in ghe-restore-es-rsync --- share/github-backup-utils/ghe-restore-es-rsync | 4 ---- 1 file changed, 4 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-es-rsync b/share/github-backup-utils/ghe-restore-es-rsync index 1ba09afed..80a7f729a 100755 --- a/share/github-backup-utils/ghe-restore-es-rsync +++ b/share/github-backup-utils/ghe-restore-es-rsync @@ -44,10 +44,6 @@ else "$snapshot_dir/elasticsearch/" \ "$(ssh_host_part "$GHE_HOSTNAME"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore" 1>&3 log_rsync "END: elasticsearch rsync" 1>&3 - # restoring in >=2.14 will remove incompatible indices created with 1.x. - if [ "$GHE_VERSION_MAJOR" -eq 2 ] && [ "$GHE_VERSION_MINOR" -ge 14 ]; then - ghe-ssh "$GHE_HOSTNAME" -- "sudo /usr/local/share/enterprise/ghe-es-remove-1x-indices" - fi fi bm_end "$(basename $0)" From 58d94fc0303750861e6e2af1e6f4a751aa98ce56 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 13:48:04 +0000 Subject: [PATCH 262/410] test create-github-app-token on test branch --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 49a7c0156..008484729 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -24,7 +24,7 @@ jobs: steps: # resulting token still gets denied by the backup-utils repo # see: https://github.com/actions/create-github-app-token/pull/46 - - uses: timreimherr/create-github-app-token@main + - uses: actions/create-github-app-token@v1 id: app-token with: # required From 2d2a3d3bb4d77af4b108752e5d50a06ba01ddfb6 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:05:06 +0000 Subject: [PATCH 263/410] update build-and-release to use action --- .github/workflows/build-and-release.yml | 201 +++++++++++------------- 1 file changed, 88 insertions(+), 113 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 008484729..a9e5cf2a5 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,10 +4,6 @@ name: Build and Release on: workflow_dispatch: inputs: - gh-token: - description: 'GitHub Token - used to create a commit in the backup-utils repo' - required: true - type: string version: description: 'Version - patch version of the release (e.g. x.y.z)' required: true @@ -22,14 +18,67 @@ jobs: build: runs-on: ubuntu-latest steps: - # resulting token still gets denied by the backup-utils repo - # see: https://github.com/actions/create-github-app-token/pull/46 - uses: actions/create-github-app-token@v1 id: app-token with: # required - app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + # required + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils @@ -37,120 +86,46 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils + ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: tims-test-branch + branch: master commit_message: "${{ github.event.inputs.version }} release" commit_user_name: "${{ github.actor }}" commit_user_email: "ghes-releases-team@github.com" commit_options: "--allow-empty" skip_dirty_check: true - - name: Log a message - if: success() - run: echo "The previous step succeeded" - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils-private - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - # steps: - # # resulting token still gets denied by the backup-utils repo - # # see: https://github.com/actions/create-github-app-token/pull/46 - # # - uses: timreimherr/create-github-app-token@main - # # id: app-token - # # with: - # # app_id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # # private_key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # # owner: ${{ github.repository_owner }} - # # repositories: backup-utils,backup-utils-private - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit - # uses: stefanzweifel/git-auto-commit-action@v4 - # id: empty-commit - # with: - # branch: master - # commit_message: "${{ github.event.inputs.version }} release" - # commit_user_name: "${{ github.actor }}" - # commit_user_email: "ghes-releases-team@github.com" - # commit_options: "--allow-empty" - # skip_dirty_check: true - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repository: github/backup-utils-private - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ github.event.inputs.gh-token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # tag: v${{ github.event.inputs.version }} - # commit: ${{ steps.empty-commit.outputs.commit_hash }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From c1a7c09973bb8ace42236631cf8ad5451fcaae5e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:05:21 +0000 Subject: [PATCH 264/410] test release notes for test --- release-notes/12.12.12.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release-notes/12.12.12.md diff --git a/release-notes/12.12.12.md b/release-notes/12.12.12.md new file mode 100644 index 000000000..445d9aada --- /dev/null +++ b/release-notes/12.12.12.md @@ -0,0 +1,7 @@ +### Changes + +* Test release notes. + +### Bug fixes + +* More test release notes. \ No newline at end of file From e69f7d1b8220f6475ce82fb1bf4af586bbd2c63d Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:50:59 +0000 Subject: [PATCH 265/410] test empty commit against master --- .github/workflows/build-and-release.yml | 189 +++++++++++++----------- 1 file changed, 102 insertions(+), 87 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a9e5cf2a5..d77906fa0 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -17,61 +17,6 @@ on: jobs: build: runs-on: ubuntu-latest - steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - # required - app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "backup-utils,backup-utils-private" - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils-private - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - release: - needs: build - runs-on: ubuntu-latest - outputs: - commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} steps: - uses: actions/create-github-app-token@v1 id: app-token @@ -86,46 +31,116 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: master - name: Create empty commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: branch: master - commit_message: "${{ github.event.inputs.version }} release" - commit_user_name: "${{ github.actor }}" + commit_message: "Test ${{ github.event.inputs.version }} release commit" + commit_user_name: "release-controller[bot]" commit_user_email: "ghes-releases-team@github.com" + commit_author: "release-controller[bot]" commit_options: "--allow-empty" + push_options: "--force" skip_dirty_check: true - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils-private - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - - name: Create Release - uses: ncipollo/release-action@v1 - with: - token: ${{ steps.app-token.outputs.token }} - repo: backup-utils - name: | - GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb - tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} - bodyFile: release-notes/${{ github.event.inputs.version }}.md - draft: ${{ github.event.inputs.draft }} - allowUpdates: true - artifactContentType: "raw" + - name: Log a message + if: success() + run: echo "The previous step succeeded" + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # release: + # needs: build + # runs-on: ubuntu-latest + # outputs: + # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + # steps: + # - uses: actions/create-github-app-token@v1 + # id: app-token + # with: + # # required + # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: "backup-utils,backup-utils-private" + # - name: Checkout backup-utils + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # - name: Create empty commit + # uses: stefanzweifel/git-auto-commit-action@v4 + # id: empty-commit + # with: + # branch: master + # commit_message: "${{ github.event.inputs.version }} release" + # commit_user_name: "${{ github.actor }}" + # commit_user_email: "ghes-releases-team@github.com" + # commit_options: "--allow-empty" + # skip_dirty_check: true + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Create Release + # uses: ncipollo/release-action@v1 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repo: backup-utils + # name: | + # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb + # tag: v${{ github.event.inputs.version }} + # commit: ${{ steps.empty-commit.outputs.commit_hash }} + # bodyFile: release-notes/${{ github.event.inputs.version }}.md + # draft: ${{ github.event.inputs.draft }} + # allowUpdates: true + # artifactContentType: "raw" From 2289bfcafe941eb04896bcf26943479c53133f59 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:54:14 +0000 Subject: [PATCH 266/410] author --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index d77906fa0..c0c35ea15 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,7 +39,7 @@ jobs: commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "ghes-releases-team@github.com" - commit_author: "release-controller[bot]" + commit_author: "release-controller" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 8553bd8c99dba00546c6e23f3a3a8d9383d03e81 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:55:54 +0000 Subject: [PATCH 267/410] author 2 --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c0c35ea15..83df40618 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,7 +39,7 @@ jobs: commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "ghes-releases-team@github.com" - commit_author: "release-controller" + commit_author: "release-controller ghes-releases-team@github.com" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 2fdfafe0bb1d0517ea6dbd91fb2efbd834e50055 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 14:57:40 +0000 Subject: [PATCH 268/410] don't set author --- .github/workflows/build-and-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 83df40618..eed8d8de9 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,9 +37,8 @@ jobs: with: branch: master commit_message: "Test ${{ github.event.inputs.version }} release commit" - commit_user_name: "release-controller[bot]" + commit_user_name: "release-controller" commit_user_email: "ghes-releases-team@github.com" - commit_author: "release-controller ghes-releases-team@github.com" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From f60549f73321af1e6adcde885eeb2d34ce9da4b0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 15:01:48 +0000 Subject: [PATCH 269/410] try release-controller again --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index eed8d8de9..006b24c9d 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,6 +39,7 @@ jobs: commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller" commit_user_email: "ghes-releases-team@github.com" + commit_author: "release-controller " commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 43978250559a6efd19b605c0bcc2b82f248cce7c Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 15:07:30 +0000 Subject: [PATCH 270/410] default author --- .github/workflows/build-and-release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 006b24c9d..8bade7a1f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,9 +37,8 @@ jobs: with: branch: master commit_message: "Test ${{ github.event.inputs.version }} release commit" - commit_user_name: "release-controller" + commit_user_name: "${{ github.actor }}" commit_user_email: "ghes-releases-team@github.com" - commit_author: "release-controller " commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From a4e63ea04210538074d60e744c7e295fe405ac6b Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 17:37:38 +0000 Subject: [PATCH 271/410] set author --- .github/workflows/build-and-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8bade7a1f..d3293be75 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,8 +37,9 @@ jobs: with: branch: master commit_message: "Test ${{ github.event.inputs.version }} release commit" - commit_user_name: "${{ github.actor }}" + commit_user_name: "release-controller" commit_user_email: "ghes-releases-team@github.com" + commit_author: "release-controller " commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 05950382eedea3ed27a11fa40d345230fa31824e Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 17:41:42 +0000 Subject: [PATCH 272/410] use rc --- .github/workflows/build-and-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index d3293be75..496cefdaf 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -37,9 +37,9 @@ jobs: with: branch: master commit_message: "Test ${{ github.event.inputs.version }} release commit" - commit_user_name: "release-controller" - commit_user_email: "ghes-releases-team@github.com" - commit_author: "release-controller " + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + commit_author: "release-controller[bot] <223695+release-controller[bot]@users.noreply.github.com>" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 827d8800f25b3505eb17eaa48ba7021db3801ef9 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 17:52:19 +0000 Subject: [PATCH 273/410] remove rc --- .github/workflows/build-and-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 496cefdaf..0e89ed42b 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -39,7 +39,6 @@ jobs: commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" - commit_author: "release-controller[bot] <223695+release-controller[bot]@users.noreply.github.com>" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true From 0a9894e9a4a90d422427b4ea6e46503d92e9efc2 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:31:55 +0000 Subject: [PATCH 274/410] enable setting commit sha for release --- .github/workflows/build-and-release.yml | 198 ++++++++++++------------ 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 0e89ed42b..6d6ccbf44 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -13,6 +13,10 @@ on: required: true type: boolean default: true + backup-utils-release-commit: + description: 'Release Commit - true if the release should be a release commit' + required: false + type: string jobs: build: @@ -26,12 +30,69 @@ jobs: private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + release: + needs: build + runs-on: ubuntu-latest + outputs: + commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} + release-commit: ${{ steps.resolve-release-commit.outputs.release-commit }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils + if: !github.event.inputs.backup-utils-release-commit uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils + ref: master - name: Create empty commit + if: !github.event.inputs.backup-utils-release-commit uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -42,104 +103,45 @@ jobs: commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true - - name: Log a message - if: success() - run: echo "The previous step succeeded" - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # release: - # needs: build - # runs-on: ubuntu-latest - # outputs: - # commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - # steps: - # - uses: actions/create-github-app-token@v1 - # id: app-token - # with: - # # required - # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: "backup-utils,backup-utils-private" - # - name: Checkout backup-utils - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit - # uses: stefanzweifel/git-auto-commit-action@v4 - # id: empty-commit - # with: - # branch: master - # commit_message: "${{ github.event.inputs.version }} release" - # commit_user_name: "${{ github.actor }}" - # commit_user_email: "ghes-releases-team@github.com" - # commit_options: "--allow-empty" - # skip_dirty_check: true - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # - name: Create Release - # uses: ncipollo/release-action@v1 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repo: backup-utils - # name: | - # GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb - # tag: v${{ github.event.inputs.version }} - # commit: ${{ steps.empty-commit.outputs.commit_hash }} - # bodyFile: release-notes/${{ github.event.inputs.version }}.md - # draft: ${{ github.event.inputs.draft }} - # allowUpdates: true - # artifactContentType: "raw" + - name: Resolve release commit + id: resolve-release-commit + if: github.event.inputs.backup-utils-release-commit + run: | + echo "Useing sha from input backup-utils-release-commit" + echo "release-commit=${{ github.event.inputs.backup-utils-release-commit }}" >> $GITHUB_OUTPUT + else: + echo "Using sha from empty commit" + echo "release-commit=${{ steps.empty-commit.outputs.commit_hash }}" >> $GITHUB_OUTPUT + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Create Release + uses: ncipollo/release-action@v1 + with: + token: ${{ steps.app-token.outputs.token }} + repo: backup-utils + name: | + GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb + tag: v${{ github.event.inputs.version }} + # use release-commit value + commit: ${{ steps.resolve-release-commit.outputs.release-commit }} + bodyFile: release-notes/${{ github.event.inputs.version }}.md + draft: ${{ github.event.inputs.draft }} + allowUpdates: true + artifactContentType: "raw" From be24c6e1230efd2223abd88d2f8e9b2442b376ec Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:34:16 +0000 Subject: [PATCH 275/410] update input description --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 6d6ccbf44..23431075c 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -14,7 +14,7 @@ on: type: boolean default: true backup-utils-release-commit: - description: 'Release Commit - true if the release should be a release commit' + description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release, if empty an empty commit will be created on master' required: false type: string From d02b7d68ef495dc9737d4af8a8ba5a51fd4fb048 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:35:17 +0000 Subject: [PATCH 276/410] update desc --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 23431075c..f4adddf23 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -14,7 +14,7 @@ on: type: boolean default: true backup-utils-release-commit: - description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release, if empty an empty commit will be created on master' + description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release. If blank, an empty commit will be created on master' required: false type: string From 38b8cdd449448bfb72a7bd9a31e05610c9f39961 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:35:53 +0000 Subject: [PATCH 277/410] reorder inputs --- .github/workflows/build-and-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f4adddf23..7dd6e6897 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -8,15 +8,15 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string + backup-utils-release-commit: + description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release. If blank, an empty commit will be created on master' + required: false + type: string draft: description: 'Draft - true if the release should be a draft' required: true type: boolean default: true - backup-utils-release-commit: - description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release. If blank, an empty commit will be created on master' - required: false - type: string jobs: build: From e164d529f7956097a74fd84d58c4479f3ca77b3a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:38:43 +0000 Subject: [PATCH 278/410] fix eval statements --- .github/workflows/build-and-release.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7dd6e6897..055228c89 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -85,14 +85,15 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - if: !github.event.inputs.backup-utils-release-commit + if: github.event.inputs.backup-utils-release-commit == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit - if: !github.event.inputs.backup-utils-release-commit + # if github.event.inputs.backup-utils-release-commit is empty string, then create an empty commit + if: github.event.inputs.backup-utils-release-commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -105,7 +106,7 @@ jobs: skip_dirty_check: true - name: Resolve release commit id: resolve-release-commit - if: github.event.inputs.backup-utils-release-commit + if: github.event.inputs.backup-utils-release-commit != '' run: | echo "Useing sha from input backup-utils-release-commit" echo "release-commit=${{ github.event.inputs.backup-utils-release-commit }}" >> $GITHUB_OUTPUT From 77d98638d88690fec292dfbf8f66f426ddfe0984 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:42:03 +0000 Subject: [PATCH 279/410] fix resolve commit --- .github/workflows/build-and-release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 055228c89..c3e1d560a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -106,13 +106,14 @@ jobs: skip_dirty_check: true - name: Resolve release commit id: resolve-release-commit - if: github.event.inputs.backup-utils-release-commit != '' run: | - echo "Useing sha from input backup-utils-release-commit" - echo "release-commit=${{ github.event.inputs.backup-utils-release-commit }}" >> $GITHUB_OUTPUT - else: - echo "Using sha from empty commit" - echo "release-commit=${{ steps.empty-commit.outputs.commit_hash }}" >> $GITHUB_OUTPUT + if [[ -z "${{ github.event.inputs.backup-utils-release-commit }}" ]]; then + echo "Using empty commit sha for release commit" + echo "release-commit=${{ steps.empty-commit.outputs.commit_hash }}" >> $GITHUB_OUTPUT + else + echo "Using provided commit sha for release commit" + echo "release-commit=${{ github.event.inputs.backup-utils-release-commit }}" >> $GITHUB_OUTPUT + fi - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From 5a2c6f1248c76992b730b32ee5898d76dce672e0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:52:06 +0000 Subject: [PATCH 280/410] remove blank lines --- .github/workflows/build-and-release.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c3e1d560a..1667f5bf1 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -85,7 +85,7 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - if: github.event.inputs.backup-utils-release-commit == '' + if: ${{ github.event.inputs.backup-utils-release-commit }} == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} @@ -93,7 +93,7 @@ jobs: ref: master - name: Create empty commit # if github.event.inputs.backup-utils-release-commit is empty string, then create an empty commit - if: github.event.inputs.backup-utils-release-commit == '' + if: ${{ github.event.inputs.backup-utils-release-commit }} == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -143,8 +143,4 @@ jobs: bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true - artifactContentType: "raw" - - - - + artifactContentType: "raw" \ No newline at end of file From 00b46d937b8b62d3894705b723eaea09192bcd34 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 21:54:34 +0000 Subject: [PATCH 281/410] avoid globbing? --- .github/workflows/build-and-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 1667f5bf1..bdc2a69db 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -107,13 +107,13 @@ jobs: - name: Resolve release commit id: resolve-release-commit run: | - if [[ -z "${{ github.event.inputs.backup-utils-release-commit }}" ]]; then - echo "Using empty commit sha for release commit" - echo "release-commit=${{ steps.empty-commit.outputs.commit_hash }}" >> $GITHUB_OUTPUT + "if [[ -z '${{ github.event.inputs.backup-utils-release-commit }}' ]]; then + echo 'Using empty commit sha for release commit' + echo 'release-commit=${{ steps.empty-commit.outputs.commit_hash }}' >> $GITHUB_OUTPUT else - echo "Using provided commit sha for release commit" - echo "release-commit=${{ github.event.inputs.backup-utils-release-commit }}" >> $GITHUB_OUTPUT - fi + echo 'Using provided commit sha for release commit' + echo 'release-commit=${{ github.event.inputs.backup-utils-release-commit }}' >> $GITHUB_OUTPUT + fi" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From c506a52b6247d931ec6cc76ab03edb5d7ec80647 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 22:13:08 +0000 Subject: [PATCH 282/410] eval was correct, revert --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index bdc2a69db..aecb756c2 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -85,7 +85,7 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - if: ${{ github.event.inputs.backup-utils-release-commit }} == '' + if: github.event.inputs.backup-utils-release-commit == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} @@ -93,7 +93,7 @@ jobs: ref: master - name: Create empty commit # if github.event.inputs.backup-utils-release-commit is empty string, then create an empty commit - if: ${{ github.event.inputs.backup-utils-release-commit }} == '' + if: github.event.inputs.backup-utils-release-commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: From 70f21fffea0849dbcd6d798aa88797aaa4f588e4 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 22:21:18 +0000 Subject: [PATCH 283/410] does copilot know the answer? --- .github/workflows/build-and-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index aecb756c2..62100f420 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -107,13 +107,13 @@ jobs: - name: Resolve release commit id: resolve-release-commit run: | - "if [[ -z '${{ github.event.inputs.backup-utils-release-commit }}' ]]; then - echo 'Using empty commit sha for release commit' - echo 'release-commit=${{ steps.empty-commit.outputs.commit_hash }}' >> $GITHUB_OUTPUT + if [[ -z "${{ github.event.inputs.backup-utils-release-commit }}" ]]; then + echo "Using empty commit sha for release commit" + echo "release-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" else - echo 'Using provided commit sha for release commit' - echo 'release-commit=${{ github.event.inputs.backup-utils-release-commit }}' >> $GITHUB_OUTPUT - fi" + echo "Using provided commit sha for release commit" + echo "release-commit=\"${{ github.event.inputs.backup-utils-release-commit }}\"" >> "$GITHUB_OUTPUT" + fi - name: Checkout backup-utils-private uses: actions/checkout@v4 with: From 425162cce2c6bf2317f1ab22e7dc6c4d0f22d66d Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 23:35:35 +0000 Subject: [PATCH 284/410] don't need priavte repo in release? --- .github/workflows/build-and-release.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 62100f420..c727e45ea 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -25,7 +25,6 @@ jobs: - uses: actions/create-github-app-token@v1 id: app-token with: - # required app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} @@ -92,7 +91,6 @@ jobs: repository: github/backup-utils ref: master - name: Create empty commit - # if github.event.inputs.backup-utils-release-commit is empty string, then create an empty commit if: github.event.inputs.backup-utils-release-commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit @@ -114,11 +112,11 @@ jobs: echo "Using provided commit sha for release commit" echo "release-commit=\"${{ github.event.inputs.backup-utils-release-commit }}\"" >> "$GITHUB_OUTPUT" fi - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils-private + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 with: @@ -138,7 +136,6 @@ jobs: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - # use release-commit value commit: ${{ steps.resolve-release-commit.outputs.release-commit }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} From c7dc45529bf4eafb36fb1c86d30c6f2d45101b31 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Mon, 16 Oct 2023 23:39:13 +0000 Subject: [PATCH 285/410] add owner --- .github/workflows/build-and-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c727e45ea..62bb55b22 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -129,6 +129,7 @@ jobs: uses: ncipollo/release-action@v1 with: token: ${{ steps.app-token.outputs.token }} + owner: github repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} From ea39b5eb1112c7d1e980d8f3d3786d11896362a0 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 09:06:56 -0400 Subject: [PATCH 286/410] for release notes --- .github/workflows/build-and-release.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 62bb55b22..1bf9e9124 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -112,11 +112,12 @@ jobs: echo "Using provided commit sha for release commit" echo "release-commit=\"${{ github.event.inputs.backup-utils-release-commit }}\"" >> "$GITHUB_OUTPUT" fi - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils-private + # need backup-utils-private for release notes + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 with: From d9a2830f5d85d40ef06fa764c600f5629b49f543 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 09:13:27 -0400 Subject: [PATCH 287/410] can use branch ref also --- .github/workflows/build-and-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 1bf9e9124..efd0d16b6 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -8,8 +8,8 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string - backup-utils-release-commit: - description: 'Backup Utils Release Commit - sha of the commit to associate with backup-utils release. If blank, an empty commit will be created on master' + branch-ref-or-commit: + description: 'Branch ref or commit - the backup-utils release will associated with the branch ref or commit provided. If blank, an empty commit will be created on master' required: false type: string draft: @@ -84,14 +84,14 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - if: github.event.inputs.backup-utils-release-commit == '' + if: github.event.inputs.branch-ref-or-commit == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit - if: github.event.inputs.backup-utils-release-commit == '' + if: github.event.inputs.branch-ref-or-commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -105,12 +105,12 @@ jobs: - name: Resolve release commit id: resolve-release-commit run: | - if [[ -z "${{ github.event.inputs.backup-utils-release-commit }}" ]]; then + if [[ -z "${{ github.event.inputs.branch-ref-or-commit }}" ]]; then echo "Using empty commit sha for release commit" echo "release-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" else echo "Using provided commit sha for release commit" - echo "release-commit=\"${{ github.event.inputs.backup-utils-release-commit }}\"" >> "$GITHUB_OUTPUT" + echo "release-commit=\"${{ github.event.inputs.branch-ref-or-commit }}\"" >> "$GITHUB_OUTPUT" fi # need backup-utils-private for release notes - name: Checkout backup-utils-private From 80d4abea80c44b6db69fbb22f64f4714db4fa358 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 09:26:06 -0400 Subject: [PATCH 288/410] reuse app token --- .github/workflows/build-and-release.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index efd0d16b6..bf849a8da 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -20,7 +20,10 @@ on: jobs: build: + name: build runs-on: ubuntu-latest + outputs: + token: ${{ steps.app-token.outputs.token }} steps: - uses: actions/create-github-app-token@v1 id: app-token @@ -74,20 +77,13 @@ jobs: runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - release-commit: ${{ steps.resolve-release-commit.outputs.release-commit }} + release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils if: github.event.inputs.branch-ref-or-commit == '' uses: actions/checkout@v4 with: - token: ${{ steps.app-token.outputs.token }} + token: ${{ needs.build.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit @@ -107,16 +103,16 @@ jobs: run: | if [[ -z "${{ github.event.inputs.branch-ref-or-commit }}" ]]; then echo "Using empty commit sha for release commit" - echo "release-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" + echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" else echo "Using provided commit sha for release commit" - echo "release-commit=\"${{ github.event.inputs.branch-ref-or-commit }}\"" >> "$GITHUB_OUTPUT" + echo "release-ref-or-commit=\"${{ github.event.inputs.branch-ref-or-commit }}\"" >> "$GITHUB_OUTPUT" fi # need backup-utils-private for release notes - name: Checkout backup-utils-private uses: actions/checkout@v4 with: - token: ${{ steps.app-token.outputs.token }} + token: ${{ needs.build.outputs.token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 @@ -129,7 +125,7 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ steps.app-token.outputs.token }} + token: ${{ needs.build.outputs.token }} owner: github repo: backup-utils name: | @@ -138,7 +134,7 @@ jobs: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.resolve-release-commit.outputs.release-commit }} + commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From 5641ebcaa212449c74268e1ef9be6ebe629c6075 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 09:41:15 -0400 Subject: [PATCH 289/410] fic input name --- .github/workflows/build-and-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index bf849a8da..12a6e5f7f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -8,7 +8,7 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string - branch-ref-or-commit: + branch_ref_or_commit: description: 'Branch ref or commit - the backup-utils release will associated with the branch ref or commit provided. If blank, an empty commit will be created on master' required: false type: string @@ -80,14 +80,14 @@ jobs: release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} steps: - name: Checkout backup-utils - if: github.event.inputs.branch-ref-or-commit == '' + if: github.event.inputs.branch_ref_or_commit == '' uses: actions/checkout@v4 with: token: ${{ needs.build.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit - if: github.event.inputs.branch-ref-or-commit == '' + if: github.event.inputs.branch_ref_or_commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -101,12 +101,12 @@ jobs: - name: Resolve release commit id: resolve-release-commit run: | - if [[ -z "${{ github.event.inputs.branch-ref-or-commit }}" ]]; then + if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then echo "Using empty commit sha for release commit" echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" else echo "Using provided commit sha for release commit" - echo "release-ref-or-commit=\"${{ github.event.inputs.branch-ref-or-commit }}\"" >> "$GITHUB_OUTPUT" + echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" fi # need backup-utils-private for release notes - name: Checkout backup-utils-private From 7b286fc3c08c01d4f9a66affd5c88ed9f2ddf764 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 12:28:06 -0400 Subject: [PATCH 290/410] skip token revoke --- .github/workflows/build-and-release.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 12a6e5f7f..18a7cb596 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -23,7 +23,7 @@ jobs: name: build runs-on: ubuntu-latest outputs: - token: ${{ steps.app-token.outputs.token }} + rc-app-token: ${{ steps.app-token.outputs.token }} steps: - uses: actions/create-github-app-token@v1 id: app-token @@ -32,6 +32,7 @@ jobs: private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" + skip-token-revoke: true - name: Checkout backup-utils-private uses: actions/checkout@v4 with: @@ -83,7 +84,7 @@ jobs: if: github.event.inputs.branch_ref_or_commit == '' uses: actions/checkout@v4 with: - token: ${{ needs.build.outputs.token }} + token: ${{ needs.build.outputs.rc-app-token }} repository: github/backup-utils ref: master - name: Create empty commit @@ -108,11 +109,10 @@ jobs: echo "Using provided commit sha for release commit" echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" fi - # need backup-utils-private for release notes - - name: Checkout backup-utils-private + - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: - token: ${{ needs.build.outputs.token }} + token: ${{ needs.build.outputs.rc-app-token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 @@ -125,7 +125,7 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ needs.build.outputs.token }} + token: ${{ needs.build.outputs.rc-app-token }} owner: github repo: backup-utils name: | From fef0eec8daf907694a57dda4b1b4163b4230133f Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 13:55:51 -0400 Subject: [PATCH 291/410] use create token again --- .github/workflows/build-and-release.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 18a7cb596..31eca6eab 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -20,7 +20,6 @@ on: jobs: build: - name: build runs-on: ubuntu-latest outputs: rc-app-token: ${{ steps.app-token.outputs.token }} @@ -31,8 +30,7 @@ jobs: app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} - repositories: "backup-utils,backup-utils-private" - skip-token-revoke: true + repositories: "backup-utils-private" - name: Checkout backup-utils-private uses: actions/checkout@v4 with: @@ -80,11 +78,18 @@ jobs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils if: github.event.inputs.branch_ref_or_commit == '' uses: actions/checkout@v4 with: - token: ${{ needs.build.outputs.rc-app-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils ref: master - name: Create empty commit @@ -109,10 +114,14 @@ jobs: echo "Using provided commit sha for release commit" echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" fi + - name: Check rc-app-token value + if: needs.build.outputs.rc-app-token == '' + run: | + echo "****** rc-app-token is empty ******" - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: - token: ${{ needs.build.outputs.rc-app-token }} + token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - name: Download deb artifact uses: actions/download-artifact@v3 From a8e052d1b0c28a88c523072aba995808db9f54ec Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 14:43:19 -0400 Subject: [PATCH 292/410] use app token in release --- .github/workflows/build-and-release.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 31eca6eab..50e86dfef 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -114,10 +114,6 @@ jobs: echo "Using provided commit sha for release commit" echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" fi - - name: Check rc-app-token value - if: needs.build.outputs.rc-app-token == '' - run: | - echo "****** rc-app-token is empty ******" - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: @@ -134,7 +130,7 @@ jobs: - name: Create Release uses: ncipollo/release-action@v1 with: - token: ${{ needs.build.outputs.rc-app-token }} + token: ${{ steps.app-token.outputs.token }} owner: github repo: backup-utils name: | From 6211c8556fa48fd9ed713cf9fbf1ac333dd96ed3 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Tue, 17 Oct 2023 16:53:58 -0400 Subject: [PATCH 293/410] just create draft release --- .github/workflows/build-and-release.yml | 166 ++++++++++++------------ 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 50e86dfef..e55205902 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -19,60 +19,60 @@ on: default: true jobs: - build: - runs-on: ubuntu-latest - outputs: - rc-app-token: ${{ steps.app-token.outputs.token }} - steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "backup-utils-private" - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # build: + # runs-on: ubuntu-latest + # outputs: + # rc-app-token: ${{ steps.app-token.outputs.token }} + # steps: + # - uses: actions/create-github-app-token@v1 + # id: app-token + # with: + # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: "backup-utils-private" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: - needs: build + # needs: build runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} @@ -85,25 +85,25 @@ jobs: private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - - name: Checkout backup-utils - if: github.event.inputs.branch_ref_or_commit == '' - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - repository: github/backup-utils - ref: master - - name: Create empty commit - if: github.event.inputs.branch_ref_or_commit == '' - uses: stefanzweifel/git-auto-commit-action@v4 - id: empty-commit - with: - branch: master - commit_message: "Test ${{ github.event.inputs.version }} release commit" - commit_user_name: "release-controller[bot]" - commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" - commit_options: "--allow-empty" - push_options: "--force" - skip_dirty_check: true + # - name: Checkout backup-utils + # if: github.event.inputs.branch_ref_or_commit == '' + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # repository: github/backup-utils + # ref: master + # - name: Create empty commit + # if: github.event.inputs.branch_ref_or_commit == '' + # uses: stefanzweifel/git-auto-commit-action@v4 + # id: empty-commit + # with: + # branch: master + # commit_message: "Test ${{ github.event.inputs.version }} release commit" + # commit_user_name: "release-controller[bot]" + # commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + # commit_options: "--allow-empty" + # push_options: "--force" + # skip_dirty_check: true - name: Resolve release commit id: resolve-release-commit run: | @@ -119,14 +119,14 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: @@ -135,9 +135,9 @@ jobs: repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - github-backup-utils_${{ github.event.inputs.version }}_all.deb + # artifacts: | + # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + # github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} bodyFile: release-notes/${{ github.event.inputs.version }}.md From 067f6c1bbe6ae6a59703a71ed669f3ac3b97b1ab Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:17:29 -0400 Subject: [PATCH 294/410] create empty commit on branch ref and use commit sha in release --- .github/workflows/build-and-release.yml | 164 ++++++++++++------------ 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index e55205902..7425d3d6f 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -19,58 +19,58 @@ on: default: true jobs: - # build: - # runs-on: ubuntu-latest - # outputs: - # rc-app-token: ${{ steps.app-token.outputs.token }} - # steps: - # - uses: actions/create-github-app-token@v1 - # id: app-token - # with: - # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: "backup-utils-private" - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: # needs: build runs-on: ubuntu-latest @@ -85,35 +85,35 @@ jobs: private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - # - name: Checkout backup-utils - # if: github.event.inputs.branch_ref_or_commit == '' - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # repository: github/backup-utils - # ref: master - # - name: Create empty commit - # if: github.event.inputs.branch_ref_or_commit == '' - # uses: stefanzweifel/git-auto-commit-action@v4 - # id: empty-commit - # with: - # branch: master - # commit_message: "Test ${{ github.event.inputs.version }} release commit" - # commit_user_name: "release-controller[bot]" - # commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" - # commit_options: "--allow-empty" - # push_options: "--force" - # skip_dirty_check: true - - name: Resolve release commit - id: resolve-release-commit - run: | - if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then - echo "Using empty commit sha for release commit" - echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" - else - echo "Using provided commit sha for release commit" - echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" - fi + - name: Checkout backup-utils + # if: github.event.inputs.branch_ref_or_commit == '' + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + repository: github/backup-utils + ref: ${{ github.event.inputs.branch_ref_or_commit }}} + - name: Create empty commit + # if: github.event.inputs.branch_ref_or_commit == '' + uses: stefanzweifel/git-auto-commit-action@v4 + id: empty-commit + with: + branch: ${{ github.event.inputs.branch_ref_or_commit }} + commit_message: "Test ${{ github.event.inputs.version }} release commit" + commit_user_name: "release-controller[bot]" + commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" + commit_options: "--allow-empty" + push_options: "--force" + skip_dirty_check: true + # - name: Resolve release commit + # id: resolve-release-commit + # run: | + # if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then + # echo "Using empty commit sha for release commit" + # echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" + # else + # echo "Using provided commit sha for release commit" + # echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" + # fi - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: @@ -139,7 +139,7 @@ jobs: # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ # github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From c3900111e4f76b8988d3806c59cbd01181f6e249 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:18:30 -0400 Subject: [PATCH 295/410] just release --- .github/workflows/build-and-release.yml | 104 ++++++++++++------------ 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7425d3d6f..f602d9371 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -19,58 +19,58 @@ on: default: true jobs: - build: - runs-on: ubuntu-latest - outputs: - rc-app-token: ${{ steps.app-token.outputs.token }} - steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "backup-utils-private" - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # build: + # runs-on: ubuntu-latest + # outputs: + # rc-app-token: ${{ steps.app-token.outputs.token }} + # steps: + # - uses: actions/create-github-app-token@v1 + # id: app-token + # with: + # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: "backup-utils-private" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: # needs: build runs-on: ubuntu-latest From c597504b0cbe25e35a6a520eae6a0a2d43e99046 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:20:28 -0400 Subject: [PATCH 296/410] fix checkout --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index f602d9371..e1473663e 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -91,7 +91,7 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - ref: ${{ github.event.inputs.branch_ref_or_commit }}} + # ref: ${{ github.event.inputs.branch_ref_or_commit }}} - name: Create empty commit # if: github.event.inputs.branch_ref_or_commit == '' uses: stefanzweifel/git-auto-commit-action@v4 From 7f738f05c47262fcaae1627e59ab7e18990545d5 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:29:56 -0400 Subject: [PATCH 297/410] try one more time --- .github/workflows/build-and-release.yml | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index e1473663e..79777fadb 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -86,14 +86,14 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - # if: github.event.inputs.branch_ref_or_commit == '' + if: github.event.inputs.branch_ref_or_commit == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils # ref: ${{ github.event.inputs.branch_ref_or_commit }}} - name: Create empty commit - # if: github.event.inputs.branch_ref_or_commit == '' + if: github.event.inputs.branch_ref_or_commit == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -104,16 +104,16 @@ jobs: commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true - # - name: Resolve release commit - # id: resolve-release-commit - # run: | - # if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then - # echo "Using empty commit sha for release commit" - # echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" - # else - # echo "Using provided commit sha for release commit" - # echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" - # fi + - name: Resolve release commit + id: resolve-release-commit + run: | + if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then + echo "Using empty commit sha for release commit" + echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" + else + echo "Using provided commit sha for release commit" + echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" + fi - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: @@ -139,7 +139,7 @@ jobs: # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ # github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} + commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From 4e343a5ec8503d36cc866f325459f3789ac7e597 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:34:48 -0400 Subject: [PATCH 298/410] always empty commit on release branch --- .github/workflows/build-and-release.yml | 160 ++++++++++++------------ 1 file changed, 79 insertions(+), 81 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 79777fadb..923fdf727 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -8,9 +8,9 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string - branch_ref_or_commit: - description: 'Branch ref or commit - the backup-utils release will associated with the branch ref or commit provided. If blank, an empty commit will be created on master' - required: false + release_commit_branch: + description: 'Release Commit Branch - the branch on which the release commit will be made.' + required: true type: string draft: description: 'Draft - true if the release should be a draft' @@ -19,58 +19,58 @@ on: default: true jobs: - # build: - # runs-on: ubuntu-latest - # outputs: - # rc-app-token: ${{ steps.app-token.outputs.token }} - # steps: - # - uses: actions/create-github-app-token@v1 - # id: app-token - # with: - # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: "backup-utils-private" - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: # needs: build runs-on: ubuntu-latest @@ -86,47 +86,45 @@ jobs: owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" - name: Checkout backup-utils - if: github.event.inputs.branch_ref_or_commit == '' uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - # ref: ${{ github.event.inputs.branch_ref_or_commit }}} - name: Create empty commit - if: github.event.inputs.branch_ref_or_commit == '' + if: github.event.inputs.release_commit_branch == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: ${{ github.event.inputs.branch_ref_or_commit }} + branch: ${{ github.event.inputs.release_commit_branch }} commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true - - name: Resolve release commit - id: resolve-release-commit - run: | - if [[ -z "${{ github.event.inputs.branch_ref_or_commit }}" ]]; then - echo "Using empty commit sha for release commit" - echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" - else - echo "Using provided commit sha for release commit" - echo "release-ref-or-commit=\"${{ github.event.inputs.branch_ref_or_commit }}\"" >> "$GITHUB_OUTPUT" - fi + # - name: Resolve release commit + # id: resolve-release-commit + # run: | + # if [[ -z "${{ github.event.inputs.release_commit_branch }}" ]]; then + # echo "Using empty commit sha for release commit" + # echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" + # else + # echo "Using provided commit sha for release commit" + # echo "release-ref-or-commit=\"${{ github.event.inputs.release_commit_branch }}\"" >> "$GITHUB_OUTPUT" + # fi - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: @@ -135,11 +133,11 @@ jobs: repo: backup-utils name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} - # artifacts: | - # github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ - # github-backup-utils_${{ github.event.inputs.version }}_all.deb + artifacts: | + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} + commit: ${{ steps.empty-commit.outputs.commit_hash }} bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From 04c7806b092ccec3575555699069dd64c6168018 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:36:01 -0400 Subject: [PATCH 299/410] after build --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 923fdf727..536c8f971 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -72,7 +72,7 @@ jobs: path: | dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: - # needs: build + needs: build runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} From 3ad603b75e6dcbd9fd940b8dbd9c3c0bd2da18e7 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 09:36:52 -0400 Subject: [PATCH 300/410] comment out old output --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 536c8f971..d6681ea68 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} + # release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} steps: - uses: actions/create-github-app-token@v1 id: app-token From 1eb6431c5261242cd9f5e2c74998189219302ca1 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 10:39:35 -0400 Subject: [PATCH 301/410] one more test --- .github/workflows/build-and-release.yml | 110 ++++++++++++------------ 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index d6681ea68..5419fbbef 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -19,60 +19,60 @@ on: default: true jobs: - build: - runs-on: ubuntu-latest - outputs: - rc-app-token: ${{ steps.app-token.outputs.token }} - steps: - - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: "backup-utils-private" - - name: Checkout backup-utils-private - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} - - name: Install dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y moreutils debhelper help2man devscripts gzip - - name: Create tag # this is required for the build scripts - run: | - git config user.name "${{ github.actor }}" - git config user.email "ghes-releases-team@github.com" - git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - git push origin "v${{ github.event.inputs.version }}" - - name: Package deb - run: | - ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - - name: Upload deb artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - path: | - dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Package tarball - run: | - ./script/package-tarball - - name: Upload tarball artifact - uses: actions/upload-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - path: | - dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # build: + # runs-on: ubuntu-latest + # outputs: + # rc-app-token: ${{ steps.app-token.outputs.token }} + # steps: + # - uses: actions/create-github-app-token@v1 + # id: app-token + # with: + # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + # owner: ${{ github.repository_owner }} + # repositories: "backup-utils-private" + # - name: Checkout backup-utils-private + # uses: actions/checkout@v4 + # with: + # token: ${{ steps.app-token.outputs.token }} + # - name: Install dependencies + # run: | + # sudo apt-get update -y + # sudo apt-get install -y moreutils debhelper help2man devscripts gzip + # - name: Create tag # this is required for the build scripts + # run: | + # git config user.name "${{ github.actor }}" + # git config user.email "ghes-releases-team@github.com" + # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + # git push origin "v${{ github.event.inputs.version }}" + # - name: Package deb + # run: | + # ./script/package-deb + # # many need to remove this once release-notes compilation is automated + # - name: Rename deb artifact + # run: | + # for file in dist/github-backup-utils_*_all.deb; do + # if [[ -f "$file" ]]; then + # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + # fi + # done + # - name: Upload deb artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # path: | + # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Package tarball + # run: | + # ./script/package-tarball + # - name: Upload tarball artifact + # uses: actions/upload-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # path: | + # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: - needs: build + # needs: build runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} @@ -91,7 +91,7 @@ jobs: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - name: Create empty commit - if: github.event.inputs.release_commit_branch == '' + # if: github.event.inputs.release_commit_branch == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: @@ -137,7 +137,7 @@ jobs: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.empty-commit.outputs.commit_hash }} + commit: 3.9-main bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From 7a88c2c933aabf070d9b0cf87615ec4f5bdd786a Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 10:42:11 -0400 Subject: [PATCH 302/410] test, no artifacts --- .github/workflows/build-and-release.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 5419fbbef..953ff86b8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -117,14 +117,14 @@ jobs: with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - - name: Download deb artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - - name: Download tarball artifact - uses: actions/download-artifact@v3 - with: - name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + # - name: Download deb artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + # - name: Download tarball artifact + # uses: actions/download-artifact@v3 + # with: + # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: From 7ab47f1160153771c246d0531319447df5be84cc Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 11:50:08 -0400 Subject: [PATCH 303/410] try stable branch --- .github/workflows/build-and-release.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 953ff86b8..da58c6bb1 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -8,10 +8,6 @@ on: description: 'Version - patch version of the release (e.g. x.y.z)' required: true type: string - release_commit_branch: - description: 'Release Commit Branch - the branch on which the release commit will be made.' - required: true - type: string draft: description: 'Draft - true if the release should be a draft' required: true @@ -76,7 +72,6 @@ jobs: runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} - # release-ref-or-commit: ${{ steps.resolve-release-commit.outputs.release-ref-or-commit }} steps: - uses: actions/create-github-app-token@v1 id: app-token @@ -85,17 +80,20 @@ jobs: private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: "backup-utils,backup-utils-private" + - name: Get major-feature from version + id: get-major-feature + run: | + echo "major-feature=\"$(echo "${{ github.event.inputs.version }}" | cut -d '.' -f 1,2)\"" >> "$GITHUB_OUTPUT" - name: Checkout backup-utils uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils - name: Create empty commit - # if: github.event.inputs.release_commit_branch == '' uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: ${{ github.event.inputs.release_commit_branch }} + branch: ${{ steps.get-major-feature.outputs.major-feature}}-stable commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" @@ -134,10 +132,10 @@ jobs: name: | GitHub Enterprise Server Backup Utilities v${{ github.event.inputs.version }} artifacts: | - github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, \ + github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: 3.9-main + commit: ${{ steps.get-major-feature.outputs.major-feature}}-stable bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From 68c71b97f0ad99f34c2b1e6e38f5d960ff307379 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 13:38:05 -0400 Subject: [PATCH 304/410] handle major-feature --- .github/workflows/build-and-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index da58c6bb1..7402a17d7 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -83,7 +83,10 @@ jobs: - name: Get major-feature from version id: get-major-feature run: | - echo "major-feature=\"$(echo "${{ github.event.inputs.version }}" | cut -d '.' -f 1,2)\"" >> "$GITHUB_OUTPUT" + echo "MAJOR_FEATURE=$(echo "${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + - name: Verify major-feature + run: | + echo "major_feature: $MAJOR_FEATURE" - name: Checkout backup-utils uses: actions/checkout@v4 with: @@ -93,7 +96,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 id: empty-commit with: - branch: ${{ steps.get-major-feature.outputs.major-feature}}-stable + branch: ${{ env.MAJOR_FEATURE }}-stable commit_message: "Test ${{ github.event.inputs.version }} release commit" commit_user_name: "release-controller[bot]" commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" From 56c72781e79d7aff1607b7f5358ebc363a6c2742 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 13:56:36 -0400 Subject: [PATCH 305/410] fix major-feature --- .github/workflows/build-and-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 7402a17d7..48d0a8ca9 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -83,7 +83,7 @@ jobs: - name: Get major-feature from version id: get-major-feature run: | - echo "MAJOR_FEATURE=$(echo "${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" + echo "MAJOR_FEATURE=\"$(echo "${{ github.event.inputs.version }}" | cut -d '.' -f 1,2)\"" >> "$GITHUB_ENV" - name: Verify major-feature run: | echo "major_feature: $MAJOR_FEATURE" @@ -138,7 +138,7 @@ jobs: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz, github-backup-utils_${{ github.event.inputs.version }}_all.deb tag: v${{ github.event.inputs.version }} - commit: ${{ steps.get-major-feature.outputs.major-feature}}-stable + commit: ${{ env.MAJOR_FEATURE }}-stable bodyFile: release-notes/${{ github.event.inputs.version }}.md draft: ${{ github.event.inputs.draft }} allowUpdates: true From f147c4e8c8b67e04f80363fab1120044501eac09 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 14:13:22 -0400 Subject: [PATCH 306/410] try another fix --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 48d0a8ca9..8f03dec9a 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -83,7 +83,7 @@ jobs: - name: Get major-feature from version id: get-major-feature run: | - echo "MAJOR_FEATURE=\"$(echo "${{ github.event.inputs.version }}" | cut -d '.' -f 1,2)\"" >> "$GITHUB_ENV" + echo "MAJOR_FEATURE=$(echo ${{ github.event.inputs.version }} | cut -d '.' -f 1,2)" >> "$GITHUB_ENV" - name: Verify major-feature run: | echo "major_feature: $MAJOR_FEATURE" From aa80191b74138d6b71997976630f3bfded0440d2 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 14:17:12 -0400 Subject: [PATCH 307/410] check release --- release-notes/{12.12.12.md => 3.10.99.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename release-notes/{12.12.12.md => 3.10.99.md} (100%) diff --git a/release-notes/12.12.12.md b/release-notes/3.10.99.md similarity index 100% rename from release-notes/12.12.12.md rename to release-notes/3.10.99.md From 77dad72bc7c6516af482e973d8981bf090ccf1f4 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 14:40:37 -0400 Subject: [PATCH 308/410] should be done --- .github/workflows/build-and-release.yml | 132 +++++++++++------------- 1 file changed, 61 insertions(+), 71 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 8f03dec9a..3f6b78648 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -15,60 +15,60 @@ on: default: true jobs: - # build: - # runs-on: ubuntu-latest - # outputs: - # rc-app-token: ${{ steps.app-token.outputs.token }} - # steps: - # - uses: actions/create-github-app-token@v1 - # id: app-token - # with: - # app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} - # private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} - # owner: ${{ github.repository_owner }} - # repositories: "backup-utils-private" - # - name: Checkout backup-utils-private - # uses: actions/checkout@v4 - # with: - # token: ${{ steps.app-token.outputs.token }} - # - name: Install dependencies - # run: | - # sudo apt-get update -y - # sudo apt-get install -y moreutils debhelper help2man devscripts gzip - # - name: Create tag # this is required for the build scripts - # run: | - # git config user.name "${{ github.actor }}" - # git config user.email "ghes-releases-team@github.com" - # git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" - # git push origin "v${{ github.event.inputs.version }}" - # - name: Package deb - # run: | - # ./script/package-deb - # # many need to remove this once release-notes compilation is automated - # - name: Rename deb artifact - # run: | - # for file in dist/github-backup-utils_*_all.deb; do - # if [[ -f "$file" ]]; then - # mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - # fi - # done - # - name: Upload deb artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # path: | - # dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Package tarball - # run: | - # ./script/package-tarball - # - name: Upload tarball artifact - # uses: actions/upload-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - # path: | - # dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + build: + runs-on: ubuntu-latest + outputs: + rc-app-token: ${{ steps.app-token.outputs.token }} + steps: + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.RELEASE_CONTROLLER_APP_ID }} + private-key: ${{ secrets.RELEASE_CONTROLLER_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "backup-utils-private" + - name: Checkout backup-utils-private + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y moreutils debhelper help2man devscripts gzip + - name: Create tag # this is required for the build scripts + run: | + git config user.name "${{ github.actor }}" + git config user.email "ghes-releases-team@github.com" + git tag -a "v${{ github.event.inputs.version }}" -m "v${{ github.event.inputs.version }}" + git push origin "v${{ github.event.inputs.version }}" + - name: Package deb + run: | + ./script/package-deb + # many need to remove this once release-notes compilation is automated + - name: Rename deb artifact + run: | + for file in dist/github-backup-utils_*_all.deb; do + if [[ -f "$file" ]]; then + mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" + fi + done + - name: Upload deb artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + path: | + dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Package tarball + run: | + ./script/package-tarball + - name: Upload tarball artifact + uses: actions/upload-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + path: | + dist/github-backup-utils-v${{ github.event.inputs.version }}.tar.gz release: - # needs: build + needs: build runs-on: ubuntu-latest outputs: commit_hash: ${{ steps.empty-commit.outputs.commit_hash }} @@ -103,29 +103,19 @@ jobs: commit_options: "--allow-empty" push_options: "--force" skip_dirty_check: true - # - name: Resolve release commit - # id: resolve-release-commit - # run: | - # if [[ -z "${{ github.event.inputs.release_commit_branch }}" ]]; then - # echo "Using empty commit sha for release commit" - # echo "release-ref-or-commit=\"${{ steps.empty-commit.outputs.commit_hash }}\"" >> "$GITHUB_OUTPUT" - # else - # echo "Using provided commit sha for release commit" - # echo "release-ref-or-commit=\"${{ github.event.inputs.release_commit_branch }}\"" >> "$GITHUB_OUTPUT" - # fi - name: Checkout backup-utils-private for release notes uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} repository: github/backup-utils-private - # - name: Download deb artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils_${{ github.event.inputs.version }}_all.deb - # - name: Download tarball artifact - # uses: actions/download-artifact@v3 - # with: - # name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz + - name: Download deb artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils_${{ github.event.inputs.version }}_all.deb + - name: Download tarball artifact + uses: actions/download-artifact@v3 + with: + name: github-backup-utils-v${{ github.event.inputs.version }}.tar.gz - name: Create Release uses: ncipollo/release-action@v1 with: From 540ae1cc01cf3bc3c2172feff74078646dc26f83 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 14:50:40 -0400 Subject: [PATCH 309/410] remove test message --- .github/workflows/build-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index 3f6b78648..c9b6b66e8 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -97,7 +97,7 @@ jobs: id: empty-commit with: branch: ${{ env.MAJOR_FEATURE }}-stable - commit_message: "Test ${{ github.event.inputs.version }} release commit" + commit_message: "${{ github.event.inputs.version }} release" commit_user_name: "release-controller[bot]" commit_user_email: "223695+release-controller[bot]@users.noreply.github.com" commit_options: "--allow-empty" From 30612641edc735f268a73fdcecf6cfe8f6ca8476 Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Wed, 18 Oct 2023 15:45:57 -0400 Subject: [PATCH 310/410] remove test release notes --- release-notes/3.10.99.md | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 release-notes/3.10.99.md diff --git a/release-notes/3.10.99.md b/release-notes/3.10.99.md deleted file mode 100644 index 445d9aada..000000000 --- a/release-notes/3.10.99.md +++ /dev/null @@ -1,7 +0,0 @@ -### Changes - -* Test release notes. - -### Bug fixes - -* More test release notes. \ No newline at end of file From 5d2d9a8ada57e543383f585d08c857bee518cfad Mon Sep 17 00:00:00 2001 From: Tim Reimherr Date: Fri, 20 Oct 2023 23:20:02 +0000 Subject: [PATCH 311/410] remove file rename step --- .github/workflows/build-and-release.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index c9b6b66e8..edf115738 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -44,14 +44,6 @@ jobs: - name: Package deb run: | ./script/package-deb - # many need to remove this once release-notes compilation is automated - - name: Rename deb artifact - run: | - for file in dist/github-backup-utils_*_all.deb; do - if [[ -f "$file" ]]; then - mv "$file" "dist/github-backup-utils_${{ github.event.inputs.version }}_all.deb" - fi - done - name: Upload deb artifact uses: actions/upload-artifact@v3 with: From dfca0ab97591346f2abc3ad05fa84893de4e3fdf Mon Sep 17 00:00:00 2001 From: Quinn Murphy Date: Mon, 23 Oct 2023 14:03:40 -0400 Subject: [PATCH 312/410] Add unit test for track progress (#659) * Add unit test for track progress * add end_test * change test and fix init-progress * Make the test go green * Update ghe-backup-config --- share/github-backup-utils/ghe-backup-config | 14 +++++--------- test/test-ghe-backup.sh | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 6c33ea954..cd9972b59 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -661,17 +661,13 @@ prompt_for_confirmation(){ #initialize progress tracking by clearing out the temp files used to track init-progress() { - if [ -d /tmp/backup-utils-progress ]; then + + if [ -e /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* fi - - mkdir -p /tmp/backup-utils-progress - chmod -R 777 /tmp/backup-utils-progress - - touch /tmp/backup-utils-progress/total - touch /tmp/backup-utils-progress/type - touch /tmp/backup-utils-progress/progress - touch /tmp/backup-utils-progress/info + # shellcheck disable=SC2174 # We are fine with -m only affecting the deepest directory + mkdir -m 777 -p /tmp/backup-utils-progress + touch /tmp/backup-utils-progress/{total,type,progress,info} } diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index d6391bbef..dabce1ed4 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -1074,3 +1074,18 @@ begin_test "ghe-backup collects information on system where backup-utils is inst ) end_test + +# Check that backup-utils manages track-progress folder correctly +begin_test "ghe-backup manages progress tracking files properly" +( + set -e + + if [ -e /tmp/backup-utils-progress ]; then + rm -rf /tmp/backup-utils-progress/* + fi + + output=$(ghe-backup) + echo "$output" | grep -v "mkdir: cannot create directory /tmp/backup-utils-progress: File exists" + +) +end_test \ No newline at end of file From d9cf3a64fc521f43eb2553e06dbedaf59596be1c Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 24 Oct 2023 09:52:06 -0700 Subject: [PATCH 313/410] Use os-release for host info resolves https://github.com/github/ghes/issues/7336 --- bin/ghe-backup | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 1dfaf748b..448a5840e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -231,9 +231,11 @@ fi # Output system information of the backup host -# If /etc/issue.net exists, use it to get the OS version -if [ -f /etc/issue.net ]; then - echo "Running on: $(cat /etc/issue.net)" +# If /etc/os-release exists, use it to get the OS version +if [ -f /etc/os-release ]; then + OS_NAME=$(grep '^NAME' /etc/os-release | cut -d'"' -f2) + VERSION_ID=$(grep '^VERSION_ID' /etc/os-release | cut -d'"' -f2) + echo "Running on: $OS_NAME $VERSION_ID" else echo "Running on: Unknown OS" fi From 4b477a69b59ef625d606905bda65f74f0a984f7f Mon Sep 17 00:00:00 2001 From: Tony Truong Date: Wed, 25 Oct 2023 16:18:29 +0200 Subject: [PATCH 314/410] update warning message (#699) --- share/github-backup-utils/ghe-backup-secrets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-secrets b/share/github-backup-utils/ghe-backup-secrets index 546be420e..6d7ab5db6 100755 --- a/share/github-backup-utils/ghe-backup-secrets +++ b/share/github-backup-utils/ghe-backup-secrets @@ -54,7 +54,7 @@ backup-secret() { log_info "* Transferring $description ..." 1>&3 ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( if [ "$best_effort" = "false" ]; then - echo "Warning: $description not set" >&2 + echo "Info: $description not set. Skipping..." >&2 fi ) if [ -n "$(cat "$file+")" ]; then From c30b46422c2b4638a01b1965f41f748612907b74 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 25 Oct 2023 20:59:43 +0000 Subject: [PATCH 315/410] check for RO file-system --- bin/ghe-host-check | 35 ++++++++++++++++++++++++++++++++++- test/test-ghe-host-check.sh | 10 ++++++++++ test/testlib.sh | 4 ++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 5cd39abee..4262887e1 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -115,6 +115,39 @@ if [ -z "$version" ]; then exit 2 fi +NON_WRITABLE="" +# ensure all nodes are writable +if "$CLUSTER"; then + if [ -z "$FILE_TO_TEST" ]; then + FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" + fi + + # Iterate through each node in the cluster + nodes=$(ghe-ssh "$host" ghe-cluster-nodes) + first=true + # echo $nodes | while read -r node; do + for node in $nodes; do + if ! $(echo "set -o pipefail; ssh "$node" -- 'touch "$FILE_TO_TEST" && rm "$FILE_TO_TEST"'" | ghe-ssh "$host" /bin/bash); then + #echo "File system is writable on $node" + #else + echo "File system is not writeable on $node" 1>&2 + if [ "$first" = true ]; then + NON_WRITABLE="$node" + first=false + else + NON_WRITABLE="$NON_WRITABLE, $node" + fi + fi || true + done + # Display the comma-separated list of non-writable nodes + if [ ! -z "$NON_WRITABLE" ]; then + echo "Error: Non-writable nodes: $NON_WRITABLE . Please make sure the file systems for all Enterprise nodes are writable." 1>&2 + exit 1 + else + echo "All nodes are writable." + fi +fi + # Block restoring snapshots to older releases of GitHub Enterprise Server if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) @@ -177,7 +210,7 @@ SKIP_MSG #Display dir requirements for repositories and mysql echo -e "\nChecking host for sufficient space for a backup..." - available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') + available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." repos_disk_size=$(transfer_size repositories /tmp) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index f2c5cca54..ebe02f0e6 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -123,3 +123,13 @@ begin_test "ghe-host-check blocks restore to old release" ! GHE_TEST_REMOTE_VERSION=$bu_version_major.$((bu_version_minor-1)).$bu_version_patch ghe-restore -v ) end_test + +# Check ghe-host-check detects RO file system +begin_test "ghe-host-check fails when encountering RO file-system" +( + set -e + + enable_ro_fs + ! FILE_TO_TEST=/run/user/501/test ghe-host-check +) +end_test diff --git a/test/testlib.sh b/test/testlib.sh index edeec8ea2..b72866abe 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -728,3 +728,7 @@ setup_actions_enabled_settings_for_restore() { echo > "$GHE_DATA_DIR/1/settings.json" git config -f "$GHE_DATA_DIR/1/settings.json" --bool app.actions.enabled $1 } + +enable_ro_fs() { + ghe-ssh "$GHE_HOSTNAME" -- 'sudo mount -o remount,rw /run/user/501' +} From 44a59b55378cce06845acb46e71d981b6b91fdb0 Mon Sep 17 00:00:00 2001 From: djdefi Date: Wed, 25 Oct 2023 14:24:21 -0700 Subject: [PATCH 316/410] Update test-ghe-backup.sh --- test/test-ghe-backup.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-backup.sh b/test/test-ghe-backup.sh index dabce1ed4..30fc23fcb 100755 --- a/test/test-ghe-backup.sh +++ b/test/test-ghe-backup.sh @@ -1068,7 +1068,9 @@ begin_test "ghe-backup collects information on system where backup-utils is inst set -e output=$(ghe-backup) - echo "$output" | grep "Running on: $(cat /etc/issue.net)" + OS_NAME=$(grep '^NAME' /etc/os-release | cut -d'"' -f2) + VERSION_ID=$(grep '^VERSION_ID' /etc/os-release | cut -d'"' -f2) + echo "$output" | grep "Running on: $OS_NAME $VERSION_ID" echo "$output" | grep "CPUs: $(nproc)" echo "$output" | grep "Memory total/used/free+share/buff/cache:" @@ -1088,4 +1090,4 @@ begin_test "ghe-backup manages progress tracking files properly" echo "$output" | grep -v "mkdir: cannot create directory /tmp/backup-utils-progress: File exists" ) -end_test \ No newline at end of file +end_test From 494a521089c71e595a77983f0f4b02ee3b79685b Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Thu, 26 Oct 2023 00:30:52 +0000 Subject: [PATCH 317/410] check for tmp dir --- bin/ghe-host-check | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 4262887e1..8bcdb85f0 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -119,7 +119,11 @@ NON_WRITABLE="" # ensure all nodes are writable if "$CLUSTER"; then if [ -z "$FILE_TO_TEST" ]; then - FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" + if [ -d "/data/user/tmp" ]; then + FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" + else + FILE_TO_TEST="/tmp/test-ro-file.txt" + fi fi # Iterate through each node in the cluster From 88f8489b6ebf36703a80754f9bba581a61e556d0 Mon Sep 17 00:00:00 2001 From: djdefi Date: Mon, 30 Oct 2023 15:43:34 -0700 Subject: [PATCH 318/410] Shellcheck docker-image.yml --- .github/workflows/docker-image.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 07c44f9df..03050f1d4 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,4 +1,4 @@ -name: Docker Image CI +name: Docker Image Build CI on: push: @@ -13,16 +13,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build the Ubuntu Docker image - run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID} + run: docker build . --file Dockerfile --tag backup-utils:"${GITHUB_RUN_ID}" - name: Build the Alpine Docker image - run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID} + run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:"${GITHUB_RUN_ID}" - name: Run tests in Ubuntu Docker image run: | - docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version - docker run backup-utils:${GITHUB_RUN_ID} rsync --version + docker run backup-utils:"${GITHUB_RUN_ID}" ghe-backup --version + docker run backup-utils:"${GITHUB_RUN_ID}" rsync --version - name: Run tests in Alpine Docker image run: | - docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version - docker run backup-utils-alpine:${GITHUB_RUN_ID} rsync --version + docker run backup-utils-alpine:"${GITHUB_RUN_ID}" ghe-backup --version + docker run backup-utils-alpine:"${GITHUB_RUN_ID}" rsync --version From d4c2952dda4176289eb9b0375c4efc9786d2d4d1 Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 31 Oct 2023 15:15:50 -0700 Subject: [PATCH 319/410] Shellcheck fix backuprestore.yml --- .github/workflows/backuprestore.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backuprestore.yml b/.github/workflows/backuprestore.yml index 13bbf40e0..34cf69032 100644 --- a/.github/workflows/backuprestore.yml +++ b/.github/workflows/backuprestore.yml @@ -28,7 +28,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: github/backup-utils-private ref: ${{ inputs.ref }} @@ -51,13 +51,13 @@ jobs: run: docker load -i backup-utils.tar - uses: actions/checkout@v3 - name: Create backup directory - run: mkdir $HOME/ghe-backup-data + run: mkdir "$HOME"/ghe-backup-data - name: set up ssh SSH_KEY - run: echo -e "${SSH_KEY}\n" > $HOME/backup + run: echo -e "${SSH_KEY}\n" > "$HOME"/backup - name: set up ssh key permissions - run: chmod 0600 $HOME/backup + run: chmod 0600 "$HOME"/backup - name: change version - run: echo "3.8.0" > $HOME/version + run: echo "3.8.0" > "$HOME"/version - name: Perform backup run: | docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \ @@ -70,7 +70,7 @@ jobs: --rm \ backup-utils ghe-backup - name: Prepare for restore - run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s" + run: ssh -p122 -i "$HOME"/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -s" - name: Restore data to instance run: | docker run -e "GHE_HOSTNAME=${{ inputs.hostname }}" \ @@ -83,5 +83,5 @@ jobs: --rm \ backup-utils ghe-restore ${{ inputs.hostname }} - name: Reset maintenance mode after restore - run: ssh -p122 -i $HOME/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u" + run: ssh -p122 -i "$HOME"/backup -o StrictHostKeyChecking=no admin@${{ inputs.hostname }} "ghe-maintenance -u" From d9619a1a5578efd93cf2fea0c0c9d5181d6d9ac1 Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 31 Oct 2023 15:16:42 -0700 Subject: [PATCH 320/410] Lint fix restore.yml --- .github/workflows/restore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/restore.yml b/.github/workflows/restore.yml index ab0627751..5f4e723a1 100644 --- a/.github/workflows/restore.yml +++ b/.github/workflows/restore.yml @@ -51,7 +51,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: github/backup-utils-private ref: ${{ inputs.ref }} @@ -69,7 +69,7 @@ jobs: needs: build runs-on: group: larger-hosted-public-runners - labels: ubuntu-latest-xl + labels: ubuntu-latest env: SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }} steps: From 4f8b5de6f45ad2f7ef9819f26579b0726ecd4aa3 Mon Sep 17 00:00:00 2001 From: djdefi Date: Wed, 1 Nov 2023 09:22:12 -0700 Subject: [PATCH 321/410] Linter fix backup.yml --- .github/workflows/backup.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index 237d8e131..0d50f8f81 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -47,7 +47,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: repository: github/backup-utils-private token: "${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}" @@ -62,7 +62,7 @@ jobs: needs: build runs-on: group: larger-hosted-public-runners - labels: ubuntu-latest-xl + labels: ubuntu-latest env: SSH_KEY: ${{ secrets.BACKUP_SSH_KEY }} steps: @@ -98,7 +98,7 @@ jobs: sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" . - name: Login to Azure - if: ${{ inputs.backup-name }} != "" + if: inputs.backup-name != "" run: | az login \ --service-principal \ @@ -108,11 +108,11 @@ jobs: az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" - name: Upload backup to Azure - if: ${{ inputs.backup-name }} != "" + if: inputs.backup-name != "" run: | az storage blob upload \ --account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \ --container-name "${{ secrets.AZURE_CONTAINER_NAME }}" \ --name "${{ inputs.backup-name }}.tar.gz" \ --file "${{ inputs.backup-name }}.tar.gz" \ - --connection-string "${{ secrets.CONNECTIONSTRING }}" \ No newline at end of file + --connection-string "${{ secrets.CONNECTIONSTRING }}" From c8f1a5e28ad3fda848abeaece2bb47845fe2b0d5 Mon Sep 17 00:00:00 2001 From: djdefi Date: Wed, 1 Nov 2023 09:29:15 -0700 Subject: [PATCH 322/410] Adjust if statement syntax --- .github/workflows/backup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index 0d50f8f81..cfb154aae 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -98,7 +98,7 @@ jobs: sudo tar -czvf "${{ inputs.backup-name }}.tar.gz" -C "$HOME/ghe-backup-data/$current" . - name: Login to Azure - if: inputs.backup-name != "" + if: "${{ inputs.backup-name != '' }}" run: | az login \ --service-principal \ @@ -108,7 +108,7 @@ jobs: az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}" - name: Upload backup to Azure - if: inputs.backup-name != "" + if: "${{ inputs.backup-name != '' }}" run: | az storage blob upload \ --account-name "${{ secrets.AZURE_ACCOUNT_NAME }}" \ From ac1cbb1dc0b39e4d86490f21d7ab25cafa1cb004 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 1 Nov 2023 20:06:49 +0000 Subject: [PATCH 323/410] fix test runs --- bin/ghe-host-check | 8 +++++--- test/test-ghe-host-check.sh | 9 +++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 8bcdb85f0..2b77d11ff 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -85,7 +85,9 @@ if [ $rc -ne 0 ]; then exit $rc fi -CLUSTER=false +if [ -z "$CLUSTER" ]; then + CLUSTER=false +fi if ghe-ssh "$host" -- \ "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then CLUSTER=true @@ -117,7 +119,7 @@ fi NON_WRITABLE="" # ensure all nodes are writable -if "$CLUSTER"; then +if [ "$CLUSTER" == "true" ] ; then if [ -z "$FILE_TO_TEST" ]; then if [ -d "/data/user/tmp" ]; then FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" @@ -164,7 +166,7 @@ if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then fi if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then - if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then + if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state1 2>/dev/null || true)" = "replica" ]; then echo "Error: high availability replica detected." 1>&2 echo "Backup Utilities should be used to backup from the primary node in" 1>&2 echo "high availability environments to ensure consistent and reliable backups." 1>&2 diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index ebe02f0e6..1a32e34cd 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -129,7 +129,12 @@ begin_test "ghe-host-check fails when encountering RO file-system" ( set -e - enable_ro_fs - ! FILE_TO_TEST=/run/user/501/test ghe-host-check + ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p "~/tmp"' + # Remove write access in ~/tmp + ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"' + + # File creation fails for CLUSTER + ! FILE_TO_TEST="~/tmp/test" CLUSTER=true ghe-host-check + FILE_TO_TEST="~/tmp/test" CLUSTER=false ghe-host-check ) end_test From c86c42548cafd6e915ee708775751b7fc11d7892 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 1 Nov 2023 20:24:19 +0000 Subject: [PATCH 324/410] remove unused function --- test/testlib.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/testlib.sh b/test/testlib.sh index b72866abe..edeec8ea2 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -728,7 +728,3 @@ setup_actions_enabled_settings_for_restore() { echo > "$GHE_DATA_DIR/1/settings.json" git config -f "$GHE_DATA_DIR/1/settings.json" --bool app.actions.enabled $1 } - -enable_ro_fs() { - ghe-ssh "$GHE_HOSTNAME" -- 'sudo mount -o remount,rw /run/user/501' -} From 970cb37787700dd7590e4a4f38dd8aa27e30a77a Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Wed, 1 Nov 2023 21:17:42 +0000 Subject: [PATCH 325/410] SC updates --- bin/ghe-host-check | 2 +- test/test-ghe-host-check.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 2b77d11ff..cf62e6453 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -133,7 +133,7 @@ if [ "$CLUSTER" == "true" ] ; then first=true # echo $nodes | while read -r node; do for node in $nodes; do - if ! $(echo "set -o pipefail; ssh "$node" -- 'touch "$FILE_TO_TEST" && rm "$FILE_TO_TEST"'" | ghe-ssh "$host" /bin/bash); then + if ! echo "set -o pipefail; ssh $node -- 'touch $FILE_TO_TEST && rm $FILE_TO_TEST'" | ghe-ssh "$host" /bin/bash; then #echo "File system is writable on $node" #else echo "File system is not writeable on $node" 1>&2 diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 1a32e34cd..a772399d6 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -134,7 +134,7 @@ begin_test "ghe-host-check fails when encountering RO file-system" ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"' # File creation fails for CLUSTER - ! FILE_TO_TEST="~/tmp/test" CLUSTER=true ghe-host-check - FILE_TO_TEST="~/tmp/test" CLUSTER=false ghe-host-check + ! FILE_TO_TEST="$HOME/tmp/test" CLUSTER=true ghe-host-check + FILE_TO_TEST="$HOME/tmp/test" CLUSTER=false ghe-host-check ) end_test From 0bb47041013c696121843420bd8c4a59b1326b75 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Thu, 2 Nov 2023 20:09:21 +0000 Subject: [PATCH 326/410] review updates --- backup.config-example | 4 ++++ bin/ghe-host-check | 11 +++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backup.config-example b/backup.config-example index 535a2f636..99bcfd232 100644 --- a/backup.config-example +++ b/backup.config-example @@ -32,6 +32,10 @@ GHE_NUM_SNAPSHOTS=10 # disk space validation and software version checks on the backup-host will be disabled. #GHE_SKIP_CHECKS=false +# Cluster filesystem to check if it's writable as part of ghe-host-check +# By default it is /data/user/tmp but can be updated if needed +#GHE_FILE_SYSTEM_CHECK="/data/user/tmp" + # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname # here. The host to restore to may also be specified directly when running diff --git a/bin/ghe-host-check b/bin/ghe-host-check index cf62e6453..b5bdcd1ff 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -120,23 +120,22 @@ fi NON_WRITABLE="" # ensure all nodes are writable if [ "$CLUSTER" == "true" ] ; then - if [ -z "$FILE_TO_TEST" ]; then + if [ -z "$GHE_FILE_SYSTEM_CHECK" ]; then if [ -d "/data/user/tmp" ]; then FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" else FILE_TO_TEST="/tmp/test-ro-file.txt" fi + else + FILE_TO_TEST="$GHE_FILE_SYSTEM_CHECK/test-ro-file.txt" fi # Iterate through each node in the cluster nodes=$(ghe-ssh "$host" ghe-cluster-nodes) first=true - # echo $nodes | while read -r node; do for node in $nodes; do if ! echo "set -o pipefail; ssh $node -- 'touch $FILE_TO_TEST && rm $FILE_TO_TEST'" | ghe-ssh "$host" /bin/bash; then - #echo "File system is writable on $node" - #else - echo "File system is not writeable on $node" 1>&2 + echo "File system is not writeable or no permission on $node" 1>&2 if [ "$first" = true ]; then NON_WRITABLE="$node" first=false @@ -147,7 +146,7 @@ if [ "$CLUSTER" == "true" ] ; then done # Display the comma-separated list of non-writable nodes if [ ! -z "$NON_WRITABLE" ]; then - echo "Error: Non-writable nodes: $NON_WRITABLE . Please make sure the file systems for all Enterprise nodes are writable." 1>&2 + log_error "Error: Following nodes are non-writable - $NON_WRITABLE. Please make sure the filesystem for all GHES nodes are writable." 1>&2 exit 1 else echo "All nodes are writable." From 6d86ba0920b326e3335370939a28de86ba40a34f Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Fri, 3 Nov 2023 14:55:07 +0000 Subject: [PATCH 327/410] fix typo --- bin/ghe-host-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index b5bdcd1ff..24a9ed262 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -165,7 +165,7 @@ if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then fi if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then - if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state1 2>/dev/null || true)" = "replica" ]; then + if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then echo "Error: high availability replica detected." 1>&2 echo "Backup Utilities should be used to backup from the primary node in" 1>&2 echo "high availability environments to ensure consistent and reliable backups." 1>&2 From 6396e77e3100b9eceb5ec3c9fc8eb53977a52277 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Fri, 3 Nov 2023 16:28:48 +0000 Subject: [PATCH 328/410] log error msg when any node is offline --- bin/ghe-backup | 2 +- bin/ghe-host-check | 9 ++++++--- share/github-backup-utils/ghe-backup-config | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 1dfaf748b..eeb872e3d 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -375,7 +375,7 @@ if [ -z "$failures" ]; then else log_info "Expired and incomplete snapshots to be pruned separately" fi -else +else log_info "Skipping pruning snapshots, since some backups failed..." fi diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 5cd39abee..60bdddf21 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -91,11 +91,13 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi +set +e # ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then online_status=$(ghe-ssh "$host" ghe-cluster-host-check) if [ "$online_status" != "Cluster is ready to configure." ]; then - echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + echo "$online_status" 1>&2 + log_error "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 exit 1 fi @@ -103,10 +105,11 @@ if "$CLUSTER"; then distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then echo "Version mismatch: $node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 + log_error "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi +set -e version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}') @@ -177,7 +180,7 @@ SKIP_MSG #Display dir requirements for repositories and mysql echo -e "\nChecking host for sufficient space for a backup..." - available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') + available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." repos_disk_size=$(transfer_size repositories /tmp) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 6c33ea954..08379ce95 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -667,7 +667,7 @@ init-progress() { mkdir -p /tmp/backup-utils-progress chmod -R 777 /tmp/backup-utils-progress - + touch /tmp/backup-utils-progress/total touch /tmp/backup-utils-progress/type touch /tmp/backup-utils-progress/progress From be270eeeff0bc5684a0b7fd6c8727e5a9445f0b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:44:28 +0000 Subject: [PATCH 329/410] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backup.yml | 2 +- .github/workflows/backuprestore.yml | 2 +- .github/workflows/integration-tests.yml | 4 ++-- .github/workflows/lint.yml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/rsync-docker-bump.yml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backup.yml b/.github/workflows/backup.yml index cfb154aae..2b59e7894 100644 --- a/.github/workflows/backup.yml +++ b/.github/workflows/backup.yml @@ -71,7 +71,7 @@ jobs: name: backup-utils - name: Load docker container run: docker load -i backup-utils.tar - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create backup directory run: mkdir "$HOME/ghe-backup-data" - name: set up ssh SSH_KEY diff --git a/.github/workflows/backuprestore.yml b/.github/workflows/backuprestore.yml index 34cf69032..a87c54ec9 100644 --- a/.github/workflows/backuprestore.yml +++ b/.github/workflows/backuprestore.yml @@ -49,7 +49,7 @@ jobs: name: backup-utils - name: Load docker container run: docker load -i backup-utils.tar - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create backup directory run: mkdir "$HOME"/ghe-backup-data - name: set up ssh SSH_KEY diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 603f7affa..a2cf68531 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -27,7 +27,7 @@ jobs: - enterprise2-backup-utils-binary-backup - enterprise2-backup-utils-migration steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Queue ${{ matrix.jankyJobName }} build @@ -50,7 +50,7 @@ jobs: - enterprise2-backup-utils-cluster-binary-backup - enterprise2-backup-utils-cluster-migration steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 1 - name: Queue ${{ matrix.jankyJobName }} build diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7fc33e1cd..0f26e9aa1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50583737b..e1917684d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ jobs: tar --xz -xvf "shellcheck-stable.linux.x86_64.tar.xz" sudo cp shellcheck-stable/shellcheck /usr/bin/shellcheck - name: Get Sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Test run: | export PATH="$PATH:/snap/bin" diff --git a/.github/workflows/rsync-docker-bump.yml b/.github/workflows/rsync-docker-bump.yml index 2cfe7782c..0acaae854 100644 --- a/.github/workflows/rsync-docker-bump.yml +++ b/.github/workflows/rsync-docker-bump.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get latest rsync tag id: latest_tag From 5460d8458371414051261e62952478c7d1db8a21 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Fri, 3 Nov 2023 17:03:59 +0000 Subject: [PATCH 330/410] test update --- test/test-ghe-host-check.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index a772399d6..f1e964da5 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -134,7 +134,7 @@ begin_test "ghe-host-check fails when encountering RO file-system" ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"' # File creation fails for CLUSTER - ! FILE_TO_TEST="$HOME/tmp/test" CLUSTER=true ghe-host-check - FILE_TO_TEST="$HOME/tmp/test" CLUSTER=false ghe-host-check + ! FILE_TO_TEST="$HOME/tmp/test" CLUSTER=true GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check + FILE_TO_TEST="$HOME/tmp/test" CLUSTER=false GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check ) end_test From 8116c8a0802b5434ca6c3530568a51cec430af84 Mon Sep 17 00:00:00 2001 From: Dax Amin Date: Fri, 3 Nov 2023 13:38:15 -0400 Subject: [PATCH 331/410] remove empty line --- share/github-backup-utils/ghe-backup-config | 1 - 1 file changed, 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 9af250b5f..cd9972b59 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -665,7 +665,6 @@ init-progress() { if [ -e /tmp/backup-utils-progress ]; then rm -rf /tmp/backup-utils-progress/* fi - # shellcheck disable=SC2174 # We are fine with -m only affecting the deepest directory mkdir -m 777 -p /tmp/backup-utils-progress touch /tmp/backup-utils-progress/{total,type,progress,info} From 47abd682e9d8cb8c34bb40a0eafece6224963b67 Mon Sep 17 00:00:00 2001 From: alejndr0 Date: Mon, 6 Nov 2023 15:54:59 +0000 Subject: [PATCH 332/410] Disable compression on ghe-backup-repositories by default --- share/github-backup-utils/ghe-backup-repositories | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index e90d8c37a..636e610c8 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -194,7 +194,7 @@ sync_data (){ echo 1>&3 log_info "* Transferring auxiliary files ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 log_info "* Transferring packed-refs files ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 log_info "* Transferring refs and reflogs ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 log_info "* Transferring special data directories from $h..." 1>&3 - rsync_repository_data $h:122 -z < Date: Mon, 6 Nov 2023 15:13:51 -0500 Subject: [PATCH 333/410] Update backup.config-example Co-authored-by: Quinn Murphy --- backup.config-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 99bcfd232..66dc5731e 100644 --- a/backup.config-example +++ b/backup.config-example @@ -34,7 +34,7 @@ GHE_NUM_SNAPSHOTS=10 # Cluster filesystem to check if it's writable as part of ghe-host-check # By default it is /data/user/tmp but can be updated if needed -#GHE_FILE_SYSTEM_CHECK="/data/user/tmp" +#GHE_FILE_SYSTEM_WRITE_CHECK="/data/user/tmp" # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname From 10a2905c8c37415c21ed18504a2f9ca5f6556e36 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Mon, 6 Nov 2023 22:17:20 +0000 Subject: [PATCH 334/410] readability updates --- bin/ghe-host-check | 22 +++++++++------------- test/test-ghe-host-check.sh | 4 ++-- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 24a9ed262..a432f3f7c 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -120,36 +120,32 @@ fi NON_WRITABLE="" # ensure all nodes are writable if [ "$CLUSTER" == "true" ] ; then - if [ -z "$GHE_FILE_SYSTEM_CHECK" ]; then + if [ -z "$GHE_FILE_SYSTEM_WRITE_CHECK" ]; then if [ -d "/data/user/tmp" ]; then - FILE_TO_TEST="/data/user/tmp/test-ro-file.txt" + WRITE_CHECK_FILE="/data/user/tmp/test-ro-file.txt" else - FILE_TO_TEST="/tmp/test-ro-file.txt" + WRITE_CHECK_FILE="/tmp/test-ro-file.txt" fi else - FILE_TO_TEST="$GHE_FILE_SYSTEM_CHECK/test-ro-file.txt" + WRITE_CHECK_FILE="$GHE_FILE_SYSTEM_CHECK/test-ro-file.txt" fi # Iterate through each node in the cluster nodes=$(ghe-ssh "$host" ghe-cluster-nodes) first=true for node in $nodes; do - if ! echo "set -o pipefail; ssh $node -- 'touch $FILE_TO_TEST && rm $FILE_TO_TEST'" | ghe-ssh "$host" /bin/bash; then + if ! echo "set -o pipefail; ssh $node -- 'touch $WRITE_CHECK_FILE && rm $WRITE_CHECK_FILE'" | ghe-ssh "$host" /bin/bash; then echo "File system is not writeable or no permission on $node" 1>&2 - if [ "$first" = true ]; then - NON_WRITABLE="$node" - first=false - else - NON_WRITABLE="$NON_WRITABLE, $node" - fi + NON_WRITABLE+="$node " fi || true done # Display the comma-separated list of non-writable nodes - if [ ! -z "$NON_WRITABLE" ]; then + if [ -n "$NON_WRITABLE" ]; then + NON_WRITABLE=$(echo "$NON_WRITABLE" | sed 's/ /, /g; s/, $//') log_error "Error: Following nodes are non-writable - $NON_WRITABLE. Please make sure the filesystem for all GHES nodes are writable." 1>&2 exit 1 else - echo "All nodes are writable." + log_info "All nodes are writable." fi fi diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index f1e964da5..50b4ed996 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -134,7 +134,7 @@ begin_test "ghe-host-check fails when encountering RO file-system" ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"' # File creation fails for CLUSTER - ! FILE_TO_TEST="$HOME/tmp/test" CLUSTER=true GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check - FILE_TO_TEST="$HOME/tmp/test" CLUSTER=false GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check + ! WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=true GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check + WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=false GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check ) end_test From 265b9fa06b0fccf22cc743652d13615aa2af398a Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Tue, 7 Nov 2023 04:29:12 +0000 Subject: [PATCH 335/410] SC fix --- bin/ghe-host-check | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index a432f3f7c..fedc84852 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -132,7 +132,6 @@ if [ "$CLUSTER" == "true" ] ; then # Iterate through each node in the cluster nodes=$(ghe-ssh "$host" ghe-cluster-nodes) - first=true for node in $nodes; do if ! echo "set -o pipefail; ssh $node -- 'touch $WRITE_CHECK_FILE && rm $WRITE_CHECK_FILE'" | ghe-ssh "$host" /bin/bash; then echo "File system is not writeable or no permission on $node" 1>&2 From 18d6113aedd699df376eb25c9089304262aee193 Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Tue, 7 Nov 2023 14:26:38 +0000 Subject: [PATCH 336/410] progress file for ha instances --- share/github-backup-utils/ghe-backup-repositories | 6 ++++++ share/github-backup-utils/ghe-backup-storage | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index e90d8c37a..f4d3a4b44 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -78,6 +78,12 @@ if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi +# hostnames for HA +if ghe-ssh "$GHE_HOSTNAME" -- \ + "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then + hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i |cut -f 2) +fi + # Make sure root backup dir exists if this is the first run mkdir -p "$backup_dir" diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index 9752de21d..bac467c8a 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -51,6 +51,12 @@ if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi +# hostnames for HA +if ghe-ssh "$GHE_HOSTNAME" -- \ + "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then + hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i |cut -f 2) +fi + # Make sure root backup dir exists if this is the first run mkdir -p "$backup_dir" From eabec3431bf55db7e5e8dc92948aadc582a3371d Mon Sep 17 00:00:00 2001 From: Chuck Pathanjali Date: Tue, 7 Nov 2023 18:32:02 +0000 Subject: [PATCH 337/410] re-trigger for cluster --- share/github-backup-utils/ghe-backup-repositories | 2 +- share/github-backup-utils/ghe-backup-storage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index f4d3a4b44..abf73a6cd 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -81,7 +81,7 @@ fi # hostnames for HA if ghe-ssh "$GHE_HOSTNAME" -- \ "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i |cut -f 2) + hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i | cut -f 2) fi # Make sure root backup dir exists if this is the first run diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index bac467c8a..c64309202 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -54,7 +54,7 @@ fi # hostnames for HA if ghe-ssh "$GHE_HOSTNAME" -- \ "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i |cut -f 2) + hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i | cut -f 2) fi # Make sure root backup dir exists if this is the first run From 11a96f6d24bbd858afa1e594edbf0083bcd305ac Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 7 Nov 2023 10:54:33 -0800 Subject: [PATCH 338/410] Update docs/requirements.md --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index af0fc805f..5a8c4347a 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), and [jq][11] v1.5 or newer. See below for an update on rsync. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.6 or newer. See below for an update on rsync. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From f19c20639c7e5da9e6b0e52de0c14b6484edc349 Mon Sep 17 00:00:00 2001 From: djdefi Date: Tue, 7 Nov 2023 15:14:02 -0800 Subject: [PATCH 339/410] Update requirements.md --- docs/requirements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.md b/docs/requirements.md index 5a8c4347a..20b3a3fad 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,7 +5,7 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.6 or newer. See below for an update on rsync. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.7 or newer. See below for an update on rsync. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. From cc320085950c6bdb464f4ef1adf85bba19d81bae Mon Sep 17 00:00:00 2001 From: "Roger D. Winans" Date: Tue, 14 Nov 2023 23:43:39 +0000 Subject: [PATCH 340/410] Merge pull request #726 from github/solvaholic/soften-warning Remove "contact support" from warning about route verification --- share/github-backup-utils/ghe-backup-repositories | 2 +- share/github-backup-utils/ghe-backup-storage | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index 68442b527..543effb35 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -384,7 +384,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then done |sort|uniq > $tempdir/source_routes (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." + git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance." increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" fi diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index c64309202..c513290bf 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -155,7 +155,7 @@ if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." + git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance." increment-progress-total-count 1 bm_end "$(basename $0) - Verifying Routes" From cc6566b6185875fd12ecaf2f113e58d6646f597d Mon Sep 17 00:00:00 2001 From: Dax Amin Date: Thu, 16 Nov 2023 08:54:45 +0000 Subject: [PATCH 341/410] Merge pull request #725 from github/ha-sync Create sync file on HA replica from primary node [cluster] --- share/github-backup-utils/ghe-backup-config | 30 +++++++++++++++++++ .../ghe-backup-repositories | 25 ++++++++++++---- share/github-backup-utils/ghe-backup-storage | 25 ++++++++++++---- share/github-backup-utils/ghe-gc-disable | 18 ++--------- 4 files changed, 72 insertions(+), 26 deletions(-) diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index cd9972b59..30062a8f6 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -676,3 +676,33 @@ increment-progress-total-count() { ((PROGRESS_TOTAL += $1)) echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total } + +## +# This function is used by ghe-gc-disable, ghe-backup-repositories, and ghe-backup-storage +# This function should be used directly to disable and drain GC operations ONLY on HA-replica node +# (as done in ghe-backup-repositories and ghe-backup-storage) +# Otherwise use ghe-gc-disable which will call this function with the correct parameters. +# +# Arguments: +# $1 - path to sync-in-progress file ($SYNC_IN_PROGRESS_FILE) +# $2 - git cooldown period ($GHE_GIT_COOLDOWN_PERIOD) +## +gc_disable() { + set -e + local sync_in_progress="$1" + local git_cooldown_period="$2" + + # Touch the sync-in-progress file, disabling GC operations, and wait for all + # active GC processes to finish on the remote side. + sudo -u git touch "$sync_in_progress" + for _ in $(seq $git_cooldown_period); do + # note: the bracket synta[x] below is to prevent matches against the + # grep process itself. + if ps axo args | grep -E -e "^git( -.*)? nw-repac[k]( |$)" -e "^git( -.*)? g[c]( |$)" >/dev/null; then + sleep 1 + else + exit 0 + fi + done + exit 7 +} diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories index 68442b527..209d81367 100755 --- a/share/github-backup-utils/ghe-backup-repositories +++ b/share/github-backup-utils/ghe-backup-repositories @@ -78,10 +78,9 @@ if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi -# hostnames for HA -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i | cut -f 2) +# Replica hostnames for HA +if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then + ha_replica_hosts=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes --replica) fi # Make sure root backup dir exists if this is the first run @@ -96,11 +95,18 @@ cleanup() { # Enable remote GC operations for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { + ghe-gc-enable $ssh_config_file_opt $hostname:$port || { echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 } done + # Enable remote GC operations for HA replica + for replica_host in $ha_replica_hosts; do + echo "set -o pipefail; ssh $replica_host -- 'sudo rm -f $SYNC_IN_PROGRESS_FILE'" | ghe-ssh "$host" /bin/bash || { + echo "Re-enable gc on $replica_host failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 + } + done + ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir rm -rf $tempdir } @@ -111,6 +117,15 @@ for hostname in $hostnames; do ghe-gc-disable $ssh_config_file_opt $hostname:$port done +# Disable remote GC operations for HA replica +# gc_disable is a function defined in ghe-backup-config +# gc_disable is called on the replica node via the primary node, because replica node is not expected to be reachable from backup host. But replica node is expected to be reachable from primary node. +for replica_host in $ha_replica_hosts; do + echo "set -o pipefail; ssh $replica_host -- '$(declare -f gc_disable); gc_disable \"$SYNC_IN_PROGRESS_FILE\" \"$GHE_GIT_COOLDOWN_PERIOD\"'" | ghe-ssh "$host" /bin/bash || { + echo "Disable gc on $replica_host failed" 1>&2 + } +done + # If we have a previous increment, avoid transferring existing files via rsync's # --link-dest support. This also decreases physical space usage considerably. if [ -d "$backup_current" ]; then diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage index c64309202..82ec98f2e 100755 --- a/share/github-backup-utils/ghe-backup-storage +++ b/share/github-backup-utils/ghe-backup-storage @@ -51,10 +51,9 @@ if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" fi -# hostnames for HA -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -i | cut -f 2) +# Replica hostnames for HA +if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then + ha_replica_hosts=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes --replica) fi # Make sure root backup dir exists if this is the first run @@ -65,11 +64,18 @@ mkdir -p "$backup_dir" cleanup() { # Enable remote maintenance operations for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { + ghe-gc-enable $ssh_config_file_opt $hostname:$port || { log_warn "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 } done + # Enable remote GC operations for HA replica + for replica_host in $ha_replica_hosts; do + echo "set -o pipefail; ssh $replica_host -- 'sudo rm -f $SYNC_IN_PROGRESS_FILE'" | ghe-ssh "$host" /bin/bash || { + echo "Re-enable gc on $replica_host failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 + } + done + ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir rm -rf $tempdir } @@ -80,6 +86,15 @@ for hostname in $hostnames; do ghe-gc-disable $ssh_config_file_opt $hostname:$port done +# Disable remote GC operations for HA replica +# gc_disable is a function defined in ghe-backup-config +# gc_disable is called on the replica node via the primary node, because replica node is not expected to be reachable from backup host. But replica node is expected to be reachable from primary node. +for replica_host in $ha_replica_hosts; do + echo "set -o pipefail; ssh $replica_host -- '$(declare -f gc_disable); gc_disable \"$SYNC_IN_PROGRESS_FILE\" \"$GHE_GIT_COOLDOWN_PERIOD\"'" | ghe-ssh "$host" /bin/bash || { + echo "Disable gc on $replica_host failed" 1>&2 + } +done + # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's # --link-dest support. This also decreases physical space usage considerably. if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/storage)" ]; then diff --git a/share/github-backup-utils/ghe-gc-disable b/share/github-backup-utils/ghe-gc-disable index 1e5a63bc5..67cdef6d4 100755 --- a/share/github-backup-utils/ghe-gc-disable +++ b/share/github-backup-utils/ghe-gc-disable @@ -32,22 +32,8 @@ done # Exit early when testing [ -n "$GHE_TEST_REMOTE_VERSION" ] && exit 0 -# Touch the sync-in-progress file, disabling GC operations, and wait for all -# active GC processes to finish on the remote side. -echo " - set -e - sudo -u git touch '$SYNC_IN_PROGRESS_FILE' - for i in \$(seq $GHE_GIT_COOLDOWN_PERIOD); do - # note: the bracket synta[x] below is to prevent matches against the - # grep process itself. - if ps axo args | grep -E -e '^git( -.*)? nw-repac[k]( |$)' -e '^git( -.*)? g[c]( |$)' >/dev/null; then - sleep 1 - else - exit 0 - fi - done - exit 7 -" | ghe-ssh $opts "$host" -- /bin/bash || { +# gc_disable is a function defined in ghe-backup-config +echo "set -o pipefail; $(declare -f gc_disable); gc_disable \"$SYNC_IN_PROGRESS_FILE\" \"$GHE_GIT_COOLDOWN_PERIOD\"" | ghe-ssh $opts "$host" -- /bin/bash || { res=$? if [ $res = 7 ]; then log_error "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds. Aborting..." 1>&2 From b2131769bb948a91eb325804e41a5c197eb6d81e Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Tue, 28 Nov 2023 15:54:38 -0600 Subject: [PATCH 342/410] Update versions --- bin/ghe-host-check | 2 +- share/github-backup-utils/version | 2 +- test/testlib.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index abc0b40b2..59bc9b526 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -173,7 +173,7 @@ fi # backup-utils 2.13 onwards limits support to the current and previous two releases # of GitHub Enterprise Server. -supported_minimum_version="3.7.0" +supported_minimum_version="3.9.0" if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then supported=1 diff --git a/share/github-backup-utils/version b/share/github-backup-utils/version index 30291cba2..afad81866 100644 --- a/share/github-backup-utils/version +++ b/share/github-backup-utils/version @@ -1 +1 @@ -3.10.0 +3.11.0 diff --git a/test/testlib.sh b/test/testlib.sh index edeec8ea2..8251fa316 100755 --- a/test/testlib.sh +++ b/test/testlib.sh @@ -48,7 +48,7 @@ export GHE_BACKUP_CONFIG GHE_DATA_DIR GHE_REMOTE_DATA_DIR GHE_REMOTE_ROOT_DIR # The default remote appliance version. This may be set in the environment prior # to invoking tests to emulate a different remote vm version. -: ${GHE_TEST_REMOTE_VERSION:=3.8.0.rc1} +: ${GHE_TEST_REMOTE_VERSION:=3.11.0.rc1} export GHE_TEST_REMOTE_VERSION # Source in the backup config and set GHE_REMOTE_XXX variables based on the From bec5f580420fe65289a6bc32ee90d580516f7377 Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Tue, 28 Nov 2023 16:56:51 -0600 Subject: [PATCH 343/410] Fix ghe-host-check unsupported version test for unreleased feature versions --- test/test-ghe-host-check.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 50b4ed996..55f2e6c83 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -68,10 +68,10 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions if [ -z "$supported" ] then #BACKUP_UTILS_VERSION WAS NOT FOUND IN LATEST.JSON, CHECK IF ITS GREATER THAN LAST VERSION - if [ "$(version $bu_major_minor)" -ge "$(version ${keys[$((${#keys[@]} - 2 ))]})" ]; then + if [ "$(version $bu_major_minor)" -ge "$(version ${keys[$((${#keys[@]}))]})" ]; then GHE_TEST_REMOTE_VERSION="$bu_major_minor.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]} - 2 ))]}.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]} - 3 ))]}.0" ghe-host-check + GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]}))]}.0" ghe-host-check + GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]} - 1 ))]}.0" ghe-host-check fi else #BACKUP_UTILS_VERSION WAS FOUND IN LATEST.JSON From 3cef5b8df620226071a9e4a61d878c638545099b Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Tue, 28 Nov 2023 20:28:02 -0600 Subject: [PATCH 344/410] Fix ghe-host-check unsupported version test by sorting supported versions --- test/test-ghe-host-check.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 55f2e6c83..fb547bf34 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -63,32 +63,30 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions # Use the modified releases string as needed supported=$(echo "$releases_with_replacement" | jq -r 'select(."'${bu_major_minor}'")') # shellcheck disable=SC2207 # Command required as alternatives fail - keys=($(echo "$releases_with_replacement" | jq -r 'keys[]')) + keys=($(echo "$releases_with_replacement" | jq -r '. | keys_unsorted | sort_by( split(".") | map(tonumber) )[]')) if [ -z "$supported" ] then #BACKUP_UTILS_VERSION WAS NOT FOUND IN LATEST.JSON, CHECK IF ITS GREATER THAN LAST VERSION - if [ "$(version $bu_major_minor)" -ge "$(version ${keys[$((${#keys[@]}))]})" ]; then + if [ "$(version $bu_major_minor)" -ge "$(version ${keys[-1]})" ]; then GHE_TEST_REMOTE_VERSION="$bu_major_minor.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]}))]}.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[$(( ${#keys[@]} - 1 ))]}.0" ghe-host-check + # Test most recent 2 versions + GHE_TEST_REMOTE_VERSION="${keys[-1]}.0" ghe-host-check + GHE_TEST_REMOTE_VERSION="${keys[-2]}.0" ghe-host-check fi else #BACKUP_UTILS_VERSION WAS FOUND IN LATEST.JSON ix=0 - for i in "${keys[@]}";do + for i in "${keys[@]}";do if [ "$i" == "$bu_major_minor" ];then break fi ix=$(( $ix + 1 )) done GHE_TEST_REMOTE_VERSION="${keys[$ix]}.0" ghe-host-check - # sometimes when the latest.json is updated during a release this test gets broken. - if [ "${keys[$(( $ix - 1 ))]}" != "latest" ]; then - GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check - fi + # Test previous 2 supported versions + GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 2 ))]}.0" ghe-host-check - fi ! GHE_TEST_REMOTE_VERSION=11.340.36 ghe-host-check GHE_TEST_REMOTE_VERSION=$bu_version_major.$bu_version_minor.999 ghe-host-check From b4b127f1762b71a1153f660632f9aaf8e23e1e12 Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Wed, 29 Nov 2023 00:16:57 -0600 Subject: [PATCH 345/410] Fix ghe-host-check unsupported version test for unreleased versions --- test/test-ghe-host-check.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index fb547bf34..80a62cfc5 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -70,9 +70,10 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions #BACKUP_UTILS_VERSION WAS NOT FOUND IN LATEST.JSON, CHECK IF ITS GREATER THAN LAST VERSION if [ "$(version $bu_major_minor)" -ge "$(version ${keys[-1]})" ]; then GHE_TEST_REMOTE_VERSION="$bu_major_minor.0" ghe-host-check - # Test most recent 2 versions + # Test most recent version + # Don't test 2 versions back because it fails when we bump the version on + # master after branching for a feature release, before it's released GHE_TEST_REMOTE_VERSION="${keys[-1]}.0" ghe-host-check - GHE_TEST_REMOTE_VERSION="${keys[-2]}.0" ghe-host-check fi else #BACKUP_UTILS_VERSION WAS FOUND IN LATEST.JSON From 872f6569e73b6eed2cb7dfbf00a33becbc7d0c23 Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Wed, 29 Nov 2023 15:52:58 -0600 Subject: [PATCH 346/410] Revert "Merge pull request #717 from github/daxamin-offline-error" This reverts commit c1a2b87fba87766abe57d1b6c4bcfb4d6cb9ad65, reversing changes made to 0889f597bfa044b5d8554e86a13b440863712a3b. --- bin/ghe-backup | 2 +- bin/ghe-host-check | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 1ae6b868c..448a5840e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -377,7 +377,7 @@ if [ -z "$failures" ]; then else log_info "Expired and incomplete snapshots to be pruned separately" fi -else +else log_info "Skipping pruning snapshots, since some backups failed..." fi diff --git a/bin/ghe-host-check b/bin/ghe-host-check index abc0b40b2..fc0a0b6e2 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -93,13 +93,11 @@ if ghe-ssh "$host" -- \ CLUSTER=true fi -set +e # ensure all nodes in the cluster are online/reachable and running the same version if "$CLUSTER"; then online_status=$(ghe-ssh "$host" ghe-cluster-host-check) if [ "$online_status" != "Cluster is ready to configure." ]; then - echo "$online_status" 1>&2 - log_error "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 + echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 exit 1 fi @@ -107,11 +105,10 @@ if "$CLUSTER"; then distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) if [ "$distinct_versions" -ne 1 ]; then echo "Version mismatch: $node_version_list" 1>&2 - log_error "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 + echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 exit 1 fi fi -set -e version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}') @@ -213,7 +210,7 @@ SKIP_MSG #Display dir requirements for repositories and mysql echo -e "\nChecking host for sufficient space for a backup..." - available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') + available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." repos_disk_size=$(transfer_size repositories /tmp) From 01f42a98927398a3b49d04a4d345045ff85598aa Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Wed, 29 Nov 2023 15:53:28 -0600 Subject: [PATCH 347/410] Revert "Merge pull request #700 from github/check-ro-filesystem" This reverts commit 0aa36fdd1ed87b608feadc07a4585247d36e0970, reversing changes made to c1a2b87fba87766abe57d1b6c4bcfb4d6cb9ad65. --- backup.config-example | 4 ---- bin/ghe-host-check | 35 +---------------------------------- test/test-ghe-host-check.sh | 15 --------------- 3 files changed, 1 insertion(+), 53 deletions(-) diff --git a/backup.config-example b/backup.config-example index 66dc5731e..535a2f636 100644 --- a/backup.config-example +++ b/backup.config-example @@ -32,10 +32,6 @@ GHE_NUM_SNAPSHOTS=10 # disk space validation and software version checks on the backup-host will be disabled. #GHE_SKIP_CHECKS=false -# Cluster filesystem to check if it's writable as part of ghe-host-check -# By default it is /data/user/tmp but can be updated if needed -#GHE_FILE_SYSTEM_WRITE_CHECK="/data/user/tmp" - # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname # here. The host to restore to may also be specified directly when running diff --git a/bin/ghe-host-check b/bin/ghe-host-check index fc0a0b6e2..5cd39abee 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -85,9 +85,7 @@ if [ $rc -ne 0 ]; then exit $rc fi -if [ -z "$CLUSTER" ]; then - CLUSTER=false -fi +CLUSTER=false if ghe-ssh "$host" -- \ "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then CLUSTER=true @@ -117,37 +115,6 @@ if [ -z "$version" ]; then exit 2 fi -NON_WRITABLE="" -# ensure all nodes are writable -if [ "$CLUSTER" == "true" ] ; then - if [ -z "$GHE_FILE_SYSTEM_WRITE_CHECK" ]; then - if [ -d "/data/user/tmp" ]; then - WRITE_CHECK_FILE="/data/user/tmp/test-ro-file.txt" - else - WRITE_CHECK_FILE="/tmp/test-ro-file.txt" - fi - else - WRITE_CHECK_FILE="$GHE_FILE_SYSTEM_CHECK/test-ro-file.txt" - fi - - # Iterate through each node in the cluster - nodes=$(ghe-ssh "$host" ghe-cluster-nodes) - for node in $nodes; do - if ! echo "set -o pipefail; ssh $node -- 'touch $WRITE_CHECK_FILE && rm $WRITE_CHECK_FILE'" | ghe-ssh "$host" /bin/bash; then - echo "File system is not writeable or no permission on $node" 1>&2 - NON_WRITABLE+="$node " - fi || true - done - # Display the comma-separated list of non-writable nodes - if [ -n "$NON_WRITABLE" ]; then - NON_WRITABLE=$(echo "$NON_WRITABLE" | sed 's/ /, /g; s/, $//') - log_error "Error: Following nodes are non-writable - $NON_WRITABLE. Please make sure the filesystem for all GHES nodes are writable." 1>&2 - exit 1 - else - log_info "All nodes are writable." - fi -fi - # Block restoring snapshots to older releases of GitHub Enterprise Server if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) diff --git a/test/test-ghe-host-check.sh b/test/test-ghe-host-check.sh index 50b4ed996..f2c5cca54 100755 --- a/test/test-ghe-host-check.sh +++ b/test/test-ghe-host-check.sh @@ -123,18 +123,3 @@ begin_test "ghe-host-check blocks restore to old release" ! GHE_TEST_REMOTE_VERSION=$bu_version_major.$((bu_version_minor-1)).$bu_version_patch ghe-restore -v ) end_test - -# Check ghe-host-check detects RO file system -begin_test "ghe-host-check fails when encountering RO file-system" -( - set -e - - ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p "~/tmp"' - # Remove write access in ~/tmp - ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"' - - # File creation fails for CLUSTER - ! WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=true GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check - WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=false GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check -) -end_test From 3bc10598b490069c033ef7c05ba89a6ba24819e9 Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Wed, 29 Nov 2023 19:51:17 -0600 Subject: [PATCH 348/410] Add 3.11.0 release notes --- release-notes/3.11.0.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 release-notes/3.11.0.md diff --git a/release-notes/3.11.0.md b/release-notes/3.11.0.md new file mode 100644 index 000000000..197501c92 --- /dev/null +++ b/release-notes/3.11.0.md @@ -0,0 +1,18 @@ +### Bug Fixes + +* `ghe-backup-myql` and `ghe-restore-mysql` will now exit 1 on errors. +* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. +* When backups are run for HA instances in both primary and replica nodes a `.sync-in-progress` file will be created. This will disable `NetworkMaintenance` jobs from running and queueing up when backups are running from the primary node. + +### Changes + +* Estimated transfer sizes will be calculated on appropriate nodes for clustered environments. +* Added support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. +* `ghe-restore` avoids unnecessary `rsync` operations when restoring to non-clustered environments. +* `ghe-backup` and `ghe-restore` output their total runtime +* `rsync` compression is now disabled by default. The `-z` flag has been removed from the `ghe-rsync` command in multiple files to improve transfer speed and reduce CPU usage. To enable `rsync` compression globally, add `GHE_RSYNC_COMPRESSION_ENABLED=yes` to the `backup.config` file. +* Updates the Host OS version output to use `/etc/os-release` for better compatibility with other Linux distributions. + +### Backups and Disaster Recovery + +* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. From 5534c57f0ea41c31c8d3e4aab9330836cf241598 Mon Sep 17 00:00:00 2001 From: Andrew Mildahl Date: Wed, 29 Nov 2023 20:01:02 -0600 Subject: [PATCH 349/410] Add 3.11.0 debian/changelog --- debian/changelog | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/debian/changelog b/debian/changelog index fb6b9940b..0d6f1e8d4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,49 @@ +github-backup-utils (3.11.0) UNRELEASED; urgency=medium + + * `ghe-backup-myql` and `ghe-restore-mysql` will now exit 1 on errors. + * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. + * When backups are run for HA instances in both primary and replica nodes a `.sync-in-progress` file will be created. This will disable `NetworkMaintenance` jobs from running and queueing up when backups are running from the primary node. + * Estimated transfer sizes will be calculated on appropriate nodes for clustered environments. + * Added support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. + * `ghe-restore` avoids unnecessary `rsync` operations when restoring to non-clustered environments. + * `ghe-backup` and `ghe-restore` output their total runtime + * `rsync` compression is now disabled by default. The `-z` flag has been removed from the `ghe-rsync` command in multiple files to improve transfer speed and reduce CPU usage. To enable `rsync` compression globally, add `GHE_RSYNC_COMPRESSION_ENABLED=yes` to the `backup.config` file. + * Updates the Host OS version output to use `/etc/os-release` for better compatibility with other Linux distributions. + * When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. + + -- Andrew Mildahl Thu, 30 Nov 2023 01:57:12 +0000 + +github-backup-utils (3.10.0) UNRELEASED; urgency=medium + + -- Balwinder Sohi Wed, 30 Aug 2023 17:39:31 +0000 + +github-backup-utils (3.10.0) UNRELEASED; urgency=medium + + * Remove -o option from ps use #341 + * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 + * Move check for git for ssh muxing into ghe-ssh #378 + * Check filesystem supports hardlinks #388 + * Remove check for git from ghe-ssh #393 + * Clean up stale HA nodes on restore #396 + + -- Balwinder Sohi Wed, 09 Aug 2023 19:37:10 +0000 + +github-backup-utils (3.9.0) UNRELEASED; urgency=medium + + * Set restore status on all cluster nodes #274 + * Fix pages backups and restores in GitHub Enterprise 11.10 #275 + * Backup and restore custom CA certificates #281 + * Set the benchmark file path consistently #283 + * Suppress dd output noise #289 + * Track completeness of Elasticsearch JSON dumps #298 + * Use existing Elasticsearch indices to speed up transfer during a restore #310 + * Include the user data directory in the benchmark name #311 + * Use calculated routes when backing up storage data from a cluster #318 + * Refresh the existing indices when restoring Elasticsearch indices to cluster #328 + * Use git to generate short name for SSH multiplex control path #335 + + -- Junior Eluhu Mon, 12 Jun 2023 20:46:10 +0000 + github-backup-utils (3.8.0) focal; urgency=medium -- Daniel Johnson Tue, 07 Feb 2023 21:43:26 +0000 From 07686532d824e3384183453ae7eaf2a1621b2e56 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 11:59:38 -0500 Subject: [PATCH 350/410] Simplified the Support section --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ee951f58..8e1d1adb2 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,8 @@ GitHub Enterprise Server. ## Support -If you find a bug or would like to request a feature in Backup Utilities, please -open an issue or pull request on this repository. If you have a question related -to your specific GitHub Enterprise Server setup or would like assistance with -backup site setup or recovery, please contact our [Enterprise support team][3] -instead. +If you have a question related to your specific GitHub Enterprise Server setup, would like assistance with +backup site setup or recovery, or would like to report a bug or a feature request, please contact our [Enterprise support team][3]. [1]: https://github.com/enterprise [2]: docs/requirements.md#github-enterprise-version-requirements From a0d9dc18d3072051aac10636eee7d183af8a167d Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 12:19:02 -0500 Subject: [PATCH 351/410] Added details about the repository settings changes we made on 2023-11-30 I copied and pasted the contents from this issue (https://github.com/github/backup-utils/issues/1140) into the README file. --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/README.md b/README.md index 8e1d1adb2..8376f371c 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,63 @@ GitHub Enterprise Server. If you have a question related to your specific GitHub Enterprise Server setup, would like assistance with backup site setup or recovery, or would like to report a bug or a feature request, please contact our [Enterprise support team][3]. + +## Repository updates - November 2023 + +In October 2023 we announced a number of changes to this repository. These changes will improve our (GitHub’s) ability to ship enhancements and new features to backup-utils, as well as simplify how GitHub Enterprise Server customers interact with backup-utils. + +Our process for shipping new versions of backup-utils prior to November 2023 involved a 2-way sync between this repository and an internal repository. This 2-way sync became significantly more problematic once we started regularly shipping patches in alignment with GitHub Enterprise Server. + +As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. + +This change has not affected the functionality of the backup-utils tool or a customer’s ability to backup or restore their GitHub Enterprise Server instance. + +### Details + +There are three specific areas that have been affected by us stop the 2-way sync between our internal repository and this public repository on 2023-11-30: + +1. **Pull requests**: Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. This is necessary because we will no longer be syncing changes between this repository and our internal repository. +2. **Issues**: Customers cannot open issues in this repository. Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. This will ensure all customer requests are handled consistently. +3. **Installing/upgrading backup-utils**: Customers will not be able to use a clone of the repository to install and upgrade backup-utils. Customers will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases) (either as a Debian package or as an archive file - see below for details on how to incorporate this change). + +### Timeline + +Below is the two phase timeline we will follow to roll out the changes described above: + +* **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), and updated the repository settings so that new issues cannot be opened. Also, we have stop syncing code from our internal repository to this repository. + * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). +* **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. After this date, you will no longer be able to clone a working copy of backup-utils from the repository. Instead, you will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases). + +### Updating your backup-utils upgrade process + +#### Clone of repository + +If your current process for upgrading backup-utils involves a clone of the repository, you will need to modify your process to download a new version of backup-utils and set it up. + +For example, you could download the v3.10.0 (github-backup-utils-v3.10.0.tar.gz) artifact from the [releases page](https://github.com/github/backup-utils/releases/tag/v3.10.0) with: + +``` +$ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz +``` +And then extract it: + +``` +$ tar xzvf github-backup-utils-v3.10.0.tar.gz +``` + +This will give you a new folder, `github-backup-utils-v3.10.0`, which contains the code for version 3.10.0 of backup-utils. Once you copy over your backup.config file from a previous installation of backup-utils your new version of backup-utils will be ready to use. + +#### Docker + +For customers that currently use Docker to create a backup-utils image, their existing process may need updating as a result of this change. Previously customers could execute this command to build a Docker image of backup-utils: + +``` +$ docker build github.com/github/backup-utils +``` + +This will not work after phase 2 roles out. You will need to update your process to first download an archive from the [release page](https://github.com/github/backup-utils/releases), extract it, and then build the Dockerfile inside the extracted directory. + + [1]: https://github.com/enterprise [2]: docs/requirements.md#github-enterprise-version-requirements [3]: https://support.github.com/ From d68e166e580caf64ae3ab84ba4f8dd039547ffa1 Mon Sep 17 00:00:00 2001 From: "Roger D. Winans" Date: Thu, 30 Nov 2023 13:12:19 -0500 Subject: [PATCH 352/410] Delete docs/SUPPORT.md --- docs/SUPPORT.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docs/SUPPORT.md diff --git a/docs/SUPPORT.md b/docs/SUPPORT.md deleted file mode 100644 index a05a91f5b..000000000 --- a/docs/SUPPORT.md +++ /dev/null @@ -1,13 +0,0 @@ -# Support - -## How to file issues and get help - -**github/backup-utils** uses GitHub issues to track bugs and feature requests. Please search the existing issues before filing new issues to avoid duplicates. - -For non-urgent help, feature requests, and questions about using this project, please [open an issue](https://github.com/github/backup-utils/issues/new/choose). - -For urgent help with **backup-utils**, for example when restoring a backup to recover from a site failure or other disaster, please [open a ticket in GitHub's support portal](https://support.github.com/contact) and work with GitHub Support directly. - -## GitHub Support Policy - -**github/backup-utils** is an MIT-licensed open source project under active development and maintained by GitHub staff. We will do our best to respond to support, feature requests, and community questions in a timely manner. From 687b17889378ea16edb1372eca367060edd71031 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 14:42:41 -0500 Subject: [PATCH 353/410] Fixing linting errors --- README.md | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8376f371c..a36add80a 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,16 @@ backup site setup or recovery, or would like to report a bug or a feature reques ## Repository updates - November 2023 -In October 2023 we announced a number of changes to this repository. These changes will improve our (GitHub’s) ability to ship enhancements and new features to backup-utils, as well as simplify how GitHub Enterprise Server customers interact with backup-utils. +In October 2023 we announced a number of changes to this repository. +These changes will improve our (GitHub’s) ability to ship enhancements and new features to backup-utils, +as well as simplify how GitHub Enterprise Server customers interact with backup-utils. -Our process for shipping new versions of backup-utils prior to November 2023 involved a 2-way sync between this repository and an internal repository. This 2-way sync became significantly more problematic once we started regularly shipping patches in alignment with GitHub Enterprise Server. +Our process for shipping new versions of backup-utils prior to November 2023 involved a 2-way sync between this repository and an internal repository. +This 2-way sync became significantly more problematic once we started regularly shipping patches in alignment with GitHub Enterprise Server. -As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. +As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. +With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. +You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. This change has not affected the functionality of the backup-utils tool or a customer’s ability to backup or restore their GitHub Enterprise Server instance. @@ -62,17 +67,27 @@ This change has not affected the functionality of the backup-utils tool or a cus There are three specific areas that have been affected by us stop the 2-way sync between our internal repository and this public repository on 2023-11-30: -1. **Pull requests**: Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. This is necessary because we will no longer be syncing changes between this repository and our internal repository. -2. **Issues**: Customers cannot open issues in this repository. Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. This will ensure all customer requests are handled consistently. -3. **Installing/upgrading backup-utils**: Customers will not be able to use a clone of the repository to install and upgrade backup-utils. Customers will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases) (either as a Debian package or as an archive file - see below for details on how to incorporate this change). +1. **Pull requests**: Customers should no longer open pull requests in this repository. +These pull requests will not be reviewed or merged. +This is necessary because we will no longer be syncing changes between this repository and our internal repository. +2. **Issues**: Customers cannot open issues in this repository. +Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. +This will ensure all customer requests are handled consistently. +3. **Installing/upgrading backup-utils**: Customers will not be able to use a clone of the repository to install and upgrade backup-utils. +Customers will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases) +(either as a Debian package or as an archive file - see below for details on how to incorporate this change). ### Timeline Below is the two phase timeline we will follow to roll out the changes described above: -* **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), and updated the repository settings so that new issues cannot be opened. Also, we have stop syncing code from our internal repository to this repository. - * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). -* **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. After this date, you will no longer be able to clone a working copy of backup-utils from the repository. Instead, you will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases). +* **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), +and updated the repository settings so that new issues cannot be opened. Also, we have stop syncing code from our internal repository to this repository. + * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. + But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). +* **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. +After this date, you will no longer be able to clone a working copy of backup-utils from the repository. +Instead, you will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases). ### Updating your backup-utils upgrade process @@ -82,12 +97,12 @@ If your current process for upgrading backup-utils involves a clone of the repos For example, you could download the v3.10.0 (github-backup-utils-v3.10.0.tar.gz) artifact from the [releases page](https://github.com/github/backup-utils/releases/tag/v3.10.0) with: -``` +```shell $ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz ``` And then extract it: -``` +```shell $ tar xzvf github-backup-utils-v3.10.0.tar.gz ``` @@ -97,7 +112,7 @@ This will give you a new folder, `github-backup-utils-v3.10.0`, which contains t For customers that currently use Docker to create a backup-utils image, their existing process may need updating as a result of this change. Previously customers could execute this command to build a Docker image of backup-utils: -``` +```shell $ docker build github.com/github/backup-utils ``` From 2525298bd4ac3ce9ede78eef54bd2702c7a71f14 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 14:50:57 -0500 Subject: [PATCH 354/410] Fix linting errors with dollar sign --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a36add80a..a186c5b3f 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Below is the two phase timeline we will follow to roll out the changes described * **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), and updated the repository settings so that new issues cannot be opened. Also, we have stop syncing code from our internal repository to this repository. - * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. + * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). * **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. After this date, you will no longer be able to clone a working copy of backup-utils from the repository. @@ -98,12 +98,12 @@ If your current process for upgrading backup-utils involves a clone of the repos For example, you could download the v3.10.0 (github-backup-utils-v3.10.0.tar.gz) artifact from the [releases page](https://github.com/github/backup-utils/releases/tag/v3.10.0) with: ```shell -$ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz +\$ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz ``` And then extract it: ```shell -$ tar xzvf github-backup-utils-v3.10.0.tar.gz +\$ tar xzvf github-backup-utils-v3.10.0.tar.gz ``` This will give you a new folder, `github-backup-utils-v3.10.0`, which contains the code for version 3.10.0 of backup-utils. Once you copy over your backup.config file from a previous installation of backup-utils your new version of backup-utils will be ready to use. @@ -113,7 +113,7 @@ This will give you a new folder, `github-backup-utils-v3.10.0`, which contains t For customers that currently use Docker to create a backup-utils image, their existing process may need updating as a result of this change. Previously customers could execute this command to build a Docker image of backup-utils: ```shell -$ docker build github.com/github/backup-utils +\$ docker build github.com/github/backup-utils ``` This will not work after phase 2 roles out. You will need to update your process to first download an archive from the [release page](https://github.com/github/backup-utils/releases), extract it, and then build the Dockerfile inside the extracted directory. From c06ec959046894bdff1ad7a17470f235b461caec Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 20:12:57 -0500 Subject: [PATCH 355/410] Fixed a typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f9f7838e6..c0ceedb0e 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ Customers will need to download a specific version of backup-utils from the [rel Below is the two phase timeline we will follow to roll out the changes described above: * **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), -and updated the repository settings so that new issues cannot be opened. Also, we have stop syncing code from our internal repository to this repository. +and updated the repository settings so that new issues cannot be opened. Also, we have stopped syncing code from our internal repository to this repository. * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). * **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. From 2e668ce05f65f020127325f0348537713190eba9 Mon Sep 17 00:00:00 2001 From: David Jarzebowski Date: Thu, 30 Nov 2023 20:17:28 -0500 Subject: [PATCH 356/410] Updating requirement about rsync's '--trust-sender' flag --- docs/requirements.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.md b/docs/requirements.md index 58dfe16ae..27e490fc3 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -22,6 +22,7 @@ CPU and memory requirements are dependent on the size of the GitHub Enterprise S The [fix in rsync `3.2.5`](https://github.com/WayneD/rsync/blob/master/NEWS.md#news-for-rsync-325-14-aug-2022) for [CVE-2022-29154](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29154) can cause severe performance degradation to `backup-utils`. If you encounter this degradation you can mitigate it by using the `--trust-sender` flag, which is available in rsync >= v3.2.5. +**Note**: If you are using backup-utils 3.9 or greater, `--trust-sender` is automatically used if your rsync version supports it and no further changes are needed. If your backup host is running rsync < v3.2.5 you may or may not need to make changes to your rsync package, depending on whether your rsync package has backported the fix for CVE-2022-29154 without also backporting the `--trust-sender` flag. From d0a6f5aa9fedce484d9ee01c38abbea2a053ce2c Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Tue, 5 Dec 2023 11:12:26 -0700 Subject: [PATCH 357/410] auto close PR from fork repo to prevent custoemr submitting PRs --- .github/workflows/close-pulls.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/close-pulls.yml diff --git a/.github/workflows/close-pulls.yml b/.github/workflows/close-pulls.yml new file mode 100644 index 000000000..ab3126b25 --- /dev/null +++ b/.github/workflows/close-pulls.yml @@ -0,0 +1,21 @@ +--- +name: Auto Close Pull Requests + +on: + schedule: + - cron: '0 * * * *' + +jobs: + close-fork-pulls: + runs-on: ubuntu-latest + + steps: + - name: Close Pull Requests + uses: peter-evans/close-fork-pulls@v2 + with: + comment: | + As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. + + Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. We will automatically close all PRs opened in this repository. + + Customers cannot open issues in this repository. Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. This will ensure all customer requests are handled consistently. From 2b993ac3bd90430513cbf050178546dd012526e3 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Tue, 5 Dec 2023 11:45:34 -0700 Subject: [PATCH 358/410] Update with comments --- .github/workflows/close-pulls.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/close-pulls.yml b/.github/workflows/close-pulls.yml index ab3126b25..24f0391ba 100644 --- a/.github/workflows/close-pulls.yml +++ b/.github/workflows/close-pulls.yml @@ -14,7 +14,7 @@ jobs: uses: peter-evans/close-fork-pulls@v2 with: comment: | - As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. + As of 2023-11-30 we have stopped the 2-way sync between this repository and our internal repository, so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. We will automatically close all PRs opened in this repository. From e4f3ae78261da0b1ec9ce5ebd437db14637b5848 Mon Sep 17 00:00:00 2001 From: "release-controller[bot]" Date: Thu, 15 Feb 2024 22:34:51 +0000 Subject: [PATCH 359/410] 3.12.0 release --- docs/getting-started.md | 2 +- docs/requirements.md | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 02f2cbe9b..ac85e5073 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,6 +1,6 @@ # Getting started - 1. [Download the latest release version][1] and extract the repository using `tar`: + 1. [Download the latest version of backup-utils][1] and extract the repository using `tar`: `tar -xzvf /path/to/github-backup-utils-vMAJOR.MINOR.PATCH.tar.gz` diff --git a/docs/requirements.md b/docs/requirements.md index 27e490fc3..6fa54e567 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -7,12 +7,10 @@ storage and must have network connectivity with the GitHub Enterprise Server app Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.7 or newer. See below for an update on rsync. -Ubuntu is the operating system we use to test `backup-utils` and it’s what we recommend you use too. You are welcome to use a different operating system, and we'll do our best to help you if you run into issues. But we can't guarantee that we'll be able to resolve issues that are specific to that operating system. - -Additionally, we encourage the use of [Docker](docker.md), as it ensures compatible versions of the aforementioned software are available to backup-utils. - The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. +We encourage the use of [Docker](docker.md), as it ensures compatible versions of the aforementioned software are available to backup-utils. + The backup host must be able to establish outbound network connections to the GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise Server. CPU and memory requirements are dependent on the size of the GitHub Enterprise Server appliance. We recommend a minimum of 4 cores and 8GB of RAM for the host running [GitHub Enterprise Backup Utilities](https://github.com/github/backup-utils). We recommend monitoring the backup host's CPU and memory usage to ensure it is sufficient for your environment. @@ -67,23 +65,23 @@ Please avoid using an NFS mount for the data directory (where backup data is sto Starting with Backup Utilities v2.13.0, version support is inline with that of the [GitHub Enterprise Server upgrade requirements][8] and as such, support is limited to three versions of GitHub Enterprise Server: the version that corresponds with the version -of Backup Utilities, and the two releases prior to it. +of Backup Utilities, and the two versions prior to it. For example, Backup Utilities v2.13.0 can be used to backup and restore all patch -releases from 2.11.0 to the latest patch release of GitHub Enterprise Server 2.13. +versions from 2.11.0 to the latest patch version of GitHub Enterprise Server 2.13. Backup Utilities v2.14.0 will be released when GitHub Enterprise Server 2.14.0 is released -and will then be used to backup all releases of GitHub Enterprise Server from 2.12.0 -to the latest patch release of GitHub Enterprise Server 2.14. +and will then be used to backup all versions of GitHub Enterprise Server from 2.12.0 +to the latest patch version of GitHub Enterprise Server 2.14. Backup Utilities v2.11.4 and earlier offer support for GitHub Enterprise Server 2.10 -and earlier releases up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier +and earlier versions up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier offer support for GitHub Enterprise Server 2.1.0 and earlier. -**Note**: You can restore a snapshot that's at most two feature releases behind +**Note**: You can restore a snapshot that's at most two feature versions behind the restore target's version of GitHub Enterprise Server. For example, to restore a snapshot of GitHub Enterprise Server 2.11, the target GitHub Enterprise Server appliance must be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapshot from -2.10 to 2.13, because that's three releases ahead. +2.10 to 2.13, because that's three versions ahead. **Note**: You _cannot_ restore a backup created from a newer version of GitHub Enterprise Server to an older version. For example, an attempt to restore a snapshot of GitHub Enterprise Server 2.21 to a GitHub Enterprise Server 2.20 environment will fail with an error of `Error: Snapshot can not be restored to an older release of GitHub Enterprise Server.`. From 9a8c195637871a17b0f670d7c8f6e28482714fa2 Mon Sep 17 00:00:00 2001 From: Hao Jiang Date: Tue, 19 Mar 2024 17:47:46 -0600 Subject: [PATCH 360/410] Remove code from backup-utils --- CONTRIBUTING.md | 12 - Dockerfile | 76 -- Dockerfile.alpine | 20 - LICENSE | 20 - Makefile | 27 - STYLEGUIDE.md | 250 ---- backup.config-example | 123 -- bin/ghe-backup | 406 ------ bin/ghe-backup-progress | 56 - bin/ghe-host-check | 265 ---- bin/ghe-restore | 727 ----------- debian/changelog | 666 ---------- debian/clean | 1 - debian/compat | 1 - debian/control | 32 - debian/copyright | 33 - debian/install | 2 - debian/manpages | 1 - debian/rules | 18 - debian/source/format | 1 - ownership.yaml | 29 - release-notes/3.11.0.md | 18 - script/package-deb | 42 - script/package-tarball | 42 - script/release | 506 -------- script/test | 28 - share/github-backup-utils/bm.sh | 63 - share/github-backup-utils/ghe-backup-actions | 51 - share/github-backup-utils/ghe-backup-config | 708 ----------- .../ghe-backup-es-audit-log | 58 - share/github-backup-utils/ghe-backup-es-rsync | 86 -- share/github-backup-utils/ghe-backup-fsck | 91 -- .../github-backup-utils/ghe-backup-git-hooks | 114 -- share/github-backup-utils/ghe-backup-minio | 55 - share/github-backup-utils/ghe-backup-mssql | 363 ------ share/github-backup-utils/ghe-backup-mysql | 48 - .../ghe-backup-mysql-binary | 60 - .../ghe-backup-mysql-logical | 27 - share/github-backup-utils/ghe-backup-pages | 89 -- share/github-backup-utils/ghe-backup-redis | 55 - .../ghe-backup-repositories | 407 ------ share/github-backup-utils/ghe-backup-secrets | 186 --- share/github-backup-utils/ghe-backup-settings | 50 - share/github-backup-utils/ghe-backup-storage | 179 --- .../ghe-backup-store-version | 23 - share/github-backup-utils/ghe-backup-strategy | 20 - share/github-backup-utils/ghe-backup-userdata | 64 - .../ghe-cluster-find-nodes | 41 - .../ghe-detect-leaked-ssh-keys | 140 --- share/github-backup-utils/ghe-docker-init | 12 - share/github-backup-utils/ghe-gc-disable | 42 - share/github-backup-utils/ghe-gc-enable | 35 - .../ghe-incremental-backup-restore | 380 ------ .../ghe-maintenance-mode-status | 30 - share/github-backup-utils/ghe-prune-snapshots | 130 -- share/github-backup-utils/ghe-restore-actions | 68 - .../ghe-restore-es-audit-log | 76 -- .../github-backup-utils/ghe-restore-es-rsync | 49 - ...tore-external-database-compatibility-check | 44 - .../github-backup-utils/ghe-restore-git-hooks | 99 -- share/github-backup-utils/ghe-restore-minio | 57 - share/github-backup-utils/ghe-restore-mssql | 91 -- share/github-backup-utils/ghe-restore-mysql | 82 -- .../ghe-restore-mysql-binary | 125 -- .../ghe-restore-mysql-legacy | 55 - .../ghe-restore-mysql-logical | 52 - share/github-backup-utils/ghe-restore-pages | 166 --- share/github-backup-utils/ghe-restore-redis | 26 - .../ghe-restore-repositories | 248 ---- .../ghe-restore-repositories-gist | 175 --- share/github-backup-utils/ghe-restore-secrets | 146 --- .../github-backup-utils/ghe-restore-settings | 52 - .../ghe-restore-snapshot-path | 32 - .../github-backup-utils/ghe-restore-ssh-keys | 28 - share/github-backup-utils/ghe-restore-storage | 181 --- share/github-backup-utils/ghe-rsync | 72 -- .../ghe-rsync-feature-checker | 47 - share/github-backup-utils/ghe-rsync-size | 106 -- share/github-backup-utils/ghe-ssh | 86 -- share/github-backup-utils/ghe-ssh-config | 86 -- .../ghe-ssh-leaked-host-keys-list.txt | 531 -------- share/github-backup-utils/requirements.txt | 3 - share/github-backup-utils/track-progress | 18 - share/github-backup-utils/version | 1 - test/STYLEGUIDE.md | 36 - test/backup.config | 11 - test/bin/chown | 1 - test/bin/cron | 1 - test/bin/curl | 22 - test/bin/dgit-cluster-backup-routes | 31 - test/bin/dgit-cluster-restore-finalize | 1 - test/bin/dgit-cluster-restore-routes | 9 - test/bin/dpages-cluster-restore-finalize | 1 - test/bin/dpages-cluster-restore-routes | 9 - test/bin/enterprise-configure | 6 - test/bin/ghe-actions-console | 1 - test/bin/ghe-actions-start | 1 - test/bin/ghe-actions-stop | 1 - test/bin/ghe-cluster-config-apply | 6 - test/bin/ghe-cluster-config-update | 6 - test/bin/ghe-cluster-each | 35 - test/bin/ghe-cluster-host-check | 5 - test/bin/ghe-cluster-nodes | 41 - test/bin/ghe-config | 18 - test/bin/ghe-config-apply | 6 - test/bin/ghe-dpages | 1 - test/bin/ghe-es-load-json | 5 - test/bin/ghe-es-remove-1x-indices | 1 - test/bin/ghe-es-snapshot | 6 - test/bin/ghe-export-authorized-keys | 1 - test/bin/ghe-export-mssql | 1 - test/bin/ghe-export-mysql | 1 - test/bin/ghe-export-redis | 1 - test/bin/ghe-export-settings | 1 - test/bin/ghe-export-ssh-host-keys | 18 - test/bin/ghe-export-ssl-ca-certificates | 1 - test/bin/ghe-fake-export-command | 6 - test/bin/ghe-fake-finalize-command | 16 - test/bin/ghe-fake-import-command | 6 - test/bin/ghe-fake-true | 4 - test/bin/ghe-hook-env-update | 11 - test/bin/ghe-import-authorized-keys | 1 - test/bin/ghe-import-license | 1 - test/bin/ghe-import-mssql | 1 - test/bin/ghe-import-mysql | 1 - test/bin/ghe-import-mysql-mysqldump | 1 - test/bin/ghe-import-mysql-xtrabackup | 1 - test/bin/ghe-import-redis | 1 - test/bin/ghe-import-settings | 1 - test/bin/ghe-import-ssh-host-keys | 1 - test/bin/ghe-import-ssl-ca-certificates | 1 - test/bin/ghe-maintenance | 1 - test/bin/ghe-mssql-console | 21 - test/bin/ghe-negotiate-version | 7 - test/bin/ghe-nomad-cleanup | 5 - test/bin/ghe-nomad-jobs | 7 - test/bin/ghe-redis-cli | 37 - test/bin/ghe-reset-gh-connect | 1 - test/bin/ghe-service-ensure-elasticsearch | 6 - test/bin/ghe-service-ensure-mysql | 6 - test/bin/ghe-service-wait-mysql | 6 - test/bin/ghe-spokes | 16 - test/bin/ghe-storage | 1 - test/bin/gist-cluster-restore-finalize | 1 - test/bin/gist-cluster-restore-routes | 9 - test/bin/github-env | 1 - test/bin/ionice-stub | 3 - test/bin/nomad | 6 - test/bin/python | 28 - test/bin/service | 1 - test/bin/ssh | 51 - test/bin/storage-cluster-backup-routes | 10 - test/bin/storage-cluster-restore-finalize | 1 - test/bin/storage-cluster-restore-routes | 10 - test/bin/systemctl | 1 - test/cluster.conf | 65 - test/test-docker-build.sh | 63 - test/test-ghe-backup-config.sh | 227 ---- test/test-ghe-backup-parallel.sh | 12 - test/test-ghe-backup.sh | 1093 ----------------- test/test-ghe-cluster-find-nodes.sh | 41 - test/test-ghe-detect-leaked-ssh-keys.sh | 61 - test/test-ghe-host-check.sh | 124 -- test/test-ghe-incremental-restore.sh | 82 -- test/test-ghe-prune-snapshots.sh | 111 -- test/test-ghe-restore-external-database.sh | 194 --- test/test-ghe-restore-parallel.sh | 24 - test/test-ghe-restore.sh | 1022 --------------- test/test-ghe-rsync-feature-checker.sh | 156 --- test/test-ghe-ssh-config.sh | 103 -- test/test-ghe-ssh.sh | 53 - test/test-shellcheck.sh | 66 - test/testlib.sh | 730 ----------- 173 files changed, 14612 deletions(-) delete mode 100644 CONTRIBUTING.md delete mode 100644 Dockerfile delete mode 100644 Dockerfile.alpine delete mode 100644 LICENSE delete mode 100644 Makefile delete mode 100644 STYLEGUIDE.md delete mode 100644 backup.config-example delete mode 100755 bin/ghe-backup delete mode 100755 bin/ghe-backup-progress delete mode 100755 bin/ghe-host-check delete mode 100755 bin/ghe-restore delete mode 100644 debian/changelog delete mode 100644 debian/clean delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/copyright delete mode 100644 debian/install delete mode 100644 debian/manpages delete mode 100755 debian/rules delete mode 100644 debian/source/format delete mode 100644 ownership.yaml delete mode 100644 release-notes/3.11.0.md delete mode 100755 script/package-deb delete mode 100755 script/package-tarball delete mode 100755 script/release delete mode 100755 script/test delete mode 100755 share/github-backup-utils/bm.sh delete mode 100755 share/github-backup-utils/ghe-backup-actions delete mode 100755 share/github-backup-utils/ghe-backup-config delete mode 100755 share/github-backup-utils/ghe-backup-es-audit-log delete mode 100755 share/github-backup-utils/ghe-backup-es-rsync delete mode 100755 share/github-backup-utils/ghe-backup-fsck delete mode 100755 share/github-backup-utils/ghe-backup-git-hooks delete mode 100755 share/github-backup-utils/ghe-backup-minio delete mode 100755 share/github-backup-utils/ghe-backup-mssql delete mode 100755 share/github-backup-utils/ghe-backup-mysql delete mode 100755 share/github-backup-utils/ghe-backup-mysql-binary delete mode 100755 share/github-backup-utils/ghe-backup-mysql-logical delete mode 100755 share/github-backup-utils/ghe-backup-pages delete mode 100755 share/github-backup-utils/ghe-backup-redis delete mode 100755 share/github-backup-utils/ghe-backup-repositories delete mode 100755 share/github-backup-utils/ghe-backup-secrets delete mode 100755 share/github-backup-utils/ghe-backup-settings delete mode 100755 share/github-backup-utils/ghe-backup-storage delete mode 100755 share/github-backup-utils/ghe-backup-store-version delete mode 100755 share/github-backup-utils/ghe-backup-strategy delete mode 100755 share/github-backup-utils/ghe-backup-userdata delete mode 100755 share/github-backup-utils/ghe-cluster-find-nodes delete mode 100755 share/github-backup-utils/ghe-detect-leaked-ssh-keys delete mode 100755 share/github-backup-utils/ghe-docker-init delete mode 100755 share/github-backup-utils/ghe-gc-disable delete mode 100755 share/github-backup-utils/ghe-gc-enable delete mode 100644 share/github-backup-utils/ghe-incremental-backup-restore delete mode 100755 share/github-backup-utils/ghe-maintenance-mode-status delete mode 100755 share/github-backup-utils/ghe-prune-snapshots delete mode 100755 share/github-backup-utils/ghe-restore-actions delete mode 100755 share/github-backup-utils/ghe-restore-es-audit-log delete mode 100755 share/github-backup-utils/ghe-restore-es-rsync delete mode 100755 share/github-backup-utils/ghe-restore-external-database-compatibility-check delete mode 100755 share/github-backup-utils/ghe-restore-git-hooks delete mode 100755 share/github-backup-utils/ghe-restore-minio delete mode 100755 share/github-backup-utils/ghe-restore-mssql delete mode 100755 share/github-backup-utils/ghe-restore-mysql delete mode 100755 share/github-backup-utils/ghe-restore-mysql-binary delete mode 100755 share/github-backup-utils/ghe-restore-mysql-legacy delete mode 100755 share/github-backup-utils/ghe-restore-mysql-logical delete mode 100755 share/github-backup-utils/ghe-restore-pages delete mode 100755 share/github-backup-utils/ghe-restore-redis delete mode 100755 share/github-backup-utils/ghe-restore-repositories delete mode 100755 share/github-backup-utils/ghe-restore-repositories-gist delete mode 100755 share/github-backup-utils/ghe-restore-secrets delete mode 100755 share/github-backup-utils/ghe-restore-settings delete mode 100755 share/github-backup-utils/ghe-restore-snapshot-path delete mode 100755 share/github-backup-utils/ghe-restore-ssh-keys delete mode 100755 share/github-backup-utils/ghe-restore-storage delete mode 100755 share/github-backup-utils/ghe-rsync delete mode 100755 share/github-backup-utils/ghe-rsync-feature-checker delete mode 100644 share/github-backup-utils/ghe-rsync-size delete mode 100755 share/github-backup-utils/ghe-ssh delete mode 100755 share/github-backup-utils/ghe-ssh-config delete mode 100644 share/github-backup-utils/ghe-ssh-leaked-host-keys-list.txt delete mode 100644 share/github-backup-utils/requirements.txt delete mode 100755 share/github-backup-utils/track-progress delete mode 100644 share/github-backup-utils/version delete mode 100644 test/STYLEGUIDE.md delete mode 100644 test/backup.config delete mode 120000 test/bin/chown delete mode 120000 test/bin/cron delete mode 100755 test/bin/curl delete mode 100755 test/bin/dgit-cluster-backup-routes delete mode 120000 test/bin/dgit-cluster-restore-finalize delete mode 100755 test/bin/dgit-cluster-restore-routes delete mode 120000 test/bin/dpages-cluster-restore-finalize delete mode 100755 test/bin/dpages-cluster-restore-routes delete mode 100755 test/bin/enterprise-configure delete mode 100755 test/bin/ghe-actions-console delete mode 120000 test/bin/ghe-actions-start delete mode 120000 test/bin/ghe-actions-stop delete mode 100755 test/bin/ghe-cluster-config-apply delete mode 100755 test/bin/ghe-cluster-config-update delete mode 100755 test/bin/ghe-cluster-each delete mode 100755 test/bin/ghe-cluster-host-check delete mode 100755 test/bin/ghe-cluster-nodes delete mode 100755 test/bin/ghe-config delete mode 100755 test/bin/ghe-config-apply delete mode 120000 test/bin/ghe-dpages delete mode 100755 test/bin/ghe-es-load-json delete mode 120000 test/bin/ghe-es-remove-1x-indices delete mode 100755 test/bin/ghe-es-snapshot delete mode 120000 test/bin/ghe-export-authorized-keys delete mode 120000 test/bin/ghe-export-mssql delete mode 120000 test/bin/ghe-export-mysql delete mode 120000 test/bin/ghe-export-redis delete mode 120000 test/bin/ghe-export-settings delete mode 100755 test/bin/ghe-export-ssh-host-keys delete mode 120000 test/bin/ghe-export-ssl-ca-certificates delete mode 100755 test/bin/ghe-fake-export-command delete mode 100755 test/bin/ghe-fake-finalize-command delete mode 100755 test/bin/ghe-fake-import-command delete mode 100755 test/bin/ghe-fake-true delete mode 100755 test/bin/ghe-hook-env-update delete mode 120000 test/bin/ghe-import-authorized-keys delete mode 120000 test/bin/ghe-import-license delete mode 120000 test/bin/ghe-import-mssql delete mode 120000 test/bin/ghe-import-mysql delete mode 120000 test/bin/ghe-import-mysql-mysqldump delete mode 120000 test/bin/ghe-import-mysql-xtrabackup delete mode 120000 test/bin/ghe-import-redis delete mode 120000 test/bin/ghe-import-settings delete mode 120000 test/bin/ghe-import-ssh-host-keys delete mode 120000 test/bin/ghe-import-ssl-ca-certificates delete mode 120000 test/bin/ghe-maintenance delete mode 100755 test/bin/ghe-mssql-console delete mode 100755 test/bin/ghe-negotiate-version delete mode 100755 test/bin/ghe-nomad-cleanup delete mode 100755 test/bin/ghe-nomad-jobs delete mode 100755 test/bin/ghe-redis-cli delete mode 120000 test/bin/ghe-reset-gh-connect delete mode 100755 test/bin/ghe-service-ensure-elasticsearch delete mode 100755 test/bin/ghe-service-ensure-mysql delete mode 100755 test/bin/ghe-service-wait-mysql delete mode 100755 test/bin/ghe-spokes delete mode 120000 test/bin/ghe-storage delete mode 120000 test/bin/gist-cluster-restore-finalize delete mode 100755 test/bin/gist-cluster-restore-routes delete mode 120000 test/bin/github-env delete mode 100755 test/bin/ionice-stub delete mode 100755 test/bin/nomad delete mode 100755 test/bin/python delete mode 120000 test/bin/service delete mode 100755 test/bin/ssh delete mode 100755 test/bin/storage-cluster-backup-routes delete mode 120000 test/bin/storage-cluster-restore-finalize delete mode 100755 test/bin/storage-cluster-restore-routes delete mode 120000 test/bin/systemctl delete mode 100644 test/cluster.conf delete mode 100755 test/test-docker-build.sh delete mode 100755 test/test-ghe-backup-config.sh delete mode 100755 test/test-ghe-backup-parallel.sh delete mode 100755 test/test-ghe-backup.sh delete mode 100755 test/test-ghe-cluster-find-nodes.sh delete mode 100755 test/test-ghe-detect-leaked-ssh-keys.sh delete mode 100755 test/test-ghe-host-check.sh delete mode 100755 test/test-ghe-incremental-restore.sh delete mode 100755 test/test-ghe-prune-snapshots.sh delete mode 100755 test/test-ghe-restore-external-database.sh delete mode 100755 test/test-ghe-restore-parallel.sh delete mode 100755 test/test-ghe-restore.sh delete mode 100755 test/test-ghe-rsync-feature-checker.sh delete mode 100755 test/test-ghe-ssh-config.sh delete mode 100755 test/test-ghe-ssh.sh delete mode 100755 test/test-shellcheck.sh delete mode 100755 test/testlib.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 042d7c71a..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,12 +0,0 @@ -# Contributing - -Looking to contribute something to this project? That is great, we always appreciate pull requests! Here's how you can help: - -1. Fork the project to your account. -2. Clone the fork (`git clone https://github.com/[username]/backup-utils.git`). -3. Create a new feature branch (`git checkout -b my-feature-branch`). -4. Add and then commit your changes (`git commit -am "Add a new backup endpoint."`). -5. Push your feature branch to GitHub.com (`git push -u origin my-feature-branch`). -6. Open a [Pull Request](https://github.com/github/backup-utils/compare/) and wait for our feedback. - -Have a look at the [styleguide](https://github.com/github/backup-utils/tree/master/STYLEGUIDE.md) to make sure your code style is consistent with the code in this repository. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 710cb9d52..000000000 --- a/Dockerfile +++ /dev/null @@ -1,76 +0,0 @@ -# Multi stage build for backup-utils -# Build layer is for compiling rsync from source -# Runtime layer is for running backup-utils -# https://docs.docker.com/develop/develop-images/multistage-build/ -# https://docs.docker.com/engine/userguide/eng-image/multistage-build/ - -# Build layer -FROM ubuntu:focal AS build - -# Install build dependencies -RUN apt-get update && apt-get install --no-install-recommends -y \ - gcc \ - g++ \ - gawk \ - autoconf \ - make \ - automake \ - python3-cmarkgfm \ - acl \ - libacl1-dev \ - attr \ - libattr1-dev \ - libxxhash-dev \ - libzstd-dev \ - liblz4-dev \ - libssl-dev \ - git \ - jq \ - bc \ - curl \ - tar \ - gzip \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Download rsync source from https://github.com/WayneD/rsync/archive/refs/tags/[TAG].tar.gz pinned to specified tag -ARG RSYNC_TAG=v3.2.7 -RUN curl https://github.com/WayneD/rsync/archive/refs/tags/${RSYNC_TAG}.tar.gz -L -o ${RSYNC_TAG}.tar.gz -RUN mkdir -p /rsync-${RSYNC_TAG}&& tar -xzf ${RSYNC_TAG}.tar.gz -C /rsync-${RSYNC_TAG} --strip-components=1 && ls -la -# Change to the working directory of the rsync source -WORKDIR /rsync-${RSYNC_TAG} -RUN ls -la && ./configure -RUN make -RUN make install - -# Reset working directory -WORKDIR / - -# Runtime layer -FROM ubuntu:focal AS runtime - -# Install runtime dependencies - bash, git, OpenSSH 5.6 or newer, and jq v1.5 or newer. -RUN apt-get update && apt-get install --no-install-recommends -y \ - bash \ - git \ - openssh-client \ - jq \ - bc \ - moreutils \ - gawk \ - ca-certificates \ - xxhash \ - && rm -rf /var/lib/apt/lists/* - -# Copy rsync from build layer -COPY --from=build /usr/local/bin/rsync /usr/local/bin/rsync - -# Copy backup-utils from repository into /backup-utils -COPY ./ /backup-utils/ - -WORKDIR /backup-utils - -RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init - -ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"] -CMD ["ghe-host-check"] diff --git a/Dockerfile.alpine b/Dockerfile.alpine deleted file mode 100644 index 2c21bf2cd..000000000 --- a/Dockerfile.alpine +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:latest - -RUN apk --update --no-cache add \ - tar \ - rsync \ - ca-certificates \ - openssh \ - git \ - bash \ - gawk \ - procps \ - coreutils - -COPY ./ /backup-utils/ -WORKDIR /backup-utils - -RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init - -ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"] -CMD ["ghe-host-check"] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 4d231b456..000000000 --- a/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2014 GitHub Inc. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index c8eaa0ff1..000000000 --- a/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -SHELL = /bin/sh - -test: info - @echo Running tests - @script/test - -info: - @echo This is github/backup-utils - @echo shell is $(shell ls -l $(SHELL) | sed 's@.*/bin/sh@/bin/sh@') - @rsync --version | head -1 - @echo - -dist: - @script/package-tarball - -deb: - @script/package-deb - -clean: - rm -rf dist - -# List pull requests that need to be merged into stable -# (helpful for the release notes) -pending-prs: - @git log stable...master | grep "Merge pull request" - -.PHONY: test info dist clean pending-prs diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md deleted file mode 100644 index 9c1fc1e86..000000000 --- a/STYLEGUIDE.md +++ /dev/null @@ -1,250 +0,0 @@ -## Bash Style Guide - -If you've not done much Bash development before you may find these debugging tips useful: http://wiki.bash-hackers.org/scripting/debuggingtips. - ---- -##### Scripts must start with `#!/usr/bin/env bash` - ---- -##### Use `set -e` - -If the return value of a command can be ignored, suffix it with `|| true`: - -```bash -set -e -command_that_might_fail || true -command_that_should_not_fail -``` - -Note that ignoring an exit status with `|| true` is not a good practice though. Generally speaking, it's better to handle the error. - ---- -##### Avoid manually checking exit status with `$?` - -Rely on `set -e` instead: - -```bash -cmd -if [ $? -eq 0 ]; then - echo worked -fi -``` - -should be written as: - -```bash -set -e -if cmd; then - echo worked -fi -``` - ---- -##### Include a usage, description and optional examples - -Use this format: - -```bash -#!/usr/bin/env bash -#/ Usage: ghe-this-is-my-script [options] -#/ -#/ This is a brief description of the script's purpose. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ -l | --longopt An option. -#/ -c Another option. -#/ -#/ EXAMPLES: (optional section but nice to have when not trivial) -#/ -#/ This will do foo and bar: -#/ $ ghe-this-is-my-script --longopt foobar -c 2 -#/ -set -e -``` - -If there are no options or required arguments, the `OPTIONS` section can be ignored. - ---- -##### Customer-facing scripts must accept both -h and --help arguments - -They should also print the usage information and exit 2. - -For example: - -```bash -#!/usr/bin/env bash -#/ Usage: ghe-this-is-my-script [options] -#/ -#/ This is a brief description of the script's purpose. -set -e - -if [ "$1" = "--help" -o "$1" = "-h" ]; then - grep '^#/' <"$0" | cut -c 4- - exit 2 -fi - -``` - ---- -##### Avoid Bash arrays - -Main issues: - -* Portability -* Important bugs in Bash versions < 4.3 - ---- -##### Use `test` or `[` whenever possible - -```bash -test -f /etc/passwd -test -f /etc/passwd -a -f /etc/group -if [ "string" = "string" ]; then - true -fi -``` - ---- -##### Scripts may use `[[` for advanced bash features - -```bash -if [[ "$(hostname)" = *.iad.github.net ]]; then - true -fi -``` - ---- -##### Scripts may use Bash for loops - -Preferred: - -```bash -for i in $(seq 0 9); do -done -``` - -or: - -```bash -for ((n=0; n<10; n++)); do -done -``` - ---- -##### Use `$[x+y*z]` for mathematical expressions - -```bash -local n=1 -let n++ -n=$[n+1] # preferred -n=$[$n+1] -n=$((n+1)) -n=$(($n+1)) -``` - ---- -##### Use variables sparingly - -Short paths and other constants should be repeated liberally throughout code since they -can be search/replaced easily if they ever change. - -```bash -DATA_DB_PATH=/data/user/db -mkdir -p $DATA_DB_PATH -rsync $DATA_DB_PATH remote:$DATA_DB_PATH -``` - -versus the much more readable: - -```bash -mkdir -p /data/user/db -rsync /data/user/db remote:/data/user/db -``` - ---- -##### Use lowercase and uppercase variable names - -Use lowercase variables for locals and internal variables, and uppercase for variables inherited or exported via the environment - -```bash -#!/usr/bin/env bash -#/ Usage: [DEBUG=0] process_repo -nwo=$1 -[ -n $DEBUG ] && echo "** processing $nwo" >&2 - -export GIT_DIR=/data/repos/$nwo.git -git rev-list -``` - ---- -##### Use `${var}` for interpolation only when required - -```bash -greeting=hello -echo $greeting -echo ${greeting}world -``` - ---- -##### Use functions sparingly, opting for small/simple/sequential scripts instead whenever possible - -When defining functions, use the following style: - -```bash -my_function() { - local arg1=$1 - [ -n $arg1 ] || return - ... -} -``` - ---- -##### Use `< /etc/foo # interpolated before ssh - chmod 0600 /etc/foo -eof -``` - ---- -##### Quote variables that could reasonably have a space now or in the future - -```bash -if [ ! -z "$packages" ]; then - true -fi -``` - ---- -##### Use two space indentation - ---- -##### Scripts should not produce errors or warnings when checked with ShellCheck - -Use inline comments to disable specific tests, and explain why the test has been disabled. - -```bash -hexToAscii() { - # shellcheck disable=SC2059 # $1 needs to be interpreted as a formatted string - printf "\x$1" -} -``` - -### Testing - -See [the style guide](https://github.com/github/backup-utils/blob/master/test/STYLEGUIDE.md) diff --git a/backup.config-example b/backup.config-example deleted file mode 100644 index 535a2f636..000000000 --- a/backup.config-example +++ /dev/null @@ -1,123 +0,0 @@ -# GitHub Enterprise Server backup configuration file - -# The hostname of the GitHub Enterprise Server appliance to back up. The host -# must be reachable via SSH from the backup host. -GHE_HOSTNAME="github.example.com" - -# Path to where backup data is stored. By default this is the "data" -# directory next to this file but can be set to an absolute path -# elsewhere for backing up to a separate partition / mount point. -GHE_DATA_DIR="data" - -# The number of backup snapshots to retain. Old snapshots are pruned after each -# successful ghe-backup run. This option should be tuned based on the frequency -# of scheduled backup runs. If backups are scheduled hourly, snapshots will be -# available for the past N hours; if backups are scheduled daily, snapshots will -# be available for the past N days ... -GHE_NUM_SNAPSHOTS=10 - -# Pruning snapshots can be scheduled outside of the backup process. -# If set to 'yes', snapshots will not be pruned by ghe-backup. -# Instead, ghe-pruning-snapshots will need to be invoked separately via cron -#GHE_PRUNING_SCHEDULED=yes - -# If --incremental is used to generate incremental MySQL backups with ghe-backup, -# then you need to specify how many cycles of full and incremental backups will be -# performed before the next full backup is created. -# For example, if `GHE_INCREMENTAL_BACKUP_MAX` is set to 14, backup-utils will -# run 1 full backup and then 13 incremental backups before performing another full backup on the next cycle. -#GHE_INCREMENTAL_MAX_BACKUPS=14 - -# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check -# disk space validation and software version checks on the backup-host will be disabled. -#GHE_SKIP_CHECKS=false - -# The hostname of the GitHub appliance to restore. If you've set up a separate -# GitHub appliance to act as a standby for recovery, specify its IP or hostname -# here. The host to restore to may also be specified directly when running -# ghe-restore so use of this variable isn't strictly required. -# -#GHE_RESTORE_HOST="github-standby.example.com" - -# If set to 'yes', ghe-restore will omit the restore of audit logs. -# -#GHE_RESTORE_SKIP_AUDIT_LOGS=no - -# When verbose output is enabled with `-v`, it's written to stdout by default. If -# you'd prefer it to be written to a separate file, set this option. -# -#GHE_VERBOSE_LOG="/var/log/backup-verbose.log" - -# Any extra options passed to the SSH command. -# In a single instance environment, nothing is required by default. -# In a clustering environment, "-i abs-path-to-ssh-private-key" is required. -# -#GHE_EXTRA_SSH_OPTS="" - -# Any extra options passed to the rsync command. Nothing required by default. -# -#GHE_EXTRA_RSYNC_OPTS="" - -# If set to 'yes', rsync will be set to use compression during backups and restores transfers. Defaults to 'no'. -# -#GHE_RSYNC_COMPRESSION_ENABLED=yes - -# If enabled and set to 'no', rsync warning message during backups will be suppressed. -#RSYNC_WARNING=no - - -# If set to 'yes', logging output will be colorized. -# -#OUTPUT_COLOR=no - -# If set to 'no', GHE_DATA_DIR will not be created automatically -# and restore/backup will exit 8 -# -#GHE_CREATE_DATA_DIR=yes - -# If set to 'yes', git fsck will run on the repositories -# and print some additional info. -# -# WARNING: do not enable this, only useful for debugging/development -#GHE_BACKUP_FSCK=no - -# Cadence of MSSQL backups -# ,, all in minutes -# e.g. -# - Full backup every week (10080 minutes) -# - Differential backup every day (1440 minutes) -# - Transactionlog backup every 15 minutes -# -#GHE_MSSQL_BACKUP_CADENCE=10080,1440,15 - -# If set to 'yes', ghe-backup jobs will run in parallel. Defaults to 'no'. -# -#GHE_PARALLEL_ENABLED=yes - -# Sets the maximum number of jobs to run in parallel. Defaults to the number -# of available processing units on the machine. -# -#GHE_PARALLEL_MAX_JOBS=2 - -# Sets the maximum number of rsync jobs to run in parallel. Defaults to the -# configured GHE_PARALLEL_MAX_JOBS, or the number of available processing -# units on the machine. -# -# GHE_PARALLEL_RSYNC_MAX_JOBS=3 - -# When jobs are running in parallel wait as needed to avoid starting new jobs -# when the system's load average is not below the specified percentage. Defaults to -# unrestricted. -# -#GHE_PARALLEL_MAX_LOAD=50 - -# When running an external mysql database, run this script to trigger a MySQL backup -# rather than attempting to backup via backup-utils directly. -#EXTERNAL_DATABASE_BACKUP_SCRIPT="/bin/false" - -# When running an external mysql database, run this script to trigger a MySQL restore -# rather than attempting to backup via backup-utils directly. -#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false" - -# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes' -#GHE_BACKUP_PAGES=no diff --git a/bin/ghe-backup b/bin/ghe-backup deleted file mode 100755 index 448a5840e..000000000 --- a/bin/ghe-backup +++ /dev/null @@ -1,406 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup [-hv] [--version] -#/ -#/ Take snapshots of all GitHub Enterprise data, including Git repository data, -#/ the MySQL database, instance settings, GitHub Pages data, etc. -#/ -#/ OPTIONS: -#/ -v | --verbose Enable verbose output. -#/ -h | --help Show this message. -#/ --version Display version information. -#/ -i | --incremental Incremental backup -#/ --skip-checks Skip storage/sw version checks -#/ - -set -e - -# Parse arguments -while true; do - case "$1" in - -h|--help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -v|--verbose) - export GHE_VERBOSE=true - shift - ;; - -i|--incremental) - export GHE_INCREMENTAL=true - shift - ;; - --skip-checks) - export GHE_SKIP_CHECKS=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - break - ;; - esac -done - - -export CALLING_SCRIPT="ghe-backup" - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" - - -# Check to make sure moreutils parallel is installed and working properly -ghe_parallel_check - - -# Used to record failed backup steps -failures= -failures_file="$(mktemp -t backup-utils-backup-failures-XXXXXX)" - -# CPU and IO throttling to keep backups from thrashing around. -export GHE_NICE=${GHE_NICE:-"nice -n 19"} -export GHE_IONICE=${GHE_IONICE:-"ionice -c 3"} - -# Create the timestamped snapshot directory where files for this run will live, -# change into it, and mark the snapshot as incomplete by touching the -# 'incomplete' file. If the backup succeeds, this file will be removed -# signifying that the snapshot is complete. -mkdir -p "$GHE_SNAPSHOT_DIR" -cd "$GHE_SNAPSHOT_DIR" -touch "incomplete" - -# Exit early if the snapshot filesystem doesn't support hard links, symlinks and -# if rsync doesn't support hardlinking of dangling symlinks -trap 'rm -rf src dest1 dest2' EXIT -mkdir -p src -touch src/testfile -if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then - log_error "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links. \nGit repositories contain symbolic links that need to be preserved during a backup." 1>&2 - exit 1 -fi - -if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 dest2 2>&1); then - log_error "Error: rsync encountered an error that could indicate a problem with permissions,\n hard links, symbolic links, or another issue that may affect backups." 1>&2 - echo "$output" - exit 1 -fi - -if [ "$(stat -c %i dest1/testfile)" != "$(stat -c %i dest2/testfile)" ]; then - log_error "Error: the filesystem containing $GHE_DATA_DIR does not support hard links.\n Backup Utilities use hard links to store backup data efficiently." 1>&2 - exit 1 -fi -rm -rf src dest1 dest2 - -# To prevent multiple backup runs happening at the same time, we create a -# in-progress file with the timestamp and pid of the backup process, -# giving us a form of locking. -# -# Set up a trap to remove the in-progress file if we exit for any reason but -# verify that we are the same process before doing so. -# -# The cleanup trap also handles disabling maintenance mode on the appliance if -# it was automatically enabled. -cleanup () { - if [ -f ../in-progress ]; then - progress=$(cat ../in-progress) - snapshot=$(echo "$progress" | cut -d ' ' -f 1) - pid=$(echo "$progress" | cut -d ' ' -f 2) - if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = "$pid" ]; then - unlink ../in-progress - fi - fi - - rm -rf "$failures_file" - rm -f "${GHE_DATA_DIR}/in-progress-backup" - rm -rf /tmp/backup-utils-progress/* - - # Cleanup SSH multiplexing - ghe-ssh --clean - - bm_end "$(basename $0)" -} - -# Setup exit traps -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - - -# Check to see if there is a running restore -ghe_restore_check - -# Check to see if there is a running backup -if [ -h ../in-progress ]; then - - log_error "Detected a backup already in progress from a previous version of ghe-backup. \nIf there is no backup in progress anymore, please remove \nthe $GHE_DATA_DIR/in-progress file." >&2 - exit 1 -fi - -if [ -f ../in-progress ]; then - progress=$(cat ../in-progress) - snapshot=$(echo "$progress" | cut -d ' ' -f 1) - pid=$(echo "$progress" | cut -d ' ' -f 2) - if ! ps -p "$pid" >/dev/null 2>&1; then - # We can safely remove in-progress, ghe-prune-snapshots - # will clean up the failed backup. - unlink ../in-progress - else - log_error "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid. \nIf PID $pid is not a process related to the backup utilities, please remove \nthe $GHE_DATA_DIR/in-progress file and try again." 1>&2 - exit 1 - fi -fi - -# Perform a host connection check and establish the remote appliance version. -# The version is available in the GHE_REMOTE_VERSION variable and also written -# to a version file in the snapshot directory itself. -# ghe_remote_version_required should be run before any other instances of ghe-ssh -# to ensure that there are no problems with host key verification. -ghe_remote_version_required -echo "$GHE_REMOTE_VERSION" > version - -# Setup progress tracking -init-progress -export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total -export PROGRESS_TYPE="Backup" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type -export PROGRESS=0 # Used to track progress of backup -echo "$PROGRESS" > /tmp/backup-utils-progress/progress - -OPTIONAL_STEPS=0 -# Backup actions+mssql -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) -fi - -# Backup fsck -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -# Backup minio -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -# Backup pages -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi - -PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14 -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total - -# check that incremental settings are valid if set -is_inc=$(is_incremental_backup_feature_on) - -if [ "$is_inc" = true ]; then -if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then - log_error "Can only perform incremental backups on enterprise version 3.10 or higher" - exit 1 -fi -if [ "$GHE_VERSION_MINOR" -lt 10 ]; then - log_error "Can only perform incremental backups on enterprise version 3.10 or higher" - exit 1 -fi - - incremental_backup_check - # If everything is ok, check if we have hit GHE_MAX_INCREMENTAL_BACKUPS, performing pruning actions if necessary - check_for_incremental_max_backups - # initialize incremental backup if it hasn't been done yet - incremental_backup_init -fi - -echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress -echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup" - -bm_start "$(basename $0)" -START_TIME=$(date +%s) -log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP" - - - -if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then - echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." -fi - -# Output system information of the backup host - -# If /etc/os-release exists, use it to get the OS version -if [ -f /etc/os-release ]; then - OS_NAME=$(grep '^NAME' /etc/os-release | cut -d'"' -f2) - VERSION_ID=$(grep '^VERSION_ID' /etc/os-release | cut -d'"' -f2) - echo "Running on: $OS_NAME $VERSION_ID" -else - echo "Running on: Unknown OS" -fi - -# If nproc command exists, use it to get the number of CPUs -if command -v nproc >/dev/null 2>&1; then - echo "CPUs: $(nproc)" -else - echo "CPUs: Unknown" -fi - -# If the free command exists, use it to get the memory details -if command -v free >/dev/null 2>&1; then - echo "Memory $(free -m | grep '^Mem:' | awk '{print "total/used/free+share/buff/cache: " $2 "/" $3 "/" $4 "+" $5 "/" $6 "/" $7}')" -else - echo "Memory: Unknown" -fi - - -# Log backup start message in /var/log/syslog on remote instance -ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..." - -export GHE_BACKUP_STRATEGY=${GHE_BACKUP_STRATEGY:-$(ghe-backup-strategy)} - -# Record the strategy with the snapshot so we will know how to restore. -echo "$GHE_BACKUP_STRATEGY" > strategy - -# Create benchmark file -bm_init > /dev/null - -ghe-backup-store-version || -log_warn "Warning: storing backup-utils version remotely failed." - -log_info "Backing up GitHub secrets ..." -ghe-backup-secrets || failures="$failures secrets" - -log_info "Backing up GitHub settings ..." -ghe-backup-settings || failures="$failures settings" - -log_info "Backing up SSH authorized keys ..." -bm_start "ghe-export-authorized-keys" -ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json || -failures="$failures authorized-keys" -bm_end "ghe-export-authorized-keys" - -log_info "Backing up SSH host keys ..." -bm_start "ghe-export-ssh-host-keys" -ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar || -failures="$failures ssh-host-keys" -bm_end "ghe-export-ssh-host-keys" - -ghe-backup-mysql || failures="$failures mysql" - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - log_info "Backing up MSSQL databases ..." - ghe-backup-mssql 1>&3 || failures="$failures mssql" - - log_info "Backing up Actions data ..." - ghe-backup-actions 1>&3 || failures="$failures actions" -fi - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - log_info "Backing up Minio data ..." - ghe-backup-minio 1>&3 || failures="$failures minio" -fi - -cmd_title=$(log_info "Backing up Redis database ...") -commands=(" -echo \"$cmd_title\" -ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"") - -cmd_title=$(log_info "Backing up audit log ...") -commands+=(" -echo \"$cmd_title\" -ghe-backup-es-audit-log || printf %s \"audit-log \" >> \"$failures_file\"") - -cmd_title=$(log_info "Backing up Git repositories ...") -commands+=(" -echo \"$cmd_title\" -ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"") - -# Pages backups are skipped only if GHE_BACKUP_PAGES is explicitly set to 'no' to guarantee backward compatibility. -# If a customer upgrades backup-utils but keeps the config file from a previous version, Pages backups still work as expected. - -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - cmd_title=$(log_info "Backing up GitHub Pages artifacts ...") - commands+=(" - echo \"$cmd_title\" - ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"") -fi - -cmd_title=$(log_info "Backing up storage data ...") -commands+=(" -echo \"$cmd_title\" -ghe-backup-storage || printf %s \"storage \" >> \"$failures_file\"") - -cmd_title=$(log_info "Backing up custom Git hooks ...") -commands+=(" -echo \"$cmd_title\" -ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"") - -if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then - increment-progress-total-count 1 - cmd_title=$(log_info "Backing up Elasticsearch indices ...") - commands+=(" - echo \"$cmd_title\" - ghe-backup-es-rsync || printf %s \"elasticsearch \" >> \"$failures_file\"") -fi - -if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then - "$GHE_PARALLEL_COMMAND" "${GHE_PARALLEL_COMMAND_OPTIONS[@]}" -- "${commands[@]}" -else - for c in "${commands[@]}"; do - eval "$c" - done -fi - -if [ -s "$failures_file" ]; then - failures="$failures $(cat "$failures_file")" -fi - -# git fsck repositories after the backup -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - log_info "Running git fsck on repositories ..." - ghe-backup-fsck "$GHE_SNAPSHOT_DIR" || failures="$failures fsck" -fi - -# If everything was successful, mark the snapshot as complete, update the -# current symlink to point to the new snapshot and prune expired and failed -# snapshots. -if [ -z "$failures" ]; then - rm "incomplete" - - rm -f "../current" - ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current" - - if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then - ghe-prune-snapshots - else - log_info "Expired and incomplete snapshots to be pruned separately" - fi -else - log_info "Skipping pruning snapshots, since some backups failed..." -fi - -END_TIME=$(date +%s) -log_info "Runtime: $((END_TIME - START_TIME)) seconds" -log_info "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")" - -# Exit non-zero and list the steps that failed. -if [ -z "$failures" ]; then - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." -else - steps="${failures// /, }" - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." - log_error "Error: Snapshot incomplete. Some steps failed: ${steps}. " - ghe_backup_finished - exit 1 -fi - -# Detect if the created backup contains any leaked ssh keys -log_info "Checking for leaked ssh keys ..." -ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true - -log_info "Backup of $GHE_HOSTNAME finished." - -# Remove in-progress file -ghe_backup_finished diff --git a/bin/ghe-backup-progress b/bin/ghe-backup-progress deleted file mode 100755 index 7ab36e084..000000000 --- a/bin/ghe-backup-progress +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-progress [--once] -#/ Tracks the completed steps of a backup or restore operation. -#/ -#/ By default the progress is printed every continuously or until a key is pressed. -#/ Use the --once option to print the current progress once and exit. -#/ -#/ Options: -#/ --once Don't loop, just print the current progress once. -# -set -e - -while true; do - case "$1" in - -o|--once) - ONCE=1 - shift - ;; - -h|--help) - export GHE_SHOW_HELP=true - shift - ;; - -*) - echo "Unknown option: $1" >&2 - exit 1 - ;; - *) - break - ;; - esac -done - -check_for_progress_file() { - if [ ! -f /tmp/backup-utils-progress/info ]; then - echo "No progress file found. Has a backup or restore been started?" - exit 1 - fi -} - -if [ -n "$ONCE" ]; then - check_for_progress_file - cat /tmp/backup-utils-progress/info -else - check_for_progress_file - clear - cat /tmp/backup-utils-progress/info - while true; do - if read -r -t 1 -n 1; then - clear - exit ; - else - clear - cat /tmp/backup-utils-progress/info - fi - done -fi diff --git a/bin/ghe-host-check b/bin/ghe-host-check deleted file mode 100755 index 73a984c30..000000000 --- a/bin/ghe-host-check +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-host-check [-h] [--version] [] -#/ -#/ Verify connectivity with the GitHub Enterprise Server host. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ --version Display version information. -#/ The GitHub Enterprise Server host to check. When no -#/ is provided, the $GHE_HOSTNAME configured in -#/ backup.config is assumed. -#/ - -set -e - -while true; do - case "$1" in - -h | --help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - break - ;; - esac -done - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-backup-config" - -# Use the host provided on the command line if provided, or fallback on the -# $GHE_HOSTNAME configured in backup.config when not present. -host="${1:-$GHE_HOSTNAME}" - -# Options to pass to SSH during connection check -options=" - -o PasswordAuthentication=no - -o ConnectTimeout=5 - -o ConnectionAttempts=1 -" - -# Split host:port into parts -port=$(ssh_port_part "$host") -hostname=$(ssh_host_part "$host") - -set +e -# ghe-negotiate-version verifies if the target is a GitHub Enterprise Server instance -output=$(echo "ghe-negotiate-version backup-utils $BACKUP_UTILS_VERSION" | ghe-ssh -o BatchMode=no $options $host -- /bin/sh 2>&1) -rc=$? -set -e - -if [ $rc -ne 0 ]; then - case $rc in - 255) - if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then - exec "$(basename $0)" "$hostname:122" - fi - - echo "$output" 1>&2 - echo "Error: ssh connection with '$host' failed" 1>&2 - echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2 - echo "* https://docs.github.com/enterprise-server/admin/configuration/configuring-your-enterprise/accessing-the-administrative-shell-ssh" 1>&2 - ;; - 101) - echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2 - ;; - 1) - if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then - exec "$(basename $0)" "$hostname:122" - else - echo "$output" 1>&2 - fi - ;; - - esac - exit $rc -fi - -CLUSTER=false -if ghe-ssh "$host" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then - CLUSTER=true -fi - -# ensure all nodes in the cluster are online/reachable and running the same version -if "$CLUSTER"; then - online_status=$(ghe-ssh "$host" ghe-cluster-host-check) - if [ "$online_status" != "Cluster is ready to configure." ]; then - echo "Error: Not all nodes are online! Please ensure cluster is in a healthy state before using backup-utils." 1>&2 - exit 1 - fi - - node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) - distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) - if [ "$distinct_versions" -ne 1 ]; then - echo "Version mismatch: $node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 - exit 1 - fi -fi - -version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}') - -if [ -z "$version" ]; then - echo "Error: failed to parse version on '$host' or this isn't a GitHub appliance." 1>&2 - exit 2 -fi - -# Block restoring snapshots to older releases of GitHub Enterprise Server -if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) - # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) - if [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version $snapshot_version_major.$snapshot_version_minor.0)" ]; then - echo "Error: Snapshot can not be restored to an older release of GitHub Enterprise Server." >&2 - exit 1 - fi -fi - -if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then - if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then - echo "Error: high availability replica detected." 1>&2 - echo "Backup Utilities should be used to backup from the primary node in" 1>&2 - echo "high availability environments to ensure consistent and reliable backups." 1>&2 - exit 1 - fi -fi - -# backup-utils 2.13 onwards limits support to the current and previous two releases -# of GitHub Enterprise Server. -supported_minimum_version="3.9.0" - -if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then - supported=1 -fi - -if [ -z "$supported" ]; then - echo "Error: unsupported release of GitHub Enterprise Server detected." 1>&2 - echo "Backup Utilities v$BACKUP_UTILS_VERSION requires GitHub Enterprise Server v$supported_minimum_version or newer." 1>&2 - echo "Please update your GitHub Enterprise Server appliance or use an older version of Backup Utilities." 1>&2 - exit 1 -fi - -if [[ "$CALLING_SCRIPT" == "ghe-backup" && "$GHE_SKIP_CHECKS" != "true" ]]; then - cat << SKIP_MSG -**You can disable the following storage & version checks by running ghe-backup with option "--skip-checks" -OR updating GHE_SKIP_CHECKS to 'true' in your backup.config file. - -SKIP_MSG - - # Bring in the requirements file - min_rsync="" - min_openssh="" - min_jq="" - # shellcheck source=share/github-backup-utils/requirements.txt - . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/requirements.txt" - - #source disk size file - # shellcheck source=share/github-backup-utils/ghe-rsync-size - . "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-rsync-size" - - #Check if GHE_DATA_DIR is NFS mounted - fs_info=$(stat -f -c "%T" "$GHE_DATA_DIR") || true - if [ "$fs_info" == "nfs" ]; then - echo "Warning: NFS (Network File System) detected for $GHE_DATA_DIR" 1>&2 - echo "Please review https://gh.io/backup-utils-storage-requirements for details." 1>&2 - fi - - #Display dir requirements for repositories and mysql - echo -e "\nChecking host for sufficient space for a backup..." - available_space=$(df -B 1k $GHE_DATA_DIR | awk 'END{printf "%.0f", $4 * 1024}') - echo " We recommend allocating at least 5x the amount of storage allocated to the primary GitHub appliance for historical snapshots and growth over time." - - repos_disk_size=$(transfer_size repositories /tmp) - pages_disk_size=$(transfer_size pages /tmp) - es_disk_size=$(transfer_size elasticsearch /tmp) - stor_disk_size=$(transfer_size storage /tmp) - minio_disk_size=$(transfer_size minio /tmp) - mysql_disk_size=$(transfer_size mysql /tmp) - actions_disk_size=$(transfer_size actions /tmp) - mssql_disk_size=$(transfer_size mssql /tmp) - - min_disk_req=$((repos_disk_size + pages_disk_size + es_disk_size + stor_disk_size + minio_disk_size + mysql_disk_size + actions_disk_size + mssql_disk_size)) - recommended_disk_req=$((min_disk_req * 5)) - echo " - Available space: $((available_space / (1024 ** 2))) MB" - echo " - Min Disk required for this backup is at least $min_disk_req MB" - echo -e " - Recommended Disk requirement is $recommended_disk_req MB\n" - - printf '### Estimated Data Transfer Sizes - - - repositories: %d MB - - pages: %d MB - - elasticsearch: %d MB - - storage: %d MB - - minio: %d MB - - mysql: %d MB - - actions: %d MB - - mssql: %d MB -\n' \ - "$repos_disk_size" "$pages_disk_size" "$es_disk_size" "$stor_disk_size" "$minio_disk_size" "$mysql_disk_size" "$actions_disk_size" "$mssql_disk_size" - - if [[ $((available_space / (1024 * 1024))) -lt $min_disk_req ]]; then - echo "There is not enough disk space for the backup. Please allocate more space and continue." 1>&2 - exit 1 - fi - - #Check rsync, openssh & jq versions - commands=("jq" "rsync" "ssh") - missing_command="" - for cmd in "${commands[@]}"; do - if ! command -v "$cmd" > /dev/null 2>&1; then - missing_command+="$cmd " - fi - done - - # Check if any command is missing - if [[ -n "$missing_command" ]]; then - echo "One or more required tools not found: $missing_command" 1>&2 - echo "Please make sure the following utils are installed and available in your PATH: $missing_command" 1>&2 - exit 1 - fi - - echo "### Software versions" - rsync_version=$(rsync --version | grep 'version' | awk '{print $3}' | tr -cd '[:digit:].\n') - if awk "BEGIN {exit !($rsync_version < $min_rsync)}" &> /dev/null; then - echo "rsync version $rsync_version in backup-host does not meet minimum requirements." 1>&2 - echo "Please make sure you have the minimum required version of rsync: $min_rsync installed" 1>&2 - exit 1 - elif [[ $rsync_version < 3.2.5 ]] && [[ $RSYNC_WARNING != "no" ]]; then - printf "\n **WARNING:** rsync version %s on backup host is less than 3.2.5, which could result in performance degradation. - For more details, please read documentation at https://gh.io/april-2023-update-of-rsync-requirements - You can disable this warning by changing RSYNC_WARNING to 'no' in your backup.config file.\n\n" \ - "$rsync_version" - fi - echo " - rsync ${rsync_version} >= required ($min_rsync)" - - ssh_version=$(ssh -V 2>&1 | awk '{print $1}'|grep -oPm 1 '[\d\.]+' |head -1 | tr -cd '[:digit:].\n') - if awk "BEGIN {exit !($ssh_version < $min_openssh)}" &> /dev/null; then - echo "openSSH version $ssh_version in backup-host does not meet minimum requirements." 1>&2 - echo "Please make sure the minimum required version of openSSH: $min_openssh is installed" 1>&2 - exit 1 - else - echo " - openSSH ${ssh_version} >= required ($min_openssh)" - fi - - jq_version=$(jq --version |awk -F\- '{print $2}' | tr -cd '[:digit:].\n') - if awk "BEGIN {exit !($jq_version < $min_jq)}" &> /dev/null; then - echo "jq version $jq_version in backup-host does not meet minimum requirements." 1>&2 - echo "Please make sure you have the minimum required version of jq: $min_jq installed" 1>&2 - exit 1 - else - echo " - jq ${jq_version} >= required ($min_jq)" - fi -fi - -echo -e "\nConnect $hostname:$port OK (v$version)" diff --git a/bin/ghe-restore b/bin/ghe-restore deleted file mode 100755 index 101f19ebc..000000000 --- a/bin/ghe-restore +++ /dev/null @@ -1,727 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s ] [] -#/ -#/ Restores a GitHub instance from local backup snapshots. -#/ -#/ Note that the GitHub Enterprise host must be reachable and your SSH key must -#/ be setup as described in the following help article: -#/ -#/ -#/ -#/ OPTIONS: -#/ -c | --config Restore appliance settings and license in addition to -#/ datastores. Settings are not restored by default to -#/ prevent overwriting different configuration on the -#/ restore host. -#/ -f | --force Don't prompt for confirmation before restoring. -#/ -h | --help Show this message. -#/ -v | --verbose Enable verbose output. -#/ --skip-mysql Skip MySQL restore steps. Only applicable to external databases. -#/ --version Display version information and exit. -#/ -#/ -s Restore from the snapshot with the given id. Available -#/ snapshots may be listed under the data directory. -#/ -#/ The is the hostname or IP of the GitHub Enterprise -#/ instance. The may be omitted when the -#/ GHE_RESTORE_HOST config variable is set in backup.config. -#/ When a argument is provided, it always overrides -#/ the configured restore host. -#/ - -set -e - -# Parse arguments -: "${RESTORE_SETTINGS:=false}" -export RESTORE_SETTINGS - -: "${FORCE:=false}" -export FORCE - -: "${SKIP_MYSQL:=false}" -export SKIP_MYSQL - -while true; do - case "$1" in - --skip-mysql) - SKIP_MYSQL=true - shift - ;; - -f|--force) - FORCE=true - shift - ;; - -s) - snapshot_id="$(basename "$2")" - shift 2 - ;; - -c|--config) - RESTORE_SETTINGS=true - shift - ;; - -h|--help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -v|--verbose) - export GHE_VERBOSE=true - shift - ;; - -i|--incremental) - export GHE_INCREMENTAL=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - if [ -n "$1" ]; then - GHE_RESTORE_HOST_OPT="$1" - shift - else - break - fi - ;; - esac -done - - - -start_cron () { - log_info "Starting cron ..." - if $CLUSTER; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo timeout 120s service cron start"; then - log_warn "Failed to start cron on one or more nodes" - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo timeout 120s service cron start"; then - log_warn "Failed to start cron" - fi - fi -} - -cleanup () { - log_info " Exiting, cleaning up ..." - if [ -n "$1" ]; then - update_restore_status "$1" - fi - - if $ACTIONS_STOPPED && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - log_info "Restarting Actions after restore ..." - # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the - # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists - # in the --help output - if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start --help' | grep -q force; then - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start -f' 1>&3 - else - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start' 1>&3 - fi - fi - - if ! $CRON_RUNNING; then - start_cron - fi - - # Cleanup SSH multiplexing - log_info "Cleaning up SSH multiplexing ..." - if ! ghe-ssh --clean; then - log_info "Failed to clean up SSH multiplexing" - fi - - # Remove in-progress file - log_info "Removing in-progress file ..." 1>&3 - if ! rm -f "${GHE_DATA_DIR}/in-progress-restore"; then - log_error "Failed to remove in-progress file" 1>&3 - fi - - # Remove progress files - rm -rf /tmp/backup-utils-progress/* - - bm_end "$(basename $0)" -} - -# This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally. -# because it doesn't run locally does not redirect output to fd 3 or use log_info/log_warn/log_error. -# shellcheck disable=SC2034 -cleanup_cluster_nodes() { - uuid="$1" - if [ -z "$uuid" ]; then - log_error "Node UUID required." - exit 2 - fi - - echo "Cleaning up spokes" - ghe-spokes server evacuate "git-server-$uuid" 'Removing replica' - ghe-spokes server destroy "git-server-$uuid" - - echo "Cleaning up storage" - ghe-storage destroy-host "storage-server-$uuid" --force - - echo "Cleaning up dpages" - ghe-dpages offline "pages-server-$uuid" - ghe-dpages remove "pages-server-$uuid" - - echo "Cleaning up redis" - ghe-redis-cli del "resque:queue:maint_git-server-$uuid" - ghe-redis-cli srem resque:queues "maint_git-server-$uuid" -} - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" - - - - - -# Check to make sure moreutils parallel is installed and working properly -ghe_parallel_check - -# Check to make sure another restore process is not running -ghe_restore_check - -# Grab the host arg -GHE_HOSTNAME="${GHE_RESTORE_HOST_OPT:-$GHE_RESTORE_HOST}" - -# Hostname without any port suffix -hostname=$(echo "$GHE_HOSTNAME" | cut -f 1 -d :) - -# Show usage with no -[ -z "$GHE_HOSTNAME" ] && print_usage - -# Flag to indicate if this script has stopped Actions. -ACTIONS_STOPPED=false - -# ghe-restore-snapshot-path validates it exists, determines what current is, -# and if there's any problem, exit for us -GHE_RESTORE_SNAPSHOT_PATH="$(ghe-restore-snapshot-path "$snapshot_id")" -GHE_RESTORE_SNAPSHOT=$(basename "$GHE_RESTORE_SNAPSHOT_PATH") -export GHE_RESTORE_SNAPSHOT - -# Check to make sure backup is not running -ghe_backup_check - -# Detect if the backup we are restoring has a leaked ssh key -echo "Checking for leaked keys in the backup snapshot that is being restored ..." -ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true - -# Figure out whether to use the tarball or rsync restore strategy based on the -# strategy file written in the snapshot directory. -GHE_BACKUP_STRATEGY=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/strategy") - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -# Figure out if this instance has been configured or is entirely new. -instance_configured=false -if is_instance_configured; then - instance_configured=true -else - RESTORE_SETTINGS=true -fi - -# Figure out if we're restoring into cluster -CLUSTER=false -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then - CLUSTER=true -fi -export CLUSTER - -# Restoring a cluster backup to a standalone appliance is not supported -if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - log_error "Error: Snapshot from a GitHub Enterprise cluster cannot be restored to a standalone appliance. Aborting." >&2 - exit 1 -fi - -# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB -if ! ghe-restore-external-database-compatibility-check; then - exit 1 -fi - -# Figure out if this appliance is in a replication pair -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - log_error "Error: Restoring to an appliance with replication enabled is not supported. Please teardown replication before restoring." >&2 - exit 1 -fi - -# Prompt to verify the restore host given is correct. Restoring overwrites -# important data on the destination appliance that cannot be recovered. This is -# mostly to prevent accidents where the backup host is given to restore instead -# of a separate restore host since they're used in such close proximity. -if $instance_configured && ! $FORCE; then - echo - echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)" - echo " will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}." - echo - - if is_external_database_snapshot && $RESTORE_SETTINGS; then - echo "WARNING: This operation will also restore the external MySQL connection configuration," - echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online." - echo - fi - - prompt_for_confirmation "Please verify that this is the correct restore host before continuing." -fi - -# Prompt to verify that restoring BYODB snapshot to unconfigured instance -# will result in BYODB connection information being restored as well. -if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then - echo - echo "WARNING: This operation will also restore the external MySQL connection configuration," - echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online." - echo - - prompt_for_confirmation "Please confirm this before continuing." -fi -# Calculate the actual amounts of steps in the restore process -# taking into account the options passed to the script and the appliance configuration -# calculate restore steps -OPTIONAL_STEPS=0 - -# Restoring UUID -if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Restoring Actions + MSSQL -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2)) -fi -# Restoring minio -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Restoring Elasticsearch -if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Restoring audit log -if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then - if [[ "$GHE_RESTORE_SKIP_AUDIT_LOG" != "yes" ]]; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) - fi -fi -# Replica cleanup -if ! $CLUSTER && $instance_configured; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1)) -fi -# Restoring settings + restore-chat-integration + restore-packages -if $RESTORE_SETTINGS; then - OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3)) -fi - -# Minimum number of steps is 7 -export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7)) - -init-progress -echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total -export PROGRESS_TYPE="Restore" -echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress/type -export PROGRESS=0 # Used to track progress of restore -echo "$PROGRESS" > /tmp/backup-utils-progress/progress - -# Log restore start message locally and in /var/log/syslog on remote instance -bm_start "$(basename $0)" -START_TIME=$(date +%s) -log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT" - -if [ "$GHE_INCREMENTAL" ]; then - if [ "$GHE_VERSION_MAJOR" -lt 3 ]; then - log_error "Can only perform incremental restores on enterprise version 3.10 or higher" - exit 1 -fi -if [ "$GHE_VERSION_MINOR" -lt 10 ]; then - log_error "Can only perform incremental restores on enterprise version 3.10 or higher" - exit 1 -fi - log_info "Incremental restore from snapshot $GHE_RESTORE_SNAPSHOT" - # If we see 'inc_previous' prepended to the snapshot name, then - # we set $INC_FULL_BACKUP and $INC_SNAPSHOT_DATA to $INC_PREVIOUS_FULL_BACKUP and - # $INC_PREVIOUS_SNAPSHOT_DATA respectively. Otherwise, leave them at default setting - # so that incremental restore is from current cycle - if [[ "$GHE_RESTORE_SNAPSHOT" =~ ^inc_previous ]]; then - INC_FULL_BACKUP=$INC_PREVIOUS_FULL_BACKUP - INC_SNAPSHOT_DATA=$INC_PREVIOUS_SNAPSHOT_DATA - log_info "Incremental restore from previous cycle snapshot. Using $INC_FULL_BACKUP" - log_info "Incremental restore from previous cycle snapshot. Using $INC_SNAPSHOT_DATA" - fi - log_info "Validating snapshot $GHE_RESTORE_SNAPSHOT" - validate_inc_snapshot_data "$GHE_RESTORE_SNAPSHOT" -fi -ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..." -# Create an in-progress-restore file to prevent simultaneous backup or restore runs -echo "${START_TIME} $$" > "${GHE_DATA_DIR}/in-progress-restore" - -# Keep other processes on the VM or cluster in the loop about the restore status. -# -# Other processes will look for these states: -# "restoring" - restore is currently in progress -# "failed" - restore has failed -# "complete" - restore has completed successfully -update_restore_status () { - if $CLUSTER; then - echo "ghe-cluster-each -- \"echo '$1' | sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null\"" | - ghe-ssh "$GHE_HOSTNAME" /bin/bash - else - echo "$1" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null" - fi -} - -CRON_RUNNING=true -# Update remote restore state file and setup failure trap -trap "cleanup failed" EXIT -update_restore_status "restoring" - -# Make sure the GitHub appliance is in maintenance mode. -if $instance_configured; then - if ! ghe-maintenance-mode-status "$GHE_HOSTNAME"; then - log_error "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2 - exit 1 - fi -fi - -# Get GHES release version in major.minor format -RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-version' | cut -d '.' -f 1,2) - -# If the backup being restored is from an appliance with Actions disabled, restoring it onto an appliance with Actions enabled will cause -# mismatches in the secrets needed for Actions which ultimately results in Actions not working properly. Note: xargs is to remove whitespace -ACTIONS_ENABLED_IN_BACKUP=$(git config -f "$GHE_RESTORE_SNAPSHOT_PATH/settings.json" --bool app.actions.enabled | xargs) -if [[ $ACTIONS_ENABLED_IN_BACKUP != true ]] && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - log_error "Restoring a backup with Actions disabled onto an appliance with Actions enabled is not supported." >&2 - exit 1 -fi - -# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data. -# If above is true, also check if ac is present in appliance then snapshot should also contains ac databases -if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash) - ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ") - if [[ $ac_db_ghe -gt 0 && $ac_db_snapshot -eq 0 ]]; then - log_error "$GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting \n Please disable Actions cache service in $GHE_HOSTNAME and retry\nTo disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2 - exit 1 - fi - if [[ $ac_db_ghe -eq 0 && $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then - log_error "$GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting. \n Please enable Actions cache service in $GHE_HOSTNAME and retry \n To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2 - exit 1 - fi - else - log_error "$GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting. \n Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2 - exit 1 - fi -fi - -# Create benchmark file -bm_init > /dev/null - -ghe-backup-store-version || -log_warn "Warning: storing backup-utils version remotely failed." - -# Stop cron and timerd, as scheduled jobs may disrupt the restore process. -log_info "Stopping cron and github-timerd ..." -if $CLUSTER; then - bm_start "$(basename $0) - Stopping cron and github-timerd on cluster" - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then - log_warn "Failed to stop cron on one or more nodes" 1>&3 - fi - bm_end "$(basename $0) - Stopping cron and github-timerd on cluster" - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 - fi - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then - log_warn "Failed to stop github-timerd on one or more nodes" 1>&3 - fi - fi - -else - bm_start "$(basename $0) - Stopping cron and github-timerd" - echo "$(basename $0) - Stopping cron and github-timerd" - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then - log_warn "Failed to stop cron" 1>&3 - fi - bm_end "$(basename $0) - Stopping cron and github-timerd" - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - log_warn "Failed to stop github-timerd" 1>&3 - fi - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then - log_warn "Failed to stop github-timerd" 1>&3 - fi - fi - -fi -CRON_RUNNING=false - -ghe-restore-secrets "$GHE_HOSTNAME" - -# Restore settings and license if restoring to an unconfigured appliance or when -# specified manually. -if $RESTORE_SETTINGS; then - ghe-restore-settings "$GHE_HOSTNAME" -fi - -# Make sure mysql and elasticsearch are prep'd and running before restoring. -# These services will not have been started on appliances that have not been -# configured yet. -if ! $CLUSTER; then - echo "sudo ghe-service-ensure-mysql && sudo ghe-service-ensure-elasticsearch" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# Restore UUID if present and not restoring to cluster. -if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then - log_info "Restoring UUID ..." - - bm_start "$(basename $0) - Restore UUID" - ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" <"$GHE_RESTORE_SNAPSHOT_PATH/uuid" - ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true - ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft" - bm_end "$(basename $0) - Restore UUID" -fi - - -if is_external_database_snapshot; then - appliance_strategy="external" - backup_snapshot_strategy="external" -else - if is_binary_backup_feature_on; then - appliance_strategy="binary" - else - appliance_strategy="logical" - fi - - if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then - backup_snapshot_strategy="binary" - else - backup_snapshot_strategy="logical" - fi -fi - -if is_external_database_target_or_snapshot && $SKIP_MYSQL; then - log_info "Skipping MySQL restore." -else - log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." - increment-progress-total-count 2 - ghe-restore-mysql "$GHE_HOSTNAME" 1>&3 -fi - - - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - log_info "Stopping Actions before restoring databases ..." - # We mark Actions as stopped even if the `ghe-actions-stop` - # fails to ensure that we cleanly start actions when performing cleanup. - ACTIONS_STOPPED=true - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3 - - log_info "Restoring MSSQL databases ..." - ghe-restore-mssql "$GHE_HOSTNAME" 1>&3 - - log_info "Restoring Actions data ..." - ghe-restore-actions "$GHE_HOSTNAME" 1>&3 - echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning." - echo "See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners." -fi - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - log_info "Restoring MinIO data ..." - ghe-restore-minio "$GHE_HOSTNAME" 1>&3 -fi - -# log input into a variable for the parallel command, as the functions don't work with eval -cmd_title=$(log_info "Restoring Redis database ...") -commands=(" -echo \"$cmd_title\" -ghe-restore-redis \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") - -cmd_title=$(log_info "Restoring Git Repositories ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-repositories \"$GHE_HOSTNAME\"") - -cmd_title=$(log_info "Restoring Gists ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") - -if [ "$GHE_BACKUP_PAGES" != "no" ]; then - cmd_title=$(log_info "Restoring Pages ...") - commands+=(" - echo \"$cmd_title\" - ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3") -fi - -cmd_title=$(log_info "Restoring SSH authorized keys ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-ssh-keys \"$GHE_HOSTNAME\" \"$GHE_RESTORE_SNAPSHOT_PATH\"") - -cmd_title=$(log_info "Restoring storage data ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-storage \"$GHE_HOSTNAME\" 1>&3") - -cmd_title=$(log_info "Restoring custom Git hooks ...") -commands+=(" -echo \"$cmd_title\" -ghe-restore-git-hooks \"$GHE_HOSTNAME\" 1>&3") - -if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then - cmd_title=$(log_info "Restoring Elasticsearch indices ...") - commands+=(" - echo \"$cmd_title\" - ghe-restore-es-rsync \"$GHE_HOSTNAME\" 1>&3") -fi - -# Restore the audit log migration sentinel file, if it exists in the snapshot -if test -f "$GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete"; then - log_info "Restoring Elasticsearch audit log migration sentinel file ..." 1>&3 - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then - log_info "Failed to restore Elasticsearch audit log migration sentinel file." 1>&3 - fi -fi - -# Restore exported audit logs to 2.12.9 and newer single nodes and -# all releases of cluster -if $CLUSTER || [ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.12.9)" ]; then - if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then - log_info "Skipping restore of audit logs." - else - cmd_title=$(log_info "Restoring Audit logs ...") - commands+=(" - echo \"$cmd_title\" - ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3") - fi - -fi - -if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then - log_info "Restoring data in parallel ..." - "$GHE_PARALLEL_COMMAND" "${GHE_PARALLEL_COMMAND_OPTIONS[@]}" -- "${commands[@]}" -else - log_info "Restoring data serially ..." 1>&3 - for c in "${commands[@]}"; do - . "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/bm.sh" - eval "$c" - done -fi - -# Restart an already running memcached to reset the cache after restore -log_info "Restarting memcached ..." 1>&3 -bm_start "$(basename $0) - Restarting memcached" -echo "sudo restart -q memcached 2>/dev/null || true" | -ghe-ssh "$GHE_HOSTNAME" -- /bin/sh -bm_end "$(basename $0) - Restarting memcached" - -# Prevent GitHub Connect jobs running before we've had a chance to reset -# the configuration by setting the last run date to now. -if ! $RESTORE_SETTINGS; then - log_info "Setting last run date for GitHub Connect jobs ..." 1>&3 - echo "now=$(date +%s.0000000); ghe-redis-cli mset timer:UpdateConnectInstallationInfo \$now timer:UploadEnterpriseServerUserAccountsJob \$now timer:UploadConnectMetricsJob \$now timer:GitHubConnectPushNewContributionsJob \$now" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# When restoring to a host that has already been configured, kick off a -# config run to perform data migrations. -if $CLUSTER; then - log_info "Configuring cluster ..." - bm_start "$(basename $0) - configure cluster" - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3 - elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 - fi - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 - bm_end "$(basename $0) - configure cluster" -elif $instance_configured; then - log_info "Configuring appliance ..." - bm_start "$(basename $0) - configure appliance" - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 - elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 - fi - ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3 - bm_end "$(basename $0) - configure appliance" -fi - -# Clear GitHub Connect settings stored in the restored database. -# This needs to happen after `ghe-config-apply` to ensure all migrations have run. -if ! $RESTORE_SETTINGS; then - log_info "Clearing GitHub Connect settings ..." 1>&3 - echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# Start cron. Timerd will start automatically as part of the config run. -start_cron -CRON_RUNNING=true - -# Clean up all stale replicas on configured instances. -if ! $CLUSTER && $instance_configured; then - log_info "Cleaning up replicas..." 1>&3 - bm_start "$(basename $0) - Cleanup replicas" - restored_uuid=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid") - other_nodes=$(echo " - set -o pipefail; \ - ghe-spokes server show --json \ - | jq -r '.[] | select(.host | contains(\"git-server\")).host' \ - | sed 's/^git-server-//g' \ - | ( grep -F -x -v \"$restored_uuid\" || true )" \ - | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash) - if [ -n "$other_nodes" ]; then - log_info "Cleaning up stale nodes ..." - for uuid in $other_nodes; do - # shellcheck disable=SC2034 - echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3 - done - fi - bm_end "$(basename $0) - Cleanup replicas" -fi - -# Update the remote status to "complete". This has to happen before importing -# ssh host keys because subsequent commands will fail due to the host key -# changing otherwise. -trap "cleanup" EXIT -update_restore_status "complete" - -# Log restore complete message in /var/log/syslog on remote instance -ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}." - -if ! $CLUSTER; then - log_info "Restoring SSH host keys ..." - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 -else - # This will make sure that Git over SSH host keys (babeld) are - # copied to all the cluster nodes so babeld uses the same keys. - log_info "Restoring Git over SSH host keys ..." - ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -xpf - -C $GHE_REMOTE_DATA_USER_DIR/common" < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 - ghe-ssh "$GHE_HOSTNAME" -- "sudo chown babeld:babeld $GHE_REMOTE_DATA_USER_DIR/common/ssh_host_*" 1>&3 - echo "if [ -f /usr/local/share/enterprise/ghe-cluster-config-update ]; then /usr/local/share/enterprise/ghe-cluster-config-update -s; else ghe-cluster-config-update -s; fi" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -END_TIME=$(date +%s) -log_info "Runtime: $((END_TIME - START_TIME)) seconds" -log_info "Completed restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT at $(date +"%H:%M:%S")" - -log_info "Restore of $GHE_HOSTNAME finished." - -if ! $instance_configured; then - echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration." -fi - diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 011d50f79..000000000 --- a/debian/changelog +++ /dev/null @@ -1,666 +0,0 @@ -github-backup-utils (3.11.0) UNRELEASED; urgency=medium - - * `ghe-backup-myql` and `ghe-restore-mysql` will now exit 1 on errors. - * On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. - * When backups are run for HA instances in both primary and replica nodes a `.sync-in-progress` file will be created. This will disable `NetworkMaintenance` jobs from running and queueing up when backups are running from the primary node. - * Estimated transfer sizes will be calculated on appropriate nodes for clustered environments. - * Added support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. - * `ghe-restore` avoids unnecessary `rsync` operations when restoring to non-clustered environments. - * `ghe-backup` and `ghe-restore` output their total runtime - * `rsync` compression is now disabled by default. The `-z` flag has been removed from the `ghe-rsync` command in multiple files to improve transfer speed and reduce CPU usage. To enable `rsync` compression globally, add `GHE_RSYNC_COMPRESSION_ENABLED=yes` to the `backup.config` file. - * Updates the Host OS version output to use `/etc/os-release` for better compatibility with other Linux distributions. - * When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. - - -- Andrew Mildahl Thu, 30 Nov 2023 01:57:12 +0000 - -github-backup-utils (3.10.0) UNRELEASED; urgency=medium - - * Remove -o option from ps use #341 - * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 - * Move check for git for ssh muxing into ghe-ssh #378 - * Check filesystem supports hardlinks #388 - * Remove check for git from ghe-ssh #393 - * Clean up stale HA nodes on restore #396 - - -- Balwinder Sohi Wed, 09 Aug 2023 19:37:10 +0000 - -github-backup-utils (3.9.0) UNRELEASED; urgency=medium - - * Set restore status on all cluster nodes #274 - * Fix pages backups and restores in GitHub Enterprise 11.10 #275 - * Backup and restore custom CA certificates #281 - * Set the benchmark file path consistently #283 - * Suppress dd output noise #289 - * Track completeness of Elasticsearch JSON dumps #298 - * Use existing Elasticsearch indices to speed up transfer during a restore #310 - * Include the user data directory in the benchmark name #311 - * Use calculated routes when backing up storage data from a cluster #318 - * Refresh the existing indices when restoring Elasticsearch indices to cluster #328 - * Use git to generate short name for SSH multiplex control path #335 - - -- Junior Eluhu Mon, 12 Jun 2023 20:46:10 +0000 - -github-backup-utils (3.8.0) focal; urgency=medium - - -- Daniel Johnson Tue, 07 Feb 2023 21:43:26 +0000 - -github-backup-utils (3.7.0) UNRELEASED; urgency=medium - - - -- Devin Dooley Tue, 25 Oct 2022 00:35:38 +0000 - -github-backup-utils (3.6.0) UNRELEASED; urgency=medium - - - -- Joe Franks Wed, 17 Aug 2022 19:20:54 +0000 - -github-backup-utils (3.5.0) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Bon Sohi Thu, 12 May 2022 21:06:56 +0000 - -github-backup-utils (3.4.1) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Donal Ellis Fri, 22 Apr 2022 04:00:00 +0000 - -github-backup-utils (3.3.2) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Donal Ellis Fri, 22 Apr 2022 00:53:45 +0000 - -github-backup-utils (3.4.0) UNRELEASED; urgency=medium - - * Add anchor to usage doc for settings restore #865 - - -- Steve Culver Tue, 15 Feb 2022 19:25:09 +0000 - -github-backup-utils (3.3.1) UNRELEASED; urgency=medium - - * Fix compat issue with ghe-actions-start during maintenance mode #836 - - -- Balwinder Sohi Tue, 21 Dec 2021 23:38:01 +0000 - -github-backup-utils (3.3.0) UNRELEASED; urgency=medium - - -- Balwinder Sohi Wed, 08 Dec 2021 03:12:53 +0000 - -github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium - - -- Nick Iodice Tue, 09 Nov 2021 19:56:08 +0000 - -github-backup-utils (3.2.0) UNRELEASED; urgency=medium - - -- Brett Westover Tue, 28 Sep 2021 16:50:00 +0000 - -github-backup-utils (3.2.0.rc3) UNRELEASED; urgency=medium - - * Move GitHub Connect reset to after ghe-config-apply #783 - - -- Brett Westover Fri, 17 Sep 2021 00:44:59 +0000 - -github-backup-utils (3.2.0.rc1) UNRELEASED; urgency=medium - - * Leaked host key check: Avoid false positives from FIPS mode #748 - * Always restore user-password-secrets #762 - * Make some of the actions setting best effort #767 - * Remove reference to `ghe-cluster-cleanup-nodes` #768 - * Stop/Restart Actions in ghe-restore #769 - * Clear GitHub Connect settings when not restoring settings #770 - * GitHub Connect Reset Issue #776 - - -- Brett Westover Fri, 10 Sep 2021 20:10:07 +0000 - -github-backup-utils (3.2.0) UNRELEASED; urgency=medium - - * Leaked host key check: Avoid false positives from FIPS mode #748 - * Always restore user-password-secrets #762 - * Make some of the actions setting best effort #767 - * Remove reference to `ghe-cluster-cleanup-nodes` #768 - * Stop/Restart Actions in ghe-restore #769 - * Clear GitHub Connect settings when not restoring settings #770 - - -- Brett Westover Thu, 09 Sep 2021 16:42:24 +0000 - -github-backup-utils (3.1.0) UNRELEASED; urgency=medium - - -- Zachary Mark Thu, 03 Jun 2021 16:55:16 +0000 - -github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium - - [ Zachary Mark ] - * Update repository backups to use ghe-gc-* for lock file management #188 - * A faster way to restore storage blobs (clusters) #212 - * Bug fix: Be more specific in restore routes globbing #715 - * fix(docker): add coreutils to get flags for date #717 - * Add backup cadence variable to the appliance #719 - * Fix is_default_external_database_snapshot function #720 - - [ Hideki Yamane ] - * debian/rules - - Drop unnecessary build-indep: - - Add manpages generation. - * debian/changelog - - Fix some lintian warnings for old entries. - * debian/source/format - - Upgrade to newer source format 3.0 (native). - * debian/copyright - - Format it as Machine-readable debian/copyright format 1.0 - - Add entries Twan Wolthof and me for debian/* since - enough code was written by non-github.com employees, at least. - * debian/control - - Update descriptions in debian/control - - Fix "Architecture: all", instead of "any". - - Declare its Homepage as https://github.com/github/backup-utils - - Add Vcs-* metadata field. - - Specify "Rules-Requires-Root: no" - - Drop unnecessary "Build-Depends: devscripts" - - Set Standards-Version: 4.5.1 - - Update Maintainer as Zachary Mark - * debian/{clean,manpages} - - Add files to support manpages handles. - - -- Zachary Mark Thu, 06 May 2021 17:11:18 +0000 - -github-backup-utils (3.0.0) UNRELEASED; urgency=medium - - * Fix restoring the password pepper for already configured instances #683 - - -- Michael Dang Tue, 16 Feb 2021 22:32:25 +0000 - -github-backup-utils (3.0.0.rc1) UNRELEASED; urgency=medium - - * Cleanup nomad container when restore for 2.22 #670 - * Use ghe-cluster-nomad-cleanup for cluster mode #663 - * Only run ghe-nomad-cleanup in 3.0 #662 - * Revert backup-utils gitHub env and a few more fixes #661 - * Note how to test filesystem symlink / hardlink support #660 - * stop github-timerd based on its running environment #659 - * Backup and restore password pepper #656 - * github-env -> github-env-dispatch #654 - * Rename redis-cli to ghe-redis-cli #639 - - -- Michael Dang Thu, 14 Jan 2021 21:17:53 +0000 - -github-backup-utils (2.22.0) UNRELEASED; urgency=medium - - * Added basic timing around the ghe-restore process #625 - * Improve python3 & finding moreutils parallel #627 - * Turn off POSIX for ghe-backup-config #632 - * Add parallelized restore capability to ghe-restore-storage #635 - * Update backup-utils for new features in 2.22 #641 - - -- Hao Jiang Wed, 23 Sep 2020 15:48:54 +0000 - -github-backup-utils (2.21.0) UNRELEASED; urgency=medium - - * Introduce option to skip restoring of audit logs #596 - * Beta: Execute ghe-backup tasks in parallel #597 - * Beta: Execute ghe-restore tasks in parallel #601 - * Run repositories restore in parallel #603 - * Fix mismatched `bm_start` and `bm_end` commands #607 - * remove rsync restore method used by GHES versions prior to 2.13 #608 - * Output MySQL backup strategy for clarity during backup and restore #610 - - -- Hao Jiang Tue, 09 Jun 2020 17:59:06 +0000 - -github-backup-utils (2.19.5) UNRELEASED; urgency=medium - - * In legacy mode we should use ghe-import-mysql #581 - - -- Caine Jette Fri, 21 Feb 2020 19:13:17 +0000 - -github-backup-utils (2.20.2) UNRELEASED; urgency=medium - - * In legacy mode we should use ghe-import-mysql #581 - - -- Hao Jiang Thu, 20 Feb 2020 22:47:09 +0000 - -github-backup-utils (2.20.1) UNRELEASED; urgency=medium - - * Fixes gist route calculation for GHES version 2.20 - - -- Caine Jette Wed, 19 Feb 2020 21:47:18 +0000 - -github-backup-utils (2.19.4) UNRELEASED; urgency=medium - - * Fix the way we connect to mysql master using ssh forwarding for binary backups #567 - - -- Hao Jiang Tue, 18 Feb 2020 17:54:31 +0000 - -github-backup-utils (2.20.0) UNRELEASED; urgency=medium - - * Fix `ghe-backup-repositories` performance for large instances #541 - * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551 - - -- Alejandro Figueroa Tue, 11 Feb 2020 20:47:17 +0000 - -github-backup-utils (2.19.3) UNRELEASED; urgency=medium - - * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551 - - -- Hao Jiang Tue, 11 Feb 2020 19:31:55 +0000 - -github-backup-utils (2.19.2) UNRELEASED; urgency=medium - - * Fix `ghe-backup-repositories` performance for large instances #541 - - -- Evgenii Khramkov Fri, 31 Jan 2020 19:13:46 +0000 - -github-backup-utils (2.19.1) UNRELEASED; urgency=medium - - * Cater for more explicit gist paths used in routes file #524 - * Suppress "*.rsync': No such file or directory" errors when no data to backup or restore #525 - - -- Colin Seymour Wed, 11 Dec 2019 09:33:01 +0000 - -github-backup-utils (2.19.0) UNRELEASED; urgency=medium - - * Remove temporary exclude file from the backup host not the target GHES appliance #516 - * Update Debian package depends to include git #520 - - -- Colin Seymour Tue, 12 Nov 2019 18:57:12 +0000 - -github-backup-utils (2.18.0) UNRELEASED; urgency=medium - - * Replace "sed -E" in ghe-host-check with a more portable solution #509 - - -- Colin Seymour Tue, 20 Aug 2019 18:49:44 +0000 - -github-backup-utils (2.17.1) UNRELEASED; urgency=medium - - * Redirect ghe-export-audit-logs stderr output unless using verbose output #497 - - -- Colin Seymour Wed, 05 Jun 2019 08:43:25 +0000 - -github-backup-utils (2.17.0) UNRELEASED; urgency=medium - - * Restore target is ignored when specified on the command line #476 - * Support incremental backups for MySQL-stored audit logs #485 - - -- Colin Seymour Thu, 23 May 2019 08:20:15 +0000 - -github-backup-utils (2.16.1) UNRELEASED; urgency=medium - - * Detect storage user on each cluster host being backed up or restored #472 - - -- Colin Seymour Tue, 26 Feb 2019 16:37:04 +0000 - -github-backup-utils (2.16.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 22 Jan 2019 20:25:34 +0000 - -github-backup-utils (2.15.1) UNRELEASED; urgency=medium - - * Restoring to an un-configured appliance fails due to a missing license file #449 - - -- Colin Seymour Tue, 13 Nov 2018 17:34:21 +0000 - -github-backup-utils (2.15.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 16 Oct 2018 16:40:03 +0000 - -github-backup-utils (2.15.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 16 Oct 2018 16:07:36 +0000 - -github-backup-utils (2.14.3) UNRELEASED; urgency=medium - - * Improve multi-platform detection of simultaneous ghe-backup runs #435 - - -- Colin Seymour Tue, 11 Sep 2018 17:03:42 +0000 - -github-backup-utils (2.14.2) UNRELEASED; urgency=medium - - * Capture and display repository/gist warnings during cluster restore #423 - * Use remote tempdir when finalizing Pages routes #424 - * Use old rsync restore method for pages prior to 2.13 #426 - - -- Colin Seymour Tue, 21 Aug 2018 13:57:20 +0000 - -github-backup-utils (2.14.1) UNRELEASED; urgency=medium - - * Don't fail a backup if the Management Console password isn't set #416 - * Fix permissions issues when repeat restoring to configured cluster instance #417 - * Add missing dependencies to debian packaging #418 - * Prevent restoring snapshots to older releases #420 - - -- Colin Seymour Tue, 07 Aug 2018 16:00:36 +0000 - -github-backup-utils (2.14.0) UNRELEASED; urgency=medium - - * Disable pager and context when running git-diff #411 - * Optimise hookshot and audit log backups and restores and MySQL restores #413 - - -- Colin Seymour Thu, 12 Jul 2018 15:11:11 +0000 - -github-backup-utils (2.13.2) UNRELEASED; urgency=medium - - * Treat missing repository networks, gists, and storage objects as a non-critical error #386 - * Clean up stale HA nodes on restore #396 - * Cleanup all SSH muxes in a non blocking way #402 - * Raise an error if the current symlink doesn't exist when attempting to restore it #404 - - -- Colin Seymour Fri, 22 Jun 2018 10:08:22 +0000 - -github-backup-utils (2.13.1) UNRELEASED; urgency=medium - - * Retry with the admin ssh port on network unreachable too. #377 - * Output backup utils version on backup and restore #384 - * Check filesystem supports hardlinks #388 - * Switch back to optimised restore route calculation #389 - * Optionally log verbose output to a file instead of stdout #391 - * Switch back to optimised backup route calculation #392 - * Remove check for git from ghe-ssh #393 - * Use remote's mktemp to create temp dir on remote host #395 - - -- Colin Seymour Wed, 09 May 2018 10:16:18 +0000 - -github-backup-utils (2.13.0) UNRELEASED; urgency=medium - - * Unify the backup & restore process #375 - - -- Colin Seymour Tue, 27 Mar 2018 16:01:43 +0000 - -github-backup-utils (2.11.4) UNRELEASED; urgency=medium - - * Move check for git for ssh muxing into ghe-ssh #378 - * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 - - -- Colin Seymour Tue, 27 Mar 2018 13:20:18 +0000 - -github-backup-utils (2.11.3) UNRELEASED; urgency=medium - - * Update argument parsing and help/usage consistency #320 - * Fix failures variable #353 - * Remove other snapshot contents before removing the "incomplete" file #358 - * Backup and restore the management console password #361 - * Check for git before allowing SSH multiplex #362 - * Cleanup SSH multiplexing on exit #363 - * Filter cluster nodes by role during backup and restore #367 - * Optimise route generation and finalisation during cluster restores of pages #369 - * Allow extra rsync options to override default options #370 - - -- Colin Seymour Wed, 28 Feb 2018 16:32:07 +0000 - -github-backup-utils (2.11.2) UNRELEASED; urgency=medium - - * Allow the restoration of configuration to Cluster #347 - * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 - - -- Colin Seymour Thu, 09 Nov 2017 12:16:23 +0000 - -github-backup-utils (2.11.1) UNRELEASED; urgency=medium - - * Refresh the existing indices when restoring Elasticsearch indices to cluster #328 - * Fix failure to restore 2.9/2.10 backups to 2.11 prevented by incorrect detection of the audit log migration #333 - * Use git to generate short name for SSH multiplex control path #335 - * Remove use of --literally when computing arbitrary shasum #338 - * Remove -o option from ps use #341 - - -- Colin Seymour Thu, 05 Oct 2017 14:47:56 +0000 - -github-backup-utils (2.11.0) UNRELEASED; urgency=medium - - * Use calculated routes when backing up storage data from a cluster #318 - * Add SSH multiplexing support #321 - * Optimise route generation and finalisation during cluster restores #322 - * Prefer the SSH port specified on the command line #324 - - -- Colin Seymour Wed, 13 Sep 2017 16:31:20 +0000 - -github-backup-utils (2.10.0) UNRELEASED; urgency=medium - - * Include the user data directory in the benchmark name #311 - * Use existing Elasticsearch indices to speed up transfer during a restore #310 - * Improve detection of failures in cluster backup rsync threads #301 - * Improve redis backup robustness #306 - * Use default niceness for restores #308 - * Add additional case to SSH port detection logic #304 - * Suppress additional dd output noise #289 - * Track completeness of Elasticsearch JSON dumps #298 - - -- Steven Honson Thu, 08 Jun 2017 09:06:16 +1000 - -github-backup-utils (2.9.0) UNRELEASED; urgency=medium - - * Block restores to appliances with HA configured #291 - * Add a `--version` flag #284 - * Check backup-utils are not being run on GitHub Enterprise host #286 - * Backup and restore custom CA certificates #281 - * Hookshot backup/restores optimisations #276 - - -- Sergio Rubio Wed, 01 Mar 2017 09:39:26 -0800 - -github-backup-utils (2.8.3) UNRELEASED; urgency=medium - - * Set restore status on all cluster nodes #274 - * Fix pages backups and restores in GitHub Enterprise 11.10 #275 - - -- Steven Honson Wed, 21 Dec 2016 21:01:20 +1100 - -github-backup-utils (2.8.2) UNRELEASED; urgency=medium - - * Backup and restore the appliance UUID #272 - - -- Sergio Rubio Thu, 17 Nov 2016 15:58:08 +0100 - -github-backup-utils (2.8.1) UNRELEASED; urgency=medium - - * Stop cron and timerd during restore #269 - * Fix compatibility issue with older versions of OpenSSH #263 - - -- Sergio Rubio Mon, 14 Nov 2016 22:04:48 +0100 - -github-backup-utils (2.8.0) UNRELEASED; urgency=low - - * Adds support for GitHub Enterprise 2.8.0 - * Speedup storage restores #247 - * More portable backup-utils #260 - - -- rubiojr Wed, 09 Nov 2016 06:35:21 -0800 - -github-backup-utils (2.7.1) UNRELEASED; urgency=medium - - * Cluster: fix offline cluster node detection #250 - * Detect leaked ssh keys in backup snapshots #253 - - -- Sergio Rubio Tue, 20 Sep 2016 20:15:12 +0200 - -github-backup-utils (2.7.0) UNRELEASED; urgency=medium - - * GitHub Enterprise 2.7.0 support - - -- Sergio Rubio Wed, 03 Aug 2016 20:25:31 +0200 - -github-backup-utils (2.6.4) UNRELEASED; urgency=medium - - * Instrument/benchmark backups #238 - * Cluster: remove restoring cluster.conf on restore #242 - * Cluster: Prevent restoring to a standalone GHE appliance #244 - - -- Sergio Rubio Wed, 27 Jul 2016 19:15:53 +0200 - -github-backup-utils (2.6.3) UNRELEASED; urgency=medium - - * Cluster: git-hooks backup fixes #235 - - -- Sergio Rubio Wed, 29 Jun 2016 21:05:21 +0200 - -github-backup-utils (2.6.2) UNRELEASED; urgency=medium - - * git-hooks fixes #231 - * Cluster: speedup repositories restore #232 (requires GitHub Enterprise - 2.6.4) - * Cluster: restore Git over SSH keys #230 - * Benchmark restores #219 - - -- Sergio Rubio Wed, 22 Jun 2016 19:36:06 +0200 - -github-backup-utils (2.6.1) UNRELEASED; urgency=medium - - * Cluster: faster gist restores #220 - * Cluster: faster storage restores #212 - * Cluster: fix git-hooks restores #204 - - -- Sergio Rubio Tue, 31 May 2016 20:54:11 +0200 - -github-backup-utils (2.6.0) UNRELEASED; urgency=medium - - * Adds support for GitHub Enterprise 2.6 - * Adds an extra supported location for the backup configuration #197 - * New config option to check for corrupted repositories after the backup #195 - * General improvements and bug fixes - - -- Sergio Rubio Tue, 26 Apr 2016 18:03:01 +0200 - -github-backup-utils (2.5.2) UNRELEASED; urgency=medium - - * New configuration variable: GHE_CREATE_DATA_DIR #186 - * Require that snapshots originated from an instance running GitHub - Enterprise 2.5.0 or above when restoring to a cluster #182 - * Prevent Git GC operations and some other maintenance jobs from running - while repositories are being restored #179 - * Fix Solaris and SmartOS support, using Bash everywhere #177 - - -- Sergio Rubio Wed, 30 Mar 2016 14:32:15 +0200 - -github-backup-utils (2.5.1) UNRELEASED; urgency=medium - - * Fixes for cluster restores #173 - * Fix Elasticsearch backups for GitHub Enterprise <= 2.2 #175 - * Removed experimental S3 support #167 - * Remote logging output fixes #170 - * Update ghe-host-check to detect extra port 22 error #162 - - -- Sergio Rubio Wed, 09 Mar 2016 14:44:05 +0100 - -github-backup-utils (2.5.0) UNRELEASED; urgency=medium - - * Adds GitHub Enterprise 2.5 support - * Adds GitHub Enterprise Clustering support - * Backups and restores SAML keypairs - - -- Sergio Rubio Tue, 9 Feb 2016 00:02:37 +0000 - -github-backup-utils (2.4.0) UNRELEASED; urgency=medium - - * Moves the in-progress detection to a separate file with PID which is - removed if the process is no longer running after the backup. #145, #99 - * Updates the README to explain why backup-utils is useful even if you have - the high availability replica running. #140 - * Changes the use of the --link-dest option to only occur when backing up - populated directories. #138 - * Adds logging to /var/log/syslog on the remote GitHub Enterprise appliance - to both ghe-backup and ghe-restore. #131 - * Restarts memcached after restoring to an already configured appliance to - ensure it doesn't contain out-of-sync information. #130 - * Removes the temporary /data/user/repositories-nw-backup directory that - remains after successfully migrating the repository storage layout to the - new format used on GitHub Enterprise 2.2.0 and later after restoring a - backup from an older release of GitHub Enterprise. #129 - * Add devscripts to Debian's build-depends for checkbashisms. #101 - * Documents the -c option which forces the restoration of the configuration - information to an already configured appliance. #96 - - -- Colin Seymour Tue, 20 Oct 2015 00:02:37 +0000 - -github-backup-utils (2.2.0) UNRELEASED; urgency=medium - - * Adds support for the new repositories filesystem layout include in - GitHub Enterprise v2.2. #122, #124 - * ghe-restore now performs a config run on the instance after an incremental - restore to 11.10.x and 2.x instances. #100 - * ghe-restore now fails fast when run against a GHE instance with replication - enabled. Replication should be disabled during a restore and then setup - after the restore completes. #121 - * Fixes an issue with special port 122 detection failing when port is - overridden in an ssh config file. #102 - * Removes a warning message when running ghe-backup against an instance with - GitHub Pages disabled. #117 - * backup-utils release version numbers now track GitHub Enterprise releases - to ease the process of determining which version of backup-utils is - required for a given GitHub Enterprise version. - - -- Ryan Tomayko Wed, 29 Apr 2015 07:29:04 +0000 - -github-backup-utils (2.0.2) UNRELEASED; urgency=medium - - * ghe-restore now requires that an already-configured appliance be put into - maintenance mode manually. This is a safeguard against accidentally - overwriting data on the wrong instance. #62, #84 - * ghe-backup and ghe-restore now run a ghe-negotiate-version program on the - appliance to determine whether the backup-utils and GHE versions are - compatible. #91 - * Various portability fixes for problems surfaced when running on Solaris - and FreeBSD. #86, #87 - * Fixes an issue in ghe-backup where mysqldump failures weren't being - reported properly. #90 - * Automated builds are now run on Travis CI. #77 - - -- Ryan Tomayko Tue, 20 Jan 2015 16:00:00 +0000 - -github-backup-utils (2.0.1) UNRELEASED; urgency=medium - - * Adds /etc/github-backup-utils/backup.config as a default config file search - location for deb / system installs. - * Enables SSH BatchMode for all remote command invocation except initial host - check / version identification. - * Fixes a bug in ghe-backup where Git GC process detection would misclassify - long-running server processes matching /git.*gc/, causing the backup operation - to timeout. - * Adds a note and link to the Migrating from GitHub Enterprise v11.10.34x to - v2.0 documentation in the README. - * Adds example / documentation for the GHE_EXTRA_SSH_OPTS config value to the - backup.config-example file. - - -- Ryan Tomayko Mon, 17 Nov 2014 12:47:22 +0000 - -github-backup-utils (2.0.0) UNRELEASED; urgency=medium - - * Support for GitHub Enterprise 2.0. - * Support for migrating from GitHub Enterprise 11.10.34x to 2.0 (including from - VMware to AWS). - * ghe-backup retains hardlinks present on VM in backup snapshots, saving space. - * ghe-restore retains hardlinks present in backup snapshot when restoring to VM. - * backup-utils now includes debian packaging support. - * Fixes an issue with ghe-restore -s not using the snapshot specified. - * Fixes an issue with ghe-backup not waiting for nw-repack processes to finish - in some instances. - - -- Ryan Tomayko Mon, 10 Nov 2014 10:48:36 +0000 - -github-backup-utils (1.1.0) UNRELEASED; urgency=medium - - * Updated documentation on minimum GitHub Enterprise version requirements for - online and incremental backups from v11.10.341 to at least v11.10.342. - * The ghe-restore command now prompts for confirmation of the host to restore to - before performing any destructive operation. This is to reduce the chances of - restoring to the wrong host. The prompt may be bypassed in automated scenarios - by providing the --force option. - * Added a -c option to ghe-restore for restoring base appliance settings in - addition to primary datastores. See ghe-restore --help for more information. - * Added a note about disabling maintenance mode on the appliance after a - successful ghe-restore operation. - * Added support for filesystem layout changes and upgraded server components in - * future versions of GitHub Enterprise. - - -- Twan Wolthof Sat, 18 Oct 2014 19:14:47 +0000 - -github-backup-utils (1.0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- Twan Wolthof Tue, 23 Sep 2014 08:34:55 +0000 diff --git a/debian/clean b/debian/clean deleted file mode 100644 index 0f651869c..000000000 --- a/debian/clean +++ /dev/null @@ -1 +0,0 @@ -debian/*.1 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index b2cd7c945..000000000 --- a/debian/control +++ /dev/null @@ -1,32 +0,0 @@ -Source: github-backup-utils -Maintainer: Zachary Mark -Section: misc -Priority: optional -Standards-Version: 4.5.1 -Build-Depends: debhelper (>= 9), git, moreutils, jq, rsync (>= 2.6.4), help2man, -Homepage: https://github.com/github/backup-utils -Vcs-Git: https://github.com/github/backup-utils.git -Vcs-Browser: https://github.com/github/backup-utils -Rules-Requires-Root: no - -Package: github-backup-utils -Architecture: all -Depends: ${misc:Depends}, rsync (>= 2.6.4), moreutils, jq, git -Description: Backup and recovery utilities for GitHub Enterprise Server - The backup utilities implement a number of advanced capabilities for backup - hosts, built on top of the backup and restore features already included in - GitHub Enterprise Server. - . - These advanced features include: - - Complete GitHub Enterprise Server backup and recovery system via two simple - utilities: `ghe-backup` and `ghe-restore`. - - Online backups. The GitHub appliance need not be put in maintenance mode for - the duration of the backup run. - - Incremental backup of Git repository data. Only changes since the last - snapshot are transferred, leading to faster backup runs and lower network - bandwidth and machine utilization. - - Efficient snapshot storage. Only data added since the previous snapshot - consumes new space on the backup host. - - Multiple backup snapshots with configurable retention periods. - - Backup runs under the lowest CPU/IO priority on the GitHub appliance, - reducing performance impact while backups are in progress. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 8879cfae8..000000000 --- a/debian/copyright +++ /dev/null @@ -1,33 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: GitHub Enterprise Server Backup Utilities -Source: https://github.com/github/backup-utils - -Files: * -Copyright: 2014-2021, GitHub Inc. -License: MIT - -Files: debian/* -Copyright: 2014-2021, GitHub Inc. - 2014 Twan Wolthof - 2021 Hideki Yamane -License: MIT - -License: MIT - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - . - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/debian/install b/debian/install deleted file mode 100644 index bc992fc49..000000000 --- a/debian/install +++ /dev/null @@ -1,2 +0,0 @@ -bin/* usr/bin -share/* usr/share diff --git a/debian/manpages b/debian/manpages deleted file mode 100644 index 0f651869c..000000000 --- a/debian/manpages +++ /dev/null @@ -1 +0,0 @@ -debian/*.1 diff --git a/debian/rules b/debian/rules deleted file mode 100755 index af3d39d08..000000000 --- a/debian/rules +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/make -f - -VERSION=$$(cat $(CURDIR)/share/github-backup-utils/version) - -override_dh_auto_build: - # generate manpages for ghe-backup, ghe-host-check and ghe-restore - help2man $(CURDIR)/bin/ghe-backup -N -o $(CURDIR)/debian/ghe-backup.1 \ - -n "Take snapshots of all GitHub Enterprise data" \ - --version-string="ghe-backup $(VERSION)" - help2man $(CURDIR)/bin/ghe-host-check -N -o $(CURDIR)/debian/ghe-host-check.1 \ - -n "Restores a GitHub instance from local backup snapshots" \ - --version-string="ghe-host-check $(VERSION)" - help2man $(CURDIR)/bin/ghe-restore -N -o $(CURDIR)/debian/ghe-restore.1 \ - -n "Verify connectivity with the GitHub Enterprise Server host" \ - --version-string="ghe-restore $(VERSION)" - -%: - dh $@ diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db8f..000000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/ownership.yaml b/ownership.yaml deleted file mode 100644 index bde619edc..000000000 --- a/ownership.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -version: 1 -ownership: -- name: ghes-backup-utilities - long_name: GHES Backup Utilities - description: GitHub Enterprise Disaster Recover Solution - kind: logical - repo: https://github.com/github/backup-utils-private - qos: best_effort - team_slack: ghes-lifecycle-aor - team: github/ghes-lifecycle - maintainer: whitneyimura - exec_sponsor: jakuboleksy - tier: 3 - product_manager: davidjarzebowski - sev1: - slack: ghes-on-call - alert_slack: ghes-backup-utils - pagerduty: https://github.pagerduty.com/escalation_policies#PBQWK20 - tta: 30 minutes - sev2: - issue: https://github.com/github/ghes/issues/new - tta: 1 business day - sev3: - issue: https://github.com/github/ghes/issues - tta: 1 week - support_squad: - slack: support-squad-infrastructure - issue: https://github.com/github/support-squad-infrastructure/issues diff --git a/release-notes/3.11.0.md b/release-notes/3.11.0.md deleted file mode 100644 index 197501c92..000000000 --- a/release-notes/3.11.0.md +++ /dev/null @@ -1,18 +0,0 @@ -### Bug Fixes - -* `ghe-backup-myql` and `ghe-restore-mysql` will now exit 1 on errors. -* On an instance with Actions enabled, incorrect backup and restore settings prevented the storage container name from being restored. This made the logs from that container inaccessible, and caused Actions to create a new storage container in a different location. -* When backups are run for HA instances in both primary and replica nodes a `.sync-in-progress` file will be created. This will disable `NetworkMaintenance` jobs from running and queueing up when backups are running from the primary node. - -### Changes - -* Estimated transfer sizes will be calculated on appropriate nodes for clustered environments. -* Added support for finding the `parallel` command from the `moreutils` tool suite on more Linux distributions, including Arch Linux and Alpine Linux. -* `ghe-restore` avoids unnecessary `rsync` operations when restoring to non-clustered environments. -* `ghe-backup` and `ghe-restore` output their total runtime -* `rsync` compression is now disabled by default. The `-z` flag has been removed from the `ghe-rsync` command in multiple files to improve transfer speed and reduce CPU usage. To enable `rsync` compression globally, add `GHE_RSYNC_COMPRESSION_ENABLED=yes` to the `backup.config` file. -* Updates the Host OS version output to use `/etc/os-release` for better compatibility with other Linux distributions. - -### Backups and Disaster Recovery - -* When a NFS mount is detected for snapshots on backup hosts, backup logs will show a warning to notify the user that such a setup may incur performance issues as highlighted in [storage requirements](https://github.com/github/backup-utils-private/blob/master/docs/requirements.md#storage-requirements) documentation. diff --git a/script/package-deb b/script/package-deb deleted file mode 100755 index 15a698e9c..000000000 --- a/script/package-deb +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/package-deb -# Script to build a deb release package from the current HEAD version. -# The package version comes from the debian/changelog file so that should -# be edited before running this. -set -e - -# Change into project root -cd "$(dirname "$0")"/.. - -# Fetch tags from remote repository -git fetch --tags - -# Basic package name and version. -PKG_BASE="github-backup-utils" -PKG_VERS="$(git describe --tags)" -PKG_NAME="${PKG_BASE}-${PKG_VERS}" -PKG_HEAD="$(git rev-parse HEAD)" - -# Run git-archive to generate tarball -rm -rf dist/debuild -trap "rm -rf dist/debuild" EXIT -mkdir -p dist/debuild - -distdir="$(pwd)/dist/debuild/$PKG_NAME" -git clone -q . "$distdir" -cd "$distdir" - -echo "Removing files listed in .releaseignore ..." -while IFS= read -r line; do - rm -rf "$line" -done < .releaseignore - -echo "Removing .releaseignore ..." -rm -f .releaseignore -git checkout -q "$PKG_HEAD" - -debuild -uc -us 1>&2 -cd .. -files=$(ls -1 *.deb *.tar.xz *.dsc *.changes) -mv $files ../ -for f in $files; do echo "dist/$f"; done diff --git a/script/package-tarball b/script/package-tarball deleted file mode 100755 index bf1510e8f..000000000 --- a/script/package-tarball +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/package-tarball -# Script to build a tarball release package from the current HEAD version. -# The package version comes from `git-describe --tags' so the release tag should -# be in place before this command is run. -set -e - -# Change into project root -cd "$(dirname "$0")"/.. - -# Fetch tags from remote repository -git fetch --tags - -# Basic package name and version. -PKG_BASE="github-backup-utils" -PKG_VERS="$(git describe --tags)" -PKG_NAME="${PKG_BASE}-${PKG_VERS}" - -# Remove all files or directories listed in .releaseignore -echo "Removing files listed in .releaseignore ..." -while IFS= read -r line; do - rm -rf "$line" -done < .releaseignore - -# Remove the .releaseignore file itself -echo "Removing .releaseignore ..." -rm -f .releaseignore - -# Run git-archive to generate tarball -echo "Creating ${PKG_NAME}.tar.gz ..." -mkdir -p dist -git archive \ - --format=tar.gz \ - --prefix="$PKG_NAME/" \ - --output="dist/${PKG_NAME}.tar.gz" \ - HEAD - -# List archive contents for review -gzip -dc < "dist/${PKG_NAME}.tar.gz" | tar tf - - -# Output location -echo "Package dist/${PKG_NAME}.tar.gz OK" diff --git a/script/release b/script/release deleted file mode 100755 index 9629cacff..000000000 --- a/script/release +++ /dev/null @@ -1,506 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -#/ Usage: release [--dry-run] [--skip-version-bump-check] [min_version] -#/ -#/ Publish a backup-utils release: -#/ * Updates the package changelog -#/ * Bumps the backup-utils version if required -#/ * Creates the release pull request -#/ * Merges the release pull request -#/ * Creates the release draft -#/ * Tags the release -#/ * Builds the release assets and uploads them -#/ -#/ Notes: -#/ * Needs GH_RELEASE_TOKEN and GH_AUTHOR set in the environment. -#/ * Export GH_OWNER and GH_REPO if you want to use a different owner/repo -#/ * Only pull requests labeled with bug, feature or enhancement will show up in the -#/ release page and the changelog. -#/ * If this is a X.Y.0 release, a minimum supported version needs to be supplied too. -#/ -require "json" -require "net/http" -require "time" -require "erb" -require "English" - -API_HOST = ENV["GH_HOST"] || "api.github.com" -API_PORT = 443 -GH_REPO = ENV["GH_REPO"] || "backup-utils" -GH_OWNER = ENV["GH_OWNER"] || "github" -GH_AUTHOR = ENV["GH_AUTHOR"] -DEB_PKG_NAME = "github-backup-utils" -GH_BASE_BRANCH = ENV["GH_BASE_BRANCH"] || "master" # TODO: should we even allow a default or require all params get set explicitly? -GH_STABLE_BRANCH = "" - -# If PUBLISH is false, we leave the release in a draft state to be manually published later through the UI -PUBLISH = ENV["PUBLISH"] == "true" || false - -CHANGELOG_TMPL = "" '<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium - -<%- changes.each do |ch| -%> - * <%= ch.strip.chomp %> -<% end -%> - - -- <%= GH_AUTHOR %> <%= Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S %z") %> - -' "" - -# Override Kernel.warn -def warn(msg) - Kernel.warn msg unless @no_warn -end - -def client(host = API_HOST, port = API_PORT) - @http ||= begin - c = Net::HTTP.new(host, port) - c.use_ssl = true - c - end -end - -def get(path) - req = Net::HTTP::Get.new(path) - req["Authorization"] = "token #{release_token}" - client.request(req) -end - -def post(path, body) - req = Net::HTTP::Post.new(path) - req["Authorization"] = "token #{release_token}" - req.body = body - client.request(req) -end - -def post_file(path, body) - req = Net::HTTP::Post.new(path) - req["Authorization"] = "token #{release_token}" - req["Content-Type"] = path.match?(/.*\.tar\.gz$/) ? "application/tar+gzip" : "application/vnd.debian.binary-package" - req.body = body - client.request(req) -end - -def put(path, body) - req = Net::HTTP::Put.new(path) - req["Authorization"] = "token #{release_token}" - req.body = body - client.request(req) -end - -def patch(path, body) - req = Net::HTTP::Patch.new(path) - req["Authorization"] = "token #{release_token}" - req.body = body - client.request(req) -end - -def release_token - token = ENV["GH_RELEASE_TOKEN"] - raise "GH_RELEASE_TOKEN environment variable not set" if token.nil? - - token -end - -# Create a lightweight tag -def tag(name, sha) - body = { - "ref": "refs/tags/#{name}", - "sha": sha, - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/git/refs", body) - - raise "Creating tag ref failed (#{res.code})" unless res.is_a? Net::HTTPSuccess -end - -def bug_or_feature?(issue_hash) - return true if issue_hash["labels"].find { |label| ["bug", "feature", "enhancement"].include?(label["name"]) } - false -end - -def issue_from(issue) - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/issues/#{issue}") - raise "Issue ##{issue} not found in #{GH_OWNER}/#{GH_REPO}" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def beautify_changes(changes) - out = [] - changes.each do |chg| - next unless chg =~ /#(\d+)/ - begin - issue = issue_from Regexp.last_match(1) - out << "#{issue["title"]} ##{Regexp.last_match(1)}" if bug_or_feature?(issue) - rescue => e - warn "Warning: #{e.message}" - end - end - - out -end - -def changelog - puts "building changelog by comparing origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH}" - changes = `git log --pretty=oneline origin/#{GH_STABLE_BRANCH}...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) - raise "Building the changelog failed" if $CHILD_STATUS != 0 - - changes -end - -def build_changelog(changes, package_name, package_version) - ERB.new(CHANGELOG_TMPL, nil, "-").result(binding) -end - -def update_changelog(changes, name, version, path = "debian/changelog") - raise "debian/changelog not found" unless File.exist?(path) - File.open("#{path}.new", "w") do |f| - f.puts build_changelog changes, name, version - f.puts(File.read(path)) - end - File.rename("#{path}.new", path) -end - -def create_release(tag_name, branch, rel_name, rel_body, draft = true) - body = { - 'tag_name': tag_name, - 'target_commitish': branch, - 'name': rel_name, - 'body': rel_body, - 'draft': draft, - 'prerelease': false, - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/releases", body) - - raise "Failed to create release (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def publish_release(release_id) - body = { - 'draft': false, - }.to_json - res = patch("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}", body) - - raise "Failed to update release (#{res.code})" unless res.is_a? Net::HTTPSuccess -end - -def list_releases - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/releases") - raise "Failed to retrieve releases" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def release_available?(tag_name) - return true if list_releases.find { |r| r["tag_name"] == tag_name } - - false -end - -def bump_version(new_version, min_version = nil, path = "share/github-backup-utils/version") - current_version = Gem::Version.new(File.read(path).strip.chomp) - if !@skip_version_bump_check && (Gem::Version.new(new_version) < current_version) - raise "New version should be newer than #{current_version}" - end - File.open("#{path}.new", "w") { |f| f.puts new_version } - File.rename("#{path}.new", path) - - unless min_version.nil? - content = File.read("bin/ghe-host-check") - new_content = content.gsub(/supported_minimum_version="[0-9]\.[0-9]+\.0"/, "supported_minimum_version=\"#{min_version}\"") - File.open("bin/ghe-host-check", "w") { |file| file.puts new_content } - - content = File.read("test/testlib.sh") - new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/, "GHE_TEST_REMOTE_VERSION:=#{new_version}") - File.open("test/testlib.sh", "w") { |file| file.puts new_content } - end -end - -def push_release_branch(version) - unless (out = `git checkout --quiet -b release-#{version}`) - raise "Creating release branch failed:\n\n#{out}" - end - - unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/test`) - raise "Error committing changelog and version:\n\n#{out}" - end - - unless (out = `git push --quiet origin release-#{version}`) - raise "Failed pushing the release branch:\n\n#{out}" - end -end - -def update_stable_branch - `git checkout --quiet #{GH_STABLE_BRANCH}` - unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`) - warn "Merging #{GH_BASE_BRANCH} into #{GH_STABLE_BRANCH} failed:\n\n#{out}" - end - unless (out = `git push --quiet origin #{GH_STABLE_BRANCH}`) - warn "Failed pushing the #{GH_STABLE_BRANCH} branch:\n\n#{out}" - end -end - -def create_release_pr(version, release_body) - body = { - 'title': "Bump version: #{version}", - 'body': release_body, - 'head': "release-#{version}", - 'base': GH_BASE_BRANCH, - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body) - raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def merge_pr(number, sha, version) - body = { - 'commit_title': "Merge pull request ##{number} from github/release-#{version}", - 'commit_message': "Bump version: #{version}", - 'sha': sha, - 'merge_method': "merge", - }.to_json - pr_mergeable? number - res = put("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}/merge", body) - raise "Merging PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -class RetryError < StandardError -end - -def pr_mergeable?(number) - begin - retries ||= 5 - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}") - raise RetryError if JSON.parse(res.body)["mergeable"].nil? - mergeable = JSON.parse(res.body)["mergeable"] - rescue RetryError - sleep 1 - retry unless (retries -= 1).zero? - raise "PR is unmergable." - end - - mergeable || false -end - -def can_auth? - !ENV["GH_RELEASE_TOKEN"].nil? -end - -def repo_exists? - res = get("/repos/#{GH_OWNER}/#{GH_REPO}") - res.is_a? Net::HTTPSuccess -end - -def can_build_deb? - system("which debuild > /dev/null 2>&1") -end - -def package_tarball - unless (out = `script/package-tarball 2>&1`) - raise "Failed to package tarball:\n\n#{out}" - end - out -end - -def package_deb - unless (out = `DEB_BUILD_OPTIONS=nocheck script/package-deb 2>&1`) - raise "Failed to package Debian package:\n\n#{out}" - end - out -end - -def attach_assets_to_release(upload_url, release_id, files) - @http = nil - client(URI(upload_url.gsub(/{.*}/, "")).host) - begin - files.each do |file| - raw_file = File.open(file).read - res = post_file("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}/assets?name=#{File.basename(file)}", raw_file) - raise "Failed to attach #{file} to release (#{res.code})" unless res.is_a? Net::HTTPSuccess - end - rescue => e - raise e - end - @http = nil -end - -def clean_up(version) - `git checkout --quiet #{GH_BASE_BRANCH}` - `git fetch --quiet origin --prune` - `git pull --quiet origin #{GH_BASE_BRANCH} --prune` - `git branch --quiet -D release-#{version} >/dev/null 2>&1` - `git push --quiet origin :release-#{version} >/dev/null 2>&1` - `git branch --quiet -D tmp-packaging >/dev/null 2>&1` -end - -def is_base_branch_valid?(branch) - if branch == "master" || branch.match(/^\d+\.\d+-main$/) - return true - else - return false - end -end - -def get_stable_branch_name(branch) - ## derive the proper stable branch. if the base branch is "master" the stable branch is just "stable" - ## if the base branch is a release branch, the stable branch will be "x.y-stable" - result = "" - if branch == "master" - result = "stable" - else - result = branch.gsub(/-main$/, "-stable") - end - - result -end - -#### All the action starts #### -if $PROGRAM_NAME == __FILE__ - begin - ## validate base branch. this must either be "master" or a release branch which will match the pattern "x.y-main" - raise "The branch #{GH_BASE_BRANCH} is not valid for releasing backup-utils. branch name must be master or match x.y-main" if !is_base_branch_valid?(GH_BASE_BRANCH) - - GH_STABLE_BRANCH = get_stable_branch_name(GH_BASE_BRANCH) - - puts "base branch = " + GH_BASE_BRANCH - puts "stable branch = " + GH_STABLE_BRANCH - - args = ARGV.dup - dry_run = false - skip_version_bump_check = false - if args.include?("--dry-run") - dry_run = true - args.delete "--dry-run" - end - - if args.include?("--no-warn") - @no_warn = true - args.delete "--no-warn" - end - - if args.include?("--skip-version-bump-check") - @skip_version_bump_check = true - args.delete "--skip-version-bump-check" - end - - raise "Usage: release [--dry-run] [--skip-version-bump-check] [min_version]" if args.empty? - - begin - version = Gem::Version.new(args[0]) - min_version = args[1] ? args[1] : nil - rescue ArgumentError - raise "Error parsing version #{args[0]}" - end - - raise "Minimum supported version is required for X.Y.0 releases\n\nUsage: release [--dry-run] [min_version]" if /[0-9]\.[0-9]+\.0/ =~ version.to_s && min_version.nil? - - raise "The repo #{GH_REPO} does not exist for #{GH_OWNER}" unless repo_exists? - - raise "GH_AUTHOR environment variable is not set" if GH_AUTHOR.nil? - - release_changes = [] - release_changes = beautify_changes changelog if can_auth? - release_a = false - release_a = release_available? "v#{version}" - - puts "Bumping version to #{version}..." - bump_version version, min_version - - if dry_run - puts "Existing release?: #{release_a}" - puts "New version: #{version}" - puts "Min version: #{min_version}" unless min_version.nil? - puts "Owner: #{GH_OWNER}" - puts "Repo: #{GH_REPO}" - puts "Author: #{GH_AUTHOR}" - puts "Token: #{ENV["GH_RELEASE_TOKEN"] && "set" || "unset"}" - puts "Base branch: #{GH_BASE_BRANCH}" - puts "Changelog:" - if release_changes.empty? - puts " => No new bug fixes, enhancements or features." - else - release_changes.each { |c| puts " * #{c}" } - end - puts "Changes:" - puts `git diff --color` - `git checkout -- share/github-backup-utils/version` - `git checkout -- bin/ghe-host-check` - `git checkout -- test/testlib.sh` - exit - end - - raise 'Unable to build Debian pkg: "debuild" not found.' unless can_build_deb? - - raise "Release #{version} already exists." if release_a - - `git fetch --quiet origin --prune` - branches = `git branch --all | grep release-#{version}$` - unless branches.empty? - out = "Release branch release-#{version} already exists. " - out += "Branches found:" - branches.each_line { |l| out += "\n* #{l.strip.chomp}" } - raise out - end - - puts "Updating changelog..." - update_changelog release_changes, DEB_PKG_NAME, version - release_body = "Includes general improvements & bug fixes" - release_body += " and support for GitHub Enterprise Server v#{version}" unless min_version.nil? - release_changes.each do |c| - release_body += "\n* #{c}" - end - - puts "Pushing release branch and creating release PR..." - push_release_branch version - res = create_release_pr(version, "#{release_body}\n\n/cc @github/backup-utils") - - puts "Merging release PR..." - res = merge_pr res["number"], res["head"]["sha"], version - - puts "Tagging and publishing release..." - tag "v#{version}", res["sha"] - - puts "Creating release..." - release_title = "GitHub Enterprise Server Backup Utilities v#{version}" - res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true - - # Tidy up before building tarball and deb pkg - clean_up version - - puts "Building release tarball..." - package_tarball - - puts "Building Debian pkg..." - package_deb - - puts "Attaching Debian pkg and tarball to release..." - base_dir = File.expand_path(File.join(File.dirname(__FILE__), "..")) - attach_assets_to_release res["upload_url"], res["id"], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] - attach_assets_to_release res["upload_url"], res["id"], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - - if PUBLISH - puts "Publishing release..." - publish_release res["id"] - end - - puts "Cleaning up..." - clean_up version - - puts "Updating #{GH_STABLE_BRANCH} branch..." - update_stable_branch - - if !PUBLISH - puts 'Release left in a "Draft" state. Go to the https://github.com/github/backup-utils/releases and publish when ready.' - end - - puts "Released!" - rescue RuntimeError => e - $stderr.puts "Error: #{e}" - exit 1 - end -end diff --git a/script/test b/script/test deleted file mode 100755 index d30bfa8df..000000000 --- a/script/test +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/test -set -e - -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -TMPDIR="$ROOTDIR/test/tmp" - -# Remove possible remnants of previous test runs -rm -rf "${TMPDIR:?}/*" - -print_test_results() { - if [ -n "$GITHUB_STEP_SUMMARY" ]; then - echo -e "### Test results\n" >> "$GITHUB_STEP_SUMMARY" - echo "| Test suite | Result | Successful | Failed | Skipped | Duration |" >> "$GITHUB_STEP_SUMMARY" - echo "|---|---|--:|--:|--:|--:|" >> "$GITHUB_STEP_SUMMARY" - sort -V "$TMPDIR/results" >> "$GITHUB_STEP_SUMMARY" - fi -} - -# Enable verbose logging of ssh commands -export GHE_VERBOSE_SSH=true - -if ! find test -name "test-*.sh" -print0 | sort -z |xargs -0 -n 1 /bin/bash; then - print_test_results - exit 1 -fi - -print_test_results diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh deleted file mode 100755 index 561bcfffb..000000000 --- a/share/github-backup-utils/bm.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# bm.sh: benchmarking Bash code blocks -# -# Example: -# bm_start "wget request" -# wget --quiet https://www.google.com -# bm_end "wget request" -# -# Sample output: -# $ bash test.sh -# wget request took 2s - -bm_desc_to_varname(){ - echo "__bm$(echo "$@" | tr -cd '[[:alnum:]]')" -} - -bm_start() -{ - eval "$(bm_desc_to_varname "$@")_start=$(date +%s)" - if [ -n "$GHE_DEBUG" ]; then - echo "Debug: $1 (bm_start)" - fi - bm_init > /dev/null -} - -bm_init() { - if [ -n "$BM_FILE_PATH" ]; then - echo $BM_FILE_PATH - return - fi - - local logfile="benchmark.${BM_TIMESTAMP:-$(date +"%Y%m%dT%H%M%S")}.log" - if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - export BM_FILE_PATH=$GHE_RESTORE_SNAPSHOT_PATH/benchmarks/$logfile - else - export BM_FILE_PATH=$GHE_SNAPSHOT_DIR/benchmarks/$logfile - fi - - mkdir -p "$(dirname $BM_FILE_PATH)" - echo $BM_FILE_PATH -} - -bm_end() { - if [ -z "$BM_FILE_PATH" ]; then - echo "Call bm_start first" >&2 - exit 1 - fi - - local tend tstart total - tend=$(date +%s) - tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start") - - total=$(($tend - $tstart)) - - echo "$1 took ${total}s" >> $BM_FILE_PATH - # also log timing information in the verbose log - echo "$1 took ${total}s" 1>&3 - if [ -n "$GHE_DEBUG" ]; then - echo "Debug: $1 took ${total}s (bm_end)" - fi - # track progress - progress "$1 took ${total}s" -} diff --git a/share/github-backup-utils/ghe-backup-actions b/share/github-backup-utils/ghe-backup-actions deleted file mode 100755 index b3f51741e..000000000 --- a/share/github-backup-utils/ghe-backup-actions +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-actions -#/ Take an online, incremental snapshot of all Actions data (excluding -#/ what is stored in MSSQL) -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") -backup_dir="$GHE_SNAPSHOT_DIR/actions" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/actions" ] && [ "$(ls -A $GHE_DATA_DIR/current/actions)" ]; then - link_dest="--link-dest=$GHE_DATA_DIR/current/actions" -fi - -# Transfer all Actions data from the user data directory using rsync. -ghe_verbose "* Transferring Actions files from $host ..." -log_rsync "BEGIN: actions rsync" 1>&3 -ghe-rsync -av \ - -e "ghe-ssh -p $port" \ - --rsync-path='sudo -u actions rsync' \ - --exclude "mutexes" --exclude "dumps" --exclude "tmp" \ - $link_dest \ - "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" \ - "$GHE_SNAPSHOT_DIR/actions" 1>&3 -log_rsync "END: actions rsync" 1>&3 - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config deleted file mode 100755 index 30062a8f6..000000000 --- a/share/github-backup-utils/ghe-backup-config +++ /dev/null @@ -1,708 +0,0 @@ -#!/usr/bin/env bash -# Usage: . ghe-backup-config -# GitHub Enterprise backup shell configuration. -# -# This file is sourced by the various utilities under bin and share/github-backup-utils to -# load in backup configuration and ensure things are configured properly. -# -# All commands in share/github-backup-utils/ should start with the following: -# -# . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config -# -# And all commands in bin/ should start with the following: -# -# . $( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config -# -set +o posix -# Terminal colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - - -# Assume this script lives in share/github-backup-utils/ when setting the root -GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# Get the version from the version file. -BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" - -# If a version check was requested, show the current version and exit -if [ -n "$GHE_SHOW_VERSION" ]; then - echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" - exit 0 -fi - -# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's -print_usage() { - grep '^#/' <"$0" | cut -c 4- - exit "${1:-1}" -} - -if [ -n "$GHE_SHOW_HELP" ]; then - print_usage -else - for a in "$@"; do - if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then - print_usage - fi - done -fi - -# Save off GHE_HOSTNAME from the environment since we want it to override the -# backup.config value when set. -GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" - -# Source in the backup config file from the copy specified in the environment -# first and then fall back to the backup-utils root, home directory and system. -config_found=false -for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \ - "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do - if [ -f "$f" ]; then - GHE_BACKUP_CONFIG="$f" - # shellcheck disable=SC1090 # This is a user-supplied value that can't be predicted - . "$GHE_BACKUP_CONFIG" - config_found=true - break - fi -done - -GHE_RESTORE_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-restore") -GHE_BACKUP_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-backup") - -export GHE_RESTORE_IN_PROGRESS -export GHE_BACKUP_IN_PROGRESS - -# Logging display and formatting functions - -log_level() { - local level=$1 - shift - local message=$* - local display="" - local timestamp - timestamp=$(date -u "+%FT%TZ") - - - if [ "$TERM" = "dumb" ] || [[ "$OUTPUT_COLOR" != "yes" ]]; then - if [ "$level" = "info" ]; then - display="INFO" - elif [ "$level" = "warn" ]; then - display="WARN" - elif [ "$level" = "error" ]; then - display="ERROR" - elif [ "$level" = "verbose" ]; then - display="INFO" - elif [ "$level" = "rsync" ]; then - display="RSYNC" - elif [ "$level" = "ssh" ]; then - display="SSH" - else - display="-" - fi - else - if [ "$level" = "info" ]; then - display="${GREEN}INFO${NC}" - elif [ "$level" = "warn" ]; then - display="${YELLOW}WARN${NC}" - elif [ "$level" = "error" ]; then - display="${RED}ERROR${NC}" - elif [ "$level" = "verbose" ]; then - display="${GREEN}INFO${NC}" - elif [ "$level" = "rsync" ]; then - display="${GREEN}RSYNC${NC}" - elif [ "$level" = "ssh" ]; then - display="${GREEN}SSH${NC}" - else - display="-" - fi - fi - echo -e "$timestamp $display $message" -} - -log_info(){ - log_level "info" "$1" -} - -log_warn(){ - log_level "warn" "$1" -} - -log_error(){ - log_level "error" "$1" -} - -log_verbose(){ - log_level "verbose" "$1" -} - -log_rsync(){ - log_level "rsync" "$1" -} - -log_ssh(){ - log_level "ssh" "$1" -} - -# Add the bin and share/github-backup-utils dirs to PATH -PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" -# shellcheck source=share/github-backup-utils/bm.sh -. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" -# shellcheck source=share/github-backup-utils/ghe-incremental-backup-restore -. "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-incremental-backup-restore" -# shellcheck source=share/github-backup-utils/track-progress -. "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress" - - -ghe_restore_check() { - if [ -h "$GHE_RESTORE_IN_PROGRESS" ]; then - echo " Error: detected a restore already in progress from a previous version of ghe-restore." 1>&2 - echo " If there is no restore in progress anymore, please remove" 1>&2 - echo " the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi - - if [ -f "$GHE_RESTORE_IN_PROGRESS" ]; then - progress=$(cat "$GHE_RESTORE_IN_PROGRESS") - pid=$(echo "$progress" | cut -d ' ' -f 2) - echo " Error: A restore of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo " If PID $pid is not a process related to the restore utilities, please remove" 1>&2 - echo " the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi -} - -ghe_backup_check() { - if [ -h "$GHE_BACKUP_IN_PROGRESS" ]; then - echo " Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2 - echo " If there is no backup in progress anymore, please remove" 1>&2 - echo " the $GHE_DATA_DIR/$GHE_BACKUP_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi - - if [ -f "$GHE_BACKUP_IN_PROGRESS" ]; then - progress=$(cat "$GHE_BACKUP_IN_PROGRESS") - pid=$(echo "$progress" | cut -d ' ' -f 2) - echo " Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo " If PID $pid is not a process related to the backup utilities, please remove" 1>&2 - echo " the $GHE_BACKUP_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi -} - -ghe_restore_finished() { - if [ -f "$GHE_RESTORE_IN_PROGRESS" ]; then - rm -f "$GHE_RESTORE_IN_PROGRESS" - fi -} - -ghe_backup_finished() { - if [ -f "$GHE_BACKUP_IN_PROGRESS" ]; then - rm -f "$GHE_BACKUP_IN_PROGRESS" - fi -} - -ghe_parallel_check() { - GHE_PARALLEL_COMMAND_OPTIONS=() - GHE_PARALLEL_RSYNC_COMMAND_OPTIONS=() - - if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then - return 0 - fi - - # Some machines may have both moreutils parallel and GNU parallel installed. - # Check some variants to find it - GHE_PARALLEL_COMMAND="parallel" - local x - for x in \ - /usr/bin/parallel-moreutils \ - /usr/bin/parallel.moreutils \ - /usr/bin/parallel_moreutils \ - /usr/bin/moreutils-parallel \ - /usr/bin/moreutils.parallel \ - /usr/bin/moreutils_parallel \ - ; do - if [ -x "${x}" ]; then - GHE_PARALLEL_COMMAND="${x}" - break - fi - done - - # Check that the GHE_PARALLEL_COMMAND is pointing to moreutils parallel - if ! "$GHE_PARALLEL_COMMAND" -h | grep -q "parallel \[OPTIONS\] command -- arguments"; then - echo "Error: moreutils not found. Please install https://joeyh.name/code/moreutils" 1>&2 - exit 1 - fi - - if [ -n "$GHE_PARALLEL_MAX_JOBS" ]; then - GHE_PARALLEL_COMMAND_OPTIONS+=(-j "$GHE_PARALLEL_MAX_JOBS") - # Default to the number of max rsync jobs to the same as GHE_PARALLEL_MAX_JOBS, if not set. - # This is only applicable to ghe-restore-repositories currently. - : "${GHE_PARALLEL_RSYNC_MAX_JOBS:="$GHE_PARALLEL_MAX_JOBS"}" - fi - - if [ -n "$GHE_PARALLEL_RSYNC_MAX_JOBS" ]; then - GHE_PARALLEL_RSYNC_COMMAND_OPTIONS+=(-j "$GHE_PARALLEL_RSYNC_MAX_JOBS") - fi - - if [ -n "$GHE_PARALLEL_MAX_LOAD" ]; then - GHE_PARALLEL_COMMAND_OPTIONS+=(-l "$GHE_PARALLEL_MAX_LOAD") - GHE_PARALLEL_RSYNC_COMMAND_OPTIONS+=(-l "$GHE_PARALLEL_MAX_LOAD") - fi -} - -# Check that the config file exists before we source it in. -if ! $config_found; then - echo "Error: No backup configuration file found. Tried:" 1>&2 - [ -n "$GHE_BACKUP_CONFIG" ] && echo " - $GHE_BACKUP_CONFIG" 1>&2 - echo " - $GHE_BACKUP_ROOT/backup.config" 1>&2 - echo " - $HOME/.github-backup-utils/backup.config" 1>&2 - echo " - /etc/github-backup-utils/backup.config" 1>&2 - exit 2 -fi - -# If verbose logging is enabled, redirect fd 3 to stdout or the specified log file; -# otherwise, redirect it to /dev/null. Write verbose output to fd 3. -if [ -n "$GHE_VERBOSE" ]; then - if [ -n "$GHE_VERBOSE_LOG" ]; then - if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then - exec 3>>"$GHE_VERBOSE_LOG" - else - calling_script_name="$(caller | sed 's:.*/::')" - if [ "$TERM" = "dumb" ] || [[ "$OUTPUT_COLOR" != "yes" ]]; then - exec 3>>"$GHE_VERBOSE_LOG" - log_info "$calling_script_name $*" 1>&3 - else - # colorize the input if supported. - display_caller="${BLUE}$calling_script_name${NC}" - exec 3>>"$GHE_VERBOSE_LOG" - log_info "$display_caller $*" 1>&3 - fi - fi - else - exec 3>&1 - fi -else - exec 3>/dev/null -fi - - -# Restore saved off hostname. -[ -n "$GHE_HOSTNAME_PRESERVE" ] && GHE_HOSTNAME="$GHE_HOSTNAME_PRESERVE" - -# Check that the GHE hostname is set. -if [ -z "$GHE_HOSTNAME" ]; then - echo "Error: GHE_HOSTNAME not set in config file." 1>&2 - exit 2 -fi - -# Check that the GHE data directory is set. -if [ -z "$GHE_DATA_DIR" ]; then - echo "Error: GHE_DATA_DIR not set in config file." 1>&2 - exit 2 -fi - -# Convert the data directory path to an absolute path, basing any relative -# paths on the backup-utils root, and use readlink to canonicalize the path. -if [ "${GHE_DATA_DIR:0:1}" != "/" ]; then - GHE_DATA_DIR="$(cd "$GHE_BACKUP_ROOT" && readlink -m "$GHE_DATA_DIR")" -fi -export GHE_DATA_DIR - -# Assign the Release File path if it hasn't been provided (eg: by test suite) -: "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" - -# Check that utils are not being run directly on GHE appliance. -if [ -f "$GHE_RELEASE_FILE" ]; then - echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 - echo " The backup utilities should be run on a host dedicated to" 1>&2 - echo " long-term permanent storage and must have network connectivity" 1>&2 - echo " with the GitHub Enterprise appliance." 1>&2 - exit 1 -fi - -GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} - -# Check that the data directory is set and create it if it doesn't exist. -if [ ! -d "$GHE_DATA_DIR" ] && [ "$GHE_CREATE_DATA_DIR" = "yes" ]; then - echo "Creating the backup data directory ..." 1>&3 - mkdir -p "$GHE_DATA_DIR" -fi - -if [ ! -d "$GHE_DATA_DIR" ]; then - echo "Error: GHE_DATA_DIR $GHE_DATA_DIR does not exist." >&2 - exit 8 -fi - -# Set some defaults if needed. -: "${GHE_NUM_SNAPSHOTS:=10}" - -# Generate a backup timestamp if one has not already been generated. -# We export the variable so the process group shares the same value. -: "${GHE_SNAPSHOT_TIMESTAMP:=$(date +"%Y%m%dT%H%M%S")}" -export GHE_SNAPSHOT_TIMESTAMP - -# Set the current snapshot directory to /. This is where -# all backups should be written for the current invocation. -GHE_SNAPSHOT_DIR="$GHE_DATA_DIR"/"$GHE_SNAPSHOT_TIMESTAMP" -export GHE_SNAPSHOT_DIR - -# The root filesystem location. This must be used so that tests can override -# the root as a local directory location. -: "${GHE_REMOTE_ROOT_DIR:=""}" - -# The root location of persistent data and applications on the remote side. This -# is always "/data" for GitHub instances. Use of this variable allows -# the location to be overridden in tests. -: "${GHE_REMOTE_DATA_DIR:="/data"}" - -# The root location of user data stores such as git repositories, pages sites, -# elasticsearch indices, etc. This is "/data" under 1.x filesystem layouts and -# "/data/user" under the 2.x filesystem layout. The location is adjusted -# dynamically in ghe_remote_version_config() immediately after obtaining the -# remote version. Utilities that transfer data in and out of the appliance -# should use this variable to ensure proper behavior under different versions. -: "${GHE_REMOTE_DATA_USER_DIR:="$GHE_REMOTE_DATA_DIR"}" - -# The location of the license file on the remote side. This is always -# "/data/enterprise/enterprise.ghl" for GitHub instances. Use of this variable -# allows the location to be overridden in tests. -: "${GHE_REMOTE_LICENSE_FILE:="$GHE_REMOTE_DATA_DIR/enterprise/enterprise.ghl"}" - -# The number of seconds to wait for in progress git-gc processes to complete -# before starting the sync of git data. See share/github-backup-utils/ghe-backup-repositories-rsync -# for more information. Default: 10 minutes. -: "${GHE_GIT_COOLDOWN_PERIOD:=600}" - -# Set "true" to get verbose logging of all ssh commands on stderr -: "${GHE_VERBOSE_SSH:=false}" - -# The location of the cluster configuration file on the remote side. -# This is always "/data/user/common/cluster.conf" for GitHub Cluster instances. -# Use of this variable allows the location to be overridden in tests. -: "${GHE_REMOTE_CLUSTER_CONF_FILE:="$GHE_REMOTE_DATA_DIR/user/common/cluster.conf"}" - -# The location of the file used to disable GC operations on the remote side. -: "${SYNC_IN_PROGRESS_FILE:="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress"}" - -# Base path for temporary directories and files. -: "${TMPDIR:="/tmp"}" - -# Backup cadence for MS SQL. Determines the kind of backup taken, either full, differential, -# or transaction log, based on when the last backup of that kind was taken. This defaults to -# taking a full backup once a week, a differential backup once a day, and transaction logs every -# 15 minutes. -: "${GHE_MSSQL_BACKUP_CADENCE:=10080,1440,15}" - -############################################################################### -### Dynamic remote version config - -# Adjusts remote paths based on the version of the remote appliance. This is -# called immediately after the remote version is obtained by -# ghe_remote_version_required(). Child processes inherit the values set here. -ghe_remote_version_config() { - GHE_REMOTE_DATA_USER_DIR="$GHE_REMOTE_DATA_DIR/user" - export GHE_REMOTE_DATA_DIR GHE_REMOTE_DATA_USER_DIR - export GHE_REMOTE_LICENSE_FILE -} - -############################################################################### -### Utility functions - -# Run ghe-host-check and establish the version of the remote GitHub instance in -# the exported GHE_REMOTE_VERSION variable. If the remote version has already -# been established then don't perform the host check again. Utilities in share/github-backup-utils -# that need the remote version should use this function instead of calling -# ghe-host-check directly to reduce ssh roundtrips. The top-level ghe-backup and -# ghe-restore commands establish the version for all subcommands. -# shellcheck disable=SC2120 # Our arguments are optional and not meant to be the owning script's -ghe_remote_version_required() { - if [ -z "$GHE_REMOTE_VERSION" ]; then - _out=$(ghe-host-check "$@") - echo "$_out" - _out_hostname=$(echo "$_out" | tail -n 1) - - # override hostname w/ ghe-host-check output because the port could have - # been autodetected to 122. - GHE_HOSTNAME="${_out_hostname/Connect /}" - GHE_HOSTNAME="${GHE_HOSTNAME/ OK*/}" - export GHE_HOSTNAME - - GHE_REMOTE_VERSION="${_out_hostname#*\(}" - GHE_REMOTE_VERSION="${GHE_REMOTE_VERSION%%\)*}" - export GHE_REMOTE_VERSION - - ghe_parse_remote_version "$GHE_REMOTE_VERSION" - ghe_remote_version_config "$GHE_REMOTE_VERSION" - fi - true -} - -# Parse a version string into major, minor and patch parts and echo. -ghe_parse_version() { - local version_major version_minor version_patch - - IFS=. read -r version_major version_minor version_patch _ <<<"${1#v}" - version_patch=${version_patch%%[a-zA-Z]*} - - echo "$version_major $version_minor $version_patch" -} -# Parse major, minor, and patch parts of the remote appliance version and store -# in GHE_VERSION_MAJOR, GHE_VERSION_MINOR, and GHE_VERSION_PATCH variables. All -# parts are numeric. This is called automatically from -# ghe_remote_version_required so shouldn't be used directly. -# -# Scripts use these variables to alter behavior based on what's supported on the -# appliance version. -ghe_parse_remote_version() { - # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH < <(ghe_parse_version "$1") - export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH -} - -# In 3.11 we started to install 2 different version parallel(s) -# moreutils parallel and GNU parallel. When gnu parallel is installed, -# it renames moreutils parallel to parallel.moreutils -# set $PARALLEL_CMD envvar to be used in place of parallel commands -ghe_remote_parallel() { - if [ -z "$GHE_REMOTE_VERSION" ]; then - echo "Error: ghe_remote_version_required needs to be invoked before ghe_remote_parallel" 1>&2 - exit 1 - fi - - if [ "$GHE_VERSION_MINOR" -lt 11 ]; then - PARALLEL_CMD="parallel" - else - PARALLEL_CMD="parallel.moreutils" - fi - export PARALLEL_CMD -} - -# Parses the part out of a ":" or just "" string. -# This is used primarily to break hostspecs with non-standard ports down for -# rsync commands. -ssh_host_part() { - [ "${1##*:}" = "$1" ] && echo "$1" || echo "${1%:*}" -} - -# Parses the part out of a ":" or just "" string. -# This is used primarily to break hostspecs with non-standard ports down for -# rsync commands. -ssh_port_part() { - if [ "${1##*:}" != "$1" ] && [ "${1##*:}" -ne "122" ]; then - echo "Error: SSH port has to be 122 connecting to GitHub Enterprise Server, current value is ${1##*:} for $1." 1>&2 - exit 1 - fi - - echo 122 -} - -# Usage: ghe_remote_logger ... -# Log a message to /var/log/syslog on the remote instance. -# Note: Use sparingly. Remote logging requires an ssh connection per invocation. -ghe_remote_logger() { - echo "$@" | - ghe-ssh "$GHE_HOSTNAME" -- logger -t backup-utils || true -} - -# Usage: ghe_verbose -# Log if verbose mode is enabled (GHE_VERBOSE or `-v`). -ghe_verbose() { - if [ -n "$GHE_VERBOSE" ]; then - log_verbose "$@" 1>&3 - fi -} - -# Usage: ghe_debug OR echo | ghe_debug -# Log if debug mode is enabled (GHE_DEBUG). -ghe_debug() { - [ -z "$GHE_DEBUG" ] && return - - if [ $# -ne 0 ]; then - echo -e "Debug: $*" 1>&3 - elif [ -p /dev/stdin ]; then - echo -e "\n" 1>&3 - while read -r line; do - echo -e "Debug: $line" 1>&3 - done /dev/null || true -} - -# The list of gists returned by the source changed in 2.16.23, 2.17.14, -# 2.18.8, and 2.19.3. We need to account for this difference here. -# In older versions, all paths need to be truncated with `dirname`. -# In newer versions, gist paths are unmodified, and only other repo types -# are truncated with `dirname`. -fix_paths_for_ghe_version() { - if [[ "$GHE_REMOTE_VERSION" =~ 2.16. && "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.16.23)" ]] || - [[ "$GHE_REMOTE_VERSION" =~ 2.17. && "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.17.14)" ]] || - [[ "$GHE_REMOTE_VERSION" =~ 2.18. && "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.18.8)" ]] || - [[ "$(version "$GHE_REMOTE_VERSION")" -ge "$(version 2.19.3)" ]]; then - GIST_FILTER=(-e "/gist/b") - else - unset GIST_FILTER - fi - - # This sed expression is equivalent to running `dirname` on each line, - # but without all the fork+exec overhead of calling `dirname` that many - # times: - # 1. strip off trailing slashes - # 2. if the result has no slashes in it, the dirname is "." - # 3. truncate from the final slash (if any) to the end - # If the GIST_FILTER was set above (because we're on a modern version of - # GHES), then don't modify lines with "gist" in them. - sed "${GIST_FILTER[@]}" -e 's/\/$//; s/^[^\/]*$/./; s/\/[^\/]*$//' -} - -is_binary_backup_feature_on() { - ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary" -} - -# Check if the backup is binary by looking up the sentinel file -is_binary_backup() { - test -f "$1/mysql-binary-backup-sentinel" -} - -# Check if a given service is managed externally on the appliance or in a snapshot. -# Usage: is_service_external $service [$config_file] -# Pass in the config file to check if the service is managed externally in the snapshot. -is_service_external(){ - service=$1 - config_file=$2 - case $service in - "mysql") - if [ -n "$config_file" ]; then - enabled=$(GIT_CONFIG="$config_file" git config mysql.external.enabled) - [ "$enabled" == "true" ]; - else - ghe-ssh "$GHE_HOSTNAME" -- ghe-config --true "mysql.external.enabled" - fi - ;; - *) - return 1 - ;; - esac -} - -is_instance_configured(){ - ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/configured' ]" -} - -# Helper method that returns true if: -# - the target appliance uses the internal MySQL database (aka NOT BYODB), and -# - the snapshot being restored is from an appliance using an external MySQL database (BYODB) -external_database_snapshot_to_internal_database(){ - ! is_external_database_target && is_external_database_snapshot -} - -# Helper method that returns true if: -# - the target appliance uses an external MySQL database (BYODB), and -# - the snapshot being restored is from an appliance using an internal MySQL database (aka NOT BYODB) -internal_database_snapshot_to_external_database(){ - is_external_database_target && ! is_external_database_snapshot -} - -is_external_database_target_or_snapshot(){ - # If restoring settings, only check if the snapshot being restored was from an appliance with external DB configured. - if $RESTORE_SETTINGS; then - is_external_database_snapshot - else - # Check if restoring a snapshot with an external database configured, or restoring - # to an appliance with an external database configured. - is_external_database_snapshot || is_external_database_target - fi -} - -is_external_database_target(){ - is_service_external "mysql" -} - -is_external_database_snapshot(){ - is_service_external "mysql" "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/settings.json" -} - -# This file exists if this is a backup for an external database AND the backup was -# taken via our logical backup strategy. -is_default_external_database_snapshot(){ - is_external_database_snapshot && test -f "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/logical-external-database-backup-sentinel" -} - -prompt_for_confirmation(){ - echo "$1" - printf "Type 'yes' to continue: " - - while read -r response; do - case $response in - yes|Yes|YES) - break - ;; - '') - printf "Type 'yes' to continue: " - ;; - *) - echo "Restore aborted." 1>&2 - exit 1 - ;; - esac - done - - echo -} - -#initialize progress tracking by clearing out the temp files used to track -init-progress() { - - if [ -e /tmp/backup-utils-progress ]; then - rm -rf /tmp/backup-utils-progress/* - fi - # shellcheck disable=SC2174 # We are fine with -m only affecting the deepest directory - mkdir -m 777 -p /tmp/backup-utils-progress - touch /tmp/backup-utils-progress/{total,type,progress,info} -} - - -#increase total count of progress -increment-progress-total-count() { - ((PROGRESS_TOTAL += $1)) - echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress/total -} - -## -# This function is used by ghe-gc-disable, ghe-backup-repositories, and ghe-backup-storage -# This function should be used directly to disable and drain GC operations ONLY on HA-replica node -# (as done in ghe-backup-repositories and ghe-backup-storage) -# Otherwise use ghe-gc-disable which will call this function with the correct parameters. -# -# Arguments: -# $1 - path to sync-in-progress file ($SYNC_IN_PROGRESS_FILE) -# $2 - git cooldown period ($GHE_GIT_COOLDOWN_PERIOD) -## -gc_disable() { - set -e - local sync_in_progress="$1" - local git_cooldown_period="$2" - - # Touch the sync-in-progress file, disabling GC operations, and wait for all - # active GC processes to finish on the remote side. - sudo -u git touch "$sync_in_progress" - for _ in $(seq $git_cooldown_period); do - # note: the bracket synta[x] below is to prevent matches against the - # grep process itself. - if ps axo args | grep -E -e "^git( -.*)? nw-repac[k]( |$)" -e "^git( -.*)? g[c]( |$)" >/dev/null; then - sleep 1 - else - exit 0 - fi - done - exit 7 -} diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log deleted file mode 100755 index 84da8953a..000000000 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-es-audit-log -#/ Take a backup of audit logs in Elasticsearch. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root elastic backup directory based on config -host="$GHE_HOSTNAME" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Make sure root backup dir exists if this is the first run -mkdir -p "$GHE_SNAPSHOT_DIR/audit-log" - -if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:9201/_cat/indices/audit_log*?h=index,pri.store.size&bytes=b\""); then - log_error "ghe-backup-es-audit-log: Failed to retrieve audit log indices." 1>&2 - exit 1 -fi - -# Exit if no indices were found -[ -z "$indices" ] && exit - -# Determine if the audit log migration has occurred or is needed. -if echo 'set -o pipefail; ! test -e /data/user/common/es-scan-complete && test -f /usr/local/share/enterprise/run-audit-log-transitions.sh' | ghe-ssh "$host" /bin/bash; then - if echo 'set -o pipefail; echo n | /usr/local/share/enterprise/run-audit-log-transitions.sh > /dev/null 2>&1 && touch /data/user/common/es-scan-complete' | ghe-ssh "$host" /bin/bash; then - touch $GHE_SNAPSHOT_DIR/es-scan-complete - fi -fi - -IFS=$'\n' -for index in $indices; do - IFS=' ' - set $index - index_name=$1 - index_size=$2 - - if [[ -f $GHE_DATA_DIR/current/audit-log/$index_name.gz && $(cat $GHE_DATA_DIR/current/audit-log/$index_name.gz.size 2>/dev/null || true) -eq $index_size ]]; then - ghe_verbose "* Linking unchanged audit log index: $index_name" - # Hard link any indices that have not changed since the last backup - ln $GHE_DATA_DIR/current/audit-log/$index_name.gz $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz - ln $GHE_DATA_DIR/current/audit-log/$index_name.gz.size $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size - else - ghe_verbose "* Performing audit log export for index: $index_name" - echo "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name/" | gzip" | ghe-ssh "$host" -- /bin/bash > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz - echo $index_size > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size - fi -done - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync deleted file mode 100755 index fce7043bb..000000000 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-es-rsync -#/ Take an online, incremental snapshot of Elasticsearch indices. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup when the rsync strategy is used. -# shellcheck disable=SC2086 -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root elastic backup directory based on config -host="$GHE_HOSTNAME" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch" - -# Verify that the /data/elasticsearch directory exists. -if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then - ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." - exit 0 -fi - -# If we have a previous increment, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then - link_dest="--link-dest=../../current/elasticsearch" -fi - -# Transfer ES indices from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. -ghe_verbose "* Performing initial sync of ES indices ..." -log_rsync "BEGIN elasticsearch rsync" 1>&3 -ghe-rsync -av \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path="sudo -u elasticsearch rsync" \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ - "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 -log_rsync "END elasticsearch rsync" 1>&3 -# Set up a trap to re-enable flushing on exit and remove temp file -cleanup () { - ghe_verbose "* Enabling ES index flushing ..." - echo '{"index":{"translog.flush_threshold_size":"512MB"}}' | - ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null -} -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - -# Disable ES flushing and force a flush right now -ghe_verbose "* Disabling ES index flushing ..." -echo '{"index":{"translog.flush_threshold_size":"1PB"}}' | -ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null -ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null - -# Transfer all ES indices again -ghe_verbose "* Performing follow-up sync of ES indices ..." -log_rsync "BEGIN: elasticsearch followup rsync" 1>&3 -ghe-rsync -av \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path="sudo -u elasticsearch rsync" \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ - "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 -log_rsync "END: elasticsearch followup rsync" 1>&3 - -# "Backup" audit log migration sentinel file -if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then - touch $GHE_SNAPSHOT_DIR/es-scan-complete -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-fsck b/share/github-backup-utils/ghe-backup-fsck deleted file mode 100755 index d255e036f..000000000 --- a/share/github-backup-utils/ghe-backup-fsck +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-fsck [--print-nwo] -#/ -#/ Run git fsck on backed up repositories. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -echo "Running git fsck on repos..." - -# Verify git is available. -if ! git --version 1>/dev/null 2>&1; then - log_error "git not found." 1>&2 - exit 1 -fi - -sdir=$1 -repos=0 -errors=0 -log=$(mktemp -t ghe-backup-fsck-XXXXXX) -t_start=$(date +%s) -if git fsck -h | grep -q '\-\-dangling'; then - git_cmd='git fsck --no-dangling' -else - log_warn "ghe-backup-fsck: old git version, --no-dangling not available" 1>&3 - git_cmd='git fsck' -fi - -if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then - print_usage -fi - -if [ ! -d "$sdir/repositories" ]; then - log_error "ghe-backup-fsck: $sdir is not a valid snapshot." >&2 - exit 1 -fi - -# shellcheck disable=SC2044 # Snapshot and repository directory names are safe for find iteration. -for repo in $(find $sdir/repositories/ -type d -name \*.git); do - repos=$(($repos+1)) - before_time=$(date +%s) - - status=$( - set -e - - cd $repo - - nwo="-" - if [ "$2" = "--print-nwo" ] && [ -f info/nwo ]; then - nwo="$(cat info/nwo)" - fi - - if [ ! -f objects/info/alternates ] || grep -q '^\.\.' objects/info/alternates; then - $git_cmd >$log 2>&1 && { - echo "OK $repo $nwo"; exit - } - else - GIT_ALTERNATE_OBJECT_DIRECTORIES=../network.git/objects $git_cmd >$log 2>&1 && { - echo "WARN $repo $nwo (alternates absolute path)"; exit - } - fi - - echo "ERROR $repo $nwo" - ) - - elapsed_time=$(($(date +%s) - before_time)) - - if [[ ! "$status" =~ ^OK ]] || [ $elapsed_time -gt 5 ]; then - echo "$status ${elapsed_time}s" 1>&3 - [ -n "$GHE_VERBOSE" ] && cat $log - fi - - case "$status" in - OK*) - ;; - ERROR*) - errors=$(($errors+1)) - ;; - esac - -done - -log_info "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s" - -rm -f $log - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-git-hooks b/share/github-backup-utils/ghe-backup-git-hooks deleted file mode 100755 index 1763501c7..000000000 --- a/share/github-backup-utils/ghe-backup-git-hooks +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-git-hooks -#/ Take an online, incremental snapshot of custom Git hooks configuration. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -backup_dir="$GHE_SNAPSHOT_DIR/git-hooks" -# Location of last good backup for rsync --link-dest -backup_current="$GHE_DATA_DIR/current/git-hooks" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX) -opts="$GHE_EXTRA_SSH_OPTS" - -# git server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "git-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - rm -rf $tempdir -} -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - -# Transfer Git hooks data from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. A set of rsync filter rules are provided on stdin -# for each invocation. -rsync_git_hooks_data () { - port=$(ssh_port_part "$1") - host=$(ssh_host_part "$1") - - subpath=$2 - shift 2 - - # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's - # --link-dest support. This also decreases physical space usage considerably. - if [ -d "$backup_current/$subpath" ] && [ "$(ls -A $backup_current/$subpath)" ]; then - subdir="git-hooks/$subpath" - link_path=".." - while true; do - if [ "$(dirname $subdir)" = "." ]; then - break - fi - - if [ "$(dirname $subdir)" = "/" ]; then - break - fi - - link_path="../$link_path" - subdir=$(dirname $subdir) - done - - local link_dest="--link-dest=../${link_path}/current/git-hooks/$subpath" - fi - - # Ensure target directory exists, is needed with subdirectories - mkdir -p "$backup_dir/$subpath" - log_rsync "BEGIN: git-hooks sync" 1>&3 - ghe-rsync -av \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" $link_dest \ - --rsync-path='sudo -u git rsync' \ - "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \ - "$backup_dir/$subpath" 1>&3 - log_rsync "END: git-hooks sync" 1>&3 -} - -hostname=$(echo $hostnames | awk '{ print $1; }') -if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs' ]"; then - rsync_git_hooks_data $hostname:122 environments/tarballs -else - ghe_verbose "git-hooks environment tarballs not found. Skipping ..." -fi - -if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos' ]"; then - rsync_git_hooks_data $hostname:122 repos -else - ghe_verbose "git-hooks repositories not found. Skipping ..." -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-minio b/share/github-backup-utils/ghe-backup-minio deleted file mode 100755 index bf7a429d4..000000000 --- a/share/github-backup-utils/ghe-backup-minio +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-minio -#/ Take an online, incremental snapshot of all minio data -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" - -bm_start "$(basename "${0}")" - -# Set up remote host and root backup snapshot directory based on config -port="$(ssh_port_part "${GHE_HOSTNAME}")" -host="$(ssh_host_part "${GHE_HOSTNAME}")" -backup_dir="${GHE_SNAPSHOT_DIR}/minio" - -# Verify rsync is available. -if ! command -v rsync 1> /dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "${host}" - -# Make sure root backup dir exists if this is the first run -mkdir -p "${backup_dir}" - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -# Hilariously, this HAS to stay unquoted when you call `rsync` further -# down because when the shell interpolates this out, `rsync` will throw -# an absolute fit if this variable is quoted. Surprise! -if [[ -d "${GHE_DATA_DIR}/current/minio" ]] && - [[ "$(ls -A "${GHE_DATA_DIR}/current/minio")" ]]; then - link_dest="--link-dest=${GHE_DATA_DIR}/current/minio" -fi - -# Transfer all minio data from the user data directory using rsync. -ghe_verbose "* Transferring minio files from ${host} ..." -log_rsync "BEGIN: minio rsync" 1>&3 -ghe-rsync \ - --archive \ - --verbose \ - --rsh="ghe-ssh -p ${port}" \ - --rsync-path='sudo -u minio rsync' \ - --exclude=".minio.sys" \ - ${link_dest} \ - "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" \ - "${GHE_SNAPSHOT_DIR}/minio" 1>&3 -log_rsync "END: minio rsync" 1>&3 -bm_end "$(basename "${0}")" diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql deleted file mode 100755 index 0975dea11..000000000 --- a/share/github-backup-utils/ghe-backup-mssql +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mssql -#/ -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Set up remote host and root backup snapshot directory based on config -backup_dir="$GHE_SNAPSHOT_DIR/mssql" -last_mssql= -backup_command= -backup_type= -full_expire= -diff_expire= -tran_expire= - -# Check if the export tool is available in this version -export_tool_available() { - if [ -z "$GHE_TEST_REMOTE_VERSION" ]; then - ghe_ssh_mssql "test -e /usr/local/bin/ghe-export-mssql" - else - # Always return available for test - return 0 - fi -} - -ghe_ssh_mssql() { - ghe-ssh "${opts[@]}" "${ssh_config_file_opt[@]}" "$GHE_MSSQL_PRIMARY_HOST" "$@" -} - -cleanup() { - rm -rf "$tempdir" -} -trap 'cleanup' EXIT INT - -# use the mssql primary host if GHES cluster configuration contains a mssql-master or use the ghe server if the mssql-master is not available. -GHE_MSSQL_PRIMARY_NODE="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.mssql-master" || true)" -GHE_MSSQL_PRIMARY_HOST="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.$GHE_MSSQL_PRIMARY_NODE.hostname" || true)" - -if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then - GHE_MSSQL_PRIMARY_HOST="$GHE_HOSTNAME" -fi - -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -ssh_config_file_opt=() -opts=() - -isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" - -# get server hostnames under cluster and HA -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt=("-F" "$ssh_config_file") - opts=("-o" "UserKnownHostsFile=/dev/null" "-o" "StrictHostKeyChecking=no" "-o" "PasswordAuthentication=no") - ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file" -fi - -if ! export_tool_available ; then - log_error "ghe-export-mssql is not available" 1>&2 - exit 1 -fi - -add_minute() { - # Expect date string in the format of yyyymmddTHHMMSS - # Here parse date differently depending on GNU Linux vs BSD MacOS - if date -v -1d > /dev/null 2>&1; then - date -v +"$2"M -ujf'%Y%m%dT%H%M%S' "$1" +%Y%m%dT%H%M%S - else - dt=$1 - date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes" - fi -} - -find_timestamp() { - filename="${1##*/}" - IFS='@' read -ra parts <<< "$filename" - datetime_part=${parts[1]:0:15} - echo "$datetime_part" -} - -actions_dbs() { - all_dbs=$(echo 'set -o pipefail; ghe-mssql-console -y -n -q "SET NOCOUNT ON; SELECT name FROM sys.databases"' | ghe_ssh_mssql /bin/bash) - for db in $all_dbs; do - if [[ ! "$db" =~ ^(master|tempdb|model|msdb)$ ]] && [[ "$db" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "$db" - fi - done -} - -ensure_same_dbs() { - locals=() - while read -r file; do - filename=$(basename "$file") - locals+=("$filename") - done < <(find "$1" \( -name "*.bak" -o -name "*.diff" -o -name "*.log" \)) - - for remote in $(actions_dbs); do - remaining_locals=() - for local in "${locals[@]}"; do - if ! [[ "$local" == "$remote"* ]]; then - remaining_locals+=("$local") - fi - done - locals=("${remaining_locals[@]}") - done - - if [[ "${#locals[@]}" -ne 0 ]]; then - log_warn "Warning: Found following ${#locals[@]} backup files that can't be traced back to the specified GHES host." - log_warn "Warning: Did you recently reconfigure the GHES host? Move or delete these backup files if no longer needed." - for local in "${locals[@]}"; do - ghe_verbose "$1/$local" - done - - exit 1 - fi -} - -run_query() { - echo "set -o pipefail; ghe-mssql-console -y -n -q \"SET NOCOUNT ON; $1\"" | ghe_ssh_mssql /bin/bash | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' -} - -get_latest_backup_file() { - backups_dir=$1 - db=$2 - ext=$3 - - latest_full_backup=$(find "$backups_dir" -type f -name "$db*.$ext" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -n 1) - latest_full_backup_file="${latest_full_backup##*/}" - echo "$latest_full_backup_file" -} - -get_backup_val() { - db=$1 - filename=$2 - column=$3 - run_query " - SELECT s.$column - FROM msdb.dbo.backupset s - JOIN msdb.dbo.backupmediafamily f - ON s.media_set_id = f.media_set_id - WHERE s.database_name = '$db' AND f.physical_device_name LIKE '%$filename'" -} - -get_backup_checkpoint_lsn() { - get_backup_val "$1" "$2" "checkpoint_lsn" -} - -get_backup_last_lsn() { - get_backup_val "$1" "$2" "last_lsn" -} - -get_next_log_backup_starting_lsn() { - db=$1 - # last_log_backup_lsn: The starting log sequence number of the next log backup - # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-database-recovery-status-transact-sql - run_query " - SELECT last_log_backup_lsn - FROM sys.database_recovery_status drs - JOIN sys.databases db on drs.database_id = db.database_id - WHERE db.name = '$db'" -} - -get_next_diff_backup_base_lsn() { - db=$1 - # differential_base_lsn: Base for differential backups. Data extents changed after this LSN will be included in a differential backup. - # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql - run_query " - SELECT differential_base_lsn - FROM sys.master_files mf - WHERE mf.name = '$db'" -} - -last_mssql=$GHE_DATA_DIR/current/mssql - -if [ ! -d "$last_mssql" ] \ - || [ -z "$(find "$last_mssql" -type f -name '*.bak' | head -n 1)" ]; then - ghe_verbose "Taking first full backup" - backup_type="full" -else - ensure_same_dbs "$last_mssql" - - # Check schedule to determine backup type - IFS=',' read -ra cadence <<< "$GHE_MSSQL_BACKUP_CADENCE" - - current=$(date -u +%Y%m%d%H%M%S) - - full=$(find "$last_mssql" -type f -name "*.bak" | head -n 1) - full=$(find_timestamp "$full") - full_expire=$(add_minute "$full" "${cadence[0]}") - full_expire="${full_expire//T}" - - diff=$(find "$last_mssql" -type f -name "*.diff" | head -n 1) - if [ -f "$diff" ]; then - diff=$(find_timestamp "$diff") - diff_expire=$(add_minute "$diff" "${cadence[1]}") - diff_expire="${diff_expire//T}" - else - diff_expire=$(add_minute "$full" "${cadence[1]}") - diff_expire="${diff_expire//T}" - fi - - tran=$(find "$last_mssql" -type f -name "*.log" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -1) - tran=$(find_timestamp "$tran") - tran_expire=$(add_minute "$tran" "${cadence[2]}") - tran_expire="${tran_expire//T}" - - ghe_verbose "current $current, full expire $full_expire, \ -diff expire $diff_expire, tran expire $tran_expire" - - # Determine the type of backup to take based on expiry time - if [ "$current" -gt "$full_expire" ]; then - backup_type='full' - elif [ "$current" -gt "$diff_expire" ]; then - backup_type='diff' - elif [ "$current" -gt "$tran_expire" ]; then - backup_type='transaction' - fi - - # Upgrade to a full backup if the diff/transaction backup might not be restorable due to other backup mechanisms interfering - # with the transaction LSN chain or differential base LSN. - if [ "$backup_type" == 'diff' ] || [ "$backup_type" == 'transaction' ]; then - ghe_verbose "Checking for conflicting backups to ensure a $backup_type backup is sufficient" - - for db in $(actions_dbs); do - # Ensure that a diff backup will be based on the full backup file we have (rather than one another backup mechanism took) - if [ "$backup_type" == 'diff' ]; then - full_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "bak") - if [[ "$full_backup_file" == "" ]]; then - log_warn "Taking a full backup instead of a diff backup because for $db a full backup file wasn't found" - backup_type="full" - break - fi - - full_backup_file_checkpoint_lsn=$(get_backup_checkpoint_lsn "$db" "$full_backup_file") - if [[ "$full_backup_file_checkpoint_lsn" = "NULL" ]] || [[ "$full_backup_file_checkpoint_lsn" == "" ]]; then - log_warn "Taking a full backup instead of a diff backup because for $db the checkpoint LSN for $full_backup_file couldn't be determined" - backup_type="full" - break - fi - - next_diff_backup_base_lsn=$(get_next_diff_backup_base_lsn "$db") - if [[ "$next_diff_backup_base_lsn" = "NULL" ]] || [[ "$next_diff_backup_base_lsn" == "" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db the base LSN for the next diff backup couldn't be determined" - backup_type="full" - break - fi - - # The base of the diff backup we're about to take must exactly match the checkpoint LSN of the full backup file we have - if [[ "$next_diff_backup_base_lsn" -ne "$full_backup_file_checkpoint_lsn" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db the diff would have base LSN $next_diff_backup_base_lsn yet our full backup has checkpoint LSN $full_backup_file_checkpoint_lsn" - backup_type="full" - break - fi - fi - - # Ensure that a transaction log backup will immediately follow the previous one - latest_log_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "log") - if [[ "$latest_log_backup_file" == "" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db a previous transaction log backup wasn't found" - backup_type="full" - break - fi - - latest_log_backup_last_lsn=$(get_backup_last_lsn "$db" "$latest_log_backup_file") - if [[ "$latest_log_backup_last_lsn" = "NULL" ]] || [[ "$latest_log_backup_last_lsn" == "" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db the LSN range for $latest_log_backup_file couldn't be determined" - backup_type="full" - break - fi - - next_log_backup_starting_lsn=$(get_next_log_backup_starting_lsn "$db") - if [[ "$next_log_backup_starting_lsn" = "NULL" ]] || [[ "$next_log_backup_starting_lsn" == "" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db the starting LSN for the next log backup couldn't be determined" - backup_type="full" - break - fi - - # The starting LSN of the backup we're about to take must be equal to (or before) the last LSN from the last backup, - # otherwise there'll be a gap and the logfiles won't be restorable - if [[ "$next_log_backup_starting_lsn" -gt "$latest_log_backup_last_lsn" ]]; then - log_warn "Taking a full backup instead of a $backup_type backup because for $db a gap would exist between the last backup ending at LSN $latest_log_backup_last_lsn and next backup starting at $next_log_backup_starting_lsn" - backup_type="full" - break - fi - done - fi -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Use hard links to "copy" over previous applicable backups to the new snapshot folder to save disk space and time -if [ -d "$last_mssql" ]; then - for p in "$last_mssql"/* - do - [[ -e "$p" ]] || break - - filename="${p##*/}" - extension="${filename##*.}" - transfer= - - # Copy full backups unless we're taking a new full backup - if [ "$extension" = "bak" ] && [ "$backup_type" != 'full' ]; then - transfer=1 - fi - - # Copy diff backups unless we're taking a new full or diff backup - if [ "$extension" = "diff" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then - transfer=1 - fi - - # Copy transaction log backups unless we're taking a new full or diff backup - if [ "$extension" = "log" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then - transfer=1 - fi - - if [ -n "$transfer" ]; then - ghe_verbose "Creating hard link to $filename" - ln "$last_mssql"/"$filename" "$backup_dir"/"$filename" - fi - done -fi - -if [ -n "$backup_type" ]; then - ghe_verbose "Taking $backup_type backup" - - backup_command='ghe-export-mssql' - if [ "$backup_type" = "diff" ]; then - backup_command='ghe-export-mssql -d' - elif [ "$backup_type" = "transaction" ]; then - backup_command='ghe-export-mssql -t' - fi - - backup_failed= - - bm_start "$(basename "$0")" - # record if generating the backup failed, this will allow us to collect any backups that may have been produced, even if they are not complete they are better than nothing - ghe_ssh_mssql -- "$backup_command" || backup_failed='true' - bm_end "$(basename "$0")" - - # Configure the backup cadence on the appliance, which is used for diagnostics - ghe_ssh_mssql "ghe-config mssql.backup.cadence $GHE_MSSQL_BACKUP_CADENCE" - - # Transfer backup files from appliance to backup host - appliance_dir="$GHE_REMOTE_DATA_DIR/user/mssql/backups" - backups=$(echo "set -o pipefail; if sudo test -d \"$appliance_dir\"; then \ - sudo ls \"$appliance_dir\"; fi" | ghe_ssh_mssql /bin/bash) - for b in $backups - do - ghe_verbose "Transferring to backup host $b" - ghe_ssh_mssql "sudo cat $appliance_dir/$b" > "$backup_dir"/"$b" - done - - if [ -n "$backup_failed" ]; then - log_error 'ghe-export-mssql failed to backup at least one mssql database' 1>&2 - exit 1 - fi -fi diff --git a/share/github-backup-utils/ghe-backup-mysql b/share/github-backup-utils/ghe-backup-mysql deleted file mode 100755 index 733051bc4..000000000 --- a/share/github-backup-utils/ghe-backup-mysql +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql -#/ Backup MySQL from a GitHub instance. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -if is_external_database_target; then - if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then - log_info "Backing up external MySQL database using customer-provided script..." - $EXTERNAL_DATABASE_BACKUP_SCRIPT - bm_end "$(basename $0)" - exit 0 - else - if is_binary_backup_feature_on; then - log_warn "Binary backups are configured on the target environment." - log_warn "Binary backup is not supported with an external MySQL database. Backing up using logical backup strategy. Please disable binary backups with 'ghe-config mysql.backup.binary false', or provide a custom backup script using EXTERNAL_DATABASE_BACKUP_SCRIPT" - fi - - ghe-backup-mysql-logical - fi -else - if is_binary_backup_feature_on; then - ghe-backup-mysql-binary - else - # if incremental backups are turned on, we can't do them with - # logical backups, so we need to tell the user and exit - is_inc=$(is_incremental_backup_feature_on) - if [ $is_inc = true ]; then - log_warn "Incremental backups are configured on the target environment." - log_warn "Incremental backup is not supported with a logical MySQL backup. Please disable incremental backups with 'ghe-config mysql.backup.incremental false'" - exit 1 - fi - ghe-backup-mysql-logical - fi -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-mysql-binary b/share/github-backup-utils/ghe-backup-mysql-binary deleted file mode 100755 index 106757c71..000000000 --- a/share/github-backup-utils/ghe-backup-mysql-binary +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql-binary -#/ Backup MySQL from a GitHub instance using binary backup strategy. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -log_verbose "Backing up MySQL database using binary backup strategy ..." -is_inc=$(is_incremental_backup_feature_on) -if [ $is_inc = true ]; then - log_verbose "Incremental backups are configured on the target environment." - log_info "Performing incremental backup of MySQL database ..." 1>&3 - INC_TYPE=$(full_or_incremental_backup) - INC_LSN="" - if [ "$INC_TYPE" == "full" ]; then - log_info "Incremental backup type: $INC_TYPE" 1>&3 - INC_LSN=0 # 0 means full backup - else - validate_inc_snapshot_data - log_info "Incremental backup type: $INC_TYPE" 1>&3 - INC_LSN=$(retrieve_last_lsn) - fi - echo "set -o pipefail; env INC_BACKUP=$INC_LSN ghe-export-mysql" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" - echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel" - # Ensure that we capture the xtrabackup_checkpoints file from the remote host - log_info "Checking if incremental backup is part of a cluster" - GET_LSN=$(get_cluster_lsn "$GHE_HOSTNAME") - ghe-ssh "$GHE_HOSTNAME" "$GET_LSN" > "$GHE_SNAPSHOT_DIR/xtrabackup_checkpoints" - if [ "$INC_TYPE" == "full" ]; then - log_info "Adding $GHE_SNAPSHOT_DIR to the list of full backups" 1>&3 - update_inc_full_backup "$GHE_SNAPSHOT_DIR" - else - log_info "Adding $GHE_SNAPSHOT_DIR to the list of incremental backups" 1>&3 - update_inc_snapshot_data "$GHE_SNAPSHOT_DIR" - fi - bm_end "$(basename $0)" - exit 0 -fi -# if incremental backup isn't enabled, or we are performing a full backup as part of the process, -# fall through and do a full backup - echo "set -o pipefail; ghe-export-mysql" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" - echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel" - is_inc=$(is_incremental_backup_feature_on) - if [ $is_inc = true ]; then - update_inc_full_backup "$GHE_SNAPSHOT_DIR" - fi -bm_end "$(basename $0)" - diff --git a/share/github-backup-utils/ghe-backup-mysql-logical b/share/github-backup-utils/ghe-backup-mysql-logical deleted file mode 100755 index 77040b8b9..000000000 --- a/share/github-backup-utils/ghe-backup-mysql-logical +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql-logical -#/ Backup MySQL from a GitHub instance using logical backup strategy. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -log_verbose "Backing up MySQL database using logical backup strategy ..." - -echo "set -o pipefail; ghe-export-mysql | pigz" | -ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" - -if is_external_database_target; then - echo "LOGICAL_EXTERNAL_BACKUP" > "$GHE_SNAPSHOT_DIR/logical-external-database-backup-sentinel" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages deleted file mode 100755 index 2b79825cf..000000000 --- a/share/github-backup-utils/ghe-backup-pages +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-pages -#/ Take an online, incremental snapshot of all Pages data -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/pages" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX) -opts="$GHE_EXTRA_SSH_OPTS" - -# Pages server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "pages-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/pages)" ]; then - link_dest="--link-dest=../../current/pages" -fi - -count=0 -for hostname in $hostnames; do - bm_start "$(basename $0) - $hostname" - echo 1>&3 - ghe_verbose "* Starting backup for host: $hostname" - # Sync all auxiliary repository data. This includes files and directories like - # HEAD, audit_log, config, description, info/, etc. No refs or object data - # should be transferred here. - echo 1>&3 - ghe_verbose "* Transferring pages files ..." - log_rsync "BEGIN: pages rsync" 1>&3 - # Transfer all data from the user data directory using rsync. - ghe-rsync -av \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - --rsync-path='sudo -u git rsync' \ - $link_dest \ - "$hostname:$GHE_REMOTE_DATA_USER_DIR/pages/" \ - "$GHE_SNAPSHOT_DIR/pages" 1>&3 - log_rsync "END: pages rsync" 1>&3 - bm_end "$(basename $0) - $hostname" - count=$((count + 1)) -done -increment-progress-total-count $count -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-redis b/share/github-backup-utils/ghe-backup-redis deleted file mode 100755 index b0eb44949..000000000 --- a/share/github-backup-utils/ghe-backup-redis +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-redis -#/ Take a snapshot of all Redis data. This is needed because older versions of -#/ the remote side ghe-export-redis command use a blocking SAVE instead of a -#/ non-blocking BGSAVE. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$GHE_HOSTNAME" - -# Force a redis BGSAVE, and wait for it to complete. -ghe-ssh "$GHE_HOSTNAME" /bin/bash < /dev/null; then - redis_cli=ghe-redis-cli - redis_arg=--remote - else - redis_cli=redis-cli - redis_arg= - fi - redis_host=\$(ghe-config cluster.redis-master 2>/dev/null || echo "localhost") - timestamp=\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE) - - for i in \$(seq 10); do - if ! \$redis_cli \$redis_arg -h \$redis_host BGSAVE | grep -q ERR; then - break - fi - sleep 15 - done - for n in \$(seq 3600); do - if [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ]; then - break - fi - sleep 1 - done - [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ] # exits 1 if bgsave didn't work - - if [ "\$redis_host" != "localhost" ]; then - ssh \$redis_host sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb' - else - sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb' - fi -EOF - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories deleted file mode 100755 index 4e142e29f..000000000 --- a/share/github-backup-utils/ghe-backup-repositories +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-repositories -#/ Take an online, incremental snapshot of all Git repository data. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# This command is designed to allow for transferring active Git repository data -# from a GitHub instance to a backup site in a way that ensures data is -# captured in a consistent state even when being written to. -# -# - All Git GC operations are disabled on the GitHub instance for the duration of -# the backup. This removes the possibly of objects or packs being removed -# while the backup is in progress. -# -# - In progress Git GC operations are given a cooldown window to complete. The -# script will sleep for up to 60 seconds waiting for GC operations to finish. -# -# - Git repository data is transferred in a specific order: auxiliary files, -# packed refs, loose refs, reflogs, and finally objects and pack files in that -# order. This ensures that all referenced objects are captured. -# -# - Git GC operations are re-enabled on the GitHub instance. -# -# The script uses multiple runs of rsync to transfer repository files. Each run -# includes a list of filter rules that ensure only specific types of files are -# transferred. -# -# See the "FILTER RULES" and "INCLUDE/EXCLUDE PATTERN RULES" sections of the -# rsync(1) manual for more information: -# - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/repositories" - -# Location of last good backup for rsync --link-dest -backup_current="$GHE_DATA_DIR/current/repositories" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) -routes_list=$tempdir/routes_list -remote_routes_list=$remote_tempdir/remote_routes_list -opts="$GHE_EXTRA_SSH_OPTS" - -# git server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "git-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Replica hostnames for HA -if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - ha_replica_hosts=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes --replica) -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - for pid in $(jobs -p); do - kill -KILL $pid > /dev/null 2>&1 || true - done - - # Enable remote GC operations - for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { - echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - # Enable remote GC operations for HA replica - for replica_host in $ha_replica_hosts; do - echo "set -o pipefail; ssh $replica_host -- 'sudo rm -f $SYNC_IN_PROGRESS_FILE'" | ghe-ssh "$host" /bin/bash || { - echo "Re-enable gc on $replica_host failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# Disable remote GC operations -for hostname in $hostnames; do - ghe-gc-disable $ssh_config_file_opt $hostname:$port -done - -# Disable remote GC operations for HA replica -# gc_disable is a function defined in ghe-backup-config -# gc_disable is called on the replica node via the primary node, because replica node is not expected to be reachable from backup host. But replica node is expected to be reachable from primary node. -for replica_host in $ha_replica_hosts; do - echo "set -o pipefail; ssh $replica_host -- '$(declare -f gc_disable); gc_disable \"$SYNC_IN_PROGRESS_FILE\" \"$GHE_GIT_COOLDOWN_PERIOD\"'" | ghe-ssh "$host" /bin/bash || { - echo "Disable gc on $replica_host failed" 1>&2 - } -done - -# If we have a previous increment, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$backup_current" ]; then - link_dest="--link-dest=../../current/repositories" -fi - -# Calculate sync routes. This will store the healthy repo paths for each node -# -# This gets a repo path and stores the path in the $node.sync file -# a/nw/a8/3f/02/100000855 dgit-node1 >> dgit-node1.sync -# a/nw/a8/bc/8d/100000880 dgit-node3 >> dgit-node3.sync -# a/nw/a5/06/81/100000659 dgit-node2 >> dgit-node2.sync -# ... -# One route per line. -# -# NOTE: The route generation is performed on the appliance as it is considerably -# more performant than performing over an SSH pipe. -# -bm_start "$(basename $0) - Generating routes" -echo "github-env ./bin/dgit-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash -ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug -bm_end "$(basename $0) - Generating routes" - -bm_start "$(basename $0) - Fetching routes" -ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list -< $routes_list ghe_debug -bm_end "$(basename $0) - Fetching routes" - -bm_start "$(basename $0) - Processing routes" -if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then - server=$host -fi -< $routes_list awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' -ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" -bm_end "$(basename $0) - Processing routes" - -if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - log_warn "no routes found, skipping repositories backup ..." - exit 0 -else - increment-progress-total-count 3 -fi - -# Transfer repository data from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. A set of rsync filter rules are provided on stdin -# for each invocation. -rsync_repository_data () { - port=$(ssh_port_part "$1") - host=$(ssh_host_part "$1") - - #check if we are syncing from a given file list - if [[ "$2" == *".rsync" ]]; then - files_list="$2" - shift - shift - log_rsync "BEGIN: repositories rsync" 1>&3 - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path='sudo -u git rsync' \ - --include-from=- --exclude=\* \ - --files-from="$files_list" \ - --ignore-missing-args \ - "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ - "$backup_dir" 1>&3 2>&3 - log_rsync "END: repositories rsync" 1>&3 - else - shift - log_rsync "BEGIN: repositories rsync" 1>&3 - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path='sudo -u git rsync' \ - --include-from=- --exclude=\* \ - --ignore-missing-args \ - "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ - "$backup_dir" 1>&3 2>&3 - log_rsync "END: repositories rsync" 1>&3 - fi -} - -sync_data (){ - # Sync all auxiliary repository data. This includes files and directories like - # HEAD, audit_log, config, description, info/, etc. No refs or object data - # should be transferred here. - echo 1>&3 - - log_info "* Transferring auxiliary files ..." 1>&3 - rsync_repository_data $1:122 $2 <&3 - log_info "* Transferring packed-refs files ..." 1>&3 - rsync_repository_data $1:122 $2 <&3 - log_info "* Transferring refs and reflogs ..." 1>&3 - rsync_repository_data $1:122 $2 <&3 - log_info "* Transferring objects and packs ..." 1>&3 - rsync_repository_data $1:122 $2 -H <&3 - -} - -# rsync all the repositories -bm_start "$(basename $0) - Repo sync" -for file_list in $tempdir/*.rsync; do - hostname=$(basename $file_list .rsync) - - repo_num=$(< $file_list wc -l) - ghe_verbose "* Transferring $repo_num repositories from $hostname" - - sync_data $hostname $file_list & -done - -for pid in $(jobs -p); do - wait $pid -done -bm_end "$(basename $0) - Repo sync" - -# Since there are no routes for special data directories, we need to do this -# serially for all hostnames. Good candidate for future optimizations. - -bm_start "$(basename $0) - Special Data Directories Sync" -for h in $hostnames; do - # Sync __special__ data directories, including the __alambic_assets__, - # __hookshot__, and __purgatory__ directories. The __nodeload_archives__, - # __gitmon__, and __render__ directories are excludes since they act only as - # caches. - # - # Under v2.x and greater, only the special __purgatory__ directory remains under - # /data/repositories. All other special user data directories have been moved under - # the /data/user directory. - echo 1>&3 - log_info "* Transferring special data directories from $h..." 1>&3 - rsync_repository_data $h:122 <&3 -done -bm_end "$(basename $0) - Special Data Directories Sync" - -if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then - bm_start "$(basename $0) - Verifying Routes" - for file_lst in $tempdir/*.rsync; do - < $file_lst sort | uniq - done |sort|uniq > $tempdir/source_routes - (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes - - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance." - increment-progress-total-count 1 - bm_end "$(basename $0) - Verifying Routes" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-secrets b/share/github-backup-utils/ghe-backup-secrets deleted file mode 100755 index 6d7ab5db6..000000000 --- a/share/github-backup-utils/ghe-backup-secrets +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-secrets -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Grab the host -host="$GHE_HOSTNAME" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - - -# Function to backup a secret setting to a file. -# backup-secret [--best-effort] -backup-secret() { - best_effort=false - description="" - file="" - setting="" - count=0 - - while [ $# -gt 0 ]; do - case "$1" in - --best-effort) - shift 1 - best_effort=true - ;; - *) - case $count in - 0) - description=$1 - ;; - 1) - file=$1 - ;; - 2) - setting=$1 - ;; - *) - >&2 echo "Too many arguments" - ;; - esac - count=$((count+1)) - shift 1 - esac - done - - log_info "* Transferring $description ..." 1>&3 - ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( - if [ "$best_effort" = "false" ]; then - echo "Info: $description not set. Skipping..." >&2 - fi - ) - if [ -n "$(cat "$file+")" ]; then - mv "$file+" "$file" - else - unlink "$file+" - fi -} - -bm_start "$(basename $0)" - -# Create the snapshot directory if needed and change into it. -mkdir -p "$GHE_SNAPSHOT_DIR" -cd "$GHE_SNAPSHOT_DIR" - -log_info "* Transferring secrets data ..." 1>&3 - -backup-secret "management console password" "manage-password" "secrets.manage" -backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" -backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" -backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" - -# backup encryption keying material and create backup value current encryption for GHES 3.7.0 onwards -# this is for forwards compatibility with GHES 3.8.0 onwards -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.7.0)" ]; then - backup-secret "encrypted column encryption keying material" "encrypted-column-encryption-keying-material" "secrets.github.encrypted-column-keying-material" - cat "$GHE_SNAPSHOT_DIR/encrypted-column-encryption-keying-material" | sed 's:.*;::' > "$GHE_SNAPSHOT_DIR/encrypted-column-current-encryption-key" -fi - -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then - backup-secret "secret scanning encrypted secrets current storage key" "secret-scanning-encrypted-secrets-current-storage-key" "secrets.secret-scanning.encrypted-secrets-current-storage-key" - backup-secret "secret scanning encrypted secrets delimited storage keys" "secret-scanning-encrypted-secrets-delimited-storage-keys" "secrets.secret-scanning.encrypted-secrets-delimited-storage-keys" - backup-secret "secret scanning encrypted secrets current shared transit key" "secret-scanning-encrypted-secrets-current-shared-transit-key" "secrets.secret-scanning.encrypted-secrets-current-shared-transit-key" - backup-secret "secret scanning encrypted secrets delimited shared transit keys" "secret-scanning-encrypted-secrets-delimited-shared-transit-keys" "secrets.secret-scanning.encrypted-secrets-delimited-shared-transit-keys" -fi - -if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.11.0)" ]; then - backup-secret "secret scanning encrypted content keys" "secret-scanning-user-content-delimited-encryption-root-keys" "secrets.secret-scanning.secret-scanning-user-content-delimited-encryption-root-keys" -fi - -# Backup argon secrets for multiuser from ghes version 3.8 onwards -if [[ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" && "$(version $GHE_REMOTE_VERSION)" -lt "$(version 3.8.2)" ]]; then - backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" -fi - -# Backup external MySQL password if running external MySQL DB. -if is_service_external 'mysql'; then - backup-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" -fi - -# Backup Actions settings. -if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then - backup-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" - backup-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" - backup-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" --best-effort - backup-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" - backup-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" - backup-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" - backup-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" - backup-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" - backup-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" - backup-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" --best-effort - backup-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" --best-effort - backup-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" --best-effort - backup-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" - backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" - backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort - backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" - backup-secret "Actions storage container prefix" "actions-storage-container-prefix" "secrets.actions.storage.container-prefix" - - backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" - backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" - backup-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" - backup-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" - backup-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" - backup-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" - backup-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" - backup-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" - backup-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" - backup-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" - backup-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" - backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" -fi - -if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then - backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" - backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" - backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" - backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" - backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" - backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" - backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" -fi - -# Backup Chat Integration settings -if ghe-ssh "$host" -- ghe-config --true app.chatops.enabled; then - backup-secret "Chat Integration MSTeams app id" "chatops-msteams-app-id" "secrets.chatops.msteams.app-id" - backup-secret "Chat Integration MSTeams app password" "chatops-msteams-app-password" "secrets.chatops.msteams.app-password" - backup-secret "Chat Integration MSTeams public endpoint" "chatops-msteams-app-public-endpoint" "secrets.chatops.msteams.public-endpoint" - backup-secret "Chat Integration MSTeams bot handle" "chatops-msteams-bot-handle" "secrets.chatops.msteams.bot-handle" - backup-secret "Chat Integration MSTeams bot name" "chatops-msteams-bot-name" "secrets.chatops.msteams.bot-name" - backup-secret "Chat Integration Slack app id" "chatops-slack-app-id" "secrets.chatops.slack.app-id" - backup-secret "Chat Integration Slack client id" "chatops-slack-client-id" "secrets.chatops.slack.client-id" - backup-secret "Chat Integration Slack client secret" "chatops-slack-client-secret" "secrets.chatops.slack.client-secret" - backup-secret "Chat Integration Slack verification token" "chatops-slack-verification-token" "secrets.chatops.slack.verification-token" - backup-secret "Chat Integration Slack config token" "chatops-slack-config-token" "secrets.chatops.slack.config-token" - backup-secret "Chat Integration Slack public endpoint" "chatops-slack-public-endpoint" "secrets.chatops.slack.public-endpoint" - backup-secret "Chat Integration Slack signing secret" "chatops-slack-signing-secret" "secrets.chatops.slack.signing-secret" - backup-secret "Chat Integration Slack app level token" "chatops-slack-app-level-token" "secrets.chatops.slack.app-level-token" - backup-secret "Chat Integration Slack slack command" "chatops-slack-slash-command" "secrets.chatops.slack.slash-command" - backup-secret "Chat Integration Slack app name" "chatops-slack.app-name" "secrets.chatops.slack.app-name" - backup-secret "Chat Integration Slack socket mode" "chatops-slack.socket-mode" "secrets.chatops.slack.socket-mode" - backup-secret "Chat Integration public endpoint" "chatops-public-endpoint" "secrets.chatops.public-endpoint" - backup-secret "Chat Integration app type" "chatops-app-type" "secrets.chatops.app-type" - backup-secret "Chat Integration app id teams" "chatops-app-id-teams" "secrets.chatops.app-id-teams" - backup-secret "Chat Integration webhook secret teams" "chatops-webhook-secret-teams" "secrets.chatops.webhook-secret-teams" - backup-secret "Chat Integration client secret teams" "chatops-client-secret-teams" "secrets.chatops.client-secret-teams" - backup-secret "Chat Integration clien id teams" "chatops-client-id-teams" "secrets.chatops.client-id-teams" - backup-secret "Chat Integration storage secret" "chatops-storage-secret" "secrets.chatops.storage-secret" - backup-secret "Chat Integration session secret" "chatops-session-secret" "secrets.chatops.session-secret" - backup-secret "Chat Integration app id slack" "chatops-app-id-slack" "secrets.chatops.app-id-slack" - backup-secret "Chat Integration webhook secret slack" "chatops-webhook-secret-slack" "secrets.chatops.webhook-secret-slack" - backup-secret "Chat Integration client secret slack" "chatops-client-secret-slack" "secrets.chatops.client-secret-slack" - backup-secret "Chat Integration client id slack" "chatops-client-id-slack" "secrets.chatops.client-id-slack" -fi - -bm_end "$(basename $0)" - -exit 0 diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings deleted file mode 100755 index f3d262293..000000000 --- a/share/github-backup-utils/ghe-backup-settings +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-settings -#/ Backup settings from a snapshot to the given . -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -bm_start "$(basename $0)" - -# Grab the host -host="$GHE_HOSTNAME" - -# Create the snapshot directory if needed and change into it. -mkdir -p "$GHE_SNAPSHOT_DIR" -cd "$GHE_SNAPSHOT_DIR" - -log_info "* Transferring settings data ..." 1>&3 -ghe-ssh "$host" -- 'ghe-export-settings' > settings.json - -log_info "* Transferring license data ..." 1>&3 -ghe-ssh "$host" -- "sudo cat '$GHE_REMOTE_LICENSE_FILE'" > enterprise.ghl - -if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then - log_info "* Transferring SAML keys ..." 1>&3 - ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar -fi - -if ghe-ssh "$host" -- "which ghe-export-ssl-ca-certificates 1>/dev/null"; then - log_info "* Transferring CA certificates ..." 1>&3 - ghe-ssh "$host" -- "ghe-export-ssl-ca-certificates" > ssl-ca-certificates.tar -fi - -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - log_info "* Transferring cluster configuration ..." 1>&3 - if ! ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_CLUSTER_CONF_FILE 2>/dev/null" > cluster.conf; then - log_error "Error: Enterprise Cluster is not configured yet, backup will fail" >&2 - exit 1 - fi -else - if ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_DATA_USER_DIR/common/uuid 2>/dev/null" > uuid; then - log_info "* Transferring UUID ..." 1>&3 - fi -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage deleted file mode 100755 index c80c39a29..000000000 --- a/share/github-backup-utils/ghe-backup-storage +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-storage -#/ Take an online, incremental snapshot of all Alambic Storage data using the -#/ calculated routes method. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/storage" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) -routes_list=$tempdir/routes_list -remote_routes_list=$remote_tempdir/remote_routes_list -opts="$GHE_EXTRA_SSH_OPTS" - -# storage server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "storage-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Replica hostnames for HA -if ghe-ssh "$GHE_HOSTNAME" -- "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - ha_replica_hosts=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes --replica) -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - # Enable remote maintenance operations - for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { - log_warn "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - # Enable remote GC operations for HA replica - for replica_host in $ha_replica_hosts; do - echo "set -o pipefail; ssh $replica_host -- 'sudo rm -f $SYNC_IN_PROGRESS_FILE'" | ghe-ssh "$host" /bin/bash || { - echo "Re-enable gc on $replica_host failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# Disable remote maintenance operations -for hostname in $hostnames; do - ghe-gc-disable $ssh_config_file_opt $hostname:$port -done - -# Disable remote GC operations for HA replica -# gc_disable is a function defined in ghe-backup-config -# gc_disable is called on the replica node via the primary node, because replica node is not expected to be reachable from backup host. But replica node is expected to be reachable from primary node. -for replica_host in $ha_replica_hosts; do - echo "set -o pipefail; ssh $replica_host -- '$(declare -f gc_disable); gc_disable \"$SYNC_IN_PROGRESS_FILE\" \"$GHE_GIT_COOLDOWN_PERIOD\"'" | ghe-ssh "$host" /bin/bash || { - echo "Disable gc on $replica_host failed" 1>&2 - } -done - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/storage)" ]; then - link_dest="--link-dest=../../current/storage" -fi - -# Calculate sync routes. This will store the healthy object paths for each node -# -# This gets a repo path and stores the path in the $node.sync file -# a/nw/a8/3f/02/100000855 storage-server-node1 >> storage-server-node1.sync -# a/nw/a8/bc/8d/100000880 storage-server-node3 >> storage-server-node3.sync -# a/nw/a5/06/81/100000659 storage-server-node2 >> storage-server-node2.sync -# ... -#one route per line. -# -# NOTE: The route generation is performed on the appliance as it is considerably -# more performant than performing over an SSH pipe. -# -bm_start "$(basename $0) - Generating routes" -echo "github-env ./bin/storage-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash -ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug -bm_end "$(basename $0) - Generating routes" - -bm_start "$(basename $0) - Fetching routes" -ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list -cat $routes_list | ghe_debug -bm_end "$(basename $0) - Fetching routes" - -bm_start "$(basename $0) - Processing routes" -if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then - server=$host -fi -cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' -ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" -bm_end "$(basename $0) - Processing routes" - -if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - log_warn "no routes found, skipping storage backup ..." - exit 0 -else - increment-progress-total-count 2 -fi - -# rsync all the storage objects -bm_start "$(basename $0) - Storage object sync" -for file_list in $tempdir/*.rsync; do - hostname=$(basename $file_list .rsync) - storage_user=$(ghe-ssh $ssh_config_file_opt $hostname:$port -- stat -c %U /data/user/storage || echo git) - - object_num=$(cat $file_list | wc -l) - ghe_verbose "* Transferring $object_num objects from $hostname" - log_rsync "BEGIN: storage rsync" 1>&3 - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path="sudo -u $storage_user rsync" \ - --files-from="$file_list" \ - --ignore-missing-args \ - --size-only \ - "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \ - "$backup_dir" 1>&3 & - log_rsync "END: storage rsync" 1>&3 -done - -for pid in $(jobs -p); do - wait $pid -done -bm_end "$(basename $0) - Storage object sync" - -if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then - bm_start "$(basename $0) - Verifying Routes" - - cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes - (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes - - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance." - - increment-progress-total-count 1 - bm_end "$(basename $0) - Verifying Routes" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-store-version b/share/github-backup-utils/ghe-backup-store-version deleted file mode 100755 index 6faffff99..000000000 --- a/share/github-backup-utils/ghe-backup-store-version +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-store-version -#/ Stores information about the used version of backup-utils on -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -version_info="$BACKUP_UTILS_VERSION" -if [ -d $GHE_BACKUP_ROOT/.git ]; then - ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true) - if [ -n "$ref" ]; then - version_info="$version_info:$ref" - fi -fi - -echo "$version_info" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1" - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-strategy b/share/github-backup-utils/ghe-backup-strategy deleted file mode 100755 index b9a919f3c..000000000 --- a/share/github-backup-utils/ghe-backup-strategy +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-strategy -#/ -#/ Determine the backup strategy that will be used. -#/ -#/ The rsync strategy should be used for single VMs and all HA configurations. -#/ -#/ The cluster strategy should be used to backup GHE clusters. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ ! -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - echo "cluster" -else - echo "rsync" -fi diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata deleted file mode 100755 index d7c7dc233..000000000 --- a/share/github-backup-utils/ghe-backup-userdata +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-userdata -#/ Take an online, incremental snapshot of a user data directory. This is used -#/ for a number of different simple datastores kept under /data/user on the -#/ remote appliance, including: hookshot, alambic_assets, and pages data. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0) - $1" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - log_error "rsync not found." 1>&2 - exit 1 -fi - -# Grab the host and /data/user directory name. -host="$GHE_HOSTNAME" -dirname="$1" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Verify that the user data directory exists. Bail out if not, which may be due -# to an older version of GHE or no data has been added to this directory yet. -ghe-ssh "$host" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0 - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/$dirname" ] && [ "$(ls -A $GHE_DATA_DIR/current/$dirname)" ]; then - - subdir=$dirname - link_path=".." - while true; do - if [ "$(dirname $subdir)" = "." ]; then - break - fi - - if [ "$(dirname $subdir)" = "/" ]; then - break - fi - - link_path="../$link_path" - subdir=$(dirname $subdir) - done - - link_dest="--link-dest=../${link_path}/current/$dirname" -fi - -# Ensure target directory exists, is needed with subdirectories -mkdir -p "$GHE_SNAPSHOT_DIR/$dirname" -log_rsync "BEGIN: userdata rsync" 1>&3 -# Transfer all data from the user data directory using rsync. -ghe-rsync -av \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path='sudo -u git rsync' \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \ - "$GHE_SNAPSHOT_DIR/$dirname" 1>&3 -log_rsync "END: userdata rsync" 1>&3 -bm_end "$(basename $0) - $1" diff --git a/share/github-backup-utils/ghe-cluster-find-nodes b/share/github-backup-utils/ghe-cluster-find-nodes deleted file mode 100755 index 2a1f7ef48..000000000 --- a/share/github-backup-utils/ghe-cluster-find-nodes +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-cluster-find-nodes -#/ -#/ Finds all nodes of the cluster using the config on . -#/ If it is a 2.8 and later cluster version the results are returned as -#/ prefix-uuid, otherwise the configured hostnames are returned. -#/ Also filters nodes based on the prefix role. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup-* and ghe-restore-* commands in cluster environments. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Check if the REMOTE DATA USER directory is set -if [ -z $GHE_REMOTE_DATA_USER_DIR ]; then - log_error "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting" - exit 1 -fi - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -GHE_HOSTNAME="$1" -prefix="$2" -role=$(echo "$prefix" | cut -d '-' -f1) - -if ghe-ssh "$GHE_HOSTNAME" test -f $GHE_REMOTE_ROOT_DIR/etc/github/cluster; then - node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -r "$role" -u | cut -f 2) - hostnames='' - for uuid in $node_uuids; do - hostnames+="$prefix-$uuid " - done -else - uuid=$(ghe-ssh "$GHE_HOSTNAME" cat $GHE_REMOTE_DATA_USER_DIR/common/uuid) - hostnames="$prefix-$uuid" -fi - -echo "$hostnames" diff --git a/share/github-backup-utils/ghe-detect-leaked-ssh-keys b/share/github-backup-utils/ghe-detect-leaked-ssh-keys deleted file mode 100755 index a71fd812e..000000000 --- a/share/github-backup-utils/ghe-detect-leaked-ssh-keys +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-detect-leaked-ssh-key [-s ] -#/ -#/ This utility will check each snapshot's existing SSH host keys against the list -#/ of known leaked SSH host keys from GitHub Enterprise packages. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ -s |--snapshot Scan the snapshot with the given id. -#/ Available snapshots may be listed under the data directory. -#/ -set -e - -usage() { - grep '^#/' < "$0" | cut -c 4- - exit 2 -} - -TEMPDIR=$(mktemp -d) - -while [ $# -gt 0 ]; do - case "$1" in - -h|--help) - usage - ;; - -s|--snapshot) - snapshot=$2 - shift - ;; - *) - usage - ;; - esac - shift -done - -ppid_script=$(ps -o args= $PPID 2>/dev/null | awk '{print $2}') -if [ -n "$ppid_script" ]; then - ppid_name=$(basename $ppid_script) -fi - -sshkeygen_multiple_hash_formats=false -if (ssh-keygen -E 2>&1 | head -1 | grep -q 'option requires an argument'); then - sshkeygen_multiple_hash_formats=true -fi - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -FINGERPRINT_BLACKLIST="${FINGERPRINT_BLACKLIST:-$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-ssh-leaked-host-keys-list.txt")}" - -keys="ssh_host_dsa_key.pub ssh_host_ecdsa_key.pub ssh_host_ed25519_key.pub ssh_host_rsa_key.pub" - -# Get all the host ssh keys tar from all snapshots directories -if [ -n "$snapshot" ]; then - if [ ! -d "$snapshot" ]; then - echo "Invalid snapshot directory: $snapshot" >&2 - exit 1 - fi - ssh_tars=$(find "$snapshot" -maxdepth 1 -type f -iname 'ssh-host-keys.tar') -else - ssh_tars=$(find "$GHE_DATA_DIR" -maxdepth 2 -type f -iname 'ssh-host-keys.tar') -fi - -# Store the current backup snapshot folder -if [ -L "$GHE_DATA_DIR/current" ]; then - current_dir=$(cd "$GHE_DATA_DIR/current"; pwd -P) -fi - -leaked_keys_found=false -leaked_keys_skippedcheck=false -current_bkup=false -for tar_file in $ssh_tars; do - for key in $keys; do - if tar -tvf "$tar_file" $key &>/dev/null; then - tar -C $TEMPDIR -xvf "$tar_file" $key &>/dev/null - if $sshkeygen_multiple_hash_formats; then - fingerprint=$(ssh-keygen -l -E md5 -f $TEMPDIR/$key | cut -d ' ' -f 2 | cut -f2- -d':') - else - fingerprint=$(ssh-keygen -lf $TEMPDIR/$key | cut -d ' ' -f 2) - fi - if [ -z "$fingerprint" ]; then - leaked_keys_skippedcheck=true - elif echo "$FINGERPRINT_BLACKLIST" | grep -q "$fingerprint"; then - leaked_keys_found=true - if [ "$current_dir" == "$(dirname "$tar_file")" ]; then - current_bkup=true - log_warn "* Leaked key found in current backup snapshot." - else - log_warn "* Leaked key found in backup snapshot." - fi - echo "* Snapshot file: $tar_file" - echo "* Key file: $key" - echo "* Key: $fingerprint" - echo - fi - fi - done -done - -if $leaked_keys_found; then - if echo "$ppid_name" | grep -q 'ghe-restore'; then - echo - echo "* The snapshot that is being restored contains a leaked SSH host key." - echo "* We recommend rolling the SSH host keys after completing the restore." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - echo - elif echo "$ppid_name" | grep -q 'ghe-backup'; then - echo "* The current backup contains leaked SSH host keys." - echo "* We strongly recommend rolling your SSH host keys and making a new backup." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - else - if $current_bkup; then - echo "* The current backup contains leaked SSH host keys." - echo "* Current backup directory: $current_dir" - echo "* We strongly recommend rolling your SSH host keys and making a new backup." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - fi - echo - echo "* One or more older backup snapshots contain leaked SSH host keys." - echo "* No immediate action is needed but when you use one of these older snapshots for a restore, " - echo "* please make sure to roll the SSH host keys after restore." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - echo - fi -else - if $leaked_keys_skippedcheck; then - log_info "* No result - check not performed since host key fingerprint was empty" - else - log_info "* No leaked keys found" - fi -fi - -# Cleanup temp dir -rm -rf $TEMPDIR diff --git a/share/github-backup-utils/ghe-docker-init b/share/github-backup-utils/ghe-docker-init deleted file mode 100755 index 76e6086b8..000000000 --- a/share/github-backup-utils/ghe-docker-init +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -e - -PATH=$PATH:/backup-utils/bin - -mkdir -p /etc/github-backup-utils - -touch /etc/github-backup-utils/backup.config - -env | grep ^GHE_ | sed -r "s/(.[^=]+)=(.*)/\1=\"\2\"/g" >> /etc/github-backup-utils/backup.config - -exec "$@" diff --git a/share/github-backup-utils/ghe-gc-disable b/share/github-backup-utils/ghe-gc-disable deleted file mode 100755 index 67cdef6d4..000000000 --- a/share/github-backup-utils/ghe-gc-disable +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-gc-disable [