From d8e88ee9444837b997ea28fa2423cc67df6e8747 Mon Sep 17 00:00:00 2001 From: Tomoaki Nosaka Date: Wed, 11 Nov 2015 22:12:42 +0900 Subject: [PATCH 0001/2608] add GHE_EXTRA_RSYNC_OPTS variable for backup.config --- backup.config-example | 4 ++++ share/github-backup-utils/ghe-rsync | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index 17e43074a..fa7e1b0e7 100644 --- a/backup.config-example +++ b/backup.config-example @@ -27,6 +27,10 @@ GHE_NUM_SNAPSHOTS=10 # #GHE_EXTRA_SSH_OPTS="" +# Any extra options passed to the rsync command. Nothing required by default +# +#GHE_EXTRA_RSYNC_OPTS="" + # Add s3 bucket for configuring which bucket to use in ghe-s3-backup and # ghe-s3-restore # GHE_S3_BUCKET="" diff --git a/share/github-backup-utils/ghe-rsync b/share/github-backup-utils/ghe-rsync index 7a234dccc..7b4151d54 100755 --- a/share/github-backup-utils/ghe-rsync +++ b/share/github-backup-utils/ghe-rsync @@ -7,11 +7,15 @@ set -o pipefail +# Bring in the backup configuration +cd $(dirname "$0")/../.. +. share/github-backup-utils/ghe-backup-config + # Filter vanished file warnings from both stdout (rsync versions < 3.x) and # stderr (rsync versions >= 3.x). The complex redirections are necessary to # filter stderr while also keeping stdout and stderr separated. IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)' -(rsync "${@}" 3>&1 1>&2 2>&3 3>&- | +(rsync $GHE_EXTRA_RSYNC_OPTS "${@}" 3>&1 1>&2 2>&3 3>&- | (egrep -v "$IGNOREOUT" || true)) 3>&1 1>&2 2>&3 3>&- | (egrep -v "$IGNOREOUT" || true) res=$? From 2ca97c84ac7ecbef8eefbe98d395dc8666305fcb Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Thu, 18 Feb 2016 12:29:56 +1100 Subject: [PATCH 0002/2608] Update ghe-host-check to detect extra port 22 error Treat `port 22: No route to host` errors from `ssh` the same as `port 22: Connection refused` errors. --- bin/ghe-host-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 039ce168e..7d5833411 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -37,7 +37,7 @@ set -e if [ $rc -ne 0 ]; then case $rc in 255) - if echo "$output" | grep -i "port 22: connection refused\|Connection timed out during banner exchange" >/dev/null; then + if echo "$output" | grep -i "port 22: connection refused\|port 22: no route to host\|Connection timed out during banner exchange" >/dev/null; then exec "bin/$(basename $0)" "$hostname:122" fi From f350b6fec34393d36c49e97d970f263271f40bd7 Mon Sep 17 00:00:00 2001 From: Daniel Hwang Date: Thu, 18 Feb 2016 14:07:22 -0800 Subject: [PATCH 0003/2608] clustering: Add notes for GHE_EXTRA_SSH_OPTS --- README.md | 2 ++ backup.config-example | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd143f167..032d96723 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ download the most recent GitHub Enterprise version. host name. Additional options are available and documented in the configuration file but none are required for basic backup functionality. + * In a clustering environment, the `GHE_EXTRA_SSH_OPTS` key must be configured. + 3. Add the backup host's SSH key to the GitHub appliance as an *Authorized SSH key*. See [Adding an SSH key for shell access][3] for instructions. diff --git a/backup.config-example b/backup.config-example index fa7e1b0e7..ab71140bc 100644 --- a/backup.config-example +++ b/backup.config-example @@ -23,11 +23,13 @@ GHE_NUM_SNAPSHOTS=10 # #GHE_RESTORE_HOST="github-standby.example.com" -# Any extra options passed to the SSH command. Nothing required by default +# Any extra options passed to the SSH command. +# In a single instance environment, nothing is required by default. +# In a clustering environment, "-i path-to-ssh-private-key" is required. # #GHE_EXTRA_SSH_OPTS="" -# Any extra options passed to the rsync command. Nothing required by default +# Any extra options passed to the rsync command. Nothing required by default. # #GHE_EXTRA_RSYNC_OPTS="" From a5b27f357b8531f78ce8751d9519d90406ed68e4 Mon Sep 17 00:00:00 2001 From: Daniel Hwang Date: Thu, 18 Feb 2016 14:29:54 -0800 Subject: [PATCH 0004/2608] clustering: Explicitly use absolute path --- backup.config-example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.config-example b/backup.config-example index ab71140bc..239c8d37b 100644 --- a/backup.config-example +++ b/backup.config-example @@ -25,7 +25,7 @@ GHE_NUM_SNAPSHOTS=10 # Any extra options passed to the SSH command. # In a single instance environment, nothing is required by default. -# In a clustering environment, "-i path-to-ssh-private-key" is required. +# In a clustering environment, "-i abs-path-to-ssh-private-key" is required. # #GHE_EXTRA_SSH_OPTS="" From 4cccc7bf70ff7033b2fceb836c503454c5d14b73 Mon Sep 17 00:00:00 2001 From: Daniel Hwang Date: Thu, 18 Feb 2016 15:46:13 -0800 Subject: [PATCH 0005/2608] clustering: Be explicit in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 032d96723..d29ba9e43 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ download the most recent GitHub Enterprise version. host name. Additional options are available and documented in the configuration file but none are required for basic backup functionality. - * In a clustering environment, the `GHE_EXTRA_SSH_OPTS` key must be configured. + * In a clustering environment, the `GHE_EXTRA_SSH_OPTS` key must be configured with the `-i ` SSH option. 3. Add the backup host's SSH key to the GitHub appliance as an *Authorized SSH key*. See [Adding an SSH key for shell access][3] for instructions. From ab1bdda371d526850e773f93b74605443255f8e3 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Mon, 22 Feb 2016 20:26:58 +1100 Subject: [PATCH 0006/2608] Prevent multiple simultaneous restores --- bin/ghe-restore | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 53742873c..c4ca82a9b 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -146,10 +146,6 @@ fi echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT" ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP ..." -# Update remote restore state file and setup failure trap -trap "update_restore_status failed" EXIT -update_restore_status "restoring" - # Verify the host has been fully configured at least once if when running # against v11.10.x appliances and the -c option wasn't specified. if [ "$GHE_VERSION_MAJOR" -le 1 ] && ! $restore_settings && ! $instance_configured; then @@ -176,6 +172,16 @@ if $instance_configured; then fi fi +# Make sure the appliance doesn't already have a restore underway +if [ "$GHE_VERSION_MAJOR" -ge 2 ] && ghe-ssh "$GHE_HOSTNAME" -- "sudo grep -q 'restoring' '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' 2>/dev/null"; then + echo "Error: $GHE_HOSTNAME already has a restore underway. Aborting." 1>&2 + exit 1 +fi + +# Update remote restore state file and setup failure trap +trap "update_restore_status failed" EXIT +update_restore_status "restoring" + # Restore settings and license if restoring to an unconfigured appliance or when # specified manually. if $restore_settings; then From a4ab951b0325263863bc39b0ffa1efa325a22228 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Mon, 22 Feb 2016 20:57:42 +1100 Subject: [PATCH 0007/2608] Tests for simultaneous restores --- test/test-ghe-restore.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index 5ca5e5f0f..f761d0f85 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -410,3 +410,37 @@ begin_test "ghe-restore with tarball strategy" echo "$output" | grep -q 'fake ghe-export-repositories data' ) end_test + +begin_test "ghe-restore aborts when another restore is underway" +( + set -e + # This test is only valid for version 2 and above + if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then + rm -rf "$GHE_REMOTE_ROOT_DIR" + setup_remote_metadata + + # create file used to determine if instance has been configured. + touch "$GHE_REMOTE_ROOT_DIR/etc/github/configured" + + # create file used to determine if instance is in maintenance mode. + mkdir -p "$GHE_REMOTE_DATA_DIR/github/current/public/system" + touch "$GHE_REMOTE_DATA_DIR/github/current/public/system/maintenance.html" + + # create file to indicate restore is underway + echo "restoring" > "$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status" + + # set restore host environ var + GHE_RESTORE_HOST=127.0.0.1 + export GHE_RESTORE_HOST + + # run ghe-restore and write output to file for asserting against + # this should fail due to the appliance being in an unconfigured state + ! ghe-restore -v > "$TRASHDIR/restore-out" 2>&1 + + cat $TRASHDIR/restore-out + + # verify that ghe-restore failed due to the appliance not being configured + grep -q -e "already has a restore underway" "$TRASHDIR/restore-out" + fi +) +end_test From 2143e95daf97b2517514c242b32a769d29693950 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Mon, 22 Feb 2016 21:18:10 +1100 Subject: [PATCH 0008/2608] Fix test comment --- test/test-ghe-restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index f761d0f85..b467eaf4e 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -439,7 +439,7 @@ begin_test "ghe-restore aborts when another restore is underway" cat $TRASHDIR/restore-out - # verify that ghe-restore failed due to the appliance not being configured + # verify that ghe-restore failed due a restore already being underway grep -q -e "already has a restore underway" "$TRASHDIR/restore-out" fi ) From f77595b8063a4f3338c614f255dbbdb7aaa84689 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Mon, 22 Feb 2016 21:33:02 +1100 Subject: [PATCH 0009/2608] Minor style tidy up --- bin/ghe-restore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index c4ca82a9b..b7776ee9d 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -173,7 +173,7 @@ if $instance_configured; then fi # Make sure the appliance doesn't already have a restore underway -if [ "$GHE_VERSION_MAJOR" -ge 2 ] && ghe-ssh "$GHE_HOSTNAME" -- "sudo grep -q 'restoring' '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' 2>/dev/null"; then +if [ "$GHE_VERSION_MAJOR" -ge 2 ] && ghe-ssh "$GHE_HOSTNAME" -- "sudo grep -q restoring $GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status 2>/dev/null"; then echo "Error: $GHE_HOSTNAME already has a restore underway. Aborting." 1>&2 exit 1 fi From 0925b39ef602669903dd3517c0ad890e265f5294 Mon Sep 17 00:00:00 2001 From: Daniel Hwang Date: Tue, 23 Feb 2016 10:01:59 -0800 Subject: [PATCH 0010/2608] Remove experimental s3 support --- backup.config-example | 4 ---- share/github-backup-utils/ghe-s3-backup | 26 ------------------------ share/github-backup-utils/ghe-s3-restore | 22 -------------------- 3 files changed, 52 deletions(-) delete mode 100755 share/github-backup-utils/ghe-s3-backup delete mode 100755 share/github-backup-utils/ghe-s3-restore diff --git a/backup.config-example b/backup.config-example index 239c8d37b..89dd3ce85 100644 --- a/backup.config-example +++ b/backup.config-example @@ -32,7 +32,3 @@ GHE_NUM_SNAPSHOTS=10 # Any extra options passed to the rsync command. Nothing required by default. # #GHE_EXTRA_RSYNC_OPTS="" - -# Add s3 bucket for configuring which bucket to use in ghe-s3-backup and -# ghe-s3-restore -# GHE_S3_BUCKET="" diff --git a/share/github-backup-utils/ghe-s3-backup b/share/github-backup-utils/ghe-s3-backup deleted file mode 100755 index 63501a793..000000000 --- a/share/github-backup-utils/ghe-s3-backup +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -#/ Usage: ghe-s3-backup-all -#/ Take snapshots of all GitHub Enterprise data, including the mysql database -#/ and backup to S3. -set -e - -# Bring in the backup configuration -cd $(dirname "$0")/../.. -. share/github-backup-utils/ghe-backup-config - -if [ -e $HOME/.s3cfg ]; then - echo "Using existing ~/.s3cfg ..." -else - # Configure s3cmd if there isn't a config file already. - s3cmd --configure -fi - -# Run the backup script. -ghe-backup - -# Create the bucket if it doesn't exist. -s3cmd mb s3://$GHE_S3_BUCKET - -# Upload to S3. -# --delete-removed is to delete items in s3 that have been removed on the host -s3cmd sync --delete-removed "$GHE_DATA_DIR"/current/ s3://$GHE_S3_BUCKET diff --git a/share/github-backup-utils/ghe-s3-restore b/share/github-backup-utils/ghe-s3-restore deleted file mode 100755 index 2554da8da..000000000 --- a/share/github-backup-utils/ghe-s3-restore +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh -#/ Usage: ghe-s3-restore -#/ Restores backup files from S3. -set -e - -# Bring in the backup configuration. -cd $(dirname "$0")/../.. -. share/github-backup-utils/ghe-backup-config - -if [ -e $HOME/.s3cfg ]; then - echo "Using existing ~/.s3cfg ..." -else - # Configure s3cmd if there isn't a config file already. - s3cmd --configure -fi - -# Restore from S3 into a new snapshot directory. -mkdir -p "$GHE_SNAPSHOT_DIR" -s3cmd --preserve get s3://$GHE_S3_BUCKET/* "$GHE_SNAPSHOT_DIR" - -# Run the restore script. -ghe-restore -s "$GHE_SNAPSHOT_DIR" From 6c5dc101d566dfdc66e97918f1e9e1442c4d1e01 Mon Sep 17 00:00:00 2001 From: Keeran Raj Hawoldar Date: Tue, 23 Feb 2016 23:31:17 +0000 Subject: [PATCH 0011/2608] Highlight symlink importance in snapshots --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index d29ba9e43..4c53d0845 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,11 @@ date and time the snapshot was taken. Each snapshot directory contains a full backup snapshot of all relevant data stores. Repository, Search, and Pages data is stored efficiently via hard links. +*Please note* Symlinks must be maintained when archiving backup snapshots. +Dereferencing or excluding symlinks, or storing the snapshot contents on a +filesystem which does not support symlinks (e.g. Amazon S3) will result in operational +problems when the data is restored. + The following example shows a snapshot file hierarchy for hourly frequency. There are five snapshot directories, with the `current` symlink pointing to the most recent successful snapshot: From 777bb4e66af904c6ac11cf73fc7c58ba87522334 Mon Sep 17 00:00:00 2001 From: Keeran Raj Hawoldar Date: Wed, 24 Feb 2016 00:24:48 +0000 Subject: [PATCH 0012/2608] Remove unnecessary S3 reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c53d0845..9fcb034f2 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ is stored efficiently via hard links. *Please note* Symlinks must be maintained when archiving backup snapshots. Dereferencing or excluding symlinks, or storing the snapshot contents on a -filesystem which does not support symlinks (e.g. Amazon S3) will result in operational +filesystem which does not support symlinks will result in operational problems when the data is restored. The following example shows a snapshot file hierarchy for hourly frequency. From 540460bbe63c15444067573af7d68b4f69108973 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Wed, 24 Feb 2016 16:24:05 +1100 Subject: [PATCH 0013/2608] Fix logging output --- bin/ghe-backup | 2 +- bin/ghe-restore | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 564597dfe..5b990db27 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -94,7 +94,7 @@ if ghe-ssh "$GHE_HOSTNAME" -- \ fi # Log backup start message in /var/log/syslog on remote instance -ghe_remote_logger "Starting backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP ..." +ghe_remote_logger "Starting backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP ..." # Determine whether to use the rsync or tarball backup strategy based on the # remote appliance version. The tarball strategy must be used with GitHub diff --git a/bin/ghe-restore b/bin/ghe-restore index 53742873c..9d427fff5 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -144,7 +144,7 @@ fi # Log restore start message locally and in /var/log/syslog on remote instance echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT" -ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP ..." +ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT ..." # Update remote restore state file and setup failure trap trap "update_restore_status failed" EXIT @@ -275,7 +275,7 @@ trap "" EXIT update_restore_status "complete" # Log restore complete message in /var/log/syslog on remote instance -ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_SNAPSHOT_TIMESTAMP}." +ghe_remote_logger "Completed restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT successfully." if ! $cluster; then echo "Restoring SSH host keys ..." From 50f34334cfda8d57a1f16a7d94c1558556eecf09 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Wed, 24 Feb 2016 17:54:01 +1100 Subject: [PATCH 0014/2608] Remote logging wording --- bin/ghe-backup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 5b990db27..5b297d7b2 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -94,7 +94,7 @@ if ghe-ssh "$GHE_HOSTNAME" -- \ fi # Log backup start message in /var/log/syslog on remote instance -ghe_remote_logger "Starting backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP ..." +ghe_remote_logger "Starting backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP ..." # Determine whether to use the rsync or tarball backup strategy based on the # remote appliance version. The tarball strategy must be used with GitHub @@ -213,10 +213,10 @@ echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $ # Exit non-zero and list the steps that failed. if [ -z "$failures" ]; then - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." + ghe_remote_logger "Completed backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." else steps="$(echo $failures | sed 's/ /, /g')" - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." + ghe_remote_logger "Completed backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." echo "Error: Snapshot incomplete. Some steps failed: ${steps}. " exit 1 fi From 5fec8c0223dba3458820a6a238928517cf7625a3 Mon Sep 17 00:00:00 2001 From: Daniel Hwang Date: Tue, 23 Feb 2016 22:57:09 -0800 Subject: [PATCH 0015/2608] Use consistent logging format as ghe-backup --- bin/ghe-restore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 9d427fff5..0244d5c51 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -144,7 +144,7 @@ fi # Log restore start message locally and in /var/log/syslog on remote instance echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT" -ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT ..." +ghe_remote_logger "Starting restore from $(hostname) in snapshot $GHE_RESTORE_SNAPSHOT ..." # Update remote restore state file and setup failure trap trap "update_restore_status failed" EXIT @@ -275,7 +275,7 @@ trap "" EXIT update_restore_status "complete" # Log restore complete message in /var/log/syslog on remote instance -ghe_remote_logger "Completed restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT successfully." +ghe_remote_logger "Completed restore from $(hostname) in snapshot $GHE_RESTORE_SNAPSHOT successfully." if ! $cluster; then echo "Restoring SSH host keys ..." From 8c7cbe7bf201702af5e4cc3834ace4e436672e52 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Thu, 25 Feb 2016 09:41:57 +1100 Subject: [PATCH 0016/2608] Revert log message format --- bin/ghe-backup | 4 ++-- bin/ghe-restore | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/ghe-backup b/bin/ghe-backup index 5b297d7b2..564597dfe 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -213,10 +213,10 @@ echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $ # Exit non-zero and list the steps that failed. if [ -z "$failures" ]; then - ghe_remote_logger "Completed backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." + ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." else steps="$(echo $failures | sed 's/ /, /g')" - ghe_remote_logger "Completed backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." + ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." echo "Error: Snapshot incomplete. Some steps failed: ${steps}. " exit 1 fi diff --git a/bin/ghe-restore b/bin/ghe-restore index 0244d5c51..52b650138 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -144,7 +144,7 @@ fi # Log restore start message locally and in /var/log/syslog on remote instance echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT" -ghe_remote_logger "Starting restore from $(hostname) in snapshot $GHE_RESTORE_SNAPSHOT ..." +ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT ..." # Update remote restore state file and setup failure trap trap "update_restore_status failed" EXIT @@ -275,7 +275,7 @@ trap "" EXIT update_restore_status "complete" # Log restore complete message in /var/log/syslog on remote instance -ghe_remote_logger "Completed restore from $(hostname) in snapshot $GHE_RESTORE_SNAPSHOT successfully." +ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}." if ! $cluster; then echo "Restoring SSH host keys ..." From f36f431386c3128512566a25040fa5f1bdf17dae Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Tue, 16 Feb 2016 17:28:25 +0100 Subject: [PATCH 0017/2608] Ignore archived gists The server will send a gist-not-found line when the gist is not in the gists database table. --- share/github-backup-utils/ghe-restore-repositories-gist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index a4337bcce..07eafa0ed 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -85,6 +85,11 @@ for gist_path in $gist_paths; do echo "$gist_id" >&4 read routes < ssh_routes_in + if [ "$routes" = 'gist-not-found' ]; then + echo " Warning: gist $gist_id not found in the database (deleted?), ignoring." + continue + fi + for route in $routes; do ghe-rsync -aHR --delete \ -e "ssh -q $opts -p $port -F $config_file -l $user" \ From 716e792f291c28bb72c9dcd6e56efd9c6d037cfc Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Mon, 22 Feb 2016 13:58:43 +0100 Subject: [PATCH 0018/2608] Ignore invalid page IDs Needs https://github.com/github/github/pull/51274 --- share/github-backup-utils/ghe-restore-pages-dpages | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-pages-dpages b/share/github-backup-utils/ghe-restore-pages-dpages index ff2746f47..e22fa31de 100755 --- a/share/github-backup-utils/ghe-restore-pages-dpages +++ b/share/github-backup-utils/ghe-restore-pages-dpages @@ -85,6 +85,11 @@ for pages_path in $pages_paths; do echo "$page_id" >&4 read routes < ssh_routes_in + if [ "$routes" = 'page-id-not-found' ]; then + echo " Warning: page ID $page_id not found in the database, ignoring." + continue + fi + for route in $routes; do ghe-rsync -aHR --delete \ -e "ssh -q $opts -p $port -F $config_file -l $user" \ From 5aa5f6dd9337aa387d316484f1537ec21cb752c1 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Tue, 23 Feb 2016 12:00:54 +0100 Subject: [PATCH 0019/2608] Read output from storage-cluster-import-finalize Needs #XXXXX --- share/github-backup-utils/ghe-restore-alambic-cluster | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster b/share/github-backup-utils/ghe-restore-alambic-cluster index 368c545e3..c91b9e731 100755 --- a/share/github-backup-utils/ghe-restore-alambic-cluster +++ b/share/github-backup-utils/ghe-restore-alambic-cluster @@ -108,14 +108,15 @@ for storage_path in $storage_paths; do done echo "$oid $routes" >&5 + read output < ssh_finalize_in done exec 4>&- exec 5>&- # Ensure to flush these and close the pipes -cat ssh_routes_in > /dev/null -cat ssh_finalize_in > /dev/null +cat ssh_routes_in > /dev/null & +cat ssh_finalize_in > /dev/null & wait $ssh_routes_pid > /dev/null 1>&2 || true wait $ssh_finalize_pid > /dev/null 1>&2 || true From fd8d9de2f13156b576acdda8fa9224c21395b853 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Tue, 23 Feb 2016 14:18:03 +0100 Subject: [PATCH 0020/2608] Add ServerAliveInterval Mostly for consistency with other scripts. --- share/github-backup-utils/ghe-restore-alambic-cluster | 1 + 1 file changed, 1 insertion(+) diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster b/share/github-backup-utils/ghe-restore-alambic-cluster index c91b9e731..a20998a6b 100755 --- a/share/github-backup-utils/ghe-restore-alambic-cluster +++ b/share/github-backup-utils/ghe-restore-alambic-cluster @@ -47,6 +47,7 @@ hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | for hostname in $hostnames; do config="$config Host $hostname + ServerAliveInterval 60 ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p" done From c94486190f2e0f0880d7490e8356d6fae729af5e Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Wed, 24 Feb 2016 14:28:27 +0100 Subject: [PATCH 0021/2608] ssh_finalize_in pipe not required Also fixed some stderr redirections. --- .../github-backup-utils/ghe-restore-alambic-cluster | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster b/share/github-backup-utils/ghe-restore-alambic-cluster index a20998a6b..a20f5f6b5 100755 --- a/share/github-backup-utils/ghe-restore-alambic-cluster +++ b/share/github-backup-utils/ghe-restore-alambic-cluster @@ -60,22 +60,21 @@ cleanup() { for pid in $(jobs -p); do kill -KILL $pid > /dev/null 1>&2 || true done - rm -rf $config_file ssh_routes_in ssh_routes_out ssh_finalize_in ssh_finalize_out + rm -rf $config_file ssh_routes_in ssh_routes_out ssh_finalize_out } trap 'cleanup' INT TERM EXIT -rm -rf ssh_routes_in ssh_routes_out ssh_finalize_in ssh_finalize_out +rm -rf ssh_routes_in ssh_routes_out ssh_finalize_out mkfifo ssh_routes_in mkfifo ssh_routes_out -mkfifo ssh_finalize_in mkfifo ssh_finalize_out echo "Setting up storage processes" ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-routes - < ssh_routes_out > ssh_routes_in & ssh_routes_pid=$! -ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-finalize - < ssh_finalize_out > ssh_finalize_in & +ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-finalize - < ssh_finalize_out & ssh_finalize_pid=$! echo "Set up storage processes" @@ -109,7 +108,6 @@ for storage_path in $storage_paths; do done echo "$oid $routes" >&5 - read output < ssh_finalize_in done exec 4>&- @@ -117,7 +115,6 @@ exec 5>&- # Ensure to flush these and close the pipes cat ssh_routes_in > /dev/null & -cat ssh_finalize_in > /dev/null & -wait $ssh_routes_pid > /dev/null 1>&2 || true -wait $ssh_finalize_pid > /dev/null 1>&2 || true +wait $ssh_routes_pid > /dev/null 2>&1 || true +wait $ssh_finalize_pid > /dev/null 2>&1 || true From 0b1c4d0ace1575f332ba86b282982fbe166147d1 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Thu, 25 Feb 2016 18:15:01 +0100 Subject: [PATCH 0022/2608] Clustering: ghe-restore fixes More robust restores when restoring gists, pages and storage data into a cluster. --- .../ghe-restore-alambic-cluster | 15 +++++++-------- .../github-backup-utils/ghe-restore-pages-dpages | 5 +++++ .../ghe-restore-repositories-gist | 5 +++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/share/github-backup-utils/ghe-restore-alambic-cluster b/share/github-backup-utils/ghe-restore-alambic-cluster index 368c545e3..a20f5f6b5 100755 --- a/share/github-backup-utils/ghe-restore-alambic-cluster +++ b/share/github-backup-utils/ghe-restore-alambic-cluster @@ -47,6 +47,7 @@ hostnames=$(ghe-ssh "$GHE_HOSTNAME" ghe-config --get-regexp cluster.*.hostname | for hostname in $hostnames; do config="$config Host $hostname + ServerAliveInterval 60 ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p" done @@ -59,22 +60,21 @@ cleanup() { for pid in $(jobs -p); do kill -KILL $pid > /dev/null 1>&2 || true done - rm -rf $config_file ssh_routes_in ssh_routes_out ssh_finalize_in ssh_finalize_out + rm -rf $config_file ssh_routes_in ssh_routes_out ssh_finalize_out } trap 'cleanup' INT TERM EXIT -rm -rf ssh_routes_in ssh_routes_out ssh_finalize_in ssh_finalize_out +rm -rf ssh_routes_in ssh_routes_out ssh_finalize_out mkfifo ssh_routes_in mkfifo ssh_routes_out -mkfifo ssh_finalize_in mkfifo ssh_finalize_out echo "Setting up storage processes" ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-routes - < ssh_routes_out > ssh_routes_in & ssh_routes_pid=$! -ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-finalize - < ssh_finalize_out > ssh_finalize_in & +ghe-ssh "$GHE_HOSTNAME" github-env ./bin/storage-cluster-import-finalize - < ssh_finalize_out & ssh_finalize_pid=$! echo "Set up storage processes" @@ -114,8 +114,7 @@ exec 4>&- exec 5>&- # Ensure to flush these and close the pipes -cat ssh_routes_in > /dev/null -cat ssh_finalize_in > /dev/null +cat ssh_routes_in > /dev/null & -wait $ssh_routes_pid > /dev/null 1>&2 || true -wait $ssh_finalize_pid > /dev/null 1>&2 || true +wait $ssh_routes_pid > /dev/null 2>&1 || true +wait $ssh_finalize_pid > /dev/null 2>&1 || true diff --git a/share/github-backup-utils/ghe-restore-pages-dpages b/share/github-backup-utils/ghe-restore-pages-dpages index ff2746f47..e22fa31de 100755 --- a/share/github-backup-utils/ghe-restore-pages-dpages +++ b/share/github-backup-utils/ghe-restore-pages-dpages @@ -85,6 +85,11 @@ for pages_path in $pages_paths; do echo "$page_id" >&4 read routes < ssh_routes_in + if [ "$routes" = 'page-id-not-found' ]; then + echo " Warning: page ID $page_id not found in the database, ignoring." + continue + fi + for route in $routes; do ghe-rsync -aHR --delete \ -e "ssh -q $opts -p $port -F $config_file -l $user" \ diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index a4337bcce..07eafa0ed 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -85,6 +85,11 @@ for gist_path in $gist_paths; do echo "$gist_id" >&4 read routes < ssh_routes_in + if [ "$routes" = 'gist-not-found' ]; then + echo " Warning: gist $gist_id not found in the database (deleted?), ignoring." + continue + fi + for route in $routes; do ghe-rsync -aHR --delete \ -e "ssh -q $opts -p $port -F $config_file -l $user" \ From 11385242fcf4e60f9aebdede3a018b59158c2088 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Thu, 25 Feb 2016 18:19:56 +0100 Subject: [PATCH 0023/2608] Consistent warning message --- share/github-backup-utils/ghe-restore-repositories-gist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index 07eafa0ed..a76866b06 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -86,7 +86,7 @@ for gist_path in $gist_paths; do read routes < ssh_routes_in if [ "$routes" = 'gist-not-found' ]; then - echo " Warning: gist $gist_id not found in the database (deleted?), ignoring." + echo " Warning: gist $gist_id not found in the database, ignoring." continue fi From 011bdd8f0ddc488abd7607b4e563fffc29cbace8 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Thu, 25 Feb 2016 18:47:49 +0100 Subject: [PATCH 0024/2608] :lipstick: warning message --- share/github-backup-utils/ghe-restore-repositories-gist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-restore-repositories-gist b/share/github-backup-utils/ghe-restore-repositories-gist index a76866b06..ea399e786 100755 --- a/share/github-backup-utils/ghe-restore-repositories-gist +++ b/share/github-backup-utils/ghe-restore-repositories-gist @@ -86,7 +86,7 @@ for gist_path in $gist_paths; do read routes < ssh_routes_in if [ "$routes" = 'gist-not-found' ]; then - echo " Warning: gist $gist_id not found in the database, ignoring." + echo " Warning: gist ID $gist_id not found in the database, ignoring." continue fi From 0a078b4a598e639d32fa6e4c2f395deaa2fb745d Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Thu, 25 Feb 2016 19:50:53 +0100 Subject: [PATCH 0025/2608] Revert "Prevent multiple restores from running at the same time" --- bin/ghe-restore | 14 ++++---------- test/test-ghe-restore.sh | 34 ---------------------------------- 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/bin/ghe-restore b/bin/ghe-restore index 0ba3ff90d..52b650138 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -146,6 +146,10 @@ fi echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT" ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT ..." +# Update remote restore state file and setup failure trap +trap "update_restore_status failed" EXIT +update_restore_status "restoring" + # Verify the host has been fully configured at least once if when running # against v11.10.x appliances and the -c option wasn't specified. if [ "$GHE_VERSION_MAJOR" -le 1 ] && ! $restore_settings && ! $instance_configured; then @@ -172,16 +176,6 @@ if $instance_configured; then fi fi -# Make sure the appliance doesn't already have a restore underway -if [ "$GHE_VERSION_MAJOR" -ge 2 ] && ghe-ssh "$GHE_HOSTNAME" -- "sudo grep -q restoring $GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status 2>/dev/null"; then - echo "Error: $GHE_HOSTNAME already has a restore underway. Aborting." 1>&2 - exit 1 -fi - -# Update remote restore state file and setup failure trap -trap "update_restore_status failed" EXIT -update_restore_status "restoring" - # Restore settings and license if restoring to an unconfigured appliance or when # specified manually. if $restore_settings; then diff --git a/test/test-ghe-restore.sh b/test/test-ghe-restore.sh index b467eaf4e..5ca5e5f0f 100755 --- a/test/test-ghe-restore.sh +++ b/test/test-ghe-restore.sh @@ -410,37 +410,3 @@ begin_test "ghe-restore with tarball strategy" echo "$output" | grep -q 'fake ghe-export-repositories data' ) end_test - -begin_test "ghe-restore aborts when another restore is underway" -( - set -e - # This test is only valid for version 2 and above - if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then - rm -rf "$GHE_REMOTE_ROOT_DIR" - setup_remote_metadata - - # create file used to determine if instance has been configured. - touch "$GHE_REMOTE_ROOT_DIR/etc/github/configured" - - # create file used to determine if instance is in maintenance mode. - mkdir -p "$GHE_REMOTE_DATA_DIR/github/current/public/system" - touch "$GHE_REMOTE_DATA_DIR/github/current/public/system/maintenance.html" - - # create file to indicate restore is underway - echo "restoring" > "$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status" - - # set restore host environ var - GHE_RESTORE_HOST=127.0.0.1 - export GHE_RESTORE_HOST - - # run ghe-restore and write output to file for asserting against - # this should fail due to the appliance being in an unconfigured state - ! ghe-restore -v > "$TRASHDIR/restore-out" 2>&1 - - cat $TRASHDIR/restore-out - - # verify that ghe-restore failed due a restore already being underway - grep -q -e "already has a restore underway" "$TRASHDIR/restore-out" - fi -) -end_test From 02b4552675a1dc3d4e8298a0c23f108c6887cc55 Mon Sep 17 00:00:00 2001 From: Ben Gollmer Date: Thu, 25 Feb 2016 14:42:23 -0700 Subject: [PATCH 0026/2608] Ask for index field instead of using cut --- share/github-backup-utils/ghe-backup-es-audit-log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log index 7ef6e40e3..c67e7c7dd 100755 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ b/share/github-backup-utils/ghe-backup-es-audit-log @@ -19,7 +19,7 @@ ghe_remote_version_required "$host" # Make sure root backup dir exists if this is the first run mkdir -p "$GHE_SNAPSHOT_DIR/audit-log" -indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/audit_log*"' | cut -d ' ' -f 3) +indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/audit_log*?h=index"') || true current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"') for index in $indices; do From 061905edcef6a03f62284c1e28c152d7f43bf734 Mon Sep 17 00:00:00 2001 From: Ben Gollmer Date: Thu, 25 Feb 2016 16:09:03 -0700 Subject: [PATCH 0027/2608] Return empty index list for ghe-backup-es-audit-log --- test/bin/curl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/bin/curl b/test/bin/curl index 64545c39a..d1234b872 100755 --- a/test/bin/curl +++ b/test/bin/curl @@ -3,6 +3,11 @@ # Fake curl command stub for tests. set -e +# Return empty list of indexes for ghe-backup-es-audit-log +if echo "$@" | grep -q '_cat/indices/audit_log\*?h=index$'; then + exit 0 +fi + # Write args to stdout echo "$@" From d4c6de828bf0f167696ca4ba739f955dab109574 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Fri, 26 Feb 2016 20:23:42 +1100 Subject: [PATCH 0028/2608] Use .sync_in_progress lock file during restore --- .../ghe-restore-repositories-rsync | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/share/github-backup-utils/ghe-restore-repositories-rsync b/share/github-backup-utils/ghe-restore-repositories-rsync index d0ae56aeb..b99c74756 100755 --- a/share/github-backup-utils/ghe-restore-repositories-rsync +++ b/share/github-backup-utils/ghe-restore-repositories-rsync @@ -19,13 +19,52 @@ GHE_HOSTNAME="$1" # Perform a host-check and establish GHE_REMOTE_XXX variables. ghe_remote_version_required "$GHE_HOSTNAME" +# Remote sync-in-progress file location. When this file exists, Git GC +# operations are disabled on the GitHub instance. +sync_in_progress_file="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress" + # The snapshot to restore should be set by the ghe-restore command but this lets # us run this script directly. : ${GHE_RESTORE_SNAPSHOT:=current} +# Removes the remote sync-in-progress file on exit, re-enabling GC operations +# on the remote instance. +cleanup() { + ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -f '$sync_in_progress_file'" +} +trap 'cleanup' EXIT +trap 'exit $?' INT # ^C always terminate + +# Touch the sync-in-progress file, disabling GC operations, and wait for all +# active GC processes to finish on the remote side. +echo " + set -e + sudo -u git touch '$sync_in_progress_file' + + sanity=0 + while [ \$sanity -lt $GHE_GIT_COOLDOWN_PERIOD ]; do + # note: the bracket synta[x] below is to prevent matches against the + # grep process itself. + if ps axo args | grep -E -e '^git( -.*)? nw-repac[k]( |$)' -e '^git( -.*)? g[c]( |$)' >/dev/null; then + sleep 1 + sanity=\$(( sanity + 1 )) + else + exit 0 + fi + done + exit 7 +" | ghe-ssh "$GHE_HOSTNAME" -- /bin/sh || { + res=$? + if [ $res = 7 ]; then + echo "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds. Aborting..." 1>&2 + fi + exit $res +} + # Transfer all git repository data from the latest snapshot to the GitHub # instance in a single rsync invocation. ghe-rsync -avH --delete \ + --exclude ".sync_in_progress" \ -e "ghe-ssh -p $(ssh_port_part "$GHE_HOSTNAME")" \ --rsync-path="sudo -u git rsync" \ "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/repositories/" \ From a89ec70c61377904a643fea1f0a13241617c87b5 Mon Sep 17 00:00:00 2001 From: Sergio Rubio Date: Mon, 29 Feb 2016 13:18:52 +0100 Subject: [PATCH 0029/2608] Exit 1 if ssh/curl fail to retrieve the indices --- share/github-backup-utils/ghe-backup-es-audit-log | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log index c67e7c7dd..014b66fb9 100755 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ b/share/github-backup-utils/ghe-backup-es-audit-log @@ -19,7 +19,11 @@ ghe_remote_version_required "$host" # Make sure root backup dir exists if this is the first run mkdir -p "$GHE_SNAPSHOT_DIR/audit-log" -indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/audit_log*?h=index"') || true +if ! indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/audit_log*?h=index"'); then + echo "Error: failed to retrieve audit log indices." 1>&2 + exit 1 +fi + current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"') for index in $indices; do From 5e3c9096d6fa7653136a3f9965ee0639d7670817 Mon Sep 17 00:00:00 2001 From: Steven Honson Date: Tue, 1 Mar 2016 17:44:55 +1100 Subject: [PATCH 0030/2608] Change to bash --- README.md | 6 +++--- bin/ghe-backup | 2 +- bin/ghe-host-check | 2 +- bin/ghe-restore | 2 +- script/cibuild | 2 +- script/package-deb | 2 +- script/package-tarball | 2 +- share/github-backup-utils/ghe-backup-alambic-cluster | 2 +- share/github-backup-utils/ghe-backup-config | 2 +- share/github-backup-utils/ghe-backup-es-audit-log | 2 +- share/github-backup-utils/ghe-backup-es-hookshot | 2 +- share/github-backup-utils/ghe-backup-es-rsync | 2 +- share/github-backup-utils/ghe-backup-es-tarball | 2 +- share/github-backup-utils/ghe-backup-pages-cluster | 2 +- share/github-backup-utils/ghe-backup-pages-rsync | 2 +- share/github-backup-utils/ghe-backup-pages-tarball | 2 +- share/github-backup-utils/ghe-backup-redis | 2 +- share/github-backup-utils/ghe-backup-redis-cluster | 2 +- share/github-backup-utils/ghe-backup-repositories-cluster | 2 +- share/github-backup-utils/ghe-backup-repositories-rsync | 2 +- share/github-backup-utils/ghe-backup-repositories-tarball | 2 +- share/github-backup-utils/ghe-backup-settings | 2 +- share/github-backup-utils/ghe-backup-userdata | 2 +- share/github-backup-utils/ghe-maintenance-mode-disable | 2 +- share/github-backup-utils/ghe-maintenance-mode-enable | 2 +- share/github-backup-utils/ghe-maintenance-mode-status | 2 +- share/github-backup-utils/ghe-prune-snapshots | 2 +- share/github-backup-utils/ghe-restore-es-audit-log | 2 +- share/github-backup-utils/ghe-restore-es-rsync | 2 +- share/github-backup-utils/ghe-restore-es-tarball | 2 +- share/github-backup-utils/ghe-restore-pages-rsync | 2 +- share/github-backup-utils/ghe-restore-pages-tarball | 2 +- share/github-backup-utils/ghe-restore-repositories-rsync | 2 +- share/github-backup-utils/ghe-restore-repositories-tarball | 2 +- share/github-backup-utils/ghe-restore-settings | 2 +- share/github-backup-utils/ghe-restore-snapshot-path | 2 +- share/github-backup-utils/ghe-restore-userdata | 2 +- share/github-backup-utils/ghe-ssh | 2 +- test/bin/chown | 2 +- test/bin/curl | 2 +- test/bin/enterprise-configure | 2 +- test/bin/ghe-config-apply | 2 +- test/bin/ghe-es-snapshot | 2 +- test/bin/ghe-fake-export-command | 2 +- test/bin/ghe-fake-import-command | 2 +- test/bin/ghe-maintenance | 2 +- test/bin/ghe-service-ensure-elasticsearch | 2 +- test/bin/ghe-service-ensure-mysql | 2 +- test/bin/ionice-stub | 2 +- test/bin/python | 2 +- test/bin/redis-cli | 2 +- test/bin/ssh | 2 +- test/test-bashisms.sh | 2 +- test/test-ghe-backup-config.sh | 2 +- test/test-ghe-backup-repositories-rsync-nw.sh | 2 +- test/test-ghe-backup-repositories-rsync.sh | 2 +- test/test-ghe-backup.sh | 2 +- test/test-ghe-host-check.sh | 2 +- test/test-ghe-prune-snapshots.sh | 2 +- test/test-ghe-restore.sh | 2 +- test/test-ghe-ssh.sh | 2 +- test/testlib.sh | 2 +- 62 files changed, 64 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 9fcb034f2..1f527009a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ storage and must have network connectivity with the GitHub Enterprise appliance. ##### Backup host requirements Backup host software requirements are modest: Linux or other modern Unix -operating system with [rsync][4] v2.6.4 or newer. +operating system with bash and [rsync][4] v2.6.4 or newer. The backup host must be able to establish network connections outbound to the GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise 2.0 or newer instances, and TCP port 22 is used for older versions (11.10.34X). @@ -211,8 +211,8 @@ backup snapshot of all relevant data stores. Repository, Search, and Pages data is stored efficiently via hard links. *Please note* Symlinks must be maintained when archiving backup snapshots. -Dereferencing or excluding symlinks, or storing the snapshot contents on a -filesystem which does not support symlinks will result in operational +Dereferencing or excluding symlinks, or storing the snapshot contents on a +filesystem which does not support symlinks will result in operational problems when the data is restored. The following example shows a snapshot file hierarchy for hourly frequency. diff --git a/bin/ghe-backup b/bin/ghe-backup index 564597dfe..39048db1e 100755 --- a/bin/ghe-backup +++ b/bin/ghe-backup @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup [-v] #/ Take snapshots of all GitHub Enterprise data, including Git repository data, #/ the MySQL database, instance settings, GitHub Pages data, etc. diff --git a/bin/ghe-host-check b/bin/ghe-host-check index 7d5833411..b75a3b59d 100755 --- a/bin/ghe-host-check +++ b/bin/ghe-host-check @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-host-check [] #/ Verify connectivity with the GitHub Enterprise host. When no is #/ provided, the $GHE_HOSTNAME configured in backup.config is assumed. diff --git a/bin/ghe-restore b/bin/ghe-restore index 52b650138..7146de8ec 100755 --- a/bin/ghe-restore +++ b/bin/ghe-restore @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore [-v] [-s ] [] #/ Restores a GitHub instance from local backup snapshots. The is the #/ hostname or IP of the GitHub instance. The may be omitted when diff --git a/script/cibuild b/script/cibuild index 553a8d035..28839c30c 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Usage: script/cibuild [--no-package] set -e diff --git a/script/package-deb b/script/package-deb index 6e3d9e829..b42f0d5fd 100755 --- a/script/package-deb +++ b/script/package-deb @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Usage: script/package-deb # Script to build a deb release package from the current HEAD version. # The package version comes from the debian/changelog file so that should diff --git a/script/package-tarball b/script/package-tarball index 6229cc33d..65f4ed51d 100755 --- a/script/package-tarball +++ b/script/package-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Usage: script/package-tarball # Script to build a tarball release package from the current HEAD version. # The package version comes from `git-describe --tags' so the release tag should diff --git a/share/github-backup-utils/ghe-backup-alambic-cluster b/share/github-backup-utils/ghe-backup-alambic-cluster index 7d4276692..f92a7180e 100755 --- a/share/github-backup-utils/ghe-backup-alambic-cluster +++ b/share/github-backup-utils/ghe-backup-alambic-cluster @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-alambic-cluster #/ Take an online, incremental snapshot of all Alambic Storage data #/ diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config index 521a91d2d..c2ff85059 100755 --- a/share/github-backup-utils/ghe-backup-config +++ b/share/github-backup-utils/ghe-backup-config @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Usage: . ghe-backup-config # GitHub Enterprise backup shell configuration. # diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log index 7ef6e40e3..454349f59 100755 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ b/share/github-backup-utils/ghe-backup-es-audit-log @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-es-audit-log #/ Take a backup of audit logs in ElasticSearch. #/ diff --git a/share/github-backup-utils/ghe-backup-es-hookshot b/share/github-backup-utils/ghe-backup-es-hookshot index 7a69a7e52..9ec41f46a 100755 --- a/share/github-backup-utils/ghe-backup-es-hookshot +++ b/share/github-backup-utils/ghe-backup-es-hookshot @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-es-hookshot #/ Take a backup of hookshot logs in ElasticSearch. #/ diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync index c3d31cca5..50cb3606c 100755 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ b/share/github-backup-utils/ghe-backup-es-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-es-rsync #/ Take an online, incremental snapshot of Elasticsearch indices. #/ diff --git a/share/github-backup-utils/ghe-backup-es-tarball b/share/github-backup-utils/ghe-backup-es-tarball index ebce38c47..febfdb718 100755 --- a/share/github-backup-utils/ghe-backup-es-tarball +++ b/share/github-backup-utils/ghe-backup-es-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-es-tarball #/ Take a tarball snapshot of all Elasticsearch data. #/ diff --git a/share/github-backup-utils/ghe-backup-pages-cluster b/share/github-backup-utils/ghe-backup-pages-cluster index a0807fe5f..9cdd54e43 100755 --- a/share/github-backup-utils/ghe-backup-pages-cluster +++ b/share/github-backup-utils/ghe-backup-pages-cluster @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-pages-cluster #/ Take an online, incremental snapshot of all Pages data #/ diff --git a/share/github-backup-utils/ghe-backup-pages-rsync b/share/github-backup-utils/ghe-backup-pages-rsync index 9de913183..318562a91 100755 --- a/share/github-backup-utils/ghe-backup-pages-rsync +++ b/share/github-backup-utils/ghe-backup-pages-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-pages-rsync #/ Take an online, incremental snapshot of all Pages data. #/ diff --git a/share/github-backup-utils/ghe-backup-pages-tarball b/share/github-backup-utils/ghe-backup-pages-tarball index 7e8bda265..9c56620ef 100755 --- a/share/github-backup-utils/ghe-backup-pages-tarball +++ b/share/github-backup-utils/ghe-backup-pages-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-pages-tarball #/ Take a tarball snapshot of all Pages data. #/ diff --git a/share/github-backup-utils/ghe-backup-redis b/share/github-backup-utils/ghe-backup-redis index dbd1e796f..2ebe3cd3a 100755 --- a/share/github-backup-utils/ghe-backup-redis +++ b/share/github-backup-utils/ghe-backup-redis @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-redis #/ Take a snapshot of all Redis data. This is needed because older versions of #/ the remote side ghe-export-redis command use a blocking SAVE instead of a diff --git a/share/github-backup-utils/ghe-backup-redis-cluster b/share/github-backup-utils/ghe-backup-redis-cluster index d571adb8d..fb4e7e13f 100755 --- a/share/github-backup-utils/ghe-backup-redis-cluster +++ b/share/github-backup-utils/ghe-backup-redis-cluster @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-redis-cluster #/ Take a snapshot of all Redis data. This is needed because older versions of #/ the remote side ghe-export-redis command use a blocking SAVE instead of a diff --git a/share/github-backup-utils/ghe-backup-repositories-cluster b/share/github-backup-utils/ghe-backup-repositories-cluster index 54a6931cc..2ccb1dfed 100755 --- a/share/github-backup-utils/ghe-backup-repositories-cluster +++ b/share/github-backup-utils/ghe-backup-repositories-cluster @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-repositories-cluster #/ Take an online, incremental snapshot of all Git repository data. #/ diff --git a/share/github-backup-utils/ghe-backup-repositories-rsync b/share/github-backup-utils/ghe-backup-repositories-rsync index be94160bb..9cc31f02f 100755 --- a/share/github-backup-utils/ghe-backup-repositories-rsync +++ b/share/github-backup-utils/ghe-backup-repositories-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-repositories-rsync #/ Take an online, incremental snapshot of all Git repository data. #/ diff --git a/share/github-backup-utils/ghe-backup-repositories-tarball b/share/github-backup-utils/ghe-backup-repositories-tarball index 09ee15ac4..87f695608 100755 --- a/share/github-backup-utils/ghe-backup-repositories-tarball +++ b/share/github-backup-utils/ghe-backup-repositories-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-repositories-tarball #/ Take a tarball snapshot of all Git repository data. #/ diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings index a6691c740..25e6d7ae7 100755 --- a/share/github-backup-utils/ghe-backup-settings +++ b/share/github-backup-utils/ghe-backup-settings @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-settings #/ Restore settings from a snapshot to the given . set -e diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata index 2c39a9827..0800b54d5 100755 --- a/share/github-backup-utils/ghe-backup-userdata +++ b/share/github-backup-utils/ghe-backup-userdata @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-backup-userdata #/ Take an online, incremental snapshot of a user data directory. This is used #/ for a number of different simple datastores kept under /data/user on the diff --git a/share/github-backup-utils/ghe-maintenance-mode-disable b/share/github-backup-utils/ghe-maintenance-mode-disable index 5df69d785..9c04cff2c 100755 --- a/share/github-backup-utils/ghe-maintenance-mode-disable +++ b/share/github-backup-utils/ghe-maintenance-mode-disable @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-maintenance-mode-disable #/ Disable maintenance mode on GitHub appliance at . This opens up access #/ to the appliance. diff --git a/share/github-backup-utils/ghe-maintenance-mode-enable b/share/github-backup-utils/ghe-maintenance-mode-enable index eee86cd3d..74673e402 100755 --- a/share/github-backup-utils/ghe-maintenance-mode-enable +++ b/share/github-backup-utils/ghe-maintenance-mode-enable @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-maintenance-mode-enable [-w] #/ Enable maintenance mode on GitHub appliance at . This locks down all #/ access to the appliance to prevent writes to datastores and waits for all diff --git a/share/github-backup-utils/ghe-maintenance-mode-status b/share/github-backup-utils/ghe-maintenance-mode-status index 679ebb894..2922989cc 100755 --- a/share/github-backup-utils/ghe-maintenance-mode-status +++ b/share/github-backup-utils/ghe-maintenance-mode-status @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-maintenance-mode-status #/ Checks the status of maintenance mode on GitHub appliance at . set -e diff --git a/share/github-backup-utils/ghe-prune-snapshots b/share/github-backup-utils/ghe-prune-snapshots index 78a30e2e4..051fd7d75 100755 --- a/share/github-backup-utils/ghe-prune-snapshots +++ b/share/github-backup-utils/ghe-prune-snapshots @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-prune-snapshots #/ Keep N latest backup snapshots. set -e diff --git a/share/github-backup-utils/ghe-restore-es-audit-log b/share/github-backup-utils/ghe-restore-es-audit-log index e224dcfc6..7014d0332 100755 --- a/share/github-backup-utils/ghe-restore-es-audit-log +++ b/share/github-backup-utils/ghe-restore-es-audit-log @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-es-audit-log #/ Take a backup of audit logs in ElasticSearch. #/ diff --git a/share/github-backup-utils/ghe-restore-es-rsync b/share/github-backup-utils/ghe-restore-es-rsync index d1b769339..f8d93af88 100755 --- a/share/github-backup-utils/ghe-restore-es-rsync +++ b/share/github-backup-utils/ghe-restore-es-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-es-rsync #/ Restore an rsync snapshot of all Elasticsearch data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-es-tarball b/share/github-backup-utils/ghe-restore-es-tarball index df5e0e91a..e876b6666 100755 --- a/share/github-backup-utils/ghe-restore-es-tarball +++ b/share/github-backup-utils/ghe-restore-es-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-es-tarball #/ Restore a tarball snapshot of all ES data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-pages-rsync b/share/github-backup-utils/ghe-restore-pages-rsync index ecda7113b..927459f44 100755 --- a/share/github-backup-utils/ghe-restore-pages-rsync +++ b/share/github-backup-utils/ghe-restore-pages-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-pages-rsync #/ Restore an rsync snapshot of all Pages data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-pages-tarball b/share/github-backup-utils/ghe-restore-pages-tarball index 9a4daf5e1..e7a40c840 100755 --- a/share/github-backup-utils/ghe-restore-pages-tarball +++ b/share/github-backup-utils/ghe-restore-pages-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-pages-tarball #/ Restore a tarball snapshot of all Pages data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-repositories-rsync b/share/github-backup-utils/ghe-restore-repositories-rsync index d0ae56aeb..418f50a23 100755 --- a/share/github-backup-utils/ghe-restore-repositories-rsync +++ b/share/github-backup-utils/ghe-restore-repositories-rsync @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-repositories-rsync #/ Restore an rsync snapshot of all Git repository data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-repositories-tarball b/share/github-backup-utils/ghe-restore-repositories-tarball index 3b41830a6..1345c9f74 100755 --- a/share/github-backup-utils/ghe-restore-repositories-tarball +++ b/share/github-backup-utils/ghe-restore-repositories-tarball @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-repositories-tarball #/ Restore a tarball snapshot of all Git repository data to a GitHub instance. #/ diff --git a/share/github-backup-utils/ghe-restore-settings b/share/github-backup-utils/ghe-restore-settings index e3cfe46c9..20116ee87 100755 --- a/share/github-backup-utils/ghe-restore-settings +++ b/share/github-backup-utils/ghe-restore-settings @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-settings #/ Restore settings from a snapshot to the given . set -e diff --git a/share/github-backup-utils/ghe-restore-snapshot-path b/share/github-backup-utils/ghe-restore-snapshot-path index a5a0db8db..18f155917 100755 --- a/share/github-backup-utils/ghe-restore-snapshot-path +++ b/share/github-backup-utils/ghe-restore-snapshot-path @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-snapshot-path [snapshot] #/ #/ Print the path to the given snapshot. Defaults to current if no argument given. diff --git a/share/github-backup-utils/ghe-restore-userdata b/share/github-backup-utils/ghe-restore-userdata index 3ee5942fc..7757546e3 100755 --- a/share/github-backup-utils/ghe-restore-userdata +++ b/share/github-backup-utils/ghe-restore-userdata @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-restore-userdata #/ Restore a special user data directory via rsync. This is used #/ for a number of different simple datastores kept under /data/user on the diff --git a/share/github-backup-utils/ghe-ssh b/share/github-backup-utils/ghe-ssh index 67f7e0fff..7cc3811ae 100755 --- a/share/github-backup-utils/ghe-ssh +++ b/share/github-backup-utils/ghe-ssh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #/ Usage: ghe-ssh [