diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 53028715a..000000000 --- a/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -.git/ -.github/ -.gitattributes -.gitignore diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 9afc93bcc..000000000 --- a/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -.gitignore export-ignore -.gitattributes export-ignore diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index fab2f101d..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,16 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "docker" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" - - - package-ecosystem: "github-actions" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "weekly" diff --git a/.github/workflows/close-pulls.yml b/.github/workflows/close-pulls.yml new file mode 100644 index 000000000..8a95bccc0 --- /dev/null +++ b/.github/workflows/close-pulls.yml @@ -0,0 +1,21 @@ +--- + name: Auto Close Pull Requests + + on: + schedule: + - cron: '0 * * * *' + + jobs: + close-fork-pulls: + runs-on: ubuntu-latest + + steps: + - name: Close Pull Requests + uses: peter-evans/close-fork-pulls@v2 + with: + comment: | + As of 2023-11-30 we have stopped the 2-way sync between this repository and our internal repository, so that our internal repository becomes the source of truth for the backup-utils source code. With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. + + Customers should no longer open pull requests in this repository. These pull requests will not be reviewed or merged. We will automatically close all PRs opened in this repository. + + Customers cannot open issues in this repository. Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. This will ensure all customer requests are handled consistently. \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 0ca099a7f..000000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - -jobs: - - build-docker: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Build the Debian Docker image - run: docker build . --file Dockerfile --tag backup-utils:${GITHUB_RUN_ID} - - name: Build the Alpine Docker image - run: docker build . --file Dockerfile.alpine --tag backup-utils-alpine:${GITHUB_RUN_ID} - - name: Run tests in Debian Docker image - run: docker run backup-utils:${GITHUB_RUN_ID} ghe-backup --version - - name: Run tests in Alpine Docker image - run: docker run backup-utils-alpine:${GITHUB_RUN_ID} ghe-backup --version - diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 9d97805de..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Lint Code Base - -on: - push: - branches-ignore: [master] - pull_request: - branches: [master] - -jobs: - lint: - name: Lint Code Base - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 - - name: Lint Code Base - uses: github/super-linter@v4 - env: - VALIDATE_ALL_CODEBASE: false - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index b4ceacdcd..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Test and build - -on: [pull_request] - - -jobs: - build: - strategy: - matrix: - # macos-latest references are kept here for historical purposes. removed macos-latest from the - #matrix as it is not a typical case for users and causes a lot of friction with other linux-based - # installs. Recommend developing on codespaces or using an ubuntu container. - os: ['ubuntu-22.04', 'ubuntu-20.04', 'ubuntu-18.04'] - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Install Dependencies (Linux) - run: | - sudo apt-get update -y - sudo apt-get install -y devscripts debhelper moreutils fakeroot jq pigz help2man - wget "https://github.com/koalaman/shellcheck/releases/download/latest/shellcheck-latest.linux.x86_64.tar.xz" - tar --xz -xvf "shellcheck-latest.linux.x86_64.tar.xz" - sudo cp shellcheck-latest/shellcheck /usr/bin/shellcheck - if: matrix.os != 'macos-latest' - - name: Install Dependencies (macOS) - run: | - brew install gnu-tar shellcheck jq pigz coreutils gnu-sed gnu-getopt wget - brew install moreutils gawk - if: matrix.os == 'macos-latest' - - name: Get Sources - uses: actions/checkout@v3 - - name: Test - run: | - export PATH="$PATH:/snap/bin" - make test - shell: bash - - name: Build (Linux) - run: DEB_BUILD_OPTIONS=nocheck debuild -us -uc - if: matrix.os != 'macos-latest' diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 246384181..000000000 --- a/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/backup.config -/data -/dist diff --git a/.ruby-version b/.ruby-version deleted file mode 100644 index 197c4d5c2..000000000 --- a/.ruby-version +++ /dev/null @@ -1 +0,0 @@ -2.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 042d7c71a..000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,12 +0,0 @@ -# Contributing - -Looking to contribute something to this project? That is great, we always appreciate pull requests! Here's how you can help: - -1. Fork the project to your account. -2. Clone the fork (`git clone https://github.com/[username]/backup-utils.git`). -3. Create a new feature branch (`git checkout -b my-feature-branch`). -4. Add and then commit your changes (`git commit -am "Add a new backup endpoint."`). -5. Push your feature branch to GitHub.com (`git push -u origin my-feature-branch`). -6. Open a [Pull Request](https://github.com/github/backup-utils/compare/) and wait for our feedback. - -Have a look at the [styleguide](https://github.com/github/backup-utils/tree/master/STYLEGUIDE.md) to make sure your code style is consistent with the code in this repository. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 9e016dc03..000000000 --- a/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM ubuntu:focal - -RUN apt-get -q -y update && \ - apt-get install -y --no-install-recommends \ - tar \ - rsync \ - ca-certificates \ - ssh \ - git \ - moreutils \ - gawk \ - && rm -rf /var/lib/apt/lists/* - -COPY ./ /backup-utils/ -WORKDIR /backup-utils - -RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init - -ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"] -CMD ["ghe-host-check"] diff --git a/Dockerfile.alpine b/Dockerfile.alpine deleted file mode 100644 index 2c21bf2cd..000000000 --- a/Dockerfile.alpine +++ /dev/null @@ -1,20 +0,0 @@ -FROM alpine:latest - -RUN apk --update --no-cache add \ - tar \ - rsync \ - ca-certificates \ - openssh \ - git \ - bash \ - gawk \ - procps \ - coreutils - -COPY ./ /backup-utils/ -WORKDIR /backup-utils - -RUN chmod +x /backup-utils/share/github-backup-utils/ghe-docker-init - -ENTRYPOINT ["/backup-utils/share/github-backup-utils/ghe-docker-init"] -CMD ["ghe-host-check"] diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 4d231b456..000000000 --- a/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2014 GitHub Inc. - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index 7ca5cd24c..000000000 --- a/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -SHELL = /bin/sh - -test: info - @script/cibuild --no-package - -info: - @echo This is github/backup-utils - @echo shell is $(shell ls -l $(SHELL) | sed 's@.*/bin/sh@/bin/sh@') - @rsync --version | head -1 - @echo - -dist: - @script/package-tarball - -deb: - @script/package-deb - -clean: - rm -rf dist - -# List pull requests that need to be merged into stable -# (helpful for the release notes) -pending-prs: - @git log stable...master | grep "Merge pull request" - -.PHONY: test info dist clean pending-prs diff --git a/README.md b/README.md index 4ee951f58..c0ceedb0e 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,84 @@ GitHub Enterprise Server. - **[Backup snapshot file structure](docs/backup-snapshot-file-structure.md)** - **[How does Backup Utilities differ from a High Availability replica?](docs/faq.md)** - **[Docker](docs/docker.md)** +- **[Releases](https://github.com/github/enterprise-releases/blob/master/docs/release-backup-utils.md)** ## Support -If you find a bug or would like to request a feature in Backup Utilities, please -open an issue or pull request on this repository. If you have a question related -to your specific GitHub Enterprise Server setup or would like assistance with -backup site setup or recovery, please contact our [Enterprise support team][3] -instead. +If you have a question related to your specific GitHub Enterprise Server setup, would like assistance with +backup site setup or recovery, or would like to report a bug or a feature request, please contact our [Enterprise support team][3]. + + +## Repository updates - November 2023 + +In October 2023 we announced a number of changes to this repository. +These changes will improve our (GitHub’s) ability to ship enhancements and new features to backup-utils, +as well as simplify how GitHub Enterprise Server customers interact with backup-utils. + +Our process for shipping new versions of backup-utils prior to November 2023 involved a 2-way sync between this repository and an internal repository. +This 2-way sync became significantly more problematic once we started regularly shipping patches in alignment with GitHub Enterprise Server. + +As of 2023-11-30 we have stopped this 2-way sync so that our internal repository becomes the source of truth for the backup-utils source code. +With the the 2-way sync stopped, this public repository will be used to host documentation about backup-utils and to publish new versions of backup-utils. +You will be able to access a specific version of backup-utils (which includes the full source code) from the [release page](https://github.com/github/backup-utils/releases) of this repository. + +This change has not affected the functionality of the backup-utils tool or a customer’s ability to backup or restore their GitHub Enterprise Server instance. + +### Details + +There are three specific areas that have been affected by us stop the 2-way sync between our internal repository and this public repository on 2023-11-30: + +1. **Pull requests**: Customers should no longer open pull requests in this repository. +These pull requests will not be reviewed or merged. +This is necessary because we will no longer be syncing changes between this repository and our internal repository. +2. **Issues**: Customers cannot open issues in this repository. +Instead, customers will need to follow the standard support process and open a support ticket for any questions/concerns/problems with backup-utils. +This will ensure all customer requests are handled consistently. +3. **Installing/upgrading backup-utils**: Customers will not be able to use a clone of the repository to install and upgrade backup-utils. +Customers will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases) +(either as a Debian package or as an archive file - see below for details on how to incorporate this change). + +### Timeline + +Below is the two phase timeline we will follow to roll out the changes described above: + +* **Phase 1 (rolled out on 2023-11-30):** We have closed all open pull requests and issues (after reviewing each one and porting them to our internal repository if merited), +and updated the repository settings so that new issues cannot be opened. Also, we have stopped syncing code from our internal repository to this repository. + * As of 2023-11-30, you can still get a working copy of backup-utils by cloning the repository. + But the code will not be updated in the repository; you can access updated versions of backup-utils via the [release page](https://github.com/github/backup-utils/releases). +* **Phase 2 (rolling out 2024-02-20):** The backup-utils code will be removed and the repository will be used to host documentation for backup-utils. +After this date, you will no longer be able to clone a working copy of backup-utils from the repository. +Instead, you will need to download a specific version of backup-utils from the [release page](https://github.com/github/backup-utils/releases). + +### Updating your backup-utils upgrade process + +#### Clone of repository + +If your current process for upgrading backup-utils involves a clone of the repository, you will need to modify your process to download a new version of backup-utils and set it up. + +For example, you could download the v3.10.0 (github-backup-utils-v3.10.0.tar.gz) artifact from the [releases page](https://github.com/github/backup-utils/releases/tag/v3.10.0) with: + +```shell +\$ wget https://github.com/github/backup-utils/releases/download/v3.10.0/github-backup-utils-v3.10.0.tar.gz +``` +And then extract it: + +```shell +\$ tar xzvf github-backup-utils-v3.10.0.tar.gz +``` + +This will give you a new folder, `github-backup-utils-v3.10.0`, which contains the code for version 3.10.0 of backup-utils. Once you copy over your backup.config file from a previous installation of backup-utils your new version of backup-utils will be ready to use. + +#### Docker + +For customers that currently use Docker to create a backup-utils image, their existing process may need updating as a result of this change. Previously customers could execute this command to build a Docker image of backup-utils: + +```shell +\$ docker build github.com/github/backup-utils +``` + +This will not work after phase 2 roles out. You will need to update your process to first download an archive from the [release page](https://github.com/github/backup-utils/releases), extract it, and then build the Dockerfile inside the extracted directory. + [1]: https://github.com/enterprise [2]: docs/requirements.md#github-enterprise-version-requirements diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md deleted file mode 100644 index 9c1fc1e86..000000000 --- a/STYLEGUIDE.md +++ /dev/null @@ -1,250 +0,0 @@ -## Bash Style Guide - -If you've not done much Bash development before you may find these debugging tips useful: http://wiki.bash-hackers.org/scripting/debuggingtips. - ---- -##### Scripts must start with `#!/usr/bin/env bash` - ---- -##### Use `set -e` - -If the return value of a command can be ignored, suffix it with `|| true`: - -```bash -set -e -command_that_might_fail || true -command_that_should_not_fail -``` - -Note that ignoring an exit status with `|| true` is not a good practice though. Generally speaking, it's better to handle the error. - ---- -##### Avoid manually checking exit status with `$?` - -Rely on `set -e` instead: - -```bash -cmd -if [ $? -eq 0 ]; then - echo worked -fi -``` - -should be written as: - -```bash -set -e -if cmd; then - echo worked -fi -``` - ---- -##### Include a usage, description and optional examples - -Use this format: - -```bash -#!/usr/bin/env bash -#/ Usage: ghe-this-is-my-script [options] -#/ -#/ This is a brief description of the script's purpose. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ -l | --longopt An option. -#/ -c Another option. -#/ -#/ EXAMPLES: (optional section but nice to have when not trivial) -#/ -#/ This will do foo and bar: -#/ $ ghe-this-is-my-script --longopt foobar -c 2 -#/ -set -e -``` - -If there are no options or required arguments, the `OPTIONS` section can be ignored. - ---- -##### Customer-facing scripts must accept both -h and --help arguments - -They should also print the usage information and exit 2. - -For example: - -```bash -#!/usr/bin/env bash -#/ Usage: ghe-this-is-my-script [options] -#/ -#/ This is a brief description of the script's purpose. -set -e - -if [ "$1" = "--help" -o "$1" = "-h" ]; then - grep '^#/' <"$0" | cut -c 4- - exit 2 -fi - -``` - ---- -##### Avoid Bash arrays - -Main issues: - -* Portability -* Important bugs in Bash versions < 4.3 - ---- -##### Use `test` or `[` whenever possible - -```bash -test -f /etc/passwd -test -f /etc/passwd -a -f /etc/group -if [ "string" = "string" ]; then - true -fi -``` - ---- -##### Scripts may use `[[` for advanced bash features - -```bash -if [[ "$(hostname)" = *.iad.github.net ]]; then - true -fi -``` - ---- -##### Scripts may use Bash for loops - -Preferred: - -```bash -for i in $(seq 0 9); do -done -``` - -or: - -```bash -for ((n=0; n<10; n++)); do -done -``` - ---- -##### Use `$[x+y*z]` for mathematical expressions - -```bash -local n=1 -let n++ -n=$[n+1] # preferred -n=$[$n+1] -n=$((n+1)) -n=$(($n+1)) -``` - ---- -##### Use variables sparingly - -Short paths and other constants should be repeated liberally throughout code since they -can be search/replaced easily if they ever change. - -```bash -DATA_DB_PATH=/data/user/db -mkdir -p $DATA_DB_PATH -rsync $DATA_DB_PATH remote:$DATA_DB_PATH -``` - -versus the much more readable: - -```bash -mkdir -p /data/user/db -rsync /data/user/db remote:/data/user/db -``` - ---- -##### Use lowercase and uppercase variable names - -Use lowercase variables for locals and internal variables, and uppercase for variables inherited or exported via the environment - -```bash -#!/usr/bin/env bash -#/ Usage: [DEBUG=0] process_repo -nwo=$1 -[ -n $DEBUG ] && echo "** processing $nwo" >&2 - -export GIT_DIR=/data/repos/$nwo.git -git rev-list -``` - ---- -##### Use `${var}` for interpolation only when required - -```bash -greeting=hello -echo $greeting -echo ${greeting}world -``` - ---- -##### Use functions sparingly, opting for small/simple/sequential scripts instead whenever possible - -When defining functions, use the following style: - -```bash -my_function() { - local arg1=$1 - [ -n $arg1 ] || return - ... -} -``` - ---- -##### Use `< /etc/foo # interpolated before ssh - chmod 0600 /etc/foo -eof -``` - ---- -##### Quote variables that could reasonably have a space now or in the future - -```bash -if [ ! -z "$packages" ]; then - true -fi -``` - ---- -##### Use two space indentation - ---- -##### Scripts should not produce errors or warnings when checked with ShellCheck - -Use inline comments to disable specific tests, and explain why the test has been disabled. - -```bash -hexToAscii() { - # shellcheck disable=SC2059 # $1 needs to be interpreted as a formatted string - printf "\x$1" -} -``` - -### Testing - -See [the style guide](https://github.com/github/backup-utils/blob/master/test/STYLEGUIDE.md) diff --git a/bin/ghe-backup b/bin/ghe-backup deleted file mode 100755 index fe1816030..000000000 --- a/bin/ghe-backup +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup [-hv] [--version] -#/ -#/ Take snapshots of all GitHub Enterprise data, including Git repository data, -#/ the MySQL database, instance settings, GitHub Pages data, etc. -#/ -#/ OPTIONS: -#/ -v | --verbose Enable verbose output. -#/ -h | --help Show this message. -#/ --version Display version information. -#/ - -set -e - -# Parse arguments -while true; do - case "$1" in - -h|--help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -v|--verbose) - export GHE_VERBOSE=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - break - ;; - esac -done - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" - -# Check to make sure moreutils parallel is installed and working properly -ghe_parallel_check - - -# Used to record failed backup steps -failures= -failures_file="$(mktemp -t backup-utils-backup-failures-XXXXXX)" - -# CPU and IO throttling to keep backups from thrashing around. -export GHE_NICE=${GHE_NICE:-"nice -n 19"} -export GHE_IONICE=${GHE_IONICE:-"ionice -c 3"} - -# Create the timestamped snapshot directory where files for this run will live, -# change into it, and mark the snapshot as incomplete by touching the -# 'incomplete' file. If the backup succeeds, this file will be removed -# signifying that the snapshot is complete. -mkdir -p "$GHE_SNAPSHOT_DIR" -cd "$GHE_SNAPSHOT_DIR" -touch "incomplete" - -# Exit early if the snapshot filesystem doesn't support hard links, symlinks and -# if rsync doesn't support hardlinking of dangling symlinks -trap 'rm -rf src dest1 dest2' EXIT -mkdir -p src -touch src/testfile -if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then - echo "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links." 1>&2 - echo "Git repositories contain symbolic links that need to be preserved during a backup." 1>&2 - exit 1 -fi - -if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 dest2 2>&1); then - echo "Error: rsync encountered an error that could indicate a problem with permissions," 1>&2 - echo "hard links, symbolic links, or another issue that may affect backups." 1>&2 - echo "$output" - exit 1 -fi - -if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile | awk '{ print $1 }')" ]; then - echo "Error: the filesystem containing $GHE_DATA_DIR does not support hard links." 1>&2 - echo "Backup Utilities use hard links to store backup data efficiently." 1>&2 - exit 1 -fi -rm -rf src dest1 dest2 - -# To prevent multiple backup runs happening at the same time, we create a -# in-progress file with the timestamp and pid of the backup process, -# giving us a form of locking. -# -# Set up a trap to remove the in-progress file if we exit for any reason but -# verify that we are the same process before doing so. -# -# The cleanup trap also handles disabling maintenance mode on the appliance if -# it was automatically enabled. -cleanup () { - if [ -f ../in-progress ]; then - progress=$(cat ../in-progress) - snapshot=$(echo "$progress" | cut -d ' ' -f 1) - pid=$(echo "$progress" | cut -d ' ' -f 2) - if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = $pid ]; then - unlink ../in-progress - fi - fi - - rm -rf "$failures_file" - rm -f ${GHE_DATA_DIR}/in-progress-backup - - # Cleanup SSH multiplexing - ghe-ssh --clean -} - -# Setup exit traps -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - - -# Check to see if there is a running restore -ghe_restore_check - -# Check to see if there is a running backup -if [ -h ../in-progress ]; then - echo "Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2 - echo "If there is no backup in progress anymore, please remove" 1>&2 - echo "the $GHE_DATA_DIR/in-progress file and try again." 1>&2 - exit 1 -fi - -if [ -f ../in-progress ]; then - progress=$(cat ../in-progress) - snapshot=$(echo "$progress" | cut -d ' ' -f 1) - pid=$(echo "$progress" | cut -d ' ' -f 2) - if ! ps -p "$pid" >/dev/null 2>&1; then - # We can safely remove in-progress, ghe-prune-snapshots - # will clean up the failed backup. - unlink ../in-progress - else - echo "Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo 1>&2 - echo " If PID $pid is not a process related to the backup utilities, please remove" 1>&2 - echo " the $GHE_DATA_DIR/in-progress file and try again." 1>&2 - exit 1 - fi -fi - -echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress -echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ${GHE_DATA_DIR}/in-progress-backup - -START_TIME=$(date +%s) -echo 'Start time:' $START_TIME -echo "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP" - -# Perform a host connection check and establish the remote appliance version. -# The version is available in the GHE_REMOTE_VERSION variable and also written -# to a version file in the snapshot directory itself. -ghe_remote_version_required -echo "$GHE_REMOTE_VERSION" > version - -if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then - echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups." -fi - -# Log backup start message in /var/log/syslog on remote instance -ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..." - -export GHE_BACKUP_STRATEGY=${GHE_BACKUP_STRATEGY:-$(ghe-backup-strategy)} - -# Record the strategy with the snapshot so we will know how to restore. -echo "$GHE_BACKUP_STRATEGY" > strategy - -# Create benchmark file -bm_init > /dev/null - -ghe-backup-store-version || -echo "Warning: storing backup-utils version remotely failed." - -echo "Backing up GitHub settings ..." -ghe-backup-settings || failures="$failures settings" - -echo "Backing up SSH authorized keys ..." -bm_start "ghe-export-authorized-keys" -ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json || -failures="$failures authorized-keys" -bm_end "ghe-export-authorized-keys" - -echo "Backing up SSH host keys ..." -bm_start "ghe-export-ssh-host-keys" -ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar || -failures="$failures ssh-host-keys" -bm_end "ghe-export-ssh-host-keys" - -ghe-backup-mysql || failures="$failures mysql" - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Backing up MSSQL databases ..." - ghe-backup-mssql 1>&3 || failures="$failures mssql" - - echo "Backing up Actions data ..." - ghe-backup-actions 1>&3 || failures="$failures actions" -fi - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - echo "Backing up Minio data ..." - ghe-backup-minio 1>&3 || failures="$failures minio" -fi - -commands=(" -echo \"Backing up Redis database ...\" -ghe-backup-redis > redis.rdb || printf %s \"redis \" >> \"$failures_file\"") - -commands+=(" -echo \"Backing up audit log ...\" -ghe-backup-es-audit-log || printf %s \"audit-log \" >> \"$failures_file\"") - -commands+=(" -echo \"Backing up Git repositories ...\" -ghe-backup-repositories || printf %s \"repositories \" >> \"$failures_file\"") - -commands+=(" -echo \"Backing up GitHub Pages artifacts ...\" -ghe-backup-pages || printf %s \"pages \" >> \"$failures_file\"") - -commands+=(" -echo \"Backing up storage data ...\" -ghe-backup-storage || printf %s \"storage \" >> \"$failures_file\"") - -commands+=(" -echo \"Backing up custom Git hooks ...\" -ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"") - -if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then - commands+=(" - echo \"Backing up Elasticsearch indices ...\" - ghe-backup-es-rsync || printf %s \"elasticsearch \" >> \"$failures_file\"") -fi - -if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then - $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}" -else - for c in "${commands[@]}"; do - eval "$c" - done -fi - -if [ -s "$failures_file" ]; then - failures="$failures $(cat "$failures_file")" -fi - -# git fsck repositories after the backup -if [ "$GHE_BACKUP_FSCK" = "yes" ]; then - ghe-backup-fsck $GHE_SNAPSHOT_DIR || failures="$failures fsck" -fi - -# If everything was successful, mark the snapshot as complete, update the -# current symlink to point to the new snapshot and prune expired and failed -# snapshots. -if [ -z "$failures" ]; then - rm "incomplete" - - rm -f "../current" - ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current" - - ghe-prune-snapshots -fi - -END_TIME=$(date +%s) -echo 'End time:' $END_TIME -echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds' - -echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")" - -# Exit non-zero and list the steps that failed. -if [ -z "$failures" ]; then - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully." -else - steps="$(echo $failures | sed 's/ /, /g')" - ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}." - echo "Error: Snapshot incomplete. Some steps failed: ${steps}. " - exit 1 -fi - -# Detect if the created backup contains any leaked ssh keys -echo "Checking for leaked ssh keys ..." -ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true - -# Make sure we exit zero after the conditional -true - -# Remove in-progress file -ghe_backup_finished diff --git a/bin/ghe-host-check b/bin/ghe-host-check deleted file mode 100755 index ebdf12b96..000000000 --- a/bin/ghe-host-check +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-host-check [-h] [--version] [] -#/ -#/ Verify connectivity with the GitHub Enterprise Server host. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ --version Display version information. -#/ The GitHub Enterprise Server host to check. When no -#/ is provided, the $GHE_HOSTNAME configured in -#/ backup.config is assumed. -#/ - -set -e - -while true; do - case "$1" in - -h | --help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - break - ;; - esac -done - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-backup-config" - -# Use the host provided on the command line if provided, or fallback on the -# $GHE_HOSTNAME configured in backup.config when not present. -host="${1:-$GHE_HOSTNAME}" - -# Options to pass to SSH during connection check -options=" - -o PasswordAuthentication=no - -o ConnectTimeout=5 - -o ConnectionAttempts=1 -" - -# Split host:port into parts -port=$(ssh_port_part "$host") -hostname=$(ssh_host_part "$host") - -set +e -# ghe-negotiate-version verifies if the target is a GitHub Enterprise Server instance -output=$(echo "ghe-negotiate-version backup-utils $BACKUP_UTILS_VERSION" | ghe-ssh -o BatchMode=no $options $host -- /bin/sh 2>&1) -rc=$? -set -e - -if [ $rc -ne 0 ]; then - case $rc in - 255) - if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then - exec "$(basename $0)" "$hostname:122" - fi - - echo "$output" 1>&2 - echo "Error: ssh connection with '$host' failed" 1>&2 - echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2 - echo "* https://docs.github.com/enterprise-server/admin/configuration/configuring-your-enterprise/accessing-the-administrative-shell-ssh" 1>&2 - ;; - 101) - echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2 - ;; - 1) - if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then - exec "$(basename $0)" "$hostname:122" - else - echo "$output" 1>&2 - fi - ;; - - esac - exit $rc -fi - -CLUSTER=false -if ghe-ssh "$host" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then - CLUSTER=true -fi - -# ensure all nodes in the cluster are running the same version -if "$CLUSTER"; then - node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version) - distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l) - if [ "$distinct_versions" -ne 1 ]; then - echo "$node_version_list" 1>&2 - echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2 - exit 1 - fi -fi - -version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}') - -if [ -z "$version" ]; then - echo "Error: failed to parse version on '$host' or this isn't a GitHub appliance." 1>&2 - exit 2 -fi - -# Block restoring snapshots to older releases of GitHub Enterprise Server -if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version) - # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r snapshot_version_major snapshot_version_minor _ <<<$(ghe_parse_version $snapshot_version) - if [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version $snapshot_version_major.$snapshot_version_minor.0)" ]; then - echo "Error: Snapshot can not be restored to an older release of GitHub Enterprise Server." >&2 - exit 1 - fi -fi - -if [ -z "$GHE_ALLOW_REPLICA_BACKUP" ]; then - if [ "$(ghe-ssh $host -- cat $GHE_REMOTE_ROOT_DIR/etc/github/repl-state 2>/dev/null || true)" = "replica" ]; then - echo "Error: high availability replica detected." 1>&2 - echo "Backup Utilities should be used to backup from the primary node in" 1>&2 - echo "high availability environments to ensure consistent and reliable backups." 1>&2 - exit 1 - fi -fi - -# backup-utils 2.13 onwards limits support to the current and previous two releases -# of GitHub Enterprise Server. -supported_minimum_version="3.6.0" - -if [ "$(version $version)" -ge "$(version $supported_minimum_version)" ]; then - supported=1 -fi - -if [ -z "$supported" ]; then - echo "Error: unsupported release of GitHub Enterprise Server detected." 1>&2 - echo "Backup Utilities v$BACKUP_UTILS_VERSION requires GitHub Enterprise Server v$supported_minimum_version or newer." 1>&2 - echo "Please update your GitHub Enterprise Server appliance or use an older version of Backup Utilities." 1>&2 - exit 1 -fi - -echo "Connect $hostname:$port OK (v$version)" diff --git a/bin/ghe-restore b/bin/ghe-restore deleted file mode 100755 index 5c196faea..000000000 --- a/bin/ghe-restore +++ /dev/null @@ -1,597 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-restore [-cfhv] [--version] [--skip-mysql] [-s ] [] -#/ -#/ Restores a GitHub instance from local backup snapshots. -#/ -#/ Note that the GitHub Enterprise host must be reachable and your SSH key must -#/ be setup as described in the following help article: -#/ -#/ -#/ -#/ OPTIONS: -#/ -c | --config Restore appliance settings and license in addition to -#/ datastores. Settings are not restored by default to -#/ prevent overwriting different configuration on the -#/ restore host. -#/ -f | --force Don't prompt for confirmation before restoring. -#/ -h | --help Show this message. -#/ -v | --verbose Enable verbose output. -#/ --skip-mysql Skip MySQL restore steps. Only applicable to external databases. -#/ --version Display version information and exit. -#/ -#/ -s Restore from the snapshot with the given id. Available -#/ snapshots may be listed under the data directory. -#/ -#/ The is the hostname or IP of the GitHub Enterprise -#/ instance. The may be omitted when the -#/ GHE_RESTORE_HOST config variable is set in backup.config. -#/ When a argument is provided, it always overrides -#/ the configured restore host. -#/ - -set -e - -# Parse arguments -: ${RESTORE_SETTINGS:=false} -export RESTORE_SETTINGS - -: ${FORCE:=false} -export FORCE - -: ${SKIP_MYSQL:=false} -export SKIP_MYSQL - -while true; do - case "$1" in - --skip-mysql) - SKIP_MYSQL=true - shift - ;; - -f|--force) - FORCE=true - shift - ;; - -s) - snapshot_id="$(basename "$2")" - shift 2 - ;; - -c|--config) - RESTORE_SETTINGS=true - shift - ;; - -h|--help) - export GHE_SHOW_HELP=true - shift - ;; - --version) - export GHE_SHOW_VERSION=true - shift - ;; - -v|--verbose) - export GHE_VERBOSE=true - shift - ;; - -*) - echo "Error: invalid argument: '$1'" 1>&2 - exit 1 - ;; - *) - if [ -n "$1" ]; then - GHE_RESTORE_HOST_OPT="$1" - shift - else - break - fi - ;; - esac -done - -start_cron () { - echo "Starting cron ..." - if $CLUSTER; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo timeout 120s service cron start"; then - echo "* Warning: Failed to start cron on one or more nodes" - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo timeout 120s service cron start"; then - echo "* Warning: Failed to start cron" - fi - fi -} - -cleanup () { - if [ -n "$1" ]; then - update_restore_status "$1" - fi - - if $ACTIONS_STOPPED && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Restarting Actions after restore ..." - # In GHES 3.3+, ghe-actions-start no longer has a -f (force) flag. In GHES 3.2 and below, we must provide the - # force flag to make sure it can start in maintenance mode. Use it conditionally based on whether it exists - # in the --help output - if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start --help' | grep -q force ; then - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start -f' 1>&3 - else - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-start' 1>&3 - fi - fi - - if ! $CRON_RUNNING; then - start_cron - fi - - # Cleanup SSH multiplexing - ghe-ssh --clean - # Remove in-progress file - rm -f ${GHE_DATA_DIR}/in-progress-restore -} - -# This function's type definition is being passed to a remote host via `ghe-ssh` but is not used locally. -# shellcheck disable=SC2034 -cleanup_cluster_nodes() { - uuid="$1" - if [ -z "$uuid" ]; then - echo "Node UUID required." - exit 2 - fi - - ghe-spokes server evacuate git-server-$uuid 'Removing replica' - ghe-spokes server destroy git-server-$uuid - - ghe-storage destroy-host storage-server-$uuid --force - - ghe-dpages offline pages-server-$uuid - ghe-dpages remove pages-server-$uuid - - ghe-redis-cli del resque:queue:maint_git-server-$uuid - ghe-redis-cli srem resque:queues maint_git-server-$uuid -} - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config" - -# Check to make sure moreutils parallel is installed and working properly -ghe_parallel_check - -# Check to make sure another restore process is not running -ghe_restore_check - -# Grab the host arg -GHE_HOSTNAME="${GHE_RESTORE_HOST_OPT:-$GHE_RESTORE_HOST}" - -# Hostname without any port suffix -hostname=$(echo "$GHE_HOSTNAME" | cut -f 1 -d :) - -# Show usage with no -[ -z "$GHE_HOSTNAME" ] && print_usage - -# Flag to indicate if this script has stopped Actions. -ACTIONS_STOPPED=false - -# ghe-restore-snapshot-path validates it exists, determines what current is, -# and if there's any problem, exit for us -GHE_RESTORE_SNAPSHOT_PATH="$(ghe-restore-snapshot-path "$snapshot_id")" -GHE_RESTORE_SNAPSHOT=$(basename "$GHE_RESTORE_SNAPSHOT_PATH") -export GHE_RESTORE_SNAPSHOT - -# Check to make sure backup is not running -ghe_backup_check - -# Detect if the backup we are restoring has a leaked ssh key -echo "Checking for leaked keys in the backup snapshot that is being restored ..." -ghe-detect-leaked-ssh-keys -s "$GHE_RESTORE_SNAPSHOT_PATH" || true - -# Figure out whether to use the tarball or rsync restore strategy based on the -# strategy file written in the snapshot directory. -GHE_BACKUP_STRATEGY=$(cat "$GHE_RESTORE_SNAPSHOT_PATH/strategy") - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -# Figure out if this instance has been configured or is entirely new. -instance_configured=false -if is_instance_configured; then - instance_configured=true -else - RESTORE_SETTINGS=true -fi - -# Figure out if we're restoring into cluster -CLUSTER=false -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then - CLUSTER=true -fi -export CLUSTER - -# Restoring a cluster backup to a standalone appliance is not supported -if ! $CLUSTER && [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - echo "Error: Snapshot from a GitHub Enterprise cluster cannot be restored" \ - "to a standalone appliance. Aborting." >&2 - exit 1 -fi - -# Ensure target appliance and restore snapshot are a compatible combination with respect to BYODB -if ! ghe-restore-external-database-compatibility-check; then - exit 1 -fi - -# Figure out if this appliance is in a replication pair -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - echo "Error: Restoring to an appliance with replication enabled is not supported." >&2 - echo " Please teardown replication before restoring." >&2 - exit 1 -fi - -# Prompt to verify the restore host given is correct. Restoring overwrites -# important data on the destination appliance that cannot be recovered. This is -# mostly to prevent accidents where the backup host is given to restore instead -# of a separate restore host since they're used in such close proximity. -if $instance_configured && ! $FORCE; then - echo - echo "WARNING: All data on GitHub Enterprise appliance $hostname ($GHE_REMOTE_VERSION)" - echo " will be overwritten with data from snapshot ${GHE_RESTORE_SNAPSHOT}." - echo - - if is_external_database_snapshot && $RESTORE_SETTINGS; then - echo "WARNING: This operation will also restore the external MySQL connection configuration," - echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online." - echo - fi - - prompt_for_confirmation "Please verify that this is the correct restore host before continuing." -fi - -# Prompt to verify that restoring BYODB snapshot to unconfigured instance -# will result in BYODB connection information being restored as well. -if is_external_database_snapshot && ! $instance_configured && ! $FORCE; then - echo - echo "WARNING: This operation will also restore the external MySQL connection configuration," - echo " which may be dangerous if the GHES appliance the snapshot was taken from is still online." - echo - - prompt_for_confirmation "Please confirm this before continuing." -fi - -# Log restore start message locally and in /var/log/syslog on remote instance -START_TIME=$(date +%s) -echo 'Start time:' $START_TIME -echo "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT" -ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..." -# Create an in-progress-restore file to prevent simultaneous backup or restore runs -echo "${START_TIME} $$" > ${GHE_DATA_DIR}/in-progress-restore - -# Keep other processes on the VM or cluster in the loop about the restore status. -# -# Other processes will look for these states: -# "restoring" - restore is currently in progress -# "failed" - restore has failed -# "complete" - restore has completed successfully -update_restore_status () { - if $CLUSTER; then - echo "ghe-cluster-each -- \"echo '$1' | sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null\"" | - ghe-ssh "$GHE_HOSTNAME" /bin/bash - else - echo "$1" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status' >/dev/null" - fi -} - -CRON_RUNNING=true -# Update remote restore state file and setup failure trap -trap "cleanup failed" EXIT -update_restore_status "restoring" - -# Make sure the GitHub appliance is in maintenance mode. -if $instance_configured; then - if ! ghe-maintenance-mode-status "$GHE_HOSTNAME"; then - echo "Error: $GHE_HOSTNAME must be put in maintenance mode before restoring. Aborting." 1>&2 - exit 1 - fi -fi - -# Get GHES release version in major.minor format -RELEASE_VERSION=$(ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --get core.package-version' | cut -d '.' -f 1,2) - -# If the backup being restored is from an appliance with Actions disabled, restoring it onto an appliance with Actions enabled will cause -# mismatches in the secrets needed for Actions which ultimately results in Actions not working properly. Note: xargs is to remove whitespace -ACTIONS_ENABLED_IN_BACKUP=$(git config -f $GHE_RESTORE_SNAPSHOT_PATH/settings.json --bool app.actions.enabled | xargs) -if [[ $ACTIONS_ENABLED_IN_BACKUP != true ]] && ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Error: Restoring a backup with Actions disabled onto an appliance with Actions enabled is not supported." >&2 - exit 1 -fi - -# Make sure the GitHub appliance has Actions enabled if the snapshot contains Actions data. -# If above is true, also check if ac is present in appliance then snapshot should also contains ac databases -if [ -d "$GHE_RESTORE_SNAPSHOT_PATH/mssql" ] || [ -d "$GHE_RESTORE_SNAPSHOT_PATH/actions" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - ac_db_ghe=$(echo 'ghe-mssql-console -y -n -q "SELECT name FROM sys.databases" | grep -i "ArtifactCache" | wc -l | tr -d " "' | ghe-ssh "$GHE_HOSTNAME" /bin/bash) - ac_db_snapshot=$(find "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/mssql/" -maxdepth 1 -name 'ArtifactCache*.bak' | wc -l | tr -d " ") - if [[ $ac_db_ghe -gt 0 && $ac_db_snapshot -eq 0 ]]; then - echo "Error: $GHE_HOSTNAME has Actions Cache service enabled but no Actions Cache data is present in snapshot to restore. Aborting" 1>&2 - echo "Please disable Actions cache service in $GHE_HOSTNAME and retry" 1>&2 - echo "To disable Actions Cache service run as admin: ghe-actions-cache-disable" 1>&2 - exit 1 - fi - if [[ $ac_db_ghe -eq 0 && $ac_db_snapshot -gt 0 && ! $RESTORE_SETTINGS ]]; then - echo "Error: $GHE_HOSTNAME has Actions Cache service disabled but the snapshot is attempting to restore data for the service. Aborting" 1>&2 - echo "Please enable Actions cache service in $GHE_HOSTNAME and retry" 1>&2 - echo "To enable Actions Cache service run as admin: ghe-actions-cache-enable" 1>&2 - exit 1 - fi - else - echo "Error: $GHE_HOSTNAME must have GitHub Actions enabled before restoring since the snapshot contains Actions data. Aborting." 1>&2 - echo "Setup details for enabling Actions can be found here: https://docs.github.com/en/enterprise-server@$RELEASE_VERSION/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled" 1>&2 - exit 1 - fi -fi - -# Create benchmark file -bm_init > /dev/null - -ghe-backup-store-version || -echo "Warning: storing backup-utils version remotely failed." - -# Stop cron and timerd, as scheduled jobs may disrupt the restore process. -echo "Stopping cron and github-timerd ..." -if $CLUSTER; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron stop"; then - ghe_verbose "* Warning: Failed to stop cron on one or more nodes" - fi - - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes" - fi - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service github-timerd stop"; then - ghe_verbose "* Warning: Failed to stop github-timerd on one or more nodes" - fi - fi -else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron stop"; then - ghe_verbose "* Warning: Failed to stop cron" - fi - - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - if ghe-ssh "$GHE_HOSTNAME" -- "systemctl -q is-active nomad && nomad job status --short github-timerd &>/dev/null"; then - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo nomad stop github-timerd 1>/dev/null"; then - ghe_verbose "* Warning: Failed to stop github-timerd" - fi - fi - else - if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service github-timerd stop"; then - ghe_verbose "* Warning: Failed to stop github-timerd" - fi - fi -fi -CRON_RUNNING=false - -# Restore settings and license if restoring to an unconfigured appliance or when -# specified manually. -if $RESTORE_SETTINGS; then - ghe-restore-settings "$GHE_HOSTNAME" -fi - -# Make sure mysql and elasticsearch are prep'd and running before restoring. -# These services will not have been started on appliances that have not been -# configured yet. -if ! $CLUSTER; then - echo "sudo ghe-service-ensure-mysql && sudo ghe-service-ensure-elasticsearch" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# Restore UUID if present and not restoring to cluster. -if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then - echo "Restoring UUID ..." - cat "$GHE_RESTORE_SNAPSHOT_PATH/uuid" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo sponge '$GHE_REMOTE_DATA_USER_DIR/common/uuid' 2>/dev/null" - ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl stop consul" || true - ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft" - fi - -if is_external_database_snapshot; then - appliance_strategy="external" - backup_snapshot_strategy="external" -else - if is_binary_backup_feature_on; then - appliance_strategy="binary" - else - appliance_strategy="logical" - fi - - if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then - backup_snapshot_strategy="binary" - else - backup_snapshot_strategy="logical" - fi -fi - -if is_external_database_target_or_snapshot && $SKIP_MYSQL; then - echo "Skipping MySQL restore." -else - echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..." - ghe-restore-mysql "$GHE_HOSTNAME" 1>&3 -fi - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then - echo "Stopping Actions before restoring databases ..." - # We mark Actions as stopped even if the `ghe-actions-stop` - # fails to ensure that we cleanly start actions when performing cleanup. - ACTIONS_STOPPED=true - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-actions-stop' 1>&3 - - echo "Restoring MSSQL databases ..." - ghe-restore-mssql "$GHE_HOSTNAME" 1>&3 - - echo "Restoring Actions data ..." - ghe-restore-actions "$GHE_HOSTNAME" 1>&3 - echo "* WARNING: Every self-hosted Actions runner that communicates with the restored GHES server must be restarted or reconfigured in order to continue functioning." - echo " See https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners for more details on how to reconfigure self-hosted Actions runners." -fi - -if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then - echo "Restoring MinIO data ..." - ghe-restore-minio "$GHE_HOSTNAME" 1>&3 -fi - -commands=(" -echo \"Restoring Redis database ...\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-redis' < \"$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb\" 1>&3") - -commands+=(" -echo \"Restoring Git repositories ...\" -ghe-restore-repositories \"$GHE_HOSTNAME\"") - -commands+=(" -echo \"Restoring Gists ...\" -ghe-restore-repositories-gist \"$GHE_HOSTNAME\"") - -commands+=(" -echo \"Restoring GitHub Pages artifacts ...\" -ghe-restore-pages \"$GHE_HOSTNAME\" 1>&3") - -commands+=(" -echo \"Restoring SSH authorized keys ...\" -ghe-ssh \"$GHE_HOSTNAME\" -- 'ghe-import-authorized-keys' < \"$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json\" 1>&3") - -commands+=(" -echo \"Restoring storage data ...\" -ghe-restore-storage \"$GHE_HOSTNAME\" 1>&3") - -commands+=(" -echo \"Restoring custom Git hooks ...\" -ghe-restore-git-hooks \"$GHE_HOSTNAME\" 1>&3") - -if ! $CLUSTER && [ -d "$GHE_RESTORE_SNAPSHOT_PATH/elasticsearch" ]; then - commands+=(" - echo \"Restoring Elasticsearch indices ...\" - ghe-restore-es-rsync \"$GHE_HOSTNAME\" 1>&3") -fi - -# Restore the audit log migration sentinel file, if it exists in the snapshot -if test -f $GHE_RESTORE_SNAPSHOT_PATH/es-scan-complete; then - ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete" -fi - -# Restore exported audit logs to 2.12.9 and newer single nodes and -# all releases of cluster -if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; then - if [[ "$GHE_RESTORE_SKIP_AUDIT_LOGS" = "yes" ]]; then - echo "Skipping restore of audit logs." - else - commands+=(" - echo \"Restoring Audit logs ...\" - ghe-restore-es-audit-log \"$GHE_HOSTNAME\" 1>&3") - fi - -fi - -if [ "$GHE_PARALLEL_ENABLED" = "yes" ]; then - $GHE_PARALLEL_COMMAND $GHE_PARALLEL_COMMAND_OPTIONS -- "${commands[@]}" -else - for c in "${commands[@]}"; do - eval "$c" - done -fi - -# Restart an already running memcached to reset the cache after restore -echo "Restarting memcached ..." 1>&3 -echo "sudo restart -q memcached 2>/dev/null || true" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh - -# Prevent GitHub Connect jobs running before we've had a chance to reset -# the configuration by setting the last run date to now. -if ! $RESTORE_SETTINGS; then - echo "Setting last run date for GitHub Connect jobs ..." 1>&3 - echo "now=$(date +%s.0000000); ghe-redis-cli mset timer:UpdateConnectInstallationInfo \$now timer:UploadEnterpriseServerUserAccountsJob \$now timer:UploadConnectMetricsJob \$now timer:GitHubConnectPushNewContributionsJob \$now" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# When restoring to a host that has already been configured, kick off a -# config run to perform data migrations. -if $CLUSTER; then - echo "Configuring cluster ..." - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3 - elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 - fi - ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3 -elif $instance_configured; then - echo "Configuring appliance ..." - if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3 - elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then - ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3 - fi - ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3 -fi - -# Clear GitHub Connect settings stored in the restored database. -# This needs to happen after `ghe-config-apply` to ensure all migrations have run. -if ! $RESTORE_SETTINGS; then - echo "if [ -f /usr/local/share/enterprise/ghe-reset-gh-connect ]; then /usr/local/share/enterprise/ghe-reset-gh-connect -y; fi" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -# Start cron. Timerd will start automatically as part of the config run. -start_cron -CRON_RUNNING=true - -# Clean up all stale replicas on configured instances. -if ! $CLUSTER && $instance_configured; then - restored_uuid=$(cat $GHE_RESTORE_SNAPSHOT_PATH/uuid) - other_nodes=$(echo " - set -o pipefail; \ - ghe-spokes server show --json \ - | jq -r '.[] | select(.host | contains(\"git-server\")).host' \ - | sed 's/^git-server-//g' \ - | ( grep -F -x -v \"$restored_uuid\" || true )" \ - | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash) - if [ -n "$other_nodes" ]; then - echo "Cleaning up stale nodes ..." - for uuid in $other_nodes; do - # shellcheck disable=SC2034 - echo "set -o pipefail; $(typeset -f cleanup_cluster_nodes); cleanup_cluster_nodes $uuid" | ghe-ssh "$GHE_HOSTNAME" 1>&3 - done - fi -fi - -# Update the remote status to "complete". This has to happen before importing -# ssh host keys because subsequent commands will fail due to the host key -# changing otherwise. -trap "cleanup" EXIT -update_restore_status "complete" - -# Log restore complete message in /var/log/syslog on remote instance -ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_RESTORE_SNAPSHOT}." - -if ! $CLUSTER; then - echo "Restoring SSH host keys ..." - ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 -else - # This will make sure that Git over SSH host keys (babeld) are - # copied to all the cluster nodes so babeld uses the same keys. - echo "Restoring Git over SSH host keys ..." - ghe-ssh "$GHE_HOSTNAME" -- "sudo tar -xpf - -C $GHE_REMOTE_DATA_USER_DIR/common" < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3 - ghe-ssh "$GHE_HOSTNAME" -- "sudo chown babeld:babeld $GHE_REMOTE_DATA_USER_DIR/common/ssh_host_*" 1>&3 - echo "if [ -f /usr/local/share/enterprise/ghe-cluster-config-update ]; then /usr/local/share/enterprise/ghe-cluster-config-update -s; else ghe-cluster-config-update -s; fi" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3 -fi - -END_TIME=$(date +%s) -echo 'End time:' $END_TIME -echo 'Runtime:' $(($END_TIME - $START_TIME)) 'seconds' - -echo "Restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT finished." -ghe_restore_finished - -if ! $instance_configured; then - echo "To complete the restore process, please visit https://$hostname/setup/settings to review and save the appliance configuration." -fi - diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index fb6b9940b..000000000 --- a/debian/changelog +++ /dev/null @@ -1,624 +0,0 @@ -github-backup-utils (3.8.0) focal; urgency=medium - - -- Daniel Johnson Tue, 07 Feb 2023 21:43:26 +0000 - -github-backup-utils (3.7.0) UNRELEASED; urgency=medium - - - -- Devin Dooley Tue, 25 Oct 2022 00:35:38 +0000 - -github-backup-utils (3.6.0) UNRELEASED; urgency=medium - - - -- Joe Franks Wed, 17 Aug 2022 19:20:54 +0000 - -github-backup-utils (3.5.0) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Bon Sohi Thu, 12 May 2022 21:06:56 +0000 - -github-backup-utils (3.4.1) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Donal Ellis Fri, 22 Apr 2022 04:00:00 +0000 - -github-backup-utils (3.3.2) UNRELEASED; urgency=medium - - * Simplify complex redirects for ghe-rsync #881 - * On restore failure, restart cron on target host #883 - - -- Donal Ellis Fri, 22 Apr 2022 00:53:45 +0000 - -github-backup-utils (3.4.0) UNRELEASED; urgency=medium - - * Add anchor to usage doc for settings restore #865 - - -- Steve Culver Tue, 15 Feb 2022 19:25:09 +0000 - -github-backup-utils (3.3.1) UNRELEASED; urgency=medium - - * Fix compat issue with ghe-actions-start during maintenance mode #836 - - -- Balwinder Sohi Tue, 21 Dec 2021 23:38:01 +0000 - -github-backup-utils (3.3.0) UNRELEASED; urgency=medium - - -- Balwinder Sohi Wed, 08 Dec 2021 03:12:53 +0000 - -github-backup-utils (3.3.0.rc1) UNRELEASED; urgency=medium - - -- Nick Iodice Tue, 09 Nov 2021 19:56:08 +0000 - -github-backup-utils (3.2.0) UNRELEASED; urgency=medium - - -- Brett Westover Tue, 28 Sep 2021 16:50:00 +0000 - -github-backup-utils (3.2.0.rc3) UNRELEASED; urgency=medium - - * Move GitHub Connect reset to after ghe-config-apply #783 - - -- Brett Westover Fri, 17 Sep 2021 00:44:59 +0000 - -github-backup-utils (3.2.0.rc1) UNRELEASED; urgency=medium - - * Leaked host key check: Avoid false positives from FIPS mode #748 - * Always restore user-password-secrets #762 - * Make some of the actions setting best effort #767 - * Remove reference to `ghe-cluster-cleanup-nodes` #768 - * Stop/Restart Actions in ghe-restore #769 - * Clear GitHub Connect settings when not restoring settings #770 - * GitHub Connect Reset Issue #776 - - -- Brett Westover Fri, 10 Sep 2021 20:10:07 +0000 - -github-backup-utils (3.2.0) UNRELEASED; urgency=medium - - * Leaked host key check: Avoid false positives from FIPS mode #748 - * Always restore user-password-secrets #762 - * Make some of the actions setting best effort #767 - * Remove reference to `ghe-cluster-cleanup-nodes` #768 - * Stop/Restart Actions in ghe-restore #769 - * Clear GitHub Connect settings when not restoring settings #770 - - -- Brett Westover Thu, 09 Sep 2021 16:42:24 +0000 - -github-backup-utils (3.1.0) UNRELEASED; urgency=medium - - -- Zachary Mark Thu, 03 Jun 2021 16:55:16 +0000 - -github-backup-utils (3.1.0~rc1) UNRELEASED; urgency=medium - - [ Zachary Mark ] - * Update repository backups to use ghe-gc-* for lock file management #188 - * A faster way to restore storage blobs (clusters) #212 - * Bug fix: Be more specific in restore routes globbing #715 - * fix(docker): add coreutils to get flags for date #717 - * Add backup cadence variable to the appliance #719 - * Fix is_default_external_database_snapshot function #720 - - [ Hideki Yamane ] - * debian/rules - - Drop unnecessary build-indep: - - Add manpages generation. - * debian/changelog - - Fix some lintian warnings for old entries. - * debian/source/format - - Upgrade to newer source format 3.0 (native). - * debian/copyright - - Format it as Machine-readable debian/copyright format 1.0 - - Add entries Twan Wolthof and me for debian/* since - enough code was written by non-github.com employees, at least. - * debian/control - - Update descriptions in debian/control - - Fix "Architecture: all", instead of "any". - - Declare its Homepage as https://github.com/github/backup-utils - - Add Vcs-* metadata field. - - Specify "Rules-Requires-Root: no" - - Drop unnecessary "Build-Depends: devscripts" - - Set Standards-Version: 4.5.1 - - Update Maintainer as Zachary Mark - * debian/{clean,manpages} - - Add files to support manpages handles. - - -- Zachary Mark Thu, 06 May 2021 17:11:18 +0000 - -github-backup-utils (3.0.0) UNRELEASED; urgency=medium - - * Fix restoring the password pepper for already configured instances #683 - - -- Michael Dang Tue, 16 Feb 2021 22:32:25 +0000 - -github-backup-utils (3.0.0.rc1) UNRELEASED; urgency=medium - - * Cleanup nomad container when restore for 2.22 #670 - * Use ghe-cluster-nomad-cleanup for cluster mode #663 - * Only run ghe-nomad-cleanup in 3.0 #662 - * Revert backup-utils gitHub env and a few more fixes #661 - * Note how to test filesystem symlink / hardlink support #660 - * stop github-timerd based on its running environment #659 - * Backup and restore password pepper #656 - * github-env -> github-env-dispatch #654 - * Rename redis-cli to ghe-redis-cli #639 - - -- Michael Dang Thu, 14 Jan 2021 21:17:53 +0000 - -github-backup-utils (2.22.0) UNRELEASED; urgency=medium - - * Added basic timing around the ghe-restore process #625 - * Improve python3 & finding moreutils parallel #627 - * Turn off POSIX for ghe-backup-config #632 - * Add parallelized restore capability to ghe-restore-storage #635 - * Update backup-utils for new features in 2.22 #641 - - -- Hao Jiang Wed, 23 Sep 2020 15:48:54 +0000 - -github-backup-utils (2.21.0) UNRELEASED; urgency=medium - - * Introduce option to skip restoring of audit logs #596 - * Beta: Execute ghe-backup tasks in parallel #597 - * Beta: Execute ghe-restore tasks in parallel #601 - * Run repositories restore in parallel #603 - * Fix mismatched `bm_start` and `bm_end` commands #607 - * remove rsync restore method used by GHES versions prior to 2.13 #608 - * Output MySQL backup strategy for clarity during backup and restore #610 - - -- Hao Jiang Tue, 09 Jun 2020 17:59:06 +0000 - -github-backup-utils (2.19.5) UNRELEASED; urgency=medium - - * In legacy mode we should use ghe-import-mysql #581 - - -- Caine Jette Fri, 21 Feb 2020 19:13:17 +0000 - -github-backup-utils (2.20.2) UNRELEASED; urgency=medium - - * In legacy mode we should use ghe-import-mysql #581 - - -- Hao Jiang Thu, 20 Feb 2020 22:47:09 +0000 - -github-backup-utils (2.20.1) UNRELEASED; urgency=medium - - * Fixes gist route calculation for GHES version 2.20 - - -- Caine Jette Wed, 19 Feb 2020 21:47:18 +0000 - -github-backup-utils (2.19.4) UNRELEASED; urgency=medium - - * Fix the way we connect to mysql master using ssh forwarding for binary backups #567 - - -- Hao Jiang Tue, 18 Feb 2020 17:54:31 +0000 - -github-backup-utils (2.20.0) UNRELEASED; urgency=medium - - * Fix `ghe-backup-repositories` performance for large instances #541 - * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551 - - -- Alejandro Figueroa Tue, 11 Feb 2020 20:47:17 +0000 - -github-backup-utils (2.19.3) UNRELEASED; urgency=medium - - * Fix two issues with binary backup (slow gzip compression and support for restore from instances other than SQL master) #551 - - -- Hao Jiang Tue, 11 Feb 2020 19:31:55 +0000 - -github-backup-utils (2.19.2) UNRELEASED; urgency=medium - - * Fix `ghe-backup-repositories` performance for large instances #541 - - -- Evgenii Khramkov Fri, 31 Jan 2020 19:13:46 +0000 - -github-backup-utils (2.19.1) UNRELEASED; urgency=medium - - * Cater for more explicit gist paths used in routes file #524 - * Suppress "*.rsync': No such file or directory" errors when no data to backup or restore #525 - - -- Colin Seymour Wed, 11 Dec 2019 09:33:01 +0000 - -github-backup-utils (2.19.0) UNRELEASED; urgency=medium - - * Remove temporary exclude file from the backup host not the target GHES appliance #516 - * Update Debian package depends to include git #520 - - -- Colin Seymour Tue, 12 Nov 2019 18:57:12 +0000 - -github-backup-utils (2.18.0) UNRELEASED; urgency=medium - - * Replace "sed -E" in ghe-host-check with a more portable solution #509 - - -- Colin Seymour Tue, 20 Aug 2019 18:49:44 +0000 - -github-backup-utils (2.17.1) UNRELEASED; urgency=medium - - * Redirect ghe-export-audit-logs stderr output unless using verbose output #497 - - -- Colin Seymour Wed, 05 Jun 2019 08:43:25 +0000 - -github-backup-utils (2.17.0) UNRELEASED; urgency=medium - - * Restore target is ignored when specified on the command line #476 - * Support incremental backups for MySQL-stored audit logs #485 - - -- Colin Seymour Thu, 23 May 2019 08:20:15 +0000 - -github-backup-utils (2.16.1) UNRELEASED; urgency=medium - - * Detect storage user on each cluster host being backed up or restored #472 - - -- Colin Seymour Tue, 26 Feb 2019 16:37:04 +0000 - -github-backup-utils (2.16.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 22 Jan 2019 20:25:34 +0000 - -github-backup-utils (2.15.1) UNRELEASED; urgency=medium - - * Restoring to an un-configured appliance fails due to a missing license file #449 - - -- Colin Seymour Tue, 13 Nov 2018 17:34:21 +0000 - -github-backup-utils (2.15.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 16 Oct 2018 16:40:03 +0000 - -github-backup-utils (2.15.0) UNRELEASED; urgency=medium - - * (There was no descriptions) - - -- Colin Seymour Tue, 16 Oct 2018 16:07:36 +0000 - -github-backup-utils (2.14.3) UNRELEASED; urgency=medium - - * Improve multi-platform detection of simultaneous ghe-backup runs #435 - - -- Colin Seymour Tue, 11 Sep 2018 17:03:42 +0000 - -github-backup-utils (2.14.2) UNRELEASED; urgency=medium - - * Capture and display repository/gist warnings during cluster restore #423 - * Use remote tempdir when finalizing Pages routes #424 - * Use old rsync restore method for pages prior to 2.13 #426 - - -- Colin Seymour Tue, 21 Aug 2018 13:57:20 +0000 - -github-backup-utils (2.14.1) UNRELEASED; urgency=medium - - * Don't fail a backup if the Management Console password isn't set #416 - * Fix permissions issues when repeat restoring to configured cluster instance #417 - * Add missing dependencies to debian packaging #418 - * Prevent restoring snapshots to older releases #420 - - -- Colin Seymour Tue, 07 Aug 2018 16:00:36 +0000 - -github-backup-utils (2.14.0) UNRELEASED; urgency=medium - - * Disable pager and context when running git-diff #411 - * Optimise hookshot and audit log backups and restores and MySQL restores #413 - - -- Colin Seymour Thu, 12 Jul 2018 15:11:11 +0000 - -github-backup-utils (2.13.2) UNRELEASED; urgency=medium - - * Treat missing repository networks, gists, and storage objects as a non-critical error #386 - * Clean up stale HA nodes on restore #396 - * Cleanup all SSH muxes in a non blocking way #402 - * Raise an error if the current symlink doesn't exist when attempting to restore it #404 - - -- Colin Seymour Fri, 22 Jun 2018 10:08:22 +0000 - -github-backup-utils (2.13.1) UNRELEASED; urgency=medium - - * Retry with the admin ssh port on network unreachable too. #377 - * Output backup utils version on backup and restore #384 - * Check filesystem supports hardlinks #388 - * Switch back to optimised restore route calculation #389 - * Optionally log verbose output to a file instead of stdout #391 - * Switch back to optimised backup route calculation #392 - * Remove check for git from ghe-ssh #393 - * Use remote's mktemp to create temp dir on remote host #395 - - -- Colin Seymour Wed, 09 May 2018 10:16:18 +0000 - -github-backup-utils (2.13.0) UNRELEASED; urgency=medium - - * Unify the backup & restore process #375 - - -- Colin Seymour Tue, 27 Mar 2018 16:01:43 +0000 - -github-backup-utils (2.11.4) UNRELEASED; urgency=medium - - * Move check for git for ssh muxing into ghe-ssh #378 - * Make it clear the settings need to be applied after restoring to an unconfigured instance #381 - - -- Colin Seymour Tue, 27 Mar 2018 13:20:18 +0000 - -github-backup-utils (2.11.3) UNRELEASED; urgency=medium - - * Update argument parsing and help/usage consistency #320 - * Fix failures variable #353 - * Remove other snapshot contents before removing the "incomplete" file #358 - * Backup and restore the management console password #361 - * Check for git before allowing SSH multiplex #362 - * Cleanup SSH multiplexing on exit #363 - * Filter cluster nodes by role during backup and restore #367 - * Optimise route generation and finalisation during cluster restores of pages #369 - * Allow extra rsync options to override default options #370 - - -- Colin Seymour Wed, 28 Feb 2018 16:32:07 +0000 - -github-backup-utils (2.11.2) UNRELEASED; urgency=medium - - * Allow the restoration of configuration to Cluster #347 - * Switch to TMPDIR before initiating SSH multiplexing workaround to prevent locking the destination filesystem #348 - - -- Colin Seymour Thu, 09 Nov 2017 12:16:23 +0000 - -github-backup-utils (2.11.1) UNRELEASED; urgency=medium - - * Refresh the existing indices when restoring Elasticsearch indices to cluster #328 - * Fix failure to restore 2.9/2.10 backups to 2.11 prevented by incorrect detection of the audit log migration #333 - * Use git to generate short name for SSH multiplex control path #335 - * Remove use of --literally when computing arbitrary shasum #338 - * Remove -o option from ps use #341 - - -- Colin Seymour Thu, 05 Oct 2017 14:47:56 +0000 - -github-backup-utils (2.11.0) UNRELEASED; urgency=medium - - * Use calculated routes when backing up storage data from a cluster #318 - * Add SSH multiplexing support #321 - * Optimise route generation and finalisation during cluster restores #322 - * Prefer the SSH port specified on the command line #324 - - -- Colin Seymour Wed, 13 Sep 2017 16:31:20 +0000 - -github-backup-utils (2.10.0) UNRELEASED; urgency=medium - - * Include the user data directory in the benchmark name #311 - * Use existing Elasticsearch indices to speed up transfer during a restore #310 - * Improve detection of failures in cluster backup rsync threads #301 - * Improve redis backup robustness #306 - * Use default niceness for restores #308 - * Add additional case to SSH port detection logic #304 - * Suppress additional dd output noise #289 - * Track completeness of Elasticsearch JSON dumps #298 - - -- Steven Honson Thu, 08 Jun 2017 09:06:16 +1000 - -github-backup-utils (2.9.0) UNRELEASED; urgency=medium - - * Block restores to appliances with HA configured #291 - * Add a `--version` flag #284 - * Check backup-utils are not being run on GitHub Enterprise host #286 - * Backup and restore custom CA certificates #281 - * Hookshot backup/restores optimisations #276 - - -- Sergio Rubio Wed, 01 Mar 2017 09:39:26 -0800 - -github-backup-utils (2.8.3) UNRELEASED; urgency=medium - - * Set restore status on all cluster nodes #274 - * Fix pages backups and restores in GitHub Enterprise 11.10 #275 - - -- Steven Honson Wed, 21 Dec 2016 21:01:20 +1100 - -github-backup-utils (2.8.2) UNRELEASED; urgency=medium - - * Backup and restore the appliance UUID #272 - - -- Sergio Rubio Thu, 17 Nov 2016 15:58:08 +0100 - -github-backup-utils (2.8.1) UNRELEASED; urgency=medium - - * Stop cron and timerd during restore #269 - * Fix compatibility issue with older versions of OpenSSH #263 - - -- Sergio Rubio Mon, 14 Nov 2016 22:04:48 +0100 - -github-backup-utils (2.8.0) UNRELEASED; urgency=low - - * Adds support for GitHub Enterprise 2.8.0 - * Speedup storage restores #247 - * More portable backup-utils #260 - - -- rubiojr Wed, 09 Nov 2016 06:35:21 -0800 - -github-backup-utils (2.7.1) UNRELEASED; urgency=medium - - * Cluster: fix offline cluster node detection #250 - * Detect leaked ssh keys in backup snapshots #253 - - -- Sergio Rubio Tue, 20 Sep 2016 20:15:12 +0200 - -github-backup-utils (2.7.0) UNRELEASED; urgency=medium - - * GitHub Enterprise 2.7.0 support - - -- Sergio Rubio Wed, 03 Aug 2016 20:25:31 +0200 - -github-backup-utils (2.6.4) UNRELEASED; urgency=medium - - * Instrument/benchmark backups #238 - * Cluster: remove restoring cluster.conf on restore #242 - * Cluster: Prevent restoring to a standalone GHE appliance #244 - - -- Sergio Rubio Wed, 27 Jul 2016 19:15:53 +0200 - -github-backup-utils (2.6.3) UNRELEASED; urgency=medium - - * Cluster: git-hooks backup fixes #235 - - -- Sergio Rubio Wed, 29 Jun 2016 21:05:21 +0200 - -github-backup-utils (2.6.2) UNRELEASED; urgency=medium - - * git-hooks fixes #231 - * Cluster: speedup repositories restore #232 (requires GitHub Enterprise - 2.6.4) - * Cluster: restore Git over SSH keys #230 - * Benchmark restores #219 - - -- Sergio Rubio Wed, 22 Jun 2016 19:36:06 +0200 - -github-backup-utils (2.6.1) UNRELEASED; urgency=medium - - * Cluster: faster gist restores #220 - * Cluster: faster storage restores #212 - * Cluster: fix git-hooks restores #204 - - -- Sergio Rubio Tue, 31 May 2016 20:54:11 +0200 - -github-backup-utils (2.6.0) UNRELEASED; urgency=medium - - * Adds support for GitHub Enterprise 2.6 - * Adds an extra supported location for the backup configuration #197 - * New config option to check for corrupted repositories after the backup #195 - * General improvements and bug fixes - - -- Sergio Rubio Tue, 26 Apr 2016 18:03:01 +0200 - -github-backup-utils (2.5.2) UNRELEASED; urgency=medium - - * New configuration variable: GHE_CREATE_DATA_DIR #186 - * Require that snapshots originated from an instance running GitHub - Enterprise 2.5.0 or above when restoring to a cluster #182 - * Prevent Git GC operations and some other maintenance jobs from running - while repositories are being restored #179 - * Fix Solaris and SmartOS support, using Bash everywhere #177 - - -- Sergio Rubio Wed, 30 Mar 2016 14:32:15 +0200 - -github-backup-utils (2.5.1) UNRELEASED; urgency=medium - - * Fixes for cluster restores #173 - * Fix Elasticsearch backups for GitHub Enterprise <= 2.2 #175 - * Removed experimental S3 support #167 - * Remote logging output fixes #170 - * Update ghe-host-check to detect extra port 22 error #162 - - -- Sergio Rubio Wed, 09 Mar 2016 14:44:05 +0100 - -github-backup-utils (2.5.0) UNRELEASED; urgency=medium - - * Adds GitHub Enterprise 2.5 support - * Adds GitHub Enterprise Clustering support - * Backups and restores SAML keypairs - - -- Sergio Rubio Tue, 9 Feb 2016 00:02:37 +0000 - -github-backup-utils (2.4.0) UNRELEASED; urgency=medium - - * Moves the in-progress detection to a separate file with PID which is - removed if the process is no longer running after the backup. #145, #99 - * Updates the README to explain why backup-utils is useful even if you have - the high availability replica running. #140 - * Changes the use of the --link-dest option to only occur when backing up - populated directories. #138 - * Adds logging to /var/log/syslog on the remote GitHub Enterprise appliance - to both ghe-backup and ghe-restore. #131 - * Restarts memcached after restoring to an already configured appliance to - ensure it doesn't contain out-of-sync information. #130 - * Removes the temporary /data/user/repositories-nw-backup directory that - remains after successfully migrating the repository storage layout to the - new format used on GitHub Enterprise 2.2.0 and later after restoring a - backup from an older release of GitHub Enterprise. #129 - * Add devscripts to Debian's build-depends for checkbashisms. #101 - * Documents the -c option which forces the restoration of the configuration - information to an already configured appliance. #96 - - -- Colin Seymour Tue, 20 Oct 2015 00:02:37 +0000 - -github-backup-utils (2.2.0) UNRELEASED; urgency=medium - - * Adds support for the new repositories filesystem layout include in - GitHub Enterprise v2.2. #122, #124 - * ghe-restore now performs a config run on the instance after an incremental - restore to 11.10.x and 2.x instances. #100 - * ghe-restore now fails fast when run against a GHE instance with replication - enabled. Replication should be disabled during a restore and then setup - after the restore completes. #121 - * Fixes an issue with special port 122 detection failing when port is - overridden in an ssh config file. #102 - * Removes a warning message when running ghe-backup against an instance with - GitHub Pages disabled. #117 - * backup-utils release version numbers now track GitHub Enterprise releases - to ease the process of determining which version of backup-utils is - required for a given GitHub Enterprise version. - - -- Ryan Tomayko Wed, 29 Apr 2015 07:29:04 +0000 - -github-backup-utils (2.0.2) UNRELEASED; urgency=medium - - * ghe-restore now requires that an already-configured appliance be put into - maintenance mode manually. This is a safeguard against accidentally - overwriting data on the wrong instance. #62, #84 - * ghe-backup and ghe-restore now run a ghe-negotiate-version program on the - appliance to determine whether the backup-utils and GHE versions are - compatible. #91 - * Various portability fixes for problems surfaced when running on Solaris - and FreeBSD. #86, #87 - * Fixes an issue in ghe-backup where mysqldump failures weren't being - reported properly. #90 - * Automated builds are now run on Travis CI. #77 - - -- Ryan Tomayko Tue, 20 Jan 2015 16:00:00 +0000 - -github-backup-utils (2.0.1) UNRELEASED; urgency=medium - - * Adds /etc/github-backup-utils/backup.config as a default config file search - location for deb / system installs. - * Enables SSH BatchMode for all remote command invocation except initial host - check / version identification. - * Fixes a bug in ghe-backup where Git GC process detection would misclassify - long-running server processes matching /git.*gc/, causing the backup operation - to timeout. - * Adds a note and link to the Migrating from GitHub Enterprise v11.10.34x to - v2.0 documentation in the README. - * Adds example / documentation for the GHE_EXTRA_SSH_OPTS config value to the - backup.config-example file. - - -- Ryan Tomayko Mon, 17 Nov 2014 12:47:22 +0000 - -github-backup-utils (2.0.0) UNRELEASED; urgency=medium - - * Support for GitHub Enterprise 2.0. - * Support for migrating from GitHub Enterprise 11.10.34x to 2.0 (including from - VMware to AWS). - * ghe-backup retains hardlinks present on VM in backup snapshots, saving space. - * ghe-restore retains hardlinks present in backup snapshot when restoring to VM. - * backup-utils now includes debian packaging support. - * Fixes an issue with ghe-restore -s not using the snapshot specified. - * Fixes an issue with ghe-backup not waiting for nw-repack processes to finish - in some instances. - - -- Ryan Tomayko Mon, 10 Nov 2014 10:48:36 +0000 - -github-backup-utils (1.1.0) UNRELEASED; urgency=medium - - * Updated documentation on minimum GitHub Enterprise version requirements for - online and incremental backups from v11.10.341 to at least v11.10.342. - * The ghe-restore command now prompts for confirmation of the host to restore to - before performing any destructive operation. This is to reduce the chances of - restoring to the wrong host. The prompt may be bypassed in automated scenarios - by providing the --force option. - * Added a -c option to ghe-restore for restoring base appliance settings in - addition to primary datastores. See ghe-restore --help for more information. - * Added a note about disabling maintenance mode on the appliance after a - successful ghe-restore operation. - * Added support for filesystem layout changes and upgraded server components in - * future versions of GitHub Enterprise. - - -- Twan Wolthof Sat, 18 Oct 2014 19:14:47 +0000 - -github-backup-utils (1.0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- Twan Wolthof Tue, 23 Sep 2014 08:34:55 +0000 diff --git a/debian/clean b/debian/clean deleted file mode 100644 index 0f651869c..000000000 --- a/debian/clean +++ /dev/null @@ -1 +0,0 @@ -debian/*.1 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index ec635144f..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/debian/control b/debian/control deleted file mode 100644 index b2cd7c945..000000000 --- a/debian/control +++ /dev/null @@ -1,32 +0,0 @@ -Source: github-backup-utils -Maintainer: Zachary Mark -Section: misc -Priority: optional -Standards-Version: 4.5.1 -Build-Depends: debhelper (>= 9), git, moreutils, jq, rsync (>= 2.6.4), help2man, -Homepage: https://github.com/github/backup-utils -Vcs-Git: https://github.com/github/backup-utils.git -Vcs-Browser: https://github.com/github/backup-utils -Rules-Requires-Root: no - -Package: github-backup-utils -Architecture: all -Depends: ${misc:Depends}, rsync (>= 2.6.4), moreutils, jq, git -Description: Backup and recovery utilities for GitHub Enterprise Server - The backup utilities implement a number of advanced capabilities for backup - hosts, built on top of the backup and restore features already included in - GitHub Enterprise Server. - . - These advanced features include: - - Complete GitHub Enterprise Server backup and recovery system via two simple - utilities: `ghe-backup` and `ghe-restore`. - - Online backups. The GitHub appliance need not be put in maintenance mode for - the duration of the backup run. - - Incremental backup of Git repository data. Only changes since the last - snapshot are transferred, leading to faster backup runs and lower network - bandwidth and machine utilization. - - Efficient snapshot storage. Only data added since the previous snapshot - consumes new space on the backup host. - - Multiple backup snapshots with configurable retention periods. - - Backup runs under the lowest CPU/IO priority on the GitHub appliance, - reducing performance impact while backups are in progress. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 8879cfae8..000000000 --- a/debian/copyright +++ /dev/null @@ -1,33 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: GitHub Enterprise Server Backup Utilities -Source: https://github.com/github/backup-utils - -Files: * -Copyright: 2014-2021, GitHub Inc. -License: MIT - -Files: debian/* -Copyright: 2014-2021, GitHub Inc. - 2014 Twan Wolthof - 2021 Hideki Yamane -License: MIT - -License: MIT - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - . - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/debian/install b/debian/install deleted file mode 100644 index bc992fc49..000000000 --- a/debian/install +++ /dev/null @@ -1,2 +0,0 @@ -bin/* usr/bin -share/* usr/share diff --git a/debian/manpages b/debian/manpages deleted file mode 100644 index 0f651869c..000000000 --- a/debian/manpages +++ /dev/null @@ -1 +0,0 @@ -debian/*.1 diff --git a/debian/rules b/debian/rules deleted file mode 100755 index af3d39d08..000000000 --- a/debian/rules +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/make -f - -VERSION=$$(cat $(CURDIR)/share/github-backup-utils/version) - -override_dh_auto_build: - # generate manpages for ghe-backup, ghe-host-check and ghe-restore - help2man $(CURDIR)/bin/ghe-backup -N -o $(CURDIR)/debian/ghe-backup.1 \ - -n "Take snapshots of all GitHub Enterprise data" \ - --version-string="ghe-backup $(VERSION)" - help2man $(CURDIR)/bin/ghe-host-check -N -o $(CURDIR)/debian/ghe-host-check.1 \ - -n "Restores a GitHub instance from local backup snapshots" \ - --version-string="ghe-host-check $(VERSION)" - help2man $(CURDIR)/bin/ghe-restore -N -o $(CURDIR)/debian/ghe-restore.1 \ - -n "Verify connectivity with the GitHub Enterprise Server host" \ - --version-string="ghe-restore $(VERSION)" - -%: - dh $@ diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db8f..000000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/docs/README.md b/docs/README.md index 153549d14..e84d313df 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,7 @@ - **[GitHub Enterprise Server version requirements](requirements.md#github-enterprise-version-requirements)** - **[Getting started](getting-started.md)** - **[Using the backup and restore commands](usage.md)** -- **[Scheduling backups](scheduling-backups.md)** +- **[Scheduling backups & snapshot pruning](scheduling-backups.md)** - **[Backup snapshot file structure](backup-snapshot-file-structure.md)** - **[How does Backup Utilities differ from a High Availability replica?](faq.md)** - **[Docker](docker.md)** diff --git a/docs/backup-snapshot-file-structure.md b/docs/backup-snapshot-file-structure.md index 2a950d63e..0a7032397 100644 --- a/docs/backup-snapshot-file-structure.md +++ b/docs/backup-snapshot-file-structure.md @@ -52,13 +52,41 @@ Actions service uses MS SQL Server as backend data store. Each snapshot includes To save time in backup, a three-level backup strategy is implemented. Based on the `GHE_MSSQL_BACKUP_CADENCE` setting, at each snapshot, either a (**F**)ull backup, a (**D**)ifferential or a (**T**)ransaction log backup is taken. As a result, a suite always contains following for each database: a full backup, possibly a differential backup and at least one transaction log backup. Their relationship with timeline is demonstrated below: -``` + +```text M---8:00--16:00---T---8:00--16:00---W... (timeline) F-----------------F-----------------F... (full backup) #-----D-----D-----#-----D-----D-----#... (differential backup) T--T--T--T--T--T--T--T--T--T--T--T--T... (transaction log backup) ``` + To save disk space, at each snapshot, hard links are created to point to previous backup files. Only newly-created backup files are transferred from appliance to backup host. When a new full/differential backup is created, they become the new source for hard links and new base line for transaction log backups, for subsequent snapshots. -During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log. \ No newline at end of file +During restore, a suite of backup files are restored in the sequence of full -> differential -> chronological transaction log. + +## Benchmark data + +Benchmark data for each snapshot is stored as a log file within the `benchmarks` directory within a snapshot directory. The benchmark log can be used to determine the duration of each backup step. For example: + +```text +ghe-backup-store-version took 0s +ghe-backup-settings took 2s +ghe-export-authorized-keys took 0s +ghe-export-ssh-host-keys took 0s +ghe-backup-mysql-binary took 9s +ghe-backup-mysql took 9s +ghe-backup-minio took 0s +ghe-backup-redis took 1s +ghe-backup-es-audit-log took 1s +ghe-backup-repositories - Generating routes took 3s +ghe-backup-repositories - Fetching routes took 0s +ghe-backup-repositories - Processing routes took 0s +ghe-backup-pages - hostname took 1s +ghe-backup-pages took 1s +ghe-backup-storage - Generating routes took 2s +ghe-backup-storage - Fetching routes took 0s +ghe-backup-storage - Processing routes took 0s +ghe-backup-git-hooks took 0s +ghe-backup-es-rsync took 2s +``` diff --git a/backup.config-example b/docs/backup.config-example similarity index 63% rename from backup.config-example rename to docs/backup.config-example index 8c98655ab..39a4d417f 100644 --- a/backup.config-example +++ b/docs/backup.config-example @@ -16,6 +16,28 @@ GHE_DATA_DIR="data" # be available for the past N days ... GHE_NUM_SNAPSHOTS=10 +# Pruning snapshots can be scheduled outside of the backup process. +# If set to 'yes', snapshots will not be pruned by ghe-backup. +# Instead, ghe-prune-snapshots will need to be invoked separately via cron +#GHE_PRUNING_SCHEDULED=yes + +# If GHE_ROUTE_VERIFICATION is set to true then ghe-repository-backup and +# ghe-storage-backup will issue a warning if the repositories and objects in +# the backup do not match the pre-backup inventory of routes. +#GHE_ROUTE_VERIFICATION=false + +# If GHE_MANAGE_CONSOLE_PW_RESTORE is set to false then management-console password +# will not be restored from backed-up snapshot data, it is restored by default +#GHE_MANAGE_CONSOLE_PW_RESTORE=true + +# If GHE_SKIP_CHECKS is set to true (or if --skip-checks is used with ghe-backup) then ghe-host-check +# disk space validation and software version checks on the backup-host will be disabled. +#GHE_SKIP_CHECKS=false + +# Cluster filesystem to check if it's writable as part of ghe-host-check +# By default it is /data/user/tmp but can be updated if needed +#GHE_FILE_SYSTEM_WRITE_CHECK="/data/user/tmp" + # The hostname of the GitHub appliance to restore. If you've set up a separate # GitHub appliance to act as a standby for recovery, specify its IP or hostname # here. The host to restore to may also be specified directly when running @@ -27,6 +49,10 @@ GHE_NUM_SNAPSHOTS=10 # #GHE_RESTORE_SKIP_AUDIT_LOGS=no +# If set to 'yes', backup and restore of Elasticsearch indices will be skipped +# +#GHE_SKIP_SEARCH_INDICES=no + # When verbose output is enabled with `-v`, it's written to stdout by default. If # you'd prefer it to be written to a separate file, set this option. # @@ -37,11 +63,30 @@ GHE_NUM_SNAPSHOTS=10 # In a clustering environment, "-i abs-path-to-ssh-private-key" is required. # #GHE_EXTRA_SSH_OPTS="" +# +# All backup processes are ran with the lowest priority for scheduling by default. +# To change throttling behaviour/allow higher priority for backup processes, set higher values for following variables. +# default value for GHENICE=nice -n 19 +# default value for GHE_IONICE=ionice -c 3 +#GHE_NICE="" +#GHE_IONICE="" # Any extra options passed to the rsync command. Nothing required by default. # #GHE_EXTRA_RSYNC_OPTS="" +# If set to 'yes', rsync will be set to use compression during backups and restores transfers. Defaults to 'no'. +# +#GHE_RSYNC_COMPRESSION_ENABLED=yes + +# If enabled and set to 'no', rsync warning message during backups will be suppressed. +#RSYNC_WARNING=no + + +# If set to 'yes', logging output will be colorized. +# +#OUTPUT_COLOR=no + # If set to 'no', GHE_DATA_DIR will not be created automatically # and restore/backup will exit 8 # @@ -90,3 +135,6 @@ GHE_NUM_SNAPSHOTS=10 # When running an external mysql database, run this script to trigger a MySQL restore # rather than attempting to backup via backup-utils directly. #EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false" + +# If set to 'yes', Pages data will be included in backup and restore. Defaults to 'yes' +#GHE_BACKUP_PAGES=no diff --git a/docs/getting-started.md b/docs/getting-started.md index 58054f3bf..f8806e616 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,13 +1,9 @@ # Getting started - 1. [Download the latest release version][1] and extract the repository using `tar`: + 1. [Download the latest version of backup-utils][1] and extract the repository using `tar`: `tar -xzvf /path/to/github-backup-utils-vMAJOR.MINOR.PATCH.tar.gz` - *or* clone the repository using Git: - - `git clone -b stable https://github.com/github/backup-utils.git` - **Note**: you will need to use [Backup Utilities v2.11.x][2] or the `legacy` branch to backup and restore GitHub Enterprise Server 2.10 and earlier. @@ -41,5 +37,5 @@ [1]: https://github.com/github/backup-utils/releases [2]: https://github.com/github/backup-utils/releases/tag/v2.11.4 -[3]: https://github.com/github/enterprise-backup-site/blob/master/backup.config-example +[3]: https://github.com/github/backup-utils/blob/master/docs/backup.config-example [4]: https://docs.github.com/enterprise-server/admin/configuration/configuring-your-enterprise/accessing-the-administrative-shell-ssh diff --git a/docs/requirements.md b/docs/requirements.md index f6b9b8f31..c87a0775a 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -5,15 +5,38 @@ storage and must have network connectivity with the GitHub Enterprise Server app ## Backup host requirements -Backup host software requirements are modest: Linux or other modern Unix operating -system (Ubuntu is highly recommended) with [bash][1], [git][2], [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer, and [jq][11] v1.5 or newer. +Backup host software requirements are modest: Linux or other modern Unix operating system (Ubuntu is highly recommended) with [bash][1], [git][2] 1.7.6 or newer, [OpenSSH][3] 5.6 or newer, [rsync][4] v2.6.4 or newer* (see [below](april-2023-update-of-rsync-requirements) for exceptions), [jq][11] v1.5 or newer and [bc][12] v1.0.7 or newer. See below for an update on rsync. The parallel backup and restore feature will require [GNU awk][10] and [moreutils][9] to be installed. We encourage the use of [Docker](docker.md), as it ensures compatible versions of the aforementioned software are available to backup-utils. -The backup host must be able to establish outbound network connections to the -GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise Server. +The backup host must be able to establish outbound network connections to the GitHub appliance over SSH. TCP port 122 is used to backup GitHub Enterprise Server. + +CPU and memory requirements are dependent on the size of the GitHub Enterprise Server appliance. We recommend a minimum of 4 cores and 8GB of RAM for the host running [GitHub Enterprise Backup Utilities](https://github.com/github/backup-utils). We recommend monitoring the backup host's CPU and memory usage to ensure it is sufficient for your environment. + +### April 2023 Update of Rsync Requirements + +The [fix in rsync `3.2.5`](https://github.com/WayneD/rsync/blob/master/NEWS.md#news-for-rsync-325-14-aug-2022) for [CVE-2022-29154](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-29154) can cause severe performance degradation to `backup-utils`. + +If you encounter this degradation you can mitigate it by using the `--trust-sender` flag, which is available in rsync >= v3.2.5. +**Note**: If you are using backup-utils 3.9 or greater, `--trust-sender` is automatically used if your rsync version supports it and no further changes are needed. + +If your backup host is running rsync < v3.2.5 you may or may not need to make changes to your rsync package, depending on whether your rsync package has backported the fix for CVE-2022-29154 without also backporting the `--trust-sender` flag. + +If your rsync package has backported the CVE fix _and_ the `--trust-sender` flag then you don't need to change anything. + +However, if your rsync package has backported the CVE fix without backporting the `--trust-sender` flag then you have three options: + +1. Downgrade (using the package manager on your host) the rsync package to a version before the CVE fix was backported +2. Upgrade (using the package manager on your host) the rsync package to v3.2.5 or newer +3. Manually download rsync v3.2.5 or newer and build the rsync binary + +Option #3 is required if your operating system's package manager does not have access to rsync v3.2.5 or later (e.g. Ubuntu Focal). + +Please note that some operating systems have their own versioning scheme for packages (including `rsync`). +If your backup host is using one of these operating systems, you will not be able to rely on a version check to determine whether you are +affected by the `rsync` performance degredation described above. ## Storage requirements @@ -37,28 +60,32 @@ ls -la Using a [case sensitive][7] file system is also required to avoid conflicts. +Performance of backup and restore operations are also dependent on the backup host's storage. We recommend using a high performance storage system with low latency and high IOPS. + +Please avoid using an NFS mount for the data directory (where backup data is stored) as this can cause performance issues and timeouts during backups. + ## GitHub Enterprise Server version requirements Starting with Backup Utilities v2.13.0, version support is inline with that of the [GitHub Enterprise Server upgrade requirements][8] and as such, support is limited to three versions of GitHub Enterprise Server: the version that corresponds with the version -of Backup Utilities, and the two releases prior to it. +of Backup Utilities, and the two versions prior to it. For example, Backup Utilities v2.13.0 can be used to backup and restore all patch -releases from 2.11.0 to the latest patch release of GitHub Enterprise Server 2.13. +versions from 2.11.0 to the latest patch version of GitHub Enterprise Server 2.13. Backup Utilities v2.14.0 will be released when GitHub Enterprise Server 2.14.0 is released -and will then be used to backup all releases of GitHub Enterprise Server from 2.12.0 -to the latest patch release of GitHub Enterprise Server 2.14. +and will then be used to backup all versions of GitHub Enterprise Server from 2.12.0 +to the latest patch version of GitHub Enterprise Server 2.14. Backup Utilities v2.11.4 and earlier offer support for GitHub Enterprise Server 2.10 -and earlier releases up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier +and earlier versions up to GitHub Enterprise Server 2.2.0. Backup Utilities v2.11.0 and earlier offer support for GitHub Enterprise Server 2.1.0 and earlier. -**Note**: You can restore a snapshot that's at most two feature releases behind +**Note**: You can restore a snapshot that's at most two feature versions behind the restore target's version of GitHub Enterprise Server. For example, to restore a snapshot of GitHub Enterprise Server 2.11, the target GitHub Enterprise Server appliance must be running GitHub Enterprise Server 2.12.x or 2.13.x. You can't restore a snapshot from -2.10 to 2.13, because that's three releases ahead. +2.10 to 2.13, because that's three versions ahead. **Note**: You _cannot_ restore a backup created from a newer version of GitHub Enterprise Server to an older version. For example, an attempt to restore a snapshot of GitHub Enterprise Server 2.21 to a GitHub Enterprise Server 2.20 environment will fail with an error of `Error: Snapshot can not be restored to an older release of GitHub Enterprise Server.`. @@ -75,7 +102,8 @@ Due to how some components of Backup Utilities (e.g. MSSQL) take incremental bac [5]: https://en.wikipedia.org/wiki/Hard_link [6]: https://en.wikipedia.org/wiki/Symbolic_link [7]: https://en.wikipedia.org/wiki/Case_sensitivity -[8]: https://help.github.com/enterprise/admin/guides/installation/upgrade-requirements/ +[8]: https://docs.github.com/enterprise-server/admin/monitoring-managing-and-updating-your-instance/updating-the-virtual-machine-and-physical-resources/upgrade-requirements [9]: https://joeyh.name/code/moreutils [10]: https://www.gnu.org/software/gawk [11]: https://stedolan.github.io/jq/ +[12]: https://www.gnu.org/software/bc/ diff --git a/docs/scheduling-backups.md b/docs/scheduling-backups.md index 8d291021a..3a7b0d12c 100644 --- a/docs/scheduling-backups.md +++ b/docs/scheduling-backups.md @@ -1,11 +1,20 @@ -# Scheduling backups +# Scheduling backups & snapshot pruning Regular backups should be scheduled using `cron(8)` or similar command scheduling service on the backup host. The backup frequency will dictate the worst case [recovery point objective (RPO)][1] in your backup plan. We recommend -hourly backups at the least. +hourly backups as a starting point. -## Example scheduling usage +It's important to consider the duration of each backup operation on the +GitHub Enterprise Server (GHES) appliance. Backups of large datasets or +over slow network links can take more than an hour. Additionally, +maintenance queues are paused during a portion of a backup runs. +We recommend scheduling backups to allow sufficient time for jobs +waiting in maintenance queues to process between backup runs + +Only one backup may be in progress at a time. + +## Example scheduling of backups The following examples assume the Backup Utilities are installed under `/opt/backup-utils`. The crontab entry should be made under the same user that @@ -20,15 +29,35 @@ storage. To schedule hourly backup snapshots with verbose informational output written to a log file and errors generating an email: - MAILTO=admin@example.com +```shell +MAILTO=admin@example.com - 0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1 +0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1 +``` To schedule nightly backup snapshots instead, use: - MAILTO=admin@example.com +```shell +MAILTO=admin@example.com + +0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1 +``` + +## Example snapshot pruning + +By default all expired and incomplete snapshots are deleted at the end of the main +backup process `ghe-backup`. If pruning these snapshots takes a long time you can +choose to disable the pruning process from the backup run and schedule it separately. +This can be achieved by enabling the `GHE_PRUNING_SCHEDULED` option in `backup.config`. +Please note that this option is only avilable for `backup-utils` >= `v3.10.0`. +If this option is enabled you will need to schedule the pruning script `ghe-prune-snapshots` using `cron` or a similar command scheduling service on the backup host. + +To schedule daily snapshot pruning, use: - 0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1 +```shell +MAILTO=admin@example.com +0 3 * * * /opt/backup-utils/share/github-backup-utils/ghe-prune-snapshots 1>>/opt/backup-utils/prune-snapshots.log 2>&1 +``` [1]: https://en.wikipedia.org/wiki/Recovery_point_objective diff --git a/docs/usage.md b/docs/usage.md index b4db45e73..58a816bc8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,11 +2,10 @@ After the initial backup, use the following commands: - - The `ghe-backup` command creates incremental snapshots of repository data, - along with full snapshots of all other pertinent data stores. - - The `ghe-restore` command restores snapshots to the same or separate GitHub - Enterprise appliance. You must add the backup host's SSH key to the target - GitHub Enterprise Server appliance before using this command. +- The `ghe-backup` command creates incremental snapshots of repository data, long with full snapshots of all other pertinent data stores. +- The `ghe-restore` command restores snapshots to the same or separate GitHub +Enterprise appliance. You must add the backup host's SSH key to the target +GitHub Enterprise Server appliance before using this command. These commands are run on the host you [installed][1] Backup Utilities on. @@ -14,9 +13,9 @@ These commands are run on the host you [installed][1] Backup Utilities on. You can supply your own configuration file or use the example configuration file as a template where you can set up your environment for backing up and restoring. -An example configuration file with documentation on possible settings can found in [backup.config-example](../backup.config-example). +An example configuration file with documentation on possible settings can found in [backup.config-example](backup.config-example). -There are a number of command line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`. +There are a number of command-line options that can also be passed to the `ghe-restore` command. Of particular note, if you use an external MySQL service but are restoring from a snapshot prior to enabling this, or vice versa, you must migrate the MySQL data outside of the context of backup-utils first, then pass the `--skip-mysql` flag to `ghe-restore`. ## Example backup and restore usage @@ -73,14 +72,19 @@ appliance at IP "5.5.5.5": Restore of 5.5.5.5:122 from snapshot 20180326T020444 finished. To complete the restore process, please visit https://5.5.5.5/setup/settings to review and save the appliance configuration. -A different backup snapshot may be selected by passing the `-s` argument and the -datestamp-named directory from the backup location. +A different backup snapshot may be selected by passing the `-s` argument to `ghe-restore` and specifying the +datestamp-named directory from the backup location as the value. The `ghe-backup` and `ghe-restore` commands also have a verbose output mode (`-v`) that lists files as they're being transferred. It's often useful to enable when output is logged to a file. -### Restoring settings, TLS certificate, and license +Every time you execute `ghe-backup` we verify the storage and software setup of the host +you [installed][1] Backup Utilities on, to make sure our [requirements][2] for the host are +met. You can disable this check using the `--skip-checks` argument or by +adding `GHE_SKIP_CHECKS=true` to your configuration file. + +### Restoring settings, TLS certificate, and license When restoring to a new GitHub Enterprise Server instance, settings, certificate, and license data *are* restored. These settings must be reviewed and saved before @@ -95,15 +99,25 @@ forcing settings, certificate, and license data to be overwritten with the backu ## Backup and restore with GitHub Actions enabled GitHub Actions data on your external storage provider is not included in regular GitHub Enterprise Server -backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with +backups, and must be backed up separately. When restoring a GitHub Enterprise Server backup with GitHub Actions enabled, the following steps are required: 1. Enable GitHub Actions on the replacement appliance and configure it to use the same GitHub Actions external storage configuration as the original appliance. -2. Put replacement appliance into maintenance mode. +2. Put replacement appliance into maintenance mode. 3. Use `ghe-restore` to restore the backup. 4. Re-register your self-hosted runners on the replacement appliance. Please refer to [GHES Documentation](https://docs.github.com/en/enterprise-server/admin/github-actions/advanced-configuration-and-troubleshooting/backing-up-and-restoring-github-enterprise-server-with-github-actions-enabled) for more details. +## Incremental MySQL Backups and Restores + +Incremental MySQL backup has been deprecated since 3.17 due to data integrity concerns. Restoring backups created with incremental backups remains supported for compatibility reasons. +## Rsync compression + +From backup-utils v3.11.0 onwards, we have disabled rsync compression by default to improve transfer speed and reduce CPU usage during the transfer process. + +If you would like to use compression with rsync, you can add `GHE_RSYNC_COMPRESSION_ENABLED=true` in your `backup.config` file. + [1]: https://github.com/github/backup-utils/blob/master/docs/getting-started.md +[2]: requirements.md diff --git a/script/cibuild b/script/cibuild deleted file mode 100755 index 7f57f444d..000000000 --- a/script/cibuild +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/cibuild [--no-package] -set -e - -# Enable verbose logging of ssh commands -export GHE_VERBOSE_SSH=true - -if ! find test -name "test-*.sh" -print0 | xargs -0 -P 4 -n 1 /bin/bash; then - exit 1 -fi - -# Bail out when --no-package given -[ "$1" = "--no-package" ] && exit 0 - -# files we'll md5sum at the end -pkg_files= - -# Build the tarball -ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -TMPDIR="$ROOTDIR/test/tmp" - -echo "Building tar.gz package ..." -if script/package-tarball 1>$TMPDIR/package-tarball.txt 2>&1; then - pkg_files=$(grep '^Package ' < $TMPDIR/package-tarball.txt | cut -f 2 -d ' ') -else - echo "Packaging tar.gz failed:" - cat $TMPDIR/package-tarball.txt | sed 's/^/ /' 1>&2 - exit 1 -fi - -# Skip deb packaging if debuild not installed -if ! type debuild 1>/dev/null 2>&1; then - echo "debuild not installed, skipping deb packaging ..." - exit 0 -fi - -# Build the deb related packages -echo "Building deb package ..." -if DEB_BUILD_OPTIONS=nocheck script/package-deb 1>$TMPDIR/package-deb-out.txt 2>$TMPDIR/package-deb-err.txt; then - pkg_files="$pkg_files $(cat $TMPDIR/package-deb-out.txt)" -else - echo "Package build failed:" - cat $TMPDIR/package-deb-out.txt $TMPDIR/package-deb-err.txt >&2 - echo >&2 - cat dist/debuild/github-backup-utils*.build >&2 - exit 1 -fi - -# Generate md5sums -md5sum $pkg_files diff --git a/script/package-deb b/script/package-deb deleted file mode 100755 index dda90cbe3..000000000 --- a/script/package-deb +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/package-deb -# Script to build a deb release package from the current HEAD version. -# The package version comes from the debian/changelog file so that should -# be edited before running this. -set -e - -# Change into project root -cd "$(dirname "$0")"/.. - -# Basic package name and version. -PKG_BASE="github-backup-utils" -PKG_VERS="$(git describe --tags)" -PKG_NAME="${PKG_BASE}-${PKG_VERS}" -PKG_HEAD="$(git rev-parse HEAD)" - -# Run git-archive to generate tarball -rm -rf dist/debuild -trap "rm -rf dist/debuild" EXIT -mkdir -p dist/debuild - -distdir="$(pwd)/dist/debuild/$PKG_NAME" -git clone -q . "$distdir" -cd "$distdir" -git checkout -q "$PKG_HEAD" - -debuild -uc -us 1>&2 -cd .. -files=$(ls -1 *.deb *.tar.xz *.dsc *.changes) -mv $files ../ -for f in $files; do echo "dist/$f"; done diff --git a/script/package-tarball b/script/package-tarball deleted file mode 100755 index be653b77a..000000000 --- a/script/package-tarball +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# Usage: script/package-tarball -# Script to build a tarball release package from the current HEAD version. -# The package version comes from `git-describe --tags' so the release tag should -# be in place before this command is run. -set -e - -# Change into project root -cd "$(dirname "$0")"/.. - -# Basic package name and version. -PKG_BASE="github-backup-utils" -PKG_VERS="$(git describe --tags)" -PKG_NAME="${PKG_BASE}-${PKG_VERS}" - -# Run git-archive to generate tarball -echo "Creating ${PKG_NAME}.tar.gz ..." -mkdir -p dist -git archive \ - --format=tar.gz \ - --prefix="$PKG_NAME/" \ - --output="dist/${PKG_NAME}.tar.gz" \ - HEAD - -# List archive contents for review -gzip -dc < "dist/${PKG_NAME}.tar.gz" | tar tf - - -# Output location -echo "Package dist/${PKG_NAME}.tar.gz OK" diff --git a/script/release b/script/release deleted file mode 100755 index a27194d4b..000000000 --- a/script/release +++ /dev/null @@ -1,466 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -#/ Usage: release [--dry-run] [--skip-version-bump-check] [min_version] -#/ -#/ Publish a backup-utils release: -#/ * Updates the package changelog -#/ * Bumps the backup-utils version if required -#/ * Creates the release pull request -#/ * Merges the release pull request -#/ * Creates the release draft -#/ * Tags the release -#/ * Builds the release assets and uploads them -#/ -#/ Notes: -#/ * Needs GH_RELEASE_TOKEN and GH_AUTHOR set in the environment. -#/ * Export GH_OWNER and GH_REPO if you want to use a different owner/repo -#/ * Only pull requests labeled with bug, feature or enhancement will show up in the -#/ release page and the changelog. -#/ * If this is a X.Y.0 release, a minimum supported version needs to be supplied too. -#/ -require 'json' -require 'net/http' -require 'time' -require 'erb' -require 'English' - -API_HOST = ENV['GH_HOST'] || 'api.github.com' -API_PORT = 443 -GH_REPO = ENV['GH_REPO'] || 'backup-utils' -GH_OWNER = ENV['GH_OWNER'] || 'github' -GH_AUTHOR = ENV['GH_AUTHOR'] -DEB_PKG_NAME = 'github-backup-utils' -GH_BASE_BRANCH = ENV['GH_BASE_BRANCH'] || 'master' - -CHANGELOG_TMPL = '''<%= package_name %> (<%= package_version %>) UNRELEASED; urgency=medium - -<%- changes.each do |ch| -%> - * <%= ch.strip.chomp %> -<% end -%> - - -- <%= GH_AUTHOR %> <%= Time.now.utc.strftime("%a, %d %b %Y %H:%M:%S %z") %> - -''' - -# Override Kernel.warn -def warn(msg) - Kernel.warn msg unless @no_warn -end - -def client(host = API_HOST, port = API_PORT) - @http ||= begin - c = Net::HTTP.new(host, port) - c.use_ssl = true - c - end -end - -def get(path) - req = Net::HTTP::Get.new(path) - req['Authorization'] = "token #{release_token}" - client.request(req) -end - -def post(path, body) - req = Net::HTTP::Post.new(path) - req['Authorization'] = "token #{release_token}" - req.body = body - client.request(req) -end - -def post_file(path, body) - req = Net::HTTP::Post.new(path) - req['Authorization'] = "token #{release_token}" - req['Content-Type'] = path.match?(/.*\.tar\.gz$/) ? 'application/tar+gzip' : 'application/vnd.debian.binary-package' - req.body = body - client.request(req) -end - -def put(path, body) - req = Net::HTTP::Put.new(path) - req['Authorization'] = "token #{release_token}" - req.body = body - client.request(req) -end - -def patch(path, body) - req = Net::HTTP::Patch.new(path) - req['Authorization'] = "token #{release_token}" - req.body = body - client.request(req) -end - -def release_token - token = ENV['GH_RELEASE_TOKEN'] - raise 'GH_RELEASE_TOKEN environment variable not set' if token.nil? - - token -end - -# Create a lightweight tag -def tag(name, sha) - body = { - "ref": "refs/tags/#{name}", - "sha": sha - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/git/refs", body) - - raise "Creating tag ref failed (#{res.code})" unless res.is_a? Net::HTTPSuccess -end - -def bug_or_feature?(issue_hash) - return true if issue_hash['labels'].find { |label| ['bug', 'feature', 'enhancement'].include?(label['name']) } - false -end - -def issue_from(issue) - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/issues/#{issue}") - raise "Issue ##{issue} not found in #{GH_OWNER}/#{GH_REPO}" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def beautify_changes(changes) - out = [] - changes.each do |chg| - next unless chg =~ /#(\d+)/ - begin - issue = issue_from Regexp.last_match(1) - out << "#{issue['title']} ##{Regexp.last_match(1)}" if bug_or_feature?(issue) - rescue => e - warn "Warning: #{e.message}" - end - end - - out -end - -def changelog - changes = `git log --pretty=oneline origin/stable...origin/#{GH_BASE_BRANCH} --reverse --grep "Merge pull request" | sort -t\# -k2`.lines.map(&:strip) - raise 'Building the changelog failed' if $CHILD_STATUS != 0 - - changes -end - -def build_changelog(changes, package_name, package_version) - ERB.new(CHANGELOG_TMPL, nil, '-').result(binding) -end - -def update_changelog(changes, name, version, path = 'debian/changelog') - raise 'debian/changelog not found' unless File.exist?(path) - File.open("#{path}.new", 'w') do |f| - f.puts build_changelog changes, name, version - f.puts(File.read(path)) - end - File.rename("#{path}.new", path) -end - -def create_release(tag_name, branch, rel_name, rel_body, draft = true) - body = { - 'tag_name': tag_name, - 'target_commitish': branch, - 'name': rel_name, - 'body': rel_body, - 'draft': draft, - 'prerelease': false - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/releases", body) - - raise "Failed to create release (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def publish_release(release_id) - body = { - 'draft': false - }.to_json - res = patch("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}", body) - - raise "Failed to update release (#{res.code})" unless res.is_a? Net::HTTPSuccess -end - -def list_releases - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/releases") - raise 'Failed to retrieve releases' unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def release_available?(tag_name) - return true if list_releases.find { |r| r['tag_name'] == tag_name } - - false -end - -def bump_version(new_version, min_version = nil, path = 'share/github-backup-utils/version') - current_version = Gem::Version.new(File.read(path).strip.chomp) - if !@skip_version_bump_check && (Gem::Version.new(new_version) < current_version) - raise "New version should be newer than #{current_version}" - end - File.open("#{path}.new", 'w') { |f| f.puts new_version } - File.rename("#{path}.new", path) - - unless min_version.nil? - content = File.read('bin/ghe-host-check') - new_content = content.gsub(/supported_minimum_version="[0-9]\.[0-9]+\.0"/, "supported_minimum_version=\"#{min_version}\"") - File.open('bin/ghe-host-check', 'w') {|file| file.puts new_content } - - content = File.read('test/testlib.sh') - new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/,"GHE_TEST_REMOTE_VERSION:=#{new_version}") - File.open('test/testlib.sh', 'w') {|file| file.puts new_content } - end -end - -def push_release_branch(version) - unless (out = `git checkout --quiet -b release-#{version}`) - raise "Creating release branch failed:\n\n#{out}" - end - - unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/cibuild`) - raise "Error committing changelog and version:\n\n#{out}" - end - - unless (out = `git push --quiet origin release-#{version}`) - raise "Failed pushing the release branch:\n\n#{out}" - end -end - -def update_stable_branch - `git checkout --quiet stable` - unless (out = `git merge --quiet --ff-only origin/#{GH_BASE_BRANCH}`) - warn "Merging #{GH_BASE_BRANCH} into stable failed:\n\n#{out}" - end - unless (out = `git push --quiet origin stable`) - warn "Failed pushing the stable branch:\n\n#{out}" - end -end - -def create_release_pr(version, release_body) - body = { - 'title': "Bump version: #{version}", - 'body': release_body, - 'head': "release-#{version}", - 'base': GH_BASE_BRANCH - }.to_json - res = post("/repos/#{GH_OWNER}/#{GH_REPO}/pulls", body) - raise "Creating release PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -def merge_pr(number, sha, version) - body = { - 'commit_title': "Merge pull request ##{number} from github/release-#{version}", - 'commit_message': "Bump version: #{version}", - 'sha': sha, - 'merge_method': 'merge' - }.to_json - pr_mergeable? number - res = put("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}/merge", body) - raise "Merging PR failed (#{res.code})" unless res.is_a? Net::HTTPSuccess - - JSON.parse(res.body) -end - -class RetryError < StandardError -end - -def pr_mergeable?(number) - begin - retries ||= 5 - res = get("/repos/#{GH_OWNER}/#{GH_REPO}/pulls/#{number}") - raise RetryError if JSON.parse(res.body)['mergeable'].nil? - mergeable = JSON.parse(res.body)['mergeable'] - rescue RetryError - sleep 1 - retry unless (retries -= 1).zero? - raise 'PR is unmergable.' - end - - mergeable || false -end - -def can_auth? - !ENV['GH_RELEASE_TOKEN'].nil? -end - -def repo_exists? - res = get("/repos/#{GH_OWNER}/#{GH_REPO}") - res.is_a? Net::HTTPSuccess -end - -def can_build_deb? - system('which debuild > /dev/null 2>&1') -end - -def package_tarball - unless (out = `script/package-tarball 2>&1`) - raise "Failed to package tarball:\n\n#{out}" - end - out -end - -def package_deb - unless (out = `DEB_BUILD_OPTIONS=nocheck script/package-deb 2>&1`) - raise "Failed to package Debian package:\n\n#{out}" - end - out -end - -def attach_assets_to_release(upload_url, release_id, files) - @http = nil - client(URI(upload_url.gsub(/{.*}/, '')).host) - begin - files.each do |file| - raw_file = File.open(file).read - res = post_file("/repos/#{GH_OWNER}/#{GH_REPO}/releases/#{release_id}/assets?name=#{File.basename(file)}", raw_file) - raise "Failed to attach #{file} to release (#{res.code})" unless res.is_a? Net::HTTPSuccess - end - rescue => e - raise e - end - @http = nil -end - -def clean_up(version) - `git checkout --quiet #{GH_BASE_BRANCH}` - `git fetch --quiet origin --prune` - `git pull --quiet origin #{GH_BASE_BRANCH} --prune` - `git branch --quiet -D release-#{version} >/dev/null 2>&1` - `git push --quiet origin :release-#{version} >/dev/null 2>&1` - `git branch --quiet -D tmp-packaging >/dev/null 2>&1` -end - -#### All the action starts #### -if $PROGRAM_NAME == __FILE__ - begin - args = ARGV.dup - dry_run = false - skip_version_bump_check = false - if args.include?('--dry-run') - dry_run = true - args.delete '--dry-run' - end - - if args.include?('--no-warn') - @no_warn = true - args.delete '--no-warn' - end - - if args.include?('--skip-version-bump-check') - @skip_version_bump_check = true - args.delete '--skip-version-bump-check' - end - - raise 'Usage: release [--dry-run] [--skip-version-bump-check] [min_version]' if args.empty? - - begin - version = Gem::Version.new(args[0]) - min_version = args[1] ? args[1] : nil - rescue ArgumentError - raise "Error parsing version #{args[0]}" - end - - raise "Minimum supported version is required for X.Y.0 releases\n\nUsage: release [--dry-run] [min_version]" if /[0-9]\.[0-9]+\.0/ =~ version.to_s && min_version.nil? - - raise "The repo #{GH_REPO} does not exist for #{GH_OWNER}" unless repo_exists? - - raise 'GH_AUTHOR environment variable is not set' if GH_AUTHOR.nil? - - release_changes = [] - release_changes = beautify_changes changelog if can_auth? - release_a = false - release_a = release_available? "v#{version}" - - puts "Bumping version to #{version}..." - bump_version version, min_version - - if dry_run - puts "Existing release?: #{release_a}" - puts "New version: #{version}" - puts "Min version: #{min_version}" unless min_version.nil? - puts "Owner: #{GH_OWNER}" - puts "Repo: #{GH_REPO}" - puts "Author: #{GH_AUTHOR}" - puts "Token: #{ENV['GH_RELEASE_TOKEN'] && 'set' || 'unset'}" - puts "Base branch: #{GH_BASE_BRANCH}" - puts 'Changelog:' - if release_changes.empty? - puts ' => No new bug fixes, enhancements or features.' - else - release_changes.each { |c| puts " * #{c}" } - end - puts "Changes:" - puts `git diff --color` - `git checkout -- share/github-backup-utils/version` - `git checkout -- bin/ghe-host-check` - `git checkout -- test/testlib.sh` - exit - end - - raise 'Unable to build Debian pkg: "debuild" not found.' unless can_build_deb? - - raise "Release #{version} already exists." if release_a - - `git fetch --quiet origin --prune` - branches = `git branch --all | grep release-#{version}$` - unless branches.empty? - out = "Release branch release-#{version} already exists. " - out += 'Branches found:' - branches.each_line { |l| out += "\n* #{l.strip.chomp}" } - raise out - end - - puts 'Updating changelog...' - update_changelog release_changes, DEB_PKG_NAME, version - release_body = "Includes general improvements & bug fixes" - release_body += " and support for GitHub Enterprise Server v#{version}" unless min_version.nil? - release_changes.each do |c| - release_body += "\n* #{c}" - end - - puts 'Pushing release branch and creating release PR...' - push_release_branch version - res = create_release_pr(version, "#{release_body}\n\n/cc @github/backup-utils") - - puts 'Merging release PR...' - res = merge_pr res['number'], res['head']['sha'], version - - puts 'Tagging and publishing release...' - tag "v#{version}", res['sha'] - - puts 'Creating release...' - release_title = "GitHub Enterprise Server Backup Utilities v#{version}" - res = create_release "v#{version}", GH_BASE_BRANCH, release_title, release_body, true - - # Tidy up before building tarball and deb pkg - clean_up version - - puts 'Building release tarball...' - package_tarball - - puts 'Building Debian pkg...' - package_deb - - puts 'Attaching Debian pkg and tarball to release...' - base_dir = File.expand_path(File.join(File.dirname(__FILE__), '..')) - attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}-v#{version}.tar.gz"] - attach_assets_to_release res['upload_url'], res['id'], ["#{base_dir}/dist/#{DEB_PKG_NAME}_#{version}_all.deb"] - - puts 'Publishing release...' - publish_release res['id'] - - puts 'Cleaning up...' - clean_up version - - puts 'Updating stable branch...' - update_stable_branch - - puts 'Released!' - rescue RuntimeError => e - $stderr.puts "Error: #{e}" - exit 1 - end -end diff --git a/share/github-backup-utils/bm.sh b/share/github-backup-utils/bm.sh deleted file mode 100755 index c512f3d44..000000000 --- a/share/github-backup-utils/bm.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# bm.sh: benchmarking Bash code blocks -# -# Example: -# bm_start "wget request" -# wget --quiet https://www.google.com -# bm_end "wget request" -# -# Sample output: -# $ bash test.sh -# wget request took 2s - -bm_desc_to_varname(){ - echo "__bm$(echo "$@" | tr -cd '[[:alnum:]]')" -} - -bm_start() -{ - eval "$(bm_desc_to_varname "$@")_start=$(date +%s)" - if [ -n "$GHE_DEBUG" ]; then - echo "Debug: $1 (bm_start)" - fi - bm_init > /dev/null -} - -bm_init() { - if [ -n "$BM_FILE_PATH" ]; then - echo $BM_FILE_PATH - return - fi - - local logfile="benchmark.${BM_TIMESTAMP:-$(date +"%Y%m%dT%H%M%S")}.log" - if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then - export BM_FILE_PATH=$GHE_RESTORE_SNAPSHOT_PATH/benchmarks/$logfile - else - export BM_FILE_PATH=$GHE_SNAPSHOT_DIR/benchmarks/$logfile - fi - - mkdir -p "$(dirname $BM_FILE_PATH)" - echo $BM_FILE_PATH -} - -bm_end() { - if [ -z "$BM_FILE_PATH" ]; then - echo "Call bm_start first" >&2 - exit 1 - fi - - local tend tstart total - tend=$(date +%s) - tstart=$(eval "echo \$$(bm_desc_to_varname "$@")_start") - total=$(($tend - $tstart)) - - echo "$1 took ${total}s" >> $BM_FILE_PATH - if [ -n "$GHE_DEBUG" ]; then - echo "Debug: $1 took ${total}s (bm_end)" - fi -} diff --git a/share/github-backup-utils/ghe-backup-actions b/share/github-backup-utils/ghe-backup-actions deleted file mode 100755 index f19777930..000000000 --- a/share/github-backup-utils/ghe-backup-actions +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-actions -#/ Take an online, incremental snapshot of all Actions data (excluding -#/ what is stored in MSSQL) -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") -backup_dir="$GHE_SNAPSHOT_DIR/actions" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/actions" ] && [ "$(ls -A $GHE_DATA_DIR/current/actions)" ]; then - link_dest="--link-dest=$GHE_DATA_DIR/current/actions" -fi - -# Transfer all Actions data from the user data directory using rsync. -ghe_verbose "* Transferring Actions files from $host ..." - -ghe-rsync -avz \ - -e "ghe-ssh -p $port" \ - --rsync-path='sudo -u actions rsync' \ - --exclude "mutexes" --exclude "dumps" --exclude "tmp" \ - $link_dest \ - "$host:$GHE_REMOTE_DATA_USER_DIR/actions/" \ - "$GHE_SNAPSHOT_DIR/actions" 1>&3 - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-config b/share/github-backup-utils/ghe-backup-config deleted file mode 100755 index ab0e24d4d..000000000 --- a/share/github-backup-utils/ghe-backup-config +++ /dev/null @@ -1,563 +0,0 @@ -#!/usr/bin/env bash -# Usage: . ghe-backup-config -# GitHub Enterprise backup shell configuration. -# -# This file is sourced by the various utilities under bin and share/github-backup-utils to -# load in backup configuration and ensure things are configured properly. -# -# All commands in share/github-backup-utils/ should start with the following: -# -# . $( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config -# -# And all commands in bin/ should start with the following: -# -# . $( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config -# -set +o posix -# Assume this script lives in share/github-backup-utils/ when setting the root -GHE_BACKUP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -# Get the version from the version file. -BACKUP_UTILS_VERSION="$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/version")" - -# If a version check was requested, show the current version and exit -if [ -n "$GHE_SHOW_VERSION" ]; then - echo "GitHub backup-utils v$BACKUP_UTILS_VERSION" - exit 0 -fi - -# Check for "--help|-h" in args or GHE_SHOW_HELP=true and show usage -# shellcheck disable=SC2120 # the script name is always referenced -print_usage() { - grep '^#/' <"$0" | cut -c 4- - exit "${1:-1}" -} - -if [ -n "$GHE_SHOW_HELP" ]; then - print_usage -else - for a in "$@"; do - if [ "$a" = "--help" ] || [ "$a" = "-h" ]; then - print_usage - fi - done -fi - -# Add the bin and share/github-backup-utils dirs to PATH -PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH" -# shellcheck source=share/github-backup-utils/bm.sh -. "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh" - -# Save off GHE_HOSTNAME from the environment since we want it to override the -# backup.config value when set. -GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME" - -# Source in the backup config file from the copy specified in the environment -# first and then fall back to the backup-utils root, home directory and system. -config_found=false -for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \ - "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do - if [ -f "$f" ]; then - GHE_BACKUP_CONFIG="$f" - # shellcheck disable=SC1090 # This is a user-supplied value that can't be predicted - . "$GHE_BACKUP_CONFIG" - config_found=true - break - fi -done - -GHE_RESTORE_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-restore") -GHE_BACKUP_IN_PROGRESS=$(readlink -fm "${GHE_DATA_DIR}/in-progress-backup") - -export GHE_RESTORE_IN_PROGRESS -export GHE_BACKUP_IN_PROGRESS - -ghe_restore_check() { - if [ -h $GHE_RESTORE_IN_PROGRESS ]; then - echo " Error: detected a restore already in progress from a previous version of ghe-restore." 1>&2 - echo " If there is no restore in progress anymore, please remove" 1>&2 - echo " the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi - - if [ -f $GHE_RESTORE_IN_PROGRESS ]; then - progress=$(cat $GHE_RESTORE_IN_PROGRESS) - pid=$(echo "$progress" | cut -d ' ' -f 2) - echo " Error: A restore of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo " If PID $pid is not a process related to the restore utilities, please remove" 1>&2 - echo " the $GHE_RESTORE_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi -} - -ghe_backup_check() { - if [ -h $GHE_BACKUP_IN_PROGRESS ]; then - echo " Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2 - echo " If there is no backup in progress anymore, please remove" 1>&2 - echo " the $GHE_DATA_DIR/$GHE_BACKUP_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi - - if [ -f $GHE_BACKUP_IN_PROGRESS ]; then - progress=$(cat $GHE_BACKUP_IN_PROGRESS) - pid=$(echo "$progress" | cut -d ' ' -f 2) - echo " Error: A backup of $GHE_HOSTNAME may still be running on PID $pid." 1>&2 - echo " If PID $pid is not a process related to the backup utilities, please remove" 1>&2 - echo " the $GHE_BACKUP_IN_PROGRESS file and try again." 1>&2 - exit 1 - fi -} - -ghe_restore_finished() { - if [ -f $GHE_RESTORE_IN_PROGRESS ]; then - rm -f $GHE_RESTORE_IN_PROGRESS - fi -} - -ghe_backup_finished() { - if [ -f $GHE_BACKUP_IN_PROGRESS ]; then - rm -f $GHE_BACKUP_IN_PROGRESS - fi -} - -ghe_parallel_check() { - if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then - return 0 - fi - - # Some machines may have both moreutils parallel and GNU parallel installed. - # Check some variants to find it - GHE_PARALLEL_COMMAND="parallel" - local x - for x in \ - /usr/bin/parallel.moreutils \ - /usr/bin/parallel_moreutils \ - /usr/bin/moreutils.parallel \ - /usr/bin/moreutils_parallel \ - ; do - if [ -x "${x}" ]; then - GHE_PARALLEL_COMMAND="${x}" - break - fi - done - - # Check that the GHE_PARALLEL_COMMAND is pointing to moreutils parallel - if ! $GHE_PARALLEL_COMMAND -h | grep -q "parallel \[OPTIONS\] command -- arguments"; then - echo "Error: moreutils not found. Please install https://joeyh.name/code/moreutils" 1>&2 - exit 1 - fi - - if [ -n "$GHE_PARALLEL_MAX_JOBS" ]; then - GHE_PARALLEL_COMMAND_OPTIONS="-j $GHE_PARALLEL_MAX_JOBS" - # Default to the number of max rsync jobs to the same as GHE_PARALLEL_MAX_JOBS, if not set. - # This is only applicable to ghe-restore-repositories currently. - : "${GHE_PARALLEL_RSYNC_MAX_JOBS:="$GHE_PARALLEL_MAX_JOBS"}" - fi - - if [ -n "$GHE_PARALLEL_RSYNC_MAX_JOBS" ]; then - GHE_PARALLEL_RSYNC_COMMAND_OPTIONS="-j $GHE_PARALLEL_RSYNC_MAX_JOBS" - fi - - if [ -n "$GHE_PARALLEL_MAX_LOAD" ]; then - GHE_PARALLEL_COMMAND_OPTIONS+=" -l $GHE_PARALLEL_MAX_LOAD" - GHE_PARALLEL_RSYNC_COMMAND_OPTIONS+=" -l $GHE_PARALLEL_MAX_LOAD" - fi -} - -# Check that the config file exists before we source it in. -if ! $config_found; then - echo "Error: No backup configuration file found. Tried:" 1>&2 - [ -n "$GHE_BACKUP_CONFIG" ] && echo " - $GHE_BACKUP_CONFIG" 1>&2 - echo " - $GHE_BACKUP_ROOT/backup.config" 1>&2 - echo " - $HOME/.github-backup-utils/backup.config" 1>&2 - echo " - /etc/github-backup-utils/backup.config" 1>&2 - exit 2 -fi - -# If verbose logging is enabled, redirect fd 3 to stdout or the specified log file; -# otherwise, redirect it to /dev/null. Write verbose output to fd 3. -if [ -n "$GHE_VERBOSE" ]; then - if [ -n "$GHE_VERBOSE_LOG" ]; then - if [ "$GHE_PARALLEL_ENABLED" != "yes" ]; then - exec 3>>"$GHE_VERBOSE_LOG" - else - if ! echo | awk '{ print strftime("%b %d %H:%M:%S"); fflush(); }' &>/dev/null; then - echo "Error: awk command failed. Please install https://www.gnu.org/software/gawk" 1>&2 - exit 1 - fi - calling_script_name="$(caller | sed 's:.*/::')" - exec 3> >(awk -v c="$calling_script_name" '{ print strftime("%b %d %H:%M:%S"), c":", $0; fflush(); }' >>"$GHE_VERBOSE_LOG") - fi - else - exec 3>&1 - fi -else - exec 3>/dev/null -fi - -# Restore saved off hostname. -[ -n "$GHE_HOSTNAME_PRESERVE" ] && GHE_HOSTNAME="$GHE_HOSTNAME_PRESERVE" - -# Check that the GHE hostname is set. -if [ -z "$GHE_HOSTNAME" ]; then - echo "Error: GHE_HOSTNAME not set in config file." 1>&2 - exit 2 -fi - -# Check that the GHE data directory is set. -if [ -z "$GHE_DATA_DIR" ]; then - echo "Error: GHE_DATA_DIR not set in config file." 1>&2 - exit 2 -fi - -# Convert the data directory path to an absolute path, basing any relative -# paths on the backup-utils root, and using readlink, if available, to -# canonicalize the path. -if [ "${GHE_DATA_DIR:0:1}" != "/" ]; then - GHE_DATA_DIR="$(cd "$GHE_BACKUP_ROOT" && readlink -m "$GHE_DATA_DIR" 2>/dev/null || echo "$GHE_BACKUP_ROOT/$GHE_DATA_DIR")" -fi -export GHE_DATA_DIR - -# Assign the Release File path if it hasn't been provided (eg: by test suite) -: "${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}" - -# Check that utils are not being run directly on GHE appliance. -if [ -f "$GHE_RELEASE_FILE" ]; then - echo "Error: Backup Utils cannot be run on the GitHub Enterprise host." 1>&2 - echo " The backup utilities should be run on a host dedicated to" 1>&2 - echo " long-term permanent storage and must have network connectivity" 1>&2 - echo " with the GitHub Enterprise appliance." 1>&2 - exit 1 -fi - -GHE_CREATE_DATA_DIR=${GHE_CREATE_DATA_DIR:-yes} - -# Check that the data directory is set and create it if it doesn't exist. -if [ ! -d "$GHE_DATA_DIR" ] && [ "$GHE_CREATE_DATA_DIR" = "yes" ]; then - echo "Creating the backup data directory ..." 1>&3 - mkdir -p "$GHE_DATA_DIR" -fi - -if [ ! -d "$GHE_DATA_DIR" ]; then - echo "Error: GHE_DATA_DIR $GHE_DATA_DIR does not exist." >&2 - exit 8 -fi - -# Set some defaults if needed. -: ${GHE_NUM_SNAPSHOTS:=10} - -# Generate a backup timestamp if one has not already been generated. -# We export the variable so the process group shares the same value. -: ${GHE_SNAPSHOT_TIMESTAMP:=$(date +"%Y%m%dT%H%M%S")} -export GHE_SNAPSHOT_TIMESTAMP - -# Set the current snapshot directory to /. This is where -# all backups should be written for the current invocation. -GHE_SNAPSHOT_DIR="$GHE_DATA_DIR"/"$GHE_SNAPSHOT_TIMESTAMP" -export GHE_SNAPSHOT_DIR - -# The root filesystem location. This must be used so that tests can override -# the root as a local directory location. -: ${GHE_REMOTE_ROOT_DIR:=""} - -# The root location of persistent data and applications on the remote side. This -# is always "/data" for GitHub instances. Use of this variable allows -# the location to be overridden in tests. -: ${GHE_REMOTE_DATA_DIR:="/data"} - -# The root location of user data stores such as git repositories, pages sites, -# elasticsearch indices, etc. This is "/data" under 1.x filesystem layouts and -# "/data/user" under the 2.x filesystem layout. The location is adjusted -# dynamically in ghe_remote_version_config() immediately after obtaining the -# remote version. Utilities that transfer data in and out of the appliance -# should use this variable to ensure proper behavior under different versions. -: ${GHE_REMOTE_DATA_USER_DIR:="$GHE_REMOTE_DATA_DIR"} - -# The location of the license file on the remote side. This is always -# "/data/enterprise/enterprise.ghl" for GitHub instances. Use of this variable -# allows the location to be overridden in tests. -: ${GHE_REMOTE_LICENSE_FILE:="$GHE_REMOTE_DATA_DIR/enterprise/enterprise.ghl"} - -# The number of seconds to wait for in progress git-gc processes to complete -# before starting the sync of git data. See share/github-backup-utils/ghe-backup-repositories-rsync -# for more information. Default: 10 minutes. -: ${GHE_GIT_COOLDOWN_PERIOD:=600} - -# Set "true" to get verbose logging of all ssh commands on stderr -: ${GHE_VERBOSE_SSH:=false} - -# The location of the cluster configuration file on the remote side. -# This is always "/data/user/common/cluster.conf" for GitHub Cluster instances. -# Use of this variable allows the location to be overridden in tests. -: ${GHE_REMOTE_CLUSTER_CONF_FILE:="$GHE_REMOTE_DATA_DIR/user/common/cluster.conf"} - -# The location of the file used to disable GC operations on the remote side. -: ${SYNC_IN_PROGRESS_FILE:="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress"} - -# Base path for temporary directories and files. -: ${TMPDIR:="/tmp"} - -# Backup cadence for MS SQL. Determines the kind of backup taken, either full, differential, -# or transaction log, based on when the last backup of that kind was taken. This defaults to -# taking a full backup once a week, a differential backup once a day, and transaction logs every -# 15 minutes. -: ${GHE_MSSQL_BACKUP_CADENCE:=10080,1440,15} - -############################################################################### -### Dynamic remote version config - -# Adjusts remote paths based on the version of the remote appliance. This is -# called immediately after the remote version is obtained by -# ghe_remote_version_required(). Child processes inherit the values set here. -ghe_remote_version_config() { - GHE_REMOTE_DATA_USER_DIR="$GHE_REMOTE_DATA_DIR/user" - export GHE_REMOTE_DATA_DIR GHE_REMOTE_DATA_USER_DIR - export GHE_REMOTE_LICENSE_FILE -} - -############################################################################### -### Utility functions - -# If we don't have a readlink command, parse ls -l output. -if ! type readlink 1>/dev/null 2>&1; then - readlink() { - if [ -x "$1" ]; then - ls -ld "$1" | sed 's/.*-> //' - else - return 1 - fi - } -fi - -# Run ghe-host-check and establish the version of the remote GitHub instance in -# the exported GHE_REMOTE_VERSION variable. If the remote version has already -# been established then don't perform the host check again. Utilities in share/github-backup-utils -# that need the remote version should use this function instead of calling -# ghe-host-check directly to reduce ssh roundtrips. The top-level ghe-backup and -# ghe-restore commands establish the version for all subcommands. -ghe_remote_version_required() { - if [ -z "$GHE_REMOTE_VERSION" ]; then - _out=$(ghe-host-check "$@") - echo "$_out" - - # override hostname w/ ghe-host-check output because the port could have - # been autodetected to 122. - GHE_HOSTNAME=$(echo "$_out" | sed 's/Connect \(.*:[0-9]*\) OK.*/\1/') - export GHE_HOSTNAME - - GHE_REMOTE_VERSION=$(echo "$_out" | sed 's/.*(\(.*\))/\1/') - export GHE_REMOTE_VERSION - - ghe_parse_remote_version "$GHE_REMOTE_VERSION" - ghe_remote_version_config "$GHE_REMOTE_VERSION" - fi - true -} - -# Parse a version string into major, minor and patch parts and echo. -ghe_parse_version() { - local version_major version_minor version_patch - version_major=$(echo "${1#v}" | cut -f 1 -d .) - version_minor=$(echo "$1" | cut -f 2 -d .) - version_patch=$(echo "$1" | cut -f 3 -d .) - version_patch=${version_patch%%[a-zA-Z]*} - - echo "$version_major $version_minor $version_patch" -} -# Parse major, minor, and patch parts of the remote appliance version and store -# in GHE_VERSION_MAJOR, GHE_VERSION_MINOR, and GHE_VERSION_PATCH variables. All -# parts are numeric. This is called automatically from -# ghe_remote_version_required so shouldn't be used directly. -# -# Scripts use these variables to alter behavior based on what's supported on the -# appliance version. -ghe_parse_remote_version() { - # shellcheck disable=SC2046 # Word splitting is required to populate the variables - read -r GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH <<<$(ghe_parse_version $1) - export GHE_VERSION_MAJOR GHE_VERSION_MINOR GHE_VERSION_PATCH -} - -# Parses the part out of a ":" or just "" string. -# This is used primarily to break hostspecs with non-standard ports down for -# rsync commands. -ssh_host_part() { - [ "${1##*:}" = "$1" ] && echo "$1" || echo "${1%:*}" -} - -# Parses the part out of a ":" or just "" string. -# This is used primarily to break hostspecs with non-standard ports down for -# rsync commands. -ssh_port_part() { - if [ "${1##*:}" != "$1" ] && [ "${1##*:}" -ne "122" ]; then - echo "Error: SSH port has to be 122 connecting to GitHub Enterprise Server, current value is ${1##*:} for $1." 1>&2 - exit 1 - fi - - echo 122 -} - -# Usage: ghe_remote_logger ... -# Log a message to /var/log/syslog on the remote instance. -# Note: Use sparingly. Remote logging requires an ssh connection per invocation. -ghe_remote_logger() { - echo "$@" | - ghe-ssh "$GHE_HOSTNAME" -- logger -t backup-utils || true -} - -# Usage: ghe_verbose -# Log if verbose mode is enabled (GHE_VERBOSE or `-v`). -ghe_verbose() { - if [ -n "$GHE_VERBOSE" ]; then - echo "$@" 1>&3 - fi -} - -# Usage: ghe_debug OR echo | ghe_debug -# Log if debug mode is enabled (GHE_DEBUG). -ghe_debug() { - [ -z "$GHE_DEBUG" ] && return - - if [ $# -ne 0 ]; then - echo -e "Debug: $*" 1>&3 - elif [ -p /dev/stdin ]; then - echo "\n" 1>&3 - while read line; do - echo -e "Debug: $line" 1>&3 - done &2 - exit 1 - ;; - esac - done - - echo -} - -# Function to restore a secret setting stored in a file. -# restore-secret -restore-secret() { - if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/$2" ]; then - echo "Restoring $1 ..." - echo "ghe-config '$3' '$(cat "$GHE_RESTORE_SNAPSHOT_PATH/$2")'" | - ghe-ssh "$GHE_HOSTNAME" -- /bin/bash - fi -} diff --git a/share/github-backup-utils/ghe-backup-es-audit-log b/share/github-backup-utils/ghe-backup-es-audit-log deleted file mode 100755 index 42ef9314a..000000000 --- a/share/github-backup-utils/ghe-backup-es-audit-log +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-es-audit-log -#/ Take a backup of audit logs in Elasticsearch. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root elastic backup directory based on config -host="$GHE_HOSTNAME" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Make sure root backup dir exists if this is the first run -mkdir -p "$GHE_SNAPSHOT_DIR/audit-log" - -if ! indices=$(ghe-ssh "$host" "curl -s \"localhost:9201/_cat/indices/audit_log*?h=index,pri.store.size&bytes=b\""); then - echo "Error: failed to retrieve audit log indices." 1>&2 - exit 1 -fi - -# Exit if no indices were found -[ -z "$indices" ] && exit - -# Determine if the audit log migration has occurred or is needed. -if echo 'set -o pipefail; ! test -e /data/user/common/es-scan-complete && test -f /usr/local/share/enterprise/run-audit-log-transitions.sh' | ghe-ssh "$host" /bin/bash; then - if echo 'set -o pipefail; echo n | /usr/local/share/enterprise/run-audit-log-transitions.sh > /dev/null 2>&1 && touch /data/user/common/es-scan-complete' | ghe-ssh "$host" /bin/bash; then - touch $GHE_SNAPSHOT_DIR/es-scan-complete - fi -fi - -IFS=$'\n' -for index in $indices; do - IFS=' ' - set $index - index_name=$1 - index_size=$2 - - if [[ -f $GHE_DATA_DIR/current/audit-log/$index_name.gz && $(cat $GHE_DATA_DIR/current/audit-log/$index_name.gz.size 2>/dev/null || true) -eq $index_size ]]; then - ghe_verbose "* Linking unchanged audit log index: $index_name" - # Hard link any indices that have not changed since the last backup - ln $GHE_DATA_DIR/current/audit-log/$index_name.gz $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz - ln $GHE_DATA_DIR/current/audit-log/$index_name.gz.size $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size - else - ghe_verbose "* Performing audit log export for index: $index_name" - echo "/usr/local/share/enterprise/ghe-es-dump-json \"http://localhost:9201/$index_name/" | gzip" | ghe-ssh "$host" -- /bin/bash > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz - echo $index_size > $GHE_SNAPSHOT_DIR/audit-log/$index_name.gz.size - fi -done - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-es-rsync b/share/github-backup-utils/ghe-backup-es-rsync deleted file mode 100755 index 11fa99f70..000000000 --- a/share/github-backup-utils/ghe-backup-es-rsync +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-es-rsync -#/ Take an online, incremental snapshot of Elasticsearch indices. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup when the rsync strategy is used. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root elastic backup directory based on config -host="$GHE_HOSTNAME" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch" - -# Verify that the /data/elasticsearch directory exists. -if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then - ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist." - exit 0 -fi - -# If we have a previous increment, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/elasticsearch" ]; then - link_dest="--link-dest=../../current/elasticsearch" -fi - -# Transfer ES indices from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. -ghe_verbose "* Performing initial sync of ES indices ..." -ghe-rsync -avz \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path="sudo -u elasticsearch rsync" \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ - "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 - -# Set up a trap to re-enable flushing on exit and remove temp file -cleanup () { - ghe_verbose "* Enabling ES index flushing ..." - echo '{"index":{"translog.disable_flush":false}}' | - ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null -} -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - -# Disable ES flushing and force a flush right now -ghe_verbose "* Disabling ES index flushing ..." -echo '{"index":{"translog.disable_flush":true}}' | -ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null -ghe-ssh "$host" -- curl -s -XPOST "localhost:9200/_flush" >/dev/null - -# Transfer all ES indices again -ghe_verbose "* Performing follow-up sync of ES indices ..." -ghe-rsync -avz \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path="sudo -u elasticsearch rsync" \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \ - "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3 - -# "Backup" audit log migration sentinel file -if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/es-scan-complete"; then - touch $GHE_SNAPSHOT_DIR/es-scan-complete -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-fsck b/share/github-backup-utils/ghe-backup-fsck deleted file mode 100755 index 004abe412..000000000 --- a/share/github-backup-utils/ghe-backup-fsck +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-fsck [--print-nwo] -#/ -#/ Run git fsck on backed up repositories. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -echo "Running git fsck on repos..." - -# Verify git is available. -if ! git --version 1>/dev/null 2>&1; then - echo "Error: git not found." 1>&2 - exit 1 -fi - -sdir=$1 -repos=0 -errors=0 -log=$(mktemp -t ghe-backup-fsck-XXXXXX) -t_start=$(date +%s) -if git fsck -h | grep -q '\-\-dangling'; then - git_cmd='git fsck --no-dangling' -else - echo "Warning: old git version, --no-dangling not available" - git_cmd='git fsck' -fi - -if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then - print_usage -fi - -if [ ! -d "$sdir/repositories" ]; then - echo "Error: $sdir is not a valid snapshot." >&2 - exit 1 -fi - -# shellcheck disable=SC2044 # Snapshot and repository directory names are safe for find iteration. -for repo in $(find $sdir/repositories/ -type d -name \*.git); do - repos=$(($repos+1)) - before_time=$(date +%s) - - status=$( - set -e - - cd $repo - - nwo="-" - if [ "$2" = "--print-nwo" ] && [ -f info/nwo ]; then - nwo="$(cat info/nwo)" - fi - - if [ ! -f objects/info/alternates ] || grep -q '^\.\.' objects/info/alternates; then - $git_cmd >$log 2>&1 && { - echo "OK $repo $nwo"; exit - } - else - GIT_ALTERNATE_OBJECT_DIRECTORIES=../network.git/objects $git_cmd >$log 2>&1 && { - echo "WARN $repo $nwo (alternates absolute path)"; exit - } - fi - - echo "ERROR $repo $nwo" - ) - - elapsed_time=$(($(date +%s) - before_time)) - - if [[ ! "$status" =~ ^OK ]] || [ $elapsed_time -gt 5 ]; then - echo "$status ${elapsed_time}s" 1>&3 - [ -n "$GHE_VERBOSE" ] && cat $log - fi - - case "$status" in - OK*) - ;; - ERROR*) - errors=$(($errors+1)) - ;; - esac - -done - -echo "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s" - -rm -f $log - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-git-hooks b/share/github-backup-utils/ghe-backup-git-hooks deleted file mode 100755 index 3229ee009..000000000 --- a/share/github-backup-utils/ghe-backup-git-hooks +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-git-hooks -#/ Take an online, incremental snapshot of custom Git hooks configuration. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -backup_dir="$GHE_SNAPSHOT_DIR/git-hooks" -# Location of last good backup for rsync --link-dest -backup_current="$GHE_DATA_DIR/current/git-hooks" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX) -opts="$GHE_EXTRA_SSH_OPTS" - -# git server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "git-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - rm -rf $tempdir -} -trap 'cleanup' EXIT -trap 'exit $?' INT # ^C always terminate - -# Transfer Git hooks data from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. A set of rsync filter rules are provided on stdin -# for each invocation. -rsync_git_hooks_data () { - port=$(ssh_port_part "$1") - host=$(ssh_host_part "$1") - - subpath=$2 - shift 2 - - # If we have a previous increment and it is not empty, avoid transferring existing files via rsync's - # --link-dest support. This also decreases physical space usage considerably. - if [ -d "$backup_current/$subpath" ] && [ "$(ls -A $backup_current/$subpath)" ]; then - subdir="git-hooks/$subpath" - link_path=".." - while true; do - if [ "$(dirname $subdir)" = "." ]; then - break - fi - - if [ "$(dirname $subdir)" = "/" ]; then - break - fi - - link_path="../$link_path" - subdir=$(dirname $subdir) - done - - local link_dest="--link-dest=../${link_path}/current/git-hooks/$subpath" - fi - - # Ensure target directory exists, is needed with subdirectories - mkdir -p "$backup_dir/$subpath" - - ghe-rsync -av \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" $link_dest \ - --rsync-path='sudo -u git rsync' \ - "$host:$GHE_REMOTE_DATA_USER_DIR/git-hooks/$subpath/" \ - "$backup_dir/$subpath" 1>&3 -} - -hostname=$(echo $hostnames | awk '{ print $1; }') -if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/environments/tarballs' ]"; then - rsync_git_hooks_data $hostname:122 environments/tarballs -else - ghe_verbose "git-hooks environment tarballs not found. Skipping ..." -fi - -if ghe-ssh $ssh_config_file_opt "$hostname:122" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/git-hooks/repos' ]"; then - rsync_git_hooks_data $hostname:122 repos -else - ghe_verbose "git-hooks repositories not found. Skipping ..." -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-minio b/share/github-backup-utils/ghe-backup-minio deleted file mode 100755 index 59dc7f8b3..000000000 --- a/share/github-backup-utils/ghe-backup-minio +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-minio -#/ Take an online, incremental snapshot of all minio data -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$(dirname "${BASH_SOURCE[0]}")/ghe-backup-config" - -bm_start "$(basename "${0}")" - -# Set up remote host and root backup snapshot directory based on config -port="$(ssh_port_part "${GHE_HOSTNAME}")" -host="$(ssh_host_part "${GHE_HOSTNAME}")" -backup_dir="${GHE_SNAPSHOT_DIR}/minio" - -# Verify rsync is available. -if ! command -v rsync 1> /dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "${host}" - -# Make sure root backup dir exists if this is the first run -mkdir -p "${backup_dir}" - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -# Hilariously, this HAS to stay unquoted when you call `rsync` further -# down because when the shell interpolates this out, `rsync` will throw -# an absolute fit if this variable is quoted. Surprise! -if [[ -d "${GHE_DATA_DIR}/current/minio" ]] && - [[ "$(ls -A "${GHE_DATA_DIR}/current/minio")" ]]; then - link_dest="--link-dest=${GHE_DATA_DIR}/current/minio" -fi - -# Transfer all minio data from the user data directory using rsync. -ghe_verbose "* Transferring minio files from ${host} ..." - -ghe-rsync \ - --archive \ - --verbose \ - --compress \ - --rsh="ghe-ssh -p ${port}" \ - --rsync-path='sudo -u minio rsync' \ - --exclude=".minio.sys" \ - ${link_dest} \ - "${host}:${GHE_REMOTE_DATA_USER_DIR}/minio/" \ - "${GHE_SNAPSHOT_DIR}/minio" 1>&3 - -bm_end "$(basename "${0}")" diff --git a/share/github-backup-utils/ghe-backup-mssql b/share/github-backup-utils/ghe-backup-mssql deleted file mode 100755 index c2ab49ce4..000000000 --- a/share/github-backup-utils/ghe-backup-mssql +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mssql -#/ -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Set up remote host and root backup snapshot directory based on config -backup_dir="$GHE_SNAPSHOT_DIR/mssql" -last_mssql= -backup_command= -backup_type= -full_expire= -diff_expire= -tran_expire= - -# Check if the export tool is available in this version -export_tool_available() { - if [ -z "$GHE_TEST_REMOTE_VERSION" ]; then - ghe_ssh_mssql "test -e /usr/local/bin/ghe-export-mssql" - else - # Always return available for test - return 0 - fi -} - -ghe_ssh_mssql() { - ghe-ssh $opts $ssh_config_file_opt "$GHE_MSSQL_PRIMARY_HOST" "$@" -} - -cleanup() { - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# use the mssql primary host if GHES cluster configuration contains a mssql-master or use the ghe server if the mssql-master is not available. -GHE_MSSQL_PRIMARY_NODE="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.mssql-master" || true)" -GHE_MSSQL_PRIMARY_HOST="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.$GHE_MSSQL_PRIMARY_NODE.hostname" || true)" - -if [ -z "$GHE_MSSQL_PRIMARY_HOST" ]; then - GHE_MSSQL_PRIMARY_HOST="$GHE_HOSTNAME" -fi - -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -ssh_config_file_opt= -opts= - -isHA="$(ghe-ssh "$GHE_HOSTNAME" -- "ghe-config cluster.ha" || true)" - -# get server hostnames under cluster and HA -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ] || [ "$isHA" = "true" ] ; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - ghe-ssh-config "$GHE_HOSTNAME" "$GHE_MSSQL_PRIMARY_HOST" > "$ssh_config_file" -fi - -if ! export_tool_available ; then - ghe_verbose "ghe-export-mssql is not available" - exit -fi - -add_minute() { - # Expect date string in the format of yyyymmddTHHMMSS - # Here parse date differently depending on GNU Linux vs BSD MacOS - if date -v -1d > /dev/null 2>&1; then - echo "$(date -v +$2M -ujf'%Y%m%dT%H%M%S' $1 +%Y%m%dT%H%M%S)" - else - dt=$1 - echo "$(date -u '+%Y%m%dT%H%M%S' -d "${dt:0:8} ${dt:9:2}:${dt:11:2}:${dt:13:2} $2 minutes")" - fi -} - -find_timestamp() { - filename="${1##*/}" - IFS='@' read -ra parts <<< "$filename" - datetime_part=${parts[1]:0:15} - echo "$datetime_part" -} - -actions_dbs() { - all_dbs=$(echo 'set -o pipefail; ghe-mssql-console -y -n -q "SET NOCOUNT ON; SELECT name FROM sys.databases"' | ghe_ssh_mssql /bin/bash) - for db in $all_dbs; do - if [[ ! "$db" =~ ^(master|tempdb|model|msdb)$ ]] && [[ "$db" =~ ^[a-zA-Z0-9_-]+$ ]]; then - echo "$db" - fi - done -} - -ensure_same_dbs() { - locals=() - while read -r file; do - filename=$(basename "$file") - locals+=("$filename") - done < <(find "$1" \( -name "*.bak" -o -name "*.diff" -o -name "*.log" \)) - - for remote in $(actions_dbs); do - remaining_locals=() - for local in "${locals[@]}"; do - if ! [[ "$local" == "$remote"* ]]; then - remaining_locals+=("$local") - fi - done - locals=("${remaining_locals[@]}") - done - - if [[ "${#locals[@]}" -ne 0 ]]; then - ghe_verbose "Warning: Found following ${#locals[@]} backup files that can't be traced back to the specified GHES host." - ghe_verbose "Warning: Did you recently reconfigure the GHES host? Move or delete these backup files if no longer needed." - for local in "${locals[@]}"; do - ghe_verbose "$1/$local" - done - - exit 1 - fi -} - -run_query() { - echo "set -o pipefail; ghe-mssql-console -y -n -q \"SET NOCOUNT ON; $1\"" | ghe_ssh_mssql /bin/bash | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' -} - -get_latest_backup_file() { - backups_dir=$1 - db=$2 - ext=$3 - - latest_full_backup=$(find "$backups_dir" -type f -name "$db*.$ext" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -n 1) - latest_full_backup_file="${latest_full_backup##*/}" - echo "$latest_full_backup_file" -} - -get_backup_val() { - db=$1 - filename=$2 - column=$3 - run_query " - SELECT s.$column - FROM msdb.dbo.backupset s - JOIN msdb.dbo.backupmediafamily f - ON s.media_set_id = f.media_set_id - WHERE s.database_name = '$db' AND f.physical_device_name LIKE '%$filename'" -} - -get_backup_checkpoint_lsn() { - get_backup_val "$1" "$2" "checkpoint_lsn" -} - -get_backup_last_lsn() { - get_backup_val "$1" "$2" "last_lsn" -} - -get_next_log_backup_starting_lsn() { - db=$1 - # last_log_backup_lsn: The starting log sequence number of the next log backup - # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-database-recovery-status-transact-sql - run_query " - SELECT last_log_backup_lsn - FROM sys.database_recovery_status drs - JOIN sys.databases db on drs.database_id = db.database_id - WHERE db.name = '$db'" -} - -get_next_diff_backup_base_lsn() { - db=$1 - # differential_base_lsn: Base for differential backups. Data extents changed after this LSN will be included in a differential backup. - # https://docs.microsoft.com/en-us/sql/relational-databases/system-catalog-views/sys-master-files-transact-sql - run_query " - SELECT differential_base_lsn - FROM sys.master_files mf - WHERE mf.name = '$db'" -} - -last_mssql=$GHE_DATA_DIR/current/mssql - -if [ ! -d "$last_mssql" ] \ - || [ -z "$(find "$last_mssql" -type f -name '*.bak' | head -n 1)" ]; then - ghe_verbose "Taking first full backup" - backup_type="full" -else - ensure_same_dbs "$last_mssql" - - # Check schedule to determine backup type - IFS=',' read -ra cadence <<< "$GHE_MSSQL_BACKUP_CADENCE" - - current=$(date -u +%Y%m%d%H%M%S) - - full=$(find "$last_mssql" -type f -name "*.bak" | head -n 1) - full=$(find_timestamp "$full") - full_expire=$(add_minute "$full" "${cadence[0]}") - full_expire="${full_expire//T}" - - diff=$(find "$last_mssql" -type f -name "*.diff" | head -n 1) - if [ -f "$diff" ]; then - diff=$(find_timestamp "$diff") - diff_expire=$(add_minute "$diff" "${cadence[1]}") - diff_expire="${diff_expire//T}" - else - diff_expire=$(add_minute "$full" "${cadence[1]}") - diff_expire="${diff_expire//T}" - fi - - tran=$(find "$last_mssql" -type f -name "*.log" | grep -E '[0-9]{8}T[0-9]{6}' | sort | tail -1) - tran=$(find_timestamp "$tran") - tran_expire=$(add_minute "$tran" "${cadence[2]}") - tran_expire="${tran_expire//T}" - - ghe_verbose "current $current, full expire $full_expire, \ -diff expire $diff_expire, tran expire $tran_expire" - - # Determine the type of backup to take based on expiry time - if [ "$current" -gt "$full_expire" ]; then - backup_type='full' - elif [ "$current" -gt "$diff_expire" ]; then - backup_type='diff' - elif [ "$current" -gt "$tran_expire" ]; then - backup_type='transaction' - fi - - # Upgrade to a full backup if the diff/transaction backup might not be restorable due to other backup mechanisms interfering - # with the transaction LSN chain or differential base LSN. - if [ "$backup_type" == 'diff' ] || [ "$backup_type" == 'transaction' ]; then - ghe_verbose "Checking for conflicting backups to ensure a $backup_type backup is sufficient" - - for db in $(actions_dbs); do - # Ensure that a diff backup will be based on the full backup file we have (rather than one another backup mechanism took) - if [ "$backup_type" == 'diff' ]; then - full_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "bak") - if [[ "$full_backup_file" == "" ]]; then - ghe_verbose "Taking a full backup instead of a diff backup because for $db a full backup file wasn't found" - backup_type="full" - break - fi - - full_backup_file_checkpoint_lsn=$(get_backup_checkpoint_lsn "$db" "$full_backup_file") - if [[ "$full_backup_file_checkpoint_lsn" = "NULL" ]] || [[ "$full_backup_file_checkpoint_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a diff backup because for $db the checkpoint LSN for $full_backup_file couldn't be determined" - backup_type="full" - break - fi - - next_diff_backup_base_lsn=$(get_next_diff_backup_base_lsn "$db") - if [[ "$next_diff_backup_base_lsn" = "NULL" ]] || [[ "$next_diff_backup_base_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the base LSN for the next diff backup couldn't be determined" - backup_type="full" - break - fi - - # The base of the diff backup we're about to take must exactly match the checkpoint LSN of the full backup file we have - if [[ "$next_diff_backup_base_lsn" -ne "$full_backup_file_checkpoint_lsn" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the diff would have base LSN $next_diff_backup_base_lsn yet our full backup has checkpoint LSN $full_backup_file_checkpoint_lsn" - backup_type="full" - break - fi - fi - - # Ensure that a transaction log backup will immediately follow the previous one - latest_log_backup_file=$(get_latest_backup_file "$last_mssql" "$db" "log") - if [[ "$latest_log_backup_file" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a previous transaction log backup wasn't found" - backup_type="full" - break - fi - - latest_log_backup_last_lsn=$(get_backup_last_lsn "$db" "$latest_log_backup_file") - if [[ "$latest_log_backup_last_lsn" = "NULL" ]] || [[ "$latest_log_backup_last_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the LSN range for $latest_log_backup_file couldn't be determined" - backup_type="full" - break - fi - - next_log_backup_starting_lsn=$(get_next_log_backup_starting_lsn "$db") - if [[ "$next_log_backup_starting_lsn" = "NULL" ]] || [[ "$next_log_backup_starting_lsn" == "" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db the starting LSN for the next log backup couldn't be determined" - backup_type="full" - break - fi - - # The starting LSN of the backup we're about to take must be equal to (or before) the last LSN from the last backup, - # otherwise there'll be a gap and the logfiles won't be restorable - if [[ "$next_log_backup_starting_lsn" -gt "$latest_log_backup_last_lsn" ]]; then - ghe_verbose "Taking a full backup instead of a $backup_type backup because for $db a gap would exist between the last backup ending at LSN $latest_log_backup_last_lsn and next backup starting at $next_log_backup_starting_lsn" - backup_type="full" - break - fi - done - fi -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Use hard links to "copy" over previous applicable backups to the new snapshot folder to save disk space and time -if [ -d "$last_mssql" ]; then - for p in "$last_mssql"/* - do - [[ -e "$p" ]] || break - - filename="${p##*/}" - extension="${filename##*.}" - transfer= - - # Copy full backups unless we're taking a new full backup - if [ "$extension" = "bak" ] && [ "$backup_type" != 'full' ]; then - transfer=1 - fi - - # Copy diff backups unless we're taking a new full or diff backup - if [ "$extension" = "diff" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then - transfer=1 - fi - - # Copy transaction log backups unless we're taking a new full or diff backup - if [ "$extension" = "log" ] && [ "$backup_type" != 'full' ] && [ "$backup_type" != 'diff' ]; then - transfer=1 - fi - - if [ -n "$transfer" ]; then - ghe_verbose "Creating hard link to $filename" - ln "$last_mssql"/"$filename" "$backup_dir"/"$filename" - fi - done -fi - -if [ -n "$backup_type" ]; then - ghe_verbose "Taking $backup_type backup" - - backup_command='ghe-export-mssql' - if [ "$backup_type" = "diff" ]; then - backup_command='ghe-export-mssql -d' - elif [ "$backup_type" = "transaction" ]; then - backup_command='ghe-export-mssql -t' - fi - - bm_start "$(basename "$0")" - ghe_ssh_mssql -- "$backup_command" || failures="$failures mssql" - bm_end "$(basename "$0")" - - # Configure the backup cadence on the appliance, which is used for diagnostics - ghe_ssh_mssql "ghe-config mssql.backup.cadence $GHE_MSSQL_BACKUP_CADENCE" - - # Transfer backup files from appliance to backup host - appliance_dir="$GHE_REMOTE_DATA_DIR/user/mssql/backups" - backups=$(echo "set -o pipefail; if sudo test -d \"$appliance_dir\"; then \ - sudo ls \"$appliance_dir\"; fi" | ghe_ssh_mssql /bin/bash) - for b in $backups - do - ghe_verbose "Transferring to backup host $b" - ghe_ssh_mssql "sudo cat $appliance_dir/$b" > "$backup_dir"/"$b" - done -fi diff --git a/share/github-backup-utils/ghe-backup-mysql b/share/github-backup-utils/ghe-backup-mysql deleted file mode 100755 index 7e3fd57e3..000000000 --- a/share/github-backup-utils/ghe-backup-mysql +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql -#/ Backup MySQL from a GitHub instance. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -if is_external_database_target; then - if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then - echo "Backing up external MySQL database using customer-provided script..." - $EXTERNAL_DATABASE_BACKUP_SCRIPT - bm_end "$(basename $0)" - exit 0 - else - if is_binary_backup_feature_on; then - echo "Warning: Binary backups are configured on the target environment." - echo "Binary backup is not supported with an external MySQL database. Backing up using logical backup strategy." - echo - echo "Please disable binary backups with 'ghe-config mysql.backup.binary false', or" - echo "provide a custom backup script using EXTERNAL_DATABASE_BACKUP_SCRIPT" - fi - - ghe-backup-mysql-logical - fi -else - if is_binary_backup_feature_on; then - ghe-backup-mysql-binary - else - ghe-backup-mysql-logical - fi -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-mysql-binary b/share/github-backup-utils/ghe-backup-mysql-binary deleted file mode 100755 index 37218c82e..000000000 --- a/share/github-backup-utils/ghe-backup-mysql-binary +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql-binary -#/ Backup MySQL from a GitHub instance using binary backup strategy. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -echo "Backing up MySQL database using binary backup strategy ..." - -echo "set -o pipefail; ghe-export-mysql" | -ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" -echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel" - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-mysql-logical b/share/github-backup-utils/ghe-backup-mysql-logical deleted file mode 100755 index 3dc2478ec..000000000 --- a/share/github-backup-utils/ghe-backup-mysql-logical +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-mysql-logical -#/ Backup MySQL from a GitHub instance using logical backup strategy. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish the remote version in GHE_REMOTE_VERSION. -ghe_remote_version_required "$GHE_HOSTNAME" - -echo "Backing up MySQL database using logical backup strategy ..." - -echo "set -o pipefail; ghe-export-mysql | pigz" | -ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz" - -if is_external_database_target; then - echo "LOGICAL_EXTERNAL_BACKUP" > "$GHE_SNAPSHOT_DIR/logical-external-database-backup-sentinel" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-pages b/share/github-backup-utils/ghe-backup-pages deleted file mode 100755 index 634e7a417..000000000 --- a/share/github-backup-utils/ghe-backup-pages +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-pages -#/ Take an online, incremental snapshot of all Pages data -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/pages" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-restore-XXXXXX) -opts="$GHE_EXTRA_SSH_OPTS" - -# Pages server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "pages-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/pages)" ]; then - link_dest="--link-dest=../../current/pages" -fi - -for hostname in $hostnames; do - bm_start "$(basename $0) - $hostname" - echo 1>&3 - ghe_verbose "* Starting backup for host: $hostname" - # Sync all auxiliary repository data. This includes files and directories like - # HEAD, audit_log, config, description, info/, etc. No refs or object data - # should be transferred here. - echo 1>&3 - ghe_verbose "* Transferring pages files ..." - - # Transfer all data from the user data directory using rsync. - ghe-rsync -avz \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - --rsync-path='sudo -u git rsync' \ - $link_dest \ - "$hostname:$GHE_REMOTE_DATA_USER_DIR/pages/" \ - "$GHE_SNAPSHOT_DIR/pages" 1>&3 - bm_end "$(basename $0) - $hostname" -done - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-redis b/share/github-backup-utils/ghe-backup-redis deleted file mode 100755 index b0eb44949..000000000 --- a/share/github-backup-utils/ghe-backup-redis +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-redis -#/ Take a snapshot of all Redis data. This is needed because older versions of -#/ the remote side ghe-export-redis command use a blocking SAVE instead of a -#/ non-blocking BGSAVE. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup command. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$GHE_HOSTNAME" - -# Force a redis BGSAVE, and wait for it to complete. -ghe-ssh "$GHE_HOSTNAME" /bin/bash < /dev/null; then - redis_cli=ghe-redis-cli - redis_arg=--remote - else - redis_cli=redis-cli - redis_arg= - fi - redis_host=\$(ghe-config cluster.redis-master 2>/dev/null || echo "localhost") - timestamp=\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE) - - for i in \$(seq 10); do - if ! \$redis_cli \$redis_arg -h \$redis_host BGSAVE | grep -q ERR; then - break - fi - sleep 15 - done - for n in \$(seq 3600); do - if [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ]; then - break - fi - sleep 1 - done - [ "\$(\$redis_cli \$redis_arg -h \$redis_host LASTSAVE)" != "\$timestamp" ] # exits 1 if bgsave didn't work - - if [ "\$redis_host" != "localhost" ]; then - ssh \$redis_host sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb' - else - sudo cat '$GHE_REMOTE_DATA_USER_DIR/redis/dump.rdb' - fi -EOF - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-repositories b/share/github-backup-utils/ghe-backup-repositories deleted file mode 100755 index 60dd5e38e..000000000 --- a/share/github-backup-utils/ghe-backup-repositories +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-repositories -#/ Take an online, incremental snapshot of all Git repository data. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# This command is designed to allow for transferring active Git repository data -# from a GitHub instance to a backup site in a way that ensures data is -# captured in a consistent state even when being written to. -# -# - All Git GC operations are disabled on the GitHub instance for the duration of -# the backup. This removes the possibly of objects or packs being removed -# while the backup is in progress. -# -# - In progress Git GC operations are given a cooldown window to complete. The -# script will sleep for up to 60 seconds waiting for GC operations to finish. -# -# - Git repository data is transferred in a specific order: auxiliary files, -# packed refs, loose refs, reflogs, and finally objects and pack files in that -# order. This ensures that all referenced objects are captured. -# -# - Git GC operations are re-enabled on the GitHub instance. -# -# The script uses multiple runs of rsync to transfer repository files. Each run -# includes a list of filter rules that ensure only specific types of files are -# transferred. -# -# See the "FILTER RULES" and "INCLUDE/EXCLUDE PATTERN RULES" sections of the -# rsync(1) manual for more information: -# - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/repositories" - -# Location of last good backup for rsync --link-dest -backup_current="$GHE_DATA_DIR/current/repositories" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) -routes_list=$tempdir/routes_list -remote_routes_list=$remote_tempdir/remote_routes_list -opts="$GHE_EXTRA_SSH_OPTS" - -# git server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "git-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - for pid in $(jobs -p); do - kill -KILL $pid > /dev/null 2>&1 || true - done - - # Enable remote GC operations - for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { - echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# Disable remote GC operations -for hostname in $hostnames; do - ghe-gc-disable $ssh_config_file_opt $hostname:$port -done - -# If we have a previous increment, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$backup_current" ]; then - link_dest="--link-dest=../../current/repositories" -fi - -# Calculate sync routes. This will store the healthy repo paths for each node -# -# This gets a repo path and stores the path in the $node.sync file -# a/nw/a8/3f/02/100000855 dgit-node1 >> dgit-node1.sync -# a/nw/a8/bc/8d/100000880 dgit-node3 >> dgit-node3.sync -# a/nw/a5/06/81/100000659 dgit-node2 >> dgit-node2.sync -# ... -# One route per line. -# -# NOTE: The route generation is performed on the appliance as it is considerably -# more performant than performing over an SSH pipe. -# -bm_start "$(basename $0) - Generating routes" -echo "github-env ./bin/dgit-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash -ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug -bm_end "$(basename $0) - Generating routes" - -bm_start "$(basename $0) - Fetching routes" -ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list -< $routes_list ghe_debug -bm_end "$(basename $0) - Fetching routes" - -bm_start "$(basename $0) - Processing routes" -if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then - server=$host -fi -< $routes_list awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' -ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" -bm_end "$(basename $0) - Processing routes" - -if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping repositories backup ..." - exit 0 -fi - -# Transfer repository data from a GitHub instance to the current snapshot -# directory, using a previous snapshot to avoid transferring files that have -# already been transferred. A set of rsync filter rules are provided on stdin -# for each invocation. -rsync_repository_data () { - port=$(ssh_port_part "$1") - host=$(ssh_host_part "$1") - - #check if we are syncing from a given file list - if [[ "$2" == *".rsync" ]]; then - files_list="$2" - shift - shift - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path='sudo -u git rsync' \ - --include-from=- --exclude=\* \ - --files-from="$files_list" \ - --ignore-missing-args \ - "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ - "$backup_dir" 1>&3 2>&3 - else - shift - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path='sudo -u git rsync' \ - --include-from=- --exclude=\* \ - --ignore-missing-args \ - "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \ - "$backup_dir" 1>&3 2>&3 - fi -} - -sync_data (){ - # Sync all auxiliary repository data. This includes files and directories like - # HEAD, audit_log, config, description, info/, etc. No refs or object data - # should be transferred here. - echo 1>&3 - - echo "* Transferring auxiliary files ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring packed-refs files ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring refs and reflogs ..." 1>&3 - rsync_repository_data $1:122 $2 -z <&3 - echo "* Transferring objects and packs ..." 1>&3 - rsync_repository_data $1:122 $2 -H <&3 - -} - -# rsync all the repositories -bm_start "$(basename $0) - Repo sync" -for file_list in $tempdir/*.rsync; do - hostname=$(basename $file_list .rsync) - - repo_num=$(< $file_list wc -l) - ghe_verbose "* Transferring $repo_num repositories from $hostname" - - sync_data $hostname $file_list & -done - -for pid in $(jobs -p); do - wait $pid -done -bm_end "$(basename $0) - Repo sync" - -# Since there are no routes for special data directories, we need to do this -# serially for all hostnames. Good candidate for future optimizations. - -bm_start "$(basename $0) - Special Data Directories Sync" -for h in $hostnames; do - # Sync __special__ data directories, including the __alambic_assets__, - # __hookshot__, and __purgatory__ directories. The __nodeload_archives__, - # __gitmon__, and __render__ directories are excludes since they act only as - # caches. - # - # Under v2.x and greater, only the special __purgatory__ directory remains under - # /data/repositories. All other special user data directories have been moved under - # the /data/user directory. - echo 1>&3 - echo "* Transferring special data directories from $h..." 1>&3 - rsync_repository_data $h:122 -z <&3 -done -bm_end "$(basename $0) - Special Data Directories Sync" - -if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then - bm_start "$(basename $0) - Verifying Routes" - for file_lst in $tempdir/*.rsync; do - < $file_lst sort | uniq - done |sort|uniq > $tempdir/source_routes - (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git | fix_paths_for_ghe_version | uniq | sort | uniq) > $tempdir/destination_routes - - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." - - bm_end "$(basename $0) - Verifying Routes" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-settings b/share/github-backup-utils/ghe-backup-settings deleted file mode 100755 index 5516e811b..000000000 --- a/share/github-backup-utils/ghe-backup-settings +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-settings -#/ Backup settings from a snapshot to the given . -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -bm_start "$(basename $0)" - -# Grab the host -host="$GHE_HOSTNAME" - -# Create the snapshot directory if needed and change into it. -mkdir -p "$GHE_SNAPSHOT_DIR" -cd "$GHE_SNAPSHOT_DIR" - -echo "* Transferring settings data ..." 1>&3 -ghe-ssh "$host" -- 'ghe-export-settings' > settings.json - -echo "* Transferring license data ..." 1>&3 -ghe-ssh "$host" -- "sudo cat '$GHE_REMOTE_LICENSE_FILE'" > enterprise.ghl - -# Function to backup a secret setting to a file. -# backup-secret [--best-effort] -backup-secret() { - - best_effort=false - description="" - file="" - setting="" - count=0 - - while [ $# -gt 0 ]; do - case "$1" in - --best-effort) - shift 1 - best_effort=true - ;; - *) - case $count in - 0) - description=$1 - ;; - 1) - file=$1 - ;; - 2) - setting=$1 - ;; - *) - >&2 echo "Too many arguments" - ;; - esac - count=$((count+1)) - shift 1 - esac - done - - echo "* Transferring $description ..." 1>&3 - ghe-ssh "$host" -- ghe-config "$setting" > "$file+" || ( - if [ "$best_effort" = "false" ]; then - echo "Warning: $description not set" >&2 - fi - ) - if [ -n "$(cat "$file+")" ]; then - mv "$file+" "$file" - else - unlink "$file+" - fi -} - -backup-secret "management console password" "manage-password" "secrets.manage" -backup-secret "password pepper" "password-pepper" "secrets.github.user-password-secrets" -backup-secret "kredz.credz HMAC key" "kredz-credz-hmac" "secrets.kredz.credz-hmac-secret" -backup-secret "kredz.varz HMAC key" "kredz-varz-hmac" "secrets.kredz.varz-hmac-secret" - -# Backup argon secrets for multiuser from ghes version 3.8 onwards -if [ "$(version $GHE_REMOTE_VERSION)" -gt "$(version 3.7.0)" ]; then - backup-secret "management console argon2 secret" "manage-argon-secret" "secrets.manage-auth.argon-secret" -fi - -# Backup external MySQL password if running external MySQL DB. -if is_service_external 'mysql'; then - backup-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql" -fi - -# Backup Actions settings. -if ghe-ssh "$host" -- ghe-config --true app.actions.enabled; then - backup-secret "Actions configuration database login" "actions-config-db-login" "secrets.actions.ConfigurationDatabaseSqlLogin" - backup-secret "Actions configuration database password" "actions-config-db-password" "secrets.actions.ConfigurationDatabaseSqlPassword" - backup-secret "Actions framework access token key secret" "actions-framework-access-token" "secrets.actions.FrameworkAccessTokenKeySecret" --best-effort - backup-secret "Actions Url signing HMAC key primary" "actions-url-signing-hmac-key-primary" "secrets.actions.UrlSigningHmacKeyPrimary" - backup-secret "Actions Url signing HMAC key secondary" "actions-url-signing-hmac-key-secondary" "secrets.actions.UrlSigningHmacKeySecondary" - backup-secret "Actions OAuth S2S signing cert" "actions-oauth-s2s-signing-cert" "secrets.actions.OAuthS2SSigningCert" - backup-secret "Actions OAuth S2S signing key" "actions-oauth-s2s-signing-key" "secrets.actions.OAuthS2SSigningKey" - backup-secret "Actions OAuth S2S signing cert thumbprint" "actions-oauth-s2s-signing-cert-thumbprint" "secrets.actions.OAuthS2SSigningCertThumbprint" - backup-secret "Actions primary encryption cert thumbprint" "actions-primary-encryption-cert-thumbprint" "secrets.actions.PrimaryEncryptionCertificateThumbprint" - backup-secret "Actions AAD cert thumbprint" "actions-aad-cert-thumbprint" "secrets.actions.AADCertThumbprint" --best-effort - backup-secret "Actions delegated auth cert thumbprint" "actions-delegated-auth-cert-thumbprint" "secrets.actions.DelegatedAuthCertThumbprint" --best-effort - backup-secret "Actions runtime service principal cert" "actions-runtime-service-principal-cert" "secrets.actions.RuntimeServicePrincipalCertificate" --best-effort - backup-secret "Actions S2S encryption cert" "actions-s2s-encryption-cert" "secrets.actions.S2SEncryptionCertificate" - backup-secret "Actions secondary encryption cert thumbprint" "actions-secondary-encryption-cert-thumbprint" "secrets.actions.SecondaryEncryptionCertificateThumbprint" - backup-secret "Actions service principal cert" "actions-service-principal-cert" "secrets.actions.ServicePrincipalCertificate" --best-effort - backup-secret "Actions SPS validation cert thumbprint" "actions-sps-validation-cert-thumbprint" "secrets.actions.SpsValidationCertThumbprint" - - backup-secret "Actions Launch secrets encryption/decryption" "actions-launch-secrets-private-key" "secrets.launch.actions-secrets-private-key" - backup-secret "Actions Launch deployer HMAC key" "actions-launch-deployer-hmac" "secrets.launch.deployer-hmac-secret" - backup-secret "Actions Launch Client id" "actions-launch-client-id" "secrets.launch.client-id" - backup-secret "Actions Launch Client secret" "actions-launch-client-secret" "secrets.launch.client-secret" - backup-secret "Actions Launch receiver webhook secret" "actions-launch-receiver-webhook-secret" "secrets.launch.receiver-webhook-secret" - backup-secret "Actions Launch app private key" "actions-launch-app-private-key" "secrets.launch.app-private-key" - backup-secret "Actions Launch app public key" "actions-launch-app-public-key" "secrets.launch.app-public-key" - backup-secret "Actions Launch app id" "actions-launch-app-id" "secrets.launch.app-id" - backup-secret "Actions Launch app relay id" "actions-launch-app-relay-id" "secrets.launch.app-relay-id" - backup-secret "Actions Launch action runner secret" "actions-launch-action-runner-secret" "secrets.launch.action-runner-secret" - backup-secret "Actions Launch service cert" "actions-launch-azp-app-cert" "secrets.launch.azp-app-cert" - backup-secret "Actions Launch service private key" "actions-launch-app-app-private-key" "secrets.launch.azp-app-private-key" -fi - -if ghe-ssh "$host" -- ghe-config --true app.packages.enabled; then - backup-secret "Packages aws access key" "packages-aws-access-key" "secrets.packages.aws-access-key" - backup-secret "Packages aws secret key" "packages-aws-secret-key" "secrets.packages.aws-secret-key" - backup-secret "Packages s3 bucket" "packages-s3-bucket" "secrets.packages.s3-bucket" - backup-secret "Packages storage service url" "packages-service-url" "secrets.packages.service-url" - backup-secret "Packages blob storage type" "packages-blob-storage-type" "secrets.packages.blob-storage-type" - backup-secret "Packages azure connection string" "packages-azure-connection-string" "secrets.packages.azure-connection-string" - backup-secret "Packages azure container name" "packages-azure-container-name" "secrets.packages.azure-container-name" -fi - -if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then - echo "* Transferring SAML keys ..." 1>&3 - ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar -fi - -if ghe-ssh "$host" -- "which ghe-export-ssl-ca-certificates 1>/dev/null"; then - echo "* Transferring CA certificates ..." 1>&3 - ghe-ssh "$host" -- "ghe-export-ssl-ca-certificates" > ssl-ca-certificates.tar -fi - -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - echo "* Transferring cluster configuration ..." 1>&3 - if ! ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_CLUSTER_CONF_FILE 2>/dev/null" > cluster.conf; then - echo "Error: Enterprise Cluster is not configured yet, backup will fail" >&2 - exit 1 - fi -else - if ghe-ssh "$host" -- "sudo cat $GHE_REMOTE_DATA_USER_DIR/common/uuid 2>/dev/null" > uuid; then - echo "* Transferring UUID ..." 1>&3 - fi -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-storage b/share/github-backup-utils/ghe-backup-storage deleted file mode 100755 index bb8117021..000000000 --- a/share/github-backup-utils/ghe-backup-storage +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-storage -#/ Take an online, incremental snapshot of all Alambic Storage data using the -#/ calculated routes method. -#/ -#/ Note: This command typically isn't called directly. It's invoked by -#/ ghe-backup. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -# Set up remote host and root backup snapshot directory based on config -host="$GHE_HOSTNAME" -backup_dir="$GHE_SNAPSHOT_DIR/storage" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Split host:port into parts -port=$(ssh_port_part "$GHE_HOSTNAME") -host=$(ssh_host_part "$GHE_HOSTNAME") - -# Add user / -l option -user="${host%@*}" -[ "$user" = "$host" ] && user="admin" - -hostnames=$host -ssh_config_file_opt= -tempdir=$(mktemp -d -t backup-utils-backup-XXXXXX) -remote_tempdir=$(ghe-ssh "$GHE_HOSTNAME" -- mktemp -d -t backup-utils-backup-XXXXXX) -routes_list=$tempdir/routes_list -remote_routes_list=$remote_tempdir/remote_routes_list -opts="$GHE_EXTRA_SSH_OPTS" - -# storage server hostnames under cluster -if [ "$GHE_BACKUP_STRATEGY" = "cluster" ]; then - ssh_config_file="$tempdir/ssh_config" - ssh_config_file_opt="-F $ssh_config_file" - opts="$opts -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no" - hostnames=$(ghe-cluster-find-nodes "$GHE_HOSTNAME" "storage-server") - ghe-ssh-config "$GHE_HOSTNAME" "$hostnames" > "$ssh_config_file" -fi - -# Make sure root backup dir exists if this is the first run -mkdir -p "$backup_dir" - -# Removes the remote sync-in-progress file on exit, re-enabling GC operations -# on the remote instance. -cleanup() { - # Enable remote maintenance operations - for hostname in $hostnames; do - ghe-gc-enable $ssh_config_file_opt $hostname:$port || { - echo "Re-enable gc on $hostname failed, please manually delete $SYNC_IN_PROGRESS_FILE" 1>&2 - } - done - - ghe-ssh "$GHE_HOSTNAME" -- rm -rf $remote_tempdir - rm -rf $tempdir -} -trap 'cleanup' EXIT INT - -# Disable remote maintenance operations -for hostname in $hostnames; do - ghe-gc-disable $ssh_config_file_opt $hostname:$port -done - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/storage)" ]; then - link_dest="--link-dest=../../current/storage" -fi - -# Calculate sync routes. This will store the healthy object paths for each node -# -# This gets a repo path and stores the path in the $node.sync file -# a/nw/a8/3f/02/100000855 storage-server-node1 >> storage-server-node1.sync -# a/nw/a8/bc/8d/100000880 storage-server-node3 >> storage-server-node3.sync -# a/nw/a5/06/81/100000659 storage-server-node2 >> storage-server-node2.sync -# ... -#one route per line. -# -# NOTE: The route generation is performed on the appliance as it is considerably -# more performant than performing over an SSH pipe. -# -bm_start "$(basename $0) - Generating routes" -echo "github-env ./bin/storage-cluster-backup-routes > $remote_routes_list" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash -ghe-ssh "$GHE_HOSTNAME" -- cat $remote_routes_list | ghe_debug -bm_end "$(basename $0) - Generating routes" - -bm_start "$(basename $0) - Fetching routes" -ghe-ssh "$GHE_HOSTNAME" -- gzip -c $remote_routes_list | gzip -d > $routes_list -cat $routes_list | ghe_debug -bm_end "$(basename $0) - Fetching routes" - -bm_start "$(basename $0) - Processing routes" -if [ "$GHE_BACKUP_STRATEGY" != "cluster" ]; then - server=$host -fi -cat $routes_list | awk -v tempdir="$tempdir" -v server="$server" '{ for(i=2;i<=NF;i++){ server != "" ? host=server : host=$i; print $1 > (tempdir"/"host".rsync") }}' -ghe_debug "\n$(find "$tempdir" -maxdepth 1 -name '*.rsync')" -bm_end "$(basename $0) - Processing routes" - -if [ -z "$(find "$tempdir" -maxdepth 1 -name '*.rsync')" ]; then - echo "Warning: no routes found, skipping storage backup ..." - exit 0 -fi - -# rsync all the storage objects -bm_start "$(basename $0) - Storage object sync" -for file_list in $tempdir/*.rsync; do - hostname=$(basename $file_list .rsync) - storage_user=$(ghe-ssh $ssh_config_file_opt $hostname:$port -- stat -c %U /data/user/storage || echo git) - - object_num=$(cat $file_list | wc -l) - ghe_verbose "* Transferring $object_num objects from $hostname" - - ghe-rsync -avr \ - -e "ssh -q $opts -p $port $ssh_config_file_opt -l $user" \ - $link_dest "$@" \ - --rsync-path="sudo -u $storage_user rsync" \ - --files-from="$file_list" \ - --ignore-missing-args \ - --size-only \ - "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \ - "$backup_dir" 1>&3 & -done - -for pid in $(jobs -p); do - wait $pid -done -bm_end "$(basename $0) - Storage object sync" - -if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then - bm_start "$(basename $0) - Verifying Routes" - - cat $tempdir/*.rsync | uniq | sort | uniq > $tempdir/source_routes - (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | uniq | sort | uniq) > $tempdir/destination_routes - - git --no-pager diff --unified=0 --no-prefix -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance." - - bm_end "$(basename $0) - Verifying Routes" -fi - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-store-version b/share/github-backup-utils/ghe-backup-store-version deleted file mode 100755 index 6faffff99..000000000 --- a/share/github-backup-utils/ghe-backup-store-version +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-store-version -#/ Stores information about the used version of backup-utils on -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0)" - -version_info="$BACKUP_UTILS_VERSION" -if [ -d $GHE_BACKUP_ROOT/.git ]; then - ref=$(git --git-dir=$GHE_BACKUP_ROOT/.git rev-parse HEAD || true) - if [ -n "$ref" ]; then - version_info="$version_info:$ref" - fi -fi - -echo "$version_info" | - ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1" - -bm_end "$(basename $0)" diff --git a/share/github-backup-utils/ghe-backup-strategy b/share/github-backup-utils/ghe-backup-strategy deleted file mode 100755 index b9a919f3c..000000000 --- a/share/github-backup-utils/ghe-backup-strategy +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-strategy -#/ -#/ Determine the backup strategy that will be used. -#/ -#/ The rsync strategy should be used for single VMs and all HA configurations. -#/ -#/ The cluster strategy should be used to backup GHE clusters. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -if ghe-ssh "$GHE_HOSTNAME" -- \ - "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ] && [ ! -f '$GHE_REMOTE_ROOT_DIR/etc/github/repl-state' ]"; then - echo "cluster" -else - echo "rsync" -fi diff --git a/share/github-backup-utils/ghe-backup-userdata b/share/github-backup-utils/ghe-backup-userdata deleted file mode 100755 index 97f71c1a9..000000000 --- a/share/github-backup-utils/ghe-backup-userdata +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-backup-userdata -#/ Take an online, incremental snapshot of a user data directory. This is used -#/ for a number of different simple datastores kept under /data/user on the -#/ remote appliance, including: hookshot, alambic_assets, and pages data. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -bm_start "$(basename $0) - $1" - -# Verify rsync is available. -if ! rsync --version 1>/dev/null 2>&1; then - echo "Error: rsync not found." 1>&2 - exit 1 -fi - -# Grab the host and /data/user directory name. -host="$GHE_HOSTNAME" -dirname="$1" - -# Perform a host-check and establish GHE_REMOTE_XXX variables. -ghe_remote_version_required "$host" - -# Verify that the user data directory exists. Bail out if not, which may be due -# to an older version of GHE or no data has been added to this directory yet. -ghe-ssh "$host" -- "sudo -u git [ -d '$GHE_REMOTE_DATA_USER_DIR/$dirname' ]" || exit 0 - -# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's -# --link-dest support. This also decreases physical space usage considerably. -if [ -d "$GHE_DATA_DIR/current/$dirname" ] && [ "$(ls -A $GHE_DATA_DIR/current/$dirname)" ]; then - - subdir=$dirname - link_path=".." - while true; do - if [ "$(dirname $subdir)" = "." ]; then - break - fi - - if [ "$(dirname $subdir)" = "/" ]; then - break - fi - - link_path="../$link_path" - subdir=$(dirname $subdir) - done - - link_dest="--link-dest=../${link_path}/current/$dirname" -fi - -# Ensure target directory exists, is needed with subdirectories -mkdir -p "$GHE_SNAPSHOT_DIR/$dirname" - -# Transfer all data from the user data directory using rsync. -ghe-rsync -avz \ - -e "ghe-ssh -p $(ssh_port_part "$host")" \ - --rsync-path='sudo -u git rsync' \ - $link_dest \ - "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/$dirname/" \ - "$GHE_SNAPSHOT_DIR/$dirname" 1>&3 - -bm_end "$(basename $0) - $1" diff --git a/share/github-backup-utils/ghe-cluster-find-nodes b/share/github-backup-utils/ghe-cluster-find-nodes deleted file mode 100755 index c2a3cf0e1..000000000 --- a/share/github-backup-utils/ghe-cluster-find-nodes +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-cluster-find-nodes -#/ -#/ Finds all nodes of the cluster using the config on . -#/ If it is a 2.8 and later cluster version the results are returned as -#/ prefix-uuid, otherwise the configured hostnames are returned. -#/ Also filters nodes based on the prefix role. -#/ -#/ Note: This script typically isn't called directly. It's invoked by the -#/ ghe-backup-* and ghe-restore-* commands in cluster environments. -set -e - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -# Check if the REMOTE DATA USER directory is set -if [ -z $GHE_REMOTE_DATA_USER_DIR ]; then - echo "Env variable GHE_REMOTE_DATA_USER_DIR is not set. Exiting" - exit 1 -fi - -# Show usage and bail with no arguments -[ -z "$*" ] && print_usage - -GHE_HOSTNAME="$1" -prefix="$2" -role=$(echo "$prefix" | cut -d '-' -f1) - -if ghe-ssh "$GHE_HOSTNAME" test -f $GHE_REMOTE_ROOT_DIR/etc/github/cluster; then - node_uuids=$(ghe-ssh "$GHE_HOSTNAME" ghe-cluster-nodes -r "$role" -u | cut -f 2) - hostnames='' - for uuid in $node_uuids; do - hostnames+="$prefix-$uuid " - done -else - uuid=$(ghe-ssh "$GHE_HOSTNAME" cat $GHE_REMOTE_DATA_USER_DIR/common/uuid) - hostnames="$prefix-$uuid" -fi - -echo "$hostnames" diff --git a/share/github-backup-utils/ghe-detect-leaked-ssh-keys b/share/github-backup-utils/ghe-detect-leaked-ssh-keys deleted file mode 100755 index 79978ddf0..000000000 --- a/share/github-backup-utils/ghe-detect-leaked-ssh-keys +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-detect-leaked-ssh-key [-s ] -#/ -#/ This utility will check each snapshot's existing SSH host keys against the list -#/ of known leaked SSH host keys from GitHub Enterprise packages. -#/ -#/ OPTIONS: -#/ -h | --help Show this message. -#/ -s |--snapshot Scan the snapshot with the given id. -#/ Available snapshots may be listed under the data directory. -#/ -set -e - -usage() { - grep '^#/' < "$0" | cut -c 4- - exit 2 -} - -TEMPDIR=$(mktemp -d) - -while [ $# -gt 0 ]; do - case "$1" in - -h|--help) - usage - ;; - -s|--snapshot) - snapshot=$2 - shift - ;; - *) - usage - ;; - esac - shift -done - -ppid_script=$(ps -o args= $PPID 2>/dev/null | awk '{print $2}') -if [ -n "$ppid_script" ]; then - ppid_name=$(basename $ppid_script) -fi - -sshkeygen_multiple_hash_formats=false -if (ssh-keygen -E 2>&1 | head -1 | grep -q 'option requires an argument'); then - sshkeygen_multiple_hash_formats=true -fi - -# Bring in the backup configuration -# shellcheck source=share/github-backup-utils/ghe-backup-config -. "$( dirname "${BASH_SOURCE[0]}" )/ghe-backup-config" - -FINGERPRINT_BLACKLIST="${FINGERPRINT_BLACKLIST:-$(cat "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-ssh-leaked-host-keys-list.txt")}" - -keys="ssh_host_dsa_key.pub ssh_host_ecdsa_key.pub ssh_host_ed25519_key.pub ssh_host_rsa_key.pub" - -# Get all the host ssh keys tar from all snapshots directories -if [ -n "$snapshot" ]; then - if [ ! -d "$snapshot" ]; then - echo "Invalid snapshot directory: $snapshot" >&2 - exit 1 - fi - ssh_tars=$(find "$snapshot" -maxdepth 1 -type f -iname 'ssh-host-keys.tar') -else - ssh_tars=$(find "$GHE_DATA_DIR" -maxdepth 2 -type f -iname 'ssh-host-keys.tar') -fi - -# Store the current backup snapshot folder -if [ -L "$GHE_DATA_DIR/current" ]; then - current_dir=$(cd "$GHE_DATA_DIR/current"; pwd -P) -fi - -leaked_keys_found=false -leaked_keys_skippedcheck=false -current_bkup=false -for tar_file in $ssh_tars; do - for key in $keys; do - if tar -tvf "$tar_file" $key &>/dev/null; then - tar -C $TEMPDIR -xvf "$tar_file" $key &>/dev/null - if $sshkeygen_multiple_hash_formats; then - fingerprint=$(ssh-keygen -l -E md5 -f $TEMPDIR/$key | cut -d ' ' -f 2 | cut -f2- -d':') - else - fingerprint=$(ssh-keygen -lf $TEMPDIR/$key | cut -d ' ' -f 2) - fi - if [ -z "$fingerprint" ]; then - leaked_keys_skippedcheck=true - elif echo "$FINGERPRINT_BLACKLIST" | grep -q "$fingerprint"; then - leaked_keys_found=true - if [ "$current_dir" == "$(dirname "$tar_file")" ]; then - current_bkup=true - echo "* Leaked key found in current backup snapshot." - else - echo "* Leaked key found in backup snapshot." - fi - echo "* Snapshot file: $tar_file" - echo "* Key file: $key" - echo "* Key: $fingerprint" - echo - fi - fi - done -done - -if $leaked_keys_found; then - if echo "$ppid_name" | grep -q 'ghe-restore'; then - echo - echo "* The snapshot that is being restored contains a leaked SSH host key." - echo "* We recommend rolling the SSH host keys after completing the restore." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - echo - elif echo "$ppid_name" | grep -q 'ghe-backup'; then - echo "* The current backup contains leaked SSH host keys." - echo "* We strongly recommend rolling your SSH host keys and making a new backup." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - else - if $current_bkup; then - echo "* The current backup contains leaked SSH host keys." - echo "* Current backup directory: $current_dir" - echo "* We strongly recommend rolling your SSH host keys and making a new backup." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - fi - echo - echo "* One or more older backup snapshots contain leaked SSH host keys." - echo "* No immediate action is needed but when you use one of these older snapshots for a restore, " - echo "* please make sure to roll the SSH host keys after restore." - echo "* Roll the keys either manually or with ghe-ssh-roll-host-keys on the appliance." - echo "* (An upgrade may be required)" - echo - fi -else - if $leaked_keys_skippedcheck; then - echo "* No result - check not performed since host key fingerprint was empty" - else - echo "* No leaked keys found" - fi -fi - -# Cleanup temp dir -rm -rf $TEMPDIR diff --git a/share/github-backup-utils/ghe-docker-init b/share/github-backup-utils/ghe-docker-init deleted file mode 100755 index 76e6086b8..000000000 --- a/share/github-backup-utils/ghe-docker-init +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -e - -PATH=$PATH:/backup-utils/bin - -mkdir -p /etc/github-backup-utils - -touch /etc/github-backup-utils/backup.config - -env | grep ^GHE_ | sed -r "s/(.[^=]+)=(.*)/\1=\"\2\"/g" >> /etc/github-backup-utils/backup.config - -exec "$@" diff --git a/share/github-backup-utils/ghe-gc-disable b/share/github-backup-utils/ghe-gc-disable deleted file mode 100755 index bf37572ca..000000000 --- a/share/github-backup-utils/ghe-gc-disable +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -#/ Usage: ghe-gc-disable [