diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..b48538b --- /dev/null +++ b/.mailmap @@ -0,0 +1,2 @@ +Jack Yuan <55564584+yongleyuan@users.noreply.github.com> +Tobias Pietzsch diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/check-branch.sh b/check-branch.sh index 8496e55..fdbfc4c 100755 --- a/check-branch.sh +++ b/check-branch.sh @@ -4,7 +4,13 @@ # recording whether the build passes or fails for each. commits=$@ -test "$commits" || commits=$(git rev-list HEAD ^master | sed '1!G;h;$!d') + +remote=$(git rev-parse --symbolic-full-name HEAD@{u}) +remote=${remote#refs/remotes/} +remote=${remote%%/*} +headBranch=$(git remote show "$remote" | grep HEAD | sed 's/ *HEAD branch: //') + +test "$commits" || commits=$(git rev-list HEAD "^$remote/$headBranch" | sed '1!G;h;$!d') # NB: The sed line above reverses the order of the commits. # See: http://stackoverflow.com/a/744093 @@ -18,7 +24,15 @@ do prefix="$(printf %04d $count)" filename="tmp/$prefix-$commit" start=$(date +%s) - mvn clean verify > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + if [ -f Makefile ] + then + make test > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + elif [ -f pom.xml ] + then + mvn clean verify > "$filename" 2>&1 && result=SUCCESS || result=FAILURE + else + result=SKIPPED + fi end=$(date +%s) time=$(expr "$end" - "$start") echo "$prefix $commit $result $time" diff --git a/ci-build.sh b/ci-build.sh new file mode 100755 index 0000000..d9ab7bf --- /dev/null +++ b/ci-build.sh @@ -0,0 +1,363 @@ +#!/bin/bash + +# +# ci-build.sh - A script to build and/or release SciJava-based projects +# automatically using a continuous integration service. +# +# Optional environment variables: +# BUILD_REPOSITORY - the repository URL running the current build + +dir="$(dirname "$0")" + +platform=$(uname -s) + +success=0 +checkSuccess() { + # Log non-zero exit code. + test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 + + if [ $1 -ne 0 -a -f "$2" ] + then + # The operation failed and a log file was provided. + # Do some heuristics, because we like being helpful! + javadocErrors=$(grep error: "$2") + generalErrors=$(grep -i '\b\(errors\?\|fail\|failures\?\)\b' "$2") + if [ "$javadocErrors" ] + then + echo + echo '/----------------------------------------------------------\' + echo '| ci-build.sh analysis: I noticed probable javadoc errors: |' + echo '\----------------------------------------------------------/' + echo "$javadocErrors" + elif [ "$generalErrors" ] + then + echo + echo '/-------------------------------------------------------\' + echo '| ci-build.sh analysis: I noticed the following errors: |' + echo '\-------------------------------------------------------/' + echo "$generalErrors" + else + echo + echo '/----------------------------------------------------------------------\' + echo '| ci-build.sh analysis: I see no problems in the operation log. Sorry! |' + echo '\----------------------------------------------------------------------/' + echo + fi + fi + + # Record the first non-zero exit code. + test $success -eq 0 && success=$1 +} + +# Credit: https://stackoverflow.com/a/12873723/1207769 +escapeXML() { + echo "$1" | sed 's/&/\&/g; s//\>/g; s/"/\"/g; s/'"'"'/\'/g' +} + +mavenEvaluate() { + mvn -B -U -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args="$1" --non-recursive validate exec:exec 2>&1 +} + +# Build Maven projects. +if [ -f pom.xml ]; then + echo ::group::"= Maven build =" + + # --== MAVEN SETUP ==-- + + echo + echo "== Configuring Maven ==" + + # NB: Suppress "Downloading/Downloaded" messages. + # See: https://stackoverflow.com/a/35653426/1207769 + export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" + + # Populate the settings.xml configuration. + mkdir -p "$HOME/.m2" + settingsFile="$HOME/.m2/settings.xml" + customSettings=.ci/settings.xml + if [ "$OSSRH_USER" -o "$OSSRH_PASS" ]; then + echo "[WARNING] Obsolete OSSRH vars detected. Secrets may need updating to deploy to Maven Central." + fi + if [ -f "$customSettings" ]; then + cp "$customSettings" "$settingsFile" + elif [ -z "$BUILD_REPOSITORY" ]; then + echo "Skipping settings.xml generation (no BUILD_REPOSITORY; assuming we are running locally)" + else + # settings.xml header + cat >"$settingsFile" < + +EOL + # settings.xml scijava servers + if [ "$MAVEN_USER" -a "$MAVEN_PASS" ]; then + cat >>"$settingsFile" < + scijava.releases + $MAVEN_USER + $(escapeXML "$MAVEN_PASS") + + + scijava.snapshots + $MAVEN_USER + $(escapeXML "$MAVEN_PASS") + +EOL + else + echo "[WARNING] Skipping settings.xml scijava servers (no MAVEN deployment credentials)." + fi + # settings.xml central server + if [ "$CENTRAL_USER" -a "$CENTRAL_PASS" ]; then + cat >>"$settingsFile" < + central + $CENTRAL_USER + $(escapeXML "$CENTRAL_PASS") + +EOL + else + echo "[WARNING] Skipping settings.xml central server (no CENTRAL deployment credentials)." + fi + cat >>"$settingsFile" < +EOL + # settings.xml GPG profile + if [ "$GPG_KEY_NAME" -a "$GPG_PASSPHRASE" ]; then + cat >>"$settingsFile" < + + gpg + + + $HOME/.gnupg + + + + $GPG_KEY_NAME + $(escapeXML "$GPG_PASSPHRASE") + + + +EOL + else + echo "[WARNING] Skipping settings.xml gpg profile (no GPG credentials)." + fi + # settings.xml footer + cat >>"$settingsFile" < +EOL + fi + + # --== DEPLOYMENT CHECKS ==-- + + # Determine whether deploying is both possible and warranted. + echo "Performing deployment checks" + deployOK= + + scmURL=$(mavenEvaluate '${project.scm.url}') + result=$? + checkSuccess $result + if [ $result -ne 0 ]; then + echo "No deploy -- could not extract ciManagement URL" + echo "Output of failed attempt follows:" + echo "$scmURL" + else + scmURL=${scmURL%.git} + scmURL=${scmURL%/} + if [ "$NO_DEPLOY" ]; then + echo "No deploy -- the NO_DEPLOY flag is set" + elif [ "$BUILD_REPOSITORY" -a "$BUILD_REPOSITORY" != "$scmURL" ]; then + echo "No deploy -- repository fork: $BUILD_REPOSITORY != $scmURL" + elif [ "$BUILD_BASE_REF" -o "$BUILD_HEAD_REF" ]; then + echo "No deploy -- proposed change: $BUILD_HEAD_REF -> $BUILD_BASE_REF" + else + # Are we building a snapshot version, or a release version? + version=$(mavenEvaluate '${project.version}') + result=$? + checkSuccess $result + if [ $result -ne 0 ]; then + echo "No deploy -- could not extract version string" + echo "Output of failed attempt follows:" + echo "$version" + else + case "$version" in + *-SNAPSHOT) + # Snapshot version -- ensure release.properties not present. + if [ -f release.properties ]; then + echo "[ERROR] Spurious release.properties file is present" + echo "Remove the file from version control and try again." + exit 1 + fi + + # Check for SciJava Maven repository credentials. + if [ "$MAVEN_USER" -a "$MAVEN_PASS" ]; then + deployOK=1 + else + echo "No deploy -- MAVEN environment variables not available" + fi + ;; + *) + # Release version -- ensure release.properties is present. + if [ ! -f release.properties ]; then + echo "[ERROR] Release version, but release.properties not found" + echo "You must use release-version.sh to release -- see https://imagej.net/develop/releasing" + exit 1 + fi + + # To which repository are we releasing? + releaseProfiles=$(mavenEvaluate '${releaseProfiles}') + result=$? + checkSuccess $result + if [ $result -ne 0 ]; then + echo "No deploy -- could not extract releaseProfiles string" + echo "Output of failed attempt follows:" + echo "$releaseProfiles" + fi + case "$releaseProfiles" in + *deploy-to-scijava*) + # Check for SciJava Maven repository credentials. + if [ "$MAVEN_USER" -a "$MAVEN_PASS" ]; then + deployOK=1 + else + echo "[ERROR] Cannot deploy: MAVEN environment variables not available" + exit 1 + fi + ;; + *sonatype-oss-release*) + # Check for Central Portal deployment credentials. + # Deploy to Central requires GPG-signed artifacts. + if [ "$CENTRAL_USER" -a "$CENTRAL_PASS" -a "$SIGNING_ASC" -a "$GPG_KEY_NAME" -a "$GPG_PASSPHRASE" ]; then + deployOK=1 + else + echo "[ERROR] Cannot deploy: CENTRAL environment variables not available" + exit 1 + fi + ;; + *) + echo "Unknown deploy target -- attempting to deploy anyway" + deployOK=1 + ;; + esac + ;; + esac + fi + fi + fi + if [ "$deployOK" ]; then + echo "All checks passed for artifact deployment" + fi + + # --== Maven build arguments ==-- + + BUILD_ARGS="$BUILD_ARGS -B -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2" + + # --== GPG SETUP ==-- + + if [ "$GPG_KEY_NAME" -a "$GPG_PASSPHRASE" ]; then + # Install GPG on macOS + if [ "$platform" = Darwin ]; then + HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 + fi + + # Avoid "signing failed: Inappropriate ioctl for device" error. + export GPG_TTY=$(tty) + + # Import the GPG signing key. + keyFile=.ci/signingkey.asc + if [ "$deployOK" ]; then + echo "== Importing GPG keypair ==" + mkdir -p .ci + echo "$SIGNING_ASC" > "$keyFile" + ls -la "$keyFile" + gpg --version + gpg --batch --fast-import "$keyFile" + checkSuccess $? + fi + + # HACK: Use maven-gpg-plugin 3.0.1+. Avoids "signing failed: No such file or directory" error. + maven_gpg_plugin_version=$(mavenEvaluate '${maven-gpg-plugin.version}') + case "$maven_gpg_plugin_version" in + 0.*|1.*|2.*|3.0.0) + echo "--> Forcing maven-gpg-plugin version from $maven_gpg_plugin_version to 3.0.1" + BUILD_ARGS="$BUILD_ARGS -Dmaven-gpg-plugin.version=3.0.1 -Darguments=-Dmaven-gpg-plugin.version=3.0.1" + ;; + *) + echo "--> maven-gpg-plugin version OK: $maven_gpg_plugin_version" + ;; + esac + + # HACK: Install pinentry helper program if missing. Avoids "signing failed: No pinentry" error. + if ! which pinentry >/dev/null 2>&1; then + echo '--> Installing missing pinentry helper for GPG' + sudo apt-get install -y pinentry-tty + # HACK: Restart the gpg agent, to notice the newly installed pinentry. + if { pgrep gpg-agent >/dev/null && which gpgconf >/dev/null 2>&1; } then + echo '--> Restarting gpg-agent' + gpgconf --reload gpg-agent + checkSuccess $? + fi + fi + else + echo "[WARNING] Skipping gpg setup (no GPG credentials)." + fi + + # --== BUILD EXECUTION ==-- + + # Run the build. + if [ "$deployOK" -a -f release.properties ]; then + echo + echo "== Cutting and deploying release version ==" + BUILD_ARGS="$BUILD_ARGS release:perform" + elif [ "$deployOK" ]; then + echo + echo "== Building and deploying main branch SNAPSHOT ==" + BUILD_ARGS="-Pdeploy-to-scijava $BUILD_ARGS deploy" + else + echo + echo "== Building the artifact locally only ==" + BUILD_ARGS="$BUILD_ARGS install javadoc:javadoc" + fi + # Check the build result. + { (set -x; mvn $BUILD_ARGS); echo $? > exit-code; } | tee mvn-log + checkSuccess "$(cat exit-code)" mvn-log + + # --== POST-BUILD ACTIONS ==-- + + # Dump logs for any failing unit tests. + if [ -d target/surefire-reports ] + then + find target/surefire-reports -name '*.txt' | while read report + do + if grep -qF 'FAILURE!' "$report" + then + echo + echo "[$report]" + cat "$report" + fi + done + fi + + echo ::endgroup:: +fi + +# Execute Jupyter notebooks. +if which jupyter >/dev/null 2>&1; then + echo ::group::"= Jupyter notebooks =" + # NB: This part is fiddly. We want to loop over files even with spaces, + # so we use the "find ... | while read ..." idiom. + # However, that runs the piped expression in a subshell, which means + # that any updates to the success variable will not persist outside + # the loop. So we store non-zero success values into a temporary file, + # then capture the value back into the parent shell's success variable. + find . -name '*.ipynb' | while read nbf + do + echo + echo "== $nbf ==" + jupyter nbconvert --to python --stdout --execute "$nbf" + checkSuccess $? + test "$success" -eq 0 || echo "$success" > success.tmp + done + test -f success.tmp && success=$(cat success.tmp) && rm success.tmp + echo ::endgroup:: +fi + +exit $success diff --git a/ci-setup-github-actions.sh b/ci-setup-github-actions.sh new file mode 100755 index 0000000..a0e7a66 --- /dev/null +++ b/ci-setup-github-actions.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# +# ci-setup-github-actions.sh - Set CI-related environment variables from GitHub Actions. +# + +echo "BUILD_REPOSITORY=https://github.com/$GITHUB_REPOSITORY" +echo "BUILD_REPOSITORY=https://github.com/$GITHUB_REPOSITORY" >> $GITHUB_ENV + +echo "BUILD_OS=$RUNNER_OS" +echo "BUILD_OS=$RUNNER_OS" >> $GITHUB_ENV + +echo "BUILD_BASE_REF=$GITHUB_BASE_REF" +echo "BUILD_BASE_REF=$GITHUB_BASE_REF" >> $GITHUB_ENV + +echo "BUILD_HEAD_REF=$GITHUB_HEAD_REF" +echo "BUILD_HEAD_REF=$GITHUB_HEAD_REF" >> $GITHUB_ENV diff --git a/class-version.sh b/class-version.sh index 54251d5..ed77a66 100755 --- a/class-version.sh +++ b/class-version.sh @@ -2,19 +2,9 @@ # class-version.sh - find the Java version which wrote a JAR file -for jar in "$@" -do - # find the first class of the JAR - class="$(jar tf "$jar" | grep '\.class' | head -n 1 | sed 's/\//./g' | sed 's/\.class$//')" - - if [ -z "$class" ] - then - echo "$jar: No classes" - continue - fi - +class_version() { # extract bytes 4-7 - info="$(unzip -p "$jar" "$(jar tf "$jar" | grep \.class$ | head -n 1)" | head -c 8 | hexdump -s 4 -e '4/1 "%d\n" "\n"')" + info=$(head -c 8 | hexdump -e '4/1 "%d\n" "\n"' | tail -n4) minor1="$(echo "$info" | sed -n 1p)" minor2="$(echo "$info" | sed -n 2p)" major1="$(echo "$info" | sed -n 3p)" @@ -27,27 +17,24 @@ do # derive Java version case $major in 45) - version="JDK 1.1" + version="Java 1.0/1.1" ;; 46) - version="JDK 1.2" + version="Java 1.2" ;; 47) - version="JDK 1.3" + version="Java 1.3" ;; 48) - version="JDK 1.4" + version="Java 1.4" ;; 49) - version="J2SE 5.0" - ;; - 50) - version="J2SE 6.0" + version="Java 5" ;; *) - if [ "$major" -gt 50 ] + if [ "$major" -gt 49 ] then - version="J2SE $(expr $major - 44)" + version="Java $(expr $major - 44)" else version="Unknown" fi @@ -55,5 +42,61 @@ do esac # report the results - echo "$jar: $version ($major.$minor)" + echo "$version ($major.$minor)" +} + +first_class() { + jar tf "$1" | + grep '\.class$' | + grep -v '^META-INF/' | + grep -v 'module-info\.class' | + head -n 1 +} + +for arg in "$@" +do + # Resolve Maven dependency coordinates into local files. + case "$arg" in + *:*:*:*) # g:a:v:c + gav=${arg%:*} + g=${gav%%:*} + av=${gav#*:} + a=${av%:*} + v=${av#*:} + c=${arg##*:} + f="$HOME/.m2/repository/$(echo "$g" | tr '.' '/')/$a/$v/$a-$v-$c.jar" + test -f "$f" || mvn dependency:get -Dartifact="$g:$a:$v:jar:$c" + arg="$f" + ;; + *:*:*) # g:a:v + ga=${arg%:*} + g=${ga%%:*} + a=${ga#*:} + v=${arg##*:} + f="$HOME/.m2/repository/$(echo "$g" | tr '.' '/')/$a/$v/$a-$v.jar" + test -f "$f" || mvn dependency:get -Dartifact="$arg" + arg="$f" + ;; + esac + # Handle the various local file cases. + case "$arg" in + *.class) + version=$(cat "$arg" | class_version) + ;; + *.jar) + class=$(first_class "$arg") + if [ -z "$class" ] + then + echo "$arg: No classes" + continue + fi + version=$(unzip -p "$arg" "$class" | class_version) + ;; + *) + >&2 echo "Unsupported argument: $arg" + continue + esac + + # Report the results. + echo "$arg: $version" done diff --git a/dep-versions.pl b/dep-versions.pl index 3698568..96bf595 100755 --- a/dep-versions.pl +++ b/dep-versions.pl @@ -5,7 +5,7 @@ use strict; -my @deps = `mvn dependency:list`; +my @deps = `mvn -B dependency:list`; my $active = 0; for my $dep (@deps) { diff --git a/disk-usage.sh b/disk-usage.sh deleted file mode 100755 index bbc5180..0000000 --- a/disk-usage.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh - -# disk-usage.sh - report when disk space starts getting low - -# Credit to: -# http://www.cyberciti.biz/faq/mac-osx-unix-get-an-alert-when-my-disk-is-full/ - -check() { - FS=$1 - CURRENT=$(df -P "$FS" | tail -n1 | awk '{print $5}' | sed 's/%//') - test "$CURRENT" -gt "$threshold" && - echo "$FS: file system usage at $CURRENT%" && - return 1 - return 0 -} - -threshold=80 -code=0 -while test $# -gt 0 -do - case "$1" in - --threshold|-t) - shift - threshold="$1" - ;; - *) - check "$1" || code=$((code+1)) - ;; - esac - shift -done - -exit $code diff --git a/find-duplicate-classes.py b/find-duplicate-classes.py new file mode 100755 index 0000000..6050d40 --- /dev/null +++ b/find-duplicate-classes.py @@ -0,0 +1,47 @@ +#!/bin/env python + +import os, subprocess + +def extract_classes(f): + lines = subprocess.check_output(['jar', 'tf', f]).split() + result = set() + for line in lines: + v = line.decode('utf-8').strip() + if v.endswith('.class') and not v.endswith('module-info.class'): + result.add(v) + return result + +print('Reading JAR files... ', end='', flush=True) +paths = [] +for root, dirs, files in os.walk('.'): + for name in files: + if not name.lower().endswith('.jar'): continue + paths.append(os.path.join(root, name)) +paths.sort() + +classes = {} +count = 0 +perc = '' +for path in paths: + count += 1 + classes[path] = extract_classes(path) + print('\b' * len(perc), end='') + perc = str(round(100 * count / len(paths))) + '% ' + print(perc, end='', flush=True) + +print() +print('Scanning for duplicate classes...') +for i1 in range(len(paths)): + p1 = paths[i1] + duplist = [] + for i2 in range(i1 + 1, len(paths)): + p2 = paths[i2] + dups = classes[p1].intersection(classes[p2]) + if len(dups) > 0: + duplist.append(f'==> {p2} (e.g. {next(iter(dups))})') + if len(duplist) > 0: + print(p1) + for line in duplist: + print(line) + +print('Done!') diff --git a/github-actionify.sh b/github-actionify.sh new file mode 100755 index 0000000..a88d24f --- /dev/null +++ b/github-actionify.sh @@ -0,0 +1,293 @@ +#!/bin/sh + +# github-actionify.sh +# +# Script for enabling or updating GitHub Action builds for a given repository. + +#set -e + +dir="$(dirname "$0")" + +ciDir=.github +ciSlugBuild=workflows/build.yml +ciConfigBuild=$ciDir/$ciSlugBuild +ciSetupScript=$ciDir/setup.sh +ciBuildScript=$ciDir/build.sh +pomMinVersion='17.1.1' +tmpFile=github-actionify.tmp +msgPrefix="CI: " + +info() { echo "- $@"; } +warn() { echo "[WARNING] $@" 1>&2; } +err() { echo "[ERROR] $@" 1>&2; } +die() { err "$@"; exit 1; } + +check() { + for tool in $@ + do + which "$tool" >/dev/null || + die "The '$tool' utility is required but not found" + done +} + +update() { + file=$1 + msg=$2 + exe=$3 + test "$msg" || msg="update $file" + if [ -e "$file" ] + then + if diff -q "$file" "$tmpFile" >/dev/null + then + info "$file is already OK" + else + info "Updating $file" + $EXEC rm -rf "$file" + $EXEC mv -f "$tmpFile" "$file" + fi + else + info "Creating $file" + $EXEC mkdir -p "$(dirname "$file")" + $EXEC mv "$tmpFile" "$file" + fi + rm -rf "$tmpFile" + $EXEC git add "$file" + if [ -n "$exe" ] + then + info "Adding execute permission to $file" + $EXEC git update-index --chmod=+x "$file" + fi + $EXEC git diff-index --quiet HEAD -- || $EXEC git commit -m "$msgPrefix$msg" +} + +process() { + cd "$1" + + # -- Git sanity checks -- + + repoSlug=$(grep '' pom.xml | sed 's;.*github.com[/:]\(.*/.*\).*;\1;') + test "$repoSlug" && info "Repository = $repoSlug" || die 'Could not determine GitHub repository slug' + case "$repoSlug" in + *.git) + die "GitHub repository slug ('$repoSlug') ends in '.git'; please fix the POM" + ;; + esac + git fetch >/dev/null + git diff-index --quiet HEAD -- || die "Dirty working copy" + currentBranch=$(git rev-parse --abbrev-ref HEAD) + upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) + remote=${upstreamBranch%/*} + defaultBranch=$(git remote show "$remote" | grep "HEAD" | sed 's/.*: //') + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch" + git merge --ff --ff-only 'HEAD@{u}' >/dev/null || + die "Cannot fast forward (local diverging?)" +# test "$(git rev-parse HEAD)" = "$(git rev-parse 'HEAD@{u}')" || +# die "Mismatch with upstream branch (local ahead?)" + + # -- POM sanity checks -- + + parent=$(grep -A4 '' pom.xml | grep '' | sed 's;.*\(.*\).*;\1;') + if [ -z "$SKIP_PARENT_CHECK" ] + then + test "$parent" = "pom-scijava" || + die "Not pom-scijava parent: $parent. Run with -p flag to skip this check." + fi + + # Change pom.xml from Travis CI to GitHub Actions + domain="github.com" + sed 's/Travis CI/GitHub Actions/g' pom.xml | + sed "s;travis-ci.*;github.com/$repoSlug/actions;g" >"$tmpFile" + update pom.xml "switch from Travis CI to GitHub Actions" + + # -- GitHub Action sanity checks -- + + test -e "$ciDir" -a ! -d "$ciDir" && die "$ciDir is not a directory" + test -e "$ciConfigBuild" -a ! -f "$ciConfigBuild" && die "$ciConfigBuild is not a regular file" + test -e "$ciConfigBuild" && warn "$ciConfigBuild already exists" + test -e "$ciBuildScript" && warn "$ciBuildScript already exists" + test -e "$ciSetupScript" && warn "$ciSetupScript already exists" + + # -- GitHub Action steps -- + + actionCheckout="uses: actions/checkout@v2" + actionSetupJava="name: Set up Java + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'zulu' + cache: 'maven'" + actionSetupConda="name: Set up conda + uses: s-weigand/setup-conda@v1 + - name: Install conda packages + run: conda env update -f environment.yml -n base" + actionSetupCI="name: Set up CI environment + run: $ciSetupScript" + actionExecuteBuild="name: Execute the build + run: $ciBuildScript" + actionSecrets="env: + GPG_KEY_NAME: \${{ secrets.GPG_KEY_NAME }} + GPG_PASSPHRASE: \${{ secrets.GPG_PASSPHRASE }} + MAVEN_USER: \${{ secrets.MAVEN_USER }} + MAVEN_PASS: \${{ secrets.MAVEN_PASS }} + CENTRAL_USER: \${{ secrets.CENTRAL_USER }} + CENTRAL_PASS: \${{ secrets.CENTRAL_PASS }} + SIGNING_ASC: \${{ secrets.SIGNING_ASC }}" + + # -- Do things -- + + # Add/update the GitHub Actions build configuration file. + cat >"$tmpFile" <>"$tmpFile" + cat >>"$tmpFile" <"$tmpFile" <"$tmpFile" <' pom.xml | grep '' | sed 's;.*\(.*\).*;\1;') + # HACK: Using a lexicographic comparison here is imperfect. + if [ "$version" \< "$pomMinVersion" ] + then + info 'Upgrading pom-scijava version' + sed "s|^ $version$| $pomMinVersion|" pom.xml >"$tmpFile" + update pom.xml "update pom-scijava parent to $pomMinVersion" + else + info "Version of pom-scijava ($version) is OK" + fi + fi + + # ensure section is present + releaseProfile=$(grep '' pom.xml 2>/dev/null | sed 's/[^>]*>//' | sed 's/<.*//') + if [ "$releaseProfile" ] + then + case "$releaseProfile" in + sign,deploy-to-scijava) + info 'No changes needed to property' + ;; + deploy-to-scijava) + info 'Updating property' + sed 's;\(\).*\(\);\1sign,deploy-to-scijava\2;' pom.xml >"$tmpFile" + update pom.xml 'sign JARs when deploying releases' + ;; + *) + warn "Unknown release profile: $releaseProfile" + ;; + esac + else + info 'Adding property' + cp pom.xml "$tmpFile" + perl -0777 -i -pe 's/(\n\t<\/properties>\n)/\n\n\t\t\n\t\tsign,deploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" + update pom.xml 'deploy releases to the SciJava repository' + fi + + # update the README + # https://docs.github.com/en/actions/managing-workflow-runs/adding-a-workflow-status-badge + if grep -q "travis-ci.*svg" README.md >/dev/null 2>&1 + then + info "Updating README.md GitHub Action badge" + sed "s;travis-ci.*;$domain/$repoSlug/actions/$ciSlugBuild/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuild);g" README.md >"$tmpFile" + update README.md 'update README.md badge link' + elif grep -qF "$domain/$repoSlug/actions/$ciSlugBuild/badge.svg" README.md >/dev/null 2>&1 + then + info "GitHub Action badge already present in README.md" + else + info "Adding GitHub Action badge to README.md" + echo "[![Build Status](https://$domain/$repoSlug/actions/$ciSlugBuild/badge.svg)](https://$domain/$repoSlug/actions/$ciSlugBuild)" >"$tmpFile" + echo >>"$tmpFile" + test -f README.md && cat README.md >>"$tmpFile" + update README.md 'add README.md badge link' + fi + + # remove old Travis CI configuration + test ! -e .travis.yml || $EXEC git rm -rf .travis.yml + test ! -e .travis || $EXEC git rm -rf .travis + $EXEC git diff-index --quiet HEAD -- && + info "No old CI configuration to remove." || + $EXEC git commit -m "${msgPrefix}remove Travis CI configuration" +} + +cat <&2; break;; + *) break;; + esac + shift +done + +test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." + +# process arguments +if [ $# -gt 0 ] +then + for d in $@ + do ( + echo "[$d]" + process "$d" + ) done +else + process . +fi diff --git a/maven-helper.sh b/maven-helper.sh deleted file mode 100755 index 54766c2..0000000 --- a/maven-helper.sh +++ /dev/null @@ -1,567 +0,0 @@ -#!/bin/sh - -# This script uses the SciJava Maven repository at https://maven.scijava.org/ -# to fetch an artifact, or to determine the state of it. - -# error out whenever a command fails -set -e - -root_url () { - test snapshots != "$2" || { - if curl -fs https://maven.scijava.org/service/local/repositories/sonatype-snapshots/content/"$1"/maven-metadata.xml > /dev/null 2>&1 - then - echo https://maven.scijava.org/service/local/repositories/sonatype-snapshots/content - else - echo https://maven.scijava.org/content/repositories/snapshots - fi - return - } - echo https://maven.scijava.org/service/local/repo_groups/public/content -} - -die () { - echo "$*" >&2 - exit 1 -} - -# Helper (thanks, BSD!) - -get_mtime () { - stat -c %Y "$1" -} -case "$(uname -s 2> /dev/null)" in -MINGW*) - get_mtime () { - date -r "$1" +%s - } - ;; -Darwin) - get_mtime () { - stat -f %m "$1" - } - ;; -esac - -# Parse :: triplets (i.e. GAV parameters) - -groupId () { - echo "${1%%:*}" -} - -artifactId () { - result="${1#*:}" - echo "${result%%:*}" -} - -version () { - result="${1#*:}" - case "$result" in - *:*) - echo "${1##*:}" - ;; - esac -} - -# Given an xml, extract the first - -extract_tag () { - result="${2%%*}" - case "$result" in - "$2") - ;; - *) - echo "${result#*<$1>}" - ;; - esac -} - -# Given an xml, extract the last - -extract_last_tag () { - result="${2##*<$1>}" - case "$result" in - "$2") - ;; - *) - echo "${result%%*}" - ;; - esac -} - -# Given an xml, skip all sections - -skip_tag () { - result="$2" - while true - do - case "$result" in - *"<$1>"*) - result="${result%%<$1>*}${result#*}" - ;; - *) - break - ;; - esac - done - echo "$result" -} - -# Given the xml of a POM, find the parent GAV - -parent_gav_from_pom_xml () { - pom="$1" - parent="$(extract_tag parent "$pom")" - test -n "$parent" || return - groupId="$(extract_tag groupId "$parent")" - artifactId="$(extract_tag artifactId "$parent")" - version="$(extract_tag version "$parent")" - echo "$groupId:$artifactId:$version" -} - -# Given a GAV parameter, determine the base URL of the project - -project_url () { - gav="$1" - artifactId="$(artifactId "$gav")" - infix="$(groupId "$gav" | tr . /)/$artifactId" - version="$(version "$gav")" - case "$version" in - *SNAPSHOT) - echo "$(root_url $infix snapshots)/$infix" - ;; - *) - # Release could be in either releases or thirdparty; try releases first - project_url="$(root_url $infix releases)/$infix" - header=$(curl -Is "$project_url/") - case "$header" in - HTTP/1.?" 200 OK"*) - ;; - *) - project_url="$(root_url $infix thirdparty)/$infix" - ;; - esac - echo "$project_url" - ;; - esac -} - -# Given a GAV parameter, determine the URL of the .jar file - -jar_url () { - gav="$1" - artifactId="$(artifactId "$gav")" - version="$(version "$gav")" - infix="$(groupId "$gav" | tr . /)/$artifactId/$version" - case "$version" in - *-SNAPSHOT) - url="$(root_url $infix snapshots)/$infix/maven-metadata.xml" - metadata="$(curl -s "$url")" - timestamp="$(extract_tag timestamp "$metadata")" - buildNumber="$(extract_tag buildNumber "$metadata")" - version=${version%-SNAPSHOT}-$timestamp-$buildNumber - echo "$(root_url $infix snapshots)/$infix/$artifactId-$version.jar" - ;; - *) - echo "$(root_url $infix releases)/$infix/$artifactId-$version.jar" - ;; - esac -} - -# Given a GAV parameter, return the URL to the corresponding .pom file - -pom_url () { - url="$(jar_url "$1")" - echo "${url%.jar}.pom" -} - -# Given a POM file, find its GAV parameter - -gav_from_pom () { - pom="$(cat "$1")" - parent="$(extract_tag parent "$pom")" - pom="$(skip_tag parent "$pom")" - pom="$(skip_tag dependencies "$pom")" - pom="$(skip_tag profiles "$pom")" - pom="$(skip_tag build "$pom")" - groupId="$(extract_tag groupId "$pom")" - test -n "$groupId" || groupId="$(extract_tag groupId "$parent")" - artifactId="$(extract_tag artifactId "$pom")" - version="$(extract_tag version "$pom")" - test -n "$version" || version="$(extract_tag version "$parent")" - echo "$groupId:$artifactId:$version" -} - -# Given a GAV parameter, find its parent's GAV - -parent_gav () { - gav="$1" - groupId="$(groupId "$gav")" - artifactId="$(artifactId "$gav")" - version="$(version "$gav")" - test -n "$version" || version="$(latest_version "$gav")" - pom="$(read_pom "$groupId:$artifactId:$version")" - parent_gav_from_pom_xml "$pom" -} - -# Given a POM file, find its parent's GAV - -parent_gav_from_pom () { - pom="$(cat "$1")" - parent_gav_from_pom_xml "$pom" -} - -# Given a POM file, extract its packaging - -packaging_from_pom () { - pom="$(cat "$1")" - pom="$(skip_tag parent "$pom")" - pom="$(skip_tag dependencies "$pom")" - pom="$(skip_tag profiles "$pom")" - pom="$(skip_tag build "$pom")" - packaging="$(extract_tag packaging "$pom")" - echo "${packaging:-jar}" -} - -# Given a GAV parameter possibly lacking a version, determine the latest version - -latest_version () { - metadata="$(curl -s "$(project_url "$1")"/maven-metadata.xml)" - latest="$(extract_tag release "$metadata")" - test -n "$latest" || latest="$(extract_tag latest "$metadata")" - test -n "$latest" || latest="$(extract_last_tag version "$metadata")" - echo "$latest" -} - -# Given a GA parameter, invalidate the cache in SciJava's Nexus' group/public - -SONATYPE_DATA_CACHE_URL=https://maven.scijava.org/service/local/data_cache/repositories/sonatype/content -SONATYPE_SNAPSHOTS_DATA_CACHE_URL=https://maven.scijava.org/service/local/data_cache/repositories/sonatype-snapshots/content -invalidate_cache () { - ga="$1" - artifactId="$(artifactId "$ga")" - infix="$(groupId "$ga" | tr . /)/$artifactId" - curl --netrc -i -X DELETE \ - $SONATYPE_DATA_CACHE_URL/$infix/maven-metadata.xml && - curl --netrc -i -X DELETE \ - $SONATYPE_SNAPSHOTS_DATA_CACHE_URL/$infix/maven-metadata.xml && - version="$(latest_version "$ga")" && - infix="$infix/$version" && - curl --netrc -i -X DELETE \ - $SONATYPE_DATA_CACHE_URL/$infix/$artifactId-$version.pom && - if test "$artifactId" = "${artifactId#pom-}" - then - curl --netrc -i -X DELETE \ - $SONATYPE_DATA_CACHE_URL/$infix/$artifactId-$version.jar - fi -} - -# Generate a temporary file; not thread-safe - -tmpfile () { - i=1 - while test -f /tmp/precompiled.$i"$1" - do - i=$(($i+1)) - done - echo /tmp/precompiled.$i"$1" -} - -# Given a GAV or a path, read the POM - -read_pom () { - case "$1" in - pom.xml|*/pom.xml|*\\pom.xml) - cat "$1" - ;; - *) - curl -s "$(pom_url "$1")" - ;; - esac -} - -# Given a GAV parameter (or pom.xml path) and a name, resolve a property (falling back to parents) - -get_property () { - gav="$1" - key="$2" - case "$key" in - imagej1.version) - latest_version net.imagej:ij - return - ;; - project.groupId) - groupId "$gav" - return - ;; - project.version) - version "$gav" - return - ;; - esac - while test -n "$gav" - do - pom="$(read_pom "$gav")" - properties="$(extract_tag properties "$pom")" - property="$(extract_tag "$key" "$properties")" - if test -n "$property" - then - echo "$property" - return - fi - gav="$(parent_gav_from_pom_xml "$pom")" - done - die "Could not resolve \${$2} in $1" -} - -# Given a GAV parameter and a string, expand properties - -expand () { - gav="$1" - string="$2" - result= - while true - do - case "$string" in - *'${'*'}'*) - result="$result${string%%\$\{*}" - string="${string#*\$\{}" - key="${string%\}*}" - result="$result$(get_property "$gav" "$key")" - string="${string#$key\}}" - ;; - *) - echo "$result$string" - break - ;; - esac - done -} - -# Given a GAV parameter, make a list of its dependencies (as GAV parameters) - -get_dependencies () { - pom="$(read_pom "$1")" - while true - do - case "$pom" in - *''*) - dependency="$(extract_tag dependency "$pom")" - scope="$(extract_tag scope "$dependency")" - case "$scope" in - ''|compile) - groupId="$(expand "$1" "$(extract_tag groupId "$dependency")")" - artifactId="$(extract_tag artifactId "$dependency")" - version="$(expand "$1" "$(extract_tag version "$dependency")")" - echo "$groupId:$artifactId:$version" - ;; - esac - pom="${pom#*}" - ;; - *) - break; - esac - done -} - -# Given a GAV parameter and a space-delimited list of GAV parameters, expand -# the list by the first parameter and its dependencies (unless the list already -# contains said parameter) - -get_all_dependencies () { - case " $2 " in - *" $1 "*) - ;; # list already contains the depdendency - *) - gav="$1" - set "" "$2 $1" - for dependency in $(get_dependencies "$gav") - do - set "" "$(get_all_dependencies "$dependency" "$2")" - done - ;; - esac - echo "$2" -} - -# Given a GAV parameter, download the .jar file - -get_jar () { - url="$(jar_url "$1")" - tmpfile="$(tmpfile .jar)" - curl -s "$url" > "$tmpfile" - test " "$tmpfile" - test PK = "$(head -c 2 "$tmpfile")" - echo "$tmpfile" -} - -# Given a GAV parameter, get the commit from the manifest of the deployed .jar - -commit_from_gav () { - jar="$(get_jar "$1")" - unzip -p "$jar" META-INF/MANIFEST.MF | - sed -n -e 's/^Implementation-Build: *//pi' | - tr -d '\r' - rm "$jar" -} - -# Given a GAV parameter, determine whether the .jar file is already in plugins/ -# or jars/ - -is_jar_installed () { - artifactId="$(artifactId "$1")" - version="$(version "$1")" - file=$artifactId-$version.jar - test -f "$file" || file=../plugins/$file - test -f "$file" || return 1 - case "$version" in - *-SNAPSHOT) - # is the file younger than a day? - mtime="$(get_mtime "$file")" - test "$(($mtime-$(date +%s)))" -gt -86400 - ;; - esac -} - -# Given a .jar file, determine whether it is an ImageJ 1.x plugin - -is_ij1_plugin () { - unzip -l "$1" plugins.config > /dev/null 2>&1 -} - -# Given a GAV parameter, download the .jar file and its dependencies as needed -# and install them into plugins/ or jars/, respectively - -install_jar () { - for gav in $(get_all_dependencies "$1") - do - if ! is_jar_installed "$gav" - then - tmp="$(get_jar "$gav")" - name="$(artifactId "$gav")-$(version "$gav").jar" - if test -d ../plugins && is_ij1_plugin "$tmp" - then - mv "$tmp" "../plugins/$name" - else - mv "$tmp" "$name" - fi - fi - done -} - -# Determine whether a local project (specified as pom.xml) needs to be deployed - -is_deployed () { - gav="$(gav_from_pom "$1")" && - commit="$(commit_from_gav "$gav")" && - test -n "$commit" && - dir="$(dirname "$1")" && - (cd "$dir" && - git diff --quiet "$commit".. -- .) -} - -# The main part - -case "$1" in -commit) - commit_from_gav "$2" - ;; -deps|dependencies) - get_dependencies "$2" - ;; -all-deps|all-dependencies) - get_all_dependencies "$2" | - tr ' ' '\n' | - grep -v '^$' - ;; -latest-version) - latest_version "$2" - ;; -invalidate-cache) - invalidate_cache "$2" - ;; -gav-from-pom) - gav_from_pom "$2" - ;; -parent-gav) - parent_gav "$2" - ;; -pom-url) - pom_url "$2" - ;; -parent-gav-from-pom) - parent_gav_from_pom "$2" - ;; -packaging-from-pom) - packaging_from_pom "$2" - ;; -property-from-pom|get-property|property) - if test $# -lt 3 - then - get_property pom.xml "$2" - else - get_property "$2" "$3" - fi - ;; -install) - install_jar "$2" - ;; -is-deployed) - is_deployed "$2" - ;; -*) - test $# -eq 0 || echo "Unknown command: $1" >&2 - die "Usage: $0 [command] [argument...]"' - -Commands: - -commit :: - Gets the commit from which the given artifact was built. - -dependencies :: - Lists the direct dependencies of the given artifact. - -all-dependencies :: - Lists all dependencies of the given artifact, including itself and - transitive dependencies. - -latest-version :[:] - Prints the current version of the given artifact (if "SNAPSHOT" is - passed as version, it prints the current snapshot version rather - than the release one). - -invalidate-cache : - Invalidates the version cached in the SciJava Nexus from OSS Sonatype, - e.g. after releasing a new version to Sonatype. Requires appropriate - credentials in $HOME/.netrc for https://maven.scijava.org/. - -parent-gav :[:] - Prints the GAV parameter of the parent project of the given artifact. - -pom-url :: - Gets the URL of the POM describing the given artifact. - -gav-from-pom - Prints the GAV parameter described in the given pom.xml file. - -parent-gav-from-pom - Prints the GAV parameter of the parent project of the pom.xml file. - -packaging-from-pom - Prints the packaging type of the given project. - -property-from-pom - Prints the property specified in the pom.xml file (or in its parents). - -install :: - Installs the given artifact and all its dependencies; if the artifact - or dependency to install is an ImageJ 1.x plugin and the parent - directory contains a subdirectory called "plugins", it will be - installed there, otherwise into the current directory. - -is-deployed - Tests whether the specified project is deployed alright. Fails - with exit code 1 if not. -' - ;; -esac diff --git a/melting-pot.sh b/melting-pot.sh index 45c3fd6..29de050 100755 --- a/melting-pot.sh +++ b/melting-pot.sh @@ -5,50 +5,43 @@ # ============================================================================ # Tests all components of a project affected by changes in its dependencies. # -# First, an anecdote illustrating the problem this script solves: +# In particular, this script detects problems caused by diamond dependency +# structures: # -# Suppose you have a large application, org:app:1.0.0, with many dependencies: -# org:foo:1.2.3, org:bar:3.4.5, and many others. +# https://jlbp.dev/what-is-a-diamond-dependency-conflict # -# Now suppose you make some changes to foo, and want to know whether deploying -# them (i.e., releasing a new foo and updating app to depend on that release) -# will break the app. So you manually update your local copy of app to depend -# on org:foo:1.3.0-SNAPSHOT, and run the build (including tests, of course). +# This "melting pot" build rebuilds every dependency of a project, but at +# unified dependency versions matching those of the toplevel project. # -# The build passes, but this alone is insufficient: org:bar:3.4.5 also depends -# on org:foo:1.2.3, so you manually update bar to use org:foo:1.3.0-SNAPSHOT, -# then build bar to verify that it also is not broken by the update. +# For example, net.imagej:imagej:2.5.0 depends on many components including +# org.scijava:scijava-common:2.88.1 and net.imagej:imagej-ops:0.46.1, +# both of which depend on org.scijava:parsington. But: # -# This process quickly becomes very tedious when there are dozens of -# components of app which all depend on foo. +# - org.scijava:scijava-common:2.88.1 depends on org.scijava:parsington:3.0.0 +# - net.imagej:imagej-ops:0.46.1 depends on org.scijava:parsington:2.0.0 # -# And more importantly, testing each component individually in this manner is -# still insufficient to determine whether all of them will truly work together -# at runtime, where only a single version of each component is deployed. +# ImageJ2 can only depend on one of these versions at runtime. The newer one, +# ideally. SciJava projects use the pom-scijava parent POM as a Bill of +# Materials (BOM) to declare these winning versions, which works great... +# EXCEPT for when newer versions break backwards compatibility, as happened +# here: it's a SemVer-versioned project at different major version numbers. # -# For example: suppose org:bar:3.4.5 depends on org:lib:8.0.0, while -# org:foo:1.2.3 depends on org:lib:7.0.0. The relevant facts are: +# Enter this melting-pot script. It rebuilds each project dependency from +# source and runs the unit tests, but with all dependency versions pinned to +# match those of the toplevel project. # -# * Your new foo (org:foo:1.3.0-SNAPSHOT) builds against lib 7, and portions -# of it rely on lib-7-specific API. +# So in the example above, this script: # -# * The bar component pinned to foo 1.3.0-SNAPSHOT builds against lib 8; it -# compiles with passing tests because bar only invokes portions of the foo -# API which do not require lib-7-specific API. +# 1. gathers the dependencies of net.imagej:imagej:2.5.0; +# 2. clones each dependency from SCM at the correct release tag; +# 3. rebuilds each dependency, but with dependency versions overridden to +# those of net.imagej:imagej:2.5.0 rather than those originally used for +# that dependency at that release. # -# In this scenario, it is lib 8 that is actually deployed at runtime with the -# app, so parts of foo will be broken, even though both foo and bar build with -# passing tests individually. -# -# This "melting pot" build seeks to overcome many of these issues by unifying -# all components of the app into a single multi-module build, with all -# versions uniformly pinned to the ones that will actually be deployed at -# runtime. -# -# This goal is achieved by synthesizing a multi-module build including all -# affected components (or optionally, all components period) of the specific -# project, and then executing a Maven build with uniformly overridden versions -# of all components to the ones resolved for the project itself. +# So e.g. in the above scenario, net.imagej:imagej-ops:0.46.1 will be rebuilt +# against org.scijava:parsington:3.0.0, and we will discover whether any of +# parsington's breaking API changes from 2.0.0 to 3.0.0 actually impact the +# compilation or (tested) runtime behavior of imagej-ops. # # IMPORTANT IMPLEMENTATION DETAIL! The override works by setting a version # property for each component of the form "artifactId.version"; it is assumed @@ -70,122 +63,102 @@ # Author: Curtis Rueden # ============================================================================ -# -- Functions -- +# -- Constants -- -stderr() { - >&2 echo "$@" -} +meltingPotCache="$HOME/.cache/scijava/melting-pot" -debug() { - test "$debug" && - stderr "+ $@" -} - -info() { - test "$verbose" && - stderr "[INFO] $@" -} - -warn() { - stderr "[WARNING] $@" -} - -error() { - stderr "[ERROR] $@" -} - -die() { - error $1 - exit $2 -} +# -- Functions -- -unknownArg() { - error "Unknown option: $@" - usage=1 -} +stderr() { >&2 printf "$@\n"; } +debug() { test "$debug" && stderr "+ $@"; } +info() { test "$verbose" && stderr "\e[0;37m[INFO] $@\e[0m"; } +warn() { stderr "\e[0;33m[WARNING] $@\e[0m"; } +error() { stderr "\e[0;31m[ERROR] $@\e[0m"; } +die() { error $1; exit $2; } +unknownArg() { error "Unknown option: $@"; usage=1; } checkPrereqs() { - while [ $# -gt 0 ] - do - which $1 > /dev/null 2> /dev/null - test $? -ne 0 && die "Missing prerequisite: $1" 255 - shift - done + while [ $# -gt 0 ] + do + which $1 > /dev/null 2> /dev/null + test $? -ne 0 && die "Missing prerequisite: $1" 255 + shift + done } verifyPrereqs() { - checkPrereqs git mvn xmllint - git --version | grep -q 'git version 2' || - die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" 254 + checkPrereqs git mvn xmllint + git --version | grep -q 'git version 2' || + die "Please use git v2.x; older versions (<=1.7.9.5 at least) mishandle 'git clone --depth 1'" 254 } parseArguments() { - while [ $# -gt 0 ] - do - case "$1" in - -b|--branch) - branch="$2" - shift - ;; - -c|--changes) - test "$changes" && changes="$changes,$2" || changes="$2" - shift - ;; - -i|--includes) - test "$includes" && includes="$includes,$2" || includes="$2" - shift - ;; - -e|--excludes) - test "$excludes" && excludes="$excludes,$2" || excludes="$2" - shift - ;; - -r|--remoteRepos) - test "$remoteRepos" && remoteRepos="$remoteRepos,$2" || remoteRepos="$2" - shift - ;; - -l|--localRepo) - repoBase="$2" - shift - ;; - -o|--outputDir) - outputDir="$2" - shift - ;; - -p|--prune) - prune=1 - ;; - -v|--verbose) - verbose=1 - ;; - -d|--debug) - debug=1 - ;; - -f|--force) - force=1 - ;; - -s|--skipBuild) - skipBuild=1 - ;; - -h|--help) - usage=1 - ;; - -*) - unknownArg "$1" - ;; - *) - test -z "$project" && project="$1" || - unknownArg "$1" - ;; - esac - shift - done - - test -z "$project" -a -z "$usage" && - error "No project specified!" && usage=1 - - if [ "$usage" ] - then - echo "Usage: $(basename "$0") [-b ] [-c ] \\ + while [ $# -gt 0 ] + do + case "$1" in + -b|--branch) + branch="$2" + shift + ;; + -c|--changes) + test "$changes" && changes="$changes,$2" || changes="$2" + shift + ;; + -i|--includes) + test "$includes" && includes="$includes,$2" || includes="$2" + shift + ;; + -e|--excludes) + test "$excludes" && excludes="$excludes,$2" || excludes="$2" + shift + ;; + -r|--remoteRepos) + test "$remoteRepos" && remoteRepos="$remoteRepos,$2" || remoteRepos="$2" + shift + ;; + -l|--localRepo) + repoBase="$2" + shift + ;; + -o|--outputDir) + outputDir="$2" + shift + ;; + -p|--prune) + prune=1 + ;; + -v|--verbose) + verbose=1 + ;; + -d|--debug) + debug=1 + ;; + -f|--force) + force=1 + ;; + -s|--skipBuild) + skipBuild=1 + ;; + -h|--help) + usage=1 + ;; + -*) + unknownArg "$1" + ;; + *) + test -z "$project" && project="$1" || + unknownArg "$1" + ;; + esac + shift + done + + test -z "$project" -a -z "$usage" && + error "No project specified!" && usage=1 + + if [ "$usage" ] + then + echo "Usage: $(basename "$0") [-b ] [-c ] \\ [-i ] [-e ] [-r ] [-l ] [-o ] [-pvfsh] @@ -246,378 +219,547 @@ groupIds org.scijava, net.imagej, net.imglib2 and io.scif in the pot. The -e flag is used to exclude net.imglib2:imglib2-roi from the pot. " - exit 1 - fi + exit 1 + fi - # If project is a local directory path, get its absolute path. - test -d "$project" && project=$(cd "$project" && pwd) + # If project is a local directory path, get its absolute path. + test -d "$project" && project=$(cd "$project" && pwd) - # Assign default parameter values. - test "$outputDir" || outputDir="melting-pot" - test "$repoBase" || repoBase="$HOME/.m2/repository" + # Assign default parameter values. + test "$outputDir" || outputDir="melting-pot" + test "$repoBase" || repoBase="$HOME/.m2/repository" } createDir() { - test -z "$force" -a -e "$1" && - die "Directory already exists: $1" 2 + test -z "$force" -a -e "$1" && + die "Directory already exists: $1" 2 - rm -rf "$1" - mkdir -p "$1" - cd "$1" + rm -rf "$1" + mkdir -p "$1" + cd "$1" } groupId() { - echo "${1%%:*}" + echo "${1%%:*}" } artifactId() { - local result="${1#*:}" # strip groupId - echo "${result%%:*}" + local result="${1#*:}" # strip groupId + echo "${result%%:*}" } version() { - local result="${1#*:}" # strip groupId - case "$result" in - *:*) - result="${result#*:}" # strip artifactId - case "$result" in - *:*:*:*) - # G:A:P:C:V:S - result="${result#*:}" # strip packaging - result="${result#*:}" # strip classifier - ;; - *:*:*) - # G:A:P:V:S - result="${result#*:}" # strip packaging - ;; - *) - # G:A:V or G:A:V:? - ;; - esac - echo "${result%%:*}" - ;; - esac + local result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + result="${result#*:}" # strip classifier + ;; + *:*:*) + # G:A:P:V:S + result="${result#*:}" # strip packaging + ;; + *) + # G:A:V or G:A:V:? + ;; + esac + echo "${result%%:*}" + ;; + esac } classifier() { - local result="${1#*:}" # strip groupId - case "$result" in - *:*) - result="${result#*:}" # strip artifactId - case "$result" in - *:*:*:*) - # G:A:P:C:V:S - result="${result#*:}" # strip packaging - ;; - *:*:*) - # G:A:P:V:S - result="" - ;; - *:*) - # G:A:V:C - result="${result#*:}" # strip version - ;; - *) - # G:A:V - result="" - ;; - esac - echo "${result%%:*}" - ;; - esac + local result="${1#*:}" # strip groupId + case "$result" in + *:*) + result="${result#*:}" # strip artifactId + case "$result" in + *:*:*:*) + # G:A:P:C:V:S + result="${result#*:}" # strip packaging + ;; + *:*:*) + # G:A:P:V:S + result="" + ;; + *:*) + # G:A:V:C + result="${result#*:}" # strip version + ;; + *) + # G:A:V + result="" + ;; + esac + echo "${result%%:*}" + ;; + esac } # Converts the given GAV into a path in the local repository cache. repoPath() { - local gPath="$(echo "$(groupId "$1")" | tr :. /)" - local aPath="$(artifactId "$1")" - local vPath="$(version "$1")" - echo "$repoBase/$gPath/$aPath/$vPath" + local gPath="$(echo "$(groupId "$1")" | tr :. /)" + local aPath="$(artifactId "$1")" + local vPath="$(version "$1")" + echo "$repoBase/$gPath/$aPath/$vPath" } # Gets the path to the given GAV's POM file in the local repository cache. pomPath() { - local pomFile="$(artifactId "$1")-$(version "$1").pom" - echo "$(repoPath "$1")/$pomFile" + local pomFile="$(artifactId "$1")-$(version "$1").pom" + echo "$(repoPath "$1")/$pomFile" } # Fetches the POM for the given GAV into the local repository cache. downloadPOM() { - local g="$(groupId "$1")" - local a="$(artifactId "$1")" - local v="$(version "$1")" - debug "mvn dependency:get \\ - -DrepoUrl=\"$remoteRepos\" \\ - -DgroupId=\"$g\" \\ - -DartifactId=\"$a\" \\ - -Dversion=\"$v\" \\ - -Dpackaging=pom" - mvn dependency:get \ - -DrepoUrl="$remoteRepos" \ - -DgroupId="$g" \ - -DartifactId="$a" \ - -Dversion="$v" \ - -Dpackaging=pom > /dev/null || - die "Problem fetching $g:$a:$v from $remoteRepos" 4 + local g="$(groupId "$1")" + local a="$(artifactId "$1")" + local v="$(version "$1")" + debug "mvn dependency:get \\ + -DremoteRepositories=\"$remoteRepos\" \\ + -DgroupId=\"$g\" \\ + -DartifactId=\"$a\" \\ + -Dversion=\"$v\" \\ + -Dpackaging=pom" + mvn dependency:get \ + -DremoteRepositories="$remoteRepos" \ + -DgroupId="$g" \ + -DartifactId="$a" \ + -Dversion="$v" \ + -Dpackaging=pom > /dev/null || + die "Problem fetching $g:$a:$v from $remoteRepos" 4 } # Gets the POM path for the given GAV, ensuring it exists locally. pom() { - local pomPath="$(pomPath "$1")" - test -f "$pomPath" || downloadPOM "$1" - test -f "$pomPath" || die "Cannot access POM: $pomPath" 9 - echo "$pomPath" + local pomPath="$(pomPath "$1")" + test -f "$pomPath" || downloadPOM "$1" + test -f "$pomPath" || die "Cannot access POM: $pomPath" 9 + echo "$pomPath" } # For the given XML file on disk ($1), gets the value of the # specified XPath expression of the form "//$2/$3/$4/...". xpath() { - local xmlFile="$1" - shift - local xpath="/" - while [ $# -gt 0 ] - do - # NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075 - xpath="$xpath/*[local-name()='$1']" - shift - done - debug "xmllint --xpath \"$xpath\" \"$xmlFile\"" - xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | - sed -E 's/^[^>]*>(.*)<[^<]*$/\1/' + local xmlFile="$1" + shift + local expression="$@" + local xpath="/" + while [ $# -gt 0 ] + do + # NB: Ignore namespace issues; see: http://stackoverflow.com/a/8266075 + xpath="$xpath/*[local-name()='$1']" + shift + done + local value=$(xmllint --xpath "$xpath" "$xmlFile" 2> /dev/null | + sed -E 's/^[^>]*>(.*)<[^<]*$/\1/') + debug "xpath $xmlFile $expression -> $value" + echo "$value" } # For the given GAV ($1), recursively gets the value of the # specified XPath expression of the form "//$2/$3/$4/...". pomValue() { - local pomPath="$(pom "$1")" - test "$pomPath" || die "Cannot discern POM path for $1" 6 - shift - local value="$(xpath "$pomPath" $@)" - if [ "$value" ] - then - echo "$value" - else - # Path not found in POM; look in the parent POM. - local pg="$(xpath "$pomPath" project parent groupId)" - if [ "$pg" ] - then - # There is a parent POM declaration in this POM. - local pa="$(xpath "$pomPath" project parent artifactId)" - local pv="$(xpath "$pomPath" project parent version)" - pomValue "$pg:$pa:$pv" $@ - fi - fi + local pomPath="$(pom "$1")" + test "$pomPath" || die "Cannot discern POM path for $1" 6 + shift + local value="$(xpath "$pomPath" $@)" + if [ "$value" ] + then + echo "$value" + else + # Path not found in POM; look in the parent POM. + local pg="$(xpath "$pomPath" project parent groupId)" + if [ "$pg" ] + then + # There is a parent POM declaration in this POM. + local pa="$(xpath "$pomPath" project parent artifactId)" + local pv="$(xpath "$pomPath" project parent version)" + pomValue "$pg:$pa:$pv" $@ + fi + fi } # Gets the SCM URL for the given GAV. scmURL() { - pomValue "$1" project scm connection | sed -E 's/^scm:git://' + pomValue "$1" project scm connection | sed 's/^scm:git://' | + sed 's_git:\(//github.com/\)_https:\1_' } # Gets the SCM tag for the given GAV. scmTag() { - local tag=$(pomValue "$1" project scm tag) - if [ -z "$tag" -o "$tag" = "HEAD" ] - then - # The value was not set properly, - # so we try to guess the tag naming scheme. :-/ - warn "$1: improper scm tag value; scanning remote tags..." - local a=$(artifactId "$1") - local v=$(version "$1") - local scmURL="$(scmURL "$1")" - local allTags=$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || - error "$1: Invalid scm url: $scmURL") - for tag in "$a-$v" "$v" "v$v" - do - echo "$allTags" | grep -q "^$tag$" && { - info "$1: inferred tag: $tag" - echo "$tag" - return - } - done - error "$1: inscrutable tag scheme" - else - echo "$tag" - fi -} - -# Fetches the source code for the given GAV. Returns the directory. -retrieveSource() { - local scmURL="$(scmURL "$1")" - test "$scmURL" || die "Cannot glean SCM URL for $1" 10 - local scmBranch - test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" - local dir="$(groupId "$1")/$(artifactId "$1")" - debug "git clone \"$scmURL\" --branch \"$scmBranch\" --depth 1 \"$dir\"" - git clone "$scmURL" --branch "$scmBranch" --depth 1 "$dir" 2> /dev/null || - die "Could not fetch project source for $1" 3 - - # Now verify that the cloned pom.xml contains the expected version! - local expectedVersion=$(version "$1") - local actualVersion=$(xpath "$dir/pom.xml" project version) - test "$expectedVersion" = "$actualVersion" || - die "POM for $1 contains wrong version: $actualVersion" 14 - - echo "$dir" + local tag=$(pomValue "$1" project scm tag) + if [ -z "$tag" -o "$tag" = "HEAD" ] + then + # The value was not set properly, + # so we try to guess the tag naming scheme. :-/ + warn "$1: improper scm tag value; scanning remote tags..." + local a=$(artifactId "$1") + local v=$(version "$1") + local scmURL="$(scmURL "$1")" + # TODO: Avoid network use. We can scan the locally cached repo. + # But this gets complicated when the locally cached repo is + # out of date, and the needed tag is not there yet... + debug "git ls-remote --tags \"$scmURL\" | sed 's/.*refs\/tags\///'" + local allTags="$(git ls-remote --tags "$scmURL" | sed 's/.*refs\/tags\///' || + error "$1: Invalid scm url: $scmURL")" + for tag in "$a-$v" "$v" "v$v" + do + echo "$allTags" | grep -q "^$tag$" && { + info "$1: inferred tag: $tag" + echo "$tag" + return + } + done + error "$1: inscrutable tag scheme -- using default branch" + else + echo "$tag" + fi +} + +# Ensures the source code for the given GAV exists in the melting-pot +# structure, and is up-to-date with the remote. Returns the directory. +resolveSource() { + local g=$(groupId "$1") + local a=$(artifactId "$1") + local cachedRepoDir="$meltingPotCache/$g/$a" + if [ ! -d "$cachedRepoDir" ] + then + # Source does not exist locally. Clone it into the melting pot cache. + local scmURL="$(scmURL "$1")" + test "$scmURL" || die "$1: cannot glean SCM URL" 10 + info "$1: cached repository not found; cloning from remote: $scmURL" + debug "git clone --bare \"$scmURL\" \"$cachedRepoDir\"" + git clone --bare "$scmURL" "$cachedRepoDir" 2> /dev/null || + die "$1: could not clone project source from $scmURL" 3 + fi + + # Check whether the needed branch/tag exists. + local scmBranch + test "$2" && scmBranch="$2" || scmBranch="$(scmTag "$1")" + + if [ "$scmBranch" ] + then + # Successfully gleaned SCM branch/tag. + debug "git ls-remote \"file://$cachedRepoDir\" | grep -q \"\brefs/tags/$scmBranch$\"" + git ls-remote "file://$cachedRepoDir" | grep -q "\brefs/tags/$scmBranch$" || { + # Couldn't find the scmBranch as a tag in the cached repo. Either the + # tag is new, or it's not a tag ref at all (e.g. it's a branch). + # So let's update from the original remote repository. + info "$1: local tag not found for ref '$scmBranch'" + info "$1: updating cached repository: $cachedRepoDir" + cd "$cachedRepoDir" + debug "git fetch --tags" + if [ "$debug" ] + then + git fetch --tags + else + git fetch --tags > /dev/null + fi + cd - > /dev/null + } + else + # No SCM branch/tag; fall back to the default branch. + info "$1: updating cached repository: $cachedRepoDir" + cd "$cachedRepoDir" + debug "git fetch" + if [ "$debug" ] + then + git fetch + else + git fetch > /dev/null + fi + head=$(cat HEAD) + scmBranch=${head##*/} + info "$1: detected default branch as $scmBranch" + cd - > /dev/null + fi + + # Shallow clone the source at the given version into melting-pot structure. + local destDir="$g/$a" + debug "git clone \"file://$cachedRepoDir\" --branch \"$scmBranch\" --depth 1 \"$destDir\"" + git clone "file://$cachedRepoDir" --branch "$scmBranch" --depth 1 "$destDir" 2> /dev/null || + die "$1: could not clone branch '$scmBranch' from local cache" 15 + + # Save the GAV string to a file, for convenience. + echo "$1" > "$destDir/gav" + + # Now verify that the cloned pom.xml contains the expected version! + local expectedVersion=$(version "$1") + local actualVersion=$(xpath "$destDir/pom.xml" project version) + test "$expectedVersion" = "$actualVersion" || + warn "$1: POM contains wrong version: $actualVersion" + + echo "$destDir" } # Gets the list of dependencies for the project in the CWD. deps() { - cd "$1" - debug "mvn dependency:list" - local depList="$(mvn -B dependency:list)" || - die "Problem fetching dependencies!" 5 - echo "$depList" | grep '^\[INFO\] [^ ]' | - sed 's/\[INFO\] //' | sed 's/ *(optional) *$//' | sort - cd - > /dev/null + cd "$1" || die "No such directory: $1" 16 + debug "mvn -B -DincludeScope=runtime dependency:list" + local depList="$(mvn -B -DincludeScope=runtime dependency:list)" || + die "Problem fetching dependencies!" 5 + echo "$depList" | grep '^\[INFO\] \w' | + sed 's/\[INFO\] //' | sed 's/ .*//' | sort + cd - > /dev/null } # Checks whether the given GA(V) matches the specified filter pattern. gaMatch() { - local ga="$1" - local filter="$2" - local g="$(groupId "$ga")" - local a="$(artifactId "$ga")" - local fg="$(groupId "$filter")" - local fa="$(artifactId "$filter")" - test "$fg" = "$g" -o "$fg" = "*" || return - test "$fa" = "$a" -o "$fa" = "*" || return - echo 1 + local ga="$1" + local filter="$2" + local g="$(groupId "$ga")" + local a="$(artifactId "$ga")" + local fg="$(groupId "$filter")" + local fa="$(artifactId "$filter")" + test "$fg" = "$g" -o "$fg" = "*" || return + test "$fa" = "$a" -o "$fa" = "*" || return + echo 1 } # Determines whether the given GA(V) version is being overridden. isChanged() { - local IFS="," + local IFS="," - local change - for change in $changes - do - test "$(gaMatch "$1" "$change")" && echo 1 && return - done + local change + for change in $changes + do + test "$(gaMatch "$1" "$change")" && echo 1 && return + done } # Determines whether the given GA(V) meets the inclusion criteria. isIncluded() { - # do not include the changed artifacts we are testing against - test "$(isChanged "$1")" && return - - local IFS="," - - # ensure GA is not excluded - local exclude - for exclude in $excludes - do - test "$(gaMatch "$1" "$exclude")" && return - done - - # ensure GA is included - test -z "$includes" && echo 1 && return - local include - for include in $includes - do - test "$(gaMatch "$1" "$include")" && echo 1 && return - done + # do not include the changed artifacts we are testing against + test "$(isChanged "$1")" && return + + local IFS="," + + # ensure GA is not excluded + local exclude + for exclude in $excludes + do + test "$(gaMatch "$1" "$exclude")" && return + done + + # ensure GA is included + test -z "$includes" && echo 1 && return + local include + for include in $includes + do + test "$(gaMatch "$1" "$include")" && echo 1 && return + done } # Deletes components which do not depend on a changed GAV. pruneReactor() { - local dir - for dir in */* - do - info "Checking relevance of component $dir" - local deps="$(deps "$dir")" - test "$deps" || die "Cannot glean dependencies for '$dir'" 8 - - # Determine whether the component depends on a changed GAV. - local keep - unset keep - local dep - for dep in $deps - do - test "$(isChanged "$dep")" && keep=1 && break - done - - # If the component is irrelevant, prune it. - if [ -z "$keep" ] - then - info "Pruning irrelevant component: $dir" - rm -rf "$dir" - fi - done + local dir + for dir in */* + do + info "Checking relevance of component $dir" + local deps="$(deps "$dir")" + test "$deps" || die "Cannot glean dependencies for '$dir'" 8 + + # Determine whether the component depends on a changed GAV. + local keep + unset keep + local dep + for dep in $deps + do + test "$(isChanged "$dep")" && keep=1 && break + done + + # If the component is irrelevant, prune it. + if [ -z "$keep" ] + then + info "Pruning irrelevant component: $dir" + rm -rf "$dir" + fi + done } # Tests if the given directory contains the appropriate source code. isProject() { - local a="$(xpath "$1/pom.xml" project artifactId)" - test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 -} - -# Generates an aggregator POM for all modules in the current directory. -generatePOM() { - echo '' > pom.xml - echo '> pom.xml - echo ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' >> pom.xml - echo ' xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' >> pom.xml - echo ' http://maven.apache.org/xsd/maven-4.0.0.xsd">' >> pom.xml - echo ' 4.0.0' >> pom.xml - echo >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' melting-pot' >> pom.xml - echo ' 0.0.0-SNAPSHOT' >> pom.xml - echo ' pom' >> pom.xml - echo >> pom.xml - echo ' Melting Pot' >> pom.xml - echo >> pom.xml - echo ' ' >> pom.xml - local dir - for dir in */* - do - if [ "$(isProject "$dir")" ] - then - echo " $dir" >> pom.xml - else - # Check for a child component of a multi-module project. - local childDir="$dir/$(basename "$dir")" - test "$(isProject "$childDir")" && - echo " $childDir" >> pom.xml - fi - done - echo ' ' >> pom.xml - echo '' >> pom.xml -} - -# Generates melt.sh script for all modules in the current directory. -generateScript() { - echo '#!/bin/sh' > melt.sh - echo 'trap "exit" INT' >> melt.sh - echo 'echo "Melting the pot..."' >> melt.sh - echo 'dir=$(pwd)' >> melt.sh - echo 'failCount=0' >> melt.sh - echo 'for f in \' >> melt.sh - local dir - for dir in */* - do - if [ "$(isProject "$dir")" ] - then - echo " $dir \\" >> melt.sh - else - # Check for a child component of a multi-module project. - local childDir="$dir/$(basename "$dir")" - test "$(isProject "$childDir")" && - echo " $childDir \\" >> melt.sh - fi - done - echo >> melt.sh - echo 'do (' >> melt.sh - echo ' cd "$f"' >> melt.sh - echo ' sh "$dir/build.sh" > build.log 2>&1 &&' >> melt.sh - echo ' echo "[SUCCESS] $f" || {' >> melt.sh - echo ' echo "[FAILURE] $f"' >> melt.sh - echo ' failCount=$((failCount+1))' >> melt.sh - echo ' }' >> melt.sh - echo ') done' >> melt.sh - echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh - echo 'exit "$failCount"' >> melt.sh + local a="$(xpath "$1/pom.xml" project artifactId)" + test "$1" = "LOCAL/PROJECT" -o "$a" = "$(basename "$1")" && echo 1 +} + +# Generates melt.sh, covering all projects in the current directory. +generateMeltScript() { + echo '#!/bin/sh' > melt.sh + echo 'trap "exit" INT' >> melt.sh + echo 'echo "Melting the pot..."' >> melt.sh + echo 'dir=$(cd "$(dirname "$0")" && pwd)' >> melt.sh + echo 'failCount=0' >> melt.sh + echo 'for f in \' >> melt.sh + local dir + for dir in */* + do + if [ "$(isProject "$dir")" ] + then + echo " $dir \\" >> melt.sh + else + # Check for a child component of a multi-module project. + local childDir="$dir/$(basename "$dir")" + test "$(isProject "$childDir")" && + echo " $childDir \\" >> melt.sh + fi + done + echo >> melt.sh + echo 'do' >> melt.sh + echo ' if [ "$("$dir/prior-success.sh" "$f")" ]' >> melt.sh + echo ' then' >> melt.sh + echo ' printf "\e[0;36m[SKIPPED] $f (prior success)\e[0m\n"' >> melt.sh + echo ' continue' >> melt.sh + echo ' fi' >> melt.sh + echo ' cd "$f"' >> melt.sh + echo ' "$dir/build.sh" >build.log 2>&1 && {' >> melt.sh + echo ' printf "\e[0;32m[SUCCESS] $f\e[0m\n"' >> melt.sh + echo ' "$dir/record-success.sh" "$f"' >> melt.sh + echo ' } || {' >> melt.sh + echo ' printf "\e[0;31m[FAILURE] $f\e[0m\n"' >> melt.sh + echo ' failCount=$((failCount+1))' >> melt.sh + echo ' }' >> melt.sh + echo ' cd - >/dev/null' >> melt.sh + echo 'done' >> melt.sh + echo 'test "$failCount" -gt 255 && failCount=255' >> melt.sh + echo 'exit "$failCount"' >> melt.sh + chmod +x melt.sh +} + +# Generates helper scripts, including prior-success.sh and record-success.sh. +generateHelperScripts() { + cat <<\PRIOR > prior-success.sh +#!/bin/sh +test "$1" || { + printf "\e[0;31m[ERROR] Please specify project to check.\e[0m\n" + exit 1 +} + +stderr() { >&2 printf "$@\n"; } +debug() { test "$DEBUG" && stderr "\e[0;37m[DEBUG] $@\e[0m"; } +info() { stderr "\e[0;37m[INFO] $@\e[0m"; } +warn() { stderr "\e[0;33m[WARNING] $@\e[0m"; } + +dir=$(cd "$(dirname "$0")" && pwd) + +# Check build.log for BUILD SUCCESS. +buildLog="$dir/$1/build.log" +test -f "$buildLog" && tail -n6 "$buildLog" | grep -qF 'BUILD SUCCESS' && { + echo "build.log" + exit 0 +} + +# Check success.log for matching dependency configuration. +successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" +test -f "$successLog" || exit 0 +row=1 +mismatch1= +success= +for deps in $(cat "$successLog") +do + debug "$1: Checking dep config: $deps" + mismatch= + for dep in $(echo "$deps" | tr ',' '\n') + do + # g:a:p:v:s -> v + s=${dep##*:} + case "$s" in + test) continue ;; # skip test dependencies + none) continue ;; # empty dependency config + esac + gapv=${dep%:*} + g=${gapv%%:*} + apv=${gapv#*:} + a=${apv%%:*} + v=${apv##*:} + bomV=$(grep -o " <$g\.$a\.version>[^>]*" "$dir/version-pins.xml" | sed 's;[^>]*>\([^>]*\)<.*;\1;') + if [ "$bomV" != "${bomV#*-SNAPSHOT*}" ] + then + warn "$1: Snapshot dependency pin detected: $g:$a:$bomV -- forcing a rebuild" + exit 0 + elif [ "$bomV" != "$v" ] + then + # G:A property is not set to this V. + # Now check if the property is even declared. + if [ "$bomV" ] + then + # G:A version is mismatched. + mismatch="$mismatch\n* $g:$a:$v -> $bomV" + else + # G:A version is not pinned. + test "$row" -ne 1 || + warn "$1: Unpinned dependency: $dep" + fi + fi + done + if [ "$mismatch" ] + then + test "$row" -eq 1 && mismatch1=$mismatch || + debug "$1: Dependency changes since success #$row:$mismatch" + else + success=$deps + break + fi + row=$((row+1)) +done +test "$success" && echo "$success" || { + test "$mismatch1" && + info "$1: Dependency changes since last success:$mismatch1" || + info "$1: No prior successes" +} +PRIOR + chmod +x prior-success.sh + + cat <<\RECORD > record-success.sh +#!/bin/sh +test "$1" || { + printf "\e[0;31m[ERROR] Please specify project to update.\e[0m\n" + exit 1 +} + +containsLine() { + pattern=$1 + file=$2 + test -f "$file" || return + # HACK: The obvious way to do this is: + # + # grep -qxF "$pattern" "$file" + # + # Unfortunately, BSD grep dies with "out of memory" when the pattern is 5111 + # characters or longer. So let's do something needlessly complex instead! + cat "$file" | while read line + do + test "$pattern" = "$line" && echo 1 && break + done +} + +dir=$(cd "$(dirname "$0")" && pwd) +buildLog="$dir/$1/build.log" +test -f "$buildLog" || exit 1 +successLog="$HOME/.cache/scijava/melting-pot/$1.success.log" +mkdir -p "$(dirname "$successLog")" + +# Record dependency configuration of successful build. +deps=$(grep '^\[[^ ]*INFO[^ ]*\] \w' "$buildLog" | + sed -e 's/^[^ ]* *//' -e 's/ -- .*//' -e 's/ (\([^)]*\))/-\1/' | + sort | tr '\n' ',') +if [ "$deps" = "${deps%*-SNAPSHOT*}" -a -z "$(containsLine "$deps" "$successLog")" ] +then + # NB: *Prepend*, rather than append, the new successful configuration. + # We do this because it is more likely this new configuration will be + # encountered again in the future, as dependency versions are highly + # likely to repeatedly increment, rather than moving backwards. + echo "$deps" > "$successLog".new + test -f "$successLog" && cat "$successLog" >> "$successLog".new + mv -f "$successLog".new "$successLog" +fi +RECORD + chmod +x record-success.sh } # Creates and tests an appropriate multi-module reactor for the given project. @@ -625,83 +767,194 @@ generateScript() { # the multi-module build, with each changed GAV overridding the originally # specified version for the corresponding GA. meltDown() { - # Fetch the project source code. - if [ -d "$1" ] - then - # Use local directory for the specified project. - test -d "$1" || die "No such directory: $1" 11 - test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 - info "Local Maven project: $1" - mkdir -p "LOCAL" - local dir="LOCAL/PROJECT" - ln -s "$1" "$dir" - else - # Treat specified project as a GAV. - info "Fetching project source" - retrieveSource "$1" "$branch" - fi - - # Get the project dependencies. - info "Determining project dependencies" - local deps="$(deps "$dir")" - test "$deps" || die "Cannot glean project dependencies" 7 - - local args="-Denforcer.skip" - - # Process the dependencies. - info "Processing project dependencies" - local dep - for dep in $deps - do - local g="$(groupId "$dep")" - local a="$(artifactId "$dep")" - local v="$(version "$dep")" - local c="$(classifier "$dep")" - test -z "$c" || continue # skip secondary artifacts - local gav="$g:$a:$v" - - test -z "$(isChanged "$gav")" && - args="$args \\\\\n -D$a.version=$v" - - if [ "$(isIncluded "$gav")" ] - then - info "$a: fetching component source" - dir="$(retrieveSource "$gav")" - fi - done - - # Override versions of changed GAVs. - info "Processing changed components" - local TLS=, - local gav - for gav in $changes - do - local a="$(artifactId "$gav")" - local v="$(version "$gav")" - args="$args \\\\\n -D$a.version=$v" - done - unset TLS - - # Prune the build, if applicable. - test "$prune" && pruneReactor - - # Generate build scripts. - info "Generating build scripts" - generatePOM - echo "mvn $args \\\\\n test \$@" > build.sh - generateScript - - # Build everything. - if [ "$skipBuild" ] - then - info "Skipping the build; run melt.sh to do it." - else - info "Building the project!" - # NB: All code is fresh; no need to clean. - sh melt.sh || die "Melt failed" 13 - fi - - info "Melt complete: $1" + # Fetch the project source code. + if [ -d "$1" ] + then + # Use local directory for the specified project. + test -d "$1" || die "No such directory: $1" 11 + test -f "$1/pom.xml" || die "Not a Maven project: $1" 12 + case "$(uname)" in + MINGW*) + warn "Skipping inclusion of local project due to lack of symlink support." + local projectDir="$1" + ;; + *) + info "Local Maven project: $1" + mkdir -p "LOCAL" + local projectDir="LOCAL/PROJECT" + ln -s "$1" "$projectDir" + ;; + esac + else + # Treat specified project as a GAV. + info "Fetching project source" + local projectDir=$(resolveSource "$1" "$branch") + test $? -eq 0 || exit $? + fi + + # Get the project dependencies. + info "Determining project dependencies" + local deps="$(deps "$projectDir")" + test "$deps" || die "Cannot glean project dependencies" 7 + + # Generate helper scripts. We need prior-success.sh + # to decide whether to include each component. + generateHelperScripts + + # Process the dependencies. + info "Processing project dependencies" + local versionProps="" + local dep + for dep in $deps + do + local g="$(groupId "$dep")" + local a="$(artifactId "$dep")" + local aa + echo "$a" | grep -q '^[0-9]' && aa="_$a" || aa="$a" + local v="$(version "$dep")" + local c="$(classifier "$dep")" + test -z "$c" || continue # skip secondary artifacts + local gav="$g:$a:$v" + test -z "$(isChanged "$gav")" && + versionProps="$versionProps + <$g.$a.version>$v <$aa.version>$v" + done + + # Override versions of changed GAVs. + info "Processing changed components" + local TLS=, + local gav + for gav in $changes + do + local a="$(artifactId "$gav")" + local v="$(version "$gav")" + versionProps="$versionProps + <$a.version>$v" + done + unset TLS + + # Generate version-pins.xml. + info "Generating version-pins.xml configuration" + echo ' version-pins.xml + echo ' xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.1.0%20https://maven.apache.org/xsd/settings-1.1.0.xsd">' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' version-pins' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' true' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo "$versionProps" >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo ' ' >> version-pins.xml + echo '' >> version-pins.xml + + # Generate build script. + info "Generating build.sh script" + echo '#!/bin/sh' > build.sh + echo >> build.sh + echo 'dir=$(cd "$(dirname "$0")" && pwd)' >> build.sh + echo >> build.sh + echo 'mvnPin() {' >> build.sh + # NB: We do *not* include -B here, because we want build.sh to preserve + # colored output if the version of Maven is new enough. We will take care + # elsewhere when parsing it to be flexible about whether colors are present. + echo ' mvn -s "$dir/version-pins.xml" -Denforcer.skip $@' >> build.sh + echo '}' >> build.sh + echo >> build.sh + echo 'unpackArtifact() {' >> build.sh + echo ' # Download and unpack the given artifact' >> build.sh + echo ' # (G:A:V) to the specified location.' >> build.sh + echo ' gav=$1' >> build.sh + echo ' out=$2' >> build.sh + echo >> build.sh + echo ' repoPrefix=$HOME/.m2/repository # TODO: generalize this' >> build.sh + echo ' g=${gav%%:*}; r=${gav#*:}; a=${r%%:*}; v=${r##*:}' >> build.sh + echo ' gavPath="$(echo "$g" | tr "." "/")/$a/$v/$a-$v"' >> build.sh + echo ' jarPath="$repoPrefix/$gavPath.jar"' >> build.sh + echo >> build.sh + echo ' # HACK: The best goal to use would be dependency:unpack,' >> build.sh + echo ' # or failing that, dependency:copy followed by jar xf.' >> build.sh + echo ' # But those goals do not support remoteRepositories;' >> build.sh + echo ' # see https://issues.apache.org/jira/browse/MDEP-390.' >> build.sh + echo ' # So we use dependency:get and then extract it by hand.' >> build.sh + echo ' mvnPin dependency:get \' >> build.sh + echo " -DremoteRepositories=\"$remoteRepos\" \\" >> build.sh + echo ' -Dartifact="$gav" &&' >> build.sh + echo >> build.sh + echo ' test -f "$jarPath" &&' >> build.sh + echo ' mkdir -p "$out" &&' >> build.sh + echo ' cd "$out" &&' >> build.sh + echo ' jar xf "$jarPath" &&' >> build.sh + echo ' cd - >/dev/null' >> build.sh + echo '}' >> build.sh + echo >> build.sh + echo 'mvnPin dependency:list dependency:tree &&' >> build.sh + echo >> build.sh + echo 'if [ -f gav ]' >> build.sh + echo 'then' >> build.sh + echo ' echo' >> build.sh + echo ' echo "================================================"' >> build.sh + echo ' echo "========= Testing with deployed binary ========="' >> build.sh + echo ' echo "================================================"' >> build.sh + echo ' unpackArtifact "$(cat gav)" target/classes &&' >> build.sh + echo ' mvnPin \' >> build.sh + echo ' -Dmaven.main.skip=true \' >> build.sh + echo ' -Dmaven.resources.skip=true \' >> build.sh + echo ' test $@' >> build.sh + echo 'fi &&' >> build.sh + echo >> build.sh + echo 'echo &&' >> build.sh + echo 'echo "================================================" &&' >> build.sh + echo 'echo "============ Rebuilding from source ============" &&' >> build.sh + echo 'echo "================================================" &&' >> build.sh + echo 'mvnPin clean test $@' >> build.sh + chmod +x build.sh + + # Clone source code. + info "Cloning source code" + for dep in $deps + do + local g="$(groupId "$dep")" + local a="$(artifactId "$dep")" + local v="$(version "$dep")" + local c="$(classifier "$dep")" + test -z "$c" || continue # skip secondary artifacts + local gav="$g:$a:$v" + if [ "$(isIncluded "$gav")" ] + then + if [ "$v" != "${v%-SNAPSHOT}" ] + then + info "$g:$a: forcing inclusion due to SNAPSHOT version" + elif [ "$(./prior-success.sh "$g/$a")" ] + then + info "$g:$a: skipping version $v due to prior successful build" + continue + fi + info "$g:$a: resolving source for version $v" + resolveSource "$gav" >/dev/null + fi + done + + # Prune the build, if applicable. + test "$prune" && pruneReactor + + # Generate melt script. + info "Generating melt.sh script" + generateMeltScript + + # Build everything. + if [ "$skipBuild" ] + then + info "Skipping the build; run melt.sh to do it." + else + info "Building the project!" + # NB: All code is fresh; no need to clean. + sh melt.sh || die "Melt failed" 13 + fi + + info "Melt complete: $1" } # -- Main -- diff --git a/release-version.sh b/release-version.sh index 734ec1e..8f95226 100755 --- a/release-version.sh +++ b/release-version.sh @@ -8,9 +8,18 @@ # Authors: Johannes Schindelin & Curtis Rueden # ============================================================================ +# -- Avoid localized output that might confuse the script -- + +export LC_ALL=C + # -- Functions -- -die () { +debug() { + test "$DEBUG" || return + echo "[DEBUG] $@" +} + +die() { echo "$*" >&2 exit 1 } @@ -30,6 +39,7 @@ SCIJAVA_THIRDPARTY_REPOSITORY=$SCIJAVA_BASE_REPOSITORY/thirdparty # Parse command line options. BATCH_MODE=--batch-mode SKIP_VERSION_CHECK= +SKIP_BRANCH_CHECK= SKIP_LICENSE_UPDATE= SKIP_PUSH= SKIP_GPG= @@ -47,6 +57,7 @@ do --dry-run) DRY_RUN=echo;; --no-batch-mode) BATCH_MODE=;; --skip-version-check) SKIP_VERSION_CHECK=t;; + --skip-branch-check) SKIP_BRANCH_CHECK=t;; --skip-license-update) SKIP_LICENSE_UPDATE=t;; --skip-push) SKIP_PUSH=t;; --tag=*) @@ -92,6 +103,7 @@ Where is the version to release. If omitted, it will prompt you. Options include: --dry-run - Simulate the release without actually doing it. --skip-version-check - Skips the SemVer and parent pom version checks. + --skip-branch-check - Skips the default branch check. --skip-license-update - Skips update of the copyright blurbs. --skip-push - Do not push to the remote git repository. --dev-version= - Specify next development version explicitly; @@ -104,26 +116,42 @@ Options include: " # -- Extract project details -- +debug "Extracting project details" -projectDetails=$(mvn -N -Dexec.executable='echo' -Dexec.args='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' exec:exec -q) +echoArg='${project.version}:${license.licenseName}:${project.parent.groupId}:${project.parent.artifactId}:${project.parent.version}' +projectDetails=$(mvn -B -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) +test $? -eq 0 || projectDetails=$(mvn -B -U -N -Dexec.executable=echo -Dexec.args="$echoArg" exec:exec -q) +test $? -eq 0 || die "Could not extract version from pom.xml. Error follows:\n$projectDetails" +printf '%s' "$projectDetails\n" | grep -Fqv '[ERROR]' || + die "Error extracting version from pom.xml. Error follows:\n$projectDetails" +# HACK: Even with -B, some versions of mvn taint the output with the [0m +# color reset sequence. So we forcibly remove such sequences, just to be safe. +projectDetails=$(printf '%s' "$projectDetails" | sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]//g") +# And also remove extraneous newlines, particularly any trailing ones. +projectDetails=$(printf '%s' "$projectDetails" | tr -d '\n') currentVersion=${projectDetails%%:*} projectDetails=${projectDetails#*:} licenseName=${projectDetails%%:*} parentGAV=${projectDetails#*:} # -- Sanity checks -- +debug "Performing sanity checks" # Check that we have push rights to the repository. if [ ! "$SKIP_PUSH" ] then + debug "Checking repository push rights" push=$(git remote -v | grep origin | grep '(push)') - test "$push" || die 'No push URL found for remote origin' - echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only' + test "$push" || die 'No push URL found for remote origin. +Please use "git remote -v" to double check your remote settings.' + echo "$push" | grep -q 'git:/' && die 'Remote origin is read-only. +Please use "git remote set-url origin ..." to change it.' fi # Discern the version to release. +debug "Gleaning release version" pomVersion=${currentVersion%-SNAPSHOT} -test "$VERSION" || test ! -t 0 || { +test "$VERSION" -o ! -t 0 || { printf 'Version? [%s]: ' "$pomVersion" read VERSION test "$VERSION" || VERSION=$pomVersion @@ -131,79 +159,189 @@ test "$VERSION" || test ! -t 0 || { # Check that the release version number starts with a digit. test "$VERSION" || die 'Please specify the version to release!' -case "$VERSION" in -[0-9]*) - ;; -*) - die "Version '$VERSION' does not start with a digit!" -esac +test "$SKIP_VERSION_CHECK" || { + case "$VERSION" in + [0-9]*) + ;; + *) + die "Version '$VERSION' does not start with a digit! +If you are sure, try again with --skip-version-check flag." + esac +} # Check that the release version number conforms to SemVer. VALID_SEMVER_BUMP="$(cd "$(dirname "$0")" && pwd)/valid-semver-bump.sh" test -f "$VALID_SEMVER_BUMP" || die "Missing helper script at '$VALID_SEMVER_BUMP' -Do you have a full clone of git://github.com/scijava/scijava-scripts?" +Do you have a full clone of https://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" || { + debug "Checking conformance to SemVer" sh -$- "$VALID_SEMVER_BUMP" "$pomVersion" "$VERSION" || die "If you are sure, try again with --skip-version-check flag." } # Check that the project extends the latest version of pom-scijava. -MAVEN_HELPER="$(cd "$(dirname "$0")" && pwd)/maven-helper.sh" -test -f "$MAVEN_HELPER" || - die "Missing helper script at '$MAVEN_HELPER' -Do you have a full clone of git://github.com/scijava/scijava-scripts?" test "$SKIP_VERSION_CHECK" -o "$parentGAV" != "${parentGAV#$}" || { - latestParentVersion=$(sh -$- "$MAVEN_HELPER" latest-version "$parentGAV") + debug "Checking pom-scijava parent version" + psjMavenMetadata=https://repo1.maven.org/maven2/org/scijava/pom-scijava/maven-metadata.xml + latestParentVersion=$(curl -fsL "$psjMavenMetadata" | grep '' | sed 's;.*>\([^<]*\)<.*;\1;') currentParentVersion=${parentGAV##*:} test "$currentParentVersion" = "$latestParentVersion" || - die "Newer version of parent '${parentGAV%:*}' is available: $latestParentVersion. + die "Newer version of parent '$parentGAV' is available: $latestParentVersion. I recommend you update it before releasing. Or if you know better, try again with --skip-version-check flag." } # Check that the working copy is clean. +debug "Checking if working copy is clean" no_changes_pending || die 'There are uncommitted changes!' test -z "$(git ls-files -o --exclude-standard)" || die 'There are untracked files! Please stash them before releasing.' -# Check that we are on the master branch. -test refs/heads/master = "$(git rev-parse --symbolic-full-name HEAD)" || - die "Not on 'master' branch" +# Discern default branch. +debug "Discerning default branch" +currentBranch=$(git rev-parse --abbrev-ref --symbolic-full-name HEAD) +upstreamBranch=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) +remote=${upstreamBranch%/*} +defaultBranch=$(git remote show "$remote" | grep "HEAD branch" | sed 's/.*: //') + +# Check that we are on the main branch. +test "$SKIP_BRANCH_CHECK" || { + debug "Checking current branch" + test "$currentBranch" = "$defaultBranch" || die "Non-default branch: $currentBranch. +If you are certain you want to release from this branch, +try again with --skip-branch-check flag." +} -# If REMOTE is unset, use origin by default. -REMOTE="${REMOTE:-origin}" +# If REMOTE is unset, use branch's upstream remote by default. +REMOTE="${REMOTE:-$remote}" -# Check that the master branch isn't behind the upstream branch. +# Check that the main branch isn't behind the upstream branch. +debug "Ensuring local branch is up-to-date" HEAD="$(git rev-parse HEAD)" && -git fetch "$REMOTE" master && +git fetch "$REMOTE" "$defaultBranch" && FETCH_HEAD="$(git rev-parse FETCH_HEAD)" && test "$FETCH_HEAD" = HEAD || test "$FETCH_HEAD" = "$(git merge-base $FETCH_HEAD $HEAD)" || - die "'master' is not up-to-date" + die "'$defaultBranch' is not up-to-date" + +# Check for release-only files committed to the main branch. +debug "Checking for spurious release-only files" +for release_file in release.properties pom.xml.releaseBackup +do + if [ -e "$release_file" ] + then + echo "==========================================================================" + echo "NOTE: $release_file was committed to source control. Removing now." + echo "==========================================================================" + git rm -rf "$release_file" && + git commit "$release_file" \ + -m 'Remove $release_file' \ + -m 'It should only exist on release tags.' + fi +done + +# Ensure that schema location URL uses HTTPS, not HTTP. +debug "Checking that schema location URL uses HTTPS" +if grep -q http://maven.apache.org/xsd/maven-4.0.0.xsd pom.xml >/dev/null 2>/dev/null +then + echo "=====================================================================" + echo "NOTE: Your POM's schema location uses HTTP, not HTTPS. Fixing it now." + echo "=====================================================================" + sed 's;http://maven.apache.org/xsd/maven-4.0.0.xsd;https://maven.apache.org/xsd/maven-4.0.0.xsd;' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml -m 'POM: use HTTPS for schema location URL' \ + -m 'Maven no longer supports plain HTTP for the schema location.' \ + -m 'And using HTTP now generates errors in Eclipse (and probably other IDEs).' +fi + +# Check project xmlns, xmlns:xsi, and xsi:schemaLocation attributes. +debug "Checking correctness of POM project XML attributes" +grep -qF 'xmlns="http://maven.apache.org/POM/4.0.0"' pom.xml >/dev/null 2>/dev/null && + grep -qF 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' pom.xml >/dev/null 2>/dev/null && + grep -qF 'xsi:schemaLocation="http://maven.apache.org/POM/4.0.0' pom.xml >/dev/null 2>/dev/null || +{ + echo "=====================================================================" + echo "NOTE: Your POM's project attributes are incorrect. Fixing it now." + echo "=====================================================================" + sed 's;xmlns="[^"]*";xmlns="http://maven.apache.org/POM/4.0.0";' pom.xml > pom.new && + mv -f pom.new pom.xml && + sed 's;xmlns:xsi="[^"]*";xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";' pom.xml > pom.new && + mv -f pom.new pom.xml && + sed 's;xsi:schemaLocation="[^"]*";xsi:schemaLocation="http://maven.apache.org/POM/4.0.0%20https://maven.apache.org/xsd/maven-4.0.0.xsd";' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml -m 'POM: fix project attributes' \ + -m 'The XML schema for Maven POMs is located at:' \ + -m ' https://maven.apache.org/xsd/maven-4.0.0.xsd' \ + -m 'Its XML namespace is the string:' \ + -m ' http://maven.apache.org/POM/4.0.0' \ + -m 'So that exact string must be the value of xmlns. It must also +match the first half of xsi:schemaLocation, which maps that +namespace to an actual URL online where the schema resides. +Otherwise, the document is not a Maven POM.' \ + -m 'Similarly, the xmlns:xsi attribute of an XML document declaring a +particular schema should always use the string identifier:' \ + -m ' http://www.w3.org/2001/XMLSchema-instance' \ + -m "because that's the namespace identifier for instances of an XML schema." \ + -m "For details, see the specification at: https://www.w3.org/TR/xmlschema-1/" +} + +# Change forum references from forum.image.net to forum.image.sc. +debug "Checking correctness of forum URL references" +if grep -q 'https*://forum.imagej.net' pom.xml >/dev/null 2>/dev/null +then + echo "================================================================" + echo "NOTE: Your POM still references forum.imagej.net. Fixing it now." + echo "================================================================" + sed 's;https*://forum.imagej.net;https://forum.image.sc;g' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml \ + -m 'POM: fix forum.image.sc tag link' \ + -m 'The Discourse software updated the tags path from /tags/ to /tag/.' +fi + +# Ensure that references to forum.image.sc use /tag/, not /tags/. +debug "Checking correctness of forum tag references" +if grep -q forum.image.sc/tags/ pom.xml >/dev/null 2>/dev/null +then + echo "==================================================================" + echo "NOTE: Your POM has an old-style forum.image.sc tag. Fixing it now." + echo "==================================================================" + sed 's;forum.image.sc/tags/;forum.image.sc/tag/;g' pom.xml > pom.new && + mv -f pom.new pom.xml && + git commit pom.xml \ + -m 'POM: fix forum.image.sc tag link' \ + -m 'The Discourse software updated the tags path from /tags/ to /tag/.' +fi # Ensure license headers are up-to-date. test "$SKIP_LICENSE_UPDATE" -o -z "$licenseName" -o "$licenseName" = "N/A" || { + debug "Ensuring that license headers are up-to-date" mvn license:update-project-license license:update-file-header && - git add LICENSE.txt || - die 'Failed to update copyright blurbs' + git add LICENSE.txt || die 'Failed to update copyright blurbs. +You can skip the license update using the --skip-license-update flag.' no_changes_pending || die 'Copyright blurbs needed an update -- commit changes and try again. Or if the license headers are being added erroneously to certain files, exclude them by setting license.excludes in your POM; e.g.: **/script_templates/** -' + +Alternately, try again with the --skip-license-update flag.' } # Prepare new release without pushing (requires the release plugin >= 2.1). +debug "Preparing new release" $DRY_RUN mvn $BATCH_MODE release:prepare -DpushChanges=false -Dresume=false $TAG \ $PROFILE $DEV_VERSION -DreleaseVersion="$VERSION" \ - "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" && + "-Darguments=-Dgpg.skip=true ${EXTRA_ARGS# }" || + die 'The release preparation step failed -- look above for errors and fix them. +Use "mvn javadoc:javadoc | grep error" to check for javadoc syntax errors.' # Squash the maven-release-plugin's two commits into one. if test -z "$DRY_RUN" then + debug "Squashing release commits" test "[maven-release-plugin] prepare for next development iteration" = \ "$(git show -s --format=%s HEAD)" || die "maven-release-plugin's commits are unexpectedly missing!" @@ -215,6 +353,7 @@ then fi && # Extract the name of the new tag. +debug "Extracting new tag name" if test -z "$DRY_RUN" then tag=$(sed -n 's/^scm.tag=//p' < release.properties) @@ -223,14 +362,15 @@ else fi && # Rewrite the tag to include release.properties. +debug "Rewriting tag to include release.properties" test -n "$tag" && # HACK: SciJava projects use SSH (git@github.com:...) for developerConnection. # The release:perform command wants to use the developerConnection URL when # checking out the release tag. But reading from this URL requires credentials -# which we would rather Travis not need. So we replace the scm.url in the -# release.properties file to use the read-only (git://github.com/...) URL. +# which the CI system typically does not have. So we replace the scm.url in +# the release.properties file to use the public (https://github.com/...) URL. # This is OK, since release:perform does not need write access to the repo. -$DRY_RUN sed -i.bak -e 's|^scm.url=scm\\:git\\:git@github.com\\:|scm.url=scm\\:git\\:git\\://github.com/|' release.properties && +$DRY_RUN sed -i.bak -e 's|^scm.url=scm\\:git\\:git@github.com\\:|scm.url=scm\\:git\\:https\\://github.com/|' release.properties && $DRY_RUN rm release.properties.bak && $DRY_RUN git checkout "$tag" && $DRY_RUN git add -f release.properties && @@ -242,6 +382,13 @@ $DRY_RUN git checkout @{-1} && # Push the current branch and the tag. if test -z "$SKIP_PUSH" then + debug "Pushing changes" $DRY_RUN git push "$REMOTE" HEAD $tag -fi || -exit +fi + +# Remove files generated by the release process. They can end up +# committed to the mainline branch and hosing up later releases. +debug "Cleaning up" +$DRY_RUN rm -f release.properties pom.xml.releaseBackup + +debug "Release complete!" diff --git a/remote-branch-info.sh b/remote-branch-info.sh index 0228897..e41d659 100755 --- a/remote-branch-info.sh +++ b/remote-branch-info.sh @@ -24,10 +24,11 @@ esac for ref in $(git for-each-ref refs/remotes/$remote --format='%(refname)') do - + headBranch=$(git remote show "$remote" | grep HEAD | sed 's/ *HEAD branch: //') refname=${ref#refs/remotes/$remote/} - case "$refname" in contrib|master) continue;; esac - unmerged_count=$(git cherry master $ref | grep '^+' | wc -l) - info=$(git log -n 1 --format='%an - %ar' $ref) - echo $refname - $info - $unmerged_count unmerged -done + test "$refname" = "$headBranch" -o "$refname" = HEAD && continue + unmerged_count=$(git cherry "$headBranch" "$ref" | grep '^+' | wc -l) + author=$(git log -n 1 --format='%an' "$ref") + timestamp=$(git log -n 1 --format='%ar' "$ref") + echo "$refname~$author~$timestamp~$unmerged_count unmerged" +done | column -t -s '~' diff --git a/sj-version.sh b/sj-version.sh index 834e52d..a0486a3 100755 --- a/sj-version.sh +++ b/sj-version.sh @@ -15,16 +15,29 @@ props() { if [ -e "$1" ] then # extract version properties from the given file path - versions=$(cat "$1") + pomContent=$(cat "$1") else - url="$repo/org/scijava/pom-scijava/$1/pom-scijava-$1.pom" - versions=$(curl -s "$url") # assume argument is a version number of pom-scijava + pomURL="$repo/org/scijava/pom-scijava/$1/pom-scijava-$1.pom" + pomContent=$(curl -s "$pomURL") fi - echo "$versions" | \ - grep '\.version>' | \ - sed -E -e 's/^ (.*)/\1 [DEV]/' | \ - sed -E -e 's/^ *<(.*)\.version>(.*)<\/.*\.version>/\1 = \2/' | \ + + # grep the pom-scijava-base parent version of out of the POM, + # then rip out the version properties from that one as well! + psbVersion=$(echo "$pomContent" | + grep -A1 'pom-scijava-base' | + grep '' | sed 's;.*>\([^<]*\)<.*;\1;') + psbContent= + if [ "$psbVersion" ] + then + psbURL="$repo/org/scijava/pom-scijava-base/$psbVersion/pom-scijava-base-$psbVersion.pom" + psbContent=$(curl -s "$psbURL") + fi + + { echo "$pomContent"; echo "$psbContent"; } | + grep '\.version>' | + sed -E -e 's/^ (.*)/\1 [DEV]/' | + sed -E -e 's/^ *<(.*)\.version>(.*)<\/.*\.version>/\1 = \2/' | sort } diff --git a/travis-build.sh b/travis-build.sh deleted file mode 100644 index 1a759e1..0000000 --- a/travis-build.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/bin/bash - -# -# travis-build.sh - A script to build and/or release SciJava-based projects. -# - -dir="$(dirname "$0")" - -success=0 -checkSuccess() { - # Log non-zero exit code. - test $1 -eq 0 || echo "==> FAILED: EXIT CODE $1" 1>&2 - - # Record the first non-zero exit code. - test $success -eq 0 && success=$1 -} - -# Build Maven projects. -if [ -f pom.xml ] -then - echo travis_fold:start:scijava-maven - echo "= Maven build =" - echo - echo "== Configuring Maven ==" - - # NB: Suppress "Downloading/Downloaded" messages. - # See: https://stackoverflow.com/a/35653426/1207769 - export MAVEN_OPTS="$MAVEN_OPTS -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" - - # Populate the settings.xml configuration. - mkdir -p "$HOME/.m2" - settingsFile="$HOME/.m2/settings.xml" - customSettings=.travis/settings.xml - if [ -f "$customSettings" ] - then - cp "$customSettings" "$settingsFile" - else - cat >"$settingsFile" < - - - scijava.releases - travis - \${env.MAVEN_PASS} - - - scijava.snapshots - travis - \${env.MAVEN_PASS} - - - sonatype-nexus-releases - scijava-ci - \${env.OSSRH_PASS} - - -EOL - # NB: Use maven.scijava.org instead of Central if defined in repositories. - # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors - # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. - grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && - cat >>"$settingsFile" < - - scijava-mirror - SciJava mirror - https://maven.scijava.org/content/groups/public/ - central - - -EOL - cat >>"$settingsFile" < - - gpg - - - \${env.HOME}/.gnupg - - - - \${env.GPG_KEY_NAME} - \${env.GPG_PASSPHRASE} - - - - -EOL - fi - - # Determine whether deploying will be possible. - deployOK= - ciURL=$(mvn -q -Denforcer.skip=true -Dexec.executable=echo -Dexec.args='${project.ciManagement.url}' --non-recursive validate exec:exec 2>&1) - if [ $? -ne 0 ] - then - echo "No deploy -- could not extract ciManagement URL" - echo "Output of failed attempt follows:" - echo "$ciURL" - else - ciRepo=${ciURL##*/} - ciPrefix=${ciURL%/*} - ciOrg=${ciPrefix##*/} - if [ "$TRAVIS_SECURE_ENV_VARS" != true ] - then - echo "No deploy -- secure environment variables not available" - elif [ "$TRAVIS_PULL_REQUEST" != false ] - then - echo "No deploy -- pull request detected" - elif [ "$TRAVIS_REPO_SLUG" != "$ciOrg/$ciRepo" ] - then - echo "No deploy -- repository fork: $TRAVIS_REPO_SLUG != $ciOrg/$ciRepo" - # TODO: Detect travis-ci.org versus travis-ci.com? - else - echo "All checks passed for artifact deployment" - deployOK=1 - fi - fi - - # Install GPG on OSX/macOS - if [ "$TRAVIS_OS_NAME" = osx ] - then - HOMEBREW_NO_AUTO_UPDATE=1 brew install gnupg2 - fi - - # Import the GPG signing key. - keyFile=.travis/signingkey.asc - key=$1 - iv=$2 - if [ "$key" -a "$iv" -a -f "$keyFile.enc" ] - then - # NB: Key and iv values were given as arguments. - echo - echo "== Decrypting GPG keypair ==" - openssl aes-256-cbc -K "$key" -iv "$iv" -in "$keyFile.enc" -out "$keyFile" -d - checkSuccess $? - fi - if [ "$deployOK" -a -f "$keyFile" ] - then - echo - echo "== Importing GPG keypair ==" - gpg --batch --fast-import "$keyFile" - checkSuccess $? - fi - - # Run the build. - BUILD_ARGS='-B -Djdk.tls.client.protocols="TLSv1,TLSv1.1,TLSv1.2"' - if [ "$deployOK" -a "$TRAVIS_BRANCH" = master ] - then - echo - echo "== Building and deploying master SNAPSHOT ==" - mvn -Pdeploy-to-scijava $BUILD_ARGS deploy - checkSuccess $? - elif [ "$deployOK" -a -f release.properties ] - then - echo - echo "== Cutting and deploying release version ==" - mvn -B $BUILD_ARGS release:perform - checkSuccess $? - echo "== Invalidating SciJava Maven repository cache ==" - curl -fsLO https://raw.githubusercontent.com/scijava/scijava-scripts/master/maven-helper.sh && - gav=$(sh maven-helper.sh gav-from-pom pom.xml) && - ga=${gav%:*} && - echo "--> Artifact to invalidate = $ga" && - echo "machine maven.scijava.org" > "$HOME/.netrc" && - echo " login travis" >> "$HOME/.netrc" && - echo " password $MAVEN_PASS" >> "$HOME/.netrc" && - sh maven-helper.sh invalidate-cache "$ga" - checkSuccess $? - else - echo - echo "== Building the artifact locally only ==" - mvn $BUILD_ARGS install javadoc:javadoc - checkSuccess $? - fi - echo travis_fold:end:scijava-maven -fi - -# Configure conda environment, if one is needed. -if [ -f environment.yml ] -then - echo travis_fold:start:scijava-conda - echo "= Conda setup =" - - condaDir=$HOME/miniconda - condaSh=$condaDir/etc/profile.d/conda.sh - if [ ! -f "$condaSh" ]; then - echo - echo "== Installing conda ==" - if [ "$TRAVIS_PYTHON_VERSION" = "2.7" ]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - fi - rm -rf "$condaDir" - bash miniconda.sh -b -p "$condaDir" - checkSuccess $? - fi - - echo - echo "== Updating conda ==" - . "$condaSh" && - conda config --set always_yes yes --set changeps1 no && - conda update -q conda && - conda info -a - checkSuccess $? - - echo - echo "== Configuring environment ==" - condaEnv=travis-scijava - test -d "$condaDir/envs/$condaEnv" && condaAction=update || condaAction=create - conda env "$condaAction" -n "$condaEnv" -f environment.yml && - conda activate "$condaEnv" - checkSuccess $? - - echo travis_fold:end:scijava-conda -fi - -# Execute Jupyter notebooks. -if which jupyter >/dev/null 2>/dev/null -then - echo travis_fold:start:scijava-jupyter - echo "= Jupyter notebooks =" - # NB: This part is fiddly. We want to loop over files even with spaces, - # so we use the "find ... -print0 | while read $'\0' ..." idiom. - # However, that runs the piped expression in a subshell, which means - # that any updates to the success variable will not persist outside - # the loop. So we suppress all stdout inside the loop, echoing only - # the final value of success upon completion, and then capture the - # echoed value back into the parent shell's success variable. - success=$(find . -name '*.ipynb' -print0 | { - while read -d $'\0' nbf - do - echo 1>&2 - echo "== $nbf ==" 1>&2 - jupyter nbconvert --execute --stdout "$nbf" >/dev/null - checkSuccess $? - done - echo $success - }) - echo travis_fold:end:scijava-jupyter -fi - -exit $success diff --git a/travis-javadoc.sh b/travis-javadoc.sh deleted file mode 100644 index 7badbcb..0000000 --- a/travis-javadoc.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - -# -# travis-javadoc.sh - A script to build the javadocs of a SciJava-based project. -# - -# The following repositories are known to use this script: -# -# bonej-org/bonej-javadoc -# fiji/fiji-javadoc -# imagej/imagej-javadoc -# imglib/imglib2-javadoc -# scifio/scifio-javadoc -# scijava/java3d-javadoc -# scijava/scijava-javadoc -# slim-curve/slim-javadoc -# uw-loci/loci-javadoc - -# Wait for a launched background command to complete, emitting -# an occasional message to avoid long periods without output. -# Return the same exit code as the launched command. -keep_alive() { - pid="$1" - if [ "$pid" = "" ] - then - echo "[ERROR] No PID given" - return - fi - i=0 - while kill -0 "$pid" 2>/dev/null; do - i=$((i+1)) - m=$((i/60)) - s=$((i%60)) - test $s -eq 0 && echo "[$m minutes elapsed]" - sleep 1 - done - wait "$pid" -} - -if [ "$TRAVIS_SECURE_ENV_VARS" = true \ - -a "$TRAVIS_PULL_REQUEST" = false \ - -a "$TRAVIS_BRANCH" = master ] -then - project=$1 - openssl_key=$2 - openssl_iv=$3 - - # Populate the settings.xml configuration. - mkdir -p "$HOME/.m2" - settingsFile="$HOME/.m2/settings.xml" - customSettings=.travis/settings.xml - if [ -f "$customSettings" ] - then - cp "$customSettings" "$settingsFile" - else - # NB: Use maven.scijava.org as sole mirror if defined in . - # This hopefully avoids intermittent "ReasonPhrase:Forbidden" errors - # when the Travis build pings Maven Central; see travis-ci/travis-ci#6593. - test -f pom.xml && grep -A 2 '' pom.xml | grep -q 'maven.scijava.org' && - cat >"$settingsFile" < - - - scijava-mirror - SciJava mirror - https://maven.scijava.org/content/groups/public/ - * - - - -EOL - fi - - # Emit some details useful for debugging. - # NB: We run once with -q to suppress the download messages, - # then again without it to emit the desired dependency tree. - mvn -B -q dependency:tree && - mvn -B dependency:tree && - - echo && - echo "== Generating javadoc ==" && - - # Build the javadocs. - (mvn -B -q -Pbuild-javadoc) & - keep_alive $! && - test -d target/apidocs && - # Strip out date stamps, to avoid spurious changes being committed. - sed -i'' -e '/\(\n\t\tdeploy-to-scijava<\/releaseProfiles>\1/igs' "$tmpFile" - update pom.xml 'POM: deploy releases to the SciJava repository' - fi - - # update the README - if grep -q "travis-ci\.[a-zA-Z0-9/_-]*\.svg" README.md >/dev/null 2>&1 - then - info "Updating README.md Travis badge" - sed "s|travis-ci\.[a-zA-Z0-9/_-]*|$domain/$repoSlug|g" README.md >"$tmpFile" - update README.md 'Travis: fix README.md badge link' - else - info "Adding Travis badge to README.md" - echo "[![](https://$domain/$repoSlug.svg?branch=master)](https://$domain/$repoSlug)" >"$tmpFile" - echo >>"$tmpFile" - test -f README.md && cat README.md >>"$tmpFile" - update README.md 'Travis: add badge to README.md' - fi - - # encrypt key/value pairs in variables file - if [ -f "$varsFile" ] - then - while read p; do - # Skip comments. (Cannot use ${p:0:1} because it's bash-specific.) - case "$p" in - '#'*) continue;; - esac - info "Encrypting ${p%%=*}" - yes | $EXEC travis encrypt --$mode "$p" --add env.global --repo "$repoSlug" - test $? -eq 0 || die "Failed to encrypt variable '$p'" - done <"$varsFile" - $EXEC git commit "$travisConfig" -m "Travis: add encrypted environment variables" - else - warn "No $varsFile found. Travis will not have any environment variables set!" - fi - - # encrypt GPG keypair - if [ -f "$signingKeySourceFile" ] - then - info "Encrypting $signingKeyDestFile" - if [ -z "$EXEC" ] - then - rm -f "$signingKeyDestFile.enc" - encryptOutput=$(travis encrypt-file --$mode "$signingKeySourceFile" "$signingKeyDestFile.enc" --repo "$repoSlug") - test $? -eq 0 || die "Failed to encrypt signing key." - encryptResult=$(echo "$encryptOutput" | grep openssl) - test "$encryptResult" || die "No openssl variables emitted." - key=$(echo "$encryptResult" | cut -d' ' -f4) - iv=$(echo "$encryptResult" | cut -d' ' -f6) - sed -i.bak "s/\(sh travis-build.sh\)/\1 $key $iv/" "$travisBuildScript" - rm -f "$travisBuildScript.bak" - git add "$travisBuildScript" "$signingKeyDestFile.enc" - git commit -m "Travis: add encrypted GPG signing keypair" - fi - else - warn "No $signingKeySourceFile found. Travis will not be able to do GPG signing!" - fi -} - -test -d "$credentialsDir" || - die "This script requires configuration stored in $credentialsDir,\n" \ - "including $varsFile for needed environment variables,\n" \ - "and $signingKeySourceFile for signing of artifacts.\n" \ - "Please contact a SciJava administrator to receive a copy of this content." - -# check prerequisites -check git sed cut perl xmllint travis - -# parse arguments -EXEC=: -SKIP_PARENT_CHECK= -while test $# -gt 0 -do - case "$1" in - -f) EXEC=;; - -p) SKIP_PARENT_CHECK=true;; - --) break;; - -*) echo "Ignoring unknown option: $1" >&2; break;; - *) break;; - esac - shift -done - -test "$EXEC" && warn "Simulation only. Run with -f flag to go for real." - -# process arguments -if [ $# -gt 0 ] -then - for d in $@ - do ( - echo "[$d]" - process "$d" - ) done -else - process . -fi