Update download script to add parallelism, correct locking, and specific version support. (jenkinsci#291)

Update download script to add parallelism, correct locking, and specific version support.

Add support for provided plugins.

Author: apottere

install-plugins.sh

@@ -1,84 +1,138 @@
-#! /bin/bash
+#!/bin/bash
 
 # Resolve dependencies and download plugins given on the command line
 #
 # FROM jenkins
 # RUN install-plugins.sh docker-slaves github-branch-source
 
-set -e
+REF_DIR=${REF:-/usr/share/jenkins/ref/plugins}
+FAILED="$REF_DIR/failed-plugins.txt"
 
-REF=${REF:-/usr/share/jenkins/ref/plugins}
-mkdir -p "$REF"
-
-function download() {
-	local plugin="$1"; shift
-
-	if [[ ! -f "${plugin}.hpi" ]]; then
+function getLockFile() {
+	echo -n "$REF_DIR/${1}.lock"
+}
 
-		local url="${JENKINS_UC}/latest/${plugin}.hpi"
-		echo "download plugin : $plugin from $url"
+function getHpiFilename() {
+	echo -n "$REF_DIR/${1}.hpi"
+}
 
-		if ! curl -s -f -L "$url" -o "${plugin}.hpi" 
-		then
+function download() {
+	local plugin originalPlugin version lock ignoreLockFile
+	plugin="$1"
+	version="${2:-latest}"
+	ignoreLockFile="$3"
+	lock="$(getLockFile "$plugin")"
+
+	if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then
+		if ! doDownload "$plugin" "$version"; then
 			# some plugin don't follow the rules about artifact ID
 			# typically: docker-plugin
-			plugin=${plugin}-plugin
-
-			local url="${JENKINS_UC}/latest/${plugin}.hpi"
-			echo "download plugin : $plugin from $url"
-			if ! curl -s -f -L "${url}" -o "${plugin}.hpi"
-			then
-				>&2 echo "failed to download plugin ${plugin}"
-				exit -1
+			originalPlugin="$plugin"
+			plugin="${plugin}-plugin"
+			if ! doDownload "$plugin" "$version"; then
+				echo "Failed to download plugin: $originalPlugin or $plugin" >&2
+				echo "Not downloaded: ${originalPlugin}" >> "$FAILED"
+				return 1
 			fi
 		fi
-	else
-		echo "$plugin is already downloaded."
-	fi	
 
-	if [[ ! -f ${plugin}.resolved ]]; then
+		if ! checkIntegrity "$plugin"; then
+			echo "Downloaded file is not a valid ZIP: $(getHpiFilename "$plugin")" >&2
+			echo "Download integrity: ${plugin}" >> "$FAILED"
+			return 1
+		fi
+
 		resolveDependencies "$plugin"
 	fi
 }
 
+function doDownload() {
+	local plugin version url hpi
+	plugin="$1"
+	version="$2"
+	hpi="$(getHpiFilename "$plugin")"
+
+	if [[ -f $hpi ]]; then
+		echo "Using provided plugin: $plugin"
+		return 0
+	fi
+
+	url="$JENKINS_UC/download/plugins/$plugin/$version/${plugin}.hpi"
+
+	echo "Downloading plugin: $plugin from $url"
+	curl -s -f -L "$url" -o "$hpi"
+	return $?
+}
+
+function checkIntegrity() {
+	local plugin hpi
+	plugin="$1"
+	hpi="$(getHpiFilename "$plugin")"
+
+	zip -T "$hpi" >/dev/null
+	return $?
+}
+
 function resolveDependencies() {	
-	local plugin="$1"; shift
-
-	local dependencies=`jrunscript -e '\
-	java.lang.System.out.println(\
-		new java.util.jar.JarFile("'${plugin}.hpi'")\
-			.getManifest()\
-			.getMainAttributes()\
-			.getValue("Plugin-Dependencies")\
-	);'`
-
-	if [[ "$dependencies" == "null" ]]; then
-		echo " > plugin has no dependencies"
+	local plugin hpi dependencies
+	plugin="$1"
+	hpi="$(getHpiFilename "$plugin")"
+
+	# ^M below is a control character, inserted by typing ctrl+v ctrl+m
+	dependencies="$(unzip -p "$hpi" META-INF/MANIFEST.MF | sed -e 's###g' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')"
+
+	if [[ ! $dependencies ]]; then
+		echo " > $plugin has no dependencies"
 		return
 	fi
 
-	echo " > depends on  ${dependencies}"
+	echo " > $plugin depends on $dependencies"
 
-	IFS=',' read -a array <<< "${dependencies}"
-    for d in "${array[@]}"
+	IFS=',' read -a array <<< "$dependencies"
+
+	for d in "${array[@]}"
 	do
-		local p=$(echo $d | cut -d':' -f1 -)
-		if [[ $d == *"resolution:=optional"* ]] 
-		then	
-			echo "skipping optional dependency $p"
+		plugin="$(cut -d':' -f1 - <<< "$d")"
+		if [[ $d == *"resolution:=optional"* ]]; then	
+			echo "Skipping optional dependency $plugin"
 		else
-			download "$p"
+			download "$plugin" &
 		fi
 	done
-	touch "${plugin}.resolved"
+	wait
 }
 
-cd "$REF"
+main() {
+	local plugin version
+
+	mkdir -p "$REF_DIR" || exit 1
+
+	# Create lockfile manually before first run to make sure any explicit version set is used.
+	echo "Creating initial locks..."
+	for plugin in "$@"; do
+		mkdir "$(getLockFile "${plugin%%:*}")"
+	done
 
-for plugin in "$@"
-do
-    download "$plugin"
-done
+	echo -e "\nDownloading plugins..."
+	for plugin in "$@"; do
+		version=""
+
+		if [[ $plugin =~ .*:.* ]]; then
+			version="${plugin##*:}"
+			plugin="${plugin%%:*}"
+		fi
+
+		download "$plugin" "$version" "true" &
+	done				  
+	wait
+
+	if [[ -f $FAILED ]]; then
+		echo -e "\nSome plugins failed to download!\n$(<"$FAILED")" >&2
+		exit 1
+	fi
+
+	echo -e "\nCleaning up locks..."
+	rm -rv "$REF_DIR"/*.lock
+}
 
-# cleanup 'resolved' flag files
-rm -f *.resolved
+main "$@"