Merge pull request #126 from framps/other_zip_tools

Support to compress final pishrink image by using parallel compression feature of gzip and xz

README.md

@@ -1,24 +1,32 @@
 
 # PiShrink #
 PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better.
+In addition the shrinked image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image
+using multiple cores is supported.
 
 ## Usage ##
 ```
-sudo pishrink.sh [-sdrzh] imagefile.img [newimagefile.img]
+Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img]
-  -s: Skip autoexpand
+
-  -d: Debug mode on
+  -s         Don't expand filesystem when image is booted the first time
-  -r: Use advanced repair options
+  -v         Be verbose
-  -z: Gzip compress image after shrinking
+  -r         Use advanced filesystem repair option if the normal one fails
-  -h: display help text
+  -z         Compress image after shrinking with gzip
+  -Z         Compress image after shrinking with xz
+  -a         Compress image in parallel using multiple cores
+  -p         Remove logs, apt archives, dhcp leases and ssh hostkeys
+  -d         Write debug messages in a debug log file
 ```
 
 If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option.
 
-* `-s` will skip the autoexpanding part of the process.
-* `-d` will create a logfile `pishrink.log` which may help for problem analysis.
 * `-r` will attempt to repair the filesystem if regular repairs fail
-* `-z` will Gzip compress the image after shrinking. The `.gz` extension will be added to the filename.
+* `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename.
+* `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename.
+* `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel.
+* `-d` will create a logfile `pishrink.log` which may help for problem analysis.
 
+Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz.
 
 ## Prerequisites ##
 If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future.

pishrink.sh

@@ -2,9 +2,15 @@
 
 version="v0.1.2"
 
-CURRENT_DIR=$(pwd)
+CURRENT_DIR="$(pwd)"
 SCRIPTNAME="${0##*/}"
-LOGFILE=${CURRENT_DIR}/${SCRIPTNAME%.*}.log
+MYNAME="${SCRIPTNAME%.*}"
+LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log"
+REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs"
+ZIPTOOLS=("gzip xz")
+declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # parallel zip tool to use in parallel mode
+declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode
+declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files
 
 function info() {
 	echo "$SCRIPTNAME: $1 ..."
@@ -63,45 +69,42 @@ fi
 help() {
 	local help
 	read -r -d '' help << EOM
-Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img]
+Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img]
 
-  -s: Don't expand filesystem when image is booted the first time
+  -s         Don't expand filesystem when image is booted the first time
-  -d: Write debug messages in a debug log file
+  -v			 Be verbose
-  -r: Use advanced filesystem repair option if the normal one fails
+  -r         Use advanced filesystem repair option if the normal one fails
-  -p: Remove logs, apt archives, dhcp leases and ssh hostkeys
+  -z         Compress image after shrinking with gzip
-  -z: Gzip compress image after shrinking
+  -Z         Compress image after shrinking with xz
+  -a         Compress image in parallel using multiple cores
+  -p         Remove logs, apt archives, dhcp leases and ssh hostkeys
+  -d         Write debug messages in a debug log file
 EOM
 	echo "$help"
 	exit -1
 }
 
-usage() {
-	echo "Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img]"
-	echo ""
-	echo "  -s: Skip autoexpand"
-	echo "  -d: Debug mode on"
-	echo "  -r: Use advanced repair options"
-	echo "  -p: Remove logs, apt archives, dhcp leases and ssh hostkeys"
-	echo "  -z: Gzip compress image after shrinking"
-	echo "  -h: display help text"
-	exit -1
-}
-
 should_skip_autoexpand=false
 debug=false
 repair=false
-gzip_compress=false
+parallel=false
+verbose=false
 prep=false
+ziptool=""
+required_tools="$REQUIRED_TOOLS"
 
-while getopts ":sdrpzh" opt; do
+while getopts ":adhprsvzZ" opt; do
   case "${opt}" in
-    s) should_skip_autoexpand=true ;;
+    a) parallel=true;;
     d) debug=true;;
-    r) repair=true;;
-    p) prep=true;;
-    z) gzip_compress=true;;
     h) help;;
-    *) usage ;;
+    p) prep=true;;
+    r) repair=true;;
+    s) should_skip_autoexpand=true ;;
+    v) verbose=true;;
+    z) ziptool="gzip";;
+    Z) ziptool="xz";;
+    *) help;;
   esac
 done
 shift $((OPTIND-1))
@@ -121,8 +124,9 @@ img="$1"
 
 #Usage checks
 if [[ -z "$img" ]]; then
-  usage
+  help
 fi
+
 if [[ ! -f "$img" ]]; then
   error $LINENO "$img is not a file..."
   exit -2
@@ -132,8 +136,22 @@ if (( EUID != 0 )); then
   exit -3
 fi
 
+# check selected compression tool is supported and installed
+if [[ -n $ziptool ]]; then
+	if [[ ! " ${ZIPTOOLS[@]} " =~ " $ziptool " ]]; then
+		error $LINENO "$ziptool is an unsupported ziptool."
+		exit -17
+	else
+		if [[ $parallel == true && ziptool == "gzip" ]]; then
+			REQUIRED_TOOLS="$REQUIRED_TOOLS pigz"
+		else
+			REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool"
+		fi
+	fi
+fi
+
 #Check that what we need is installed
-for command in parted losetup tune2fs md5sum e2fsck resize2fs; do
+for command in $REQUIRED_TOOLS; do
   command -v $command >/dev/null 2>&1
   if (( $? != 0 )); then
     error $LINENO "$command is not installed."
@@ -159,17 +177,16 @@ trap cleanup ERR EXIT
 
 #Gather info
 info "Gathering data"
-beforesize=$(ls -lh "$img" | cut -d ' ' -f 5)
+beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)"
-parted_output=$(parted -ms "$img" unit B print | tail -n 1)
+parted_output="$(parted -ms "$img" unit B print | tail -n 1)"
-partnum=$(echo "$parted_output" | cut -d ':' -f 1)
+partnum="$(echo "$parted_output" | cut -d ':' -f 1)"
-partstart=$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B')
+partstart="$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B')"
-loopback=$(losetup -f --show -o "$partstart" "$img")
+loopback="$(losetup -f --show -o "$partstart" "$img")"
-tune2fs_output=$(tune2fs -l "$loopback")
+tune2fs_output="$(tune2fs -l "$loopback")"
-currentsize=$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)
+currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)"
-blocksize=$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)
+blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)"
-
-logVariables $LINENO tune2fs_output currentsize blocksize
 
+logVariables $LINENO beforesize parted_output partnum partstart tune2fs_output currentsize blocksize
 #Check if we should make pi expand rootfs on next boot
 if [ "$should_skip_autoexpand" = false ]; then
   #Make pi expand rootfs on next boot
@@ -180,6 +197,7 @@ if [ "$should_skip_autoexpand" = false ]; then
     echo "Creating new /etc/rc.local"
     mv "$mountdir/etc/rc.local" "$mountdir/etc/rc.local.bak"
     #####Do not touch the following lines#####
+
 cat <<\EOF1 > "$mountdir/etc/rc.local"
 #!/bin/bash
 do_expand_rootfs() {
@@ -264,7 +282,7 @@ if ! minsize=$(resize2fs -P "$loopback"); then
 	exit -10
 fi
 minsize=$(cut -d ':' -f 2 <<< "$minsize" | tr -d ' ')
-logVariables $LINENO minsize
+logVariables $LINENO currentsize minsize
 if [[ $currentsize -eq $minsize ]]; then
   error $LINENO "Image already shrunk to smallest size"
   exit -11
@@ -326,11 +344,32 @@ if ! truncate -s "$endresult" "$img"; then
 	exit -16
 fi
 
-if [[ $gzip_compress == true ]]; then
+# handle compression
-    info "Gzipping the shrunk image"
+if [[ -n $ziptool ]]; then
-    if [[ ! $(gzip -f9 "$img") ]]; then
+	options=""
-        img=$img.gz
+	envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override all options for gzip or xz
+	[[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}"
+	[[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options
+	[[ $verbose == true ]] && options="$options -v" # add verbose flag if requested
+	
+	if [[ $parallel == true ]]; then
+		parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}"
+		info "Using $parallel_tool on the shrunk image"
+		if ! $parallel_tool ${options} "$img"; then
+			rc=$?
+			error $LINENO "$parallel_tool failed with rc $rc"
+			exit -18
 		fi
+		
+	else # sequential
+		info "Using $ziptool on the shrunk image"
+		if ! $ziptool ${options} $img; then
+			rc=$?
+			error $LINENO "$ziptool failed with rc $rc"
+			exit -19
+		fi
+	fi
+	img=$img.${ZIPEXTENSIONS[$ziptool]}
 fi
 
 aftersize=$(ls -lh "$img" | cut -d ' ' -f 5)