Fixes for rc.local issues

- Display warning when no existing /etc/rc.local is found
- Only try to run old /etc/rc.local if it actually exists

Closes #151
Closes #267

README.md

@@ -1,13 +1,57 @@
+
 # PiShrink #
+
 PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better.
+In addition the shrunk image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image
+using multiple cores is supported.
 
-`Usage: ./pishrink [-s] imagefile.img [newimagefile.img]`
+## Usage ##
 
-If the `-s` option is given the script will skip the autoexpanding part of the process.  If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option.
+```
+Usage: $0 [-adhrsvzZ] imagefile.img [newimagefile.img]
+
+  -s         Don't expand filesystem when image is booted the first time
+  -v         Be verbose
+  -r         Use advanced filesystem repair option if the normal one fails
+  -z         Compress image after shrinking with gzip
+  -Z         Compress image after shrinking with xz
+  -a         Compress image in parallel using multiple cores
+  -d         Write debug messages in a debug log file
+```
+
+If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option.
+
+* `-s` prevents automatic filesystem expansion on the images next boot
+* `-v` enables more verbose output
+* `-r` will attempt to repair the filesystem using additional options if the normal repair fails
+* `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename.
+* `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename.
+* `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel.
+* `-d` will create a logfile `pishrink.log` which may help for problem analysis.
+
+Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz.
+
+## Prerequisites ##
+
+If you are running PiShrink in VirtualBox you will likely encounter an error if you
+attempt to use VirtualBox's "Shared Folder" feature. You can copy the image you wish to
+shrink on to the VM from a Shared Folder, but shrinking directctly from the Shared Folder
+is know to cause issues.
+
+If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run.
+
+## Installation ##
+
+```bash
+wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh
+chmod +x pishrink.sh
+sudo mv pishrink.sh /usr/local/bin
+```
 
 ## Example ##
+
 ```bash
-[user@localhost PiShrink]$ sudo ./shrink.sh pi.img
+[user@localhost PiShrink]$ sudo pishrink.sh pi.img
 e2fsck 1.42.9 (28-Dec-2013)
 Pass 1: Checking inodes, blocks, and sizes
 Pass 2: Checking directory structure
@@ -30,6 +74,7 @@ Shrunk pi.img from 30G to 3.1G
 ```
 
 ## Contributing ##
+
 If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it.
 
 Pull requests for new features and bug fixes are more than welcome!

pishrink.sh

@@ -1,81 +1,113 @@
 #!/bin/bash
 
-usage() { echo "Usage: $0 [-s] imagefile.img [newimagefile.img]"; exit -1; }
+version="v0.1.4"
 
-should_skip_autoexpand=false
+CURRENT_DIR="$(pwd)"
+SCRIPTNAME="${0##*/}"
+MYNAME="${SCRIPTNAME%.*}"
+LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log"
+REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs"
+ZIPTOOLS=("gzip xz")
+declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # parallel zip tool to use in parallel mode
+declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode
+declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files
 
-while getopts ":s" opt; do
-  case "${opt}" in
-    s) should_skip_autoexpand=true ;;
-    *) usage ;;
-  esac
+function info() {
+	echo "$SCRIPTNAME: $1 ..."
+}
+
+function error() {
+	echo -n "$SCRIPTNAME: ERROR occurred in line $1: "
+	shift
+	echo "$@"
+}
+
+function cleanup() {
+	if losetup "$loopback" &>/dev/null; then
+		losetup -d "$loopback"
+	fi
+	if [ "$debug" = true ]; then
+		local old_owner=$(stat -c %u:%g "$src")
+		chown "$old_owner" "$LOGFILE"
+	fi
+
+}
+
+function logVariables() {
+	if [ "$debug" = true ]; then
+		echo "Line $1" >> "$LOGFILE"
+		shift
+		local v var
+		for var in "$@"; do
+			eval "v=\$$var"
+			echo "$var: $v" >> "$LOGFILE"
 		done
-shift $((OPTIND-1))
-
-#Args
-img=$1
-
-#Usage checks
-if [[ -z $img ]]; then
-  usage
-fi
-if [[ ! -e $img ]]; then
-  echo "ERROR: $img is not a file..."
-  exit -2
-fi
-if (( EUID != 0 )); then
-  echo "ERROR: You need to be running as root."
-  exit -3
 	fi
+}
 
-#Check that what we need is installed
-A=`which parted 2>&1`
-if (( $? != 0 )); then
-  echo "ERROR: parted is not installed."
-  exit -4
-fi
+function checkFilesystem() {
+	info "Checking filesystem"
+	e2fsck -pf "$loopback"
+	(( $? < 4 )) && return
 
-#Copy to new file if requested
-if [ -n "$2" ]; then
-  echo "Copying $1 to $2..."
-  cp --reflink=auto --sparse=always "$1" "$2"
-  if (( $? != 0 )); then
-    echo "ERROR: Could not copy file..."
-    exit -5
-  fi
-  img=$2
-fi
+	info "Filesystem error detected!"
 
-#Gather info
-beforesize=`ls -lah $img | cut -d ' ' -f 5`
-partnum=`parted -m $img unit B print | tail -n 1 | cut -d ':' -f 1 | tr -d '\n'`
-partstart=`parted -m $img unit B print | tail -n 1 | cut -d ':' -f 2 | tr -d 'B\n'`
-loopback=`losetup -f --show -o $partstart $img`
-currentsize=`tune2fs -l $loopback | grep 'Block count' | tr -d ' ' | cut -d ':' -f 2 | tr -d '\n'`
-blocksize=`tune2fs -l $loopback | grep 'Block size' | tr -d ' ' | cut -d ':' -f 2 | tr -d '\n'`
+	info "Trying to recover corrupted filesystem"
+	e2fsck -y "$loopback"
+	(( $? < 4 )) && return
 
-#Check if we should make pi expand rootfs on next boot
-if [ "$should_skip_autoexpand" = false ]; then
+if [[ $repair == true ]]; then
+	info "Trying to recover corrupted filesystem - Phase 2"
+	e2fsck -fy -b 32768 "$loopback"
+	(( $? < 4 )) && return
+fi
+	error $LINENO "Filesystem recoveries failed. Giving up..."
+	exit 9
+
+}
+
+function set_autoexpand() {
     #Make pi expand rootfs on next boot
-  mountdir=`mktemp -d`
-  mount $loopback $mountdir
+    mountdir=$(mktemp -d)
+    partprobe "$loopback"
+    sleep 3
+    umount "$loopback" > /dev/null 2>&1
+    mount "$loopback" "$mountdir" -o rw
+    if (( $? != 0 )); then
+      info "Unable to mount loopback, autoexpand will not be enabled"
+      return
+    fi
 
-  if [ `md5sum $mountdir/etc/rc.local | cut -d ' ' -f 1` != "c4eb22d9aa99915af319287d194d4529" ]; then
-    echo Creating new /etc/rc.local
-    mv $mountdir/etc/rc.local $mountdir/etc/rc.local.bak
-    ###Do not touch the following 6 lines including EOF###
-cat <<\EOFE > $mountdir/etc/rc.local
+    if [ ! -d "$mountdir/etc" ]; then
+        info "/etc not found, autoexpand will not be enabled"
+        umount "$mountdir"
+        return
+    fi
+
+    if [[ ! -f "$mountdir/etc/rc.local" ]]; then
+        info "An existing /etc/rc.local was not found, autoexpand may fail..."
+    fi
+
+    if [[ -f "$mountdir/etc/rc.local" ]] && [[ "$(md5sum "$mountdir/etc/rc.local" | cut -d ' ' -f 1)" != "5c286b336c0606ed8e6f87708f7802eb" ]]; then
+      echo "Creating new /etc/rc.local"
+    if [ -f "$mountdir/etc/rc.local" ]; then
+        mv "$mountdir/etc/rc.local" "$mountdir/etc/rc.local.bak"
+    fi
+
+    #####Do not touch the following lines#####
+cat <<\EOF1 > "$mountdir/etc/rc.local"
 #!/bin/bash
 do_expand_rootfs() {
-  ROOT_PART=$(cat /proc/cmdline | tr -s ' ' '\n' | grep root= | cut -d '=' -f 2)
-  if [[ "$ROOT_PART" != /dev/mmcblk0* ]] ; then
-    echo $ROOT_PART is not an SD card...
+  ROOT_PART=$(mount | sed -n 's|^/dev/\(.*\) on / .*|\1|p')
+
+  PART_NUM=${ROOT_PART#mmcblk0p}
+  if [ "$PART_NUM" = "$ROOT_PART" ]; then
+    echo "$ROOT_PART is not an SD card. Don't know how to expand"
     return 0
   fi
-  PART_NUM=$(echo $ROOT_PART | cut -d 'p' -f 2)
 
   # Get the starting offset of the root partition
-  PART_START=$(fdisk -l /dev/mmcblk0 | grep $ROOT_PART | tr -s ' ' | cut -d ' ' -f 2)
+  PART_START=$(parted /dev/mmcblk0 -ms unit s p | grep "^${PART_NUM}" | cut -f 2 -d: | sed 's/[^0-9]//g')
   [ "$PART_START" ] || return 1
   # Return value will likely be error for fdisk as it fails to reload the
   # partition table because the root fs is mounted
@@ -88,70 +120,310 @@ p
 $PART_NUM
 $PART_START
 
+p
 w
-
 EOF
 
 cat <<EOF > /etc/rc.local &&
 #!/bin/sh
-resize2fs $ROOT_PART
-rm -f /etc/rc.local; cp -f /etc/rc.local.bak /etc/rc.local; /etc/rc.local
+echo "Expanding /dev/$ROOT_PART"
+resize2fs /dev/$ROOT_PART
+rm -f /etc/rc.local; cp -fp /etc/rc.local.bak /etc/rc.local && /etc/rc.local
 
 EOF
 reboot
 exit
 }
-do_expand_rootfs
-rm -f /etc/rc.local; cp -f /etc/rc.local.bak /etc/rc.local; /etc/rc.local
-exit 0
-EOFE
-    ###End no touch zone###
-    chmod +x $mountdir/etc/rc.local
-  fi
-  umount $mountdir
+raspi_config_expand() {
+/usr/bin/env raspi-config --expand-rootfs
+if [[ $? != 0 ]]; then
+  return -1
 else
-  echo Skipping autoexpanding process...
+  rm -f /etc/rc.local; cp -fp /etc/rc.local.bak /etc/rc.local && /etc/rc.local
+  reboot
+  exit
+fi
+}
+raspi_config_expand
+echo "WARNING: Using backup expand..."
+sleep 5
+do_expand_rootfs
+echo "ERROR: Expanding failed..."
+sleep 5
+if [[ -f /etc/rc.local.bak ]]; then
+  cp -fp /etc/rc.local.bak /etc/rc.local
+  /etc/rc.local
+fi
+exit 0
+EOF1
+    #####End no touch zone#####
+    chmod +x "$mountdir/etc/rc.local"
+    fi
+    umount "$mountdir"
+}
+
+help() {
+	local help
+	read -r -d '' help << EOM
+Usage: $0 [-adhrsvzZ] imagefile.img [newimagefile.img]
+
+  -s         Don't expand filesystem when image is booted the first time
+  -v         Be verbose
+  -r         Use advanced filesystem repair option if the normal one fails
+  -z         Compress image after shrinking with gzip
+  -Z         Compress image after shrinking with xz
+  -a         Compress image in parallel using multiple cores
+  -d         Write debug messages in a debug log file
+EOM
+	echo "$help"
+	exit 1
+}
+
+should_skip_autoexpand=false
+debug=false
+repair=false
+parallel=false
+verbose=false
+ziptool=""
+
+while getopts ":adhrsvzZ" opt; do
+  case "${opt}" in
+    a) parallel=true;;
+    d) debug=true;;
+    h) help;;
+    r) repair=true;;
+    s) should_skip_autoexpand=true ;;
+    v) verbose=true;;
+    z) ziptool="gzip";;
+    Z) ziptool="xz";;
+    *) help;;
+  esac
+done
+shift $((OPTIND-1))
+
+if [ "$debug" = true ]; then
+	info "Creating log file $LOGFILE"
+	rm "$LOGFILE" &>/dev/null
+	exec 1> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&1)
+	exec 2> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&2)
+fi
+
+echo "${0##*/} $version"
+
+#Args
+src="$1"
+img="$1"
+
+#Usage checks
+if [[ -z "$img" ]]; then
+  help
+fi
+
+if [[ ! -f "$img" ]]; then
+  error $LINENO "$img is not a file..."
+  exit 2
+fi
+if (( EUID != 0 )); then
+  error $LINENO "You need to be running as root."
+  exit 3
+fi
+
+# set locale to POSIX(English) temporarily
+# these locale settings only affect the script and its sub processes
+
+export LANGUAGE=POSIX
+export LC_ALL=POSIX
+export LANG=POSIX
+
+
+# check selected compression tool is supported and installed
+if [[ -n $ziptool ]]; then
+	if [[ ! " ${ZIPTOOLS[@]} " =~ $ziptool ]]; then
+		error $LINENO "$ziptool is an unsupported ziptool."
+		exit 17
+	else
+		if [[ $parallel == true && $ziptool == "gzip" ]]; then
+			REQUIRED_TOOLS="$REQUIRED_TOOLS pigz"
+		else
+			REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool"
+		fi
+	fi
+fi
+
+#Check that what we need is installed
+for command in $REQUIRED_TOOLS; do
+  command -v $command >/dev/null 2>&1
+  if (( $? != 0 )); then
+    error $LINENO "$command is not installed."
+    exit 4
+  fi
+done
+
+#Copy to new file if requested
+if [ -n "$2" ]; then
+  f="$2"
+  if [[ -n $ziptool && "${f##*.}" == "${ZIPEXTENSIONS[$ziptool]}" ]]; then	# remove zip extension if zip requested because zip tool will complain about extension
+    f="${f%.*}"
+  fi
+  info "Copying $1 to $f..."
+  cp --reflink=auto --sparse=always "$1" "$f"
+  if (( $? != 0 )); then
+    error $LINENO "Could not copy file..."
+    exit 5
+  fi
+  old_owner=$(stat -c %u:%g "$1")
+  chown "$old_owner" "$f"
+  img="$f"
+fi
+
+# cleanup at script exit
+trap cleanup EXIT
+
+#Gather info
+info "Gathering data"
+beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)"
+parted_output="$(parted -ms "$img" unit B print)"
+rc=$?
+if (( $rc )); then
+	error $LINENO "parted failed with rc $rc"
+	info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate"
+	exit 6
+fi
+partnum="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 1)"
+partstart="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 2 | tr -d 'B')"
+if [ -z "$(parted -s "$img" unit B print | grep "$partstart" | grep logical)" ]; then
+    parttype="primary"
+else
+    parttype="logical"
+fi
+loopback="$(losetup -f --show -o "$partstart" "$img")"
+tune2fs_output="$(tune2fs -l "$loopback")"
+rc=$?
+if (( $rc )); then
+    echo "$tune2fs_output"
+    error $LINENO "tune2fs failed. Unable to shrink this type of image"
+    exit 7
+fi
+
+currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)"
+blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)"
+
+logVariables $LINENO beforesize parted_output partnum partstart parttype tune2fs_output currentsize blocksize
+
+#Check if we should make pi expand rootfs on next boot
+if [ "$parttype" == "logical" ]; then
+  echo "WARNING: PiShrink does not yet support autoexpanding of this type of image"
+elif [ "$should_skip_autoexpand" = false ]; then
+  set_autoexpand
+else
+  echo "Skipping autoexpanding process..."
 fi
 
 #Make sure filesystem is ok
-e2fsck -f $loopback
-minsize=`resize2fs -P $loopback | cut -d ':' -f 2 | tr -d ' ' | tr -d '\n'`
+checkFilesystem
+
+if ! minsize=$(resize2fs -P "$loopback"); then
+	rc=$?
+	error $LINENO "resize2fs failed with rc $rc"
+	exit 10
+fi
+minsize=$(cut -d ':' -f 2 <<< "$minsize" | tr -d ' ')
+logVariables $LINENO currentsize minsize
 if [[ $currentsize -eq $minsize ]]; then
-  echo ERROR: Image already shrunk to smallest size
-  exit -6
+  error $LINENO "Image already shrunk to smallest size"
+  exit 11
 fi
 
 #Add some free space to the end of the filesystem
-if [[ `expr $currentsize - $minsize - 5000` -gt 0 ]]; then
-  minsize=`expr $minsize + 5000 | tr -d '\n'`
-elif [[ `expr $currentsize - $minsize - 1000` -gt 0 ]]; then
-  minsize=`expr $minsize + 1000 | tr -d '\n'`
-elif [[ `expr $currentsize - $minsize - 100` -gt 0 ]]; then
-  minsize=`expr $minsize + 100 | tr -d '\n'`
+extra_space=$(($currentsize - $minsize))
+logVariables $LINENO extra_space
+for space in 5000 1000 100; do
+  if [[ $extra_space -gt $space ]]; then
+    minsize=$(($minsize + $space))
+    break
   fi
+done
+logVariables $LINENO minsize
 
 #Shrink filesystem
-resize2fs -p $loopback $minsize
-if [[ $? != 0 ]]; then
-  echo ERROR: resize2fs failed...
-  mount $loopback $mountdir
-  mv $mountdir/etc/rc.local.bak $mountdir/etc/rc.local
-  umount $mountdir
-  losetup -d $loopback
-  exit $rc
+info "Shrinking filesystem"
+resize2fs -p "$loopback" $minsize
+rc=$?
+if (( $rc )); then
+  error $LINENO "resize2fs failed with rc $rc"
+  mount "$loopback" "$mountdir"
+  mv "$mountdir/etc/rc.local.bak" "$mountdir/etc/rc.local"
+  umount "$mountdir"
+  losetup -d "$loopback"
+  exit 12
 fi
 sleep 1
 
 #Shrink partition
-losetup -d $loopback
-partnewsize=`expr $minsize \* $blocksize | tr -d '\n'`
-newpartend=`expr $partstart + $partnewsize | tr -d '\n'`
-part1=`parted $img rm $partnum`
-part2=`parted $img unit B mkpart primary $partstart $newpartend`
+partnewsize=$(($minsize * $blocksize))
+newpartend=$(($partstart + $partnewsize))
+logVariables $LINENO partnewsize newpartend
+parted -s -a minimal "$img" rm "$partnum"
+rc=$?
+if (( $rc )); then
+	error $LINENO "parted failed with rc $rc"
+	exit 13
+fi
+
+parted -s "$img" unit B mkpart "$parttype" "$partstart" "$newpartend"
+rc=$?
+if (( $rc )); then
+	error $LINENO "parted failed with rc $rc"
+	exit 14
+fi
 
 #Truncate the file
-endresult=`parted -m $img unit B print free | tail -1 | cut -d ':' -f 2 | tr -d 'B\n'`
-truncate -s $endresult $img
-aftersize=`ls -lah $img | cut -d ' ' -f 5`
+info "Shrinking image"
+endresult=$(parted -ms "$img" unit B print free)
+rc=$?
+if (( $rc )); then
+	error $LINENO "parted failed with rc $rc"
+	exit 15
+fi
 
-echo "Shrunk $img from $beforesize to $aftersize"
+endresult=$(tail -1 <<< "$endresult" | cut -d ':' -f 2 | tr -d 'B')
+logVariables $LINENO endresult
+truncate -s "$endresult" "$img"
+rc=$?
+if (( $rc )); then
+	error $LINENO "trunate failed with rc $rc"
+	exit 16
+fi
+
+# handle compression
+if [[ -n $ziptool ]]; then
+	options=""
+	envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override all options for gzip or xz
+	[[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}"
+	[[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options
+	[[ $verbose == true ]] && options="$options -v" # add verbose flag if requested
+
+	if [[ $parallel == true ]]; then
+		parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}"
+		info "Using $parallel_tool on the shrunk image"
+		if ! $parallel_tool ${options} "$img"; then
+			rc=$?
+			error $LINENO "$parallel_tool failed with rc $rc"
+			exit 18
+		fi
+
+	else # sequential
+		info "Using $ziptool on the shrunk image"
+		if ! $ziptool ${options} "$img"; then
+			rc=$?
+			error $LINENO "$ziptool failed with rc $rc"
+			exit 19
+		fi
+	fi
+	img=$img.${ZIPEXTENSIONS[$ziptool]}
+fi
+
+aftersize=$(ls -lh "$img" | cut -d ' ' -f 5)
+logVariables $LINENO aftersize
+
+info "Shrunk $img from $beforesize to $aftersize"