From 1a731f48b21d987598a8ec425eab6867109d9e53 Mon Sep 17 00:00:00 2001 From: nhkhai Date: Fri, 24 Jan 2020 18:22:37 +0800 Subject: [PATCH 1/4] Added support for XZ compression --- pishrink.sh | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/pishrink.sh b/pishrink.sh index d2f922a..7360f81 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -69,6 +69,8 @@ Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img] -d: Write debug messages in a debug log file -r: Use advanced filesystem repair option if the normal one fails -p: Remove logs, apt archives, dhcp leases and ssh hostkeys + -x: XZ compress image after shrinking + -e: XZ extreme compress image after shrinking -z: Gzip compress image after shrinking EOM echo "$help" @@ -82,6 +84,8 @@ usage() { echo " -d: Debug mode on" echo " -r: Use advanced repair options" echo " -p: Remove logs, apt archives, dhcp leases and ssh hostkeys" + echo " -x: XZ compress image after shrinking" + echo " -e: XZ extreme compress image after shrinking" echo " -z: Gzip compress image after shrinking" echo " -h: display help text" exit -1 @@ -90,6 +94,8 @@ usage() { should_skip_autoexpand=false debug=false repair=false +xzip_compress=false +exzip_compress=false gzip_compress=false prep=false @@ -99,6 +105,8 @@ while getopts ":sdrpzh" opt; do d) debug=true;; r) repair=true;; p) prep=true;; + x) xzip_compress=true;; + e) exzip_compress=true;; z) gzip_compress=true;; h) help;; *) usage ;; @@ -115,11 +123,11 @@ fi echo "${0##*/} $version" -#Args +# Args src="$1" img="$1" -#Usage checks +# Usage checks if [[ -z "$img" ]]; then usage fi @@ -132,7 +140,7 @@ if (( EUID != 0 )); then exit -3 fi -#Check that what we need is installed +# Check that what we need is installed for command in parted losetup tune2fs md5sum e2fsck resize2fs; do command -v $command >/dev/null 2>&1 if (( $? != 0 )); then @@ -141,7 +149,7 @@ for command in parted losetup tune2fs md5sum e2fsck resize2fs; do fi done -#Copy to new file if requested +# Copy to new file if requested if [ -n "$2" ]; then info "Copying $1 to $2..." cp --reflink=auto --sparse=always "$1" "$2" @@ -154,7 +162,7 @@ if [ -n "$2" ]; then img="$2" fi -# cleanup at script exit +# Cleanup at script exit trap cleanup ERR EXIT #Gather info @@ -170,7 +178,7 @@ blocksize=$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ': logVariables $LINENO tune2fs_output currentsize blocksize -#Check if we should make pi expand rootfs on next boot +# Check if we should make pi expand rootfs on next boot if [ "$should_skip_autoexpand" = false ]; then #Make pi expand rootfs on next boot mountdir=$(mktemp -d) @@ -254,8 +262,7 @@ if [[ $prep == true ]]; then umount "$mountdir" fi - -#Make sure filesystem is ok +# Make sure filesystem is ok checkFilesystem if ! minsize=$(resize2fs -P "$loopback"); then @@ -270,7 +277,7 @@ if [[ $currentsize -eq $minsize ]]; then exit -11 fi -#Add some free space to the end of the filesystem +# Add some free space to the end of the filesystem extra_space=$(($currentsize - $minsize)) logVariables $LINENO extra_space for space in 5000 1000 100; do @@ -281,7 +288,7 @@ for space in 5000 1000 100; do done logVariables $LINENO minsize -#Shrink filesystem +# Shrink filesystem info "Shrinking filesystem" resize2fs -p "$loopback" $minsize if [[ $? != 0 ]]; then @@ -294,7 +301,7 @@ if [[ $? != 0 ]]; then fi sleep 1 -#Shrink partition +# Shrink partition partnewsize=$(($minsize * $blocksize)) newpartend=$(($partstart + $partnewsize)) logVariables $LINENO partnewsize newpartend @@ -310,7 +317,7 @@ if ! parted -s "$img" unit B mkpart primary "$partstart" "$newpartend"; then exit -14 fi -#Truncate the file +# Truncate the file info "Shrinking image" if ! endresult=$(parted -ms "$img" unit B print free); then rc=$? @@ -326,6 +333,20 @@ if ! truncate -s "$endresult" "$img"; then exit -16 fi +if [[ $xzip_compress == true ]]; then + info "XZipping the shrunk image" + if [[ ! $(xz -9v "$img") ]]; then + img=$img.xz + fi +fi + +if [[ $exzip_compress == true ]]; then + info "Extreme XZipping the shrunk image" + if [[ ! $(xz -9ve "$img") ]]; then + img=$img.xz + fi +fi + if [[ $gzip_compress == true ]]; then info "Gzipping the shrunk image" if [[ ! $(gzip -f9 "$img") ]]; then From c6ffbb02944c2bf9fb53b602ec9aca4c854f3c52 Mon Sep 17 00:00:00 2001 From: nhkhai Date: Fri, 24 Jan 2020 18:58:12 +0800 Subject: [PATCH 2/4] Added parsing of the new flags. --- pishrink.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pishrink.sh b/pishrink.sh index 7360f81..d932b82 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -99,7 +99,7 @@ exzip_compress=false gzip_compress=false prep=false -while getopts ":sdrpzh" opt; do +while getopts ":sdrpxezh" opt; do case "${opt}" in s) should_skip_autoexpand=true ;; d) debug=true;; From 4959190ae924395c2ab51c5becbc2f6e4bbce3cf Mon Sep 17 00:00:00 2001 From: nhkhai Date: Sun, 9 Aug 2020 13:53:01 +0800 Subject: [PATCH 3/4] Updated to original copy --- README.md | 143 +++++++++++-------- pishrink.sh | 405 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 315 insertions(+), 233 deletions(-) diff --git a/README.md b/README.md index b69a134..e80200a 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,81 @@ - -# PiShrink # -PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. - -## Usage ## -``` -sudo pishrink.sh [-sdrzh] imagefile.img [newimagefile.img] - -s: Skip autoexpand - -d: Debug mode on - -r: Use advanced repair options - -z: Gzip compress image after shrinking - -h: display help text -``` - -If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. - -* `-s` will skip the autoexpanding part of the process. -* `-d` will create a logfile `pishrink.log` which may help for problem analysis. -* `-r` will attempt to repair the filesystem if regular repairs fail -* `-z` will Gzip compress the image after shrinking. The `.gz` extension will be added to the filename. - - -## Prerequisites ## -If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. - -If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. - -## Installation ## -```bash -wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh -chmod +x pishrink.sh -sudo mv pishrink.sh /usr/local/bin -``` - -## Example ## -```bash -[user@localhost PiShrink]$ sudo pishrink.sh pi.img -e2fsck 1.42.9 (28-Dec-2013) -Pass 1: Checking inodes, blocks, and sizes -Pass 2: Checking directory structure -Pass 3: Checking directory connectivity -Pass 4: Checking reference counts -Pass 5: Checking group summary information -/dev/loop1: 88262/1929536 files (0.2% non-contiguous), 842728/7717632 blocks -resize2fs 1.42.9 (28-Dec-2013) -resize2fs 1.42.9 (28-Dec-2013) -Resizing the filesystem on /dev/loop1 to 773603 (4k) blocks. -Begin pass 2 (max = 100387) -Relocating blocks XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -Begin pass 3 (max = 236) -Scanning inode table XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -Begin pass 4 (max = 7348) -Updating inode references XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -The filesystem on /dev/loop1 is now 773603 blocks long. - -Shrunk pi.img from 30G to 3.1G -``` - -## Contributing ## -If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. - -Pull requests for new features and bug fixes are more than welcome! + +# PiShrink # + +PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. +In addition the shrinked image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image +using multiple cores is supported. + +## Usage ## + +``` +Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] + + -s Don't expand filesystem when image is booted the first time + -v Be verbose + -r Use advanced filesystem repair option if the normal one fails + -z Compress image after shrinking with gzip + -Z Compress image after shrinking with xz + -a Compress image in parallel using multiple cores + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -d Write debug messages in a debug log file +``` + +If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. + +* `-s` prevents automatic filesystem expantion on the images next boot +* `-v` enables more verbose output +* `-r` will attempt to repair the filesystem using aditional options if the normal repair fails +* `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename. +* `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename. +* `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel. +* `-d` will create a logfile `pishrink.log` which may help for problem analysis. + +Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. + +## Prerequisites ## + +If you are running PiShrink in VirtualBox you will likely encounter an error if you +attempt to use VirtualBox's "Shared Folder" feature. You can copy the image you wish to +shrink on to the VM from a Shared Folder, but shrinking directctly from the Shared Folder +is know to cause issues. + +If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. + +## Installation ## + +```bash +wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh +chmod +x pishrink.sh +sudo mv pishrink.sh /usr/local/bin +``` + +## Example ## + +```bash +[user@localhost PiShrink]$ sudo pishrink.sh pi.img +e2fsck 1.42.9 (28-Dec-2013) +Pass 1: Checking inodes, blocks, and sizes +Pass 2: Checking directory structure +Pass 3: Checking directory connectivity +Pass 4: Checking reference counts +Pass 5: Checking group summary information +/dev/loop1: 88262/1929536 files (0.2% non-contiguous), 842728/7717632 blocks +resize2fs 1.42.9 (28-Dec-2013) +resize2fs 1.42.9 (28-Dec-2013) +Resizing the filesystem on /dev/loop1 to 773603 (4k) blocks. +Begin pass 2 (max = 100387) +Relocating blocks XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +Begin pass 3 (max = 236) +Scanning inode table XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +Begin pass 4 (max = 7348) +Updating inode references XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +The filesystem on /dev/loop1 is now 773603 blocks long. + +Shrunk pi.img from 30G to 3.1G +``` + +## Contributing ## + +If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. + +Pull requests for new features and bug fixes are more than welcome! diff --git a/pishrink.sh b/pishrink.sh index d932b82..623f736 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -2,12 +2,18 @@ version="v0.1.2" -CURRENT_DIR=$(pwd) +CURRENT_DIR="$(pwd)" SCRIPTNAME="${0##*/}" -LOGFILE=${CURRENT_DIR}/${SCRIPTNAME%.*}.log +MYNAME="${SCRIPTNAME%.*}" +LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log" +REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs" +ZIPTOOLS=("gzip xz") +declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # parallel zip tool to use in parallel mode +declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode +declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files function info() { - echo "$SCRIPTNAME: $1..." + echo "$SCRIPTNAME: $1 ..." } function error() { @@ -56,137 +62,28 @@ if [[ $repair == true ]]; then (( $? < 4 )) && return fi error $LINENO "Filesystem recoveries failed. Giving up..." - exit -9 + exit 9 } -help() { - local help - read -r -d '' help << EOM -Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img] +function set_autoexpand() { + #Make pi expand rootfs on next boot + mountdir=$(mktemp -d) + partprobe "$loopback" + mount "$loopback" "$mountdir" - -s: Don't expand filesystem when image is booted the first time - -d: Write debug messages in a debug log file - -r: Use advanced filesystem repair option if the normal one fails - -p: Remove logs, apt archives, dhcp leases and ssh hostkeys - -x: XZ compress image after shrinking - -e: XZ extreme compress image after shrinking - -z: Gzip compress image after shrinking -EOM - echo "$help" - exit -1 -} + if [ ! -d "$mountdir/etc" ]; then + info "/etc not found, autoexpand will not be enabled" + umount "$mountdir" + return + fi -usage() { - echo "Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img]" - echo "" - echo " -s: Skip autoexpand" - echo " -d: Debug mode on" - echo " -r: Use advanced repair options" - echo " -p: Remove logs, apt archives, dhcp leases and ssh hostkeys" - echo " -x: XZ compress image after shrinking" - echo " -e: XZ extreme compress image after shrinking" - echo " -z: Gzip compress image after shrinking" - echo " -h: display help text" - exit -1 -} + if [[ -f "$mountdir/etc/rc.local" ]] && [[ "$(md5sum "$mountdir/etc/rc.local" | cut -d ' ' -f 1)" != "1c579c7d5b4292fd948399b6ece39009" ]]; then + echo "Creating new /etc/rc.local" + if [ -f "$mountdir/etc/rc.local" ]; then + mv "$mountdir/etc/rc.local" "$mountdir/etc/rc.local.bak" + fi -should_skip_autoexpand=false -debug=false -repair=false -xzip_compress=false -exzip_compress=false -gzip_compress=false -prep=false - -while getopts ":sdrpxezh" opt; do - case "${opt}" in - s) should_skip_autoexpand=true ;; - d) debug=true;; - r) repair=true;; - p) prep=true;; - x) xzip_compress=true;; - e) exzip_compress=true;; - z) gzip_compress=true;; - h) help;; - *) usage ;; - esac -done -shift $((OPTIND-1)) - -if [ "$debug" = true ]; then - info "Creating log file $LOGFILE" - rm "$LOGFILE" &>/dev/null - exec 1> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&1) - exec 2> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&2) -fi - -echo "${0##*/} $version" - -# Args -src="$1" -img="$1" - -# Usage checks -if [[ -z "$img" ]]; then - usage -fi -if [[ ! -f "$img" ]]; then - error $LINENO "$img is not a file..." - exit -2 -fi -if (( EUID != 0 )); then - error $LINENO "You need to be running as root." - exit -3 -fi - -# Check that what we need is installed -for command in parted losetup tune2fs md5sum e2fsck resize2fs; do - command -v $command >/dev/null 2>&1 - if (( $? != 0 )); then - error $LINENO "$command is not installed." - exit -4 - fi -done - -# Copy to new file if requested -if [ -n "$2" ]; then - info "Copying $1 to $2..." - cp --reflink=auto --sparse=always "$1" "$2" - if (( $? != 0 )); then - error $LINENO "Could not copy file..." - exit -5 - fi - old_owner=$(stat -c %u:%g "$1") - chown "$old_owner" "$2" - img="$2" -fi - -# Cleanup at script exit -trap cleanup ERR EXIT - -#Gather info -info "Gathering data" -beforesize=$(ls -lh "$img" | cut -d ' ' -f 5) -parted_output=$(parted -ms "$img" unit B print | tail -n 1) -partnum=$(echo "$parted_output" | cut -d ':' -f 1) -partstart=$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B') -loopback=$(losetup -f --show -o "$partstart" "$img") -tune2fs_output=$(tune2fs -l "$loopback") -currentsize=$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2) -blocksize=$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2) - -logVariables $LINENO tune2fs_output currentsize blocksize - -# Check if we should make pi expand rootfs on next boot -if [ "$should_skip_autoexpand" = false ]; then - #Make pi expand rootfs on next boot - mountdir=$(mktemp -d) - mount "$loopback" "$mountdir" - - if [ "$(md5sum "$mountdir/etc/rc.local" | cut -d ' ' -f 1)" != "0542054e9ff2d2e0507ea1ffe7d4fc87" ]; then - echo "Creating new /etc/rc.local" - mv "$mountdir/etc/rc.local" "$mountdir/etc/rc.local.bak" #####Do not touch the following lines##### cat <<\EOF1 > "$mountdir/etc/rc.local" #!/bin/bash @@ -243,13 +140,166 @@ sleep 5 do_expand_rootfs echo "ERROR: Expanding failed..." sleep 5 -rm -f /etc/rc.local; cp -f /etc/rc.local.bak /etc/rc.local; /etc/rc.local +if [[ -f /etc/rc.local.bak ]]; then + cp -f /etc/rc.local.bak /etc/rc.local + /etc/rc.local +fi exit 0 EOF1 #####End no touch zone##### chmod +x "$mountdir/etc/rc.local" + fi + umount "$mountdir" +} + +help() { + local help + read -r -d '' help << EOM +Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] + + -s Don't expand filesystem when image is booted the first time + -v Be verbose + -r Use advanced filesystem repair option if the normal one fails + -z Compress image after shrinking with gzip + -Z Compress image after shrinking with xz + -a Compress image in parallel using multiple cores + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -d Write debug messages in a debug log file +EOM + echo "$help" + exit 1 +} + +should_skip_autoexpand=false +debug=false +repair=false +parallel=false +verbose=false +prep=false +ziptool="" + +while getopts ":adhprsvzZ" opt; do + case "${opt}" in + a) parallel=true;; + d) debug=true;; + h) help;; + p) prep=true;; + r) repair=true;; + s) should_skip_autoexpand=true ;; + v) verbose=true;; + z) ziptool="gzip";; + Z) ziptool="xz";; + *) help;; + esac +done +shift $((OPTIND-1)) + +if [ "$debug" = true ]; then + info "Creating log file $LOGFILE" + rm "$LOGFILE" &>/dev/null + exec 1> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&1) + exec 2> >(stdbuf -i0 -o0 -e0 tee -a "$LOGFILE" >&2) +fi + +echo "${0##*/} $version" + +#Args +src="$1" +img="$1" + +#Usage checks +if [[ -z "$img" ]]; then + help +fi + +if [[ ! -f "$img" ]]; then + error $LINENO "$img is not a file..." + exit 2 +fi +if (( EUID != 0 )); then + error $LINENO "You need to be running as root." + exit 3 +fi + +# check selected compression tool is supported and installed +if [[ -n $ziptool ]]; then + if [[ ! " ${ZIPTOOLS[@]} " =~ $ziptool ]]; then + error $LINENO "$ziptool is an unsupported ziptool." + exit 17 + else + if [[ $parallel == true && $ziptool == "gzip" ]]; then + REQUIRED_TOOLS="$REQUIRED_TOOLS pigz" + else + REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool" + fi + fi +fi + +#Check that what we need is installed +for command in $REQUIRED_TOOLS; do + command -v $command >/dev/null 2>&1 + if (( $? != 0 )); then + error $LINENO "$command is not installed." + exit 4 fi - umount "$mountdir" +done + +#Copy to new file if requested +if [ -n "$2" ]; then + f="$2" + if [[ -n $ziptool && "${f##*.}" == "${ZIPEXTENSIONS[$ziptool]}" ]]; then # remove zip extension if zip requested because zip tool will complain about extension + f="${f%.*}" + fi + info "Copying $1 to $f..." + cp --reflink=auto --sparse=always "$1" "$f" + if (( $? != 0 )); then + error $LINENO "Could not copy file..." + exit 5 + fi + old_owner=$(stat -c %u:%g "$1") + chown "$old_owner" "$f" + img="$f" +fi + +# cleanup at script exit +trap cleanup EXIT + +#Gather info +info "Gathering data" +beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)" +parted_output="$(parted -ms "$img" unit B print)" +rc=$? +if (( $rc )); then + error $LINENO "parted failed with rc $rc" + info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate" + exit 6 +fi +partnum="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 1)" +partstart="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 2 | tr -d 'B')" +if [ -z "$(parted -s "$img" unit B print | grep "$partstart" | grep logical)" ]; then + parttype="primary" +else + parttype="logical" +fi +loopback="$(losetup -f --show -o "$partstart" "$img")" +tune2fs_output="$(tune2fs -l "$loopback")" +rc=$? +if (( $rc )); then + echo "$tune2fs_output" + error $LINENO "tune2fs failed. Unable to shrink this type of image" + exit 7 +fi + +currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)" +blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)" + +logVariables $LINENO beforesize parted_output partnum partstart parttype tune2fs_output currentsize blocksize + +#Check if we should make pi expand rootfs on next boot +if [ "$parttype" == "logical" ]; then + echo "WARNING: PiShrink does not yet support autoexpanding of this type of image" +elif [ "$should_skip_autoexpand" = false ]; then + set_autoexpand else echo "Skipping autoexpanding process..." fi @@ -258,26 +308,27 @@ if [[ $prep == true ]]; then info "Syspreping: Removing logs, apt archives, dhcp leases and ssh hostkeys" mountdir=$(mktemp -d) mount "$loopback" "$mountdir" - rm -rf "$mountdir/var/cache/apt/archives/*" "$mountdir/var/lib/dhcpcd5/*" "$mountdir/var/log/*" "$mountdir/var/tmp/*" "$mountdir/tmp/*" "$mountdir/etc/ssh/*_host_*" + rm -rf "$mountdir/var/cache/apt/archives/*" "$mountdir/var/lib/dhcpcd5/*" "$mountdir/var/log/*" "$mountdir/var/tmp/*" "$mountdir/tmp/*" "$mountdir/etc/ssh/*_host_*" umount "$mountdir" fi -# Make sure filesystem is ok + +#Make sure filesystem is ok checkFilesystem if ! minsize=$(resize2fs -P "$loopback"); then rc=$? error $LINENO "resize2fs failed with rc $rc" - exit -10 + exit 10 fi minsize=$(cut -d ':' -f 2 <<< "$minsize" | tr -d ' ') -logVariables $LINENO minsize +logVariables $LINENO currentsize minsize if [[ $currentsize -eq $minsize ]]; then error $LINENO "Image already shrunk to smallest size" - exit -11 + exit 11 fi -# Add some free space to the end of the filesystem +#Add some free space to the end of the filesystem extra_space=$(($currentsize - $minsize)) logVariables $LINENO extra_space for space in 5000 1000 100; do @@ -288,70 +339,82 @@ for space in 5000 1000 100; do done logVariables $LINENO minsize -# Shrink filesystem +#Shrink filesystem info "Shrinking filesystem" resize2fs -p "$loopback" $minsize -if [[ $? != 0 ]]; then - error $LINENO "resize2fs failed" +rc=$? +if (( $rc )); then + error $LINENO "resize2fs failed with rc $rc" mount "$loopback" "$mountdir" mv "$mountdir/etc/rc.local.bak" "$mountdir/etc/rc.local" umount "$mountdir" losetup -d "$loopback" - exit -12 + exit 12 fi sleep 1 -# Shrink partition +#Shrink partition partnewsize=$(($minsize * $blocksize)) newpartend=$(($partstart + $partnewsize)) logVariables $LINENO partnewsize newpartend -if ! parted -s -a minimal "$img" rm "$partnum"; then - rc=$? +parted -s -a minimal "$img" rm "$partnum" +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" - exit -13 + exit 13 fi -if ! parted -s "$img" unit B mkpart primary "$partstart" "$newpartend"; then - rc=$? +parted -s "$img" unit B mkpart "$parttype" "$partstart" "$newpartend" +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" - exit -14 + exit 14 fi -# Truncate the file +#Truncate the file info "Shrinking image" -if ! endresult=$(parted -ms "$img" unit B print free); then - rc=$? +endresult=$(parted -ms "$img" unit B print free) +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" - exit -15 + exit 15 fi endresult=$(tail -1 <<< "$endresult" | cut -d ':' -f 2 | tr -d 'B') logVariables $LINENO endresult -if ! truncate -s "$endresult" "$img"; then - rc=$? +truncate -s "$endresult" "$img" +rc=$? +if (( $rc )); then error $LINENO "trunate failed with rc $rc" - exit -16 + exit 16 fi -if [[ $xzip_compress == true ]]; then - info "XZipping the shrunk image" - if [[ ! $(xz -9v "$img") ]]; then - img=$img.xz - fi -fi +# handle compression +if [[ -n $ziptool ]]; then + options="" + envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override all options for gzip or xz + [[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" + [[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options + [[ $verbose == true ]] && options="$options -v" # add verbose flag if requested -if [[ $exzip_compress == true ]]; then - info "Extreme XZipping the shrunk image" - if [[ ! $(xz -9ve "$img") ]]; then - img=$img.xz - fi -fi + if [[ $parallel == true ]]; then + parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}" + info "Using $parallel_tool on the shrunk image" + if ! $parallel_tool ${options} "$img"; then + rc=$? + error $LINENO "$parallel_tool failed with rc $rc" + exit 18 + fi -if [[ $gzip_compress == true ]]; then - info "Gzipping the shrunk image" - if [[ ! $(gzip -f9 "$img") ]]; then - img=$img.gz - fi + else # sequential + info "Using $ziptool on the shrunk image" + if ! $ziptool ${options} "$img"; then + rc=$? + error $LINENO "$ziptool failed with rc $rc" + exit 19 + fi + fi + img=$img.${ZIPEXTENSIONS[$ziptool]} fi aftersize=$(ls -lh "$img" | cut -d ' ' -f 5) From 4e90956eb1cd60f7e0ed33756d01d4d637c9b01c Mon Sep 17 00:00:00 2001 From: nhkhai Date: Sun, 9 Aug 2020 16:05:02 +0800 Subject: [PATCH 4/4] Added options to change compression level for fast, default and best compression levels and added the option to compress in extreme mode for xz. --- README.md | 39 ++++++++++++++++++--------------- pishrink.sh | 63 +++++++++++++++++++++++++++++++++-------------------- 2 files changed, 61 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index e80200a..54cbd9b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # PiShrink # -PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. -In addition the shrinked image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image -using multiple cores is supported. +PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. +In addition the shrinked image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image using multiple cores is supported. ## Usage ## @@ -13,8 +12,11 @@ Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time -v Be verbose -r Use advanced filesystem repair option if the normal one fails - -z Compress image after shrinking with gzip - -Z Compress image after shrinking with xz + -z Compress image after shrinking with gzip (Default is balanced (6) compression) + -Z Compress image after shrinking with xz (Default is balanced (6) compression) + -c Change the compression level from the default, balanced (6) to fast (1 for gzip, 0 or xz) + -C Change the compression level from the default, balanced (6) to best (9) + -e Compress image in extreme mode (Only applicable to xz compression) -a Compress image in parallel using multiple cores -p Remove logs, apt archives, dhcp leases and ssh hostkeys -d Write debug messages in a debug log file @@ -22,24 +24,27 @@ Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. -* `-s` prevents automatic filesystem expantion on the images next boot -* `-v` enables more verbose output -* `-r` will attempt to repair the filesystem using aditional options if the normal repair fails -* `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename. -* `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename. -* `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel. -* `-d` will create a logfile `pishrink.log` which may help for problem analysis. +* `-s` Prevents automatic filesystem expantion on the images next boot. +* `-v` Enables more verbose output. +* `-r` Will attempt to repair the filesystem using aditional options if the normal repair fails. +* `-z` Will compress the image after shrinking using gzip. `.gz` extension will be added to the filename. +* `-Z` Will compress the image after shrinking using xz. `.xz` extension will be added to the filename. +* `-c Will use option -1 for pigz and option -0 for xz. +* `-C Will use option -9. +* `-e Will use option -e for xz. +* `-a` will use option -f for pigz and option -T0 for xz and compress in parallel. +* `-d` will create a logfile `pishrink.log` which may help for problem analysis. -Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. +Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. ## Prerequisites ## If you are running PiShrink in VirtualBox you will likely encounter an error if you attempt to use VirtualBox's "Shared Folder" feature. You can copy the image you wish to shrink on to the VM from a Shared Folder, but shrinking directctly from the Shared Folder -is know to cause issues. +is know to cause issues. -If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. +If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. ## Installation ## @@ -76,6 +81,6 @@ Shrunk pi.img from 30G to 3.1G ## Contributing ## -If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. +If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. -Pull requests for new features and bug fixes are more than welcome! +Pull requests for new features and bug fixes are more than welcome! \ No newline at end of file diff --git a/pishrink.sh b/pishrink.sh index 623f736..d162ebd 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -8,9 +8,9 @@ MYNAME="${SCRIPTNAME%.*}" LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log" REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs" ZIPTOOLS=("gzip xz") -declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # parallel zip tool to use in parallel mode -declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode -declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files +declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # Parallel zip tool to use in parallel mode +declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f" [xz]="-T0" ) # Options for zip tools in parallel mode +declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # Extensions of zipped files function info() { echo "$SCRIPTNAME: $1 ..." @@ -160,9 +160,12 @@ Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time -v Be verbose -r Use advanced filesystem repair option if the normal one fails - -z Compress image after shrinking with gzip - -Z Compress image after shrinking with xz + -z Compress image after shrinking with gzip (Default is balanced (6) compression) + -Z Compress image after shrinking with xz (Default is balanced (6) compression) -a Compress image in parallel using multiple cores + -c Change the compression level from the default, balanced (6) to fast (1 for gzip, 0 or xz) + -C Change the compression level from the default, balanced (6) to best (9) + -e Compress image in extreme mode (Only applicable to xz compression) -p Remove logs, apt archives, dhcp leases and ssh hostkeys -d Write debug messages in a debug log file EOM @@ -171,17 +174,22 @@ EOM } should_skip_autoexpand=false +compression=6 debug=false +extreme=false repair=false parallel=false verbose=false prep=false ziptool="" -while getopts ":adhprsvzZ" opt; do +while getopts ":acCdehprsvzZ" opt; do case "${opt}" in a) parallel=true;; + c) compression=0;; + C) compression=9;; d) debug=true;; + e) extreme=true;; h) help;; p) prep=true;; r) repair=true;; @@ -194,6 +202,10 @@ while getopts ":adhprsvzZ" opt; do done shift $((OPTIND-1)) +if [ $compression == 0 ] && [[ "$ziptool" == "gzip" ]]; then + compression=1 +fi + if [ "$debug" = true ]; then info "Creating log file $LOGFILE" rm "$LOGFILE" &>/dev/null @@ -203,11 +215,11 @@ fi echo "${0##*/} $version" -#Args +# Args src="$1" img="$1" -#Usage checks +# Usage checks if [[ -z "$img" ]]; then help fi @@ -221,7 +233,7 @@ if (( EUID != 0 )); then exit 3 fi -# check selected compression tool is supported and installed +# Check selected compression tool is supported and installed if [[ -n $ziptool ]]; then if [[ ! " ${ZIPTOOLS[@]} " =~ $ziptool ]]; then error $LINENO "$ziptool is an unsupported ziptool." @@ -235,7 +247,7 @@ if [[ -n $ziptool ]]; then fi fi -#Check that what we need is installed +# Check that what we need is installed for command in $REQUIRED_TOOLS; do command -v $command >/dev/null 2>&1 if (( $? != 0 )); then @@ -244,10 +256,10 @@ for command in $REQUIRED_TOOLS; do fi done -#Copy to new file if requested +# Copy to new file if requested if [ -n "$2" ]; then f="$2" - if [[ -n $ziptool && "${f##*.}" == "${ZIPEXTENSIONS[$ziptool]}" ]]; then # remove zip extension if zip requested because zip tool will complain about extension + if [[ -n $ziptool && "${f##*.}" == "${ZIPEXTENSIONS[$ziptool]}" ]]; then # Remove zip extension if zip requested because zip tool will complain about extension f="${f%.*}" fi info "Copying $1 to $f..." @@ -261,10 +273,10 @@ if [ -n "$2" ]; then img="$f" fi -# cleanup at script exit +# Cleanup at script exit trap cleanup EXIT -#Gather info +# Gather info info "Gathering data" beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)" parted_output="$(parted -ms "$img" unit B print)" @@ -295,7 +307,7 @@ blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ' logVariables $LINENO beforesize parted_output partnum partstart parttype tune2fs_output currentsize blocksize -#Check if we should make pi expand rootfs on next boot +# Check if we should make pi expand rootfs on next boot if [ "$parttype" == "logical" ]; then echo "WARNING: PiShrink does not yet support autoexpanding of this type of image" elif [ "$should_skip_autoexpand" = false ]; then @@ -313,7 +325,7 @@ if [[ $prep == true ]]; then fi -#Make sure filesystem is ok +# Make sure filesystem is ok checkFilesystem if ! minsize=$(resize2fs -P "$loopback"); then @@ -328,7 +340,7 @@ if [[ $currentsize -eq $minsize ]]; then exit 11 fi -#Add some free space to the end of the filesystem +# Add some free space to the end of the filesystem extra_space=$(($currentsize - $minsize)) logVariables $LINENO extra_space for space in 5000 1000 100; do @@ -339,7 +351,7 @@ for space in 5000 1000 100; do done logVariables $LINENO minsize -#Shrink filesystem +# Shrink filesystem info "Shrinking filesystem" resize2fs -p "$loopback" $minsize rc=$? @@ -353,7 +365,7 @@ if (( $rc )); then fi sleep 1 -#Shrink partition +# Shrink partition partnewsize=$(($minsize * $blocksize)) newpartend=$(($partstart + $partnewsize)) logVariables $LINENO partnewsize newpartend @@ -371,7 +383,7 @@ if (( $rc )); then exit 14 fi -#Truncate the file +# Truncate the file info "Shrinking image" endresult=$(parted -ms "$img" unit B print free) rc=$? @@ -389,13 +401,15 @@ if (( $rc )); then exit 16 fi -# handle compression +# Handle the compression if [[ -n $ziptool ]]; then options="" envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override all options for gzip or xz [[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" - [[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options - [[ $verbose == true ]] && options="$options -v" # add verbose flag if requested + [[ $compression != 6 ]] && options="$options -$compression" # Update the compression value if requested (default value is 6) + [[ "$ziptool" == "xz" ]] && [[ $extreme == true ]] && options="$options -e" # Add extreme flag if requested for xz + [[ -v $envVarname ]] && options="${!envVarname}" # If environment variables defined use these options + [[ $verbose == true ]] && options="$options -v" # Add verbose flag if requested if [[ $parallel == true ]]; then parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}" @@ -406,7 +420,7 @@ if [[ -n $ziptool ]]; then exit 18 fi - else # sequential + else # Sequential info "Using $ziptool on the shrunk image" if ! $ziptool ${options} "$img"; then rc=$? @@ -414,6 +428,7 @@ if [[ -n $ziptool ]]; then exit 19 fi fi + img=$img.${ZIPEXTENSIONS[$ziptool]} fi