From 9de4f53b96a89be358b17e5659437e79a56ec079 Mon Sep 17 00:00:00 2001 From: framp Date: Thu, 9 Jan 2020 13:21:55 +0100 Subject: [PATCH 01/14] Added some more dbug logs --- pishrink.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pishrink.sh b/pishrink.sh index d2f922a..81e17e5 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -159,16 +159,16 @@ trap cleanup ERR EXIT #Gather info info "Gathering data" -beforesize=$(ls -lh "$img" | cut -d ' ' -f 5) -parted_output=$(parted -ms "$img" unit B print | tail -n 1) -partnum=$(echo "$parted_output" | cut -d ':' -f 1) -partstart=$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B') -loopback=$(losetup -f --show -o "$partstart" "$img") -tune2fs_output=$(tune2fs -l "$loopback") -currentsize=$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2) -blocksize=$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2) +beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)" +parted_output="$(parted -ms "$img" unit B print | tail -n 1)" +partnum="$(echo "$parted_output" | cut -d ':' -f 1)" +partstart="$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B')" +loopback="$(losetup -f --show -o "$partstart" "$img")" +tune2fs_output="$(tune2fs -l "$loopback")" +currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)" +blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)" -logVariables $LINENO tune2fs_output currentsize blocksize +logVariables $LINENO beforesize parted_output partnum partstart tune2fs_output currentsize blocksize #Check if we should make pi expand rootfs on next boot if [ "$should_skip_autoexpand" = false ]; then @@ -250,7 +250,7 @@ if [[ $prep == true ]]; then info "Syspreping: Removing logs, apt archives, dhcp leases and ssh hostkeys" mountdir=$(mktemp -d) mount "$loopback" "$mountdir" - rm -rf "$mountdir/var/cache/apt/archives/*" "$mountdir/var/lib/dhcpcd5/*" "$mountdir/var/log/*" "$mountdir/var/tmp/*" "$mountdir/tmp/*" "$mountdir/etc/ssh/*_host_*" + rm -rf "$mountdir/var/cache/apt/archives/*" "$mountdir/var/lib/dhcpcd5/*" "$mountdir/var/log/*" "$mountdir/var/tmp/*" "$mountdir/tmp/*" "$mountdir/etc/ssh/*_host_*" umount "$mountdir" fi @@ -264,7 +264,7 @@ if ! minsize=$(resize2fs -P "$loopback"); then exit -10 fi minsize=$(cut -d ':' -f 2 <<< "$minsize" | tr -d ' ') -logVariables $LINENO minsize +logVariables $LINENO currentsize minsize if [[ $currentsize -eq $minsize ]]; then error $LINENO "Image already shrunk to smallest size" exit -11 From 4ed316228444d651432fc670dccc75e719f5a003 Mon Sep 17 00:00:00 2001 From: framp Date: Sat, 11 Jan 2020 15:21:54 +0100 Subject: [PATCH 02/14] Initial version which supports addl zip tools --- pishrink.sh | 64 +++++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/pishrink.sh b/pishrink.sh index 81e17e5..e3f7555 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -5,6 +5,10 @@ version="v0.1.2" CURRENT_DIR=$(pwd) SCRIPTNAME="${0##*/}" LOGFILE=${CURRENT_DIR}/${SCRIPTNAME%.*}.log +REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs" +ZIPTOOLS=("gzip pigz xz pxz") +declare -A ZIPOPTIONS=( [gzip]="-f9" [pigz]="-9" [xz]="-9" [pxz]="-9") # options for zip tools +declare -A ZIPEXTENSIONS=( [gzip]="gz" [pigz]="gz" [xz]="xz" [pxz]="xz") # extensions of zipped files function info() { echo "$SCRIPTNAME: $1..." @@ -63,45 +67,36 @@ fi help() { local help read -r -d '' help << EOM -Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img] +Usage: $0 [-sdirpzh] imagefile.img [newimagefile.img] - -s: Don't expand filesystem when image is booted the first time - -d: Write debug messages in a debug log file - -r: Use advanced filesystem repair option if the normal one fails - -p: Remove logs, apt archives, dhcp leases and ssh hostkeys - -z: Gzip compress image after shrinking + -s Don't expand filesystem when image is booted the first time + -d Write debug messages in a debug log file + -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) + -r Use advanced filesystem repair option if the normal one fails + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -z Compress image after shrinking EOM echo "$help" exit -1 } -usage() { - echo "Usage: $0 [-sdrpzh] imagefile.img [newimagefile.img]" - echo "" - echo " -s: Skip autoexpand" - echo " -d: Debug mode on" - echo " -r: Use advanced repair options" - echo " -p: Remove logs, apt archives, dhcp leases and ssh hostkeys" - echo " -z: Gzip compress image after shrinking" - echo " -h: display help text" - exit -1 -} - should_skip_autoexpand=false debug=false repair=false -gzip_compress=false +compress=false prep=false +ziptool="gzip" +required_tools="$REQUIRED_TOOLS" -while getopts ":sdrpzh" opt; do +while getopts ":si:drpzh" opt; do case "${opt}" in s) should_skip_autoexpand=true ;; d) debug=true;; r) repair=true;; p) prep=true;; - z) gzip_compress=true;; - h) help;; - *) usage ;; + z) compress=true;; + i) ziptool="$OPTARG";; + h,*) help;; esac done shift $((OPTIND-1)) @@ -121,8 +116,9 @@ img="$1" #Usage checks if [[ -z "$img" ]]; then - usage + help fi + if [[ ! -f "$img" ]]; then error $LINENO "$img is not a file..." exit -2 @@ -132,8 +128,18 @@ if (( EUID != 0 )); then exit -3 fi +# check selected compression tool is supported and installed +if [[ $compress == true ]]; then + if [[ ! " ${ZIPTOOLS[@]} " =~ " $ziptool " ]]; then + error $LINENO "$ziptool is an unsupported ziptool." + exit -17 + else + REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool" + fi +fi + #Check that what we need is installed -for command in parted losetup tune2fs md5sum e2fsck resize2fs; do +for command in $REQUIRED_TOOLS; do command -v $command >/dev/null 2>&1 if (( $? != 0 )); then error $LINENO "$command is not installed." @@ -326,10 +332,10 @@ if ! truncate -s "$endresult" "$img"; then exit -16 fi -if [[ $gzip_compress == true ]]; then - info "Gzipping the shrunk image" - if [[ ! $(gzip -f9 "$img") ]]; then - img=$img.gz +if [[ $compress == true ]]; then + info "Using $ziptool on the shrunk image using options ${ZIPOPTIONS[$ziptool]}" + if [[ ! $($ziptool ${ZIPOPTIONS[$ziptool]} "$img") ]]; then + img=$img.${ZIPEXTENSIONS[$ziptool]} fi fi From 3a931ddb73900aef16ef28fa37c8b4a6285aca05 Mon Sep 17 00:00:00 2001 From: framp Date: Sat, 11 Jan 2020 15:24:52 +0100 Subject: [PATCH 03/14] Updated README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b69a134..e2ae0f4 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,10 @@ PiShrink is a bash script that automatically shrink a pi image that will then re ``` sudo pishrink.sh [-sdrzh] imagefile.img [newimagefile.img] -s: Skip autoexpand + -i TOOL: Zip tool to use to compress image. TOOLS can be one of gtip pigz xz pxz (Default: gzip) -d: Debug mode on -r: Use advanced repair options - -z: Gzip compress image after shrinking + -z: Compress image after shrinking -h: display help text ``` From 7515ee5631aff6ee2e05303878be0466da197afc Mon Sep 17 00:00:00 2001 From: framp Date: Sun, 12 Jan 2020 10:23:22 +0100 Subject: [PATCH 04/14] Updated README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e2ae0f4..78fb10b 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ # PiShrink # PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. +In addition the shrinked image can be compressed with gzip, pigz, xz or pxz to create an even smaller image. ## Usage ## ``` sudo pishrink.sh [-sdrzh] imagefile.img [newimagefile.img] -s: Skip autoexpand - -i TOOL: Zip tool to use to compress image. TOOLS can be one of gtip pigz xz pxz (Default: gzip) + -i TOOL: Zip tool to use to compress image. TOOLS can be one of gzip pigz xz pxz (Default: gzip) -d: Debug mode on -r: Use advanced repair options -z: Compress image after shrinking @@ -18,8 +19,7 @@ If you specify the `newimagefile.img` parameter, the script will make a copy of * `-s` will skip the autoexpanding part of the process. * `-d` will create a logfile `pishrink.log` which may help for problem analysis. * `-r` will attempt to repair the filesystem if regular repairs fail -* `-z` will Gzip compress the image after shrinking. The `.gz` extension will be added to the filename. - +* `-z` will compress the image after shrinking using gzip, pigz, xz or pxz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. ## Prerequisites ## If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. From 965fc206d7c093b98ea77d534ba3322c43e95f25 Mon Sep 17 00:00:00 2001 From: framp Date: Sun, 12 Jan 2020 10:28:33 +0100 Subject: [PATCH 05/14] Fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78fb10b..eb75bee 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ In addition the shrinked image can be compressed with gzip, pigz, xz or pxz to c ## Usage ## ``` -sudo pishrink.sh [-sdrzh] imagefile.img [newimagefile.img] +sudo pishrink.sh [-sdirzh] imagefile.img [newimagefile.img] -s: Skip autoexpand -i TOOL: Zip tool to use to compress image. TOOLS can be one of gzip pigz xz pxz (Default: gzip) -d: Debug mode on From ab320089812913f2a5ff368dae02c411396b9321 Mon Sep 17 00:00:00 2001 From: framp Date: Fri, 7 Feb 2020 16:49:22 +0100 Subject: [PATCH 06/14] Added support for gzip and xz only --- README.md | 19 ++++++++++------ pishrink.sh | 65 +++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index eb75bee..1f992ae 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,14 @@ In addition the shrinked image can be compressed with gzip, pigz, xz or pxz to c ## Usage ## ``` sudo pishrink.sh [-sdirzh] imagefile.img [newimagefile.img] - -s: Skip autoexpand - -i TOOL: Zip tool to use to compress image. TOOLS can be one of gzip pigz xz pxz (Default: gzip) - -d: Debug mode on - -r: Use advanced repair options - -z: Compress image after shrinking - -h: display help text + -s Don't expand filesystem when image is booted the first time + -d Write debug messages in a debug log file + -r Use advanced filesystem repair option if the normal one fails + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -v Be verbose + -z Compress image after shrinking + -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) + -a Use parallel compress feature of compression tool ``` If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. @@ -19,7 +21,10 @@ If you specify the `newimagefile.img` parameter, the script will make a copy of * `-s` will skip the autoexpanding part of the process. * `-d` will create a logfile `pishrink.log` which may help for problem analysis. * `-r` will attempt to repair the filesystem if regular repairs fail -* `-z` will compress the image after shrinking using gzip, pigz, xz or pxz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. +* `-z` will compress the image after shrinking using gzip or xz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. +* `-a` will use option -f9 for gzip and option -T0 for xz. + +Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variable. ## Prerequisites ## If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. diff --git a/pishrink.sh b/pishrink.sh index e3f7555..ba97b86 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -2,16 +2,17 @@ version="v0.1.2" -CURRENT_DIR=$(pwd) +CURRENT_DIR="$(pwd)" SCRIPTNAME="${0##*/}" -LOGFILE=${CURRENT_DIR}/${SCRIPTNAME%.*}.log +MYNAME="${SCRIPTNAME%.*}" +LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log" REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs" -ZIPTOOLS=("gzip pigz xz pxz") -declare -A ZIPOPTIONS=( [gzip]="-f9" [pigz]="-9" [xz]="-9" [pxz]="-9") # options for zip tools -declare -A ZIPEXTENSIONS=( [gzip]="gz" [pigz]="gz" [xz]="xz" [pxz]="xz") # extensions of zipped files +ZIPTOOLS=("gzip xz") +declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode +declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files function info() { - echo "$SCRIPTNAME: $1..." + echo "$SCRIPTNAME: $1 ..." } function error() { @@ -67,14 +68,16 @@ fi help() { local help read -r -d '' help << EOM -Usage: $0 [-sdirpzh] imagefile.img [newimagefile.img] +Usage: $0 [-sdiarpzvh] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time -d Write debug messages in a debug log file - -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) -r Use advanced filesystem repair option if the normal one fails -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -v Be verbose -z Compress image after shrinking + -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) + -a Use parallel compress feature of compression tool EOM echo "$help" exit -1 @@ -84,19 +87,24 @@ should_skip_autoexpand=false debug=false repair=false compress=false +parallel=false +verbose=false prep=false ziptool="gzip" required_tools="$REQUIRED_TOOLS" -while getopts ":si:drpzh" opt; do +while getopts ":adhipr:svz" opt; do case "${opt}" in - s) should_skip_autoexpand=true ;; + a) parallel=true;; d) debug=true;; - r) repair=true;; - p) prep=true;; - z) compress=true;; + h) help;; i) ziptool="$OPTARG";; - h,*) help;; + p) prep=true;; + r) repair=true;; + s) should_skip_autoexpand=true ;; + z) compress=true;; + v) verbose=true;; + *) help;; esac done shift $((OPTIND-1)) @@ -175,7 +183,6 @@ currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut - blocksize="$(echo "$tune2fs_output" | grep '^Block size:' | tr -d ' ' | cut -d ':' -f 2)" logVariables $LINENO beforesize parted_output partnum partstart tune2fs_output currentsize blocksize - #Check if we should make pi expand rootfs on next boot if [ "$should_skip_autoexpand" = false ]; then #Make pi expand rootfs on next boot @@ -186,6 +193,7 @@ if [ "$should_skip_autoexpand" = false ]; then echo "Creating new /etc/rc.local" mv "$mountdir/etc/rc.local" "$mountdir/etc/rc.local.bak" #####Do not touch the following lines##### + cat <<\EOF1 > "$mountdir/etc/rc.local" #!/bin/bash do_expand_rootfs() { @@ -333,10 +341,29 @@ if ! truncate -s "$endresult" "$img"; then fi if [[ $compress == true ]]; then - info "Using $ziptool on the shrunk image using options ${ZIPOPTIONS[$ziptool]}" - if [[ ! $($ziptool ${ZIPOPTIONS[$ziptool]} "$img") ]]; then - img=$img.${ZIPEXTENSIONS[$ziptool]} - fi + options="" + envVarname="${MYNAME^^}_${ziptool^^}" + if [[ $parallel == true ]]; then + options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" + [[ -v $envVarname ]] && options="${!envVarname}" + [[ $verbose == true ]] && options="$options -v" + info "Using $ziptool on the shrunk image using options ${options}" + if ! $ziptool ${options} "$img"; then + rc=$? + error $LINENO "$ziptool failed with rc $rc" + exit -18 + fi + else # sequential + info "Using $ziptool on the shrunk image" + [[ -v $envVarname ]] && options="${!envVarname}" + [[ $verbose == true ]] && options="$options -v" + if ! $ziptool ${options} $img; then + rc=$? + error $LINENO "$ziptool failed with rc $rc" + exit -19 + fi + fi + img=$img.${ZIPEXTENSIONS[$ziptool]} fi aftersize=$(ls -lh "$img" | cut -d ' ' -f 5) From c7d74a73e746a988c6c03bed856f59ce222361c7 Mon Sep 17 00:00:00 2001 From: framp Date: Fri, 7 Feb 2020 17:30:38 +0100 Subject: [PATCH 07/14] Use -Z and -z instead of -i and use pigz now --- README.md | 138 ++++++++++++++++++++++++++-------------------------- pishrink.sh | 45 +++++++++-------- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 1f992ae..e3502c0 100644 --- a/README.md +++ b/README.md @@ -1,68 +1,70 @@ - -# PiShrink # -PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. -In addition the shrinked image can be compressed with gzip, pigz, xz or pxz to create an even smaller image. - -## Usage ## -``` -sudo pishrink.sh [-sdirzh] imagefile.img [newimagefile.img] - -s Don't expand filesystem when image is booted the first time - -d Write debug messages in a debug log file - -r Use advanced filesystem repair option if the normal one fails - -p Remove logs, apt archives, dhcp leases and ssh hostkeys - -v Be verbose - -z Compress image after shrinking - -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) - -a Use parallel compress feature of compression tool -``` - -If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. - -* `-s` will skip the autoexpanding part of the process. -* `-d` will create a logfile `pishrink.log` which may help for problem analysis. -* `-r` will attempt to repair the filesystem if regular repairs fail -* `-z` will compress the image after shrinking using gzip or xz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. -* `-a` will use option -f9 for gzip and option -T0 for xz. - -Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variable. - -## Prerequisites ## -If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. - -If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. - -## Installation ## -```bash -wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh -chmod +x pishrink.sh -sudo mv pishrink.sh /usr/local/bin -``` - -## Example ## -```bash -[user@localhost PiShrink]$ sudo pishrink.sh pi.img -e2fsck 1.42.9 (28-Dec-2013) -Pass 1: Checking inodes, blocks, and sizes -Pass 2: Checking directory structure -Pass 3: Checking directory connectivity -Pass 4: Checking reference counts -Pass 5: Checking group summary information -/dev/loop1: 88262/1929536 files (0.2% non-contiguous), 842728/7717632 blocks -resize2fs 1.42.9 (28-Dec-2013) -resize2fs 1.42.9 (28-Dec-2013) -Resizing the filesystem on /dev/loop1 to 773603 (4k) blocks. -Begin pass 2 (max = 100387) -Relocating blocks XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -Begin pass 3 (max = 236) -Scanning inode table XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -Begin pass 4 (max = 7348) -Updating inode references XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -The filesystem on /dev/loop1 is now 773603 blocks long. - -Shrunk pi.img from 30G to 3.1G -``` - -## Contributing ## -If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. - -Pull requests for new features and bug fixes are more than welcome! + +# PiShrink # +PiShrink is a bash script that automatically shrink a pi image that will then resize to the max size of the SD card on boot. This will make putting the image back onto the SD card faster and the shrunk images will compress better. +In addition the shrinked image can be compressed with gzip and xz to create an even smaller image. Parallel compression of the image +using multiple cores is supported. + +## Usage ## +``` +Usage: $0 [-sdiarpzZvh] imagefile.img [newimagefile.img] + + -s Don't expand filesystem when image is booted the first time + -d Write debug messages in a debug log file + -r Use advanced filesystem repair option if the normal one fails + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -v Be verbose + -z Compress image after shrinking with gzip + -Z Compress image after shrinking with xz + -a Compress image in parallel using multiple cores +``` + +If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. + +* `-s` will skip the autoexpanding part of the process. +* `-d` will create a logfile `pishrink.log` which may help for problem analysis. +* `-r` will attempt to repair the filesystem if regular repairs fail +* `-z` will compress the image after shrinking using gzip or xz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. +* `-a` will use option -f9 for pigz and option -T0 for xz. + +Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables. + +## Prerequisites ## +If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. + +If using Ubuntu, you will likely see an error about `e2fsck` being out of date and `metadata_csum`. The simplest fix for this is to use Ubuntu 16.10 and up, as it will save you a lot of hassle in the long run. + +## Installation ## +```bash +wget https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh +chmod +x pishrink.sh +sudo mv pishrink.sh /usr/local/bin +``` + +## Example ## +```bash +[user@localhost PiShrink]$ sudo pishrink.sh pi.img +e2fsck 1.42.9 (28-Dec-2013) +Pass 1: Checking inodes, blocks, and sizes +Pass 2: Checking directory structure +Pass 3: Checking directory connectivity +Pass 4: Checking reference counts +Pass 5: Checking group summary information +/dev/loop1: 88262/1929536 files (0.2% non-contiguous), 842728/7717632 blocks +resize2fs 1.42.9 (28-Dec-2013) +resize2fs 1.42.9 (28-Dec-2013) +Resizing the filesystem on /dev/loop1 to 773603 (4k) blocks. +Begin pass 2 (max = 100387) +Relocating blocks XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +Begin pass 3 (max = 236) +Scanning inode table XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +Begin pass 4 (max = 7348) +Updating inode references XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +The filesystem on /dev/loop1 is now 773603 blocks long. + +Shrunk pi.img from 30G to 3.1G +``` + +## Contributing ## +If you find a bug please create an issue for it. If you would like a new feature added, you can create an issue for it but I can't promise that I will get to it. + +Pull requests for new features and bug fixes are more than welcome! diff --git a/pishrink.sh b/pishrink.sh index ba97b86..8f23190 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -8,6 +8,7 @@ MYNAME="${SCRIPTNAME%.*}" LOGFILE="${CURRENT_DIR}/${SCRIPTNAME%.*}.log" REQUIRED_TOOLS="parted losetup tune2fs md5sum e2fsck resize2fs" ZIPTOOLS=("gzip xz") +declare -A ZIP_PARALLEL_TOOL=( [gzip]="pigz" [xz]="xz" ) # parallel zip tool to use in parallel mode declare -A ZIP_PARALLEL_OPTIONS=( [gzip]="-f9" [xz]="-T0" ) # options for zip tools in parallel mode declare -A ZIPEXTENSIONS=( [gzip]="gz" [xz]="xz" ) # extensions of zipped files @@ -68,16 +69,16 @@ fi help() { local help read -r -d '' help << EOM -Usage: $0 [-sdiarpzvh] imagefile.img [newimagefile.img] +Usage: $0 [-sdarpzZvh] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time -d Write debug messages in a debug log file -r Use advanced filesystem repair option if the normal one fails -p Remove logs, apt archives, dhcp leases and ssh hostkeys -v Be verbose - -z Compress image after shrinking - -i TOOL Zip tool to use to compress image. TOOLS can be one of $ZIPTOOLS (Default: gzip) - -a Use parallel compress feature of compression tool + -z Compress image after shrinking with gzip + -Z Compress image after shrinking with xz + -a Compress image in parallel using multiple cores EOM echo "$help" exit -1 @@ -86,11 +87,10 @@ EOM should_skip_autoexpand=false debug=false repair=false -compress=false parallel=false verbose=false prep=false -ziptool="gzip" +ziptool="" required_tools="$REQUIRED_TOOLS" while getopts ":adhipr:svz" opt; do @@ -98,11 +98,11 @@ while getopts ":adhipr:svz" opt; do a) parallel=true;; d) debug=true;; h) help;; - i) ziptool="$OPTARG";; p) prep=true;; r) repair=true;; s) should_skip_autoexpand=true ;; - z) compress=true;; + z) ziptool="gzip";; + Z) ziptool="xz";; v) verbose=true;; *) help;; esac @@ -137,12 +137,16 @@ if (( EUID != 0 )); then fi # check selected compression tool is supported and installed -if [[ $compress == true ]]; then +if [[ -n $ziptool ]]; then if [[ ! " ${ZIPTOOLS[@]} " =~ " $ziptool " ]]; then error $LINENO "$ziptool is an unsupported ziptool." exit -17 else - REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool" + if [[ $parallel == true && ziptool == "gzip" ]]; then + REQUIRED_TOOLS="$REQUIRED_TOOLS pigz" + else + REQUIRED_TOOLS="$REQUIRED_TOOLS $ziptool" + fi fi fi @@ -340,23 +344,24 @@ if ! truncate -s "$endresult" "$img"; then exit -16 fi -if [[ $compress == true ]]; then +if [[ -n $ziptool ]]; then options="" - envVarname="${MYNAME^^}_${ziptool^^}" + envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override options + [[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" + [[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options + [[ $verbose == true ]] && options="$options -v" # add verbose flag + if [[ $parallel == true ]]; then - options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" - [[ -v $envVarname ]] && options="${!envVarname}" - [[ $verbose == true ]] && options="$options -v" - info "Using $ziptool on the shrunk image using options ${options}" - if ! $ziptool ${options} "$img"; then + parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}" + info "Using $parallel_tool on the shrunk image" + if ! $parallel_tool ${options} "$img"; then rc=$? - error $LINENO "$ziptool failed with rc $rc" + error $LINENO "$parallel_tool failed with rc $rc" exit -18 fi + else # sequential info "Using $ziptool on the shrunk image" - [[ -v $envVarname ]] && options="${!envVarname}" - [[ $verbose == true ]] && options="$options -v" if ! $ziptool ${options} $img; then rc=$? error $LINENO "$ziptool failed with rc $rc" From fda7a38ba15331f8219d1df7f163cdd4bcf58ad7 Mon Sep 17 00:00:00 2001 From: framp Date: Fri, 7 Feb 2020 17:47:09 +0100 Subject: [PATCH 08/14] Updated README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e3502c0..12f8e78 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ If you specify the `newimagefile.img` parameter, the script will make a copy of * `-s` will skip the autoexpanding part of the process. * `-d` will create a logfile `pishrink.log` which may help for problem analysis. * `-r` will attempt to repair the filesystem if regular repairs fail -* `-z` will compress the image after shrinking using gzip or xz. Default is `gzip`. `.gz` or `.xz` extension will be added to the filename. +* `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename. +* `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename. * `-a` will use option -f9 for pigz and option -T0 for xz. Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables. From b4db6c1a64fa368cdb210f05f9486b7988be907d Mon Sep 17 00:00:00 2001 From: framp Date: Thu, 27 Feb 2020 16:42:43 +0100 Subject: [PATCH 09/14] Patch from OmegaSquad82 --- pishrink.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pishrink.sh b/pishrink.sh index 8f23190..c8e589d 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -93,7 +93,7 @@ prep=false ziptool="" required_tools="$REQUIRED_TOOLS" -while getopts ":adhipr:svz" opt; do +while getopts ":adhipr:svzZ" opt; do case "${opt}" in a) parallel=true;; d) debug=true;; From d80d48a6459d070f2d435417ae73a37834399b54 Mon Sep 17 00:00:00 2001 From: framp Date: Sat, 29 Feb 2020 14:52:35 +0100 Subject: [PATCH 10/14] Minor code cleanup and README update --- README.md | 15 +++++++-------- pishrink.sh | 17 +++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 12f8e78..d2a84e1 100644 --- a/README.md +++ b/README.md @@ -6,28 +6,27 @@ using multiple cores is supported. ## Usage ## ``` -Usage: $0 [-sdiarpzZvh] imagefile.img [newimagefile.img] +Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time - -d Write debug messages in a debug log file - -r Use advanced filesystem repair option if the normal one fails - -p Remove logs, apt archives, dhcp leases and ssh hostkeys -v Be verbose + -r Use advanced filesystem repair option if the normal one fails -z Compress image after shrinking with gzip -Z Compress image after shrinking with xz -a Compress image in parallel using multiple cores + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -d Write debug messages in a debug log file ``` If you specify the `newimagefile.img` parameter, the script will make a copy of `imagefile.img` and work off that. You will need enough space to make a full copy of the image to use that option. -* `-s` will skip the autoexpanding part of the process. -* `-d` will create a logfile `pishrink.log` which may help for problem analysis. * `-r` will attempt to repair the filesystem if regular repairs fail * `-z` will compress the image after shrinking using gzip. `.gz` extension will be added to the filename. * `-Z` will compress the image after shrinking using xz. `.xz` extension will be added to the filename. -* `-a` will use option -f9 for pigz and option -T0 for xz. +* `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel. +* `-d` will create a logfile `pishrink.log` which may help for problem analysis. -Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables. +Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. ## Prerequisites ## If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. diff --git a/pishrink.sh b/pishrink.sh index c8e589d..33f8e07 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -69,16 +69,16 @@ fi help() { local help read -r -d '' help << EOM -Usage: $0 [-sdarpzZvh] imagefile.img [newimagefile.img] +Usage: $0 [-adhrspvzZ] imagefile.img [newimagefile.img] -s Don't expand filesystem when image is booted the first time - -d Write debug messages in a debug log file - -r Use advanced filesystem repair option if the normal one fails - -p Remove logs, apt archives, dhcp leases and ssh hostkeys -v Be verbose + -r Use advanced filesystem repair option if the normal one fails -z Compress image after shrinking with gzip -Z Compress image after shrinking with xz -a Compress image in parallel using multiple cores + -p Remove logs, apt archives, dhcp leases and ssh hostkeys + -d Write debug messages in a debug log file EOM echo "$help" exit -1 @@ -93,7 +93,7 @@ prep=false ziptool="" required_tools="$REQUIRED_TOOLS" -while getopts ":adhipr:svzZ" opt; do +while getopts ":adhprsvzZ" opt; do case "${opt}" in a) parallel=true;; d) debug=true;; @@ -101,9 +101,9 @@ while getopts ":adhipr:svzZ" opt; do p) prep=true;; r) repair=true;; s) should_skip_autoexpand=true ;; + v) verbose=true;; z) ziptool="gzip";; Z) ziptool="xz";; - v) verbose=true;; *) help;; esac done @@ -344,12 +344,13 @@ if ! truncate -s "$endresult" "$img"; then exit -16 fi +# handle compression if [[ -n $ziptool ]]; then options="" - envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override options + envVarname="${MYNAME^^}_${ziptool^^}" # PISHRINK_GZIP or PISHRINK_XZ environment variables allow to override all options for gzip or xz [[ $parallel == true ]] && options="${ZIP_PARALLEL_OPTIONS[$ziptool]}" [[ -v $envVarname ]] && options="${!envVarname}" # if environment variable defined use these options - [[ $verbose == true ]] && options="$options -v" # add verbose flag + [[ $verbose == true ]] && options="$options -v" # add verbose flag if requested if [[ $parallel == true ]]; then parallel_tool="${ZIP_PARALLEL_TOOL[$ziptool]}" From f337674f580edb00b558bfd020d595fa4d5277a6 Mon Sep 17 00:00:00 2001 From: OmegaSquad82 <34405234+OmegaSquad82@users.noreply.github.com> Date: Sun, 1 Mar 2020 19:22:52 +0100 Subject: [PATCH 11/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2a84e1..77e2930 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ If you specify the `newimagefile.img` parameter, the script will make a copy of * `-a` will use option -f9 for pigz and option -T0 for xz and compress in parallel. * `-d` will create a logfile `pishrink.log` which may help for problem analysis. -Default options for parallel compression can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. +Default options for compressors can be overwritten by defining PISHRINK_GZIP or PSHRINK_XZ environment variables for gzip and xz. ## Prerequisites ## If you are trying to shrink a [NOOBS](https://github.com/raspberrypi/noobs) image it will likely fail. This is due to [NOOBS partitioning](https://github.com/raspberrypi/noobs/wiki/NOOBS-partitioning-explained) being significantly different than Raspbian's. Hopefully PiShrink will be able to support NOOBS in the near future. From c99b053afb0f7ce2f4416a8c225d467fbb087c5d Mon Sep 17 00:00:00 2001 From: kmpm Date: Thu, 9 Apr 2020 18:17:47 +0200 Subject: [PATCH 12/14] stop on parted error --- pishrink.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pishrink.sh b/pishrink.sh index d2f922a..4ddce81 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -160,7 +160,13 @@ trap cleanup ERR EXIT #Gather info info "Gathering data" beforesize=$(ls -lh "$img" | cut -d ' ' -f 5) -parted_output=$(parted -ms "$img" unit B print | tail -n 1) +if ! parted_output=$(parted -ms "$img" unit B print); then + rc=$? + error $LINENO "parted failed with rc $rc" + info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate" + exit -6 +fi +parted_output=$(echo "$parted_output" | tail -n 1) partnum=$(echo "$parted_output" | cut -d ':' -f 1) partstart=$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B') loopback=$(losetup -f --show -o "$partstart" "$img") From 9a2c7fe949ec2ffed410aa1e3adf520d4b85ef4a Mon Sep 17 00:00:00 2001 From: framp Date: Mon, 20 Apr 2020 13:24:55 +0200 Subject: [PATCH 13/14] Fixed pishrink issue #133 --- pishrink.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pishrink.sh b/pishrink.sh index 6551e28..5f04ae5 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -178,14 +178,14 @@ trap cleanup ERR EXIT #Gather info info "Gathering data" beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)" -if ! parted_output=$(parted -ms "$img" unit B print); then - rc=$? +if ! parted_output="$(parted -ms "$img" unit B print)"; then + rc=$? error $LINENO "parted failed with rc $rc" - info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate" + info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate" exit -6 fi partnum="$(echo "$parted_output" | cut -d ':' -f 1)" -partstart="$(echo "$parted_output" | cut -d ':' -f 2 | tr -d 'B')" +partstart="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 2 | tr -d 'B')" loopback="$(losetup -f --show -o "$partstart" "$img")" tune2fs_output="$(tune2fs -l "$loopback")" currentsize="$(echo "$tune2fs_output" | grep '^Block count:' | tr -d ' ' | cut -d ':' -f 2)" From 3c3f68f59fb1c11b38d22dccc5714b1d92f083f1 Mon Sep 17 00:00:00 2001 From: framp Date: Mon, 20 Apr 2020 14:58:25 +0200 Subject: [PATCH 14/14] Added missing tail when retrieving partnum --- pishrink.sh | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/pishrink.sh b/pishrink.sh index 5f04ae5..7e308aa 100755 --- a/pishrink.sh +++ b/pishrink.sh @@ -178,13 +178,14 @@ trap cleanup ERR EXIT #Gather info info "Gathering data" beforesize="$(ls -lh "$img" | cut -d ' ' -f 5)" -if ! parted_output="$(parted -ms "$img" unit B print)"; then - rc=$? +parted_output="$(parted -ms "$img" unit B print)" +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" info "Possibly invalid image. Run 'parted $img unit B print' manually to investigate" exit -6 fi -partnum="$(echo "$parted_output" | cut -d ':' -f 1)" +partnum="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 1)" partstart="$(echo "$parted_output" | tail -n 1 | cut -d ':' -f 2 | tr -d 'B')" loopback="$(losetup -f --show -o "$partstart" "$img")" tune2fs_output="$(tune2fs -l "$loopback")" @@ -308,8 +309,9 @@ logVariables $LINENO minsize #Shrink filesystem info "Shrinking filesystem" resize2fs -p "$loopback" $minsize -if [[ $? != 0 ]]; then - error $LINENO "resize2fs failed" +rc=$? +if (( $rc )); then + error $LINENO "resize2fs failed with rc $rc" mount "$loopback" "$mountdir" mv "$mountdir/etc/rc.local.bak" "$mountdir/etc/rc.local" umount "$mountdir" @@ -322,30 +324,34 @@ sleep 1 partnewsize=$(($minsize * $blocksize)) newpartend=$(($partstart + $partnewsize)) logVariables $LINENO partnewsize newpartend -if ! parted -s -a minimal "$img" rm "$partnum"; then - rc=$? +parted -s -a minimal "$img" rm "$partnum" +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" exit -13 fi -if ! parted -s "$img" unit B mkpart primary "$partstart" "$newpartend"; then - rc=$? +parted -s "$img" unit B mkpart primary "$partstart" "$newpartend" +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" exit -14 fi #Truncate the file info "Shrinking image" -if ! endresult=$(parted -ms "$img" unit B print free); then - rc=$? +endresult=$(parted -ms "$img" unit B print free) +rc=$? +if (( $rc )); then error $LINENO "parted failed with rc $rc" exit -15 fi endresult=$(tail -1 <<< "$endresult" | cut -d ':' -f 2 | tr -d 'B') logVariables $LINENO endresult -if ! truncate -s "$endresult" "$img"; then - rc=$? +truncate -s "$endresult" "$img" +rc=$? +if (( $rc )); then error $LINENO "trunate failed with rc $rc" exit -16 fi