X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=backup.sh;h=e64312f529f68b3021a2751c2c75ef186092f85a;hb=875b0af73ff25c6f85097b05820ae9f0c0894486;hp=f4ae98bb0ab7dde18f3aee1df337786472ed50ae;hpb=d3303892e4f393d994804016060a8bc704acfebb;p=backup.sh diff --git a/backup.sh b/backup.sh index f4ae98b..e64312f 100755 --- a/backup.sh +++ b/backup.sh @@ -18,27 +18,15 @@ # The script is halfway Norwegian and halfway English. Newer modifications are # in English, we should probably stick to that. -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # Absolute directory the script lives in -confdir=$SCRIPTPATH/conf/ # Directory configuration files -[ -r $confdir/env ] && . $confdir/env - -# Default configuration; used unless $confdir/env has set something different. -storagedir=${storagedir:-/backup} # Where we keep backups -exclude=${exclude:-$confdir/exclude} # List of exclude patterns -maxnumfull=${maxnumfull:-3} # Number of full backups -daysbetweenfull=${daysbetweenfull:-30} # Days between full backups - -# Days between full backups for machines with fixed full-backup date -# (in case the fixed day gets missed). -daysbetweenfullforfixed=${daysbetweenfullforfixed:-32} - -# End of configuration. - computer=$1 # The computer to backup. +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # Absolute directory the script lives in +confdir=$SCRIPTPATH/conf # Directory for configuration files +conffile=$confdir/backup.config LOCKFILE=$SCRIPTPATH/backuprun.lockfile.$computer DAY_OF_MONTH=`date "+%d" | sed s/^0//` # from 1 to 31 DATE=`date "+%Y%m%d%H%M"` # format: touch DATEs=`date "+%Y-%m-%d %H:%M"` # format: tar +TOP_PID=$$ if [ -z "$computer" ]; then echo "Usage: $0 COMPUTER" @@ -64,6 +52,22 @@ warnmsg() { infomsg() { echo `date`": $computer: $@" >&2 } +findconf() { # getconf is already taken + key=$1 + computer=$2 + filesystem=$( echo $3 | tr / . ) + + if [ "$computer" ] && [ "$filesystem" ] && git config -f $conffile $computer$filesystem.$key > /dev/null; then + git config -f $conffile $computer$filesystem.$key + elif [ "$computer" ] && git config -f $conffile $computer.$key > /dev/null; then + git config -f $conffile $computer.$key + elif git config -f $conffile defaults.$key > /dev/null; then + git config -f $conffile defaults.$key + else + warnmsg "No configuration found for $key in $conffile, aborting." + kill -TERM $TOP_PID + fi +} # Trap C-c and kill trap die SIGINT SIGTERM @@ -86,11 +90,11 @@ backup() localsnar="${storagedir}/${computer}/${sfilesystem}/.incremental.snar" remotesnar="$remotehome/.backup/${sfilesystem}.snar" - if [ "$backuplevel" = "daglig" ]; then + if [ "$backuplevel" = "daily" ]; then # If incremental backup, we need to copy the incremental status to $computer. # If it does not exist, whine a bit and then run date-based instead. if [ -s "$localsnar" ]; then - if ! scp $localsnar $username@$computer:$remotesnar; then + if ! scp $localsnar $username@$computer:$remotesnar >&2; then diemsg "Could not copy .incremental.snar to $computer" fi lastcmd="--listed-incremental=$remotesnar --no-check-device" @@ -107,6 +111,19 @@ backup() lastcmd="--listed-incremental=$remotesnar" fi + # The file is usually stored to disk with a simple cat > $TARFILE, + # but in some cases, like backing up to a remote and untrusted + # file system, it could be useful to replace it with e.g. + # a GPG command line. + storageprogram=$( findconf storageprogram $computer $sfilesystem ) + + # Find compressor and decompressor for this filesystem. + fscompressor=$( findconf compressor $computer $filesystem ) + if [ -z "$fscompressor" ]; then + fscompressor=$compressor + fi + decompressor=$( findconf decompressor $computer $filesystem ) + # We try to run tar on the remote computer # c create archive # C change to directory first @@ -118,8 +135,8 @@ backup() # Pipe the stuff over ssh to ourselves, run pee to cat the contents to a # file and build a file list, respectivly. TARFILE=$DATE.tmp - TARCMD="ssh $username@$computer \"nice -n 19 ionice -c3 tar --one-file-system --use-compress-program $compressor -cf - -C $filesystem . $lastcmd \ - --exclude-from=$remotehome/.backup/exclude\" | pee \"cat > $TARFILE\" \"tar tzvf -\"" + TARCMD="ssh -n $username@$computer \"$nice tar --one-file-system --use-compress-program $fscompressor -cf - -C $filesystem . $lastcmd \ + --exclude-from=$remotehome/.backup/exclude\" | pee \"$storageprogram > $TARFILE\" \"$decompressor | tar tvf -\"" infomsg "Running $TARCMD" eval $TARCMD > $DATE.idx @@ -127,7 +144,7 @@ backup() # File is >0 in size and neither cat or tar tzvf failed; we assume it worked. if [ "$remotesnar" ]; then - if ! scp $username@$computer:$remotesnar $localsnar.tmp; then + if ! scp $username@$computer:$remotesnar $localsnar.tmp >&2; then diemsg "Could not copy .incremental.snar from $computer" fi fi @@ -146,7 +163,7 @@ backup() chmod 644 *idx 2>/dev/null #everyone can read # Let the remote computer know that we ran a successful backup (for nagios) - ssh $username@$computer touch $filesystem/.lastbackup + ssh -n $username@$computer touch $filesystem/.lastbackup else # Something wrong happened. rm $TARFILE @@ -161,11 +178,13 @@ backup() # - No touching of .lastbackup for Nagios. # - Failed tar operations may go undetected, since we only see the error code # the last process in the pipe chain, which is gpg. +# - No storageprogram support (see above), because the file is typically already +# encrypted. nonprivbackup() { infomsg "$computer:$filesystem $backuplevel non-privileged backup" - if [ "$backuplevel" = "daglig" ]; then + if [ "$backuplevel" = "daily" ]; then lastd=`cat ../.date` lastcmd="'$lastd'" else @@ -173,7 +192,7 @@ nonprivbackup() fi TARFILE=$DATE.tmp - TARCMD="ssh $username@$computer \"sudo /usr/local/sbin/output-encrypted-backup $filesystem $lastcmd\"" + TARCMD="ssh -n $username@$computer \"sudo /usr/local/sbin/output-encrypted-backup $filesystem $lastcmd\"" infomsg "Running $TARCMD" eval $TARCMD > $TARFILE @@ -195,6 +214,11 @@ nonprivbackup() fi } +# Get basic configuration. +storagedir=$( findconf storagedir $computer ) +username=$( findconf username $computer ) +exclude=$( findconf excludefile $computer ) + # Check that the target filesystem is mounted (actually check that it's not # the root filesystem) if [ ! -d "$storagedir/$computer" ]; then @@ -206,13 +230,6 @@ if [ "$rootfilesystem" == "$targetfilesystem" ]; then diemsg "Target filesystem ($storagedir/$computer) was mounted on /." fi -# Find the user to do the backup as. -if [ -f $confdir/username.$computer ] ; then - username=`cat $confdir/username.$computer` -else - username=root -fi - infomsg "Backing up $computer" # Try to SSH to the computer without entering a password. @@ -221,17 +238,28 @@ if ! ssh -n -o NumberOfPasswordPrompts=0 $username@$computer /bin/true; then fi # Find the home directory of the backup user -remotehome=`ssh $username@$computer "echo ~"` +remotehome=`ssh -n $username@$computer "echo ~"` if [ -z "$remotehome" ]; then diemsg "Could not expand ~ for user $username" fi -# Check if pigz is available -if ssh -n $username@$computer "pigz -V 2>/dev/null"; then - compressor=pigz +# Check if pigz is available. Note that this may be overridden per-filesystem later. +compressor=$( findconf compressor $computer ) +if [ -z "$compressor" ]; then + if ssh -n $username@$computer "pigz -V 2>/dev/null"; then + compressor=pigz + else + infomsg "pigz missing; falling back to gzip." + compressor=gzip + fi +fi + +# Check if nice and ionice are available +if ssh -n $username@$computer "nice -n 19 ionice -c3 /bin/true 2>/dev/null"; then + nice="nice -n 19 ionice -c3" else - infomsg "pigz missing; falling back to gzip." - compressor=gzip + infomsg "nice and/or ionice missing; will run at normal priority." + nice="" fi # Check dump bit in fstab @@ -242,12 +270,12 @@ filesystems=`ssh -n $username@$computer "cat /etc/fstab" \ | awk '{ if ( $(NF-1) != "0" ) print $2}' ` # Clean up our dir at this client -if ! ssh $username@$computer "rm -r $remotehome/.backup ; mkdir -m 700 $remotehome/.backup"; then +if ! ssh -n $username@$computer "rm -r $remotehome/.backup ; mkdir -m 700 $remotehome/.backup"; then diemsg "Could not create backup staging area at $computer:$remotehome/.backup" fi # Try to copy $exclude to $computer -if ! scp $exclude $username@$computer:$remotehome/.backup/exclude > /dev/null; then +if ! scp $exclude $username@$computer:$remotehome/.backup/exclude >&2; then diemsg "Could not copy exclude.txt to $computer" fi @@ -264,7 +292,7 @@ if [ -f $confdir/postexec.$computer ]; then fi # Try to run preexec if it exists -if ! ssh $username@$computer "[ ! -f $remotehome/.backup/preexec ] || /bin/bash -x $remotehome/.backup/preexec" >&2; then +if ! ssh -n $username@$computer "[ ! -f $remotehome/.backup/preexec ] || /bin/bash -x $remotehome/.backup/preexec" >&2; then diemsg "Could not run $computer:$remotehome/.backup/preexec" fi @@ -274,10 +302,10 @@ for filesystem in $filesystems; do # Prepare storage area mkdir -m 755 -p $storagedir/$computer/$sfilesystem/full 2>/dev/null - mkdir -m 755 -p $storagedir/$computer/$sfilesystem/daglig 2>/dev/null + mkdir -m 755 -p $storagedir/$computer/$sfilesystem/daily 2>/dev/null # Default backuplevel - backuplevel=daglig + backuplevel=daily if [ ! -s $storagedir/$computer/$sfilesystem/.date ]; then # Take the first full backup of this filesystem on this computer @@ -285,18 +313,14 @@ for filesystem in $filesystems; do fi # Check if we want a full backup - if [ -f $confdir/fastfullbackupdag.$computer.$sfilesystem ]; then - fullbackup_min_for_this_machine=$daysbetweenfullforfixed - if [ "$DAY_OF_MONTH" = "`cat $confdir/fastfullbackupdag.$computer.$sfilesystem`" ]; then - backuplevel=full - fi - elif [ -f $confdir/fastfullbackupdag.$computer ]; then - fullbackup_min_for_this_machine=$daysbetweenfullforfixed - if [ "$DAY_OF_MONTH" = "`cat $confdir/fastfullbackupdag.$computer`" ]; then + fixedfullbackupday=$( findconf fixedfullbackupday $computer $filesystem ) + if [ "$fixedfullbackupday" ]; then + fullbackup_min_for_this_machine=$( findconf daysbetweenfullforfixed $computer $filesystem ) + if [ "$DAY_OF_MONTH" = "$fixedfullbackupday" ]; then backuplevel=full fi else - fullbackup_min_for_this_machine=$daysbetweenfull + fullbackup_min_for_this_machine=$( findconf daysbetweenfull $computer $filesystem ) fi if [ -z "`find $storagedir/$computer/$sfilesystem/full/ -name \*tgz\* -mtime -$fullbackup_min_for_this_machine`" ]; then @@ -307,22 +331,15 @@ for filesystem in $filesystems; do cd $storagedir/$computer/$sfilesystem/$backuplevel || diemsg "$storagedir/$computer/$sfilesystem/$backuplevel does not exist" # Perform the actual backup - if [ "$username" = "root" ]; then - backup - else + if [ "$( findconf nonpriv $computer )" ]; then nonprivbackup - fi - - # Check if this box has a custom number of full backups - if [ -f $confdir/maksfulle.$computer.$sfilesystem ]; then - mf=$((`cat $confdir/maksfulle.$computer`+1)) - elif [ -f $confdir/maksfulle.$computer ] ; then - mf=$((`cat $confdir/maksfulle.$computer`+1)) else - mf=$(($maxnumfull+1)) + backup fi # Delete old full backups + mf=$( findconf maxnumfull $computer $filesystem ) + mf=$(( mf + 1 )) for full in `ls -1t $storagedir/$computer/$sfilesystem/full/*tgz* | tail -n +$mf`; do prefix=`echo $full | sed "s/\.[^.]*$//"` infomsg "$computer:$filesystem Deleting full backup $prefix" @@ -332,7 +349,7 @@ for filesystem in $filesystems; do # Delete incremental backups older than the oldest full backup oldf=`ls -t1 $storagedir/$computer/$sfilesystem/full/*tgz* | tail -1` find \ - $storagedir/$computer/$sfilesystem/daglig \ + $storagedir/$computer/$sfilesystem/daily \ -type f \ \! -newer $oldf \ -printf "`date`: $computer: Deleting old incremental backup: %p\n" \ @@ -340,7 +357,7 @@ for filesystem in $filesystems; do done # Try to run postexec if it exist -if ! ssh $username@$computer "[ ! -f $remotehome/.backup/postexec ] || /bin/bash -x $remotehome/.backup/postexec" >&2; then +if ! ssh -n $username@$computer "[ ! -f $remotehome/.backup/postexec ] || /bin/bash -x $remotehome/.backup/postexec" >&2; then diemsg "Could not run $computer:$remotehome/.backup/postexec" fi