various changes around data volumes

[distro-setup] / btrbk-run

#!/bin/bash

set -eE -o pipefail
trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR

[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@"
usage() {
    echo "top of script file:"
    sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
    exit $1
}

script_dir=$(dirname $(readlink -f "$BASH_SOURCE"))

# todo: finish figuring out fai / distro-setup
# initial fstab / subvol setup.

conf_only=false
dry_run=false # mostly for testing
resume_arg=

temp=$(getopt -l help hcnrt: "$@") || usage 1
eval set -- "$temp"
while true; do
    case $1 in
        -c) conf_only=true; shift ;;
        -n) dry_run=true; dry_run_arg=-n; shift ;;
        -r) resume_arg=-r; shift ;;
        -t) IFS=, targets=($2); shift 2 ;;
        -h|--help) usage ;;
        --) shift; break ;;
        *) echo "$0: Internal error!" ; exit 1 ;;
    esac
done
read primary <<<"$@"

##### end command line parsing ########

rsync-dirs() {
    local host=$1
    local path=$2
    rsync $dry_run_arg -ahi --relative --delete "$path" "root@$host:/"
}

vol-conf() {
    cat >>/etc/btrbk.conf <<EOF
volume $vol
EOF
}
sub-conf() {
    cat >>/etc/btrbk.conf <<EOF
subvolume $sub
EOF
}
tg-conf() {
    cat >>/etc/btrbk.conf <<EOF
target send-receive ssh://$tg$vol/btrbk
EOF
}



if ! which btrbk &>/dev/null; then
    echo "$0: error: no btrbk binary found"
fi

cat >/etc/btrbk.conf <<'EOF'
ssh_identity /root/.ssh/id_rsa

# so we only run one at a time
lockfile                   /var/lock/btrbk.lock

# default format of short does not accomidate hourly preservation setting
timestamp_format long-iso

# only make a snapshot if things have changed
snapshot_create onchange
# much less snapshots because I have less space on the
# local filesystem.
#snapshot_preserve 2h 2d
# for now, keeping them equal for simplicity sake
snapshot_preserve 48h 14d 8w 24m
snapshot_preserve_min 6h
snapshot_dir btrbk

# so, total backups = ~89
target_preserve 48h 14d 8w 24m
target_preserve_min 6h

# if something fails and it's not obvious, try doing
# btrbk -l debug -v dryrun
EOF

# note q is owned by root:1000
# note p is owned 1000:1000 and chmod 700
mountpoints=(/q)
if awk '{print $2}' /etc/fstab | grep -xF /p &>/dev/null; then
    mountpoints+=(/p)
fi

# if our mountpoints are from stale snapshots,
# it doesn't make sense to do a backup.
check-subvol-stale ${mountpoints[@]} || exit 1

if [[ ! $targets ]]; then
    case $HOSTNAME in
        tp|x2)
            if ! timeout -s 9 10 ssh frodo :; then
                targets=($HOME_DOMAIN)
            fi
            ;;
    esac
    targets=(frodo)
fi


# for i, we just do a 1 way sync from master to backup,
# and manually manage any changes to that.
do_i=false
for tg in ${targets[@]}; do
    # for an initial run, btrbk requires the dir to exist
    ssh root@$tg mkdir -p /mnt/root/btrbk
    if [[ $tg == frodo && $HOSTNAME == treetowl ]]; then
        do_i=true
    fi
done


vol=/mnt/root
vol-conf
for m in ${mountpoints[@]}; do
    sub=${m##*/}
    sub-conf
    for tg in ${targets[@]}; do
        tg-conf
    done
done

if $do_i; then
    vol=/mnt/iroot
    vol-conf
    sub=i
    sub-conf
    tg=frodo
    vol=/mnt/root
    tg-conf
fi



# todo: umount first to ensure we don't have any errors
# todo: do some kill fuser stuff to make umount more reliable
# todo: run this on a systemd timer on $primary, once per hour,
# and if primary is, change that timer over to primary, and make
# sure we mount the latest



if $conf_only; then
    exit
fi

if $dry_run; then
    btrbk -n $resume_arg run
else
    # -q and just using the syslog option seemed nice,
    # but it doesn't show when a send has a parent and when it doesn't.
    btrbk $resume_arg run
fi

# if we have /p, rsync to targets without /p
if mountpoint /p >/dev/null; then
    for tg in ${targets[@]}; do
        case $tg in
            tp|li|lk)
                for x in /p/c/machine_specific/*.hosts; do
                    if grep -qxF $tg $x; then
                        dir=${x%.hosts}
                        rsync-dirs $tg $dir
                    fi
                done
                ;;
        esac
    done
fi

if ! $dry_run; then
    $script_dir/mount-latest-remote ${targets[@]}
fi


# todo: move variable data we don't care about backing up
# to /nocow and symlink it.


# background on btrbk timezones. with short/long, timestamps use local time.
# for long, if your local time moves backwards, by moving timezones or
# for an hour when daylight savings changes it, you will temporarily get
# a more aggressive retention policy for the overlapping period, and
# vice versa for the opposite timezone move. The alternative is using
# long-iso, which puts timezone info into the timestamp, which means
# that instead of shifting time, you shift the start of day/week/month
# which is used for retention to your new local time, which means for
# example, if you moved forward by 8 hours, the daily/weekly/monthly
# retention will be 8 hours more aggressive since midnight is at a new
# time, unless you fake the timzeone using the TZ env variable.
# However, in the short term, there will be no inconsistencies.
# I don't see any problem with shifting when the day starts for
# retention, so I'm using long-iso.

# note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z