X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=btrbk-run;h=70fc7043ed6520b4d353d8c6df0e4ee72df0f372;hb=50f25d1eec109c4596cfc638c9a7c98da0f0b5d0;hp=140f9e6053fdef509aace78d1186fee1c0abe26f;hpb=28a58715872fc0f002f268369c82085da0e25b3f;p=distro-setup
diff --git a/btrbk-run b/btrbk-run
index 140f9e6..70fc704 100644
--- a/btrbk-run
+++ b/btrbk-run
@@ -1,17 +1,22 @@
#!/bin/bash
-# Copyright (C) 2016 Ian Kelling
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Configure & run btrbk & related work on Ian's computers.
+# Copyright (C) 2024 Ian Kelling
-# http://www.apache.org/licenses/LICENSE-2.0
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+# SPDX-License-Identifier: GPL-3.0-or-later
# todo: if we cancel in the middle of a btrfs send, then run again
@@ -22,7 +27,7 @@
[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
-source /usr/local/lib/err
+set -e; . /usr/local/lib/bash-bear; set +e
usage() {
cat <<'EOF'
@@ -39,13 +44,98 @@ EOF
}
+pre=btrbk-run
script_name="${BASH_SOURCE[0]}"
script_name="${script_name##*/}"
-m() { if $verbose; then printf "%s\n" "$*"; fi; "$@"; }
-e() { printf "%s\n" "$*"; }
-die() { printf "%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; }
-mexit() { echo ": exiting with status $1"; exit $1; }
+d() {
+ if $dry_run || $conf_only; then
+ printf "$pre dry-run: %s\n" "$*"
+ else
+ printf "$pre running: %s\n" "$*"
+ "$@"
+ fi
+}
+m() { if $verbose; then printf "$pre %s\n" "$*"; fi; "$@"; }
+e() { printf "$pre %s\n" "$*"; }
+die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; }
+mexit() { echo "$pre exiting with status $1"; exit $1; }
+
+uninstalled-file-die() {
+ die "uninstalled file $1. run install-my-scripts or rerun with -f"
+}
+
+set-location() {
+ case $HOSTNAME in
+ kw)
+ at_work=true
+ ;;
+ kd|frodo)
+ at_home=true
+ ;;
+ x2|x3|sy)
+ if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
+ && ip n show 10.2.0.1 | grep . &>/dev/null; then
+ at_home=true
+ elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
+ && ip n show 192.168.0.26 | grep . &>/dev/null; then
+ at_work=true
+ fi
+ ;;
+ esac
+}
+
+exit-if-no-default-targets() {
+ if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
+ echo "MAIL_HOST=$MAIL_HOST, nothing to do"
+ mexit 0
+ fi
+ case $HOSTNAME in
+ kw|kd|frodo|x2|x3|sy) : ;;
+ *)
+ die "error: no default targets for this host, use -t"
+ ;;
+ esac
+}
+
+add-x3-target() {
+ # main work machine
+ if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
+ targets+=(x3.office.fsf.org)
+ elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
+ # in case we took it home
+ targets+=(x3.b8.nz)
+ elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
+ targets+=(x3w.b8.nz)
+ else
+ targets+=(x3wg.b8.nz)
+ fi
+}
+
+add-wireless-target-h() {
+ if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
+ targets+=($h.b8.nz)
+ elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
+ targets+=(${h}w.b8.nz)
+ fi
+}
+
+qconf() {
+ case $sub in
+ q)
+ # q has sensitive data i dont want to backup for so long
+ cat >>/etc/btrbk$conf_suf.conf < >(sed '/No such file or directory/d'); then
+ # shellcheck disable=SC2046 # we want word splitting
+ set -- $(< $once_args_file-tmp) "$@"
+ # i havent used this feature yet, so warn about it
+ echo "$0: btrbk-run options set in $once_args_file:"
+ cat $once_args_file-tmp
+ rm -f $once_args_file-tmp
+fi
+
+
targets=()
early=false
fast=false
kd_spread=false
+check_installed=false
orig_args=("$@")
-temp=$(getopt -l fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
+temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1
eval set -- "$temp"
while true; do
case $1 in
@@ -99,6 +201,9 @@ while true; do
-3) conf_suf=3 ;;
# only creates the config file, does not run btrbk
-c) conf_only=true ;;
+ --check-installed)
+ check_installed=true
+ ;;
# quit early, just btrbk, no extra remounting etc.
-e) early=true ;;
# avoids some default behaviors:
@@ -110,7 +215,9 @@ while true; do
# switch mail-host, no need to repeat the same checks again.
--fast) fast=true ;;
-i) incremental_strict=true ;;
- # note this implies resume and -p
+ # note this implies resume and -p because it is just meant to make
+ # other hosts have the same snapshots, not do any expiry or new
+ # backups.
-k) kd_spread=true ;;
# bytes per second, suffix k m g
-l) rate_limit=$2; shift ;;
@@ -146,6 +253,28 @@ done
cmd_arg="$1"
+
+
+if ! $force && { $check_installed || [[ ! $source ]]; } ; then
+ install_bin_files=(
+ mount-latest-subvol
+ check-subvol-stale
+ btrbk-run
+ )
+ for f in ${install_bin_files[@]}; do
+ if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then
+ uninstalled-file-die $f
+ fi
+ done
+ if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then
+ uninstalled-file-die err
+ fi
+ if $check_installed; then
+ exit 0
+ fi
+fi
+
+
if $kd_spread; then
if [[ $cmd_arg && $cmd_arg != resume ]]; then
die "dont pass -k without resume or empty run arg"
@@ -211,6 +340,7 @@ if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then
fi
cd /a/opt/btrbk
m make install
+ cd /
fi
# TODO: i wonder if there should be an option to send to the default
@@ -221,60 +351,6 @@ at_work=false
at_home=false
-set-location() {
- case $HOSTNAME in
- kw)
- at_work=true
- ;;
- kd|frodo)
- at_home=true
- ;;
- x2|x3|sy)
- if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \
- && ip n show 10.2.0.1 | grep . &>/dev/null; then
- at_home=true
- elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \
- && ip n show 192.168.0.26 | grep . &>/dev/null; then
- at_work=true
- fi
- ;;
- esac
-}
-
-exit-if-no-default-targets() {
- if ! $force && [[ $HOSTNAME != "$MAIL_HOST" ]]; then
- echo "MAIL_HOST=$MAIL_HOST, nothing to do"
- mexit 0
- fi
- case $HOSTNAME in
- kw|kd|frodo|x2|x3|sy) : ;;
- *)
- die "error: no default targets for this host, use -t"
- ;;
- esac
-}
-
-add-x3-target() {
- # main work machine
- if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then
- targets+=(x3.office.fsf.org)
- elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
- # in case we took it home
- targets+=(x3.b8.nz)
- else
- targets+=(x3wg.b8.nz)
- fi
-}
-
-add-wireless-target-h() {
- if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then
- targets+=($h.b8.nz)
- elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then
- targets+=(${h}w.b8.nz)
- fi
-}
-
-
# set default targets
if [[ ! -v targets && ! $source ]]; then
exit-if-no-default-targets
@@ -325,24 +401,24 @@ else
*)
prospective_mps=()
if [[ $source ]]; then
- source_state="$(ssh $source cat /a/bin/bash_unpublished/source-state)"
+ source_state="$(ssh $source 'cat /a/bin/bash_unpublished/source-state; echo source_host=$HOSTNAME')"
eval "$source_state"
- source_host="$(ssh $source cat /etc/hostname)"
+ # shellcheck disable=SC2154 # assigned in the above eval.
if [[ $source_host == "$MAIL_HOST" ]]; then
prospective_mps+=(/o)
fi
if [[ $source_host == "$HOST2" ]]; then
- prospective_mps+=(/a /ar /qr /qd /q)
+ prospective_mps+=(/a /qr /qd /q)
fi
else
if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
prospective_mps+=(/o)
fi
if [[ $HOSTNAME == "$HOST2" ]]; then
- prospective_mps+=(/a /ar /qr /qd /q)
+ prospective_mps+=(/a /qr /qd /q)
fi
if $kd_spread; then
- prospective_mps=(/a /ar /o /qr /qd /q)
+ prospective_mps=(/a /o /qr /qd /q)
fi
fi
# note: put q last just in case its specific retention options were to
@@ -361,7 +437,8 @@ else
done
fi
-if (( ! ${#mountpoints[@]} )); then
+tmp=$(( ${#mountpoints[@]} == 0 ))
+if (( tmp )); then
die didnt get mountpoint arg and had no defaults
fi
@@ -386,20 +463,48 @@ if ! command -v btrbk &>/dev/null; then
die "error: no btrbk binary found"
fi
+if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then
+ ssh root@$source btrbk-run --check-installed || exit 1
+fi
#### end pre-checks #####
+
+
mkdir -p /var/log/btrbk
# The journal doesnt go back to my oldest backups, and I've found myself
# wanting older logs. Not going to bother expiring old logs, since it is
# fine if they go back years.
log_path=/var/log/btrbk/$(date +%F_%T%:::z).log
echo copying output to $log_path
-exec &> >(ts "%F %T" | tee -a $log_path)
-
+exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none')
+# print some non-default opts
if $verbose; then
- printf " options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
+ opts_show=()
+ if ! $conf_only; then
+ opts_show+=(conf_only=true)
+ fi
+ if ! $dry_run; then
+ opts_show+=(dry_run=true)
+ fi
+ if [[ $rate_limit != no ]]; then
+ opts_show+=("rate_limit=$rate_limit")
+ fi
+ if [[ $cmd_arg != run ]]; then
+ opts_show+=(cmd_arg=$cmd_arg)
+ fi
+ if (( ${#opts_show[@]} >= 1 )); then
+ first=true
+ for opt in ${opts_show[@]}; do
+ if $first; then
+ printf "%s" "$opt"
+ else
+ printf " %s" "$opt"
+ fi
+ done
+ echo
+ fi
fi
if [[ -v targets ]]; then
@@ -413,27 +518,6 @@ fi
echo "mountpoints: ${mountpoints[*]}"
-uninstalled-file-die() {
- err "uninstalled file $1. run install-my-scripts or rerun with -f"
- exit 1
-}
-if ! $force; then
- install_bin_files=(
- mount-latest-subvol
- check-subvol-stale
- btrbk-run
- )
- for f in ${install_bin_files[@]}; do
- if ! diff -q /a/bin/ds/$f /usr/local/bin/$f; then
- uninstalled-file-die $f
- fi
- done
- if ! diff -q /a/bin/errhandle/err /usr/local/lib/err; then
- uninstalled-file-die err
- fi
-fi
-
-
# pull_reexec stops us from getting into an infinite loop if there is some
# kind of weird problem
pulla=false
@@ -443,10 +527,11 @@ for m in "${mountpoints[@]}"; do
break
fi
done
+
if ! $pull_reexec && [[ $source ]] && $pulla ; then
tmpf=$(mktemp)
m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin
- m rsync -ra $source:/usr/local/lib/err /usr/local/lib
+ m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib
m scp $source:/a/bin/distro-setup/btrbk-run $tmpf
if ! diff -q $tmpf ${BASH_SOURCE[0]}; then
e "found different version on host $source. reexecing"
@@ -498,7 +583,8 @@ else
IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}"
percent_used=${percent_used%%%}
- if (( ${#tmp_array[@]} != 2 )); then
+ tmp=$(( ${#tmp_array[@]} != 2 ))
+ if (( tmp )); then
die "error: didnt get 2 lines in test ssh to target $h. investigate"
fi
case $percent_used in
@@ -514,11 +600,13 @@ else
# we may be booted into a bootstrap fs or something
min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb
- if (( root_size < min_root_kb )); then
+ tmp=$(( root_size < min_root_kb ))
+ if (( tmp )); then
continue
fi
- if (( percent_used >= 98 )); then
+ tmp=$(( percent_used >= 98 ))
+ if (( tmp )); then
die "error: filesystem on target $h is $percent_used % full"
fi
@@ -595,8 +683,12 @@ snapshot_dir btrbk
target_preserve $std_preserve
target_preserve_min 6h
-# i tried this when investigating: clone no source subvolume found error
-#incremental_prefs sro:1 srn:1 sao san:1 aro:1 arn:1
+# it seems very likely that not doing this could result in clone source not found
+# errors, for example when expiry happens differently on different hosts,
+# also, as btrbk does by default, if a failed send happens, on the next run it
+# will warn about a stray subvolume, but then create a backup of a newer subvol
+# and use an older subvol as the parent.
+incremental_prefs sao:1
# if something fails and it's not obvious, try doing
# btrbk -l debug -v dryrun
@@ -610,21 +702,6 @@ incremental strict
EOF
fi
-qconf() {
- case $sub in
- q)
- # q has sensitive data i dont want to backup for so long
- cat >>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf <>/etc/btrbk$conf_suf.conf < 1 && tg_snap_count == orphan_mp_count ))
+ if (( tmp )) ; then
+ die "something went wrong checking orphans on $tg: for mountpoint ${mountpoints[$i]}, $orphan_mp_count"
+ fi
+ done
+}
+
+if [[ $source ]]; then
+ for snap in $(ssh root@$source "shopt -s nullglob; ${snap_list_cmds[*]}"); do
+ source_snaps[$snap]=t
+ done
+ get-orphan-tg-snaps
+ tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
+ if (( tmp )); then
+ d btrfs sub del ${orphan_tg_snaps[*]}
+ fi
+else # we have targets
+ for tg in ${targets[@]}; do
+ tmp_str=$(ssh root@$tg "shopt -s nullglob; ${snap_list_cmds[*]}")
+ mapfile -t tg_snaps <<<"$tmp_str"
+ get-orphan-tg-snaps
+ tmp=$(( ${#orphan_tg_snaps[*]} >= 1 ))
+ if (( tmp )); then
+ d ssh root@$tg "btrfs sub del ${orphan_tg_snaps[*]}"
+ fi
+ done
+fi
+
# todo: umount first to ensure we don't have any errors
# todo: do some kill fuser stuff to make umount more reliable
@@ -726,17 +864,18 @@ else
done
fi
+# todo, we get hostnames earlier, reuse that.
if [[ $ret == 0 ]]; then
for tg in ${targets[@]}; do
h=$(ssh $tg hostname)
if [[ $h == kd && $HOSTNAME == x3 && $HOSTNAME == "$MAIL_HOST" ]]; then
- ssh root$tg systemctl --no-block start btrbk-spread
+ m ssh root@$tg 'btrbk-spread-wrap &>/dev/null