X-Git-Url: https://iankelling.org/git/?a=blobdiff_plain;f=btrbk-run;h=8fc4c4f00b8c9f5ea3b64126767c83003e675643;hb=HEAD;hp=5ce774b7e83c84270ad228479e5ba7b0f1901c40;hpb=0f9a999aa601cd77343a3e08cf0c6c4f9f943813;p=distro-setup diff --git a/btrbk-run b/btrbk-run index 5ce774b..9bd6e9b 100644 --- a/btrbk-run +++ b/btrbk-run @@ -1,17 +1,22 @@ #!/bin/bash -# Copyright (C) 2016 Ian Kelling -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Configure & run btrbk & related work on Ian's computers. +# Copyright (C) 2024 Ian Kelling -# http://www.apache.org/licenses/LICENSE-2.0 +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later # todo: if we cancel in the middle of a btrfs send, then run again @@ -23,6 +28,7 @@ [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" set -e; . /usr/local/lib/bash-bear; set +e +shopt -s nullglob usage() { cat <<'EOF' @@ -39,25 +45,53 @@ EOF } + pre=btrbk-run + + script_name="${BASH_SOURCE[0]}" script_name="${script_name##*/}" + + +log-setup() { + if [[ ! $log_path ]]; then + mkdir -p /var/log/btrbk + log_path=/var/log/btrbk/$(date +%F_%T%:::z).log + fi +} d() { if $dry_run || $conf_only; then printf "$pre dry-run: %s\n" "$*" else - printf "$pre running: %s\n" "$*" - "$@" + log-setup + printf "$pre running: %s\n" "$*" |& pee cat 'ts "%F %T" >>'$log_path + "$@" |& pee cat 'ts "%F %T" >>'$log_path fi } m() { if $verbose; then printf "$pre %s\n" "$*"; fi; "$@"; } e() { printf "$pre %s\n" "$*"; } + +logq() { + local exit_code + exit_code=0 + log-setup + printf "$pre running: %s\n" "$*" | pee cat 'ts "%F %T" >>'$log_path + e logging to $log_path + "$@" |& ts "%F %T" >>$log_path || exit_code=$? + printf "$pre exit code:%s of %s\n" "$exit_code" "$*" | pee cat 'ts "%F %T" >>'$log_path + if (( exit_code > 0 )); then + e "error: command exit code: $exit_code. exiting after tail -n50 $log_path" + tail -n50 $log_path + exit $exit_code + fi +} + die() { printf "$pre error: %s\n" "$*" >&2; echo "$pre exiting with status 1" >&2; exit 1; } mexit() { echo "$pre exiting with status $1"; exit $1; } uninstalled-file-die() { - die "uninstalled file $1. run install-my-scripts or rerun with -f" + die "file $1 is not latest. run install-my-scripts or rerun with -f" } set-location() { @@ -68,9 +102,10 @@ set-location() { kd|frodo) at_home=true ;; - x2|x3|sy) + x2|x3|sy|so) if [[ $(dig +short @10.2.0.1 -x 10.2.0.2 2>&1 ||:) == kd.b8.nz. ]] \ && ip n show 10.2.0.1 | grep . &>/dev/null; then + # note: logic duplicated in 11-iank at_home=true elif ping -q -c1 -w1 hal.office.fsf.org &>/dev/null \ && ip n show 192.168.0.26 | grep . &>/dev/null; then @@ -86,7 +121,7 @@ exit-if-no-default-targets() { mexit 0 fi case $HOSTNAME in - kw|kd|frodo|x2|x3|sy) : ;; + kw|kd|frodo|x2|x3|sy|so) : ;; *) die "error: no default targets for this host, use -t" ;; @@ -97,20 +132,31 @@ add-x3-target() { # main work machine if ping -q -c1 -w1 x3.office.fsf.org &>/dev/null; then targets+=(x3.office.fsf.org) - elif ping -q -c1 -w1 $h.b8.nz &>/dev/null; then + elif ping -q -c1 -w1 x3.b8.nz &>/dev/null; then # in case we took it home targets+=(x3.b8.nz) + elif ping -q -c1 -w1 x3w.b8.nz &>/dev/null; then + targets+=(x3w.b8.nz) else targets+=(x3wg.b8.nz) fi } -add-wireless-target-h() { - if ping -q -c1 -w1 $h.b8.nz &>/dev/null; then - targets+=($h.b8.nz) - elif ping -q -c1 -w1 ${h}w.b8.nz &>/dev/null; then - targets+=(${h}w.b8.nz) +add-wireless-target() { + local host + if [[ ! $1 ]]; then + set -- $h fi + for host; do + # c = cabled, w = wireless + if ping -q -c1 -w1 ${host}c.b8.nz &>/dev/null; then + targets+=(${host}c.b8.nz) + elif ping -q -c1 -w1 $host.b8.nz &>/dev/null; then + targets+=($host.b8.nz) + elif ping -q -c1 -w1 ${host}w.b8.nz &>/dev/null; then + targets+=(${host}w.b8.nz) + fi + done } qconf() { @@ -146,7 +192,7 @@ ret=0 conf_only=false dry_run=false # mostly for testing rate_limit=no -verbose=true; verbose_arg=-v +verbose=true; verbose_arg="-l trace" force=false if [[ $INVOCATION_ID ]]; then # INVOCATION_ID means running as a systemd service. we cant show progress in this case, @@ -185,13 +231,17 @@ fast=false kd_spread=false check_installed=false orig_args=("$@") -temp=$(getopt -l check-installed,fast,pull-reexec,help 23cefikl:m:npqrs:t:vh "$@") || usage 1 +temp=$(getopt -l check-installed,fast,pull-reexec,help 23acefikl:m:npqrs:t:vh "$@") || usage 1 eval set -- "$temp" while true; do case $1 in # for the rare case we want to run multiple instances at the same time -2) conf_suf=2 ;; -3) conf_suf=3 ;; + -a) + # all moiuntpoints + mountpoints=(/a /o /qr /qd /q) + ;; # only creates the config file, does not run btrbk -c) conf_only=true ;; --check-installed) @@ -236,7 +286,9 @@ while true; do # snapshot. we have default hosts we will populate. -t) IFS=, targets=($2); unset IFS; shift ;; # verbose. - -v) verbose=true; verbose_arg=-v ;; + -v) + verbose=true; verbose_arg="-l trace" + ;; -h|--help) usage ;; --) shift; break ;; *) die "Internal error!" ;; @@ -260,7 +312,7 @@ if ! $force && { $check_installed || [[ ! $source ]]; } ; then fi done if ! diff -q /a/bin/bash-bear-trap/bash-bear /usr/local/lib/bash-bear; then - uninstalled-file-die err + uninstalled-file-die bash-bear fi if $check_installed; then exit 0 @@ -280,8 +332,7 @@ if $kd_spread; then fi cmd_arg=resume preserve_arg=-p - h=sy - add-wireless-target-h + add-wireless-target sy so fi if [[ ! $cmd_arg ]]; then @@ -333,6 +384,7 @@ if [[ /a/opt/btrbk/btrbk -nt /usr/bin/btrbk ]]; then fi cd /a/opt/btrbk m make install + cd / fi # TODO: i wonder if there should be an option to send to the default @@ -357,14 +409,15 @@ if [[ ! -v targets && ! $source ]]; then wireless_home_hosts=( x2 sy + so ) for h in ${wireless_home_hosts[@]}; do if [[ $HOSTNAME != "$h" ]]; then - add-wireless-target-h + add-wireless-target fi done elif $at_work; then - targets+=(i.b8.nz) + targets+=(b8.nz) for h in x2 x3 kw; do if [[ $HOSTNAME == "$h" ]]; then continue @@ -374,7 +427,7 @@ if [[ ! -v targets && ! $source ]]; then fi done else - targets+=(i.b8.nz) + targets+=(b8.nz) fi fi @@ -400,17 +453,17 @@ else prospective_mps+=(/o) fi if [[ $source_host == "$HOST2" ]]; then - prospective_mps+=(/a /ar /qr /qd /q) + prospective_mps+=(/a /qr /qd /q) fi else if [[ $HOSTNAME == "$MAIL_HOST" ]]; then prospective_mps+=(/o) fi if [[ $HOSTNAME == "$HOST2" ]]; then - prospective_mps+=(/a /ar /qr /qd /q) + prospective_mps+=(/a /qr /qd /q) fi if $kd_spread; then - prospective_mps=(/a /ar /o /qr /qd /q) + prospective_mps=(/a /o /qr /qd /q) fi fi # note: put q last just in case its specific retention options were to @@ -455,49 +508,54 @@ if ! command -v btrbk &>/dev/null; then die "error: no btrbk binary found" fi +# pull_reexec stops us from getting into an infinite loop if there is some +# kind of weird problem +pulla=false +for m in "${mountpoints[@]}"; do + if [[ $m == /a ]]; then + pulla=true + break + fi +done + if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then - ssh root@$source btrbk-run --check-installed || exit 1 + ssh root@$source btrbk-run --check-installed fi #### end pre-checks ##### -mkdir -p /var/log/btrbk -# The journal doesnt go back to my oldest backups, and I've found myself -# wanting older logs. Not going to bother expiring old logs, since it is -# fine if they go back years. -log_path=/var/log/btrbk/$(date +%F_%T%:::z).log -echo copying output to $log_path -exec &> >(pee cat 'ts "%F %T"|dd of='$log_path' status=none') - +# print some non-default opts if $verbose; then - printf " options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s\n" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg" -fi - -if [[ -v targets ]]; then - echo "targets: ${targets[*]}" -fi - -if [[ $source ]]; then - echo "source: $source" + opts_show=() + if ! $conf_only; then + opts_show+=(conf_only=true) + fi + if ! $dry_run; then + opts_show+=(dry_run=true) + fi + if [[ $rate_limit != no ]]; then + opts_show+=("rate_limit=$rate_limit") + fi + if [[ $cmd_arg != run ]]; then + opts_show+=(cmd_arg=$cmd_arg) + fi + if (( ${#opts_show[@]} >= 1 )); then + first=true + for opt in ${opts_show[@]}; do + if $first; then + printf "%s" "$opt" + first=false + else + printf " %s" "$opt" + fi + done + echo + fi fi -echo "mountpoints: ${mountpoints[*]}" - - - - -# pull_reexec stops us from getting into an infinite loop if there is some -# kind of weird problem -pulla=false -for m in "${mountpoints[@]}"; do - if [[ $m == /a ]]; then - pulla=true - break - fi -done if ! $pull_reexec && [[ $source ]] && $pulla ; then tmpf=$(mktemp) @@ -513,6 +571,19 @@ if ! $pull_reexec && [[ $source ]] && $pulla ; then fi +if [[ -v targets ]]; then + echo "targets: ${targets[*]}" +fi +if [[ $source ]]; then + echo "source: $source" +fi +echo "mountpoints: ${mountpoints[*]}" + + +# todo: check if we have no snapshots yet, because I always want to run +# archive instead of run. Likely, I should give an error unless a cli +# override is passed. perhaps check-subvol-stale could give the error. +# see the error message "no snapshots found" in that file. if ! $fast; then # if our mountpoints are from stale snapshots, # it doesn't make sense to do a backup. @@ -542,13 +613,17 @@ else sshable=() sshfail=() + remote_str_cmd="mkdir -p /mnt/root/btrbk /mnt/o/btrbk && \ +date +%z && \ +df --output=size,pcent / | tail -n1" + for h in ${targets[@]}; do if $fast || $conf_only; then # Use some typical values in this case root_size=$(( 1024 * 1024 * 2000 )) #2tb percent_used=10 zone=$(date +%z) - elif remote_str=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk /mnt/o/btrbk && date +%z && df --output=size,pcent / | tail -n1"); then + elif remote_str=$(timeout -s 9 6 ssh root@$h "$remote_str_cmd"); then mapfile -t tmp_array <<<"$remote_str" zone="${tmp_array[0]}" IFS=" " read -r root_size percent_used <<<"${tmp_array[1]}" @@ -573,6 +648,7 @@ else min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb tmp=$(( root_size < min_root_kb )) if (( tmp )); then + e "warning: $h: root_size=$root_size < 200gb, perhaps it is booted to bootstrap vol. skipping for now" continue fi @@ -610,7 +686,7 @@ else fi done if [[ ! ${sshable[*]} ]] || { $force && [[ ${sshfail[*]} ]]; }; then - die "failed to ssh to hosts: ${sshfail[*]}" + die "see skipped host warning above or sshfail hosts: ${sshfail[*]}" else if [[ ${sshfail[*]} ]]; then ret=1 @@ -625,14 +701,6 @@ cat >/etc/btrbk$conf_suf.conf </dev/null /dev/null