From 6f67d94f5a7dc838bea59795296317948ebb9df9 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Thu, 22 Aug 2024 05:57:26 -0400 Subject: [PATCH] minor fixes and improvements --- .bashrc | 6 +- brc | 114 ++++++++++++++++------------- brc2 | 2 +- distro-end | 13 ---- filesystem/usr/local/bin/btrbk-run | 91 ++++++++++++++--------- 5 files changed, 126 insertions(+), 100 deletions(-) diff --git a/.bashrc b/.bashrc index c863938..3e90a50 100644 --- a/.bashrc +++ b/.bashrc @@ -69,9 +69,9 @@ if [[ $EUID == 1000 && ! $_ran_bashrc && $TERM == linux && ! $DISPLAY && $(tty) fi _ran_bashrc=t -#### if (in -# noninteractive ssh shells or tty). tty because often i use it when -# something is going and io is slow and my bashrc is too slow. +# Test if we are in noninteractive ssh shells or tty. tty because often +# i use it when for example, io on the remote host is really slow and my +# bashrc is too slow. if [[ $LC_USEBASHRC != t && ( $SSH_CONNECTION || $TERM == linux ) ]]; then # Here we did not opt-in to running our .bashrc file so we just # return, but we still setup a function to source it without returning diff --git a/brc b/brc index 65c3cf6..c91d06b 100644 --- a/brc +++ b/brc @@ -707,7 +707,7 @@ jdo() { journalctl --since=now -qn2 -f -u "$unit_name" & jr_pid=$! # sleep 1 is too fast for x200 - sleep 2 + sleep 3 $sdrun_prefix systemd-run $sdrun_args --unit "$unit_name" --wait --collect "$cmd" "$@" || ret=$? # The sleep lets the journal output its last line # before the prompt comes up. @@ -2650,8 +2650,9 @@ resolvcat() { m s nscd -i hosts fi f=/etc/resolv.conf - echo $f:; ccat $f - hr; s ss -lpn sport = 53 + ll $f + m ccat $f + hr; m s ss -lpn sport = 53 if systemctl is-enabled dnsmasq &>/dev/null || [[ $(systemctl is-active dnsmasq ||:) != inactive ]]; then # this will fail is dnsmasq is failed hr; m ser status dnsmasq | cat || : @@ -2922,20 +2923,27 @@ skgit() { } -# sl: ssh, but firsh rsync our bashrc and related files to a special -# directory on the remote host if needed. +#### sl: ssh wrapper, but maybe first rsync files that we configure and +# always source our .bashrc on the remote. + +# TODO: this needs better documentation. + +# Usage: all the same args as ssh + a few below. Note, combining options +# that have arguments with ones that dont, like -4oOption is not +# supported. + +## Required setup -# Some environment variables and files need to be setup for this to work -# (mine are set at the beginning of this file) +# Sorry these have no defaults, you can read my settings in this file. # SL_FILES_DIR: Environment variable. Path to folder which should at -# least have a .bashrc file or symlink. This dir will be rsynced to ~ on -# remote hosts (top level symlinks are resolved) unless the host already -# has a $SL_FILES_DIR/.bashrc. In that case, we assume it is a host you -# control and sync files to separately and already has the ~/.bashrc you -# want. The remote bash will also take its .inputrc config from this -# folder (default of not existing is fine). Mine looks like this: -# https://iankelling.org/git/?p=distro-setup;a=tree;f=sl/.iank +# least have a .bashrc file or symlink to it. This dir will be rsynced +# to ~ on remote hosts (top level symlinks are resolved) unless the host +# already has a $SL_FILES_DIR/.bashrc. In that case, we assume it is a +# host you control and sync files to separately and already has the +# ~/.bashrc you want. The remote bash will also take its .inputrc config +# from this folder (default of not existing is fine). Mine looks like +# this: https://iankelling.org/git/?p=distro-setup;a=tree;f=sl/.iank # SL_INFO_DIR: Environment variable. This folder stores info about what # we detected on the remote system and when we last synced. It will be created @@ -2943,19 +2951,29 @@ skgit() { # remote system, you can use sl --rsync, or the function for that slr # below. -# SL_TEST_CMD: Env var. If set, we run this string on the remote host -# the first time sl is run (or if we run slr), and then the result is -# passed to SL_TEST_HOOK which gets run locally. It is meant to be used -# to vary the files synced depending on the remote host. For example, -# export SL_TEST_CMD=". /etc/os-release ; echo \${VERSION//[^a-zA-Z0-9]/}" -# SL_TEST_HOOK: Env var. It is run as $SL_TEST_HOOK. This can set -# $SL_FILES_DIR to vary the files synced. +## Optional settings -# SL_RSYNC_ARGS: Env var. String of arguments passed to rsync. For -# example to exclude files within a directory. Note, excluded -# files wont be deleted on rsync, you can add --delete-excluded -# to the rsync command if that is desired. +# --rsync Forget about any previous rsync we did. The most common case +# is that the ssh target does not have files we previously rsynced for +# various reasons. I have a wrapper for this called slr below. + +# SL_TEST_CMD / --sl-test-cmd CMD: Env var or cli option. If set, we run +# this string on the remote host the first time sl is run (or if we run +# slr). Its standard out is passed to SL_TEST_HOOK which gets run +# locally. It is meant to be used to vary the files synced depending on +# the remote host. For example, export SL_TEST_CMD=". /etc/os-release ; +# echo \${VERSION//[^a-zA-Z0-9]/}". cli option overrides env var. + +# SL_TEST_HOOK / --sl-test-hook HOOK: Env var or cli option. See SL_TEST_CMD +# above. It is run unquoted (with expansion). This can set $SL_FILES_DIR +# to vary the files synced. cli option overrides env var. + +# SL_RSYNC_ARGS / --sl-rsync-args ARGS: Env var or cli option. String of +# arguments passed to rsync. For example to exclude files within a +# directory. Note, excluded files wont be deleted on rsync, you can add +# --delete-excluded to the rsync command if that is desired. cli +# overrides env var. # SL_SSH_ARGS: Env var. Default arguments passed to ssh. @@ -2981,7 +2999,7 @@ sl() { # .bashrc. This means the outer shell still ran the default .bashrc, # but that is the best we can do. - local now args remote dorsync haveinfo tmpa sshinfo tmp tmp2 type info_sec force_rsync \ + local now args remote dorsync haveinfo tmpa sshinfo tmp tmp2 host_type info_sec force_rsync \ sync_dirname testcmd extra_info testbool files_sec sl_test_cmd sl_test_hook declare -a args tmpa @@ -3066,9 +3084,9 @@ sl() { return 1 fi - dorsync=false + do_rsync=false haveinfo=false - tmpa=($SL_INFO_DIR/???????????"$remote") + tmpa=($SL_INFO_DIR/??????????-????-"$remote") sshinfo=${tmpa[0]} if [[ -e $sshinfo ]]; then if $force_rsync; then @@ -3079,8 +3097,8 @@ sl() { fi if $haveinfo; then tmp=${sshinfo[0]##*/} - tmp2=${tmp::11} - type=${tmp2: -1} + tmp2=${tmp:11} # skip 11 chars + host_type=${tmp2%%-*} extra_info=$(cat $sshinfo) else # we test for string to know ssh succeeded @@ -3091,10 +3109,11 @@ sl() { command ssh -v "${args[@]}" "$remote" fi if [[ $tmp == y* ]]; then - type=a + # yes test result, no need to rsync this host. + host_type=skip else - dorsync=true - type=b + do_rsync=true + host_type=sync fi extra_info="${tmp:1}" fi @@ -3102,12 +3121,12 @@ sl() { RSYNC_RSH="ssh ${args[*]}" $sl_test_hook "$extra_info" "$remote" fi - if $haveinfo && [[ $type == b ]]; then + if $haveinfo && [[ $host_type == sync ]]; then info_sec=${tmp::10} read -r files_sec _ < <(find -L $SL_FILES_DIR -printf "%T@ %p\n" | sort -nr || [[ $? == 141 || ${PIPESTATUS[0]} == 32 ]] ) files_sec=${files_sec%%.*} if (( files_sec > info_sec )); then - dorsync=true + do_rsync=true rm -f $sshinfo fi fi @@ -3119,21 +3138,23 @@ sl() { return 1 fi - if $dorsync; then + if $do_rsync; then RSYNC_RSH="ssh ${args[*]}" m rsync -rptL --delete $sl_rsync_args $SL_FILES_DIR "$remote": fi - if $dorsync || ! $haveinfo; then - sshinfo=$SL_INFO_DIR/$EPOCHSECONDS$type"$remote" + if $do_rsync || ! $haveinfo; then + sshinfo=$SL_INFO_DIR/$EPOCHSECONDS-$host_type-"$remote" [[ -e $SL_INFO_DIR ]] || mkdir -p $SL_INFO_DIR printf "%s\n" "$extra_info" >$sshinfo chmod 666 $sshinfo fi - if [[ $type == b ]]; then + if [[ $host_type == sync ]]; then if (( ${#@} )); then - # Theres a couple ways to pass arguments, im not sure whats best, - # but relying on bash 4.4+ escape quoting seems most reliable. + # in the past, I used ${@@Q} passed to a bash subshell. Then + # later, I noticed it didn't handle spaces as normal ssh does, and + # didn't see any benefit to doing it that way (perhaps forgot what + # I originally saw in it). command ssh "${args[@]}" "$remote" \ - LC_USEBASHRC=t bash -c '.\ '$sync_dirname'/.bashrc\;"\"\$@\""' bash ${@@Q} + LC_USEBASHRC=t . $sync_dirname/.bashrc\; "$@" elif [[ ! -t 0 ]]; then # This case is when commands are being piped to ssh. # Normally, no bashrc gets sourced. @@ -3144,7 +3165,7 @@ sl() { fi else if [[ -t 0 ]]; then - LC_USEBASHRC=t command ssh "${args[@]}" "$remote" ${@@Q} + LC_USEBASHRC=t command ssh "${args[@]}" "$remote" "$@" else command ssh "${args[@]}" "$remote" LC_USEBASHRC=t bash fi @@ -3241,13 +3262,6 @@ srm () { } # scp a script then ssh and run it. -# -# note on use cases: this can be useful for running a commands locally, -# dump them into a script, then run remotely. Note, there is no way to -# include an escaped ; in an ssh arg so that it works in the remote -# command, but we can use eval to make it work. For example: -# -# ssh ahost eval cd /mnt/root/btrbk \; ls -1 a.\* srun() { scp $2 $1:/tmp ssh $1 "/tmp/${2##*/}" "$(printf "%q\n" "${@:2}")" diff --git a/brc2 b/brc2 index 05f3eef..fa226c9 100644 --- a/brc2 +++ b/brc2 @@ -2562,7 +2562,7 @@ host-info-all() { bindpushb8 # for wireguard configs - ssh iank@li.b8.nz eval conflink \; ser reload wg-quick@wgmail + ssh iank@li.b8.nz "conflink; ser reload wg-quick@wgmail" wrt-setup } diff --git a/distro-end b/distro-end index 8fbc83a..9b5d484 100755 --- a/distro-end +++ b/distro-end @@ -2259,19 +2259,6 @@ EOF sudo exportfs -rav fi -# very temporary fix. -# should be gone in a few days -# https://bugs.launchpad.net/ubuntu/+source/fail2ban/+bug/2055114 -case $codename_compat in - noble) - if [[ ! -e ~/fail2ban_1.1.0-1_all.deb ]]; then - cd - wget https://launchpad.net/ubuntu/+source/fail2ban/1.1.0-1/+build/28291332/+files/fail2ban_1.1.0-1_all.deb - sudo dpkg -i fail2ban_1.1.0-1_all.deb - fi - ;; -esac - case $codename_compat in noble) # this fails on startup. i don't use resolvconf, so it is totally pointless. diff --git a/filesystem/usr/local/bin/btrbk-run b/filesystem/usr/local/bin/btrbk-run index 7df7843..6665607 100755 --- a/filesystem/usr/local/bin/btrbk-run +++ b/filesystem/usr/local/bin/btrbk-run @@ -27,6 +27,8 @@ [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" + +# todo: try adding set -u set -e; . /usr/local/lib/bash-bear; set +e shopt -s nullglob @@ -60,7 +62,16 @@ d() { "$@" |& pee cat 'ts "%F %T" >>'$log_path fi } -m() { if $verbose; then printf "$pre %s\n" "$*"; fi; "$@"; } +m() { + if $dry_run; then + printf "$pre dry-run: %s\n" "$*" + return 0 + fi + if $verbose; then + printf "$pre %s\n" "$*" + fi + "$@" +} e() { printf "$pre %s\n" "$*"; } logq() { @@ -727,7 +738,7 @@ for m in "${mountpoints[@]}"; do done if ! $pull_reexec && [[ $source ]] && $pulla && ! $force ; then - ssh root@$source btrbk-run --check-installed + m ssh root@$source btrbk-run --check-installed fi #### end pre-checks ##### @@ -766,15 +777,19 @@ fi if ! $pull_reexec && [[ $source ]] && $pulla ; then - tmpf=$(mktemp) - m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin - m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib - m rsync $source:/usr/local/bin/btrbk-run $tmpf - if ! diff -q $tmpf ${BASH_SOURCE[0]}; then - e "found different version on host $source. reexecing" - rsync -aSAX --chmod=755 --chown=root:root $tmpf /usr/local/bin/btrbk-run - m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}" - mexit 0 + if $dry_run; then + e due to dry run, skipping check for newer files on source host + else + tmpf=$(mktemp) + m rsync -ra $source:/usr/local/bin/{mount-latest-subvol,check-subvol-stale} /usr/local/bin + m rsync -ra $source:/usr/local/lib/bash-bear /usr/local/lib + m rsync $source:/usr/local/bin/btrbk-run $tmpf + if ! diff -q $tmpf ${BASH_SOURCE[0]}; then + e "found different version on host $source. reexecing" + m rsync -aSAX --chmod=755 --chown=root:root $tmpf /usr/local/bin/btrbk-run + m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}" + mexit 0 + fi fi fi @@ -803,21 +818,20 @@ fi local_zone=$(date +%z) if [[ $source ]]; then - if $fast; then - zone=$local_zone - else - if ! ssh_info=$(ssh root@$source 'hostname && date +%z'); then - if $conf_only; then - echo "$0: warning: failed to ssh to root@$source" - else - die failed to ssh to root@$source - fi - fi - { read -r source_hostname; read -r zone; } <<<"$ssh_info" - if [[ $zone != "$local_zone" ]]; then - die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + if ! ssh_info=$(ssh root@$source 'hostname && date +%z'); then + if $conf_only; then + echo "$0: warning: failed to ssh to root@$source" + else + die failed to ssh to root@$source fi fi + { read -r source_hostname; read -r zone; } <<<"$ssh_info" + if [[ ! $source_hostname ]]; then + die "error: failed to get source hostname. source=$source" + fi + if [[ $zone != "$local_zone" ]]; then + die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost" + fi else sshable=() @@ -934,7 +948,6 @@ del-orphan-snaps if $dry_run; then m btrbk -c /etc/btrbk$conf_suf.conf -v -n $cmd_arg - mexit 0 else logq btrbk -c /etc/btrbk$conf_suf.conf $preserve_arg $verbose_arg $progress_arg $cmd_arg fi @@ -955,7 +968,7 @@ if mountpoint $rsync_mountpoint >/dev/null; then dirs+=(${x%.hosts}) fi done - m rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/ + d rsync -aSAXPH --specials --devices --delete --relative ${dirs[@]} root@$tg:/ ;; esac done @@ -969,16 +982,18 @@ if [[ $source ]]; then d mount-latest-subvol "${subvols[@]}" else for tg in ${targets[@]}; do - d /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$? + m /a/exe/mount-latest-remote "$tg" "${subvols[@]}" || ret=$? done fi ## run extra commands on targets +# shellcheck disable=SC2116 # intentional expansion local_snaps=$(echo $snap_list_glob) if [[ $ret == 0 ]]; then for tg in ${targets[@]}; do + # todo: get this in ssh_info, less sshing is faster. h=$(ssh $tg hostname) remote_snaps=$(ssh root@$tg "shopt -s nullglob; echo $snap_list_glob") # a check like this will catch the situation we aim to prevent by running purge @@ -987,10 +1002,17 @@ if [[ $ret == 0 ]]; then printf "%s\n" "$local_snaps" |tr ' ' '\n' >$localtmp remotetmp=$(mktemp) printf "%s\n" "$remote_snaps" |tr ' ' '\n' >$remotetmp - e "error: for $tg, remote and local snaps are different." - e "local: $local_snaps" - e "tg:$tg = $remote_snaps" - e "diff -u local remote" + + if $dry_run; then + e "dry run, expected error:" + fi + cat <<"EOF" | sed "s/^/$pre /" +error: for $tg, remote and local snaps are different. +local: $local_snaps +tg:$tg = $remote_snaps +diff -u local remote: +EOF + diff -u $localtmp $remotetmp rm $localtmp $remotetmp ret=1 @@ -999,17 +1021,20 @@ if [[ $ret == 0 ]]; then d ssh root@$tg 'btrbk-spread-wrap &>/dev/null