From 165008e5b82c81ebff1061c8f8294c3bc6e9dced Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sun, 10 Nov 2019 14:09:29 -0500 Subject: [PATCH] various fixes and improvements --- .bash_profile | 26 ++++++- .bashrc | 23 ++++++ brc | 25 +------ brc2 | 75 +++++++++++++++---- btrbk-run | 19 +++-- conflink | 55 +++++++++----- distro-end | 8 +- filesystem/etc/client.conf | 2 - filesystem/etc/dnsmasq.d/iank.conf | 1 + .../etc/cron.daily/mediawiki_backup | 33 -------- mail-setup | 14 ++-- switch-mail-host | 24 +++--- 12 files changed, 184 insertions(+), 121 deletions(-) delete mode 100644 filesystem/etc/client.conf delete mode 100755 machine_specific/kd/filesystem/etc/cron.daily/mediawiki_backup mode change 100755 => 100644 switch-mail-host diff --git a/.bash_profile b/.bash_profile index cd422cb..09b0001 100644 --- a/.bash_profile +++ b/.bash_profile @@ -11,10 +11,30 @@ # it is purposefully in a user startup file, not a system one, so intended for # the user to change. +# History related options first, or else +# we risk screwing up history history. And this is duplicated +# in ~/.bash_profile just for good measure +# history number. History expansion is good. +PS4='$LINENO+ ' +# history file size limit, set to unlimited. +# this needs to be different from the default because +# default HISTFILESIZE is 500 and could clobber our history +HISTFILESIZE= +# max commands 1 session can append/read from history +HISTSIZE=1000000 +# the time format display when doing the history command +# also, setting this makes the history file record time +# of each command as seconds from the epoch +HISTTIMEFORMAT="%Y-%m-%d %I:%M %p " +# consecutive duplicate lines dont go in history +HISTCONTROL=ignoredups +# works in addition to HISTCONTROL to do more flexible things +# it could also do the same things as HISTCONTROL and thus replace it, +# but meh. dunno why, but just " *" does glob expansion, so use [ ] to avoid it. +HISTIGNORE='pass *:[ ]*:otp *:oathtool *' + + [[ -f ~/.bashrc ]] && . ~/.bashrc # ensure no bad programs appending to this file will have an affect return 0 - - -export PATH="$HOME/.cargo/bin:$PATH" diff --git a/.bashrc b/.bashrc index ac3ec03..840a353 100644 --- a/.bashrc +++ b/.bashrc @@ -9,6 +9,29 @@ #exec 2>>/a/tmp/bashlog +# History related options first, or else +# we risk screwing up history history. And this is duplicated +# in ~/.bash_profile just for good measure +# history number. History expansion is good. +PS4='$LINENO+ ' +# history file size limit, set to unlimited. +# this needs to be different from the default because +# default HISTFILESIZE is 500 and could clobber our history +HISTFILESIZE= +# max commands 1 session can append/read from history +HISTSIZE=1000000 +# the time format display when doing the history command +# also, setting this makes the history file record time +# of each command as seconds from the epoch +HISTTIMEFORMAT="%Y-%m-%d %I:%M %p " +# consecutive duplicate lines dont go in history +HISTCONTROL=ignoredups +# works in addition to HISTCONTROL to do more flexible things +# it could also do the same things as HISTCONTROL and thus replace it, +# but meh. dunno why, but just " *" does glob expansion, so use [ ] to avoid it. +HISTIGNORE='pass *:[ ]*:otp *:oathtool *' + + # see comments in brc2 sl() function for background. if [[ $SSH_CONNECTION ]] \ && [[ $BRC != t ]]; then diff --git a/brc b/brc index 74cda5b..b5d4596 100644 --- a/brc +++ b/brc @@ -33,8 +33,6 @@ fi CDPATH=. -set -o pipefail - # remove all aliases. aliases provided by the system tend to get in the way, # for example, error happens if I try to define a function the same name as an alias unalias -a @@ -161,28 +159,9 @@ if [[ $- == *i* ]]; then fi - -# history number. History expansion is good. -PS4='$LINENO+ ' -# history file size limit, set to unlimited. -# this needs to be different from the default because -# default HISTFILESIZE is 500 and could clobber our history -HISTFILESIZE= -# max commands 1 session can append/read from history -HISTSIZE=1000000 -# the time format display when doing the history command -# also, setting this makes the history file record time -# of each command as seconds from the epoch -HISTTIMEFORMAT="%Y-%m-%d %I:%M %p " -# consecutive duplicate lines dont go in history -HISTCONTROL=ignoredups -# works in addition to HISTCONTROL to do more flexible things -# it could also do the same things as HISTCONTROL and thus replace it, -# but meh. dunno why, but just " *" does glob expansion, so use [ ] to avoid it. -HISTIGNORE='pass *:[ ]*:otp *:oathtool *' - export BC_LINE_LENGTH=0 +# ansible option export PROFILE_TASKS_TASK_OUTPUT_LIMIT=100 # note, if I use a machine I dont want files readable by all users, set @@ -875,7 +854,7 @@ grep ps and output in a nice format" # final grep is because some commands tend to have a lot of trailing spaces y=$(echo "$x" | grep -iP "$@" | grep -o '.*[^ ]') ||: if [[ $y ]]; then - echo "$x" | head -n 1 + echo "$x" | head -n 1 || [[ $? == 141 ]] echo "$y" fi } diff --git a/brc2 b/brc2 index a3ab32a..891e095 100644 --- a/brc2 +++ b/brc2 @@ -108,6 +108,8 @@ EOF } bbk() { # btrbk wrapper + + local pid c / local active=true systemctl is-active btrbk.timer || active=false @@ -121,10 +123,15 @@ bbk() { # btrbk wrapper fi # run latest install-my-scripts - unbuffer btrbk-run "$@" |& pee cat "systemd-cat -t btrbk-run" + jrun -p btrbk btrbk-run "$@" if $active; then - ser enable btrbk.timer + if (( $ret )); then + echo bbk: WARNING: btrbk.timer not reenabled due to failure + else + ser enable btrbk.timer + fi fi + return $ret } bfg() { java -jar /a/opt/bfg-1.12.14.jar "$@"; } @@ -133,25 +140,67 @@ bigclock() { xclock -digital -update 1 -face 'arial black-80:bold' } -bpull() { - [[ $1 ]] || return 1 - c / - # run latest - install-my-scripts - ubuffer switch-mail-host pull $1 |& pee cat "systemd-cat -t switch-mail-host" +_jrun() { # journal run. run args, log to journal, tail and grep the journal. + local pid pattern jr_pid sedscript cmd_name ended + ret=0 + case $1 in + -p) + pattern="$2\|" + shift 2 + ;; + esac + cmd_name=${1##*/} + systemd-cat -t "$cmd_name" "$@" 2>&1 & + pid=$! + sedscript="/$pattern$cmd_name/p;/^.{16}[^ ]+ $cmd_name\[$pid]: ([^ ]+ +)?exiting with status [0-9]+\$/q" + journalctl -S "4 seconds ago" -f |& sed -nr "$sedscript" 2>&1 & + jr_pid=$! + wait $pid 2>&1 || ret=$? + if (( $ret )); then + echo "$0: ERROR: $* returned $ret" + fi + jobs %'journalctl -S "4 seconds ago" -f' &>/dev/null + ended=false + # give it about 5 seconds to find the end of the log + for (( i=0; i<20; i++ )); do + if jobs %'journalctl -S "4 seconds ago" -f' &>/dev/null; then + sleep .3 2>&1 + else + ended=true + break + fi + done + if ! $ended; then + kill $jr_pid 2>&1 + fi } -bpush() { - [[ $1 ]] || return 1 +jrun() { + # ditching stderr avoids the jobs status change output. + _jrun "$@" 2>/dev/null +} + +sm() { + (( $# == 2 )) || return 1 + action=$1 + shift + host=$1 + shift + case $action in + pull|push) : ;; + *) return 1 ;; + esac c / # run latest install-my-scripts - ubuffer switch-mail-host push $1 |& pee cat "systemd-cat -t switch-mail-host" + jrun -p btrbk switch-mail-host $action $host + return $ret } + lipush() { # note, i had --delete-excluded, but that deletes all files in --exclude-from on # the remote site, which doesn't make sense, so not sure why i had it. local p a - p=(/a/bin /a/exe /a/h /a/c /p/c/machine_specific/linode{,.hosts} /a/opt/{emacs,emacs-debianstable,mu}) + p=(/a/bin /a/exe /a/h /a/c /p/c/machine_specific/linode{,.hosts} /a/opt/{emacs-debianstable,mu}) a="-ahviSAXPH --specials --devices --delete --relative --exclude-from=/p/c/li-rsync-excludes" ret=0 m rsync "$@" $a ${p[@]} /p/c/machine_specific/l2 root@l2.b8.nz:/ || ret=$? @@ -993,7 +1042,7 @@ sss() { # ssh solo } # kill off old shared socket then ssh ssk() { - m ssh -O exit "$@" + m ssh -O exit "$@" || [[ $? == 255 ]] m sl "$@" } # plain limited ssh diff --git a/btrbk-run b/btrbk-run index 10bd638..60f065d 100644 --- a/btrbk-run +++ b/btrbk-run @@ -45,7 +45,8 @@ rsync-dirs() { pre="${0##*/}: " m() { if $verbose; then printf "$pre%s\n" "$*"; fi; "$@"; } e() { printf "$pre%s\n" "$*"; } -die() { printf "$pre%s\n" "$*" >&2; exit 1; } +die() { printf "$pre%s\n" "$*" >&2; echo "exiting with status 1" >&2; exit 1; } +mexit() { echo "exiting with status $1"; exit $1; } # latest $MAIL_HOST if [[ -e /b/bash_unpublished/source-state ]]; then @@ -129,6 +130,7 @@ if $verbose; then fi ### end options parsing + # TODO: i wonder if there should be an option to send to the default # targets, plus any given on the command line. @@ -136,7 +138,7 @@ fi if [[ ! -v targets && ! $source ]]; then if [[ $HOSTNAME != "$MAIL_HOST" ]] && $cron ; then echo "MAIL_HOST=$MAIL_HOST, nothing to do" - exit 0 + mexit 0 fi at_work=false @@ -242,7 +244,7 @@ echo "mountpoints: ${mountpoints[*]}" if [[ $source ]]; then if [[ $(ssh $source systemctl is-active btrbk.service) != inactive ]]; then echo "$0: error: cron btrbk is running on source. exiting out of caution" - exit 1 + mexit 1 fi fi @@ -262,7 +264,7 @@ if ! $pull_reexec && [[ $source ]] && $pulla ; then e "found newer version on host $source. reexecing" install -T $tmpf /usr/local/bin/btrbk-run m /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}" - exit + mexit 0 fi fi @@ -399,14 +401,14 @@ done if $conf_only; then - exit + mexit 0 fi if $dry_run; then m btrbk -v -n $cmd_arg - exit 0 + mexit 0 elif [[ $cmd_arg == archive ]]; then if [[ $source ]]; then m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol @@ -415,7 +417,7 @@ elif [[ $cmd_arg == archive ]]; then m btrbk $verbose_arg $progress_arg $cmd_arg $vol ssh://$tg$vol done fi - exit 0 + mexit 0 fi # -q and just using the syslog option seemed nice, # but it doesn't show when a send has a parent and when it doesn't. @@ -443,7 +445,8 @@ else m /a/exe/mount-latest-remote ${targets[@]} fi -exit $ret +e btrbk-run COMPLETED with exit status $ret +mexit $ret # todo: move variable data we don't care about backing up # to /nocow and symlink it. diff --git a/conflink b/conflink index 55d7a02..61291f2 100755 --- a/conflink +++ b/conflink @@ -25,12 +25,7 @@ s() { sudo "$@"; } lnf() { /a/exe/lnf "$@"; } -# error prone -#f=~/.local/conflink -# fast=false -# if [[ -e $f ]] && (( $(stat -c %Y $f) > $now - 60*60*24 )); then -# fast=true -# fi +fast=false if [[ $1 == -f ]]; then # f for fast fast=true shift @@ -79,22 +74,35 @@ subdir-link-r() { common-file-setup() { - local dir fs x f systemd_reload - systemd_reload=false + local dir fs x f reload_systemd + local -a reload_services + local -a restart_services + reload_systemd=false for dir in "$@"; do fs=$dir/filesystem if [[ -e $fs && $user =~ ^iank?$ ]]; then - # this could be done with rsync, something like this, - # but I haven't looked at the symlinks. - # s rsync -n -ahviSAXPH --specials --devices --chown=root:root --chmod=g-s $fs / - # note, symlinks get resolved, not copied. - if s tar --anchored \ - --exclude etc/dovecot/users \ - --exclude etc/exim4/passwd \ - --exclude etc/exim4/*.pem \ - --mode=g-s --owner=0 --group=0 -cz -C $fs . | s tar -dz -C / | grep /etc/systemd &>/dev/null; then - systemd_reload=true - fi + while read -r line; do + file="${line:12}" + case $file in + etc/systemd/system/*) + reload_systemd=true + ;; + etc/dnsmasq.d/*) + restart_services+=(dnsmasq) + ;; + esac + # Previously did this with tar, but it doesn't + # update directory permissions. + # + # S = do spare files efficiently + # A = preserve acls + # X = preserve extended attributes + # i = itemize + done < <(s rsync -n -aiSAX --chown=root:root --chmod=g-s \ + --exclude=/etc/dovecot/users \ + --exclude='/etc/exim4/passwd*' \ + --exclude='/etc/exim4/*.pem' \ + $fs/ /) fi if [[ -e $dir/subdir_files ]]; then @@ -104,9 +112,15 @@ common-file-setup() { (( ${#x[@]} >= 1 )) || continue m lnf ${x[@]} ~ done - if $systemd_reload; then + if $reload_systemd; then m s systemctl daemon-reload fi + for service in ${restart_services[@]}; do + if systemctl is-active $service >/dev/null; then + m s systemctl reload $service + fi + done + } user=$(id -un) @@ -169,6 +183,7 @@ case $user in if [[ -x $f ]]; then $f _ fi + mkdir -p ~/.local echo 0 >~/.local/conflink ;; diff --git a/distro-end b/distro-end index 1b72e2e..d63d315 100755 --- a/distro-end +++ b/distro-end @@ -436,9 +436,12 @@ case $HOSTNAME in li) m /a/h/setup.sh iankelling.org ;; + tp) + m /a/h/setup.sh -s b8.nz + ;; *) - # allow symlinks on other hosts so i can host files in arbitrary paths - m /a/h/setup.sh -s + # allow symlinks on non-main hosts so i can host files in arbitrary paths + m /a/h/setup.sh -s -p 80 ;; esac m /a/h/build.rb @@ -1414,7 +1417,6 @@ EOF # Remove dep that came in with desktop to fix associations. m pu transmission-gtk -sudo gpasswd -a iank adm #needed for reading logs m /a/bin/buildscripts/pithosfly diff --git a/filesystem/etc/client.conf b/filesystem/etc/client.conf deleted file mode 100644 index d78e09b..0000000 --- a/filesystem/etc/client.conf +++ /dev/null @@ -1,2 +0,0 @@ -# note, this needs to be disabled to print elsewhere. -ServerName printserver0.office.fsf.org diff --git a/filesystem/etc/dnsmasq.d/iank.conf b/filesystem/etc/dnsmasq.d/iank.conf index 520b8b2..6e7a423 100644 --- a/filesystem/etc/dnsmasq.d/iank.conf +++ b/filesystem/etc/dnsmasq.d/iank.conf @@ -1 +1,2 @@ + servers-file=/etc/dnsmasq-servers.conf diff --git a/machine_specific/kd/filesystem/etc/cron.daily/mediawiki_backup b/machine_specific/kd/filesystem/etc/cron.daily/mediawiki_backup deleted file mode 100755 index 51dcf6b..0000000 --- a/machine_specific/kd/filesystem/etc/cron.daily/mediawiki_backup +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# if we get an error, keep going but return it at the end -last_error=0 -trap 'last_error=$?' ERR -source ~/mw_vars -# No strict because the host is likely not named the same as -# the domain. -ssh="ssh -oStrictHostKeyChecking=no" -logfile=/var/log/${mwdomain}_backup.log -{ - echo "#### starting backup at $(date) ####" - $ssh root@$mwdomain < ~/wiki_backups/wiki_db_backup -sed -i '\$ d' $mwc # delete read only setting -ENDSSH - # add no strict option to the defaults - - rdiff() { rdiff-backup --remote-schema "$ssh -C %s rdiff-backup --server" "$@"; } - set -x - rdiff root@$mwdomain::/root/wiki_backups ~/backup/${mwdomain}_wiki_db_backup - rdiff root@$mwdomain::$mw ~/backup/${mwdomain}_wiki_file_backup - set +x - echo "=== ending backup at $(date) ====" -} &>>$logfile -if [[ $last_error != 0 ]]; then - echo "backup for $mwdomain failed. See $logfile" -fi -exit $last_error diff --git a/mail-setup b/mail-setup index 98d5c41..3dd3e97 100755 --- a/mail-setup +++ b/mail-setup @@ -358,6 +358,8 @@ m systemctl enable mailcert.timer # * common exim4 config source /a/bin/bash_unpublished/source-state +m sudo gpasswd -a iank adm #needed for reading logs + ### make local bounces go to normal maildir # local mail that bounces goes to /Maildir or /root/Maildir @@ -832,6 +834,12 @@ EOF hostname -f >/etc/mailname + f=/p/c/filesystem/etc/exim4/passwd.client + if [[ ! -e $f ]]; then + f=/p/c/machine_specific/$HOSTNAME/filesystem/etc/exim4/passwd.client + fi + m sudo rsync -ahhi --chown=root:Debian-exim --chmod=0640 $f /etc/exim4/ + ;;& ## we use this host to monitor MAIL_HOST l2) @@ -851,13 +859,9 @@ EOF /^root:/d EOF ;; + # not l2 and not MAIL_HOST *) - f=/p/c/filesystem/etc/exim4/passwd.client - if [[ ! -e $f ]]; then - f=/p/c/machine_specific/$HOSTNAME/filesystem/etc/exim4/passwd.client - fi - sudo rsync -ahhi --chown=root:Debian-exim --chmod=0640 $f /etc/exim4/ # This ends up at alerts mailbox on MAIL_HOST, but using a user that doesn't exist elsewhere # is no good. diff --git a/switch-mail-host b/switch-mail-host old mode 100755 new mode 100644 index 8f0912f..14a11f8 --- a/switch-mail-host +++ b/switch-mail-host @@ -34,11 +34,12 @@ errcatch-cleanup() { e $old_shell sudo systemctl start btrbk.timer fi } -b + pre="${0##*/}:" m() { printf "$pre %s\n" "$*"; "$@"; } e() { printf "$pre %s\n" "$*"; } err() { echo "[$(date +'%Y-%m-%d %H:%M:%S%z')]: $pre: $*" >&2; } +mexit() { echo "exiting with status $1"; exit $1; } ##### begin command line parsing ######## @@ -50,7 +51,7 @@ while true; do -o) mp_args="-m /o"; shift ;; -h|--help) usage ;; --) shift; break ;; - *) echo "$0: Internal error! unexpected args: $*" ; exit 1 ;; + *) echo "$0: Internal error! unexpected args: $*" ; mexit 1 ;; esac done @@ -74,7 +75,7 @@ case $1 in ;; *) err invalid first argument - exit 1 + mexit 1 ;; esac @@ -82,15 +83,13 @@ esac source /a/bin/bash_unpublished/source-state if [[ $old_host != "$MAIL_HOST" ]]; then - read -r -p "warning: \$old_host != \$MAIL_HOST: $old_host != $MAIL_HOST, proceed? y/N " - if [[ $REPLY != [yY] ]]; then - exit 1 - fi + e "WARNING: \$old_host != \$MAIL_HOST. Sleeping for 5 seconds in case you want to reconsider" + sleep 5 fi if [[ ! $new_host || ! $old_host ]]; then echo "$0: bad args. see script" - exit 1 + mexit 1 fi e $new_shell hostname @@ -148,21 +147,24 @@ EOF # ipv6 i think. +e Running initial btrbk if ! m btrbk-run -v $bbk_args $mp_args; then ret=$? err "failed initial btrbk" - exit $ret + mexit $ret fi - m $old_shell /a/exe/primary-setup $new_hostname +e Running main btrbk if ! m btrbk-run -v $bbk_args -m /o; then ret=$? bang="$(printf "$(tput setaf 5)█$(tput sgr0)%.0s" 1 2 3 4 5 6 7)" e $bang failed btrbk of /o. restoring old host as primary m $old_shell /a/exe/primary-setup localhost - exit $ret + mexit $ret fi m $new_shell /a/exe/primary-setup localhost + +mexit 0 -- 2.30.2