/old-unused
/.emacs.d
/Arduino
+/a/roles/prom
--- /dev/null
+[defaults]
+forks = 200
+host_key_checking = False
+display_skipped_hosts = False
+retry_files_enabled = False
+
+[ssh_connection]
+pipelining = True
+retries = 2
--- /dev/null
+---
+- hosts: tp.b8.nz
+ roles:
+ - prom
xclock -digital -update 1 -face 'arial black-80:bold'
}
-bpush() {
- case $HOSTNAME in
- tp) target=x2 ;;
- x2) target=tp ;;
- esac
- btrbk-run -t $target && switch-mail-host $HOSTNAME $target
-}
bpull() {
- case $HOSTNAME in
- tp) source=x2 ;;
- x2) source=tp ;;
- esac
- btrbk-run -s $source && switch-mail-host $source $HOSTNAME
+ c /
+ # run latest
+ install-my-scripts
+ switch-mail-host $1 $HOSTNAMEs
}
btc() {
journalctl -n 10000 -f "$@" | jfilter
}
jr() { journalctl "$@" | jfilter | less ; }
+jrf() { journalctl -f "$@" | jfilter; }
kff() { # keyboardio firmware flash
pushd /a/bin/distro-setup/Arduino/Model01-Firmware
}
-m() { printf "%s: %s\n" "${0##*/}" "$*"; "$@"; }
+m() { if $verbose; then printf "$pre%s\n" "$*"; fi; "$@"; }
+die() { printf "$pre%s\n" "$*" >&2; exit 1; }
# latest $MAIL_HOST
if [[ -e /b/bash_unpublished/source-semi-priv ]]; then
sleep 5
fi
+cron=false
orig_args=("$@")
-temp=$(getopt -l pull-reexec,help cl:m:nps:t:vh "$@") || usage 1
+temp=$(getopt -l cpull-reexec,help cl:m:npqs:t:vh "$@") || usage 1
eval set -- "$temp"
while true; do
case $1 in
+ --cron)
+ cron=true
+ pre="${0##*/}: "
+ shift
+ ;;
# only creates the config file, does not run btrbk
-c) conf_only=true; shift ;;
# bytes per second, suffix k m g
-v) verbose=true; verbose_arg=-v; shift ;;
-h|--help) usage ;;
--) shift; break ;;
- *) echo "$0: Internal error!" ; exit 1 ;;
+ *) die "Internal error!" ;;
esac
done
cmd_arg=${1:-run}
if [[ -v targets && $source ]]; then
- echo "$0: error: -t and -s are mutually exclusive" >&2
- exit 1
+ die "error: -t and -s are mutually exclusive"
fi
-
-echo -e "$0: options: conf_only=$conf_only\ndry_run=$dry_run\nrate_limit=$rate_limit\nverbose=$verbose\ncmd_arg=$cmd_arg"
+if $verbose; then
+ printf "$0: options: conf_only=%s\ndry_run=%s\nrate_limit=%s\nverbose=%s\ncmd_arg=%s" "$conf_only" "$dry_run" "$rate_limit" "$verbose" "$cmd_arg"
+fi
### end options parsing
# set default targets
-if [[ ! -v targets && ! $source ]]; then
+if [[ ! -v targets && ! $source && $HOSTNAME == $MAIL_HOST ]]; then
case $HOSTNAME in
- x2|kw)
- if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
- targets=($HOME_DOMAIN)
+ kw|x2)
+ if ping -q -c1 -w1 iank.vpn.office.fsf.org &>/dev/null; then
+ home=iank.vpn.office.fsf.org
+ else
+ home=$HOME_DOMAIN
fi
+ ;;&
+ kw)
+ targets=($home x2)
+ ;;
+ x2)
+ targets=($home kw)
;;
tp)
- targets=(frodo)
- if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
- if timeout -s 9 10 ssh x2 :; then
- targets+=(x2)
- fi
+ targets=(frodo kd)
+ # might not be connected to the vpn
+ if timeout -s 9 10 ssh kw :; then
+ targets+=(kw)
fi
;;
- frodo)
- targets=()
+ kd)
+ targets=(frodo tp)
+ # might not be connected to the vpn
+ if timeout -s 9 10 ssh kw :; then
+ targets+=(kw)
+ fi
;;
*)
- echo "$0: error: no default targets for this host, use -t"
- exit 1
+ die "error: no default targets for this host, use -t"
;;
esac
fi
if [[ $mountpoints ]]; then
for mp in ${mountpoints[@]}; do # default mountpoints to sync
if [[ -e /nocow/btrfs-stale/$mp ]]; then
- echo "$0: error: $mp is stale, mount-latest-subvol first"
- exit 1
+ die "error: $mp is stale, mount-latest-subvol first"
fi
done
else
if ! which btrbk &>/dev/null; then
- echo "$0: error: no btrbk binary found"
- exit 1
+ die "error: no btrbk binary found"
fi
# if our mountpoints are from stale snapshots,
# it doesn't make sense to do a backup.
-check-subvol-stale ${mountpoints[@]} || exit 1
+check-subvol-stale ${mountpoints[@]} || die "found stale mountpoints in ${mountpoints[*]}"
cat >/etc/btrbk.conf <<EOF
mkdir -p /mnt/root/btrbk
local_zone=$(date +%z)
for h in ${targets[@]} $source; do
- zone=$(ssh root@$h "mkdir -p /mnt/root/btrbk; date +%z")
+ cmd="ssh root@$h mkdir -p /mnt/root/btrbk && date +%z"
+ if ! zone=$($cmd); then
+ die "error running: $cmd"
+ fi
if [[ $zone != $local_zone ]]; then
- echo "error: error. dont confuse yourself with multiple time zones. $h has different timezone than localhost" >&2
- exit 1
+ die "error: dont confuse yourself with multiple time zones. $h has different timezone than localhost"
fi
done
done
fi
-/a/bin/distro-setup/install-my-scripts
if [[ $source ]]; then
- m mount-latest-subvol
+ m mount-latest-subvol $verbose_arg
else
m /a/exe/mount-latest-remote ${targets[@]}
fi
if [[ $svp_cgen -ge $last_received_cgen ]]; then
stale=false
else
- echo "$svp stale: it's gen at creation, $svp_cgen, is earlier than the last received snapshot, $last_received's gen at creation: $last_received_cgen"
+ d "$svp stale: it's gen at creation, $svp_cgen, is earlier than the last received snapshot, $last_received's gen at creation: $last_received_cgen"
freshest_snap=$last_received
stale=true
fi
all_dirs=({/a/c,/p/c}{,/machine_specific/$HOSTNAME})
# note, we assume a group of hosts does not have the
# same name as a single host, which is no problem on our scale.
-for x in /p/c/machine_specific/*.hosts; do
+for x in /p/c/machine_specific/*.hosts /a/bin/ds/machine_specific/*.hosts; do
if grep -qxF $HOSTNAME $x; then all_dirs+=( ${x%.hosts} ); fi
done
c_dirs=(/a/c{,/machine_specific/$HOSTNAME})
case $USER in
- ian|iank)
+ iank)
+ files=(/p/c/machine_specific/*/filesystem/etc/ssh/*_key)
+ if [[ -e $files ]]; then
+ chmod 600 ${files[@]}
+ fi
# p needs to go first so .ssh link is created, then config link inside it
m common-file-setup ${all_dirs[@]}
#### begin special extra stuff ####
install -d -m700 ~/gpg-agent-socket
- files=(/var/lib/bind)
- if [[ -e $files ]]; then
+ f=/var/lib/bind
+ if [[ -e $f ]]; then
# reset to the original permissions.
- m s chgrp -R bind ${files[@]}
- m s chmod g+w ${files[@]}
+ m s chgrp -R bind $f
+ m s chmod g+w $f
fi
- sudo bash -c 'shopt -s nullglob; cd /etc/bind; for f in *.key *.private key.*; do chgrp bind $f; done'
+ sudo bash -c 'shopt -s nullglob; for f in /etc/bind/*.key /etc/bind/*.private /etc/bind/key.*; do chgrp bind $f; done'
if [[ -e /etc/davpass ]] && getent group www-data &>/dev/null; then
s chgrp www-data /etc/davpass
fi
--- /dev/null
+[main]
+kd.b8.nz
+kw.office.fsf.org
+tp.b8.nz
+frodo.b8.nz
+x2.office.fsf.org
+x3.b8.nz
kd
-frodo
\ No newline at end of file
+kw
+x2
+tp
[Service]
Type=oneshot
-ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /a/exe/btrbk-run
+ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /usr/local/bin/btrbk-run -q
+++ /dev/null
-[Unit]
-Description=Btrbk backup
-After=multi-user.target
-
-[Service]
-Type=oneshot
-ExecStart=/a/bin/log-quiet/sysd-mail-once btrbk /a/exe/btrbk-run
+++ /dev/null
-[Unit]
-Description=Run btrbk-run once every 20 mins
-
-[Timer]
-OnCalendar=*:0/20
-
-[Install]
-WantedBy=timers.target
# See the License for the specific language governing permissions and
# limitations under the License.
-# usage: mount-latest-subvol
-#
-# Note, at source location, intentionally not executable, run and read
-# install-my-scripts.
-
+script=$(readlink -f -- "$BASH_SOURCE")
cd /
-[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@"
+[[ $EUID == 0 ]] || exec sudo -E "$script" "$@"
+
+
+usage() {
+ cat <<EOF
+Usage: ${0##*/} [OPTIONS]
+
+-h|--help Print help and exit.
+
+
+Note, at source location, intentionally not executable, run and read
+install-my-scripts.
+
+Note: Uses util-linux getopt option parsing: spaces between args and
+options, short options can be combined, options before args.
+EOF
+ exit $1
+}
+
errcatch() {
set -E; shopt -s extdebug
grep -xFq "$line" "$file" || tee -a "$file"<<<"$line"
done
}
-e() { printf "%s\n" "$*"; "$@"; }
+m() {
+ if $verbose; then
+ printf "%s\n" "$*"
+ fi
+ "$@"
+}
+x() {
+ printf "%s\n" "$*"
+ "$@"
+}
+
mnt() {
dir=$1
- if ! mountpoint $dir &>/dev/null; then
+ if ! mountpoint -q $dir; then
mkdir -p $dir
- e mount $dir
+ m mount $dir
fi
}
fstab() {
return 1
}
+##### begin command line parsing ########
+
+# you can remove this if you do not have options which can have args with spaces or empty.
+
+verbose=false
force=false
-if [[ $1 == -f ]]; then
- force=true
-fi
+temp=$(getopt -l help,force,verbose hfv "$@") || usage 1
+eval set -- "$temp"
+while true; do
+ case $1 in
+ -f|--force) force=true ;;
+ -v|--verbose) verbose=true ;;
+ -h|--help) usage ;;
+ --) shift; break ;;
+ *) echo "$0: unexpected args: $*" >&2 ; usage 1 ;;
+ esac
+ shift
+done
+
+##### end command line parsing ########
ret=0
root_dev=$(awk '$2 == "/" {print $1}' /etc/mtab)
if [[ $root_dev == /dev/dm-* ]]; then
for d in /dev/mapper/*; do
- if [[ $(readlink -f $d) == $root_dev ]]; then
+ if [[ $(readlink -f $d) == "$root_dev" ]]; then
root_dev=$d
break
fi
# ownership, and ssh doesn\'t allow any group writable parent
# directories, so we are forced to use a directory structure similar
# to home directories
-f=(/mnt/root/btrbk/q.*)
+f=(/mnt/root/btrbk/q.*); f=${f[0]}
if [[ -e $f ]]; then
fstab <<EOF
$crypt_dev /q btrfs noatime,subvol=q,gid=1000$noauto 0 0
EOF
fi
-f=(/mnt/root/btrbk/o.*)
+f=(/mnt/root/btrbk/o.*); f=${f[0]}
if [[ -e $f ]]; then
fstab <<EOF
$crypt_dev /o btrfs noatime,subvol=o$noauto 0 0
# if latest is already mounted, make sure binds are mounted and move on
- e check-subvol-stale $d
+ m check-subvol-stale $d
# populated by check-subvol-stale if stale
if ! fresh_snap=$(cat /nocow/btrfs-stale/$vol 2>/dev/null); then
mnt $d
umount_ret=true
unmounted=()
for dir in $(echo $d ${binds[*]}\ |tac -s\ ); do
- if mountpoint $dir; then
- if e umount -R $dir; then
+ if mountpoint -q $dir; then
+ if m umount -R $dir; then
unmounted+=($dir)
else
if ! kill-dir TERM TERM TERM INT INT HUP HUP; then
if $force; then kill-dir KILL; fi
fi
- if e umount -R $dir; then
+ if m umount -R $dir; then
unmounted+=($dir)
else
echo "$0: failed to umount $dir"
# todo: decipher /mnt/root, like we do in check-subvol-stale
cd /mnt/root
if [[ -e $vol ]]; then
- e mv $vol $vol.leaf.$(date +%Y-%m-%dT%H:%M:%S%z)
+ leaf=$vol.leaf.$(date +%Y-%m-%dT%H:%M:%S%z)
+ m mv $vol $leaf
+ m btrfs property set -ts $leaf ro true
+
+ ### begin check if leaf is different, delete it if not ###
+ if [[ -e /a/opt/btrfs-snapshots-diff/btrfs-snapshots-diff.py ]]; then
+ parentid=$(btrfs sub show $leaf | awk '$1 == "Parent" && $2 == "UUID:" {print $3}')
+ bsubs=(/mnt/root/btrbk/$vol.*)
+ bsub=
+ # go in reverse order as its more likely to be at the end
+ for ((i=${#bsubs[@]}-1; i>=0; i--)); do
+ if [[ $parentid == $(btrfs sub show ${bsubs[i]} | awk '$1 == "UUID:" {print $2}') ]]; then
+ bsub=${bsubs[i]}
+ break
+ fi
+ done
+ if [[ $bsub ]]; then
+ tmp=$(mktemp)
+ # in testing, same subvol is 136 bytes. allow some overhead
+ btrfs send --no-data -p $bsub $leaf | head -c 1000 > $tmp
+ if (( $(stat -c%s $tmp) < 1000)); then
+ # example output for an empty diff:
+ # Found a valid Btrfs stream header, version 1
+ # o.leaf.2019-05-15T14:00:50-0400;snapshot: uuid=ba045ea30737dd449003f1ee40ec12d0, ctrasid=109533, clone_uuid=3c7e3544e486834aa71d89e5b8f30056, clone_ctransid=109533
+ lines=$(/a/opt/btrfs-snapshots-diff/btrfs-snapshots-diff.py -s -f $tmp | \
+ grep -vxF "Found a valid Btrfs stream header, version 1" | \
+ grep -cv "^[^;]*;snapshot: ") ||:
+ if [[ $lines == 0 ]]; then
+ x btrfs sub del $leaf
+ fi
+ fi
+ fi
+ fi
+ ### end check if leaf is different, delete it if not ###
+
+ ## begin expire leaf vols ##
leaf_vols=($vol.leaf.*)
+ count=1
for leaf in ${leaf_vols[@]}; do
leaf_secs=$(date -d ${leaf#$vol.leaf.} +%s)
- if (( $(date +%s) - 60*60*24*60 > leaf_secs )); then # 60 days
- e btrfs sub del $leaf
+ if (( $(date +%s) - 60*60*24*60 > leaf_secs || count > 200 )); then # 60 days
+ x btrfs sub del $leaf
fi
+ count=$((count+1))
done
+ ## end expire leaf vols ##
fi
# Note, we make a few assumptions in this script, like
# $d was not a different subvol id than $vol, and
# things otherwise didn't get mounted very strangely.
- e btrfs sub snapshot $fresh_snap $vol
+ m btrfs sub snapshot $fresh_snap $vol
for dir in $d ${binds[@]}; do
- e mnt $dir
+ m mnt $dir
done
stale_dir=/nocow/btrfs-stale
rm -f $stale_dir/$d
local network. Turn off mail receiving on OLD_HOST, run btrbk to move
mail to NEW_HOST, turn on mail receiving on NEW_HOST.
+-o Only btrbk /o, instead of all filesystems.
-w Don't try to ssh to wrt. Should only be used in unusual network situation.
-h|--help Print help and exit.
update_wrt=true # default
temp=$(getopt -l help wh "$@") || usage 1
+mp_args=
eval set -- "$temp"
while true; do
case $1 in
+ -o) mp_args="-m /o"; shift ;;
-w) update_wrt=false; shift ;;
-h|--help) usage ;;
--) shift; break ;;
/a/bin/distro-setup/install-my-scripts
if $localhost_new; then
- btrbk-run -s $old_host -m /o
+ btrbk-run -s $old_host $mp_args
else
- btrbk-run -t $new_host -m /o
+ btrbk-run -t $new_host $mp_args
fi
$new_shell primary-setup $new_hostname
iankelling.org / git / distro-setup / commitdiff