From 0cc08a73110c15c4e7bbb8c8233a6c122c27490e Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Tue, 16 Apr 2019 16:01:26 -0400 Subject: [PATCH] various fixes --- brc | 16 +- btrbk-run | 11 +- check-subvol-stale | 9 +- distro-end | 53 ++- .../system/fsf-vpn-dns-cleanup.service | 10 + fsf-dnsmasq-servers.conf | 8 + fsf-vpn-dns-cleanup | 10 + install-my-scripts | 2 +- mail-setup | 25 +- mount-latest-subvol | 2 +- offlineimap-sync | 68 ++-- pkgs | 325 +++++++++--------- switch-mail-host | 131 +++---- 13 files changed, 363 insertions(+), 307 deletions(-) create mode 100644 filesystem/etc/systemd/system/fsf-vpn-dns-cleanup.service create mode 100644 fsf-dnsmasq-servers.conf create mode 100755 fsf-vpn-dns-cleanup diff --git a/brc b/brc index 6cbc09a..e85f309 100644 --- a/brc +++ b/brc @@ -182,6 +182,8 @@ if [[ -e /a/bin/errhandle/err ]]; then err-allow fi path_add /a/exe +# end just because it seems safer +path_add --end node_modules/.bin # pip3 --user things go here: path_add --end ~/.local/bin path_add --ifexists --end /a/work/libremanage @@ -332,8 +334,8 @@ b() { c - } -bkr() { - # get latest +bbk() { + # run latest install-my-scripts btrbk-run "$@" } @@ -987,8 +989,10 @@ hrcat() { local f; for f; do [[ -f $f ]] || continue; hr; echo "$f"; cat "$f"; d # get latest hub and run it # main command to use: -# hub pull-request -# on first use, you input username/pass and it gets an oath token so you dont have to repeat +# hub pull-request --no-edit +# --no-edit means to use the first commit\'s message as the pull request message. +# Also, you need to use a feature branch, not master in your fork. +# On first use, you input username/pass and it gets an oath token so you dont have to repeat # it\'s at ~/.config/hub hub() { local up uptar updir p @@ -1751,8 +1755,8 @@ testexim() { # adding -d, etc to COMMONOPTIONS in # /etc/default/exim4 exim -d -t <<'EOF' -From: root@gnu.org -To: ian@iankelling.org +From: ian@iankelling.org +To: root@lists0p.fsf.org Subject: Testing Exim This is a test message. diff --git a/btrbk-run b/btrbk-run index 7070f20..df17880 100644 --- a/btrbk-run +++ b/btrbk-run @@ -35,6 +35,11 @@ EOF exit $1 } +# latest $MAIL_HOST +if [[ -e /b/bash_unpublished/source-semi-priv ]]; then + source /b/bash_unpublished/source-semi-priv +fi + script_dir=$(dirname $(readlink -f "$BASH_SOURCE")) # note q is owned by root:1000 @@ -147,11 +152,6 @@ case $HOSTNAME in fi ;; esac -case ${targets[0]} in - kw|kww) - prospective_mps=(/a) - ;; -esac for mp in ${prospective_mps[@]}; do # default mountpoints to sync if [[ -e /nocow/btrfs-stale/$mp ]]; then @@ -268,6 +268,7 @@ fi if $dry_run; then m btrbk -v -n $cmd_arg + exit 0 elif [[ $cmd_arg == archive ]]; then if [[ $source ]]; then m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol diff --git a/check-subvol-stale b/check-subvol-stale index 085025a..0049ff0 100644 --- a/check-subvol-stale +++ b/check-subvol-stale @@ -110,10 +110,12 @@ for d; do fi # get info on last received sub + last_received= last_received_cgen=0 for f in ${snaps[@]}; do show="$(btrfs sub show $f)" if echo "$show" | grep -E "Received UUID:\s+[[:alnum:]]" &>/dev/null; then + d found received uuid in $f cgen=$(echo "$show" | sed -rn 's,^\s*Gen at creation:\s+([0-9]+).*,\1,p') if [[ $cgen -gt $last_received_cgen ]]; then last_received_cgen=$cgen @@ -121,8 +123,8 @@ for d; do fi fi done - d last_received_cgen=$cgen - d last_received=$f + d last_received_cgen=$last_received_cgen + d last_received=$last_received # Get last_snap by date. # when a btrbk bugfix makes it into the distro, @@ -139,6 +141,7 @@ for d; do echo "$0: error: could not find latest snapshot for $svp among ${snaps[*]}" >&2 exit 1 fi + d last_snap=$last_snap if [[ ! -e $svp ]]; then echo "$0: warning: subvol does not exist: $svp" @@ -170,7 +173,7 @@ for d; do stale=true # fresh if $svp has $last_snap as a snapshot, if btrfs sub show $svp 2>/dev/null | sed '0,/^\s*Snapshot(s):/d;s/^\s*//' | \ - grep -xF btrbk/$last_snap &>/dev/null; then + grep -xF ${last_snap#$root_dir/} ; then stale=false else # or else $svp is a snapshot of $last_snap. we use a uuid # comparison, which if I remember from the docs, is a bit more diff --git a/distro-end b/distro-end index 41b9104..d51cbef 100755 --- a/distro-end +++ b/distro-end @@ -73,7 +73,6 @@ case $distro in ;; esac - # disable motd junk. case $distro in debian) @@ -119,6 +118,9 @@ if isdeb; then # docker eats up a fair amount of cpu when doing nothing, so don't enable it unless # we really need it. pi-nostart docker-ce + # and docker is even more crap, it ignores that it shouldnt start + ser stop docker + ser disable docker case $HOSTNAME in li|lj) sgo docker ;; esac @@ -171,6 +173,8 @@ case $HOSTNAME in ;; esac +pi ${p1[@]} + # website setup case $HOSTNAME in lj|li) @@ -568,8 +572,13 @@ esac ########### end section including li/lj ############### -pi ${p4[@]} $(apt-cache search ruby[.0-9]+-doc| awk '{print $1}') +# depends gcc is a way to install suggests. this is apparently the only +# way to install suggests even if the main package is already +# installed. reinstall doesn't work, uninstalling can cause removing +# dependent packages. +pi ${pall[@]} $(apt-cache search ruby[.0-9]+-doc| awk '{print $1}') $(apt-cache depends gcc|grep -i suggests:| awk '{print $2}') +sgo fsf-vpn-dns-cleanup case $distro in trisquel|ubuntu) @@ -707,10 +716,9 @@ fi case $distro in debian|trisquel|ubuntu) - # suggests because we want the resolvconf package. however, i install it earlier - # as well, so this is redundant. + # suggests resolvconf package. installing it here is redundant, but make sure anyways. # todo: check other distros to make sure it\'s installed - pi-nostart --install-suggests openvpn + pi-nostart openvpn resolvconf # pi-nostart does not disable ser disable openvpn ;; @@ -903,6 +911,7 @@ esac case $(debian-codename) in # needed for debootstrap scripts for fai since fai requires debian flidas) + curl http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg | s apt-key add - s dd of=/etc/apt/preferences.d/flidas-xenial </dev/null </dev/null <&2' ERR + +if ! echo | /a/exe/cedit fsf /etc/dnsmasq-servers.conf; then + if systemctl is-active dnsmasq >/dev/null; then + systemctl reload dnsmasq + fi +fi diff --git a/install-my-scripts b/install-my-scripts index 2f9e224..3025d84 100755 --- a/install-my-scripts +++ b/install-my-scripts @@ -31,4 +31,4 @@ x="$(readlink -f "$BASH_SOURCE")"; cd ${x%/*} e() { echo "$*"; "$@"; } # scripts that would interfere with unmounting /a, put them elsewhere -e install btrbk-run mount-latest-subvol check-subvol-stale /usr/local/bin +e install /a/bin/log-quiet/log-once btrbk-run mount-latest-subvol check-subvol-stale /usr/local/bin diff --git a/mail-setup b/mail-setup index 2f4a726..5730788 100755 --- a/mail-setup +++ b/mail-setup @@ -245,7 +245,7 @@ if postfix; then if command -v apt-get &> /dev/null; then debconf-set-selections < /etc/mailname + # MAIN_HARDCODE_PRIMARY_HOSTNAME might mess up the # smarthost config type, not sure. all other settings # would be unused in that config type. @@ -700,6 +703,19 @@ ssl_prefer_server_ciphers = yes EOF ####### end dovecot setup ######## + # https://selivan.github.io/2017/12/30/systemd-serice-always-restart.html + d=/etc/systemd/system/openvpn@mail + mkdir -p $d + cat >$d/override.conf <<'EOF' +[Service] +Restart=always +# time to sleep before restarting a service +RestartSec=1 + +[Unit] +# StartLimitIntervalSec in recent systemd versions +StartLimitInterval=0 +EOF systemctl enable offlineimapsync.timer systemctl start offlineimapsync.timer @@ -723,10 +739,13 @@ EOF debconf-set-selections < /etc/mailname fi # end $HOSTNAME != $MAIL_HOST diff --git a/mount-latest-subvol b/mount-latest-subvol index e1d4d1a..033d9b8 100644 --- a/mount-latest-subvol +++ b/mount-latest-subvol @@ -221,7 +221,7 @@ for vol in q a o i; do # if latest is already mounted, make sure binds are mounted and move on e check-subvol-stale $d # populated by check-subvol-stale if stale - if ! fresh_snap=$(/dev/null); then mnt $d for b in ${binds[@]}; do mnt $b diff --git a/offlineimap-sync b/offlineimap-sync index 2b71a69..4fac247 100755 --- a/offlineimap-sync +++ b/offlineimap-sync @@ -2,7 +2,6 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR - # mail-route can get messed up a bit randomly, I don't know why. #/b/ds/mail-route up | /b/log-quiet/log-once -1 mail-route @@ -10,16 +9,16 @@ offlineimap -u quiet shopt -s nullglob omv() { # offlineimap mv. move mail files within $src_base/$1 to /m/md/$2 - src="$1" - dst="$2" - found_files=false - for x in new cur; do - files=($src_base/"$src"/$x/*) - if [[ $files ]]; then - found_files=true - mv "${files[@]}" /m/md/"$dst"/$x - fi - done + src="$1" + dst="$2" + found_files=false + for x in new cur; do + files=($src_base/"$src"/$x/*) + if [[ $files ]]; then + found_files=true + mv "${files[@]}" /m/md/"$dst"/$x + fi + done } src_base=/m/offlineimap @@ -27,28 +26,47 @@ omv "Sent Items" "Sent" omv INBOX offlineimaptmp src_base=/m/md if $found_files; then - sieve-filter -eW ~/sieve/main.sieve offlineimaptmp &>/dev/null - # the default folder is INBOX for anything leftover - omv offlineimaptmp INBOX - # remove messages from remote host - offlineimap -u quiet - # this makes us sit and wait when we want to use mu and this is running in a cronjob. - # todo: emacs updates the index much faster. what command is it running? I'd like - # to just run that - # looks like it might be mu index --lazy-check, but that still takes like 10 seconds, - # figure out if that is the same speed, or if we can make it faster. - #mu index &>/dev/null ||: + sieve-filter -eW ~/sieve/main.sieve offlineimaptmp &>/dev/null + # the default folder is INBOX for anything leftover + omv offlineimaptmp INBOX + # remove messages from remote host + offlineimap -u quiet + # this makes us sit and wait when we want to use mu and this is running in a cronjob. + # todo: emacs updates the index much faster. what command is it running? I'd like + # to just run that + # looks like it might be mu index --lazy-check, but that still takes like 10 seconds, + # figure out if that is the same speed, or if we can make it faster. + #mu index &>/dev/null ||: fi +# find but ignore directories which dont exist, assuming first args are directories +# and a following arg starts with - +myfind() { + dirs=() + for d; do + if [[ $d == -* ]]; then + # past dirs, onto options + break + fi + shift + if [[ -e $d ]]; then + dirs+=($d) + fi + done + if [[ $dirs ]]; then + find ${dirs[@]} $@ + fi +} # qemu-devel is our biggest list by far, so occasionally # I want to hop into conversations about our mailing # systems there, but I don't need many old messages. -find /nocow/user/fsfmd/l/qemu-devel/new -type f -mtime +14 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/l/qemu-devel/new -type f -mtime +14 -execdir rm -- '{}' + + -find /nocow/user/fsfmd/{sec,Spam,Drafts,{rtcc,sysadmin,l/outreachy-mentors}/new} -type f -mtime +100 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/{sec,Spam,Drafts,{rtcc,sysadmin,l/outreachy-mentors}/new} -type f -mtime +100 -execdir rm -- '{}' + -find /nocow/user/fsfmd/log -type f -mtime +300 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/log -type f -mtime +300 -execdir rm -- '{}' + # delete based on http://deflexion.com/2006/05/imap-way-of-deleting-message diff --git a/pkgs b/pkgs index 56d48c6..d3b0744 100644 --- a/pkgs +++ b/pkgs @@ -4,41 +4,41 @@ # packages with the same name across distros. p1=( - trash-cli - cryptsetup - lvm2 - mbuffer - screen + trash-cli + cryptsetup + lvm2 + mbuffer + screen ) p2=( - # 2nd - ghc - konsole - xmonad - cabal-install - suckless-tools + # 2nd + ghc + konsole + xmonad + cabal-install + suckless-tools ) p3=( - at - bash-completion - curl - ethtool - eatmydata - fping - git - htop - iptables - mailutils - nmon - ntp - rdiff-backup - ruby - ruby-rest-client - traceroute - tree - vim - wcd - wget + at + bash-completion + curl + ethtool + eatmydata + fping + git + htop + iptables + mailutils + nmon + ntp + rdiff-backup + ruby + ruby-rest-client + traceroute + tree + vim + wcd + wget ) # swh-plugins is for karaoke pulsaudio filter. @@ -50,137 +50,140 @@ p3=( # pv is used by btrbk # libterm-readkey-perl for rt cli tool # fonts pkgs are an attempt to get less block characters +# gnat-5 & ccache is for coreboot p4=( - adb - apache2 - apache2-doc - apt-doc - apt-listchanges - apt-rdepends - aptitude-doc-en - asciidoc - backupninja - bash-doc - beets - beets-doc - binutils-doc - bind9-doc - bind9utils - bwm-ng - cloc - cpulimit - cron - debian-archive-keyring - debootstrap - debconf-doc - dirmngr - dnsutils - dnsmasq - dtrx - duplicity - eclipse - elinks - evince - fdupes - feh - filelight - flashrom - fonts-noto - fonts-roboto - fonts-thai-tlwg - fonts-thai-tlwg-otf - xfonts-intl-asian - fonts-sil-lateef - gawk-doc - gcc-doc - gdb - gdb-doc - geoip-bin - git-doc - git-email - git-svn - gitk - glibc-doc - goaccess - gnome-screenshot - gnupg-doc - guvcview - i3lock - inotify-tools - iputils-tracepath - iperf3 - iproute2-doc - jq - kid3-qt - kid3-cli - konsole - libterm-readkey-perl - libreoffice - linphone - linux-doc - lshw - make-doc - manpages - manpages-dev - markdown - mb2md - meld - moreutils - mps-youtube - mpv - mumble - nagstamon - needrestart - ncdu - nginx-doc - nmap - offlineimap - oathtool - opendkim-tools - p7zip - paprefs - parted-doc - pass - pavucontrol - pdfgrep - perl-doc - pianobar - pidgin - pidgin-otr - pip - pry - pv - python-autopep8 - python3-doc - qrencode - readline-doc - reportbug - rng-tools - sakura - schroot - sig2dot - sipcalc - sqlite3-doc - squashfs-tools - strace - swh-plugins - tar-doc - tcpdump - telnet - transmission-remote-gtk - vlc - w3m - whois - wondershaper - xawtv - xbacklight - xdot - xloadimage - xprintidle - xscreensaver - xscreensaver-data-extra - xscreensaver-gl - xscreensaver-gl-extra + adb + apache2 + apache2-doc + apt-doc + apt-listchanges + apt-rdepends + aptitude-doc-en + asciidoc + backupninja + bash-doc + beets + beets-doc + binutils-doc + bind9-doc + bind9utils + bwm-ng + ccache + cloc + cpulimit + cron + debian-archive-keyring + debootstrap + debconf-doc + dirmngr + dnsutils + dnsmasq + dtrx + duplicity + eclipse + elinks + evince + fdupes + feh + filelight + flashrom + fonts-noto + fonts-roboto + fonts-thai-tlwg + fonts-thai-tlwg-otf + xfonts-intl-asian + fonts-sil-lateef + gawk-doc + gcc-doc + gdb + gdb-doc + geoip-bin + git-doc + git-email + git-svn + gitk + glibc-doc + goaccess + gnat-5 + gnome-screenshot + gnupg-doc + guvcview + hunspell + i3lock + inotify-tools + iputils-tracepath + iperf3 + iproute2-doc + jq + kid3-qt + kid3-cli + konsole + libterm-readkey-perl + libreoffice + linphone + linux-doc + lshw + make-doc + manpages + manpages-dev + markdown + mb2md + meld + moreutils + mps-youtube + mpv + mumble + nagstamon + ncdu + nginx-doc + nmap + offlineimap + oathtool + opendkim-tools + p7zip + paprefs + parted-doc + pass + pavucontrol + pdfgrep + perl-doc + pianobar + pidgin + pidgin-otr + pry + pv + python-autopep8 + python-pip + python3-doc + qrencode + readline-doc + reportbug + rng-tools + sakura + schroot + sig2dot + sipcalc + sqlite3-doc + squashfs-tools + strace + swh-plugins + tar-doc + tcpdump + telnet + transmission-remote-gtk + vlc + w3m + whois + wondershaper + xawtv + xbacklight + xdot + xloadimage + xprintidle + xscreensaver + xscreensaver-data-extra + xscreensaver-gl + xscreensaver-gl-extra ) pall=(${p1[@]} ${p2[@]} ${p3[@]} ${p4[@]} ) diff --git a/switch-mail-host b/switch-mail-host index 9ebd2a7..a06e1ba 100755 --- a/switch-mail-host +++ b/switch-mail-host @@ -4,7 +4,7 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR usage() { - cat <