From 8793e42101bdb66e63e76deebe3958e9a1ef7c87 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Mon, 18 Dec 2017 15:11:47 -0500 Subject: [PATCH] lots of fixes and improvements --- btrbk-run | 5 +- btrfsmaint | 85 +++++++++++++++++++++++++++++ btrfsmaintstop | 1 + conflink | 4 ++ desktop-apps | 2 +- distro-begin | 2 +- distro-end | 126 ++++++++++++++++++++++++++++++++----------- dynamic-ip-update.sh | 18 ++----- mail-setup | 8 +++ mount-latest-subvol | 37 +++++++++---- mymimes | 21 ++++++++ pump-backup | 2 +- schrootupdate | 13 +++++ switch-mail-host | 2 +- 14 files changed, 265 insertions(+), 61 deletions(-) create mode 100755 btrfsmaint create mode 100755 btrfsmaintstop create mode 100755 mymimes create mode 100755 schrootupdate diff --git a/btrbk-run b/btrbk-run index f45fe36..5ff8326 100755 --- a/btrbk-run +++ b/btrbk-run @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. + +# todo: remove old leaf subvols, like keep up to 1 month or something. + set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR @@ -78,7 +81,7 @@ echo -e "$0: options: conf_only=$conf_only\ndry_run=$dry_run\nresume_arg=$resume # set default targets if [[ ! -v targets ]]; then case $HOSTNAME in - x2) + x2|fz) if [[ $HOSTNAME == "$MAIL_HOST" ]]; then targets=($HOME_DOMAIN) fi diff --git a/btrfsmaint b/btrfsmaint new file mode 100755 index 0000000..4bb2246 --- /dev/null +++ b/btrfsmaint @@ -0,0 +1,85 @@ +#!/bin/bash +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +# inspired from +# https://github.com/kdave/btrfsmaintenance + + +# Man page says we could also use a range, i suppose it would be +# logical to use a pattern like 5..10 10..20, +# but I don't know if this would help us at all. +dusage="1 5 10 20 30 40 50" +musage="1 5 10 20 30" + +e() { echo "cron: $*"; "$@"; } + +check-idle() { + export DISPLAY=:0 + idle_time=$(xprintidle 2>/dev/null) ||: + # 3 hours, assume a movie might run that long. + if [[ $idle_time ]] && (( idle_time < 1000 * 60 * 60 * 3 )); then + idle=false + else + idle=true + fi +} + +if [[ $1 == check ]]; then + check=true +else + check=false +fi + +check-idle + +fnd="findmnt --types btrfs --noheading" +for x in $($fnd --output "SOURCE" --nofsroot | sort -u); do + mnt=$($fnd --output "TARGET" --first-only --source $x) + [[ $mnt ]] || continue + + if ! $idle; then + btrfs scrub cancel $mnt &>/dev/null ||: + continue + fi + if $check; then + continue + fi + + # for comparing before and after balance. + # the log is already fairly verbose, so commented. + # e btrfs filesystem df $mnt + # e df -H $mnt + if btrfs filesystem df $mnt | grep -q "Data+Metadata"; then + for usage in $dusage; do + e btrfs balance start -dusage=$usage -musage=$usage $mnt + done + else + e btrfs balance start -dusage=0 $mnt + for usage in $dusage; do + e btrfs balance start -dusage=$usage $mnt + done + e btrfs balance start -musage=0 $mnt + for usage in $musage; do + e btrfs balance start -musage=$usage $mnt + done + fi + # e btrfs filesystem df $mnt + # e df -H $mnt + date=$( + btrfs scrub status $mnt | \ + sed -rn 's/^\s*scrub started at (.*) and finished.*/\1/p' + ) + if [[ $date ]]; then + date=$(date --date="$date" +%s) + # if date is sooner than 90 days ago + # the wiki recommends 30 days or so, but + # it makes the comp lag like shit for a day, + # so I'm going with 90 days. + if (( $date > `date +%s` - 60*60*24*30 )); then + echo "cron: skiping scrub of $mnt" + continue + fi + fi + e btrfs scrub start -Bd $mnt +done diff --git a/btrfsmaintstop b/btrfsmaintstop new file mode 100755 index 0000000..a9bf588 --- /dev/null +++ b/btrfsmaintstop @@ -0,0 +1 @@ +#!/bin/bash diff --git a/conflink b/conflink index 1bc997b..38aacf0 100755 --- a/conflink +++ b/conflink @@ -83,6 +83,10 @@ case $USER in # named[20823]: /etc/bind/db.iank.pw.jnl: create: permission denied m s chgrp bind /etc/bind/bind-writable fi + files=(/etc/bind/[Kk]*) + if [[ -e $files ]]; then + m s chgrp bind ${files[@]} + fi if [[ -e /etc/davpass ]] && getent group www-data &>/dev/null; then s chgrp www-data /etc/davpass fi diff --git a/desktop-apps b/desktop-apps index ae275f2..a0ed3c7 100755 --- a/desktop-apps +++ b/desktop-apps @@ -8,4 +8,4 @@ linphone & if ! pgrep -u $EUID -f "firefox -P sfw"; then firefox -P sfw & fi -nagstamon & +#nagstamon & diff --git a/distro-begin b/distro-begin index 4ea0842..ff3c1ba 100755 --- a/distro-begin +++ b/distro-begin @@ -65,7 +65,7 @@ if [[ $1 ]]; then export HOSTNAME=$1 fi -for f in iank-dev htpc treetowl x2 frodo tp li lj demohost kw; do +for f in iank-dev htpc treetowl x2 frodo tp li lj demohost kw fz; do eval "$f() { [[ $HOSTNAME == $f ]]; }" done has_p() { ! linode; } # when tp is tracis, then not tp either diff --git a/distro-end b/distro-end index 76d37ea..0c80a1d 100755 --- a/distro-end +++ b/distro-end @@ -166,6 +166,9 @@ case $HOSTNAME in xbacklight xprintidle xscreensaver + xscreensaver-data-extra + xscreensaver-gl + xscreensaver-gl-extra ) spa $(apt-cache search ruby[.0-9]+-doc| awk '{print $1}') ;; @@ -305,6 +308,7 @@ fi ### end docker install #### +### begin certbot install ### case $distro in debian) # note, need python-certbot-nginx for nginx, but it depends on nginx, @@ -315,27 +319,37 @@ case $distro in else pi certbot python-certbot-apache fi - # make a version of the certbot timer that emails me. - x=/systemd/system/certbot - $sed -r -f - /lib$x.timer <<'EOF' |s dd of=/etc${x}mail.timer -s,^Description.*,\0 mail version, -EOF - $sed -r -f - /lib$x.service <<'EOF' |s dd of=/etc${x}mail.service -s,(ExecStart=)(/usr/bin/certbot),\1/a/bin/log-quiet/sysd-mail-once certbotmail \2 --renew-hook /a/bin/distro-setup/certbot-renew-hook, -EOF - ser daemon-reload - sgo certbotmail.timer - ;; trisquel|ubuntu) # not packaged in xenial or flidas pi software-properties-common - s add-apt-repository -y ppa:certbot/certbot + # this fails with: + # + # gpg: key 75BCA694: public key "Launchpad PPA for certbot" imported + # gpg: Total number processed: 1 + # gpg: imported: 1 + # gpg: no valid OpenPGP data found. + # Failed to add key. + # + # but it seems to work fine, perhaps it's only failing on the second run. + s add-apt-repository -y ppa:certbot/certbot ||: p update pi python-certbot-apache ;; # todo: other distros unknown esac +# make a version of the certbot timer that emails me. +x=/systemd/system/certbot +$sed -r -f - /lib$x.timer <<'EOF' |s dd of=/etc${x}mail.timer +s,^Description.*,\0 mail version, +EOF +$sed -r -f - /lib$x.service <<'EOF' |s dd of=/etc${x}mail.service +s,(ExecStart=)(/usr/bin/certbot),\1/a/bin/log-quiet/sysd-mail-once certbotmail \2 --renew-hook /a/bin/distro-setup/certbot-renew-hook, +EOF +ser daemon-reload +sgo certbotmail.timer +### end certbot install ### + # dogcam setup case $HOSTNAME in @@ -670,6 +684,13 @@ EOF esac +# needed for checkrestart +if isdeb; then + spa debian-goodies +fi + + + ########### end section including li/lj ############### case $distro in @@ -686,10 +707,6 @@ case $distro in esac -if isdeb; then - spa debian-goodies -fi - case $distro in debian) @@ -720,8 +737,6 @@ case $distro in esac - - ### begin home vpn server setup @@ -733,22 +748,23 @@ esac # vpn-server-setup -rds # s cp -r --parents /etc/openvpn/easy-rsa/keys /p/c/filesystem # s chown -R 1000:1000 /p/c/filesystem/etc/openvpn/easy-rsa/keys +# # kw = kgpe work machine. # for host in x2 kw; do -# vpn-mk-client-cert -b $host -n home b8.nz -# dir=/p/c/machine_specific/$host/filesystem/etc/openvpn/client -# mkdir -p $dir -# s bash -c "cp /etc/openvpn/client/home* $dir" +# vpn-mk-client-cert -b $host -n home b8.nz 1196 +# dir=/p/c/machine_specific/$host/filesystem/etc/openvpn/client +# mkdir -p $dir +# s bash -c "cp /etc/openvpn/client/home* $dir" # # note: /etc/update-resolv-conf-home also exists for all systems with /p # done - +# key already exists, so this won't generate one, just the configs. vpn-server-setup -rds s tee -a /etc/openvpn/server/server.conf <<'EOF' push "dhcp-option DNS 192.168.1.1" push "route 192.168.1.0 255.255.255.0" client-connect /a/bin/distro-setup/vpn-client-connect EOF -s sed -i --follow-symlinks 's/10.8./10.9./g' /etc/openvpn/server/server.conf +s sed -i --follow-symlinks 's/10.8./10.9./g;s/^\s*port\s.*/port 1196/' /etc/openvpn/server/server.conf if [[ $HOSTNAME == tp ]]; then if [[ -e /lib/systemd/system/openvpn-server@.service ]]; then @@ -936,9 +952,12 @@ esac ####### begin misc packages ########### +if [[ ! -e ~/.linphonerc && -e /p/.linphonerc-initial ]]; then + cp /p/.linphonerc-initial ~/.linphonerc +fi -### begin spd install +### begin spd install pi libswitch-perl libdigest-md5-file-perl libgnupg-interface-perl t=$(mktemp) wget -O $t http://mirror.fsf.org/fsfsys-trisquel/fsfsys-trisquel/pool/main/s/spd-perl/spd-perl_0.2-1_amd64.deb @@ -946,11 +965,11 @@ s dpkg -i $t rm $t # this guesses at the appropriate directory, adjust if needed x=(/usr/lib/x86_64-linux-gnu/perl/5.*) -sudo ln -s ../../../perl/5.18.2/SPD/ $x +sudo ln -sf ../../../perl/5.18.2/SPD/ $x # newer distro had gpg2 as default, older one, flidas, need to make it that way x=$(which gpg2) if [[ $x ]]; then - lnf -T $x /usr/local/bin/gpg + s lnf -T $x /usr/local/bin/gpg fi ### end spd install @@ -975,7 +994,23 @@ fi # services on unreachable osts # hosts in soft state # services in soft state -# in display tab: icon in systray. +# in display tab: fullscreen + +# these translate to these settings I think +# filter_acknowledged_hosts_services = True +# filter_all_unknown_services = True +# filter_all_warning_services = True +# filter_hosts_in_soft_state = True +# filter_hosts_services_maintenance = True +# filter_services_in_soft_state = True +# filter_services_on_down_hosts = True +# filter_services_on_hosts_in_maintenance = True +# filter_services_on_unreachable_hosts = True +# notify_if_up = False +# statusbar_floating = False +# fullscreen = True +# but i'm just going to rely on the webpage plus sms for now. + case $distro in debian|trisquel|ubuntu) @@ -997,6 +1032,7 @@ tu /etc/schroot/desktop/fstab <<'EOF' /dev/shm /dev/shm none rw,bind 0 0 /run/shm /run/shm none rw,bind 0 0 /run/user/1000 /run/user/1000 none rw,bind 0 0 +/run/user/1001 /run/user/1001 none rw,bind 0 0 /run/user/0 /run/user/0 none rw,bind 0 0 EOF @@ -1024,8 +1060,31 @@ EOF cd; s schroot -c $n -- apt-get install --allow-unauthenticated -y ${apps[@]} fi s cp -P {,$d}/etc/localtime - } +s dd of=/etc/systemd/system/schrootupdate.service <<'EOF' +[Unit] +Description=schrootupdate +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/a/bin/log-quiet/sysd-mail-once schrootupdate /a/bin/distro-setup/schrootupdate +EOF +s dd of=/etc/systemd/system/schrootupdate.timer <<'EOF' +[Unit] +Description=schrootupdate + +[Timer] +OnCalendar=*-*-* 04:20:00 + +[Install] +WantedBy=timers.target +EOF +s systemctl daemon-reload +sgo schrootupdate.timer + + + # for my roommate case $distro in @@ -1610,6 +1669,7 @@ DEVICESCAN -a -o on -S on -n standby,q $sched \ ########### misc stuff +/a/bin/distro-setup/mymimes # stop autopoping windows when i plug in an android phone. @@ -1618,17 +1678,21 @@ DEVICESCAN -a -o on -S on -n standby,q $sched \ # dconf-WARNING **: failed to commit changes to dconf: Cannot autolaunch D-Bus without X11 $DISPLAY dbus-launch gsettings set org.gnome.desktop.media-handling automount-open false + +# on grub upgrade, we get prompts unless we do this devs=() for dev in $(s btrfs fi show /boot | sed -nr 's#.*path\s+(\S+)$#\1#p'); do devs+=($(devbyid $dev),) done devs[-1]=${devs[-1]%,} # jonied by commas - -# on grub upgrade, we get prompts unless we do this s debconf-set-selections <&2' ERR -cur="$(host -4 iank.life iankelling.org | sed -rn 's/.*has address (.*)/\1/p;T;q')" +cur="$(host -4 b8.nz iankelling.org | sed -rn 's/.*has address (.*)/\1/p;T;q')" ip=$(curl -s4 https://iankelling.org/cgi/pubip) # note, a simpler way to do this would be to ssh and use @@ -12,17 +12,6 @@ ip=$(curl -s4 https://iankelling.org/cgi/pubip) # to update bind if needed. if [[ $cur != $ip ]]; then - nsupdate -k /p/c/machine_specific/li/filesystem/etc/bind/Kiank.life.*.private <$f </dev/null; then if [[ $HOSTNAME == $MAIL_HOST ]]; then systemctl restart radicale systemctl enable radicale + if [[ -e /etc/logrotate.d/radicale.disabled ]]; then + mv /etc/logrotate.d/radicale{.disabled,} + fi else systemctl stop radicale systemctl disable radicale + # weekly logrotate tries to restart radicale even if it's a disabled service in flidas. + if [[ -e /etc/logrotate.d/radicale ]]; then + mv /etc/logrotate.d/radicale{,.disabled} + fi fi fi +exit 0 # if I wanted the from address to be renamed and sent to a different address, # echo "sdx@localhost development@localhost" | sudo dd of=/etc/postfix/recipient_canonical diff --git a/mount-latest-subvol b/mount-latest-subvol index d39b882..be239ed 100644 --- a/mount-latest-subvol +++ b/mount-latest-subvol @@ -81,6 +81,31 @@ fstab() { tu /etc/fstab <<<"$l" done } +kill-dir() { + found_pids=false + sig=${1:-TERM} + if pids=$(timeout 4 lsof -t $dir); then + found_pids=true + timeout 4 lsof -w $dir + kill -$sig $pids + fi + # fuser will find open sockets that lsof won't, for example from gpg-agent. + # note: -v shows kernel processes, which then doesn't return true when we want + if timeout 4 fuser -m $dir &>/dev/null; then + found_pids=true + fuser -$sig -mvk $dir + fi + if $found_pids; then + sleep .5 + return 0 + fi + return 1 +} + +force=false +if [[ $1 == -f ]]; then + force=true +fi ret=0 @@ -163,18 +188,8 @@ for vol in q a o i; do if e umount -R $dir; then unmounted+=($dir) else - if pids=$(timeout 4 lsof -t $dir); then - timeout 4 lsof -w $dir - kill $pids - fi - - # fuser will find open sockets that lsof won't, for example from gpg-agent. - # note: -v shows kernel processes, which then doesn't return true when we want - if timeout 4 fuser -m $dir &>/dev/null; then - fuser -TERM -mvk $dir - fi + kill-dir || kill-dir INT || kill-dir HUP || ! $force || kill-dir KILL ||: - sleep .5 if e umount -R $dir; then unmounted+=($dir) else diff --git a/mymimes b/mymimes new file mode 100755 index 0000000..010d418 --- /dev/null +++ b/mymimes @@ -0,0 +1,21 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" + +shopt -s extglob nullglob + +echo '[Default Applications]' >/etc/xdg/defaults.list +sed -r '/^MimeType=/!d;s/^MimeType=//;s/;/=mpv.desktop;\n/g' /usr/share/applications/mpv.desktop >>/etc/xdg/defaults.list + +# this is the new one, according to +# https://specifications.freedesktop.org/mime-apps-spec/mime-apps-spec-1.0.html +# https://wiki.debian.org/MIME +# https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +cp /etc/xdg/defaults.list /etc/xdg/mimapps.list + +for f in /usr/share/applications/!(defaults).list; do + cp /etc/xdg/defaults.list /etc/xdg/${f##*/} +done diff --git a/pump-backup b/pump-backup index 65a346a..0a26297 100755 --- a/pump-backup +++ b/pump-backup @@ -1,3 +1,3 @@ #!/bin/bash ssh li mongodump >/dev/null # too verbose by default -rsync -r --delete root@li:{/home/ian/dump,/home/pumpio/pumpdata} /w/backup/pump +rsync -r --delete root@li:{/home/iank/dump,/home/pumpio/pumpdata} /w/backup/pump diff --git a/schrootupdate b/schrootupdate new file mode 100755 index 0000000..280095d --- /dev/null +++ b/schrootupdate @@ -0,0 +1,13 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" + +for n in stretch jessie; do + if [[ -e /etc/schroot/chroot.d/$n.conf ]]; then + cd + schroot -c $n -- apt-get -y dist-upgrade --purge --auto-remove + fi +done diff --git a/switch-mail-host b/switch-mail-host index bac2148..96e03ef 100755 --- a/switch-mail-host +++ b/switch-mail-host @@ -197,7 +197,7 @@ target send-receive $btrbk_dst EOF -sudo btrbk run +sudo btrbk -l debug --progress run $new_shell mount-latest-subvol mail-setup "$new_shell" -- 2.30.2