From ebd0ea323de3eacc11e1a624f8cc4f961d8bdb6c Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Tue, 8 Aug 2017 17:27:07 -0700 Subject: [PATCH] various fixes --- btrbk-run | 149 +++++++++++++++++++++--------------------- check-subvol-stale | 33 ++++++++-- desktop-apps | 11 ++++ distro-begin | 1 + distro-end | 8 ++- dynamic-ip-update.sh | 39 +++++------ fsf-get-mail | 3 + input-setup | 46 +++++++++++-- mail-setup | 14 +++- music-tag-sync | 46 +++++++++++++ myunison | 4 +- offlineimap-sync | 9 ++- radicale-setup | 49 +++++++++++--- rew | 13 ++++ switch-mail-host | 150 +++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 456 insertions(+), 119 deletions(-) create mode 100755 desktop-apps create mode 100755 fsf-get-mail create mode 100644 music-tag-sync create mode 100755 rew create mode 100755 switch-mail-host diff --git a/btrbk-run b/btrbk-run index 7a1d946..eb8a7b5 100755 --- a/btrbk-run +++ b/btrbk-run @@ -17,6 +17,7 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR [[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" + usage() { echo "top of script file:" sed -n '1,/^[# ]*end command line/{p;b};q' "$0" @@ -27,33 +28,32 @@ script_dir=$(dirname $(readlink -f "$BASH_SOURCE")) # note q is owned by root:1000 # note p/m is owned 1000:1000 and chmod 700 -mountpoints=(/a) -private_mountpoints=(/q) + + +targets=() +mountpoints=() + rsync_mountpoint=/q conf_only=false dry_run=false # mostly for testing resume_arg= -do_i=true -if [[ $HOSTNAME == $MAIL_HOST ]]; then - do_o=true -else - do_o=false -fi default_args_file=/etc/btrbk-run.conf -if [[ -r $default_args_file ]]; then +if [[ -s $default_args_file ]]; then + echo "$0: warning: options file default options set:" + cat $default_args_file + sleep 5 set -- $(< $default_args_file) "$@" fi -temp=$(getopt -l help hcinoprt: "$@") || usage 1 +temp=$(getopt -l help hcm:nprt: "$@") || usage 1 eval set -- "$temp" while true; do case $1 in -c) conf_only=true; shift ;; - -i) do_i=false; shift ;; + -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;; -n) dry_run=true; dry_run_arg=-n; shift ;; - -o) do_o=false; shift ;; -p) progress_arg="--progress"; shift ;; # btrbk arg: Resume only. Skips snapshot creation. -r) resume_arg=-r; shift ;; @@ -64,10 +64,54 @@ while true; do esac done -if $do_o; then - private_mountpoints+=(/o) +echo "$0: options: conf_only=$conf_only, dry_run=$dry_run, resume_arg=$resume_arg" + +# set default targets +if ! (( ${#targets[@]} )); then + case $HOSTNAME in + x2) + if [[ $HOSTNAME == "$MAIL_HOST" ]]; then + targets=($HOME_DOMAIN) + fi + ;; + treetowl) + targets=(frodo) + if [[ $HOSTNAME == "$MAIL_HOST" ]]; then + if timeout -s 9 10 ssh x2 :; then + targets+=(x2) + fi + fi + ;; + *) + echo "$0: error: no default targets for this host, use -t" + exit 1 + ;; + esac +fi + +echo "targets: ${targets[*]}" + + + +# set default mountpoints +if ! (( ${#mountpoints[@]} )); then + prospective_mps=(/a /q) + if [[ $HOSTNAME == "$MAIL_HOST" ]]; then + prospective_mps+=(/o) + fi + for tg in ${targets[@]}; do + if [[ $tg == frodo && $HOSTNAME == treetowl ]]; then + prospective_mps+=(/i) + fi + done + for mp in ${prospective_mps[@]}; do # default mountpoints to sync + if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then + mountpoints+=($mp) + fi + done fi -read primary <<<"$@" # not yet used + +echo "mountpoints: ${mountpoints[*]}" ##### end command line parsing ######## @@ -132,82 +176,41 @@ target_preserve_min 4h # btrbk -l debug -v dryrun EOF -for mp in ${private_mountpoints[@]}; do # private mountpoints - if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then - mountpoints+=($mp) - fi -done # if our mountpoints are from stale snapshots, # it doesn't make sense to do a backup. check-subvol-stale ${mountpoints[@]} || exit 1 -if [[ ! $targets ]]; then - case $HOSTNAME in - tp|x2) - if ! timeout -s 9 10 ssh frodo :; then - targets=($HOME_DOMAIN) - fi - ;; - treetowl) - targets=(frodo) - if timeout -s 9 10 ssh x2 :; then - targets+=(x2) - fi - ;; - *) - targets=(frodo) - ;; - esac -fi - - -echo "targets: ${targets[*]}" - - -# for i, we just do a 1 way sync from master to backup, -# and manually manage any changes to that. -i_possible=false for tg in ${targets[@]}; do # for an initial run, btrbk requires the dir to exist ssh root@$tg mkdir -p /mnt/root/btrbk - if [[ $tg == frodo && $HOSTNAME == treetowl ]]; then - i_possible=true - fi done -if ! $i_possible; then - do_i=false -fi -vol=/mnt/root -vol-conf for m in ${mountpoints[@]}; do - sub=${m##*/} - sub-conf - for tg in ${targets[@]}; do + # for /i, some special cases. there is just one static target and direction. + if [[ $m == /i ]]; then + vol=/mnt/iroot + vol-conf + sub=i + sub-conf + tg=frodo + vol=/mnt/root tg-conf - done + else + vol=/mnt/root + vol-conf + sub=${m##*/} + sub-conf + for tg in ${targets[@]}; do + tg-conf + done + fi done -if $do_i; then - vol=/mnt/iroot - vol-conf - sub=i - sub-conf - tg=frodo - vol=/mnt/root - tg-conf -fi - - # todo: umount first to ensure we don't have any errors # todo: do some kill fuser stuff to make umount more reliable -# todo: run this on a systemd timer on $primary, once per hour, -# and if primary is, change that timer over to primary, and make -# sure we mount the latest - if $conf_only; then diff --git a/check-subvol-stale b/check-subvol-stale index 1277182..b977f76 100644 --- a/check-subvol-stale +++ b/check-subvol-stale @@ -58,7 +58,7 @@ for d; do ret=1 continue fi - stale=true + # check that $d has $last_snap as a snapshot, # or else $d is a snapshot of $last_snap. In the second # case, we use a uuid comparison, which if I remember from the @@ -67,17 +67,42 @@ for d; do grep -xF btrbk/$last_snap &>/dev/null; then stale=false else - last_uuid=$(btrfs sub show $last_snap| awk '$1 == "UUID:" {print $2}') - if btrfs sub show $d| grep "^\s*Parent UUID:\s*$last_uuid$" &>/dev/null; then + last_snap_uuid=$(btrfs sub show $last_snap| awk '$1 == "UUID:" {print $2}') + if btrfs sub show $d| grep "^\s*Parent UUID:\s*$last_snap_uuid$" &>/dev/null; then stale=false fi fi + + # check if $d is a snapshot of any of the btrbk backups + if [[ ! $stale ]]; then + for f in ${snaps[@]}; do + if [[ $f == $last_snap ]]; then continue; fi + uuid=$(btrfs sub show $f| awk '$1 == "UUID:" {print $2}') + if btrfs sub show $d| grep "^\s*Parent UUID:\s*$uuid$" &>/dev/null; then + stale=true + echo "$d stale: it's a snapshot of $f" + break + fi + done + fi + # check if $d generation is later than last_snap + if [[ ! $stale ]]; then + last_snap_gen=$(btrfs sub show $last_snap| awk '$1 == "Generation:" {print $2}') + d_gen=$(btrfs sub show $d| awk '$1 == "Generation:" {print $2}') + if (( last_snap_gen < d_gen )); then + stale=false + else + echo "$d stale: it's generation, $d_gen, is earlier than the last snapshot's, $last_snap_gen" + stale=true + fi + fi + + stale_dir=/nocow/btrfs-stale stale_file=$stale_dir/$vol if $stale; then mkdir -p $stale_dir printf "%s\n" $last_snap > $stale_file - echo "$d stale" ret=1 continue else diff --git a/desktop-apps b/desktop-apps new file mode 100755 index 0000000..ae275f2 --- /dev/null +++ b/desktop-apps @@ -0,0 +1,11 @@ +#!/bin/bash + +if ! pgrep -u $EUID emacs; then + emacs --daemon & +fi +pidgin & +linphone & +if ! pgrep -u $EUID -f "firefox -P sfw"; then + firefox -P sfw & +fi +nagstamon & diff --git a/distro-begin b/distro-begin index 2577879..1822afe 100755 --- a/distro-begin +++ b/distro-begin @@ -285,6 +285,7 @@ pi trash-cli s lnf -T /a/bin /b +s lnf -T /nocow/t /t if has_p; then lnf -T /p/News ~/News diff --git a/distro-end b/distro-end index 4fb10db..6625799 100755 --- a/distro-end +++ b/distro-end @@ -87,6 +87,7 @@ case $HOSTNAME in fdupes feh filelight + flashrom gawk-doc gcc-doc gdb @@ -118,6 +119,7 @@ case $HOSTNAME in nginx-doc nmap offlineimap + oathtool p7zip paprefs parted-doc @@ -130,6 +132,7 @@ case $HOSTNAME in python-autopep8 python3-doc python3-mutagen + qrencode reportbug $(aptitude show ruby | sed -rn 's/Depends: (.*)/\1/p')-doc sqlite3-doc @@ -592,6 +595,7 @@ EOF ############# end setup mastodon ############## + # we use nsupdate to update the ip of home pi bind9 echo "$0: $(date): ending now)" @@ -1671,7 +1675,7 @@ if [[ $HOSTNAME == treetowl ]]; then # path: k/music. Then copied the file # /p/c/subdir_files/.kodi/userdata/sources.xml to save that setting. s a2enmod dav dav_fs - web-conf -r /a/c/playlists - apache2 dav.iank.pw <<'EOF' + web-conf -r /a/c/playlists - apache2 dav.$HOME_DOMAIN <<'EOF' DAV On AuthType Basic @@ -1686,7 +1690,7 @@ if [[ $HOSTNAME == treetowl ]]; then EOF s mkdir -p /var/www/davlock s chown www-data:www-data /var/www/davlock - s sed -i "1i DavLockDB /var/www/davlock/davlock" /etc/apache2/sites-enabled/dav.iank.pw.conf + s sed -i "1i DavLockDB /var/www/davlock/davlock" /etc/apache2/sites-enabled/dav.$HOME_DOMAIN.conf ser reload apache2 teeu /etc/exports "/k/music *(ro,nohide,async,no_subtree_check,insecure)" diff --git a/dynamic-ip-update.sh b/dynamic-ip-update.sh index 923da11..42dab1f 100755 --- a/dynamic-ip-update.sh +++ b/dynamic-ip-update.sh @@ -4,7 +4,7 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR -cur="$(host -4 iank.pw iankelling.org | sed -rn 's/.*has address (.*)/\1/p;T;q')" +cur="$(host -4 iank.life iankelling.org | sed -rn 's/.*has address (.*)/\1/p;T;q')" ip=$(curl -s4 https://iankelling.org/cgi/pubip) # note, a simpler way to do this would be to ssh and use @@ -12,28 +12,29 @@ ip=$(curl -s4 https://iankelling.org/cgi/pubip) # to update bind if needed. if [[ $cur != $ip ]]; then - nsupdate -k /p/c/machine_specific/li/filesystem/etc/bind/Kiank.pw.*.private <$f <$f </tmp/fsfsieve.log diff --git a/input-setup b/input-setup index 91bede6..9cff413 100755 --- a/input-setup +++ b/input-setup @@ -1,4 +1,5 @@ #!/bin/bash -l +set -x # Copyright (C) 2016 Ian Kelling # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# set to oppsite if the order is flipped. +k2flip=falsesf- +if $k2flip; then +k2inorder=false +else + k2inorder=true + fi + case $HOSTNAME in x2|tp) type=laptop ;; treetowl*|iank-dev|frodo) type=kinesis ;; @@ -39,9 +48,15 @@ ms() { mi "$1" } set_device_id() { - if device_id=$(xinput --list | grep -m 1 "$1"); then + if [[ $2 ]] && $2; then + cmd="tail -n1" + else + cmd="head -n1" + fi + if device_id=$(xinput --list | grep "$1" | $cmd); then device_id=${device_id##*id=} device_id=${device_id%%[[:space:]]*} + echo "2:$2 device_id=$device_id" else return 1 fi @@ -68,15 +83,23 @@ case $type in xset r rate 200 13 # decrease rate delay cd / # so xbindkeys does not hold open mountpoints + killall xbindkeys # having some lag, thinking this might help. xbindkeys # configured to grab left and right scroll button presses fi - + kinesis2=false #right scroll wheel, change from button 4 & 5 to 13 and 14. # also changes the middle click to 12, even though I'm not using it anymore if set_device_id "04d9:048e"; then xinput --set-button-map "$device_id" 1 12 3 13 14 6 7 + else + kinesis2=true # if we are using the 2nd kinesis which has different device ids fi + if $kinesis2 && set_device_id "USB OPTICAL MOUSE" $k2flip; then + xinput --set-button-map "$device_id" 1 12 3 14 13 6 7 + fi + + ms 'Kensington Kensington Slimblade Trackball' 100 4000/1 7 6.5 1.5 xinput --set-button-map 'Kensington Kensington Slimblade Trackball' 0 0 0 4 5 6 7 0 9 10 11 12 @@ -98,12 +121,21 @@ case $type in fi - # disable the mouse movements of my mouse-wheel only mouse - if set_device_id "USB Optical Mouse"; then - xinput --set-prop "$device_id" 'Device Accel Constant Deceleration' 10000 - # 12 is to effectively disable the middle click button - xinput --set-button-map "$device_id" 1 12 3 10 11 6 7 + # disable the mouse movements mouse wheel + if $kinesis2; then + if set_device_id "USB OPTICAL MOUSE" $k2inorder; then + xinput --set-prop "$device_id" 'Device Accel Constant Deceleration' 10000 + # 12 is to effectively disable the middle click button + xinput --set-button-map "$device_id" 1 12 3 11 10 6 7 + fi + else + if set_device_id "USB Optical Mouse"; then + xinput --set-prop "$device_id" 'Device Accel Constant Deceleration' 10000 + # 12 is to effectively disable the middle click button + xinput --set-button-map "$device_id" 1 12 3 10 11 6 7 + fi fi + . /a/bin/bash_unpublished/duplicity-gpg-agent-setup ;; esac diff --git a/mail-setup b/mail-setup index 33ad75a..09c33d7 100755 --- a/mail-setup +++ b/mail-setup @@ -622,7 +622,7 @@ EOF # sieve option, we wouldn\'t need this, but I\'d rather not modify a # default config if not needed. This won\'t work as a symlink in /a/c # unfortunately. - sudo -u $postmaster /a/exe/lnf -T sieve/main.sieve ~$postmaster/.dovecot.sieve + sudo -u $postmaster /a/exe/lnf -T sieve/main.sieve $(eval echo ~$postmaster)/.dovecot.sieve sed -ri -f - /etc/dovecot/conf.d/10-mail.conf <<'EOF' 1i mail_location = maildir:/m/md:LAYOUT=fs:INBOX=/m/md/INBOX @@ -817,6 +817,18 @@ fi systemctl restart $type systemctl enable $type +# MAIL_HOST also does radicale, and easier to start and stop it here +# for when MAIL_HOST changes, so radicale gets the synced files and +# does not stop us from remounting /o. +if dpkg -s radicale &>/dev/null; then + if [[ $HOSTNAME == $MAIL_HOST ]]; then + systemctl restart radicale + systemctl enable radicale + else + systemctl stop radicale + systemctl disable radicale + fi +fi # if I wanted the from address to be renamed and sent to a different address, # echo "sdx@localhost development@localhost" | sudo dd of=/etc/postfix/recipient_canonical diff --git a/music-tag-sync b/music-tag-sync new file mode 100644 index 0000000..0f6e7d9 --- /dev/null +++ b/music-tag-sync @@ -0,0 +1,46 @@ +#!/bin/bash -l +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR +# cd /k/music +# find -type f -name '*.flac' | while read -r f; do +# mkdir -p "../flacs/$(dirname "$f")" +# mv -T "$f" ../flacs/"$f" +# done + + + +# todo, add settings from /etc/default/nfs-{common,kernel-server} +# todo: do mysql setup. kodi install. mysql backup. + +# in kodi, music, add files, named source, add network share, +# server address: iank.life +# path: k/music + + + +rm -f /a/tmp/y.sql + +cd /k/music +find -type f \( -name '*.flac' -or -name '*.mp3' -or -name '*.m4a' \) | while read -r f; do + rating=$(kid3-cli -c "get RATING" "$f") + if [[ ! $rating ]]; then + echo $f + continue + fi + rating=$((rating*2)) + + ## begin sql escaping + f="${f//\"/\\\"}" + f="${f//\'/\\\'}" + f="${f//_/\\_}" + f="${f//%/\\%}" + ## end sql escaping + d=${f%/*} + d=${d#./}/ # use exact dir format that is in database + cat >>/a/tmp/y.sql <&2' ERR # which did not have the whole p subvol. gen_args=() -do_snapshot=true +do_snapshot=false batch=false while [[ $1 ]]; do case $1 in -ob) gen_args+=(--ours -b); batch=true; shift ;; -b) gen_args+=(-b); batch=true; shift ;; - -n) do_snapshot=false; shift ;; + -n) do_snapshot=true; shift ;; -h|--help) echo "$0: help is head of $BASH_SOURCE:" head -n 30 "$BASH_SOURCE" diff --git a/offlineimap-sync b/offlineimap-sync index a179b36..21361b9 100755 --- a/offlineimap-sync +++ b/offlineimap-sync @@ -6,7 +6,7 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR offlineimap -u quiet shopt -s nullglob -omv() { # offlineimap mv +omv() { # offlineimap mv. move mail files within $src_base/$1 to /m/md/$2 src="$1" dst="$2" found_files=false @@ -29,5 +29,10 @@ if $found_files; then omv offlineimaptmp INBOX # remove messages from remote host offlineimap -u quiet - mu index &>/dev/null ||: + # this makes us sit and wait when we want to use mu and this is running in a cronjob. + # todo: emacs updates the index much faster. what command is it running? I'd like + # to just run that + # looks like it might be mu index --lazy-check, but that still takes like 10 seconds, + # figure out if that is the same speed, or if we can make it faster. + #mu index &>/dev/null ||: fi diff --git a/radicale-setup b/radicale-setup index 2f22e64..cb27eda 100755 --- a/radicale-setup +++ b/radicale-setup @@ -9,10 +9,28 @@ # http://radicale.org/user_documentation/ # https://davdroid.bitfire.at/configuration/ +# note on debugging: if radicale can't bind to the address, +# in the log it just says "Starting Radicale". If you run +# it in the foreground, it will give more info. Background +# plus debug does not help. +# sudo -u radicale radicale -D -f # created password file with: # htpasswd -c /etc/davpass dav +d=/etc/systemd/system/radicale.service.d +mkdir -p $d +sudo dd of=$d/override.conf <<'EOF' +[Unit] +# this unit is configured to start and stop whenever openvpn-client@mail.service +# does +After=network.target +BindsTo=openvpn-client@mail.service +After=openvpn-client@mail.service + +[Install] +RequiredBy=openvpn-client@mail.service +EOF pi radicale @@ -36,31 +54,44 @@ setini() { # comments say default is 0.0.0.0:5232 setini hosts 10.8.0.4:5232 server -sgo radicale -# davdroid from f-droid. +if [[ $HOSTNAME == $MAIL_HOST ]]; then + sgo radicale +fi + +# disable power management feature, set to 240 min sync interval, +# so it shouldn't be bad. + +# davdroid from f-druid. # login with url and user name # username ian, # url https://cal.iankelling.org # username ian # pass, see password manager -# I disabled power management feature, it's got 240 min sync interval, -# so it shouldn't be bad. # - -# when setting up davdroid, switch to groups are per-contact categories, -# per https://davdroid.bitfire.at/configuration/radicale/ +# add account dialog: # # set account name as ian@iankelling.org, per help text below the # field. # +# switch to groups are per-contact categories, +# per https://davdroid.bitfire.at/configuration/radicale/ +# +# # After setting up account, I added one address book, named -# ian. calender was already created, named ian. checked boxes under +# ianaddr. calender was already created, named ian. checked boxes under # both. synced. # +# To restore from old phone to new phone, I wiped all data out, then copied over the newly created files. I think +# # ignorable background info: # -# When debugging, tailed /var/log/radicale/radicale.log and nginx log, +# opentasks uses the calendar file. +# +# The address book I created got a uuid as a name for the file. Note +# the .props file says if it's a calendar or addressbook. +# +# When debugging, tailed /var/log/radicale/radicale.log and apache log, # both show the requests happening. Without creating the address book, # after creating a contact, a sync would delete it. # diff --git a/rew b/rew new file mode 100755 index 0000000..860a303 --- /dev/null +++ b/rew @@ -0,0 +1,13 @@ +#!/bin/bash +p="$(lsof -i tcp:31416 -Fp -a -c java | sed -rn 's/^p//p')" +if [[ $p ]]; then + kill $p +fi +cd /a/opt/gnirehtet +./gnirehtet rt &>/tmp/rt & +# note, to kill it, you need to also kill +# lsof -i tcp:31416 +# and maybe unplug adb cable +# note, ping results in log message: dropping invalid packet + +tail -F /tmp/rt ||: diff --git a/switch-mail-host b/switch-mail-host new file mode 100755 index 0000000..61e8966 --- /dev/null +++ b/switch-mail-host @@ -0,0 +1,150 @@ +#!/bin/bash +set -x +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +usage() { + cat <