From e893484a0470843582699dc41fe9d69388c63c02 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Wed, 22 Mar 2017 17:42:20 -0700 Subject: [PATCH] own mailserver, lots of improvements --- btrbk-run | 23 +- check-subvol-stale | 22 +- distro-begin | 65 --- distro-end | 181 ++++--- dynamic-ip-update.sh | 2 +- input-setup | 2 +- ...all-btrfs-sync-tools => install-my-scripts | 6 +- mail-cert-cron | 4 +- mail-route | 6 +- mail-setup | 464 +++++++++++++++--- mount-latest-remote | 2 +- mount-latest-subvol | 17 +- offlineimap-sync | 33 ++ phabricator-setup | 2 - radicale-setup.sh | 82 ++++ rootsshsync | 6 +- spamd-dns-fix | 16 + vpn-mail-forward | 25 + 18 files changed, 716 insertions(+), 242 deletions(-) mode change 100755 => 100644 check-subvol-stale rename install-btrfs-sync-tools => install-my-scripts (84%) mode change 100755 => 100644 mail-cert-cron mode change 100755 => 100644 mount-latest-subvol create mode 100755 offlineimap-sync create mode 100755 radicale-setup.sh create mode 100755 spamd-dns-fix create mode 100755 vpn-mail-forward diff --git a/btrbk-run b/btrbk-run index e812d5b..97b83e0 100755 --- a/btrbk-run +++ b/btrbk-run @@ -25,8 +25,11 @@ usage() { script_dir=$(dirname $(readlink -f "$BASH_SOURCE")) -# todo: finish figuring out fai / distro-setup -# initial fstab / subvol setup. +# note q is owned by root:1000 +# note p/m is owned 1000:1000 and chmod 700 +mountpoints=(/a) +private_mountpoints=(/q /m) +rsync_mountpoint=/q conf_only=false dry_run=false # mostly for testing @@ -114,13 +117,11 @@ target_preserve_min 4h # btrbk -l debug -v dryrun EOF -# note q is owned by root:1000 -# note p is owned 1000:1000 and chmod 700 -mountpoints=(/a) -qmnt=/q -if awk '{print $2}' /etc/fstab | grep -xF $qmnt &>/dev/null; then - mountpoints+=($qmnt) -fi +for mp in ${private_mountpoints[@]}; do # private mountpoints + if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then + mountpoints+=($mp) + fi +done # if our mountpoints are from stale snapshots, # it doesn't make sense to do a backup. @@ -204,8 +205,8 @@ else m btrbk $progress_arg $resume_arg run fi -# if we have /p, rsync to targets without /p -if mountpoint /p >/dev/null; then +# if we have it, sync to systems which don't +if mountpoint $rsync_mountpoint >/dev/null; then for tg in ${targets[@]}; do case $tg in tp|li|lk) diff --git a/check-subvol-stale b/check-subvol-stale old mode 100755 new mode 100644 index f278188..1277182 --- a/check-subvol-stale +++ b/check-subvol-stale @@ -14,7 +14,15 @@ # limitations under the License. # usage: $0 SUBVOL_MOUNTPOINT... -# if latest subvols $@ are not mounted, exit 1, print message, and touch /nocow/btrfs-stale/$subvol +# +# In git, this is not not executable because it's meant to be installed +# using ./install-my-scripts +# +# If latest subvols $@ are not mounted, exit 1, print message, and touch +# /nocow/btrfs-stale/$subvol +# +# Either SUBVOL_MOUNTPOINT is a snapshot of the latest, or +# the latest snapshot is snapshot of SUBVOL_MOUNTPOINT. [[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" @@ -31,7 +39,7 @@ ret=0 for d; do vol=${d##*/} cd /mnt/root/btrbk - snaps=($vol.20*) + snaps=($vol.20*) # Assumes we are in the 21st century. if [[ ! $snaps ]]; then # no snapshots yet continue @@ -45,12 +53,17 @@ for d; do done | sort -r | head -n 1 | awk '{print $2}' ) if [[ ! $last_snap ]]; then - echo "$d stale" + # should not happen. + echo "$0: error: could not find latest snapshot for $d among ${snaps[@]}" ret=1 continue fi stale=true - if btrfs sub show $d 2>/dev/null | sed '0,/^\t*Snapshot(s):/d;s/^\s*//' | \ + # check that $d has $last_snap as a snapshot, + # or else $d is a snapshot of $last_snap. In the second + # case, we use a uuid comparison, which if I remember from the + # docs, is a bit more robust, perhaps to renames. + if btrfs sub show $d 2>/dev/null | sed '0,/^\s*Snapshot(s):/d;s/^\s*//' | \ grep -xF btrbk/$last_snap &>/dev/null; then stale=false else @@ -72,4 +85,3 @@ for d; do fi done exit $ret - diff --git a/distro-begin b/distro-begin index fa42542..aa19cfa 100755 --- a/distro-begin +++ b/distro-begin @@ -658,71 +658,6 @@ if which systemd-hwdb; then ser restart systemd-udev-trigger fi -# work desktop doesnt need gpg stuff, but it doesnt hurt -s dd of=/etc/profile.d/environment.sh <<'EOF' -# IAN: EDIT THIS FROM /a/bin/distro-setup/distro-begin -export ACME_TINY_WRAPPER_CERT_DIR=/p/c/machine_specific/$HOSTNAME/webservercerts - -if [ -f $HOME/path_add-function ]; then - . $HOME/path_add-function - path_add /usr/sbin /usr/local/sbin /sbin - path_add /a/exe /a/opt/bin $HOME/.cabal/bin - - if [ -r /etc/alternatives/java_sdk ]; then - export JAVA_HOME=/etc/alternatives/java_sdk - path_add /etc/alternatives/java_sdk - fi -fi - -export EDITOR="emacsclient" -# this makes emacsclient file/-c start a server instance if none is running, -# instead of some alternate editor logic -export ALTERNATE_EDITOR="" - -# ubuntu starts gpg agent automatically with /etc/X11/Xsession.d/90gpg-agent. -# fedora doesn't, which left me to figure this out, and google was no help. -# fedora documentation is often quite bad :( -# This is mostly copied from that file. -# Main difference is that we eval the result of starting gpg-agent, -# while that file executes it through xsession specific var. -# Also make sourcing the pidfile make more sense. -# End result should be the same afaik. -# for gpg-agent to work when calling gpg from the command line, -# we need an environment variable that is setup via the eval. -# which is why we do this upon login, so it can propogate -# It is also written to the file $HOME/.gnupg/gpg-agent-info-$(hostname) -# I'm not aware if that is ever used, but just fyi. -# I also added the bit about xmessaging the stderr, -# because I'd like to know if the command fails -if [ -f /etc/fedora-release ]; then - : ${GNUPGHOME=$HOME/.gnupg} - - GPGAGENT=/usr/bin/gpg-agent - PID_FILE="$GNUPGHOME/gpg-agent-info-$(hostname)" - - if ! $GPGAGENT 2>/dev/null; then - temp="$(mktemp)" - eval "$($GPGAGENT --homedir /p/do-not-delete --daemon --sh --write-env-file=$PID_FILE 2>$temp)" - temperr="$(<"$temp")" - [ -n "$temperr" ] && xmessage "gpg-agent stderr: $temperr" - elif [ -r "$PID_FILE" ]; then - . "$PID_FILE" - export GPG_AGENT_INFO - fi -fi - -# ubuntu has 002, debian has 022. -# from what I've read, benefit of 002 makes shared groups read/write. -# Security concern is where some unixes put everyone in a same group, -# so if you copy files there with exact perms, that is probably not -# what you want. I don't use a system like that, and I don't really care -# either way, but I'd prefer -# being able to sync file perms with ubuntu systems at work, -# and it's easier to change the debian one. - -umask 002 -EOF - if isdeb; then # I\'ve had problems with postfix on debian: diff --git a/distro-end b/distro-end index 645f1d5..2317edd 100755 --- a/distro-end +++ b/distro-end @@ -35,6 +35,7 @@ spa() { # simple package add distro=$(distro-name) pending_reboot=false +sed="sed --follow-symlinks" # template case $distro in @@ -62,8 +63,14 @@ case $HOSTNAME in # mutagen for pithos simple_packages+=( apache2 + apache2-doc + apt-doc + aptitude-doc-en + bash-doc + binutils-doc bwm-ng chromium + cpio-doc cron debconf-doc duplicity @@ -72,15 +79,21 @@ case $HOSTNAME in fdupes feh filelight + gawk-doc gcc-doc gdb + gdb-doc + git-doc gitk + glibc-doc goaccess gnome-screenshot i3lock + iproute2-doc jq linux-doc locate + make-doc manpages manpages-dev meld @@ -89,23 +102,29 @@ case $HOSTNAME in offlineimap p7zip paprefs + parted-doc pavucontrol pdfgrep + perl-doc pianobar pidgin + python3-doc python3-mutagen reportbug + sqlite3-doc squashfs-tools swh-plugins + tar-doc tcpdump transmission-remote-gtk vlc + whois ) + spa $(apt-cache search ruby[.0-9]+-doc| awk '{print $1}') ;; esac - ########### begin section including li ################ @@ -257,7 +276,7 @@ case $HOSTNAME in #$src/phab-setup pi-nostart mumble-server - s sed -ri "s/^ *(serverpassword=).*/\1$(< /a/bin/bash_unpublished/mumble_pass)/" /etc/mumble-server.ini + s $sed -ri "s/^ *(serverpassword=).*/\1$(< /a/bin/bash_unpublished/mumble_pass)/" /etc/mumble-server.ini sgo mumble-server vpn-server-setup -d @@ -269,8 +288,8 @@ Description=Turns on iptables mail nat [Service] Type=oneshot RemainAfterExit=yes -ExecStart=/sbin/iptables -t nat -A PREROUTING -i eth0 -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25 -ExecStop=/sbin/iptables -t nat -D PREROUTING -i eth0 -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25 +ExecStart=/a/bin/distro-setup/vpn-mail-forward start +ExecStop=/a/bin/distro-setup/vpn-mail-forward stop [Install] WantedBy=openvpn.service @@ -279,7 +298,7 @@ EOF ser enable vpnmail.service acme-tiny-wrapper mail.iankelling.org sgo openvpn - tu /etc/hosts <<<"mail.iankelling.org 10.8.0.4" + tu /etc/hosts <<<"10.8.0.4 mail.iankelling.org" echo "$0: $(date): ending now)" @@ -311,8 +330,9 @@ if private-host; then fi ser enable mailroute if [[ $HOSTNAME == treetowl ]]; then - # note, this will need to be changed when the mail host changes + # note, this will need to be changed when the mail/contacts host changes sgo openvpn-client@mail + /a/bin/distro-setup/radicale-setup.sh fi ## android studio setup @@ -374,8 +394,13 @@ if [[ $HOSTNAME == treetowl ]]; then # on dekstop, top right, actions, device id # after adding, notification will appear on desktop to confirm # - # add folder to sync phone, notification will appear on desktop - # to set folder location. + # syncing folder. from phone to desktop: select desktop in the + # folder on phone's sync options, notification will appear in + # desktop's web ui within a minute. For the reverse, the + # notification will appear in android's notifications, you have to + # swipe down and tap it to add the folder. It won't appear in the + # syncthing ui, which would be intuitive, but don't wait for it + # there. # # On phone, set settings to run syncthing all the time, and # show no notification. @@ -462,8 +487,8 @@ EOF ser disable transmission-daemon sgo transmission-daemon-nn ;; - # todo: others unknown -esac + # todo: others unknown + esac fi # adapted from /var/lib/dpkg/info/transmission-daemon.postinst @@ -512,7 +537,7 @@ require 'json' p = '/etc/transmission-daemon/settings.json' puts JSON.parse(File.read(p))["rpc-password"] EOF -) + ) for f in /home/*; do d=$f/.config/transmission-remote-gtk @@ -716,7 +741,7 @@ case $distro in bridge-utils dnsmasq qemu bind-tools # otherwise we get error about accessing kvm module. # seems like there might be a better way, but google was a bit vague. - s sed -ri --follow-symlinks '/^ *user *=/d' /etc/libvirt/qemu.conf + s $sed -ri '/^ *user *=/d' /etc/libvirt/qemu.conf echo 'user = "root"' | s tee -a /etc/libvirt/qemu.conf # https://bbs.archlinux.org/viewtopic.php?id=206206 # # this should prolly go in the wiki @@ -755,48 +780,69 @@ case $distro in # other distros unknown esac -case $distro in - debian) - # if [[ `debian-archive` == testing ]]; then - # # has no unstable dependencies - # pi bitcoind/unstable - # fi - s cp /a/opt/bitcoin/contrib/init/bitcoind.service /etc/systemd/system - ser daemon-reload - - dir=/nocow/.bitcoin - s mkdir -p $dir - s chown -R bitcoin:bitcoin $dir - dir=/etc/bitcoin - s mkdir -p $dir - s chown -R root:bitcoin $dir - s chmod 750 $dir - f=$dir/bitcoin.conf - - # pruning decreases the bitcoin dir to 2 gb, keeps - # just the recent blocks. can't do a few things like - # import a wallet dump. - # pruning works, but people had to do - # some manual stuff in joinmarket. I dun need the - # disk space, so not bothering yet, maybe in a year or so. - # https://github.com/JoinMarket-Org/joinmarket/issues/431 - #https://bitcoin.org/en/release/v0.12.0#wallet-pruning - #prune=550 - - s dd of=$f </etc/systemd/system/bitcoinjm.service + + d=jm; jm=d # being clever for succinctness + for s in d jm; do + s $sed -ri "/^\s*\[Unit\]/a Conflicts=bitcoin${!s}.service" \ + /etc/systemd/system/bitcoin${s}.service + done + + ser daemon-reload + + dir=/nocow/.bitcoin + s mkdir -p $dir + s chown -R bitcoin:bitcoin $dir + dir=/etc/bitcoin + s mkdir -p $dir + s chown -R root:bitcoin $dir + s chmod 750 $dir + + # pruning decreases the bitcoin dir to 2 gb, keeps + # just the recent blocks. can\'t do a few things like + # import a wallet dump. + # pruning works, but people had to do + # some manual stuff in joinmarket. I dun need the + # disk space, so not bothering yet, maybe in a year or so. + # https://github.com/JoinMarket-Org/joinmarket/issues/431 + #https://bitcoin.org/en/release/v0.12.0#wallet-pruning + #prune=550 + + f=$dir/bitcoin.conf + s dd of=$f </dev/null </dev/null; then # run "control userpasswords2", turn on automatic login. # note: when changing devices, I just undefine, the create the vm again. - if [[ -e /a/images/win10.qcow2 ]]; then + if [[ -e /nocow/user/vms/win10.qcow2 ]]; then s virt-install --noautoconsole --graphics spice,listen=0.0.0.0 \ --disk=/a/images/win10.qcow2,bus=virtio --vcpus 2 -r 4096 -w bridge=br0 \ -n win10 --import --os-variant $variant --cpu host-model-only @@ -1280,7 +1303,7 @@ if ! s virsh list --all --name | grep -xF win10 &>/dev/null; then s virsh destroy win10 fi - if [[ -e /a/images/win7.qcow2 ]]; then + if [[ -e /nocow/user/vms/win7.qcow2 ]]; then # this one hasn\'t had the virtio fix done yet. s virt-install --noautoconsole --graphics spice,listen=0.0.0.0 \ --disk=/a/images/win7.qcow2 --vcpus 2 -r 4096 -w bridge=br0 \ diff --git a/dynamic-ip-update.sh b/dynamic-ip-update.sh index 60c9dd9..a2da871 100755 --- a/dynamic-ip-update.sh +++ b/dynamic-ip-update.sh @@ -20,7 +20,7 @@ # https://www.namecheap.com/support/knowledgebase/article.aspx/583/11/how-do-i-configure-ddclient ip=`curl -s4 echoip.com` -curl -sS "https://dynamicdns.park-your-domain.com/update?host=@&domain=$HOME_DOMAIN&password=$(cat /p/dynamic-ip-pass)&ip=$ip" > /dev/null | /a/exe/log-once dynamic-ip +curl -sS "https://dynamicdns.park-your-domain.com/update?host=@&domain=$HOME_DOMAIN&password=$(cat /p/dynamic-ip-pass)&ip=$ip" > /dev/null # an alternative, putting my ip on some known server, # allows ssh to home if I can access that server: diff --git a/input-setup b/input-setup index 5db66e9..2f595f3 100755 --- a/input-setup +++ b/input-setup @@ -90,7 +90,7 @@ case $HOSTNAME in # 12 is to effectively disable the middle click button xinput --set-button-map "$device_id" 1 12 3 10 11 6 7 fi - /a/bin/bash_unpublished/gpg + . /a/bin/bash_unpublished/duplicity-gpg-agent-setup ;; frodo*) ;; diff --git a/install-btrfs-sync-tools b/install-my-scripts similarity index 84% rename from install-btrfs-sync-tools rename to install-my-scripts index e912664..42a6d2d 100755 --- a/install-btrfs-sync-tools +++ b/install-my-scripts @@ -28,5 +28,7 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR x="$(readlink -f "$BASH_SOURCE")"; cd ${x%/*} -echo install mount-latest-subvol check-subvol-stale /usr/local/bin -install mount-latest-subvol check-subvol-stale /usr/local/bin +e() { echo "$*"; "$@"; } + +# scripts that would interfere with unmounting /a, put them elsewhere +e install mail-cert-cron mount-latest-subvol check-subvol-stale /usr/local/bin diff --git a/mail-cert-cron b/mail-cert-cron old mode 100755 new mode 100644 index fad8214..64b22eb --- a/mail-cert-cron +++ b/mail-cert-cron @@ -16,10 +16,12 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR +[[ $EUID == 0 ]] || exec sudo "$BASH_SOURCE" "$@" + source /a/bin/bash_unpublished/source-semi-priv if [[ $HOSTNAME == $MAIL_HOST ]]; then local_mx=mail.iankelling.org - rsync_common="s rsync -ogt --chown=root:Debian-exim --chmod=640 root@li:/p/c/machine_specific/li/webservercerts/$local_mx-" + rsync_common="rsync -ogt --chown=root:Debian-exim --chmod=640 root@li:/p/c/machine_specific/li/webservercerts/$local_mx-" ${rsync_common}chained.pem /etc/exim4/exim.crt ${rsync_common}domain.key /etc/exim4/exim.key fi diff --git a/mail-route b/mail-route index a508e2e..f896bfe 100755 --- a/mail-route +++ b/mail-route @@ -59,8 +59,12 @@ esac # exim is replying to. I don't know why. #iptables -t mangle -A OUTPUT -m owner --uid-owner Debian-exim -j MARK --set-mark 0x1 +# match source or dest port. when we send to 25, it picks a random high port as +# the source. -e iptables -t mangle $iptables_op OUTPUT -m tcp -p tcp -m multiport --sports 25 -j MARK --set-mark 0x1 +for port in 25 993; do # smtp and imap with ssl. +e iptables -t mangle $iptables_op OUTPUT -m tcp -p tcp -m multiport --ports $port -j MARK --set-mark 0x1 +done e iptables -t nat $iptables_op POSTROUTING -o tun0 -m mark --mark 0x1 -j SNAT --to-source 10.8.0.4 e ip rule $ip_op fwmark 1 table 1 # note, this rule does not persist when the tun interface is deleted diff --git a/mail-setup b/mail-setup index b43bde4..da91b1c 100755 --- a/mail-setup +++ b/mail-setup @@ -13,6 +13,96 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +####### begin perstent password instructions ###### +# # exim passwords: +# # for hosts which have all private files I just use the same user +# # for other hosts, each one get's their own password. +# # for generating secure pass, and storing for server too: +# # user=USUALLY_SAME_AS_HOSTNAME +# user=li +# f=$(mktemp) +# apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f +# s sed -i "/^$user:/d" /p/c/filesystem/etc/exim4/passwd +# echo "$user:$(mkpasswd -m sha-512 -s <$f)" >>/p/c/filesystem/etc/exim4/passwd +# echo "mail.iankelling.org:$user:$(<$f)" >> /p/c/machine_specific/$user/filesystem/etc/mailpass +# # then run this script, or part of it which uses /etc/mailpass + +# # dovecot password, i just need 1 as I'm the only user +# mkdir /p/c/filesystem/etc/dovecot +# echo "ian:$(doveadm pw -s ssha256)::::::" >/p/c/filesystem/etc/dovecot/users +# conflink + + + +# # for ad-hoc testing of some random new host sending mail: +# user=li # client host username & hostname +# f=$(mktemp) +# apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f +# s sed -i "/^$user:/d" /etc/exim4/passwd +# echo "$user:$(mkpasswd -m sha-512 -s <$f)" | s tee -a /etc/exim4/passwd +# echo "mail.iankelling.org:$user:$(<$f)" | ssh root@$user dd of=/etc/exim4/passwd.client +####### end perstent password instructions ###### + + +####### begin persistent dkim/dns instructions ######### +# # Remove 1 level of comments in this section, set the domain var +# # for the domain you are setting up, then run this and copy dns settings +# # into dns. +# domain=iankelling.org +# c /p/c/filesystem/etc/exim4 +# # this has several bugs addressed in comments, but it was helpful +# # https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4 + +# openssl genrsa -out $domain-private.pem 2048 -outform PEM +# openssl rsa -in $domain-private.pem -out $domain.pem -pubout -outform PEM +# # selector is needed for having multiple keys for one domain. +# # I dun do that, so just use a static one: li +# echo "txt record name: li._domainkey.$domain" +# # Debadmin page does not have v=, fastmail does, and this +# # says it's recommended in 3.6.1, default is DKIM1 anyways. +# # https://www.ietf.org/rfc/rfc6376.txt +# # Join and print all but first and last line. +# # last line: swap hold & pattern, remove newlines, print. +# # lines 2+: append to hold space +# echo "txt record contents:" +# echo "v=DKIM1; k=rsa; p=$(sed -n '${x;s/\n//gp};2,$H' $domain.pem)" +# chmod 644 $domain.pem +# chmod 640 $domain-private.pem +# # in conflink, we chown these to group debian +# conflink +# # selector was also put into /etc/exim4/conf.d/main/000_localmacros, +# # via the mail-setup scripts + +# # 2017-02 dmarc policies: +# # yahoo: p=reject, hotmail: p=none, gmail: p=none, fastmail none for legacy reasons +# # gmail will be changing to p=reject, which is expected to cause problems +# # with a few old mailing lists, copying theirs for now. +# echo "dmarc dns, name: _dmarc value: v=DMARC1; p=none; rua=mailto:mailauth-reports@$domain" + +# # 2017-02 spf policies: +# # google ~all, hotmail -all, yahoo: ?all, fastmail ?all +# # i include fastmail's settings, per their instructions, +# # and follow their policy. In mail in a box, or similar instructions, +# # I've seen recommended to not use a restrictive policy. +# echo "spf dns: name is empty, value: v=spf1 a include:spf.messagingengine.com ?all" + +# # to check if dns has updated, you do +# host -a mesmtp._domainkey.$domain + +# # mx records, +# # setting it to iankelling.org would work the same, but this +# # is more flexible, I could change where mail.iankelling.org pointed. +# cat <<'EOF' +# mx records, 2 records each, for * and empty domain +# pri 10 mail.iankelling.org +# pri 20 in1-smtp.messagingengine.com +# pri 30 in2-smtp.messagingengine.com +# EOF +####### end persistent dkim instructions ######### + # misc exim notes: # useful exim docs: @@ -43,9 +133,6 @@ #exim4 -bt ian@localhost -set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR - type=$1 postfix() { [[ $type == postfix ]]; } exim() { [[ $type == exim4 ]]; } @@ -56,15 +143,19 @@ fi local_mx=mail.iankelling.org -if [[ $HOSTNAME == $MAIL_HOST ]]; then - host=mail.messagingengine.com - relayhost="[$host]:587" # postfix - smarthost="$host::587" # exim -else - host=$local_mx - relayhost="[$host]:25" # postfix - smarthost="$host::25" # exim -fi + +host=$local_mx +relayhost="[$host]:25" # postfix +smarthost="$host::25" # exim + + +# this was for when I used the exim config type +# "mail sent by smarthost; received via SMTP or fetchmail" +# if [[ $HOSTNAME == $MAIL_HOST ]]; then +# host=mail.messagingengine.com +# relayhost="[$host]:587" # postfix +# smarthost="$host::587" # exim +# fi forward=ian@$local_mx @@ -144,9 +235,7 @@ EOF s service postfix reload else # exim. has debian specific stuff for now - # debconf settings will not work if packages are already installed, - # such as on vps images. - pu exim4-daemon-light exim4-daemon-heavy exim4-config exim4-base exim4 + # wording of question from dpkg-reconfigure exim4-config # 1. internet site; mail is sent and received directly using SMTP @@ -155,30 +244,256 @@ else # exim. has debian specific stuff for now # 4. local delivery only; not on a network # 5. no configuration at this time # - # only the one receiving host needs option 2, the rest can do option - # 1, but that host might change, so we pick option 2 and later don't - # set it up to receive anything, because it has nothing in it's - # receiving password file and we make that mandatory. Also, only - # receiving host needs dc_other_hostnames and beyond, but no harm. - - # note, another related setting is /etc/mailname, which - # is set to be $HOSTNAME.lan on stretch. this may need to be - # setup on other distros. - - # setting local_interfaces to empty listens on all interfaces. - # default is 127.0.0.1 ; ::1, so only listen to lo interface. + # Note, I have used option 2 in the past for receiving mail + # from lan hosts, sending external mail via another smtp server. + # + # Note, other than configtype, we could set all the options in + # both types of configs without harm, they would either be + # ignored or be disabled by other settings, but the default + # local_interfaces definitely makes things more secure. + + # most of these settings get translated into settings + # in /etc/exim4/update-exim4.conf.conf + # mailname setting sets /etc/mailname + s debconf-set-selections </dev/null <<'EOF' +MAIN_TLS_ENABLE = true + +DKIM_CANON = relaxed +DKIM_SELECTOR = li + +# from comments in +# https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4 + +# The file is based on the outgoing domain-name in the from-header. +DKIM_DOMAIN = ${lc:${domain:$h_from:}} +# sign if key exists +DKIM_PRIVATE_KEY= ${if exists{/etc/exim4/${dkim_domain}-private.pem} {/etc/exim4/${dkim_domain}-private.pem}} + + +# failing message on mail-tester.com: +# We check if there is a server (A Record) behind your hostname treetowl. +# You may want to publish a DNS record (A type) for the hostname treetowl or use a different hostname in your mail software +# https://serverfault.com/questions/46545/how-do-i-change-exim4s-primary-hostname-on-a-debian-box +# and this one seemed appropriate from grepping config +MAIN_HARDCODE_PRIMARY_HOSTNAME = li.iankelling.org + +# normally empty, I set this so I can set the envelope address +# when doing mail redelivery to invoke filters +MAIN_TRUSTED_GROUPS = ian + +# disabled, didn't finished configuring +#LOCAL_DELIVERY = dovecot_lmtp + +CHECK_RCPT_LOCAL_ACL_FILE = /etc/exim4/rcpt_local_acl +EOF + + + s dd of=/etc/systemd/system/offlineimapsync.timer <<'EOF' +[Unit] +Description=Run offlineimap-sync once every 5 mins + +[Timer] +OnCalendar=*:0/5 + +[Install] +WantedBy=timers.target +EOF + + s dd of=/etc/systemd/system/offlineimapsync.service <<'EOF' +[Unit] +Description=Offlineimap sync +After=multi-user.target + +[Service] +Type=oneshot +ExecStart=/a/bin/log-quiet/sysd-mail-once offlineimap-sync /a/bin/distro-setup/offlineimap-sync +EOF + s systemctl daemon-reload + s systemctl enable offlineimapsync.timer + s systemctl start offlineimapsync.timer + + else # $HOSTNAME != $MAIL_HOST + s systemctl disable offlineimapsync.timer &>/dev/null ||: + s systemctl stop offlineimapsync.timer &>/dev/null ||: + # + # + # would only exist because I wrote it i the previous condition, + # it's not part of exim + s rm -f $exim_main_dir/000_localmacros + s debconf-set-selections </dev/null; then + s dpkg-reconfigure -u -pcritical exim4-config + fi # light version does not have sasl auth support. - pi exim4-daemon-heavy + pi exim4-daemon-heavy spamassassin + + ##### begin spamassassin config + ser enable spamassassin + # per readme.debian + s sed -i '/^\s*CRON\s*=/d' /etc/default/spamassassin + s tee -a /etc/default/spamassassin <<$f - # echo "$user:$(mkpasswd -m sha-512 -s <$f)" >>/p/c/filesystem/etc/exim4/passwd - # echo "mail.iankelling.org:$user:$(<$f)" >> /p/c/machine_specific/$user/filesystem/etc/mailpass - # - # for ad-hoc testing of some random new host: - # host=testhost # client host username & hostname - # f=$(mktemp) - # apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f - # s sed "/^host:/d" /etc/exim4/passwd - # echo "$host:$(mkpasswd -m sha-512 -s <$f)" | s tee -a /etc/exim4/passwd - # echo "mail.iankelling.org:$host:$(<$f)" | ssh root@$host dd of=/etc/exim4/passwd.client + # note: this will go away s cat /etc/mailpass| while read -r domain port pass; do # reference: exim4_passwd_client(5) - printf "%s:%s" "$domain" "$pass" | s tee -a $f >/dev/null + printf "%s:%s\n" "$domain" "$pass" | s tee -a $f >/dev/null done + # end setup passwd.client # https://blog.dhampir.no/content/make-exim4-on-debian-respect-forward-and-etcaliases-when-using-a-smarthost # i only need .forwards, so just doing that one. @@ -237,33 +566,36 @@ EOF sed -r s/^\\S+:/$b:/ 600_exim4-config_$a | s dd of=$tmp 2>/dev/null if ! diff -q $tmp $of &>/dev/null; then s dd if=$tmp of=$of >/dev/null - ser restart exim4 fi -fi + ser restart exim4 + + fi -# linode image has a root alias. completely useless, remove it. -sudo sed -i '/^root:/d' /etc/aliases -s newaliases + # linode image has a root alias. completely useless, remove it. + sudo sed -i '/^root:/d' /etc/aliases -# based on http://www.postfix.org/qmgr.8.html and my notes in gnus -dir=/nocow/$type -sdir=/var/spool/$type -if [[ $(readlink -f $sdir) != $dir ]]; then - ser stop $type - if [[ ! -e $dir && -d $sdir ]]; then - s mv $sdir $dir + s newaliases + + + # based on http://www.postfix.org/qmgr.8.html and my notes in gnus + dir=/nocow/$type + sdir=/var/spool/$type + if [[ $(readlink -f $sdir) != $dir ]]; then + ser stop $type + if [[ ! -e $dir && -d $sdir ]]; then + s mv $sdir $dir + fi + s lnf -T $dir $sdir fi - s lnf -T $dir $sdir -fi -sgo $type + sgo $type -# if I wanted the from address to be renamed and sent to a different address, -# echo "sdx@localhost development@localhost" | sudo dd of=/etc/postfix/recipient_canonical -# sudo postmap hash:/etc/postfix/recipient_canonical -# sudo service postfix reload + # if I wanted the from address to be renamed and sent to a different address, + # echo "sdx@localhost development@localhost" | sudo dd of=/etc/postfix/recipient_canonical + # sudo postmap hash:/etc/postfix/recipient_canonical + # sudo service postfix reload diff --git a/mount-latest-remote b/mount-latest-remote index 43f9400..cd04faf 100755 --- a/mount-latest-remote +++ b/mount-latest-remote @@ -29,7 +29,7 @@ for tg; do if ! ssh root@$tg bash <<'EOF' set -e chmod +x /usr/local/bin/{mount-latest-subvol,check-subvol-stale} -mount-latest-subvol +/usr/local/bin/mount-latest-subvol EOF then echo "$0: warning: failed mount-latest-subvol on $tg" diff --git a/mount-latest-subvol b/mount-latest-subvol old mode 100755 new mode 100644 index 091cf44..a8ff06d --- a/mount-latest-subvol +++ b/mount-latest-subvol @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +# usage: mount-latest-subvol [[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" @@ -76,6 +76,7 @@ mnt() { ret=0 +##### begin setup fstab for subvols we care about ###### first_root_crypt=$(awk '$2 == "/" {print $1}' /etc/mtab) tu /etc/fstab </dev/null; then continue fi - binds=() - roots=($d) + ##### begin building up list of bind mounts ###### + binds=() # list of bind mounts + roots=($d) # list of bind mounts, plus the original mount while true; do new_roots=() for r in ${roots[@]}; do - # example + # eg. when r=/q/p, for lines like # /q/p /p none bind 0 0 + # output /p new_roots+=($(sed -rn "s#^$r/\S+\s+(\S+)\s+none\s+bind\s.*#\1#p" /etc/fstab)) done (( ${#new_roots} )) || break binds+=(${new_roots[@]}) roots=( ${new_roots[@]} ) done + ##### end building up list of bind mounts ###### + # if latest is already mounted, make sure binds are mounted and move on if e check-subvol-stale $d; then diff --git a/offlineimap-sync b/offlineimap-sync new file mode 100755 index 0000000..a179b36 --- /dev/null +++ b/offlineimap-sync @@ -0,0 +1,33 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +offlineimap -u quiet +shopt -s nullglob + +omv() { # offlineimap mv + src="$1" + dst="$2" + found_files=false + for x in new cur; do + files=("$src_base"/"$src"/$x/*) + if [[ $files ]]; then + found_files=true + mv "${files[@]}" /m/md/"$dst"/$x + fi + done +} + +src_base=/m/offlineimap +omv "Sent Items" "Sent" +omv INBOX offlineimaptmp +src_base=/m/md +if $found_files; then + sieve-filter -eW ~/sieve/main.sieve offlineimaptmp &>/dev/null + # the default folder is INBOX for anything leftover + omv offlineimaptmp INBOX + # remove messages from remote host + offlineimap -u quiet + mu index &>/dev/null ||: +fi diff --git a/phabricator-setup b/phabricator-setup index 053cf21..7c63e4f 100755 --- a/phabricator-setup +++ b/phabricator-setup @@ -50,8 +50,6 @@ fsetd() { fbin config set --database "$@"; } # phabricator complained about wanting arcanist first pi arcanist/unstable mercurial -for x in /a/bin/bash_unpublished/*; do source $x; done - # duplicated in mediawiki setup. todo fix that. s DEBIAN_FRONTEND=noninteractive pi mysql-server cd # mysql_secure_installation writes some temp files to the current dir, diff --git a/radicale-setup.sh b/radicale-setup.sh new file mode 100755 index 0000000..620096a --- /dev/null +++ b/radicale-setup.sh @@ -0,0 +1,82 @@ +#!/bin/bash -l + +[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" + +# davdroid setup instructions at the bottom + +# main docs: +# http://radicale.org/user_documentation/ +# https://davdroid.bitfire.at/configuration/ + + +# created password file with: +# htpasswd -c /etc/nginx/caldav/htpasswd ian + + +# python-dulwich, to track changes with git, per +# http://radicale.org/user_documentation/#idgit-support +pi nginx python-dulwich radicale + +# I moved /var/lib/radicale after it's initialization. +# I did a sudo -u radicale git init in the collections subfolder +# after it gets created, per the git docs. +lnf -T /q/radicale /var/lib/radicale + +acme-tiny -n cal.iank.pw + +# from https://www.williamjbowman.com/blog/2015/07/24/setting-up-webdav-caldav-and-carddav-servers/ +mkdir /etc/nginx/caldav +touch /etc/nginx/caldav/htpasswd +chmod 640 /etc/nginx/caldav/htpasswd +chgrp www-data /etc/nginx/caldav/htpasswd +nginx-site -f 5232 - cal.iank.pw <<'EOF' + auth_basic "Not currently available"; + auth_basic_user_file /etc/nginx/caldav/htpasswd; +EOF + +# coment in this file says this is needed for it to run on startup +sed -ri 's/^\s*#+\s*(ENABLE_RADICALE\s*=\s*yes\s*)/\1/' /etc/default/radicale +sgo radicale + +setini() { + key="$1" value="$2" section="$3" + file="/etc/radicale/config" + sed -ri "/ *\[$section\]/,/^ *\[[^]]+\]/{/^\s*$key[[:space:]=]/d};/ *\[$section\]/a $key = $value" "$file" +} + +# comments say default is 0.0.0.0:5232 +setini hosts 127.0.0.1:5232 server + +# davdroid from f-droid. username ian, +# url https://iank.pw/radicale/ian. +# I disabled power management feature, it's got 240 min sync interval, +# so it shouldn't be bad. +# + +# when setting up davdroid, switch to groups are per-contact categories, +# per https://davdroid.bitfire.at/configuration/radicale/ +# +# set account name as ian@iankelling.org, per help text below the +# field. +# +# After setting up account, I added one address book, named +# ian. calender was already created, named ian. checked boxes under +# both. synced. +# +# ignorable background info: +# +# When debugging, tailed /var/log/radicale/radicale.log and nginx log, +# both show the requests happening. Without creating the address book, +# after creating a contact, a sync would delete it. +# +# Address books correspond to .props files in the radicale dir. +# +# Some background is here, +# https://davdroid.bitfire.at/faq/entry/cant-manage-groups-on-device/ +# which shows separate vcard option is from rfc 6350, the other is 2426, +# radicale page says it implements the former not the latter, +# which conflicts with the documentation of which to select, but whatever. +# http://radicale.org/technical_choices/ +# https://davdroid.bitfire.at/faq/entry/cant-manage-groups-on-device/ +# +# Note, url above says only cayanogenmod 13+ and omnirom can manage groups. diff --git a/rootsshsync b/rootsshsync index d3e860b..e962f19 100755 --- a/rootsshsync +++ b/rootsshsync @@ -19,12 +19,14 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR [[ $EUID == 0 ]] || exec sudo "$BASH_SOURCE" "$@" if test -e /q/root/.ssh; then - /a/exe/lnf /q/root/.ssh /root + dest=/q/root/.ssh + /a/exe/lnf $dest /root else + dest=/root/.ssh mkdir -p /root/.ssh chmod 700 /root/.ssh fi # -t times, so it won't rewrite the file every time, # -L resolve links -rsync -rtL $(eval echo ~${SUDO_USER:-$USER})/.ssh /root +rsync -rtL $(eval echo ~${SUDO_USER:-$USER})/.ssh $dest chown -R root:root /root/.ssh diff --git a/spamd-dns-fix b/spamd-dns-fix new file mode 100755 index 0000000..fb042f5 --- /dev/null +++ b/spamd-dns-fix @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +[[ $EUID == 0 ]] + +# to deal with this bug until it\'s fixed +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=741521 +# I just happened to notice it in my journal. +str="dns: sendto() to \S\+ failed: Connection refused, failing over" +if journalctl --since=-9m --unit=spamassassin | \ + grep "$str" &>/dev/null; then + echo "dns bug, restarting spamassassin" + systemctl restart spamassassin +fi diff --git a/vpn-mail-forward b/vpn-mail-forward new file mode 100755 index 0000000..9a86e69 --- /dev/null +++ b/vpn-mail-forward @@ -0,0 +1,25 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +do-forward() { + cmd=$1; shift + for port; do + /sbin/iptables -t nat $cmd PREROUTING -i eth0 -p tcp -m tcp --dport $port -j DNAT --to-destination 10.8.0.4:$port + done +} + +ports=(25 993) +case $1 in + start) + do-forward -A ${ports[@]} + ;; + stop) + do-forward -D ${ports[@]} + ;; + *) + echo "$0: error: expected 1 argument of start or stop" + exit 1 + ;; +esac -- 2.30.2