From: Ian Kelling Date: Thu, 20 Apr 2017 06:20:21 +0000 (-0700) Subject: mostly refactor mail-setup, more self contained X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=8b81badab655a1768308ef8881ffb7241f7531a2 mostly refactor mail-setup, more self contained --- diff --git a/conflink b/conflink index 8431c16..948d1a2 100755 --- a/conflink +++ b/conflink @@ -43,18 +43,38 @@ subdir-link-r() { done } + + common-file-setup() { - local dir fs x + local dir fs x bdir f dst for dir in "$@"; do fs=$dir/filesystem if [[ -e $fs && $USER == ian ]]; then # note, symlinks get resolved, not copied. m s cp -RLT --preserve=mode,timestamps $fs / fi + + # one program so far does not work with a symlink, + # so we have to use a bind mount + bdir=$dir/binds + if [[ -e $bdir && $USER == ian ]]; then + for f in $(find $bdir -type f); do + dst=/home/ian/${f##$bdir} + if [[ ! -e $dst || $(stat -c%i $f $dst |uniq -u) ]]; then + if mountpoint -q $dst; then + s umount $dst + fi + # note: in future, may need to mkdir also + [[ -e $dst ]] || touch $dst + s mount --bind $f $dst + fi + done + fi + if [[ -e $dir/subdir_files ]]; then subdir-link-r $dir/subdir_files fi - local x=( $dir/!(subdir_files|filesystem|machine_specific|..|.) ) + local x=( $dir/!(binds|subdir_files|filesystem|machine_specific|..|.) ) (( ${#x[@]} >= 1 )) || continue m lnf ${x[@]} ~ done diff --git a/distro-end b/distro-end index b55fabc..e0bf4f6 100755 --- a/distro-end +++ b/distro-end @@ -65,6 +65,7 @@ case $HOSTNAME in apache2 apache2-doc apt-doc + apt-listchanges aptitude-doc-en bash-doc binutils-doc diff --git a/install-my-scripts b/install-my-scripts index 42a6d2d..e2356ca 100755 --- a/install-my-scripts +++ b/install-my-scripts @@ -31,4 +31,4 @@ x="$(readlink -f "$BASH_SOURCE")"; cd ${x%/*} e() { echo "$*"; "$@"; } # scripts that would interfere with unmounting /a, put them elsewhere -e install mail-cert-cron mount-latest-subvol check-subvol-stale /usr/local/bin +e install mount-latest-subvol check-subvol-stale /usr/local/bin diff --git a/mail-cert-cron b/mail-cert-cron deleted file mode 100644 index 64b22eb..0000000 --- a/mail-cert-cron +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# Copyright (C) 2016 Ian Kelling - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eE -o pipefail -trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR - -[[ $EUID == 0 ]] || exec sudo "$BASH_SOURCE" "$@" - -source /a/bin/bash_unpublished/source-semi-priv -if [[ $HOSTNAME == $MAIL_HOST ]]; then - local_mx=mail.iankelling.org - rsync_common="rsync -ogt --chown=root:Debian-exim --chmod=640 root@li:/p/c/machine_specific/li/webservercerts/$local_mx-" - ${rsync_common}chained.pem /etc/exim4/exim.crt - ${rsync_common}domain.key /etc/exim4/exim.key -fi diff --git a/mail-setup b/mail-setup index 7921a22..d7901e7 100755 --- a/mail-setup +++ b/mail-setup @@ -1,4 +1,4 @@ -#!/bin/bash -l +#!/bin/bash # Copyright (C) 2016 Ian Kelling # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,30 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR +[[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@" + +usage() { + cat <&2' ERR #exim4 -bt ian@localhost -type=$1 -postfix() { [[ $type == postfix ]]; } -exim() { [[ $type == exim4 ]]; } -if ! exim && ! postfix; then - echo "$1: error: expected exim4 or postfix as first arg" - exit 1 -fi +postconfin() { + local MAPFILE + mapfile -t + local s + postconf -ev "${MAPFILE[@]}" +} +e() { printf "%s\n" "$*"; } +postmaster=ian +mxhost=mail.iankelling.org +mxport=25 +forward=ian@$mxhost -local_mx=mail.iankelling.org +# old setup. left as comment for example +# mxhost=mail.messagingengine.com +# mxport=587 +# forward=ian@iankelling.org -host=$local_mx -relayhost="[$host]:25" # postfix -smarthost="$host::25" # exim +relayhost="[$mxhost]:$mxport" # postfix +smarthost="$mxhost::$mxport" # exim -# this was for when I used the exim config type -# "mail sent by smarthost; received via SMTP or fetchmail" -# if [[ $HOSTNAME == $MAIL_HOST ]]; then -# host=mail.messagingengine.com -# relayhost="[$host]:587" # postfix -# smarthost="$host::587" # exim -# fi - -forward=ian@$local_mx - if [[ $HOSTNAME == $MAIL_HOST ]]; then - # if we are MAIL_HOST, exim config sets up an /etc/alias from - # root to the postmaster, which i config to ian, as long as there - # exists an entry for root, or there was no preexisting aliases file. - # based on the postinst file. - s rm -f /etc/aliases + # afaik, these will get ignored because they are routing to my own + # machine, but rm them is safer + rm -f $(eval echo ~$postmaster)/.forward /root/.forward else - # linode image has a root alias, I think it might override our .forward - sudo sed -i '/^root:/d' /etc/aliases - s newaliases - - # background: This also works instead of ~/.forward - # s sed -i --follow-symlinks '/^root/d' /etc/aliases ||: - #echo "root: $HOSTNAME@$SOME_DOMAIN" | s tee -a /etc/aliases # this can\'t be a symlink and has permission restrictions # it might work in /etc/aliases, but this seems more proper. - e $forward > ~/.forward - e $forward | s tee /root/.forward - # 644 is required. shouldn\'t need changing, but set it just in case. - s chmod 644 ~/.forward /root/.forward + install -m 644 {-o,-g}$postmaster <(e $forward) $(eval echo ~$postmaster)/.forward fi # offlineimap uses this too, it is much easier to use one location than to # condition it\'s config and postfix\'s config -case $distro in - fedora) s lnf -T ca-certificates.crt /etc/ssl/ca-bundle.trust.crt ;; - *) : -esac +if [[ -f /etc/fedora-release ]]; then + /a/exe/lnf -T ca-certificates.crt /etc/ssl/ca-bundle.trust.crt +fi if postfix; then # dunno why, but debian installed postfix with builddep emacs # but I will just explicitly install it here since # I use it for sending mail in emacs. - if isdeb; then - s debconf-set-selections < /dev/null; then + debconf-set-selections </dev/null; then + dpkg-reconfigure -u -fnoninteractive postfix + else + apt-get -y install --purge --auto-remove postfix + fi else + source /a/bin/distro-functions/src/package-manager-abstractions pi postfix # Settings from reading the output when installing on debian, # then seeing which were different in a default install on arch. @@ -213,8 +225,8 @@ relayhost = $relayhost inet_interfaces = loopback-only EOF - s systemctl enable postfix - s systemctl start postfix + systemctl enable postfix + systemctl start postfix fi # i\'m assuming mail just won\'t work on systems without the sasl_passwd. postconfin <<'EOF' @@ -227,23 +239,23 @@ smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt inet_protocols = ipv4 EOF # msg_size_limit: I ran into a log file not sending cuz of size. double from 10 to 20 meg limit - # inet_protocols: without this, postfix tries an ipv6 lookup then gives - # up and fails. snippet from syslog: type=AAAA: Host not found, try again + # inet_protocols: without this, I've had postfix try an ipv6 lookup then gives + # up and fail forever. snippet from syslog: type=AAAA: Host not found, try again f=/etc/postfix/sasl_passwd - s rm -f $f - s touch $f - s chmod 600 $f - s cat /etc/mailpass| while read -r domain port pass; do + install -m 600 /dev/null $f + cat /etc/mailpass| while read -r domain port pass; do # format: domain port user:pass # mailpass is just a name i made up, since postfix and # exim both use a slightly crazy format to translate to # each other, it\'s easier to use my own format. - printf "[%s]:%s %s" "$domain" "$port" "${pass/@/#}" | s tee -a $f >/dev/null + printf "[%s]:%s %s" "$domain" "$port" "${pass/@/#}" >>$f done - s postmap hash:/etc/postfix/sasl_passwd - s service postfix reload + postmap hash:/etc/postfix/sasl_passwd + # need restart instead of reload when changing + # inet_protocols + service postfix restart else # exim. has debian specific stuff for now @@ -267,19 +279,16 @@ else # exim. has debian specific stuff for now # in /etc/exim4/update-exim4.conf.conf # mailname setting sets /etc/mailname - s debconf-set-selections </dev/null <<'EOF' + cat >$exim_main_dir/000_localmacros <<'EOF' MAIN_TLS_ENABLE = true DKIM_CANON = relaxed @@ -419,11 +428,13 @@ MAIN_TRUSTED_GROUPS = ian LOCAL_DELIVERY = dovecot_lmtp +# options exim has to avoid altering the default config files CHECK_RCPT_LOCAL_ACL_FILE = /etc/exim4/rcpt_local_acl +CHECK_DATA_LOCAL_ACL_FILE = /etc/exim4/data_local_acl EOF - s dd of=/etc/systemd/system/offlineimapsync.timer <<'EOF' + cat >/etc/systemd/system/offlineimapsync.timer <<'EOF' [Unit] Description=Run offlineimap-sync once every 5 mins @@ -434,7 +445,7 @@ OnCalendar=*:0/5 WantedBy=timers.target EOF - s dd of=/etc/systemd/system/offlineimapsync.service <<'EOF' + cat >/etc/systemd/system/offlineimapsync.service <<'EOF' [Unit] Description=Offlineimap sync After=multi-user.target @@ -444,45 +455,48 @@ User=ian Type=oneshot ExecStart=/a/bin/log-quiet/sysd-mail-once offlineimap-sync /a/bin/distro-setup/offlineimap-sync EOF - s systemctl daemon-reload - s systemctl enable offlineimapsync.timer - s systemctl start offlineimapsync.timer + systemctl daemon-reload + systemctl enable offlineimapsync.timer + systemctl start offlineimapsync.timer else # $HOSTNAME != $MAIL_HOST - s systemctl disable offlineimapsync.timer &>/dev/null ||: - s systemctl stop offlineimapsync.timer &>/dev/null ||: + systemctl disable offlineimapsync.timer &>/dev/null ||: + systemctl stop offlineimapsync.timer &>/dev/null ||: # # # would only exist because I wrote it i the previous condition, # it\'s not part of exim - s rm -f $exim_main_dir/000_localmacros - s debconf-set-selections </dev/null; then # gotta remove this, otherwise the set-selections are completely # ignored. It woulda been nice if this was documented somewhere! - s rm -f /etc/exim4/update-exim4.conf.conf - s dpkg-reconfigure -u -fnoninteractive exim4-config + rm -f /etc/exim4/update-exim4.conf.conf + dpkg-reconfigure -u -fnoninteractive exim4-config fi - # light version does not have sasl auth support. - pi exim4-daemon-heavy spamassassin + # light version of exim does not have sasl auth support. + apt-get -y install --purge --auto-remove exim4-daemon-heavy spamassassin + + + ##### begin spamassassin config - ser enable spamassassin + systemctl enable spamassassin # per readme.debian - s sed -i '/^\s*CRON\s*=/d' /etc/default/spamassassin - s tee -a /etc/default/spamassassin <<>/etc/default/spamassassin # just noticed this in the config file, seems like a good idea. - s sed -i '/^\s*NICE\s*=/d' /etc/default/spamassassin - s tee -a /etc/default/spamassassin <<<'NICE="--nicelevel 15"' - ser reload spamassassin + sed -i '/^\s*NICE\s*=/d' /etc/default/spamassassin + e 'NICE="--nicelevel 15"' >>/etc/default/spamassassin + systemctl reload spamassassin - s dd of=/etc/systemd/system/spamddnsfix.service <<'EOF' + cat >/etc/systemd/system/spamddnsfix.service <<'EOF' [Unit] Description=spamd dns bug fix cronjob @@ -490,7 +504,7 @@ Description=spamd dns bug fix cronjob Type=oneshot ExecStart=/a/bin/distro-setup/spamd-dns-fix EOF - s dd of=/etc/systemd/system/spamddnsfix.timer <<'EOF' + cat >/etc/systemd/system/spamddnsfix.timer <<'EOF' [Unit] Description=run spamd bug fix script every 10 minutes @@ -504,13 +518,34 @@ OnUnitActiveSec=550 [Install] WantedBy=timers.target EOF - ser daemon-reload - sgo spamddnsfix.timer + systemctl daemon-reload + systemctl restart spamddnsfix.timer + systemctl enable spamddnsfix.timer # ##### end spamassassin config - gitslink # needed to install the execstart files below - s dd of=/etc/systemd/system/mailcert.service <<'EOF' + + f=/usr/local/bin/mail-cert-cron + cat >$f <<'EOF' +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +[[ $EUID == 0 ]] || exec sudo "$BASH_SOURCE" "$@" + +f=/a/bin/bash_unpublished/source-semi-priv +if [[ -e $f ]]; then + source $f +fi +if [[ $HOSTNAME == $MAIL_HOST ]]; then + local_mx=mail.iankelling.org + rsync_common="rsync -ogt --chown=root:Debian-exim --chmod=640 root@li:/p/c/machine_specific/li/webservercerts/$local_mx-" + ${rsync_common}chained.pem /etc/exim4/exim.crt + ${rsync_common}domain.key /etc/exim4/exim.key +fi +EOF + chmod 755 $f + + cat >/etc/systemd/system/mailcert.service <<'EOF' [Unit] Description=Mail cert rsync After=multi-user.target @@ -520,7 +555,7 @@ Type=oneshot ExecStart=/usr/local/bin/sysd-mail-once mailcert /usr/local/bin/mail-cert-cron EOF - s dd of=/etc/systemd/system/mailcert.timer <<'EOF' + cat >/etc/systemd/system/mailcert.timer <<'EOF' [Unit] Description=Run mail-cert once a day @@ -530,81 +565,221 @@ OnCalendar=daily [Install] WantedBy=timers.target EOF - ser daemon-reload - ser start mailcert - sgo mailcert.timer + systemctl daemon-reload + systemctl start mailcert + systemctl restart mailcert.timer + systemctl enable mailcert.timer + + + ####### begin dovecot setup ######## + if [[ $HOSTNAME == $MAIL_HOST ]]; then + # based on a little google and package search, just the dovecot + # packages we need instead of dovecot-common. + # + # dovecot-lmtpd is for exim to deliver to dovecot instead of maildir + # directly. The reason to do this is to use dovecot\'s sieve, which + # has extensions that allow it to be almost equivalent to exim\'s + # filter capabilities, some ways probably better, some worse, and + # sieve has the benefit of being supported in postfix and + # proprietary/weird environments, so there is more examples on the + # internet. I was torn about whether to do this or not, meh. + apt-get -y install --purge --auto-remove \ + dovecot-core dovecot-imapd dovecot-sieve dovecot-lmtpd + + # if we changed 90-sieve.conf and removed the active part of the + # sieve option, we wouldn\'t need this, but I\'d rather not modify a + # default config if not needed. This won\'t work as a symlink in /a/c + # unfortunately. + sudo -u $postmaster /a/exe/lnf -T sieve/main.sieve ~$postmaster/.dovecot.sieve + + sed -ri -f - /etc/dovecot/conf.d/10-mail.conf <<'EOF' +1i mail_location = maildir:/m/md:LAYOUT=fs:INBOX=/m/md/INBOX +/^\s*mail_location\s*=/d +EOF + + cat >/etc/dovecot/conf.d/20-lmtp.conf <<'EOF' +protocol lmtp { +#per https://wiki2.dovecot.org/Pigeonhole/Sieve/Configuration + mail_plugins = $mail_plugins sieve +# default was + #mail_plugins = $mail_plugins + +# For a normal setup with exim, we need something like this, which +# removes the domain part +# auth_username_format = %Ln +# +# or else # Exim says something like +# "LMTP error after RCPT ... 550 ... User doesn't exist someuser@somedomain" +# Dovecot verbose log says something like +# "auth-worker(9048): passwd(someuser@somedomain): unknown user" +# reference: http://wiki.dovecot.org/LMTP/Exim +# +# However, I use this to direct all mail to the same inbox. +# A normal way to do this, which I did at first is to have +# a router in exim almost at the end, eg 950, +#local_catchall: +# debug_print = "R: catchall for $local_part@$domain" +# driver = redirect +# domains = +local_domains +# data = ian +# based on +# http://blog.alteholz.eu/2015/04/exim4-and-catchall-email-address/ +# with superflous options removed. +# However, this causes the envelope to be rewritten, +# which makes filtering into mailboxes a little less robust or more complicated, +# so I've done it this way instead. it also requires +# modifying the local router in exim. + auth_username_format = ian +} + +EOF + + + cat >/etc/dovecot/local.conf <<'EOF' +# so I can use a different login that my shell login for mail. this is +# worth doing solely for the reason that if this login is compromised, +# it won't also compromise my shell password. +!include conf.d/auth-passwdfile.conf.ext + +# settings derived from wiki and 10-ssl.conf +ssl = required +ssl_cert = /etc/exim4/rcpt_local_acl <<'EOF' +# Only hosts we control send to mail.iankelling.org, so make sure +# they are all authed. +# Note, if we wanted authed senders for all domains, +# we could make this condition in acl_check_mail +deny + message = ian trusted domain recepient but no auth + !authenticated = * + domains = mail.iankelling.org +EOF + cat >/etc/exim4/data_local_acl <<'EOF' +# Except for the "condition =", this was +# a comment in the check_data acl. The comment about this not +# being suitable is mostly bs. The only thing related I found was to +# add the condition =, cuz spamassassin has problems with big +# messages and spammers don't bother with big messages, +# but I've increased the size from 10k +# suggested in official docs, and 100k in the wiki example because +# those docs are rather old and I see a 110k spam message +# pretty quickly looking through my spam folder. + warn + condition = ${if < {$message_size}{2000K}} + spam = Debian-exim:true + add_header = X-Spam_score: $spam_score\n\ + X-Spam_score_int: $spam_score_int\n\ + X-Spam_bar: $spam_bar\n\ + X-Spam_report: $spam_report + +EOF + cat >/etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' +# from 30_exim4-config_examples + +plain_server: +driver = plaintext +public_name = PLAIN +server_condition = "${if crypteq{$auth3}{${extract{1}{:}{${lookup{$auth2}lsearch{CONFDIR/passwd}{$value}{*:*}}}}}{1}{0}}" +server_set_id = $auth2 +server_prompts = : +.ifndef AUTH_SERVER_ALLOW_NOTLS_PASSWORDS +server_advertise_condition = ${if eq{$tls_in_cipher}{}{}{*}} +.endif +EOF + + cat >/etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF' +### router/900_exim4-config_local_user +################################# + +# This router matches local user mailboxes. If the router fails, the error +# message is "Unknown user". + +local_user: + debug_print = "R: local_user for $local_part@$domain" + driver = accept + domains = +local_domains +# ian: commented this, in conjunction with a dovecot lmtp +# change so I get mail for all users. +# check_local_user + local_parts = ! root + transport = LOCAL_DELIVERY + cannot_route_message = Unknown user +EOF + cat >/etc/exim4/conf.d/transport/30_exim4-config_dovecot_lmtp <<'EOF' +dovecot_lmtp: + driver = lmtp + socket = /var/run/dovecot/lmtp + #maximum number of deliveries per batch, default 1 + batch_max = 200 +EOF + # begin setup passwd.client f=/etc/exim4/passwd.client - s rm -f $f - s touch $f - s chmod 640 $f - s chown root:Debian-exim $f - # note: this will go away - s cat /etc/mailpass| while read -r domain port pass; do + install -m 640 -g Debian-exim /dev/null $f + cat /etc/mailpass| while read -r domain port pass; do # reference: exim4_passwd_client(5) - printf "%s:%s\n" "$domain" "$pass" | s tee -a $f >/dev/null + printf "%s:%s\n" "$domain" "$pass" >>$f done # end setup passwd.client # https://blog.dhampir.no/content/make-exim4-on-debian-respect-forward-and-etcaliases-when-using-a-smarthost # i only need .forwards, so just doing that one. cd /etc/exim4/conf.d/router - a=userforward - b=${a}_higher_priority - tmp=$(mktemp) - of=175_$b - # sed to make the router name unique - sed -r s/^\\S+:/$b:/ 600_exim4-config_$a | s dd of=$tmp 2>/dev/null - if ! diff -q $tmp $of &>/dev/null; then - s dd if=$tmp of=$of >/dev/null - fi - - - ser restart exim4 - + b=userforward_higher_priority + # replace the router name so it is unique + sed -r s/^\\S+:/$b:/ 600_exim4-config_userforward >175_$b + systemctl restart exim4 + +fi # end if exim4 + +# /etc/alias setup is debian specific, and +# exim config sets up an /etc/alias from root to the postmaster, which i +# config to ian, as long as there exists an entry for root, or there was +# no preexisting aliases file. based on the postinst file. postfix +# won't set up a root to $postmaster alias if it's already installed. +# Since postfix is not the greatest, just set it ourselves. +if [[ $postmaster != root ]]; then + sed -i --follow-symlinks -f - /etc/aliases <