own mailserver, lots of improvements

[distro-setup] / mail-setup

#!/bin/bash -l
# Copyright (C) 2016 Ian Kelling

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eE -o pipefail
trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR

####### begin perstent password instructions ######
# # exim passwords:
# # for hosts which have all private files I just use the same user
# # for other hosts, each one get's their own password.
# # for generating secure pass, and storing for server too:
# # user=USUALLY_SAME_AS_HOSTNAME
# user=li
# f=$(mktemp)
# apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f
# s sed -i "/^$user:/d" /p/c/filesystem/etc/exim4/passwd
# echo "$user:$(mkpasswd -m sha-512 -s <$f)" >>/p/c/filesystem/etc/exim4/passwd
# echo "mail.iankelling.org:$user:$(<$f)" >> /p/c/machine_specific/$user/filesystem/etc/mailpass
# # then run this script, or part of it which uses /etc/mailpass

# # dovecot password, i just need 1 as I'm the only user
# mkdir /p/c/filesystem/etc/dovecot
# echo "ian:$(doveadm pw -s ssha256)::::::" >/p/c/filesystem/etc/dovecot/users
# conflink



# # for ad-hoc testing of some random new host sending mail:
# user=li # client host username & hostname
# f=$(mktemp)
# apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f
# s sed -i "/^$user:/d" /etc/exim4/passwd
# echo "$user:$(mkpasswd -m sha-512 -s <$f)" | s tee -a /etc/exim4/passwd
# echo "mail.iankelling.org:$user:$(<$f)" | ssh root@$user dd of=/etc/exim4/passwd.client
####### end perstent password instructions ######


####### begin persistent dkim/dns instructions #########
# # Remove 1 level of comments in this section, set the domain var
# # for the domain you are setting up, then run this and copy dns settings
# # into dns.
# domain=iankelling.org
# c /p/c/filesystem/etc/exim4
# # this has several bugs addressed in comments, but it was helpful
# # https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4

# openssl genrsa -out $domain-private.pem 2048 -outform PEM
# openssl rsa -in $domain-private.pem -out $domain.pem -pubout -outform PEM
# # selector is needed for having multiple keys for one domain.
# # I dun do that, so just use a static one: li
# echo "txt record name: li._domainkey.$domain"
# # Debadmin page does not have v=, fastmail does, and this
# # says it's recommended in 3.6.1, default is DKIM1 anyways.
# # https://www.ietf.org/rfc/rfc6376.txt
# # Join and print all but first and last line.
# # last line: swap hold & pattern, remove newlines, print.
# # lines 2+: append to hold space
# echo "txt record contents:"
# echo "v=DKIM1; k=rsa; p=$(sed -n '${x;s/\n//gp};2,$H' $domain.pem)"
# chmod 644 $domain.pem
# chmod 640 $domain-private.pem
# # in conflink, we chown these to group debian
# conflink
# # selector was also put into /etc/exim4/conf.d/main/000_localmacros,
# # via the mail-setup scripts

# # 2017-02 dmarc policies:
# # yahoo: p=reject, hotmail: p=none, gmail: p=none, fastmail none for legacy reasons
# # gmail will be changing to p=reject, which is expected to cause problems
# # with a few old mailing lists, copying theirs for now.
# echo "dmarc dns, name: _dmarc value: v=DMARC1; p=none; rua=mailto:mailauth-reports@$domain"

# # 2017-02 spf policies:
# # google ~all, hotmail -all, yahoo: ?all, fastmail ?all
# # i include fastmail's settings, per their instructions,
# # and follow their policy. In mail in a box, or similar instructions,
# # I've seen recommended to not use a restrictive policy.
# echo "spf dns: name is empty, value: v=spf1 a include:spf.messagingengine.com ?all"

# # to check if dns has updated, you do
# host -a mesmtp._domainkey.$domain

# # mx records,
# # setting it to iankelling.org would work the same, but this
# # is more flexible, I could change where mail.iankelling.org pointed.
# cat <<'EOF'
# mx records, 2 records each, for * and empty domain
# pri 10 mail.iankelling.org
# pri 20 in1-smtp.messagingengine.com
# pri 30 in2-smtp.messagingengine.com
# EOF
####### end  persistent dkim instructions #########


# misc exim notes:
# useful exim docs:
# /usr/share/doc/exim4-base/README.Debian.gz
# /usr/share/doc/exim4-base/spec.txt.gz

# routers, transports, and authenticators are sections, and you define
# driver instances in those sections, and the manual calls them driver
# types but there is also a more specific "type" of driver, which is specified
# with the driver = some_module setting in the driver.

# the driver option must precede and private options (options that are
# specific to that driver), so follow example of putting it at beginning.

# The full list of option settings for any particular driver instance,
# including all the defaulted values, can be extracted by making use of
# the -bP command line option.

# exim clear out message queue. as root:
# adapted from somewhere on stackoverflow.
# ser stop exim4; sleep 1; exim -bp | exiqgrep -i | xargs exim -Mrm; ser start exim4

# fastmail has changed their smtp server, but the old one still works,
# I see no reason to bother changing.
# New one is smtp.fastmail.com

# test delivery & rewrite settings:
#exim4 -bt ian@localhost


type=$1
postfix() { [[ $type == postfix ]]; }
exim() { [[ $type == exim4 ]]; }
if ! exim && ! postfix; then
    echo "$1: error: expected exim4 or postfix as first arg"
    exit 1
fi


local_mx=mail.iankelling.org

host=$local_mx
relayhost="[$host]:25" # postfix
smarthost="$host::25" # exim


# this was for when I used the exim config type
# "mail sent by smarthost; received via SMTP or fetchmail"
# if [[ $HOSTNAME == $MAIL_HOST ]]; then
#     host=mail.messagingengine.com
#     relayhost="[$host]:587" # postfix
#     smarthost="$host::587" # exim
# fi

forward=ian@$local_mx


# background: This also works instead of ~/.forward
# s sed -i --follow-symlinks '/^root/d' /etc/aliases ||:
#echo "root: $HOSTNAME@$SOME_DOMAIN" | s tee -a /etc/aliases
# this can't be a symlink and has permission restrictions
# it might work in /etc/aliases, but this seems more proper.
e $forward > ~/.forward
e $forward | s tee /root/.forward
# exim log complains about 664 permissions.
s chmod 644 ~/.forward /root/.forward


# offlineimap uses this too, it is much easier to use one location than to
# condition it's config and postfix's config
case $distro in
    fedora) s lnf -T ca-certificates.crt /etc/ssl/ca-bundle.trust.crt ;;
    *) :
esac

if postfix; then
    # dunno why, but debian installed postfix with builddep emacs
    # but I will just explicitly install it here since
    # I use it for sending mail in emacs.
    if isdeb; then
        s debconf-set-selections <<EOF
postfix postfix/main_mailer_type select Satellite system
postfix postfix/mailname string $HOSTNAME
postfix postfix/relayhost string $relayhost
EOF

        pi postfix
    else
        pi postfix
        # Settings from reading the output when installing on debian,
        # then seeing which were different in a default install on arch.
        # I assume the same works for fedora.
        postconfin <<EOF
mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
mailbox_size_limit = 0
relayhost = $relayhost
inet_interfaces = loopback-only
EOF

        s systemctl enable postfix
        s systemctl start postfix
    fi
    # i'm assuming mail just won't work on systems without the sasl_passwd.
    postconfin <<'EOF'
smtp_sasl_auth_enable = yes
smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd
smtp_sasl_security_options = noanonymous
smtp_tls_security_level = secure
message_size_limit =  20480000
smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt
inet_protocols = ipv4
EOF
    # msg_size_limit: I ran into a log file not sending cuz of size. double from 10 to 20 meg limit
    # inet_protocols: without this, postfix tries an ipv6 lookup then gives
    # up and fails. snippet from syslog: type=AAAA: Host not found, try again


    f=/etc/postfix/sasl_passwd
    s rm -f $f
    s touch $f
    s chmod 600 $f
    s cat /etc/mailpass| while read -r domain port pass; do
        # format: domain port user:pass
        # mailpass is just a name i made up, since postfix and
        # exim both use a slightly crazy format to translate to
        # each other, it\'s easier to use my own format.
        printf "[%s]:%s %s" "$domain" "$port" "${pass/@/#}" | s tee -a $f >/dev/null
    done
    s postmap hash:/etc/postfix/sasl_passwd
    s service postfix reload

else # exim. has debian specific stuff for now


    # wording of question from dpkg-reconfigure exim4-config
    # 1. internet site; mail is sent and received directly using SMTP
    # 2. mail sent by smarthost; received via SMTP or fetchmail
    # 3. mail sent by smarthost; no local mail
    # 4. local delivery only; not on a network
    # 5. no configuration at this time
    #
    # Note, I have used option 2 in the past for receiving mail
    # from lan hosts, sending external mail via another smtp server.
    #
    # Note, other than configtype, we could set all the options in
    # both types of configs without harm, they would either be
    # ignored or be disabled by other settings, but the default
    # local_interfaces definitely makes things more secure.

    # most of these settings get translated into settings
    # in /etc/exim4/update-exim4.conf.conf
    # mailname setting sets /etc/mailname

    s debconf-set-selections <<EOF
exim4-config exim4/use_split_config boolean true
EOF

    source /a/bin/bash_unpublished/source-semi-priv
    exim_main_dir=/etc/exim4/conf.d/main
    s mkdir -p exim_main_dir
    if [[ $HOSTNAME == $MAIL_HOST ]]; then
        # afaik, these will get ignored, routing to my own machine, but rm
        # them to make me feel better.
        s rm -f ~/.forward /root/.forward

        s debconf-set-selections <<EOF
# Mail Server configuration
# -------------------------

# Please select the mail server configuration type that best meets your needs.

# Systems with dynamic IP addresses, including dialup systems, should generally be
# configured to send outgoing mail to another machine, called a 'smarthost' for
# delivery because many receiving systems on the Internet block incoming mail from
# dynamic IP addresses as spam protection.

# A system with a dynamic IP address can receive its own mail, or local delivery can be
# disabled entirely (except mail for root and postmaster).

#   1. internet site; mail is sent and received directly using SMTP
#   2. mail sent by smarthost; received via SMTP or fetchmail
#   3. mail sent by smarthost; no local mail
#   4. local delivery only; not on a network
#   5. no configuration at this time

# General type of mail configuration: 1
exim4-config exim4/dc_eximconfig_configtype select internet site; mail is sent and received directly using SMTP



# The 'mail name' is the domain name used to 'qualify' mail addresses without a domain
# name.

# This name will also be used by other programs. It should be the single, fully
# qualified domain name (FQDN).

# Thus, if a mail address on the local host is foo@example.org, the correct value for
# this option would be example.org.

# This name won't appear on From: lines of outgoing messages if rewriting is enabled.

# System mail name:
exim4-config exim4/mailname string li.iankelling.org




# Please enter a semicolon-separated list of recipient domains for which this machine
# should consider itself the final destination. These domains are commonly called
# 'local domains'. The local hostname (treetowl.lan) and 'localhost' are always added
# to the list given here.

# By default all local domains will be treated identically. If both a.example and
# b.example are local domains, acc@a.example and acc@b.example will be delivered to the
# same final destination. If different domain names should be treated differently, it
# is necessary to edit the config files afterwards.

# Other destinations for which mail is accepted:
# iank.bid is for testing
# mail.iankelling.org is for machines i own
exim4-config exim4/dc_other_hostnames string mail.iankelling.org;iank.bid;iankelling.org;zroe.org




# Please enter a semicolon-separated list of IP addresses. The Exim SMTP listener
# daemon will listen on all IP addresses listed here.

# An empty value will cause Exim to listen for connections on all available network
# interfaces.

# If this system only receives mail directly from local services (and not from other
# hosts), it is suggested to prohibit external connections to the local Exim daemon.
# Such services include e-mail programs (MUAs) which talk to localhost only as well as
# fetchmail. External connections are impossible when 127.0.0.1 is entered here, as
# this will disable listening on public network interfaces.

# IP-addresses to listen on for incoming SMTP connections:
exim4-config exim4/dc_local_interfaces  string




# Mail for the 'postmaster', 'root', and other system accounts needs to be redirected
# to the user account of the actual system administrator.

# If this value is left empty, such mail will be saved in /var/mail/mail, which is not
# recommended.

# Note that postmaster's mail should be read on the system to which it is directed,
# rather than being forwarded elsewhere, so (at least one of) the users listed here
# should not redirect their mail off this machine. A 'real-' prefix can be used to
# force local delivery.

# Multiple user names need to be separated by spaces.

# Root and postmaster mail recipient:
exim4-config exim4/dc_postmaster string ian



# Exim is able to store locally delivered email in different formats. The most commonly
# used ones are mbox and Maildir. mbox uses a single file for the complete mail folder
# stored in /var/mail/. With Maildir format every single message is stored in a
# separate file in ~/Maildir/.

# Please note that most mail tools in Debian expect the local delivery method to be
# mbox in their default.

#   1. mbox format in /var/mail/  2. Maildir format in home directory

# Delivery method for local mail: 2
exim4-config exim4/dc_localdelivery select Maildir format in home directory
EOF
        # MAIN_HARDCODE_PRIMARY_HOSTNAME might mess up the
        # smarthost config type, not sure. all other settings
        # would be unused in that config type.
        s dd of=$exim_main_dir/000_localmacros 2>/dev/null <<'EOF'
MAIN_TLS_ENABLE = true

DKIM_CANON = relaxed
DKIM_SELECTOR = li

# from comments in
# https://debian-administration.org/article/718/DKIM-signing_outgoing_mail_with_exim4

# The file is based on the outgoing domain-name in the from-header.
DKIM_DOMAIN = ${lc:${domain:$h_from:}}
# sign if key exists
DKIM_PRIVATE_KEY= ${if exists{/etc/exim4/${dkim_domain}-private.pem} {/etc/exim4/${dkim_domain}-private.pem}}


# failing message on mail-tester.com:
# We check if there is a server (A Record) behind your hostname treetowl.
# You may want to publish a DNS record (A type) for the hostname treetowl or use a different hostname in your mail software
# https://serverfault.com/questions/46545/how-do-i-change-exim4s-primary-hostname-on-a-debian-box
# and this one seemed appropriate from grepping config
MAIN_HARDCODE_PRIMARY_HOSTNAME = li.iankelling.org

# normally empty, I set this so I can set the envelope address
# when doing mail redelivery to invoke filters
MAIN_TRUSTED_GROUPS = ian

# disabled, didn't finished configuring
#LOCAL_DELIVERY = dovecot_lmtp

CHECK_RCPT_LOCAL_ACL_FILE = /etc/exim4/rcpt_local_acl
EOF


        s dd of=/etc/systemd/system/offlineimapsync.timer <<'EOF'
[Unit]
Description=Run offlineimap-sync once every 5 mins

[Timer]
OnCalendar=*:0/5

[Install]
WantedBy=timers.target
EOF

        s dd of=/etc/systemd/system/offlineimapsync.service <<'EOF'
[Unit]
Description=Offlineimap sync
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/a/bin/log-quiet/sysd-mail-once offlineimap-sync /a/bin/distro-setup/offlineimap-sync
EOF
        s systemctl daemon-reload
        s systemctl enable offlineimapsync.timer
        s systemctl start offlineimapsync.timer

    else # $HOSTNAME != $MAIL_HOST
        s systemctl disable offlineimapsync.timer &>/dev/null ||:
        s systemctl stop offlineimapsync.timer &>/dev/null ||:
        #
        #
        # would only exist because I wrote it i the previous condition,
        # it's not part of exim
        s rm -f $exim_main_dir/000_localmacros
        s debconf-set-selections <<EOF
exim4-config exim4/dc_eximconfig_configtype select mail sent by smarthost; no local mail
exim4-config exim4/dc_smarthost string $smarthost
EOF
    fi

    # if we already have it installed, need to reconfigure, without being prompted
    if ! dpkg -s exim4-config &>/dev/null; then
        s dpkg-reconfigure -u -pcritical exim4-config
    fi
    # light version does not have sasl auth support.
    pi exim4-daemon-heavy spamassassin

    ##### begin spamassassin config
    ser enable spamassassin
    # per readme.debian
    s sed -i '/^\s*CRON\s*=/d' /etc/default/spamassassin
    s tee -a /etc/default/spamassassin <<<CRON=1
    # just noticed this in the config file, seems like a good idea.
    s sed -i '/^\s*NICE\s*=/d' /etc/default/spamassassin
    s tee -a /etc/default/spamassassin <<<'NICE="--nicelevel 15"'

    s dd of=/etc/systemd/system/spamddnsfix.service <<'EOF'
[Unit]
Description=spamd dns bug fix cronjob

[Service]
Type=oneshot
ExecStart=/a/bin/distro-setup/spamd-dns-fix
EOF
    s dd of=/etc/systemd/system/spamddnsfix.timer <<'EOF'
[Unit]
Description=run spamd bug fix script every 10 minutes

[Timer]
# the script looks back 9 minutes into the journal,
# it takes a second to run,
# so lets run every 9 minutes and 10 seconds.
OnActiveSec=550

[Install]
WantedBy=timers.target
EOF
    ser daemon-reload
    sgo spamddnsfix.timer
    #
    #####   end spamassassin config

    gitslink # needed to install the execstart files below
    s dd of=/etc/systemd/system/mailcert.service <<'EOF'
[Unit]
Description=Mail cert rsync
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/sysd-mail-once mailcert /usr/local/bin/mail-cert-cron
EOF

    s dd of=/etc/systemd/system/mailcert.timer <<'EOF'
[Unit]
Description=Run mail-cert once a day

[Timer]
OnCalendar=daily

[Install]
WantedBy=timers.target
EOF
    ser daemon-reload
    ser start mailcert
    sgo mailcert.timer


    # based on a little google and package search, just the dovecot
    # packages we need instead of dovecot-common.
    #
    # dovecot-lmtpd is for exim to deliver to dovecot instead of maildir
    # directly.  The reason to do this is to use dovecot's sieve, which
    # has extensions that allow it to be almost equivalent to exim's
    # filter capabilities, some ways probably better, some worse, and
    # sieve has the benefit of being supported in postfix and
    # proprietary/weird environments, so there is more examples on the
    # internet. I was torn about whether to do this or not, meh.
    pi dovecot-core dovecot-imapd dovecot-sieve dovecot-lmtpd

    # if we changed 90-sieve.conf and removed the active part of the
    # sieve option, we wouldn't need this, but I'd rather not modify a
    # default config if not needed. This won't work as a symlink in /a/c
    # unfortunately.
    lnf -T sieve/main.sieve ~/.dovecot.sieve

    # we have a few config files which installing exim/dovecot overwrites,
    # and might as well have this before reading /etc/mailpass,
    # which this sets up too.
    conflink

    # begin setup passwd.client
    f=/etc/exim4/passwd.client
    s rm -f $f
    s touch $f
    s chmod 640 $f
    s chown root:Debian-exim $f
    # note: this will go away
    s cat /etc/mailpass| while read -r domain port pass; do
        # reference: exim4_passwd_client(5)
        printf "%s:%s\n" "$domain" "$pass" | s tee -a $f >/dev/null
    done
    # end setup passwd.client

    # https://blog.dhampir.no/content/make-exim4-on-debian-respect-forward-and-etcaliases-when-using-a-smarthost
    # i only need .forwards, so just doing that one.
    cd /etc/exim4/conf.d/router
    a=userforward
    b=${a}_higher_priority
    tmp=$(mktemp)
    of=175_$b
    # sed to make the router name unique
    sed -r s/^\\S+:/$b:/ 600_exim4-config_$a | s dd of=$tmp 2>/dev/null
    if ! diff -q $tmp $of &>/dev/null; then
        s dd if=$tmp of=$of >/dev/null
    fi


    ser restart exim4

    fi



    # linode image has a root alias. completely useless, remove it.
    sudo sed -i '/^root:/d' /etc/aliases

    s newaliases


    # based on http://www.postfix.org/qmgr.8.html and my notes in gnus
    dir=/nocow/$type
    sdir=/var/spool/$type
    if [[ $(readlink -f $sdir) != $dir ]]; then
        ser stop $type
        if [[ ! -e $dir && -d $sdir ]]; then
            s mv $sdir $dir
        fi
        s lnf -T $dir $sdir
    fi

    sgo $type


    # if I wanted the from address to be renamed and sent to a different address,
    # echo "sdx@localhost development@localhost" | sudo dd of=/etc/postfix/recipient_canonical
    # sudo postmap hash:/etc/postfix/recipient_canonical
    # sudo service postfix reload