From f95f9128ba77e77d41389810affd475581075246 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sat, 20 Jul 2019 11:30:23 -0400 Subject: [PATCH] various improvements mostly email --- .bash_profile | 3 + .profile | 5 +- brc | 11 +- distro-end | 263 ++++++++++++++++++--------------- email-date.py | 9 ++ mail-route | 2 +- mail-setup | 40 ++++- mailclean | 61 ++++++++ offlineimap-sync | 27 ---- subdir_files/sieve/lists.sieve | 1 + 10 files changed, 266 insertions(+), 156 deletions(-) create mode 100755 email-date.py create mode 100755 mailclean diff --git a/.bash_profile b/.bash_profile index d3a6a8f..cd422cb 100644 --- a/.bash_profile +++ b/.bash_profile @@ -15,3 +15,6 @@ # ensure no bad programs appending to this file will have an affect return 0 + + +export PATH="$HOME/.cargo/bin:$PATH" diff --git a/.profile b/.profile index 96ca117..c78d022 100644 --- a/.profile +++ b/.profile @@ -1,7 +1,6 @@ # see .bashrc_profile for commentary echo this is ~/.profile, either my .bashrc_profile is unavailable or this is posix mode or not bash -# Add RVM to PATH for scripting. Make sure this is the last PATH variable change. -export PATH="$PATH:$HOME/.rvm/bin" +return 0 -[[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" # Load RVM into a shell session *as a function* +export PATH="$HOME/.cargo/bin:$PATH" diff --git a/brc b/brc index 5f12a07..b232c9e 100644 --- a/brc +++ b/brc @@ -735,7 +735,7 @@ fdup() { local p fdroidcl update if fdroidcl search -u | grep ^org.fdroid.fdroid; then - fdroidcl upgrade org.fdroid.fdroid + fdroidcl install org.fdroid.fdroid sleep 5 fdroidcl update fi @@ -754,7 +754,7 @@ fdup() { done for p in ${!installed[@]}; do if ! ${updated[$p]:-true}; then - fdroidcl upgrade $p + fdroidcl install $p sleep 5 fi done @@ -991,6 +991,9 @@ grr() { # grep recursive grep --exclude-dir='*.emacs.d' --exclude-dir='*.git' -RiIP --color=auto "$@" fi } +rg() { + command rg -i -M 200 "$@" +} hstatus() { # do git status on published repos @@ -2290,8 +2293,12 @@ path_add /usr/local/go/bin # ARDUINO_PATH=/a/opt/Arduino/build/linux/work export ARDUINO_PATH=/a/opt/arduino-1.8.9 +# They want to be added to the start, but i think +# that should be avoided unless we really need it. path_add --end ~/.npm-global +path_add --end $HOME/.cargo/bin + # taken from default changes to bashrc and bash_profile path_add --end $HOME/.rvm/bin [[ -s "$HOME/.rvm/scripts/rvm" ]] && source "$HOME/.rvm/scripts/rvm" # Load RVM into a shell session *as a function* diff --git a/distro-end b/distro-end index b883fec..bc334d6 100755 --- a/distro-end +++ b/distro-end @@ -54,7 +54,7 @@ popularity-contest popularity-contest/participate boolean true EOF ########### begin section including li ################ -pi ${p3[@]} $($src/distro-pkgs) +pi ${p3[@]} #### desktop stuff case $codename_compat in @@ -193,6 +193,11 @@ s dpkg-reconfigure -u -fnoninteractive unattended-upgrades # Setup daily reboots, so all unattended upgrades go into affect # unattended upgrades happen at 6 am + rand(60 min). echo '20 7 * * * root /usr/local/bin/zelous-unattended-reboot' | s dd of=/etc/cron.d/unattended-upgrade-reboot +case $(debian-codename) in + flidas) + if [[ ! -e /usr/local/bin/checkrestart ]]; then + ;; +esac ##### end automatic upgrades #### # office is not exposed to internet yet @@ -609,13 +614,142 @@ esac ########### end section including li/lj ############### + +case $(debian-codename) in + # needed for debootstrap scripts for fai since fai requires debian + flidas) + curl http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg | s apt-key add - + s dd of=/etc/apt/preferences.d/flidas-xenial </dev/null <$t <$t </dev/null <$t <$t </dev/null; then # manual instructions # wget https://sv.gnu.org/people/viewgpg.php?user_id=15145 -qO - | gpg --import - # echo is to get past prompt - echo | sudo -E HOME=/home/iank bash guix-install.sh + yes | sudo -E HOME=$HOME bash guix-install.sh || [[ $? == 141 ]] guix install glibc-utf8-locales guix package --install guile fi +# install rust +curl https://sh.rustup.rs -sSf | bash -s -- -y +cargo install ripgrep + #### tor case $distro in # based on @@ -1762,16 +1787,12 @@ make sure to reload the firewall to load the persistent configuration EOF pi nfs-utils - sgo nfs-server ;; debian|trisquel|ubuntu) - pi nfs-server + pi-nostart nfs-server ;; arch) pi nfs-utils || pending_reboot=true - sgo rpcbind - # this failed until I rebooted - sgo nfs-server ;; esac diff --git a/email-date.py b/email-date.py new file mode 100755 index 0000000..f090336 --- /dev/null +++ b/email-date.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +from email.parser import BytesParser, Parser +from email.policy import default +from pathlib import Path +from email.utils import mktime_tz, parsedate_tz +import sys +with open(sys.argv[1], 'rb') as fp: + headers = BytesParser(policy=default).parse(fp) +print(mktime_tz(parsedate_tz(headers['date']))) diff --git a/mail-route b/mail-route index 4289720..1dea430 100755 --- a/mail-route +++ b/mail-route @@ -118,7 +118,7 @@ modify() { for port in 25 143 587; do # smtp and imap. iptcommon="OUTPUT -m tcp -p tcp -m multiport --ports $port -j MARK --set-mark" iptmod iptables -t mangle $iptables_op $iptcommon 0x1 - iptmod iptables -t mangle $iptables_op $iptcommon 0x0 -d 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 + iptmod iptables -t mangle $iptables_op $iptcommon 0x0 -d 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.0/8 # note, we could have used a custom chain and returned instead of setting the mark again. # in case anyone was ever curious, the inverse of private ips is: #0.0.0.0/5,8.0.0.0/7,11.0.0.0/8,12.0.0.0/6,16.0.0.0/4,32.0.0.0/3,64.0.0.0/2,128.0.0.0/3,160.0.0.0/5,168.0.0.0/6,172.0.0.0/12,172.32.0.0/11,172.64.0.0/10,172.128.0.0/9,173.0.0.0/8,174.0.0.0/7,176.0.0.0/4,192.0.0.0/9,192.128.0.0/11,192.160.0.0/13,192.169.0.0/16,192.170.0.0/15,192.172.0.0/14,192.176.0.0/12,192.192.0.0/10,193.0.0.0/8,194.0.0.0/7,196.0.0.0/6,200.0.0.0/5,208.0.0.0/4,224.0.0.0/3 done diff --git a/mail-setup b/mail-setup index 0c3f0bd..9b30579 100755 --- a/mail-setup +++ b/mail-setup @@ -58,6 +58,15 @@ if ! exim && ! postfix; then fi +####### instructions for icedove ##### +# Incoming mail server: mail.iankelling.org, port 143, username iank, connection security starttls, authentication method normal password +# we could also just use 127.0.0.1 with no ssl, but todo: disable that in dovecot, so mail is secure from local programs. +# +# hamburger -> preferences -> preferences -> advanced tab -> config editor button -> security.ssl.enable_ocsp_must_staple = false +# background: ovecot does not yet have ocsp stapling support +# reference: https://community.letsencrypt.org/t/simple-guide-using-lets-encrypt-ssl-certs-with-dovecot/2921 +####### + ####### begin perstent password instructions ###### # # exim passwords: @@ -70,12 +79,12 @@ fi # apg -m 50 -x 70 -n 1 -a 1 -M CLN >$f # s sed -i "/^$user:/d" /p/c/filesystem/etc/exim4/passwd # echo "$user:$(mkpasswd -m sha-512 -s <$f)" >>/p/c/filesystem/etc/exim4/passwd -# echo "mail.iankelling.org $user $(<$f)" >> /p/c/machine_specific/$user/filesystem/etc/mailpass +# echo "mail.iankelling.org 587 $user:$(<$f)" >> /p/c/machine_specific/$user/filesystem/etc/mailpass # # then run this script, or part of it which uses /etc/mailpass # # dovecot password, i just need 1 as I\'m the only user # mkdir /p/c/filesystem/etc/dovecot -# echo "ian:$(doveadm pw -s ssha256)::::::" >/p/c/filesystem/etc/dovecot/users +# echo "iank:$(doveadm pw -s ssha256)::::::" >/p/c/filesystem/etc/dovecot/users # conflink @@ -342,6 +351,29 @@ User=$u Type=oneshot ExecStart=/a/bin/log-quiet/sysd-mail-once offlineimap-sync /a/bin/distro-setup/offlineimap-sync EOF + + cat >/etc/systemd/system/mailclean.timer <<'EOF' +[Unit] +Description=Run mailclean daily + +[Timer] +OnCalendar=monthly + +[Install] +WantedBy=timers.target +EOF + + cat >/etc/systemd/system/mailclean.service </dev/null ||: systemctl stop offlineimapsync.timer &>/dev/null ||: + systemctl disable mailclean.timer &>/dev/null ||: + systemctl stop mailclean.timer &>/dev/null ||: systemctl disable $vpn_ser@mail systemctl stop $vpn_ser@mail systemctl disable dovecot ||: diff --git a/mailclean b/mailclean new file mode 100755 index 0000000..753529e --- /dev/null +++ b/mailclean @@ -0,0 +1,61 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR +x="$(readlink -f -- "$BASH_SOURCE")"; scriptdir=${x%/*} + + + +# find but ignore directories which dont exist, assuming first args are directories +# and a following arg starts with - +myfind() { + dirs=() + for d; do + if [[ $d == -* ]]; then + # past dirs, onto options + break + fi + shift + if [[ -e $d ]]; then + dirs+=($d) + fi + done + if [[ $dirs ]]; then + find ${dirs[@]} $@ + fi +} + +# qemu-devel is our biggest list by far, so occasionally +# I want to hop into conversations about our mailing +# systems there, but I don't need many old messages. +myfind /nocow/user/fsfmd/l/qemu-devel/new -type f -mtime +14 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/{sec,Spam,Drafts,{rtcc,sysadmin,l/outreachy-mentors}/new} -type f -mtime +100 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/log -type f -mtime +300 -execdir rm -- '{}' + +myfind /nocow/user/fsfmd/dmarc -type f -mtime +14 -execdir rm -- '{}' + + + + +shopt -s nullglob +shopt -s extglob + +now=$(date +%s) +cd /m/md +for d in ./!(*archive|Drafts)/*(cur|new) ./l/!(*archive)/*(cur|new); do + madearchive=false + leafdir=${d##*/} + md=${d%/*}; md=${md##*/} + archive=${d%/*/*}/${md}-myarchive/$leafdir + archivebase=archive=${d%/*/*}/${md}-myarchive + for f in $d/1*; do + date=$($scriptdir/email-date.py $f) || echo $f + [[ $date ]] || continue + if (( date < now - 60*60*24*400 )); then + if ! $madearchive; then + mkdir -p ${archivebase}{cur,tmp,new} + madearchive=true + fi + mv $f $archive + echo mv $f $archive + fi + done +done diff --git a/offlineimap-sync b/offlineimap-sync index 3c75258..1c07702 100755 --- a/offlineimap-sync +++ b/offlineimap-sync @@ -52,33 +52,6 @@ if $found_files; then #mu index &>/dev/null ||: fi -# find but ignore directories which dont exist, assuming first args are directories -# and a following arg starts with - -myfind() { - dirs=() - for d; do - if [[ $d == -* ]]; then - # past dirs, onto options - break - fi - shift - if [[ -e $d ]]; then - dirs+=($d) - fi - done - if [[ $dirs ]]; then - find ${dirs[@]} $@ - fi -} - -# qemu-devel is our biggest list by far, so occasionally -# I want to hop into conversations about our mailing -# systems there, but I don't need many old messages. -myfind /nocow/user/fsfmd/l/qemu-devel/new -type f -mtime +14 -execdir rm -- '{}' + -myfind /nocow/user/fsfmd/{sec,Spam,Drafts,{rtcc,sysadmin,l/outreachy-mentors}/new} -type f -mtime +100 -execdir rm -- '{}' + -myfind /nocow/user/fsfmd/log -type f -mtime +300 -execdir rm -- '{}' + -myfind /nocow/user/fsfmd/dmarc -type f -mtime +14 -execdir rm -- '{}' + - # delete based on http://deflexion.com/2006/05/imap-way-of-deleting-message sieve-filter -eW -o mail_location=maildir:/nocow/user/fsfmd:LAYOUT=fs:INBOX=/nocow/user/fsfmd/INBOX ~/sieve/fsf.sieve INBOX delete &>>/tmp/fsfsieve.log diff --git a/subdir_files/sieve/lists.sieve b/subdir_files/sieve/lists.sieve index 4e17c09..6cc19db 100644 --- a/subdir_files/sieve/lists.sieve +++ b/subdir_files/sieve/lists.sieve @@ -59,6 +59,7 @@ if anyof ( header :contains "list-id" "", header :contains "list-id" "", header :contains "list-id" "", + header :contains "list-id" "", header :contains "list-id" "", header :contains "list-id" "") { if header :regex "list-id" "<([a-z_0-9-]+)[.@]" { -- 2.30.2