mainly new feature to intentionally delay sending email

diff --git a/brc b/brc
index b78e08ff865d1779f3a3065dadb483f4496821c7..4e02122cccc6f72ae3d3acf9b882783b692ec455 100644 (file)
--- a/brc
+++ b/brc
@@ -712,7 +712,7 @@ for field in {1..20}; do
 done
 # h1 = head -n1
 for num in {1..9}; do
-  eval h$num"() { head -n$num; }"
+  eval h$num"() { head -n$num || [[ \$? == 141 ]]; }"
 done
 
 
@@ -1590,6 +1590,8 @@ pst() {
   pstree -apnA
 }
 
+# journalctl with times in the format the --since= and --until= options accept
+jrt() { journalctl -e -n100000 -o short-full "$@"; }
 jr() { journalctl -e -n100000 "$@" ; }
 jrf() { journalctl -n1000 -f "$@" ; }
 jru() {
@@ -1754,6 +1756,14 @@ nags() {
   fi
   /usr/bin/nagstamon &
 }
+prof() {
+  if ! pgrep -f /usr/lib/notification-daemon/notification-daemon >/dev/null; then
+    /usr/lib/notification-daemon/notification-daemon &
+  fi
+  profanity "$@"
+}
+
+
 
 nmt() {
   # cant use s because sudo -i doesnt work for passwordless sudo command

diff --git a/brc2 b/brc2
index 23cfd89aa14b5f1342229ca97acbf9abcf4b9eca..1974c580b867ea432b9cff6b42664080888f9087 100644 (file)
--- a/brc2
+++ b/brc2
@@ -289,14 +289,41 @@ tback() {
 # sshfs example:
 # s sshfs bu@$host:/bu/home/md /bu/mnt -o reconnect,ServerAliveInterval=20,ServerAliveCountMax=30 -o allow_other
 
+edelayoff() {
+  echo all >/etc/exim4/no-delay-eximids
+}
+edelayon() {
+  echo >/etc/exim4/no-delay-eximids
+}
+
 eqgo() {
-  local -a array tmpstr
+  local -a array tmpstr delayon
+  delayon=true
+  if grep -qFx all /etc/exim4/no-delay-eximids; then
+    delayon=false
+  fi
+  if $delayon; then
+    echo all >/etc/exim4/no-delay-eximids
+  fi
   tmpstr=$(exiqgrep -i -r.\*)
   mapfile -t array <<<"$tmpstr"
   enn -M "${array[@]}"
+  if $delayon; then
+    echo >/etc/exim4/no-delay-eximids
+  fi
 }
 eqgo1() {
-  enn -M "$(exipick -i -r.\*|h1)"
+  local eid
+  eid="$(exipick -i -r.\*|h1)"
+  sed -n "/^all$/p;\$a $eid" /etc/exim4/no-delay-eximids
+  enn -M "$eid"
+}
+ennm() {
+  local eid
+  for eid; do
+    printf "%s\n" "$eid" >>/etc/exim4/no-delay-eximids
+  done
+  enn -M "$@"
 }
 
 
@@ -2751,11 +2778,21 @@ myirc() {
   # use * instead of -r since that does sorted order
   ssh root@iankelling.org "for f in ${d[*]}; do cd \$f/#$1; grep '\<iank.*' *; done" | cut --complement -c12-16
 }
+
+
+# The way pidgin logs with xmpp (maybe related to running cheogram too)
+# is that there are sometimes duplicates, and sometimes the a log file
+# is for a specific day yet logs messages for subsequent days, and the
+# only way to realize that is to notice that the timestamps rolled over
+# into a new day, you can't see it in isolation. So, basically, pidgin
+# logs are really annoying to read a grep of my messages to find the
+# date and time I said when I started and stopped working, so I'm trying
+# out a new client: profanity.
 mypidgin() {
   c /p/c/.purple/logs/jabber/iank@fsf.org/office@conference.fsf.org.chat
   for x in *.html; do html2text -o ${x%.html}.txt $x; done;
   # shellcheck disable=SC2016 # false positive on ${
-  grep -A1 ') iank:' ./*.txt \
+  grep -A1 ') iank:' *.txt \
     | sed -r 's/^(.{10})[^ ]*\.txt:\(?([^ ]*)[[:space:]](..). iank:/\1_\2_\3/
 s/^[^ ]*\.txt-//
 /^--$/d
@@ -3292,7 +3329,7 @@ enn() {
     s $ecmd "$@"
     return
   fi
-  pid=$(pgrep -f "/usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf"|h1)
+  pid=$(pgrep -f "/usr/sbin/exim4 -bd -q10m -C /etc/exim4/my.conf"|h1)
   m s nsenter -t $pid -n -m $ecmd "$@"
 }
 
@@ -3371,7 +3408,7 @@ mailnnbash() {
 
 eximbash() {
   local pid
-  pid=$(pgrep -f "/usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf"|h1)
+  pid=$(pgrep -f "/usr/sbin/exim4 -bd -q10m -C /etc/exim4/my.conf"|h1)
   if [[ ! $pid ]]; then
     echo "eximbash: failed to find exim pid. systemctl -n 30 status exim4:"
     systemctl status exim4
@@ -3630,8 +3667,8 @@ rem() {
   paths="/p/c /b"
   find $paths -not \( -name .svn -prune -o -name .git -prune \
        -o -name .hg -prune -o -name .editor-backups -prune \
-       -o -name .undo-tree-history -prune \) 2>/dev/null | grep -iP --color=auto "$*" ||:
-  rgv -m 5 "$*" $paths /a/t.org /p/w.org /a/work.org ||:
+       -o -name .undo-tree-history -prune \) 2>/dev/null | grep -iP --color=auto -- "$*" ||:
+  rgv -m 5 -- "$*" $paths /a/t.org /p/w.org /a/work.org ||:
 }
 
 # setup:
@@ -3827,6 +3864,14 @@ amoffice() {
 amls() {
   amall silence query "$@"
 }
+# amtool silence add
+amsa() {
+  amall silence add "$@"
+}
+# amtool silence force
+amsf() {
+  amall silence add x!="1"
+}
 amrmall() {
   # note: not sure if quoting of this arg is correct
   amfsf silence expire "$(amfsf silence query -q)"
@@ -3842,12 +3887,12 @@ youtube-dl-update() {
 # https://github.com/yt-dlp/yt-dlp/wiki/Installation
 yt-dlp-update() {
   sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
-sudo chmod a+rx /usr/local/bin/yt-dlp  # Make executable
-  }
+  sudo chmod a+rx /usr/local/bin/yt-dlp  # Make executable
+}
 
 mpvyt() {
   mpv --ytdl ytdl_path=/usr/local/bin/yt-dlp "$@"
-  }
+}
 
 # taken from default changes to bashrc and bash_profile
 path-add --end --ifexists $HOME/.rvm/bin

diff --git a/mail-setup b/mail-setup
index e753fbe9670ea27f7837b14be795d1ce5d7c95af..667f0ec47f85a27a14f547b54cd6cfcb691dae03 100755 (executable)
--- a/mail-setup
+++ b/mail-setup
@@ -13,6 +13,8 @@
 # which has log path
 # log_file_path = /var/log/exim4/my%s
 #
+# On non bk|MAIL_HOST, the config and log file are all standard.
+#
 # eximbackup folder is /bu/md
 # it is cleaned up by mail-backup-clean, which is run by btrbk-run
 
@@ -24,6 +26,14 @@
 #
 #&! testignore|jtuttle|eximbackup|/usr/sbin/exim4 -bpu
 
+# todo: this message seems to get dropped on the floor, it was due to a missing 2nd colon in
+#  condition = ${if def:h_fdate:}
+# Figure out how to avoid this message being discarded.
+
+# 2023-09-12 01:41:43 [722371] 1qfw9f-0031v9-0S <= ian@iankelling.org U=iank P=local S=483 id=87cyyogd7t.fsf@iankelling.org T="iank2" from <ian@iankelling.org> for testignore@amnimal.ninja
+# 2023-09-12 01:41:43 [722373] 1qfw9f-0031v9-0S H=nn.b8.nz [10.173.8.2]: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
+# 2023-09-12 01:41:43 [722372] 1qfw9f-0031v9-0S == testignore@amnimal.ninja R=smarthost T=remote_smtp_smarthost defer (-46) H=nn.b8.nz [10.173.8.2] DT=0s: SMTP error from remote mail server after pipelined end of data: 451 Temporary local problem - please try later
+
 # todo: check new macro DKIM_TIMESTAMPS
 
 # todo: check if REMOTE_SMTP_INTERFACE or REMOTE_SMTP_TRANSPORTS_HEADERS_REMOVE can simplify my or fsfs config
@@ -494,6 +504,9 @@ EOF
 
 # * clamav
 
+# old file. remove when all hosts updated, 2023-09-11
+rm -fv /etc/exim4/conf.d/clamav_data_acl
+
 m usermod -a -G Debian-exim clamav
 
 u /etc/systemd/system/clamav-daemon.service.d/fix.conf <<EOF
@@ -1329,6 +1342,8 @@ smtp_reserve_hosts = +iank_trusted
 # Rules that make receiving more liberal should be on backup hosts
 # so that we dont reject mail accepted by MAIL_HOST
 LOCAL_DENY_EXCEPTIONS_LOCAL_ACL_FILE = /etc/exim4/conf.d/local_deny_exceptions_acl
+
+acl_not_smtp = acl_check_not_smtp
 EOF
 
 if dpkg --compare-versions "$(dpkg-query -f='${Version}\n' --show exim4)" ge 4.94; then
@@ -1349,6 +1364,11 @@ DKIM_DOMAIN = ${lc:${domain:$rh_from:}}
 EOF
 fi
 
+cat >/etc/exim4/conf.d/main/30_local <<EOF
+freeze_tell =
+EOF
+
+
 rm -fv /etc/exim4/rcpt_local_acl # old path
 
 u /etc/exim4/conf.d/local_deny_exceptions_acl <<'EOF'
@@ -1402,6 +1422,38 @@ warn
 #  dmarc_status = reject:quarantine
 #  add_header = Reply-to: dmarctest@iankelling.org
 
+# This allows us to delay sending an email until a specific time,
+# allowing us time to change our mind and also to appear to have
+# sent the message at a different time. In emacs copy the
+# automcatically date header add an f to make it fdate,
+# and then change the date to whenever you want to send it.
+# In the system-status script, I check once per minute
+# or more if it should be sent.
+
+warn
+  # fdate = future date.
+  condition = ${if def:h_fdate:}
+  remove_header = fdate:
+  remove_header = date:
+  add_header = date: $h_fdate
+  control = freeze
+EOF
+sed -i 's/^freeze_tell =.*/#\0/' /etc/exim4/conf.d/main/02_exim4-config_options
+
+u /etc/exim4/conf.d/acl/41_check_not_smtp <<'EOF'
+# todo: for non MAIL_HOST machines, i'd like
+# to send to the MAIL_HOST without freezing.
+# So, only do this if we are MAIL_HOST.
+
+acl_check_not_smtp:
+warn
+  # fdate = future date.
+  condition = ${if def:h_fdate:}
+  remove_header = fdate:
+  remove_header = date:
+  add_header = Date: $h_fdate
+  control = freeze
+accept
 EOF
 
 
@@ -2791,7 +2843,8 @@ case $HOSTNAME in
     # which will overwrite any existing file
     u /etc/default/exim4 <<'EOF'
 QUEUERUNNER='combined'
-QUEUEINTERVAL='30m'
+# note: this is duplicated in brc2, 10m here is -q10m there.
+QUEUEINTERVAL='10m'
 COMMONOPTIONS='-C /etc/exim4/my.conf'
 UPEX4OPTS='-o /etc/exim4/my.conf'
 # i use epanic-clean for alerting if there are bad paniclog entries
@@ -2912,28 +2965,6 @@ EOF
 dc_relay_nets='defaultnn.b8.nz'
 EOF
 
-    # no clamav on je, it has 1.5g memory and clamav uses most of it
-    u /etc/exim4/conf.d/clamav_data_acl <<'EOF'
-warn
-!hosts = +iank_trusted
-!authenticated = plain_server:login_server
-condition = ${if def:malware_name}
-remove_header = Subject:
-add_header = Subject: [Clamav warning: $malware_name] $h_subject
-log_message = heuristic malware warning: $malware_name
-EOF
-
-    cat >>/etc/exim4/conf.d/main/000_local <<EOF
-# je.b8.nz will run out of memory with freshclam
-av_scanner = clamd:/var/run/clamav/clamd.ctl
-EOF
-
-    cat >> /etc/exim4/conf.d/data_local_acl <<'EOF'
-deny
-  malware = */defer_ok
-  !condition = ${if match {$malware_name}{\N^Heuristic\N}}
-  message = This message was detected as possible malware ($malware_name).
-EOF
 
     cat >/etc/exim4/conf.d/main/000_local-nn <<EOF
 # MAIN_HARDCODE_PRIMARY_HOSTNAME might mess up the
@@ -2979,6 +3010,38 @@ EOF
   # ** $MAIL_HOST)
   $MAIL_HOST)
 
+
+
+    u /etc/exim4/conf.d/router/155_delay <<'EOF'
+By default, delay sending email by 30-40 minutes in case I
+change my mind.
+
+# Note, if we switch mail_host, the next queue run will
+# send the message to mail_host and the delay will be reset.
+# That is fine. I could probably set some header to track
+# the delay but it is not worth it.
+delay_iank:
+  driver = redirect
+  allow_defer
+  data = :defer:
+  # It hasnt been 30 minutes since we received the message.
+  # we can avoid delay by adding the header i: or putting the exim message id into a file,
+  # or pulling "all" into a file.
+  # note, true false at the end just for easier debugging when pasting into a exim -Mset ID -be.
+  condition = ${if and { \
+{< {$tod_epoch} {${eval10:$received_time + 60*30}}} \
+{!def:h_i:} \
+{!bool{${lookup{$message_exim_id}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \
+{!bool{${lookup{all}lsearch{/etc/exim4/no-delay-eximids}{true}}}} \
+} {true}{false}}
+  headers_remove = <; i:
+  domains = ! +local_domains
+  # uncomment for testing delays to jtuttle
+  # local_parts = ! root : ! testignore : ! alerts : ! ian-pager : ! daylert
+  local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert
+  ignore_target_hosts = ROUTER_DNSLOOKUP_IGNORE_TARGET_HOSTS
+EOF
+
     u /etc/exim4/conf.d/router/195_dnslookup_vpn <<'EOF'
 # copied from /etc/exim4/conf.d/router/200_exim4-config_primary, but
 # use vpn transport. lower priority so it overrides the default route.
@@ -3010,7 +3073,7 @@ condition = ${if !bool{${lookup{$local_part@$domain}lsearch{/etc/exim4/ignore-se
 # note, to test this, i could temporarily allow testignore.
 # alerts avoids potential mail loop. root is already
 # redirected earlier, so that is just being overly cautious.
-local_parts = ! root : ! testignore : ! alerts
+local_parts = ! root : ! testignore : ! alerts : ! jtuttle : ! ian-pager : ! daylert
 unseen = true
 errors_to = alerts@iankelling.org
 EOF
@@ -3153,6 +3216,39 @@ EOF
   ## we use this host to monitor MAIL_HOST and host a mail server for someone
   bk)
 
+    # No clamav on je, it has 1.5g memory and clamav uses most of it.
+    #
+    # No clamav on MAIL_HOST because it is just a waste of useful cpu
+    # time and memory when I'm running on an x200, and it takes 30
+    # seconds to shut down.
+
+    cat >>/etc/exim4/conf.d/main/000_local <<EOF
+# je.b8.nz will run out of memory with freshclam
+av_scanner = clamd:/var/run/clamav/clamd.ctl
+EOF
+
+    cat >> /etc/exim4/conf.d/data_local_acl <<'EOF'
+deny
+  malware = */defer_ok
+  !condition = ${if match {$malware_name}{\N^Heuristic\N}}
+  message = This message was detected as possible malware ($malware_name).
+
+warn
+  !hosts = +iank_trusted
+  !authenticated = *
+  condition = ${if def:malware_name}
+  remove_header = Subject:
+  add_header = Subject: [Clamav warning: $malware_name] $h_subject
+  log_message = heuristic malware warning: $malware_name
+
+warn
+  # fdate = future date. # tdate = temporary date.
+  condition = ${if def:h_fdate}
+  remove_header = fdate:
+  add_header = tdate:
+  control = freeze
+EOF
+
 
     /a/exe/cedit nn /etc/hosts <<'EOF' || [[ $? == 1 ]]
 10.173.8.2 nn.b8.nz
@@ -3199,18 +3295,25 @@ EOF
     ;;
   # ** not MAIL_HOST|bk|je
   *)
-    # this one should be removed for all non mail hosts, but
+    echo|u /etc/exim4/conf.d/router/155_delay
+    # this one should be removed for all non mail_hosts. note
     # bk and je never become mail_host
     echo|u /etc/exim4/conf.d/router/195_dnslookup_vpn
     echo|u /etc/exim4/conf.d/router/160_backup_redir
     echo|u /etc/exim4/conf.d/router/161_backup_redir_nn
     echo|u /etc/exim4/conf.d/router/185_sentarchive
     echo|u /etc/exim4/conf.d/router/186_sentarchive_nn
+    # Note, in general we could submit to smarthosts on non MAIL_HOST.
+    # however, delayed mail makes this inconvenient, because I
+    # occasionally want to send an email from a non-MAIL_HOST and then
+    # turn off that computer or travel with it so it is disconnected.
+    # It is also probably easier to setup emacs to delay messages, but
+    # that would mean we need to keep emacs running, this is much
+    # nicer.
     echo|u /etc/exim4/conf.d/router/188_exim4-config_smarthost
     echo|u /etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost
     echo|u /etc/exim4/conf.d/rcpt_local_acl
     echo|u /etc/exim4/conf.d/main/000_local-nn
-    echo|u /etc/exim4/conf.d/clamav_data_acl
 
 
     if $bhost_t; then
@@ -3530,6 +3633,8 @@ case $HOSTNAME in
     else
       m systemctl --now enable $vpnser
     fi
+    ;;&
+  bk)
     if ! systemctl is-active clamav-daemon >/dev/null; then
       m systemctl --now enable clamav-daemon
       out=$(rsync -aiSAX --chown=root:root --chmod=g-s /a/bin/ds/filesystem/etc/systemd/system/epanicclean.service /etc/systemd/system)

diff --git a/pkgs b/pkgs
index 9c6f907206bcbbf5afa7190837230b1940af69de..efa794588610a510ae304ca2a71010033a27ceee 100644 (file)
--- a/pkgs
+++ b/pkgs
@@ -223,6 +223,7 @@ p3=(
   pidgin
   pidgin-otr
   pixz
+  profanity
   pry
   pv
   python3-doc

diff --git a/system-status b/system-status
index 87d8cf9d7939df5b2b55175e10015cca1d0f0f4f..fdf4758b5028a9f33ff03b91dfc0e9dedb22c77c 100755 (executable)
--- a/system-status
+++ b/system-status
@@ -263,13 +263,35 @@ write-status() {
   fi
   p $var_mail_msg | loday -1 var_mail
 
-  # early in install process, we dont have permission yet for exiqgrep.
-  # 1100 helps allow for system restarts
-  qlen=$(/usr/sbin/exiqgrep -o 1100 -c -b | awk '{print $1}') ||:
+  # Note, early in install process, we dont have permission yet for exiqgrep.
+  #
+  # todo: don't do this every 15 seconds, more like once every 2 minutes to
+  # save cpu cycles.
+  #
+  # 2400 = 40 mins. This should allow for system restarts, and
+  # 30 minute message delay plus 10 minute queu runs.
+  qlen=$(/usr/sbin/exiqgrep -o 2400 -c -b | awk '{print $1}') ||:
   qmsg=
   if ((qlen)); then
-    qmsg="queue length $qlen"
-    chars+=("q $qlen")
+    # Do sending of long delayed messages, and dont count them in our queue warnings.
+    for mid in $(exiqgrep -o 2400 -zi); do
+      if exim -Mvh $mid | awk 'tolower($2) == "fdate:"' | grep -q .; then
+        qlen=$(( qlen - 1 ))
+        if (( $(date -d "$(exim -Mset $mid -be <<<'$h_date:' | sed -n 's/^> *//;/./p')" +%s) < EPOCHSECONDS )); then
+          if ip a show veth0-mail &>/dev/null; then
+            pid=$(pgrep -f "/usr/sbin/exim4 -bd -q30m -C /etc/exim4/my.conf"|head -n1);
+            nsenter -t $pid -n -m /usr/sbin/exim4 -C /etc/exim4/my.conf -M $mid
+          else
+            /usr/sbin/exim4 -M $mid
+          fi
+        fi
+      fi
+    done
+
+    if ((qlen)); then
+      qmsg="queue length $qlen"
+      chars+=("q $qlen")
+    fi
   fi
   case $HOSTNAME in
     # No point in emailing about the mailq on a host where we don't