From: Ian Kelling Date: Sat, 3 Jul 2021 03:57:08 +0000 (-0400) Subject: various fixes X-Git-Url: https://iankelling.org/git/?a=commitdiff_plain;h=c886b13a93866c1977e4e70ca689ceb651db9fd2;p=distro-setup various fixes --- diff --git a/.bashrc b/.bashrc index 1aab809..9c7d638 100644 --- a/.bashrc +++ b/.bashrc @@ -35,7 +35,7 @@ HISTIGNORE='pass *:[ ]*:otp *:oathtool *' # noninteractive ssh shells, or tty. tty because often i # use it when something is going and io is slow and my bashrc # is too slow. -if [[ $SSH_CONNECTION && $LC_USEBASHRC != t || $TERM == linux ]]; then +if [[ $LC_USEBASHRC != t && ( $SSH_CONNECTION || $TERM == linux ) ]]; then # Here we did not opt-in to running our .bashrc file so we just # return, but we still setup a function to source it without returning # so if we want it we don't have to restart our ssh connection. diff --git a/brc2 b/brc2 index b563c9e..f82c3d7 100644 --- a/brc2 +++ b/brc2 @@ -203,9 +203,14 @@ bum() { bu() { fusermount -u /bu/mnt } + eqgo() { enn -M $(exiqgrep -i) } +eqgo1() { + enn -M $(exiqgrep -i|h1) +} + gnupload(){ /a/f/gnulib/build-aux/gnupload "$@" @@ -350,8 +355,8 @@ jrun() { # journal run. run args, log to journal, tail and grep the journal. fi journalctl -qn2 -f -u "$cmd_name" & # Guess of time needed to avoid missing initial lines. - # .5 was not reliable. - sleep 1 + # .5 was not reliable. 1 was not reliable + sleep 2 # We kill this in prompt-command for the case that we ctrl-c the # systemd-cat. i dont know any way to trap ctrl-c and still run the # normal action for it. There might be a way, unsure. @@ -375,7 +380,7 @@ srun() { unit=$1 journalctl -qn2 -f -u $unit & systemctl start $unit - sleep .5 + sleep 2 kill $jr_pid &>/dev/null ||: unset jr_pid fg &>/dev/null ||: @@ -403,7 +408,7 @@ lipush() { p[0]=/a/opt/emacs-ubuntu20.04 p[1]=/a/opt/emacs-ubuntu20.04-nox done - m rsync "$@" -ahviSAXPH root@li:/a/h/proposed-comments/ /a/h/proposed-comments || ret=$? + m rsync "$@" -ahviSAXPH root@li.b8.nz:/a/h/proposed-comments/ /a/h/proposed-comments || ret=$? return $ret } bkpush() { # no emacs. for running faster. @@ -1071,17 +1076,22 @@ myirc() { # use * instead of -r since that does sorted order ssh root@iankelling.org "cd $d/#$1; grep '\/dev/null; then - rg() { command rg -i -M 300 --no-ignore "$@"; } + rg() { command rg -L -i -M 300 --no-ignore "$@"; } complete -r rg else alias rg=grr diff --git a/btrbk-run b/btrbk-run index 74deab7..fd05c09 100644 --- a/btrbk-run +++ b/btrbk-run @@ -143,8 +143,13 @@ fi ### end options parsing # note, this test succeeds if not installed -if [[ /a/opt/btrbk/btrbk -nt /usr/sbin/btrbk ]]; then +if [[ /a/opt/btrbk/btrbk -nt /usr/btrbk ]]; then + if [[ -e /b/distro-functions/src/package-manager-abstractions ]]; then + . /b/distro-functions/src/package-manager-abstractions + pi asciidoctor + fi cd /a/opt/btrbk + m make m sudo make install fi @@ -321,7 +326,26 @@ else sshfail=() min_idle_ms=$((1000 * 60 * 15)) for h in ${targets[@]}; do - if zone=$(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk && date +%z"); then + if remote_info=( $(timeout -s 9 6 ssh root@$h "mkdir -p /mnt/root/btrbk && date +%z && df --output=size,pcent / | tail -n1") ); then + zone=${remote_info[0]} + root_size=${remote_info[1]} + percent_used=${remote_info[2]%%%} + + if (( ${#remote_info[@]} != 3 )); then + die "error: didnt get 3 fields in test ssh to target $h. investigate" + fi + + + # we may be booted into a bootstrap fs or something + min_root_kb=$(( 1024 * 1024 * 200 )) # 200 gb + if (( root_size < min_root_kb )); then + continue + fi + + if (( percent_used >= 98 )); then + die "error: filesystem on target $h is $percent_used % full" + fi + # This is a separate ssh because xprintidle can fail and thats ok. if $cron && idle_ms=$(timeout -s 9 6 ssh $h DISPLAY=:0 xprintidle); then if (( idle_ms < min_idle_ms )); then diff --git a/btrfsmaint b/btrfsmaint index 06570b3..f27d2b2 100755 --- a/btrfsmaint +++ b/btrfsmaint @@ -23,7 +23,7 @@ check-idle() { idle_time=$idle_limit id=999 while id $((++id)) &>/dev/null; do - new_idle_time=$(sudo -u \#$id xprintidle 2>/dev/null) ||: + new_idle_time=$(su -c xprintidle $(id -nu $id) 2>/dev/null) ||: if [[ $new_idle_time && $new_idle_time -lt $idle_time ]]; then idle_time=$new_idle_time fi @@ -35,6 +35,24 @@ check-idle() { fi } + +usage() { + cat <&2 - exit 1 + usage 1 ;; esac fi -idle=true -if ! $force; then - check-idle -fi - -fnd="findmnt --types btrfs --noheading" -for x in $($fnd --output "SOURCE" --nofsroot | sort -u); do - mnt=$($fnd --output "TARGET" --first-only --source $x) - [[ $mnt ]] || continue - if ! $idle; then - btrfs scrub cancel $mnt &>/dev/null ||: - continue - fi - if $check; then - continue +main() { + idle=true + if ! $force; then + check-idle fi - # for comparing before and after balance. - # the log is already fairly verbose, so commented. - # e btrfs filesystem df $mnt - # e df -H $mnt - if btrfs filesystem df $mnt | grep -q "Data+Metadata"; then - for usage in $dusage; do - e btrfs balance start -dusage=$usage -musage=$usage $mnt - done - else - e btrfs balance start -dusage=0 $mnt - for usage in $dusage; do - e btrfs balance start -dusage=$usage $mnt - done - e btrfs balance start -musage=0 $mnt - for usage in $musage; do - e btrfs balance start -musage=$usage $mnt - done - fi - # e btrfs filesystem df $mnt - # e df -H $mnt - date=$( - btrfs scrub status $mnt | \ - sed -rn 's/^\s*scrub started at (.*) and finished.*/\1/p' - ) - if [[ $date ]]; then - date=$(date --date="$date" +%s) - # if date is sooner than 90 days ago - # the wiki recommends 30 days or so, but - # it makes the comp lag like shit for a day, - # so I'm going with 90 days. - if (( date > $(date +%s) - 60*60*24*30 )); then - echo "cron: skiping scrub of $mnt" + fnd="findmnt --types btrfs --noheading" + for x in $($fnd --output "SOURCE" --nofsroot | sort -u); do + mnt=$($fnd --output "TARGET" --first-only --source $x) + [[ $mnt ]] || continue + + if ! $idle; then + btrfs scrub cancel $mnt &>/dev/null ||: continue fi - fi - e btrfs scrub start -Bd $mnt -done + if $check; then + continue + fi + + # for comparing before and after balance. + # the log is already fairly verbose, so commented. + # e btrfs filesystem df $mnt + # e df -H $mnt + if btrfs filesystem df $mnt | grep -q "Data+Metadata"; then + for usage in $dusage; do + e btrfs balance start -dusage=$usage -musage=$usage $mnt + done + else + e btrfs balance start -dusage=0 $mnt + for usage in $dusage; do + e btrfs balance start -dusage=$usage $mnt + done + e btrfs balance start -musage=0 $mnt + for usage in $musage; do + e btrfs balance start -musage=$usage $mnt + done + fi + # e btrfs filesystem df $mnt + # e df -H $mnt + date=$( + btrfs scrub status $mnt | \ + sed -rn 's/^\s*scrub started at (.*) and finished.*/\1/p' + ) + if [[ $date ]]; then + date=$(date --date="$date" +%s) + # if date is sooner than 90 days ago + # the wiki recommends 30 days or so, but + # it makes the comp lag like shit for a day, + # so I'm going with 90 days. + if (( date > $(date +%s) - 60*60*24*30 )); then + echo "cron: skiping scrub of $mnt" + continue + fi + fi + e btrfs scrub start -Bd $mnt + done +} + +if $check; then + # this is to prevent systemd from filling up the journal + for (( runcount=0; runcount < 90; runcount++ )); do + main + sleep 20 + done +else + main +fi diff --git a/conflink b/conflink index b0e67ad..90debe6 100755 --- a/conflink +++ b/conflink @@ -160,8 +160,8 @@ case $user in m s chmod g+w $f fi sudo bash -c 'shopt -s nullglob; for f in /etc/bind/*.key /etc/bind/*.private /etc/bind/key.*; do chgrp bind $f; done' - if [[ -e /etc/davpass ]] && getent group www-data &>/dev/null; then - s chgrp www-data /etc/davpass + if [[ -e /etc/caldav-htpasswd ]] && getent group www-data &>/dev/null; then + s chgrp www-data /etc/caldav-htpasswd fi if [[ -e /var/lib/znc ]] && getent group znc; then s chown -R znc:znc /var/lib/znc diff --git a/distro-begin b/distro-begin index 34eac6a..5388a25 100755 --- a/distro-begin +++ b/distro-begin @@ -349,12 +349,15 @@ fi if isdeb && [[ $(debian-codename) == nabia ]]; then sudo dd of=/etc/apt/preferences.d/nabia-focal-missing <<'EOF' Explanation: libkf5* are for konsole libilmbase* are needed for emacs -Package: libkf5solid5 libkf5solid5-data libilmbase-dev libilmbase24 ansible unrar-free pidgin pidgin-* libpurple0 linux-doc p7zip mumble hplip cups-filters libcupsfilters1 libfontembed1 cups-filters-core-drivers lightdm mate-desktop-environment mate-desktop-environment-* mate-menus mate-panel mate-panel-* mate-session-manager libmate-* libgoa-* ubuntu-mate-default-settings mate-control-center mate-control-center-common gir1.2-matemenu-* lightdm-gtk-greeter liblightdm-gobject-* firefox libfdk-aac1 +Package: libilmbase-dev libilmbase24 unrar-free pidgin pidgin-* libpurple0 linux-doc p7zip mumble lightdm mate-desktop-environment mate-desktop-environment-* mate-menus mate-panel mate-panel-* mate-session-manager libmate-* ubuntu-mate-default-settings mate-control-center mate-control-center-common gir1.2-matemenu-* lightdm-gtk-greeter liblightdm-gobject-* firefox libfdk-aac1 Pin: release n=focal,o=Ubuntu Pin-Priority: 500 EOF fi +# to test when these become available in trisquel, copy the package list, remove the * chars, repaplace PACKAGES below +# for x in PACKAGES do hr; e $x; if [[ $x == *- ]]; then s="^$x"; dpkg -l "$x*" | sed '0,/^+++/d'; else s="^$x\$"; fi; aptitude search "~O Trisquel ~n $s"; done + #### setup firefox backport ## ian: disabled. backports are not being published atm due to rust packaging issue diff --git a/distro-end b/distro-end index dfc0340..546850e 100755 --- a/distro-end +++ b/distro-end @@ -456,26 +456,26 @@ s rm -fv /etc/apt/preferences.d/radicale ######### end universal pinned packages ###### ### system76 things ### -case $HOSTNAME in - sy) - # note, i stored the initial popos packages at /a/bin/data/popos-pkgs - if [[ ! -e /etc/apt/sources.list.d/system76.list ]]; then - # https://blog.zackad.dev/en/2017/08/17/add-ppa-simple-way.html - sd /etc/apt/sources.list.d/system76.list < - Options +FollowSymLinks +Multiviews +Indexes - AllowOverride None - AuthType basic - AuthName "Authentication Required" - # setup one time, with root:www-data, 640 - AuthUserFile "/etc/caldav-htpasswd" - Require valid-user + + Options +FollowSymLinks +Multiviews +Indexes + AllowOverride None + AuthType basic + AuthName "Authentication Required" + # setup one time, with root:www-data, 640 + AuthUserFile "/etc/caldav-htpasswd" + Require valid-user + RequestHeader set X-Script-Name /radicale/ + RequestHeader set X-Remote-User expr=%{REMOTE_USER} + ProxyPass "http://10.8.0.4:5232/" retry=0 + ProxyPassReverse "http://10.8.0.4:5232/" EOF # nginx version of above would be: @@ -809,6 +815,10 @@ case $codename_compat in # https://bugs.launchpad.net/ubuntu/+source/im-config/+bug/1879352 eval exec $STARTUP EOF + # also, gdm wayland was default, had to do + # dpkg-reconfigure gdm3 + # and choose lightdm. + # ;; esac @@ -1693,6 +1703,11 @@ EOF # Remove dep that came in with desktop to fix associations. m pu transmission-gtk +case $HOSTNAME in + kd) + lnf -T /d/vidshare /home/iank/Videos + ;; +esac # # Based on guix manual instructions, also added code to profile. # # disabled since i'm not using it now. diff --git a/dynamic-ip-update b/dynamic-ip-update index d5102dd..354f2d0 100755 --- a/dynamic-ip-update +++ b/dynamic-ip-update @@ -1,130 +1,145 @@ #!/bin/bash source ~/.bashrc -fqdn=$(hostname -f) -domaintmp=${fqdn#*.} -hostnametmp=${fqdn%%.*} -# i for internet -fqdn=${hostnametmp}i.${domaintmp} - -up4=false - -read -r _ _ gateway _ < <(ip -4 route get 85.119.83.50) - -case $gateway in - 10.2.0.1) - dynhost=i.b8.nz - ;; - *) - exit 0 - ;; -esac - -if timeout -s 9 5 ssh-keyscan -p 2220 -t rsa $gateway 2>/dev/null | grep -qFx "[$gateway]:2220 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCH+/h1dGEfKEusBblndU2e6QT4wLCm5+yqr/sqh/0X9YfjR7BfWWm8nNmuP55cYc+Wuf5ljB1H1acXEcsl1y8e0j3agHfF0V74FE1N1zz5nn2Ep8NHnmqgEhza38ZxMPh+4p3X7zklEKU7+3SzybKBi8sg0wLzlS2LM0JaUN80zR2sK11Kye3dURUXPk78u5wodOkgcEYRwSYaDMJlUzWP+poRXIDJwFaMQnwmxbl/c84yOyaU0x/d6hFwoRscWecihX+vvBNeSyxR4xr2HDOyUWwJkctyAgt2p7w3tfkXOKcCRzTAjGVIMQLTvo0sG/yJbcyHoEFdFybCsgDvfyYn"; then - # we are on a home network - cur4="$(dig +short $dynhost @iankelling.org | tail -1)" - if ip4=$(curl -s4 https://iankelling.org/cgi/pubip); then - if [[ $cur4 && $ip4 && $cur4 != $ip4 ]]; then - up4=true # update ipv4 +main() { + + fqdn=$(hostname -f) + domaintmp=${fqdn#*.} + hostnametmp=${fqdn%%.*} + # i for internet + fqdn=${hostnametmp}i.${domaintmp} + + up4=false + + read -r _ _ gateway _ < <(ip -4 route get 85.119.83.50) + + case $gateway in + 10.2.0.1) + dynhost=i.b8.nz + ;; + *) + exit 0 + ;; + esac + + if timeout -s 9 5 ssh-keyscan -p 2220 -t rsa $gateway 2>/dev/null | grep -qFx "[$gateway]:2220 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCH+/h1dGEfKEusBblndU2e6QT4wLCm5+yqr/sqh/0X9YfjR7BfWWm8nNmuP55cYc+Wuf5ljB1H1acXEcsl1y8e0j3agHfF0V74FE1N1zz5nn2Ep8NHnmqgEhza38ZxMPh+4p3X7zklEKU7+3SzybKBi8sg0wLzlS2LM0JaUN80zR2sK11Kye3dURUXPk78u5wodOkgcEYRwSYaDMJlUzWP+poRXIDJwFaMQnwmxbl/c84yOyaU0x/d6hFwoRscWecihX+vvBNeSyxR4xr2HDOyUWwJkctyAgt2p7w3tfkXOKcCRzTAjGVIMQLTvo0sG/yJbcyHoEFdFybCsgDvfyYn"; then + # we are on a home network + cur4="$(dig +short $dynhost @iankelling.org | tail -1)" + if ip4=$(curl -s4 https://iankelling.org/cgi/pubip); then + if [[ $cur4 && $ip4 && $cur4 != $ip4 ]]; then + up4=true # update ipv4 + fi fi fi -fi -# may not be set yet so allow fail -cur6="$(host -4 -t aaaa $fqdn iankelling.org | sed -rn 's/.*has IPv6 address (.*)/\1/p;T;q')" ||: + # may not be set yet so allow fail + cur6="$(host -4 -t aaaa $fqdn iankelling.org | sed -rn 's/.*has IPv6 address (.*)/\1/p;T;q')" ||: -up6=false + up6=false -out6=$(curl -s6 https://iankelling.org/cgi/pubip) ||: # failure allowed if we have no ipv6 + out6=$(curl -s6 https://iankelling.org/cgi/pubip) ||: # failure allowed if we have no ipv6 -if [[ $out6 ]]; then - dev=$(ip -o a show to $out6 | awk '{print $2}') - # we use slaac with privacy extension, so get our less private more permanent address - mac=$(cat /sys/class/net/$dev/address) + if [[ $out6 ]]; then + dev=$(ip -o a show to $out6 | awk '{print $2}') + # we use slaac with privacy extension, so get our less private more permanent address + mac=$(cat /sys/class/net/$dev/address) - IFS=: read -a f <<<$mac; set -- ${f[@]} - ip6=${out6%:*:*:*:*}:$(printf %x $((0x$1 + 2)))$2:$3'ff:fe'$4:$5$6 - # in case we aren't using slaac - if ! ip a | grep "^ *inet6 $ip6/" &>/dev/null; then - ip6=$out6 + IFS=: read -a f <<<$mac; set -- ${f[@]} + ip6=${out6%:*:*:*:*}:$(printf %x $((0x$1 + 2)))$2:$3'ff:fe'$4:$5$6 + # in case we aren't using slaac + if ! ip a | grep "^ *inet6 $ip6/" &>/dev/null; then + ip6=$out6 + fi fi -fi -if [[ $cur6 != $ip6 ]]; then - up6=true -fi + if [[ $cur6 != $ip6 ]]; then + up6=true + fi -if ! $up4 && ! $up6; then - exit 0 -fi + if ! $up4 && ! $up6; then + exit 0 + fi -# note, a simpler way to do this would be to ssh and use -# "${SSH_CLIENT%% *} -# to update bind if needed. + # note, a simpler way to do this would be to ssh and use + # "${SSH_CLIENT%% *} + # to update bind if needed. -f=$(mktemp) -cat >>$f <>$f <>$f <>$f <>$f <>$f <>$f <>$f <>$f <>$f <$f <$f </dev/null ||: # ignore fail in bash < 4.4 set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR -[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" + +if [[ $EUID != 0 ]]; then + s=sudo +fi if [[ ! -s /var/log/exim4/paniclog ]]; then exit 0 @@ -42,6 +45,11 @@ while read -r d1 d2; do fi done < <(awk "/$regex/ "'{print $1,$2}' /var/log/exim4/paniclog) if $found && $wipe; then + if [[ ! -w /var/log/exim4/paniclog-archive ]]; then + $s touch /var/log/exim4/paniclog-archive + $s chgrp adm /var/log/exim4/paniclog-archive + $s chmod 664 /var/log/exim4/paniclog-archive + fi grep -E "$regex" /var/log/exim4/paniclog >> /var/log/exim4/paniclog-archive - sed -ri "/$regex/d" /var/log/exim4/paniclog + $s sed -ri "/$regex/d" /var/log/exim4/paniclog fi diff --git a/laptop-btrbk b/laptop-btrbk index b88d931..7a28dd9 100755 --- a/laptop-btrbk +++ b/laptop-btrbk @@ -8,7 +8,7 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?. PIPESTATUS: ${PIPESTATUS[*]}" >&2' ERR idle_limit=$((1000 * 60 * 45)) -idle_time=$(sudo -u '#1000' xprintidle 2>/dev/null) ||: +idle_time=$(su -c xprintidle $(id -nu 1000) 2>/dev/null) ||: if [[ $idle_time && $idle_limit -lt $idle_time ]]; then btrbk run fi diff --git a/mail-setup b/mail-setup index 735bb33..7faa5e1 100755 --- a/mail-setup +++ b/mail-setup @@ -3,6 +3,9 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: if we fail in the middle and rerun we can mistakenly +# have reload=false. + # todo: auto restart of je on checkrestart # todo: run mailping test after running, or otherwise @@ -1000,7 +1003,9 @@ if mailhost; then # sudo -u radicale radicale -D -f # created password file with: - # htpasswd -c /etc/davpass dav + # htpasswd -c /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd + # chmod 640 /p/c/machine_specific/li/filesystem/etc/caldav-htpasswd + # # setup chgrp www-data in ./conflink pi-nostart radicale @@ -1053,6 +1058,9 @@ EOF # comments say default is 0.0.0.0:5232 m setini hosts 10.8.0.4:5232 server + # https://radicale.org/2.1.html + m setini type http_x_remote_user auth + # disable power management feature, set to 240 min sync interval, # so it shouldn't be bad. @@ -2412,7 +2420,11 @@ case $HOSTNAME in sstart mailclean.timer ;;& $MAIL_HOST) - sstart radicale + # < 2.1 (eg: in t9), uses a different data format which required manual + # migration. dont start if we are running an old version. + if dpkg --compare-versions $(dpkg -s radicale | awk '$1 == "Version:" { print $2 }') ge 2.1; then + sstart radicale + fi ;;& esac diff --git a/pkgs b/pkgs index 1eba787..b9d9966 100644 --- a/pkgs +++ b/pkgs @@ -95,6 +95,8 @@ p3=( etckeeper evince exim4-doc-html + exfat-fuse + exfat-utils fakeroot fail2ban fdupes @@ -107,6 +109,7 @@ p3=( fonts-thai-tlwg-otf xfonts-intl-asian fonts-sil-lateef + gajim gawk-doc # gcc suggests make manpages-dev autoconf automake libtool flex bison @@ -129,12 +132,14 @@ p3=( hplip hunspell i3lock + iotop info inotify-tools ipcalc iputils-tracepath iperf3 iproute2-doc + irqbalance jq kid3-qt kid3-cli @@ -191,6 +196,7 @@ p3=( reportbug rfkill rng-tools + rygel sakura schroot shellcheck diff --git a/subdir_files/.config/rss2mail.cfg b/subdir_files/.config/rss2mail.cfg index 287272b..b10e705 100644 --- a/subdir_files/.config/rss2mail.cfg +++ b/subdir_files/.config/rss2mail.cfg @@ -1,2 +1,4 @@ [DEFAULT] from = rss2email@iankelling.org +# exim delay_warning_condition, this prevents warnings +bonus-header = 'Precedence: list' diff --git a/untrusted-network b/untrusted-network index 9167c9f..677aeb6 100755 --- a/untrusted-network +++ b/untrusted-network @@ -27,6 +27,7 @@ Domains=b8.nz DNSOverTLS=yes EOF +mkdir -p /etc/NetworkManager/conf.d cat >/etc/NetworkManager/conf.d/dns.conf <<'EOF' [main] dns=none