From: Ian Kelling Date: Fri, 19 Apr 2019 18:50:22 +0000 (-0400) Subject: lots of fixes X-Git-Url: https://iankelling.org/git/?p=distro-setup;a=commitdiff_plain;h=0b6d44c7f3d567e0a26138509c8a24cb57c69b50 lots of fixes --- diff --git a/brc b/brc index e85f309..2a1a0a7 100644 --- a/brc +++ b/brc @@ -2214,6 +2214,7 @@ fi export GOPATH=$HOME/go path_add $GOPATH/bin +path_add /usr/local/go/bin export ARDUINO_PATH=/a/opt/Arduino/build/linux/work diff --git a/btrbk-run b/btrbk-run index 71b0e6b..5e1a888 100644 --- a/btrbk-run +++ b/btrbk-run @@ -40,11 +40,7 @@ if [[ -e /b/bash_unpublished/source-semi-priv ]]; then source /b/bash_unpublished/source-semi-priv fi -script_dir=$(dirname $(readlink -f "$BASH_SOURCE")) - # note q is owned by root:1000 -# note p/m is owned 1000:1000 and chmod 700 - mountpoints=() @@ -56,13 +52,18 @@ dry_run=false # mostly for testing rate_limit=no verbose=true; verbose_arg=-v progress_arg="--progress" +pull_reexec=false default_args_file=/etc/btrbk-run.conf if [[ -s $default_args_file ]]; then set -- $(< $default_args_file) "$@" + echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):" + cat $default_args_file + sleep 5 fi -temp=$(getopt -l help cl:m:nps:t:vh "$@") || usage 1 +orig_args=("$@") +temp=$(getopt -l pull-reexec,help cl:m:nps:t:vh "$@") || usage 1 eval set -- "$temp" while true; do case $1 in @@ -74,6 +75,7 @@ while true; do -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;; -n) dry_run=true; dry_run_arg=-n; shift ;; -p) progress_arg="--progress"; shift ;; + --pull-reexec) pull_reexec=true; shift ;; -q) verbose=false; verbose_arg=; progress_arg=; shift ;; # source host to receive a backup from -s) source=$2; shift 2 ;; @@ -90,18 +92,27 @@ done # usefull commands are resume and archive cmd_arg=${1:-run} -if [[ -s $default_args_file ]]; then - echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):" - cat $default_args_file - sleep 5 -fi - if [[ -v targets && $source ]]; then echo "$0: error: -t and -s are mutually exclusive" >&2 exit 1 fi +# pull_reexec stops us from getting into an infinite loop if there is some +# kind of weird problem +if ! $pull_reexec && [[ $source ]]; then + tmpf=$(mktemp) + scp $source:/a/bin/distro-setup/btrbk-run $tmpf + if diff -q $tmpf $BASH_SOURCE; then + echo "$0: found newer version on host $source. reexecing" + install -T $tmpf /usr/local/bin/btrbk-run + /usr/local/bin/btrbk-run --pull-reexec "${orig_args[@]}" + fi +fi + + echo -e "$0: options: conf_only=$conf_only\ndry_run=$dry_run\nrate_limit=$rate_limit\nverbose=$verbose\ncmd_arg=$cmd_arg" +### end options parsing + # set default targets if [[ ! -v targets && ! $source ]]; then diff --git a/distro-end b/distro-end index d51cbef..655ae7f 100755 --- a/distro-end +++ b/distro-end @@ -14,7 +14,7 @@ # limitations under the License. ### setup -errcatch +source /a/bin/errhandle/err src="${BASH_SOURCE%/*}" source $src/pkgs @@ -110,11 +110,13 @@ if isdeb; then # https://store.docker.com/editions/community/docker-ce-server-debian?tab=description pi software-properties-common apt-transport-https curl -fsSL https://download.docker.com/linux/$(distro-name-compat)/gpg | sudo apt-key add - - sudo add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/$(distro-name-compat) \ - $(debian-codename-compat) \ - stable" - p update + url=https://download.docker.com/linux/$(distro-name-compat) + l="deb [arch=amd64] $url $(debian-codename-compat) stable" + + if ! grep -xFq "$l" /etc/apt/sources.list{,.d/*.list}; then + sudo add-apt-repository $l + p update + fi # docker eats up a fair amount of cpu when doing nothing, so don't enable it unless # we really need it. pi-nostart docker-ce @@ -144,8 +146,11 @@ case $distro in trisquel|ubuntu) # not packaged in xenial or flidas pi software-properties-common - s add-apt-repository -y ppa:certbot/certbot ||: - p update + l="deb http://ppa.launchpad.net/certbot/certbot/ubuntu xenial main" + if ! grep -xFq "$l" /etc/apt/sources.list{,.d/*.list}; then + s add-apt-repository -y ppa:certbot/certbot ||: + p update + fi pi python-certbot-apache ;; # todo: other distros unknown @@ -163,15 +168,15 @@ sgo certbotmail.timer ### end certbot install ### -# dogcam setup -case $HOSTNAME in - lj|li) - /a/bin/webcam/install-server - ;; - kw) - /a/bin/webcam/install-client - ;; -esac +# dogcam setup. not using atm +# case $HOSTNAME in +# lj|li) +# /a/bin/webcam/install-server +# ;; +# kw) +# /a/bin/webcam/install-client +# ;; +# esac pi ${p1[@]} @@ -578,25 +583,13 @@ esac # dependent packages. pi ${pall[@]} $(apt-cache search ruby[.0-9]+-doc| awk '{print $1}') $(apt-cache depends gcc|grep -i suggests:| awk '{print $2}') -sgo fsf-vpn-dns-cleanup - -case $distro in - trisquel|ubuntu) - l="deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" - f=/etc/apt/sources.list.d/ansible-ubuntu-ansible-xenial.list - if ! grep -qF "$l" $f; then - s add-apt-repository -y ppa:ansible/ansible - p update - fi - s pip install --upgrade pip - # newer 2.7 versions have a bug that incorrectly detects trisquel version. fixed once 2.8 arrives - # in 2019 - pip install --user ansible=2.7.4 - #pi ansible - ;; -esac - +if ! type pip; then + x=$(mktemp) + wget -O$x https://bootstrap.pypa.io/get-pip.py + python3 $x --user +fi +sgo fsf-vpn-dns-cleanup case $distro in @@ -604,11 +597,17 @@ case $distro in pi chromium ;; trisquel|ubuntu) wget -qO - https://downloads.iridiumbrowser.de/ubuntu/iridium-release-sign-01.pub|sudo apt-key add - - cat <$t </dev/null <$t < /dev/null; then + case $distro in + arch) + s groupadd -g 450 debian-transmission + s useradd \ + --system \ + --create-home \ + --gid 450 \ + --uid 450 \ + --home-dir /var/lib/transmission-daemon \ + --shell /bin/false \ + debian-transmission + ;; + *) + s adduser --quiet \ + --gid 450 \ + --uid 450 \ + --system \ + --group \ + --no-create-home \ + --disabled-password \ + --home /var/lib/transmission-daemon \ + debian-transmission + ;; + esac +fi +# We want group writable stuff from transmission. +# However, after setting this, I learn that transmission sets it's +# own umask based on it's settings file. Well, no harm leaving this +# so it's set right from the beginning. +s chfn debian-transmission -o umask=0002 + case $distro in debian|trisquel|ubuntu) # note i had to do this, which is persistent: @@ -1093,6 +1137,9 @@ EOF # some reason it doesn\'t seem to start automatically anyways pi-nostart transmission-daemon + # be extra sure its not started + ser disable transmission-daemon + ser stop transmission-daemon # the folder was moved here after an install around 02/2017. # it contains runtime data, @@ -1140,40 +1187,11 @@ File.write(p, JSON.pretty_generate(JSON.parse(File.read(p)).merge({ })) + "\n") EOF - # make sure its not enabled, not sure if this is needed - ser disable transmission-daemon ;; # todo: others unknown esac -# adapted from /var/lib/dpkg/info/transmission-daemon.postinst -if ! getent passwd debian-transmission > /dev/null; then - case $distro in - arch) - s useradd \ - --system \ - --create-home \ - --home-dir /var/lib/transmission-daemon \ - --shell /bin/false \ - debian-transmission - ;; - *) - s adduser --quiet \ - --system \ - --group \ - --no-create-home \ - --disabled-password \ - --home /var/lib/transmission-daemon \ - debian-transmission - ;; - esac -fi -# We want group writable stuff from transmission. -# However, after setting this, I learn that transmission sets it's -# own umask based on it's settings file. Well, no harm leaving this -# so it's set right from the beginning. -s chfn debian-transmission -o umask=0002 # trisquel 8 = openvpn, debian stretch = openvpn-client vpn_ser=openvpn-client @@ -1316,8 +1334,17 @@ for x in iank traci; do s usermod -a -G libvirt,kvm $x; done case $distro in debian|trisquel|ubuntu) + if ! type -p go &>/dev/null; then + cd $(mktemp -d) + # just the latest stable at the time of writing + # TODO, maybe put this all into a build script, + # and do some automatic updates + wget https://dl.google.com/go/go1.12.4.linux-amd64.tar.gz + s tar -C /usr/local -xzf go1.12.4.linux-amd64.tar.gz + rm -f * + fi + go get -u mvdan.cc/fdroidcl # a bit of googling, and added settings to bashrc - go get -u github.com/mvdan/fdroidcl/cmd/fdroidcl ;; esac @@ -1347,7 +1374,7 @@ case $distro in ./configure --with-exuberant-ctags=/usr/bin/ctags make s make install - s pip install pygments + pip install pygments ;; *) pi global diff --git a/distro-pkgs b/distro-pkgs index 62c877a..21d4462 100755 --- a/distro-pkgs +++ b/distro-pkgs @@ -191,14 +191,6 @@ case $distro in # others unknown esac -case $distro in - debian|trisquel|ubuntu) - # fdroidcl dependency - e golang-go - ;; - # others unknown -esac - ### stuff brought in through deps in other distros case $distro in diff --git a/mail-setup b/mail-setup index 5730788..c1db3c7 100755 --- a/mail-setup +++ b/mail-setup @@ -374,8 +374,92 @@ exim4-config exim4/use_split_config boolean true EOF source /a/bin/bash_unpublished/source-semi-priv - mkdir -p /etc/exim4/conf.d/main + mkdir -p /etc/exim4/conf.d/{main,transport,auth,router} + cat >/etc/exim4/rcpt_local_acl <<'EOF' +# Only hosts we control send to mail.iankelling.org, so make sure +# they are all authed. +# Note, if we wanted authed senders for all domains, +# we could make this condition in acl_check_mail +deny + message = ian trusted domain recepient but no auth + !authenticated = * + domains = mail.iankelling.org +EOF + cat >/etc/exim4/data_local_acl <<'EOF' +# Except for the "condition =", this was +# a comment in the check_data acl. The comment about this not +# being suitable is mostly bs. The only thing related I found was to +# add the condition =, cuz spamassassin has problems with big +# messages and spammers don't bother with big messages, +# but I've increased the size from 10k +# suggested in official docs, and 100k in the wiki example because +# those docs are rather old and I see a 110k spam message +# pretty quickly looking through my spam folder. + warn + condition = ${if < {$message_size}{2000K}} + spam = Debian-exim:true + add_header = X-Spam_score: $spam_score\n\ + X-Spam_score_int: $spam_score_int\n\ + X-Spam_bar: $spam_bar\n\ + X-Spam_report: $spam_report + +EOF + cat >/etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' +# from 30_exim4-config_examples + +plain_server: +driver = plaintext +public_name = PLAIN +server_condition = "${if crypteq{$auth3}{${extract{1}{:}{${lookup{$auth2}lsearch{CONFDIR/passwd}{$value}{*:*}}}}}{1}{0}}" +server_set_id = $auth2 +server_prompts = : +.ifndef AUTH_SERVER_ALLOW_NOTLS_PASSWORDS +server_advertise_condition = ${if eq{$tls_in_cipher}{}{}{*}} +.endif +EOF + + cat >/etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF' +### router/900_exim4-config_local_user +################################# + +# This router matches local user mailboxes. If the router fails, the error +# message is "Unknown user". + +local_user: + debug_print = "R: local_user for $local_part@$domain" + driver = accept + domains = +local_domains +# ian: commented this, in conjunction with a dovecot lmtp +# change so I get mail for all users. +# check_local_user + local_parts = ! root + transport = LOCAL_DELIVERY + cannot_route_message = Unknown user +EOF + cat >/etc/exim4/conf.d/transport/30_exim4-config_dovecot_lmtp <<'EOF' +dovecot_lmtp: + driver = lmtp + socket = /var/run/dovecot/lmtp + #maximum number of deliveries per batch, default 1 + batch_max = 200 +EOF + + cat >/etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost <<'EOF' +# smarthost for fsf mail +# ian: copied from /etc/exim4/conf.d/router/200_exim4-config_primary, and added senders = and +# replaced DCsmarthost with mail.fsf.org +fsfsmarthost: + debug_print = "R: smarthost for $local_part@$domain" + driver = manualroute + domains = ! +local_domains + senders = *@fsf.org + transport = remote_smtp_smarthost + route_list = * mail.fsf.org byname + host_find_failed = ignore + same_domain_copy_routing = yes + no_more +EOF #### begin mail cert setup ### @@ -392,7 +476,7 @@ if [[ -e $f ]]; then fi if [[ $HOSTNAME == $MAIL_HOST ]]; then local_mx=mail.iankelling.org - rsync_common="rsync -ogtL --chown=root:Debian-exim --chmod=640 root@li:/etc/letsencrypt/live/$local_mx/" + rsync_common="rsync -ogtL --chown=root:Debian-exim --chmod=640 root@li.iankelling.org:/etc/letsencrypt/live/$local_mx/" ${rsync_common}fullchain.pem /etc/exim4/exim.crt ret=$? ${rsync_common}privkey.pem /etc/exim4/exim.key @@ -776,12 +860,12 @@ EOF find / /nocow -xdev -gid $gid -exec chgrp -h 608 {} + fi + # light version of exim does not have sasl auth support. pi exim4-daemon-heavy spamassassin - ##### begin spamassassin config systemctl enable spamassassin # per readme.debian @@ -828,91 +912,6 @@ EOF - cat >/etc/exim4/rcpt_local_acl <<'EOF' -# Only hosts we control send to mail.iankelling.org, so make sure -# they are all authed. -# Note, if we wanted authed senders for all domains, -# we could make this condition in acl_check_mail -deny - message = ian trusted domain recepient but no auth - !authenticated = * - domains = mail.iankelling.org -EOF - cat >/etc/exim4/data_local_acl <<'EOF' -# Except for the "condition =", this was -# a comment in the check_data acl. The comment about this not -# being suitable is mostly bs. The only thing related I found was to -# add the condition =, cuz spamassassin has problems with big -# messages and spammers don't bother with big messages, -# but I've increased the size from 10k -# suggested in official docs, and 100k in the wiki example because -# those docs are rather old and I see a 110k spam message -# pretty quickly looking through my spam folder. - warn - condition = ${if < {$message_size}{2000K}} - spam = Debian-exim:true - add_header = X-Spam_score: $spam_score\n\ - X-Spam_score_int: $spam_score_int\n\ - X-Spam_bar: $spam_bar\n\ - X-Spam_report: $spam_report - -EOF - cat >/etc/exim4/conf.d/auth/29_exim4-config_auth <<'EOF' -# from 30_exim4-config_examples - -plain_server: -driver = plaintext -public_name = PLAIN -server_condition = "${if crypteq{$auth3}{${extract{1}{:}{${lookup{$auth2}lsearch{CONFDIR/passwd}{$value}{*:*}}}}}{1}{0}}" -server_set_id = $auth2 -server_prompts = : -.ifndef AUTH_SERVER_ALLOW_NOTLS_PASSWORDS -server_advertise_condition = ${if eq{$tls_in_cipher}{}{}{*}} -.endif -EOF - - cat >/etc/exim4/conf.d/router/900_exim4-config_local_user <<'EOF' -### router/900_exim4-config_local_user -################################# - -# This router matches local user mailboxes. If the router fails, the error -# message is "Unknown user". - -local_user: - debug_print = "R: local_user for $local_part@$domain" - driver = accept - domains = +local_domains -# ian: commented this, in conjunction with a dovecot lmtp -# change so I get mail for all users. -# check_local_user - local_parts = ! root - transport = LOCAL_DELIVERY - cannot_route_message = Unknown user -EOF - cat >/etc/exim4/conf.d/transport/30_exim4-config_dovecot_lmtp <<'EOF' -dovecot_lmtp: - driver = lmtp - socket = /var/run/dovecot/lmtp - #maximum number of deliveries per batch, default 1 - batch_max = 200 -EOF - - cat >/etc/exim4/conf.d/router/190_exim4-config_fsfsmarthost <<'EOF' -# smarthost for fsf mail -# ian: copied from /etc/exim4/conf.d/router/200_exim4-config_primary, and added senders = and -# replaced DCsmarthost with mail.fsf.org -fsfsmarthost: - debug_print = "R: smarthost for $local_part@$domain" - driver = manualroute - domains = ! +local_domains - senders = *@fsf.org - transport = remote_smtp_smarthost - route_list = * mail.fsf.org byname - host_find_failed = ignore - same_domain_copy_routing = yes - no_more -EOF - # https://blog.dhampir.no/content/make-exim4-on-debian-respect-forward-and-etcaliases-when-using-a-smarthost # i only need .forwards, so just doing that one. cd /etc/exim4/conf.d/router diff --git a/offlineimap-sync b/offlineimap-sync index 4fac247..74b23c1 100755 --- a/offlineimap-sync +++ b/offlineimap-sync @@ -5,9 +5,22 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR # mail-route can get messed up a bit randomly, I don't know why. #/b/ds/mail-route up | /b/log-quiet/log-once -1 mail-route +for f in $(awk '$1 == "localfolders" {print $NF}' ~/.offlineimaprc); do + mkdir -p $f + chmod 700 $f +done + offlineimap -u quiet shopt -s nullglob +if grep -qP '^ *accounts.*fsf' ~/.offlineimaprc; then + if [[ ! -e /nocow/user/.mufsf ]]; then + mkdir -p /nocow/user/.mufsf + chmod 700 /nocow/user/.mufsf + mu index --maildir=/nocow/user/fsfmd + fi +fi + omv() { # offlineimap mv. move mail files within $src_base/$1 to /m/md/$2 src="$1" dst="$2" diff --git a/path_add-function b/path_add-function index f03451a..540306c 100644 --- a/path_add-function +++ b/path_add-function @@ -21,9 +21,9 @@ path_add() { local help="usage: path_add [options] PATH --help: print this --end: adds to end of path, which will give it lowest priority ---force: add to path even if directory does not exist" +--ifexists: add to path only if directory exists" local found x y z ifexists end loop newpath - force=false + ifexists=false end=false loop=true # portable substring matching is ugly http://mywiki.wooledge.org/BashFAQ/041 @@ -32,8 +32,8 @@ path_add() { --*) if [ "$1" = --end ]; then end=true - elif [ "$1" = --force ]; then - force=true + elif [ "$1" = --ifexists ]; then + ifexists=true elif [ "$1" = --help ]; then echo "$help" return @@ -67,7 +67,7 @@ path_add() { unset IFS PATH="$newpath" for x in "$@"; do - if $force || [ -d "$x" ]; then + if ! $ifexists || [ -d "$x" ]; then if [ ! "$PATH" ]; then PATH="$x" elif $end; then diff --git a/pkgs b/pkgs index d3b0744..13c6ba9 100644 --- a/pkgs +++ b/pkgs @@ -53,6 +53,7 @@ p3=( # gnat-5 & ccache is for coreboot p4=( adb + ansible apache2 apache2-doc apt-doc @@ -153,7 +154,6 @@ p4=( pry pv python-autopep8 - python-pip python3-doc qrencode readline-doc