From 6459d4698ef983f7b1540589b87013c76913393d Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Sun, 11 Aug 2019 19:44:27 -0400 Subject: [PATCH] changes for buster linode --- brc | 13 +- btrbk-run | 2 +- certbot-renew-hook | 4 + checkrestart-blacklist | 1 - conflink | 6 +- disabled/docker | 24 + disabled/mastodon | 128 +++++ disabled/pump.io | 107 +++++ distro-begin | 286 +++++------ distro-end | 604 +++++++----------------- dynamic-ip-update | 9 +- filesystem/etc/ansible/hosts | 4 +- filesystem/usr/local/bin/mycheckrestart | 7 +- filesystem/usr/local/bin/myupgrade | 6 +- keyscript-off | 2 +- keyscript-on | 28 +- mail-setup | 26 +- pkgs | 11 +- rootsshsync | 12 +- switch-mail-host | 4 +- vpn-mail-forward | 8 +- 21 files changed, 648 insertions(+), 644 deletions(-) delete mode 100644 checkrestart-blacklist create mode 100644 disabled/docker create mode 100644 disabled/mastodon create mode 100644 disabled/pump.io diff --git a/brc b/brc index 5df2c27..9fff367 100644 --- a/brc +++ b/brc @@ -426,6 +426,14 @@ bpush() { install-my-scripts switch-mail-host $HOSTNAME $1 } +lipush() { + rsync $@ --delete-excluded -ahviSAXPH --specials --devices --delete --relative \ + --exclude-from=/p/c/li-rsync-excludes /a/bin /a/exe /a/h /a/c /p/c/machine_specific/li /a/opt/{emacs,emacs-debianstable,mu} root@li:/ +} +lipushnoe() { + rsync $@ --delete-excluded -ahviSAXPH --specials --devices --delete --relative \ + --exclude-from=/p/c/li-rsync-excludes /a/bin /a/exe /a/h /a/c /p/c/machine_specific/li root@li:/ +} btc() { @@ -1554,7 +1562,7 @@ rspicy() { # usage: HOST DOMAIN sudo virsh dumpxml $2|grep ".env.production <<'EOF' +REDIS_HOST=redis +REDIS_PORT=6379 +DB_HOST=db +DB_USER=postgres +DB_NAME=postgres +DB_PASS= +DB_PORT=5432 + +LOCAL_DOMAIN=mast.iankelling.org +LOCAL_HTTPS=true + +SINGLE_USER_MODE=true + +SMTP_SERVER=mail.iankelling.org +SMTP_PORT=25 +SMTP_LOGIN=li +SMTP_FROM_ADDRESS=notifications@mast.iankelling.org +SMTP_DOMAIN=mast.iankelling.org +SMTP_DELIVERY_METHOD=smtp +EOF + + for key in PAPERCLIP_SECRET SECRET_KEY_BASE OTP_SECRET; do + # 1 minute 7 seconds to run this docker command + # to generate a secret, and it has ^M chars at the end. wtf. really dumb + printf "%s=%s\n" $key "$(docker-compose run --rm web rake secret|dos2unix|tail -n1)" >>.env.production + done + found=false + while read -r domain _ pass; do + if [[ $domain == mail.iankelling.org ]]; then + found=true + # remove the username part + pass="${pass#*:}" + printf "SMTP_PASSWORD=%s\n" "$pass" >>.env.production + break + fi + done < <(s cat /etc/mailpass) + if ! $found; then + echo "$0: error, failed to find mailpass domain for mastadon" + exit 1 + fi + + # docker compose makes an interface named like br-8f3e208558f2. we need mail to + # get routed to us. + if ! s /sbin/iptables -t nat -C PREROUTING -i br-+ -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25; then + s /sbin/iptables -t nat -A PREROUTING -i br-+ -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25 + fi + + docker-compose run --rm web rake mastodon:webpush:generate_vapid_key | grep -E '^VAPID_PUBLIC_KEY=|^VAPID_PRIVATE_KEY=' >> .env.production + logq docker-compose run --rm web rake db:migrate + docker-compose run --rm web rails assets:precompile + + # avatar failed to upload, did + # docker logs mastodon_web_1 + # google lead me to this + s chown -R 991:991 public/system + + # docker daemon takes care of starting on boot. + docker-compose up -d + + s a2enmod proxy_wstunnel headers + web-conf -f 3000 - apache2 mast.iankelling.org <<'EOF' + ProxyPreserveHost On + RequestHeader set X-Forwarded-Proto "https" + ProxyPass /500.html ! + ProxyPass /oops.png ! + ProxyPass /api/v1/streaming/ ws://localhost:4000/ + ProxyPassReverse /api/v1/streaming/ ws://localhost:4000/ + ErrorDocument 500 /500.html + ErrorDocument 501 /500.html + ErrorDocument 502 /500.html + ErrorDocument 503 /500.html + ErrorDocument 504 /500.html +EOF + + + ############### !!!!!!!!!!!!!!!!! + ############### manual steps: + + # only following a few people atm, so not bothering to figure out backups + # when mastodon has not documented it at all. + # + # fsf@status.fsf.org + # cwebber@toot.cat + # dbd@status.fsf.org + # johns@status.fsf.org + + # sign in page is at https://mast.iankelling.org/auth/sign_in + # register as iank, then + # https://github.com/tootsuite/documentation/blob/master/Running-Mastodon/Administration-guide.md + # docker-compose run --rm web bundle exec rails mastodon:make_admin USERNAME=iank + + ############# end setup mastodon ############## diff --git a/disabled/pump.io b/disabled/pump.io new file mode 100644 index 0000000..ebb10b9 --- /dev/null +++ b/disabled/pump.io @@ -0,0 +1,107 @@ + ########## begin pump.io setup ########## + + # once pump adds a logrotation script, turn off nologger, + # and add + # "logfile": "/var/log/pumpio/pumpio.log", + # + s dd of=/etc/pump.io.json <<'EOF' +{ + "secret": "SECRET_REPLACE_ME", + "driver": "mongodb", + "params": { "dbname": "pumpio" }, + "noweb": false, + "site": "pump.iankelling.org", + "owner": "Ian Kelling", + "ownerURL": "https://iankelling.org/", + "port": 8001, + "urlPort": 443, + "hostname": "pump.iankelling.org", + "nologger": true, + "datadir": "/home/pumpio/pumpdata", + "enableUploads": true, + "debugClient": false, + "disableRegistration": true, + "noCDN": true, + "key": "/home/pumpio/privkey.pem", + "cert": "/home/pumpio/fullchain.pem", + "address": "localhost", + "sockjs": false +} +EOF + s sed -i "s#SECRET_REPLACE_ME#$(cat /p/c/machine_specific/li/pump-secret)#" /etc/pump.io.json + + # stretch node is too old + # https://nodejs.org/en/download/package-manager/ + curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - + pi nodejs graphicsmagick mongodb + cd /home/iank + if [[ -e pump.io ]]; then + cd pump.io + git pull + else + git clone https://github.com/pump-io/pump.io.git + cd pump.io + fi + # note: these 2 commands seem + # note: doing this or the npm install pump.io as root had problems. + npm install + npm run build + # normally, next command would be + # s npm install -g odb + # but it\'s this until a bug in pump gets fixed + # https://github.com/pump-io/pump.io/issues/1287 + s npm install -g databank-mongodb@0.19.2 + if ! getent passwd pumpio &>/dev/null; then + s useradd -Um -s /bin/false pumpio + fi + sudo -u pumpio mkdir -p /home/pumpio/pumpdata + # for testing browser when only listening to localhost, + # in the pump.io.json, set hostname localhost, urlPort 5233 + #ssh -L 5233:localhost:5233 li + + s mkdir -p /var/log/pumpio/ + s chown pumpio:pumpio /var/log/pumpio/ + + web-conf - apache2 pump.iankelling.org <<'EOF' +# currently a bug in pump that we cant terminate ssl + SSLProxyEngine On + ProxyPreserveHost On + ProxyPass / https://127.0.0.1:8001/ + ProxyPassReverse / https://127.0.0.1:8001/ + # i have sockjs disabled per people suggesting that + # it won\'t work with apache right now. + # not sure if it would work with this, + # but afaik, this is pointless atm. + + ProxyPass wss://127.0.0.1:8001/main/realtime/sockjs/ + ProxyPassReverse wss://127.0.0.1:8001/main/realtime/sockjs/ + +EOF + + sudo -i <<'EOF' +export RENEWED_LINEAGE=/etc/letsencrypt/live/pump.iankelling.org +/a/bin/distro-setup/certbot-renew-hook +EOF + + s dd of=/etc/systemd/system/pump.service <<'EOF' +[Unit] +Description=pump.io +After=syslog.target network.target mongodb.service +Requires=mongodb.service + +[Service] +Type=simple +User=pumpio +Group=pumpio +ExecStart=/home/iank/pump.io/bin/pump +Environment=NODE_ENV=production +# failed to find databank-mongodb without this. +# I just looked at my environment variables took a guess. +Environment=NODE_PATH=/usr/lib/nodejs:/usr/lib/node_modules:/usr/share/javascript + +[Install] +WantedBy=multi-user.target +EOF + ser daemon-reload + sgo pump + ########## end pump.io setup ############ diff --git a/distro-begin b/distro-begin index 633e8d1..b2b21f7 100755 --- a/distro-begin +++ b/distro-begin @@ -14,7 +14,8 @@ # limitations under the License. # for setting up a new machine -# usage: $0 [-r] HOSTNAME +# usage: $0 [-r] [HOSTNAME] +# HOSTNAME changes the machine's hostname # tips: # run any sudo command first so your pass is cached @@ -35,6 +36,7 @@ if ! $interactive; then set -x fi source /a/bin/errhandle/err +source /a/bin/distro-functions/src/package-manager-abstractions ### setup logging exec &> >(sudo tee -a /var/log/distro-begin) @@ -54,9 +56,11 @@ fi ### arg parsing recompile=false +emacs=true while [[ $1 == -* ]]; do case $1 in -r) recompile=true; shift ;; + -e) emacs=false; shift ;; esac done if [[ $1 ]]; then @@ -71,17 +75,19 @@ source $script_dir/pkgs set +x source /a/bin/distro-functions/src/identify-distros $interactive || set -x -for f in iank-dev htpc kd x2 x3 frodo tp li lj demohost kw fz; do +for f in kd x2 x3 frodo tp li lj demohost kw; do eval "$f() { [[ $HOSTNAME == $f ]]; }" done codename=$(debian-codename) -has_wayland() { [[ $codename == buster ]]; } -has_x() { [[ $codename != buster ]]; } +has_wayland() { has_monitor && [[ $codename == buster ]]; } +has_x() { has_monitor && [[ $codename != buster ]]; } has_monitor() { ! linode ; } linode() { lj || li; } +# linode actually has btrfs now, but we dont do anything with it. has_btrfs() { ! linode; } -home_network() { ! linode; } -encrypted() { has_p; } +home_network() { ! linode && ! kw; } +has_p() { ! linode; } +encrypted() { true; } shopt -s extglob export GLOBIGNORE="*/.:*/.." umask 022 @@ -138,14 +144,24 @@ EOF sudo systemctl enable keyscriptoff.service sudo systemctl start keyscriptoff.service + pi rsync + # from /usr/share/doc/dropbear-initramfs/README.initramfs.gz + tmp=$(mktemp) while read -r m _; do /sbin/modinfo -F filename "$m"; done /dev/null; then + sudo dd if=$tmp of=/etc/initramfs-tools/modules + sudo /usr/sbin update-initramfs -u -k all + fi # initram auth keys get setup with rootsshsync + $script_dir/rootsshsync # then for remote unlock, ssh and do this once per crypt disk: # echo -n PASS >/lib/cryptsetup/passfifo + # or for buster+ + # cryptroot-unlock + fi ##### end setup encryption scripts ###### @@ -155,19 +171,6 @@ fi # todo, it would be nice to cut down on some of the output -##### fedora prereq/fundamental settings -if isfedora; then - # comment out line disallowing calling sudo in scripts - sudo $sed -i 's/^Defaults *requiretty/#\0 # ian commented/' /etc/sudoers - # turn on magic sysrq commands - echo 1 > sudo dd of=/proc/sys/kernel/sysrq - echo "kernel.sysrq = 1" > /etc/sysctl.d/90-sysrq.conf - # selinux is not user friendly. Like, you enable samba, but you haven't run the magic selinux commands so it doesn't work - # and you have no idea why. - sudo $sed -i 's/^\(SELINUX=\).*/\1disabled/' /etc/selinux/config - selinuxenabled && sudo setenforce 0 -fi - #### rerun my fai-time scripts # already ran for pxe installs, but used for vps & updates @@ -183,6 +186,10 @@ case $distro in esac ###### setup hostname +if [[ $HOSTNAME != $(cat /etc/hostname) ]]; then + echo $HOSTNAME > /etc/hostname + hostname -F /etc/hostname +fi sudo $sed -i '/^127\.0\.1\.1/d' /etc/hosts echo "127.0.1.1 $HOSTNAME.b8.nz $HOSTNAME" | sudo tee -a /etc/hosts @@ -221,16 +228,20 @@ $interactive || set -x #### setup passwordless sudo -tu /etc/sudoers < *{/,/}/s/KP_Multiply/underscore/g' /usr/share/X11/xkb/symbols/keypad -fi - #### arch specific early packages @@ -385,22 +344,12 @@ case $(distro-name) in ;; esac -#### fedora specific packages -case $(distro-name) in - fedora) - # todo, this could probably come later - p -y groupinstall development-tools c-development books admin-tools - pi man-pages - ;; - # other distros unknown -esac - #### enable trim # enable trim for volume delete, other rare commands sudo $sed -ri 's/( *issue_discards\b).*/\1 = 1/' /etc/lvm/lvm.conf if encrypted; then # flidas or so, these units arent built-in - if isdeb && ! systemctl list-unit-files | grep -xF fstrim.timer &>/dev/null ; then + if isdeb && ! systemctl list-unit-files | grep ^fstrim.timer &>/dev/null ; then sudo cp /usr/share/doc/util-linux/examples/fstrim.{service,timer} /etc/systemd/system fi # does weekly trim @@ -413,15 +362,15 @@ s mkdir -p "${dirs[@]}" s chown $USER:$USER "${dirs[@]}" ###### setup /i -tu /etc/fstab <<'EOF' +if home_network; then + tu /etc/fstab <<'EOF' /i/w /w none bind,noauto 0 0 /i/k /k none bind,noauto 0 0 EOF -if ! mountpoint /kr; then - s mkdir -p /kr - s chown $USER:user2 /kr -fi -if home_network; then + if ! mountpoint /kr; then + s mkdir -p /kr + s chown $USER:user2 /kr + fi if [[ $HOSTNAME == frodo ]]; then tu /etc/fstab <<'EOF' /k /kr none bind,noauto 0 0 @@ -431,22 +380,20 @@ EOF frodo:/k /kr nfs noauto 0 0 EOF fi -fi -s mkdir -p /q /i/{w,k} -for dir in /{i,w,k}; do - if mountpoint $dir; then continue; fi # already mounted - s mkdir -p $dir - s chown $USER:$USER $dir -done -# not needed for all hosts, but rather just keep it uniform -s mkdir -p /mnt/iroot -# debian auto mounting of multi-disk encrypted btrfs is busted. It is -# in jessie, and in stretch as of 11/26/2016 I have 4 disks in cryptab, -# based on 3 of those, it creates .device units for /dev/mapper/dev... -# then waits endlessly for them on bootup, after the /dev/mapper disks -# have already been created and exist. todo: create a simple repro -# for this in a vm and report it upstream. -if has_btrfs || home_network; then + s mkdir -p /q /i/{w,k} + for dir in /{i,w,k}; do + if mountpoint $dir; then continue; fi # already mounted + s mkdir -p $dir + s chown $USER:$USER $dir + done + # not needed for all hosts, but rather just keep it uniform + s mkdir -p /mnt/iroot + # debian auto mounting of multi-disk encrypted btrfs is busted. It is + # in jessie, and in stretch as of 11/26/2016 I have 4 disks in cryptab, + # based on 3 of those, it creates .device units for /dev/mapper/dev... + # then waits endlessly for them on bootup, after the /dev/mapper disks + # have already been created and exist. todo: create a simple repro + # for this in a vm and report it upstream. pi nfs-common s dd of=/root/imount <<'EOF' #!/bin/bash @@ -465,7 +412,6 @@ for dir in /i /mnt/iroot /k /kr /w; do done EOF s chmod +x /root/imount - s dd of=/etc/systemd/system/imount.service < *{/,/}/s/KP_Multiply/underscore/g' /usr/share/X11/xkb/symbols/keypad +fi + +##### basic graphical packages +if has_monitor; then + pi konsole suckless-tools +fi -#### refix interactive ssh terminal -# the first pup command can kill off our /etc/ mod, so rerun this -/a/exe/ssh-emacs-setup +##### install emacs +if $emacs; then + if isarch; then + # emacs git build was broken last time i checked, + x=$(mktemp -d) + pushd $x + aurex emacs-git + makepkg -si --noconfirm + popd + rm -rf $x + pi hunspell hunspell-en + else + if $recompile; then + /a/bin/buildscripts/emacs + else + /a/bin/buildscripts/emacs --no-r + fi + fi + # the first pup command can kill off our /etc/ mod, so rerun this + /a/exe/ssh-emacs-setup +fi echo "$0: $(date): ending now" exit 0 diff --git a/distro-end b/distro-end index 14cc2f5..2e6b687 100755 --- a/distro-end +++ b/distro-end @@ -50,29 +50,13 @@ if isdeb; then pi aptitude fi -# avoid ptompts! +# avoid prompts s debconf-set-selections < /dev/null; then +if isdeb && pcheck apt-file; then # this condition is just a speed optimization pi apt-file s apt-file update @@ -117,30 +98,6 @@ case $distro in esac -### begin docker install #### -if isdeb; then - # https://store.docker.com/editions/community/docker-ce-server-debian?tab=description - pi software-properties-common apt-transport-https - curl -fsSL https://download.docker.com/linux/$(distro-name-compat)/gpg | sudo apt-key add - - url=https://download.docker.com/linux/$(distro-name-compat) - l="deb [arch=amd64] $url $codename_compat stable" - - if ! grep -xFq "$l" /etc/apt/sources.list{,.d/*.list}; then - sudo add-apt-repository "$l" - p update - fi - # docker eats up a fair amount of cpu when doing nothing, so don't enable it unless - # we really need it. - pi-nostart docker-ce - # and docker is even more crap, it ignores that it shouldnt start - ser stop docker - ser disable docker - case $HOSTNAME in - li|lj) sgo docker ;; - esac -fi -### end docker install #### - ### begin certbot install ### @@ -183,72 +140,39 @@ sgo certbotmail.timer # ;; # esac -pi ${p1[@]} -##### begin automatic upgrades #### - -s dd of=/etc/apt/apt.conf.d/10periodic <<'EOF' -# this file was mostly just comments. -APT::Periodic::Update-Package-Lists "1"; -APT::Periodic::Download-Upgradeable-Packages "1"; -APT::Periodic::AutocleanInterval "7"; -APT::Periodic::Unattended-Upgrade "1"; -EOF - -s dd of=/etc/apt/apt.conf.d/50unattended-upgrades < +# AllowOverride None +# AuthType basic +# AuthName "Authentication Required" +# # setup one time, with root:www-data, 640 +# AuthUserFile "/etc/prometheus-htpasswd" +# Require valid-user +# +# EOF +# fi -# old names, too verbose -s rm -f /etc/cron.d/unattended-upgrade-reboot /usr/local/bin/zelous-unattended-reboot +# website setup +case $HOSTNAME in + li) -s dd of=/etc/cron.d/myupgrade <<'EOF' -20 7 * * * root /usr/local/bin/myupgrade | /usr/local/bin/log-once -1 myupgrade -0 * * * * root /usr/local/bin/mycheckrestart | /usr/local/bin/log-once -1 mycheckrestart -EOF -##### end automatic upgrades #### + pi bind9 -# office is not exposed to internet yet -if [[ $(hostname -f) != *.office.fsf.org ]]; then - ## prometheus node exporter setup - web-conf -f 9100 -p 9101 apache2 $(hostname -f) <<'EOF' -#https://httpd.apache.org/docs/2.4/mod/mod_authn_core.html#authtype -# https://stackoverflow.com/questions/5011102/apache-reverse-proxy-with-basic-authentication - - AllowOverride None - AuthType basic - AuthName "Authentication Required" - # setup one time, with root:www-data, 640 - AuthUserFile "/etc/prometheus-htpasswd" - Require valid-user - -EOF -fi -# website setup -case $HOSTNAME in - lj|li) case $HOSTNAME in - lj) domain=iank.bid; exit 0 ;; li) domain=iankelling.org ;; esac /a/h/setup.sh $domain /a/h/build.rb - sudo -E /a/bin/mediawiki-setup/mw-setup-script - + # start mumble only when im going to use it, since i dont use it much pi-nostart mumble-server s $sed -ri "s/^ *(serverpassword=).*/\1$(< /a/bin/bash_unpublished/mumble_pass)/" /etc/mumble-server.ini @@ -262,25 +186,17 @@ export RENEWED_LINEAGE=/etc/letsencrypt/live/mumble.iankelling.org /a/bin/distro-setup/certbot-renew-hook EOF - sgo mumble-server vpn-server-setup -rd s tee /etc/openvpn/client-config/mail <<'EOF' ifconfig-push 10.8.0.4 255.255.255.0 EOF - # it\'s strange. docker seems to make the default for forward - # be drop, but then I set it to accept and it\'s stuck that way, - # I dun know why. But, let\'s make sure we can forward anyways. - s DEBIAN_FRONTEND=noninteractive pi iptables-persistent - rm /etc/iptables/rules.v6 - s tee /etc/iptables/rules.v4 <<'EOF' -*filter --A FORWARD -i tun+ -o eth0 -j ACCEPT --A FORWARD -i eth0 -o tun+ -j ACCEPT -COMMIT -EOF - + if [[ -e /lib/systemd/system/openvpn-server@.service ]]; then + vpn_service=openvpn-server@server + else + vpn_service=openvpn@server + fi sudo dd of=/etc/systemd/system/vpnmail.service < - Options +FollowSymLinks +Multiviews +Indexes - AllowOverride None - AuthType basic - AuthName "Authentication Required" - # setup one time, with root:www-data, 640 - AuthUserFile "/etc/caldav-htpasswd" - Require valid-user - + + Options +FollowSymLinks +Multiviews +Indexes + AllowOverride None + AuthType basic + AuthName "Authentication Required" + # setup one time, with root:www-data, 640 + AuthUserFile "/etc/caldav-htpasswd" + Require valid-user + EOF # nginx version of above would be: # auth_basic "Not currently available"; # auth_basic_user_file /etc/nginx/caldav/htpasswd; - ########## begin pump.io setup ########## - # once pump adds a logrotation script, turn off nologger, - # and add - # "logfile": "/var/log/pumpio/pumpio.log", - # - s dd of=/etc/pump.io.json <<'EOF' -{ - "secret": "SECRET_REPLACE_ME", - "driver": "mongodb", - "params": { "dbname": "pumpio" }, - "noweb": false, - "site": "pump.iankelling.org", - "owner": "Ian Kelling", - "ownerURL": "https://iankelling.org/", - "port": 8001, - "urlPort": 443, - "hostname": "pump.iankelling.org", - "nologger": true, - "datadir": "/home/pumpio/pumpdata", - "enableUploads": true, - "debugClient": false, - "disableRegistration": true, - "noCDN": true, - "key": "/home/pumpio/privkey.pem", - "cert": "/home/pumpio/fullchain.pem", - "address": "localhost", - "sockjs": false -} -EOF - s sed -i "s#SECRET_REPLACE_ME#$(cat /p/c/machine_specific/li/pump-secret)#" /etc/pump.io.json - - # stretch node is too old - # https://nodejs.org/en/download/package-manager/ - curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - - pi nodejs graphicsmagick mongodb - cd /home/iank - if [[ -e pump.io ]]; then - cd pump.io - git pull - else - git clone https://github.com/pump-io/pump.io.git - cd pump.io - fi - # note: these 2 commands seem - # note: doing this or the npm install pump.io as root had problems. - npm install - npm run build - # normally, next command would be - # s npm install -g odb - # but it\'s this until a bug in pump gets fixed - # https://github.com/pump-io/pump.io/issues/1287 - s npm install -g databank-mongodb@0.19.2 - if ! getent passwd pumpio &>/dev/null; then - s useradd -Um -s /bin/false pumpio - fi - sudo -u pumpio mkdir -p /home/pumpio/pumpdata - # for testing browser when only listening to localhost, - # in the pump.io.json, set hostname localhost, urlPort 5233 - #ssh -L 5233:localhost:5233 li - - s mkdir -p /var/log/pumpio/ - s chown pumpio:pumpio /var/log/pumpio/ - - web-conf - apache2 pump.iankelling.org <<'EOF' -# currently a bug in pump that we cant terminate ssl - SSLProxyEngine On - ProxyPreserveHost On - ProxyPass / https://127.0.0.1:8001/ - ProxyPassReverse / https://127.0.0.1:8001/ - # i have sockjs disabled per people suggesting that - # it won\'t work with apache right now. - # not sure if it would work with this, - # but afaik, this is pointless atm. - - ProxyPass wss://127.0.0.1:8001/main/realtime/sockjs/ - ProxyPassReverse wss://127.0.0.1:8001/main/realtime/sockjs/ - -EOF + ###### begin znc setup ##### + pi znc + # https://wiki.znc.in/FAQ seems to imply that znc doesn\'t need restart after cert change. + # to get into the web interface, + # then use non-main browser or else it doebsn't allow it based on ocsp stapling from my main site. + # https://iankelling.org:12533/ sudo -i <<'EOF' -export RENEWED_LINEAGE=/etc/letsencrypt/live/pump.iankelling.org +export RENEWED_LINEAGE=/etc/letsencrypt/live/iankelling.org /a/bin/distro-setup/certbot-renew-hook EOF - s dd of=/etc/systemd/system/pump.service <<'EOF' -[Unit] -Description=pump.io -After=syslog.target network.target mongodb.service -Requires=mongodb.service - -[Service] -Type=simple -User=pumpio -Group=pumpio -ExecStart=/home/iank/pump.io/bin/pump -Environment=NODE_ENV=production -# failed to find databank-mongodb without this. -# I just looked at my environment variables took a guess. -Environment=NODE_PATH=/usr/lib/nodejs:/usr/lib/node_modules:/usr/share/javascript - -[Install] -WantedBy=multi-user.target -EOF - ser daemon-reload - sgo pump - ########## end pump.io setup ############ - - - ############# begin setup mastodon ############## - - # main doc is Docker-Guide.md in docs repo - - # I'd like to try gnu social just cuz of gnu, but it's not being - # well maintained, for example, simple pull requests - # languishing: - # https://git.gnu.io/gnu/gnu-social/merge_requests/143 - # and I submitted my own bugs, basic docs are broken - # https://git.gnu.io/gnu/gnu-social/issues/269 - - # note, docker required, but we installed it earlier - - # i subscrubed to https://github.com/docker/compose/releases.atom - # to see release notes. - # i had some problems upgrading. blew things away with - # docker-compose down - # docker rmi $(docker images -q) - # s reboot now - # when running docker-compose run, kernel stack traces are printed to the journal. - # things seem to succeed, google says nothing, so ignoring them. - curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-$(uname -s)-$(uname -m) | s dd of=/usr/local/bin/docker-compose - s chmod +x /usr/local/bin/docker-compose - - - cd ~ - s rm -rf mastodon - i clone https://github.com/tootsuite/mastodon - cd mastodon - # subbed to atom feed to deal with updates - git checkout $(git tag | grep -v rc | tail -n1) - - # per instructions, uncomment redis/postgres persistence in docker-compose.yml - sed -i 's/^#//' docker-compose.yml - - cat >.env.production <<'EOF' -REDIS_HOST=redis -REDIS_PORT=6379 -DB_HOST=db -DB_USER=postgres -DB_NAME=postgres -DB_PASS= -DB_PORT=5432 - -LOCAL_DOMAIN=mast.iankelling.org -LOCAL_HTTPS=true - -SINGLE_USER_MODE=true - -SMTP_SERVER=mail.iankelling.org -SMTP_PORT=25 -SMTP_LOGIN=li -SMTP_FROM_ADDRESS=notifications@mast.iankelling.org -SMTP_DOMAIN=mast.iankelling.org -SMTP_DELIVERY_METHOD=smtp -EOF - - for key in PAPERCLIP_SECRET SECRET_KEY_BASE OTP_SECRET; do - # 1 minute 7 seconds to run this docker command - # to generate a secret, and it has ^M chars at the end. wtf. really dumb - printf "%s=%s\n" $key "$(docker-compose run --rm web rake secret|dos2unix|tail -n1)" >>.env.production - done - found=false - while read -r domain _ pass; do - if [[ $domain == mail.iankelling.org ]]; then - found=true - # remove the username part - pass="${pass#*:}" - printf "SMTP_PASSWORD=%s\n" "$pass" >>.env.production - break - fi - done < <(s cat /etc/mailpass) - if ! $found; then - echo "$0: error, failed to find mailpass domain for mastadon" - exit 1 - fi - - # docker compose makes an interface named like br-8f3e208558f2. we need mail to - # get routed to us. - if ! s /sbin/iptables -t nat -C PREROUTING -i br-+ -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25; then - s /sbin/iptables -t nat -A PREROUTING -i br-+ -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.8.0.4:25 - fi - - docker-compose run --rm web rake mastodon:webpush:generate_vapid_key | grep -E '^VAPID_PUBLIC_KEY=|^VAPID_PRIVATE_KEY=' >> .env.production - logq docker-compose run --rm web rake db:migrate - docker-compose run --rm web rails assets:precompile - - # avatar failed to upload, did - # docker logs mastodon_web_1 - # google lead me to this - s chown -R 991:991 public/system - - # docker daemon takes care of starting on boot. - docker-compose up -d - - s a2enmod proxy_wstunnel headers - web-conf -f 3000 - apache2 mast.iankelling.org <<'EOF' - ProxyPreserveHost On - RequestHeader set X-Forwarded-Proto "https" - ProxyPass /500.html ! - ProxyPass /oops.png ! - ProxyPass /api/v1/streaming/ ws://localhost:4000/ - ProxyPassReverse /api/v1/streaming/ ws://localhost:4000/ - ErrorDocument 500 /500.html - ErrorDocument 501 /500.html - ErrorDocument 502 /500.html - ErrorDocument 503 /500.html - ErrorDocument 504 /500.html -EOF - - - ############### !!!!!!!!!!!!!!!!! - ############### manual steps: - - # only following a few people atm, so not bothering to figure out backups - # when mastodon has not documented it at all. - # - # fsf@status.fsf.org - # cwebber@toot.cat - # dbd@status.fsf.org - # johns@status.fsf.org - - # sign in page is at https://mast.iankelling.org/auth/sign_in - # register as iank, then - # https://github.com/tootsuite/documentation/blob/master/Running-Mastodon/Administration-guide.md - # docker-compose run --rm web bundle exec rails mastodon:make_admin USERNAME=iank - - ############# end setup mastodon ############## - - # we use nsupdate to update the ip of home - pi bind9 - - pi znc # znc config generated by doing # znc --makeconf # selected port is also used in erc config @@ -588,11 +270,6 @@ EOF # sed -i 's,/home/iank/.znc/,/var/lib/znc,' /var/lib/znc/config/znc.conf # and made a copy of the config files into /p/c # /msg *status LoadMod --type=global log -sanitize - # to get into the web interface, - # cat /etc/letsencrypt/live/iankelling.org/{privkey,cert,chain}.pem > /var/lib/znc/znc.pem - # then use non-main browser or else it doesn't allow it based on ocsp stapling from my main site. - # https://iankelling.org:12533/ - # i'm going to figure out how to automate this when it expires. i know i can hook a script into the renewal. https://wiki.znc.in/FAQ seems to imply that znc doesn\'t need restart. # todo: in config file AllowWeb = true should be false. better security if that is off unless we need it. # /msg *status LoadMod --type=network perform # /msg *perform add PRIVMSG ChanServ :invite #fsf-office @@ -605,12 +282,12 @@ EOF # on killing the, # an example seems to be here: https://github.com/zenspider/elisp/blob/master/rwd-irc.el # if that was the case i could remove the module clearbufferonmsg - # alo would be nice if erc supported + # also would be nice if erc supported # https://wiki.znc.in/self-message # https://wiki.znc.in/Query_buffers \ # s useradd --create-home -d /var/lib/znc --system --shell /sbin/nologin --comment "Account to run ZNC daemon" --user-group znc || [[ $? == 9 ]] # 9 if it exists already - chmod 700 /var/lib/znc + s chmod 700 /var/lib/znc s chown -R znc:znc /var/lib/znc s dd of=/etc/systemd/system/znc.service 2>/dev/null <<'EOF' [Unit] @@ -626,20 +303,15 @@ WantedBy=multi-user.target EOF ser daemon-reload sgo znc - - echo "$0: $(date): ending now)" - exit 0 - ;; -esac - -########### end section including li/lj ############### + ###### stop znc setup ##### -case $(debian-codename) in - # needed for debootstrap scripts for fai since fai requires debian - flidas) - curl http://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg | s apt-key add - - s dd of=/etc/apt/preferences.d/flidas-xenial </dev/null </dev/null <$t <$t <$t <$t <$t <$t <&2' ERR @@ -104,6 +104,7 @@ EOF chmod 640 [kK]* -myunison -ob li -ssh li conflink -ssh li ser restart bind9 +# push here? +#myunison -ob li +#ssh li conflink +ssh li systemctl reload bind9 diff --git a/filesystem/etc/ansible/hosts b/filesystem/etc/ansible/hosts index 4717dbc..083d3bd 100644 --- a/filesystem/etc/ansible/hosts +++ b/filesystem/etc/ansible/hosts @@ -1,7 +1,7 @@ [main] kd.b8.nz -kw.office.fsf.org +kw.b8.nz tp.b8.nz frodo.b8.nz -x2.office.fsf.org +x2.b8.nz x3.b8.nz diff --git a/filesystem/usr/local/bin/mycheckrestart b/filesystem/usr/local/bin/mycheckrestart index f6da8cd..eec8584 100755 --- a/filesystem/usr/local/bin/mycheckrestart +++ b/filesystem/usr/local/bin/mycheckrestart @@ -14,4 +14,9 @@ if [[ $HOSTNAME != "$MAIL_HOST" && $hn != li.b8.nz && ! $DISPLAY ]]; then exit 0 fi -$s /usr/sbin/checkrestart -v -i bash -b /a/bin/distro-setup/checkrestart-blacklist | sed '/^Found 0 processes using old versions of upgraded files$/d' +cmd="$s /usr/sbin/checkrestart -p" +if [[ $($cmd | sed '/^Found 0 processes using old versions of upgraded files$/d' | wc -l) != 0 ]]; then + $cmd + echo "with -v:" + $cmd -v +fi diff --git a/filesystem/usr/local/bin/myupgrade b/filesystem/usr/local/bin/myupgrade index b990fbb..55e7221 100755 --- a/filesystem/usr/local/bin/myupgrade +++ b/filesystem/usr/local/bin/myupgrade @@ -13,8 +13,10 @@ if [[ $HOSTNAME == "$MAIL_HOST" || $hn == li.b8.nz ]]; then exit 0 fi -/usr/sbin/checkrestart -v -i bash -b /a/bin/distro-setup/checkrestart-blacklist | \ - sed '/^Found 0 processes using old versions of upgraded files$/d' >/var/log/checkrestart.log +cmd="$s /usr/sbin/checkrestart -p" +if [[ $($cmd | sed '/^Found 0 processes using old versions of upgraded files$/d' | wc -l) != 0 ]]; then + $cmd -v | pee cat "wall -n" +fi if [[ -s /var/log/checkrestart.log ]]; then for x in {30..1}; do diff --git a/keyscript-off b/keyscript-off index 84f253c..1cf19b0 100755 --- a/keyscript-off +++ b/keyscript-off @@ -31,7 +31,7 @@ if [[ $($sed -rn 's/^ID=(.*)/\1/p' /etc/os-release) == arch ]]; then else x=/root/keyscript if grep -q "${x}," /etc/crypttab; then - $sed -i "s#${x},#${x}-manual,#" /etc/crypttab + $sed -i "s#${x},#decrypt_keyctl,#" /etc/crypttab update-initramfs -u fi fi diff --git a/keyscript-on b/keyscript-on index e62c7ed..246a971 100755 --- a/keyscript-on +++ b/keyscript-on @@ -18,26 +18,28 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?"' ERR [[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@" if [[ $- != *i* ]]; then - exec &>>/var/log/keyscript-on.log - echo "$0: starting. $(date)" + exec &>>/var/log/keyscript-on.log + echo "$0: starting. $(date)" fi rootn=1 sed="sed --follow-symlinks" if [[ ! -e /tmp/keyscript-off ]]; then - if [[ $($sed -rn 's/^ID=(.*)/\1/p' /etc/os-release) == arch ]]; then - if ! grep -q '^\s*FILES=' /etc/mkinitcpio.conf; then - $sed -ri 's/^#(\s*FILES=.*)/\1/' /etc/mkinitcpio.conf # uncomment - mkinitcpio -p linux - fi - else - x=/root/keyscript - if grep -q "${x}-manual," /etc/crypttab; then - $sed -i "s#${x}-manual,#${x},#" /etc/crypttab - update-initramfs -u - fi + if [[ $($sed -rn 's/^ID=(.*)/\1/p' /etc/os-release) == arch ]]; then + if ! grep -q '^\s*FILES=' /etc/mkinitcpio.conf; then + $sed -ri 's/^#(\s*FILES=.*)/\1/' /etc/mkinitcpio.conf # uncomment + mkinitcpio -p linux fi + else + x=decrypt_keyctl + # old name. can remove this sometime after aug 2019 + $sed -i "s#/root/keyscript-manual,#${x},#" /etc/crypttab + if grep -q "${x}," /etc/crypttab; then + $sed -i "s#${x},#/root/keyscript,#" /etc/crypttab + update-initramfs -u + fi + fi fi # switch to easy or hard login pass which is the same as luks f=/q/root/shadow/traci-simple diff --git a/mail-setup b/mail-setup index 93d09c7..6e21394 100755 --- a/mail-setup +++ b/mail-setup @@ -162,7 +162,7 @@ pi() { # package install if [[ ! -r $f ]] || (( $(( $(date +%s) - $(stat -c %Y $f ) )) > 60*60*12 )); then apt-get update fi - apt-get -y install --purge --auto-remove "$@" + DEBIAN_FRONTEND=noninteractive apt-get -y install --purge --auto-remove "$@" } postmaster=alerts @@ -177,6 +177,9 @@ forward=$u@$mxhost smarthost="$mxhost::$mxport" # exim +# light version of exim does not have sasl auth support. +pi exim4-daemon-heavy spamassassin spf-tools-perl + # trisquel 8 = openvpn, debian stretch = openvpn-client vpn_ser=openvpn-client if [[ ! -e /lib/systemd/system/openvpn-client@.service ]]; then @@ -194,7 +197,8 @@ else fi -pi openvpn +# light version of exim does not have sasl auth support. +pi exim4-daemon-heavy spamassassin spf-tools-perl openvpn dnsmasq if [[ -e /p/c/filesystem ]]; then # allow failure of these commands when our internet is down, they are likely not needed, @@ -694,7 +698,7 @@ EOF # it\'s not part of exim rm -f /etc/exim4/conf.d/main/000_localmacros cat >>/etc/exim4/update-exim4.conf.conf <$f <<'EOF' + cat >$f <<'EOFOUTER' #!/bin/bash -echo body_test | mail -s "primary_test $(date +%s) $(date +%Y-%m-%dT%H:%M:%S%z)" iank@posteo.de +/usr/sbin/exim -t </etc/cron.d/mailtest <&1 | /usr/local/bin/log-once send-test-forward -*/10 * * * * $u /usr/local/bin/mailtest-check 2>&1 | /usr/local/bin/log-once -1 send-test-forward +*/10 * * * * $u /usr/local/bin/mailtest-check 2>&1 | /usr/local/bin/log-once -1 mailtest-check */10 * * * * root chmod -R g+rw /m/md/bounces 2>&1 | /usr/local/bin/log-once -1 bounces-chmod EOF cp /a/bin/distro-setup/filesystem/usr/local/bin/mailtest-check /usr/local/bin diff --git a/pkgs b/pkgs index d8acab1..b14a881 100644 --- a/pkgs +++ b/pkgs @@ -10,12 +10,6 @@ p1=( screen ) p2=( - # 2nd - konsole - suckless-tools -) -p3=( - at bash-completion curl ethtool @@ -49,7 +43,8 @@ p3=( # gnat-5 & ccache is for coreboot # python-dnspython is needed by some ansible module and 3 is # for when ansible becomes py3. -p4=( +p3=( + at adb ansible apache2 @@ -196,4 +191,4 @@ p4=( xscreensaver-gl-extra ) -pall=(${p1[@]} ${p2[@]} ${p3[@]} ${p4[@]} ) +pall=(${p1[@]} ${p2[@]} ${p3[@]} ) diff --git a/rootsshsync b/rootsshsync index c550149..c4cebbf 100755 --- a/rootsshsync +++ b/rootsshsync @@ -27,7 +27,12 @@ fi mkdir -p $dest chmod 700 $dest -user_ssh_dir=$(eval echo ~${SUDO_USER:-$USER})/.ssh +user=$(id -un) +if [[ $SUDO_USER ]]; then + user=$SUDO_USER +fi + +user_ssh_dir=$(eval echo ~$user)/.ssh # remove broken links, or else rsync has error about them. find $user_ssh_dir -xtype l -exec rm '{}' \; @@ -35,10 +40,13 @@ find $user_ssh_dir -xtype l -exec rm '{}' \; # -L resolve links rsync -rtL --delete $user_ssh_dir/ $dest chown -R root:root /root/.ssh + + d=/etc/initramfs-tools if [[ -e $d ]] && ! diff -q /root/.ssh/authorized_keys $d/root/.ssh/authorized_keys &>/dev/null; then - mkdir -p /etc/initramfs-tools/root/.ssh + mkdir -p $d/root/.ssh /etc/dropbear-initramfs chmod 700 $d/root $d/root/.ssh cp -p /root/.ssh/authorized_keys $d/root/.ssh/authorized_keys + cp -p /root/.ssh/authorized_keys /etc/dropbear-initramfs update-initramfs -u -k all fi diff --git a/switch-mail-host b/switch-mail-host index f26b945..1598695 100644 --- a/switch-mail-host +++ b/switch-mail-host @@ -26,10 +26,10 @@ restore_new_btrbk=false restore_old_btrbk=false cleanup() { if $restore_new_btrbk; then - $new_shell sudo systemctl start btrbk.timer + m $new_shell sudo systemctl start btrbk.timer fi if $restore_old_btrbk; then - $old_shell sudo systemctl start btrbk.timer + m $old_shell sudo systemctl start btrbk.timer fi } _errcatch_cleanup=cleanup # used by sourced err diff --git a/vpn-mail-forward b/vpn-mail-forward index 30159a1..7965263 100755 --- a/vpn-mail-forward +++ b/vpn-mail-forward @@ -2,12 +2,16 @@ set -eE -o pipefail trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR - +m() { printf "%s\n" "$*"; "$@"; } +gw=$(/usr/sbin/ip route | sed -rn 's/^default via .* dev (\S+).*/\1/p') do-forward() { cmd=$1; shift for port; do - /sbin/iptables -t nat $cmd PREROUTING -i eth0 -p tcp -m tcp --dport $port -j DNAT --to-destination 10.8.0.4:$port + m /sbin/iptables -t nat $cmd PREROUTING -i $gw -p tcp -m tcp --dport $port -j DNAT --to-destination 10.8.0.4:$port + # we could leave these on all the time but its convenient to do it here done + m /sbin/iptables $cmd FORWARD -i tun+ -o $gw -j ACCEPT + m /sbin/iptables $cmd FORWARD -i $gw -o tun+ -j ACCEPT } ports=(25 143 587) -- 2.30.2