From: Ian Kelling Date: Sat, 16 Apr 2022 22:51:57 +0000 (-0400) Subject: mostly fixes, some new scripts X-Git-Url: https://iankelling.org/git/?a=commitdiff_plain;h=4b3f043829a57215e1251122a8ab0019b717ca8d;p=distro-setup mostly fixes, some new scripts --- diff --git a/brc2 b/brc2 index 02aa77f..f3c4b24 100644 --- a/brc2 +++ b/brc2 @@ -50,6 +50,32 @@ fi # * functions +multimic() { + local i + local -a sources + + m pactl unload-module module-loopback + m pactl unload-module module-null-sink + m pactl unload-module module-remap-source + + sources=($(pacmd list-sources | sed -rn 's/.*name: <([^>]+).*/\1/p')) + + if (( ! $# )); then + i=0 + for s in ${sources[@]}; do + e $i $s + i=$(( i+1 )) + done + read -r l + set -- $l + fi + m pactl load-module module-null-sink sink_name=ianinput sink_properties=device.description=ianinputs + for i; do + m pactl load-module module-loopback source=${sources[i]} sink_dont_move=true sink=ianinput + done + pactl load-module module-remap-source source_name=iancombine master=ianinput.monitor source_properties=device.description=iancombine +} + hstest() { install-my-scripts d=$(mktemp -d) @@ -725,6 +751,10 @@ fastboot() { kdecd() { /usr/lib/x86_64-linux-gnu/libexec/kdeconnectd; } +bat() { + cat /sys/class/power_supply/BAT0/capacity +} + # List of apps to install/update # Create from existing manually installed apps by doing # fdroidcl update diff --git a/distro-end b/distro-end index 539856c..6e86d21 100755 --- a/distro-end +++ b/distro-end @@ -842,7 +842,7 @@ EOF # also would be nice if erc supported # https://wiki.znc.in/self-message # https://wiki.znc.in/Query_buffers \ - # + # # for geekshed, there was no sasl support as far as I can tell, # so I set to msg nickserv to identify upon connect. if ! getent passwd znc > /dev/null; then @@ -1916,6 +1916,31 @@ EOF sysd-prom-fail-install $ser done + ## get upstream because it has the react ui, which has localtime, and general better usability. + ## begin get latest upstream prometheus ### + cd /a/opt/promdl + url=$(curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest | jq -r '.assets[].browser_download_url | match(".*linux-amd64.tar.gz$").string') + f=${url##*/} + if [[ -e $f ]]; then + timestamp=$(stat -c %Y $f) + else + timestamp=0 + fi + m wget -nv -N $url + new_timestamp=$(stat -c %Y $f) + if [[ $timestamp != $new_timestamp || ! -e /usr/local/bin/prometheus ]]; then + ngset + to_rm=( !($f) ) + ngreset + if (( ${#to_rm[@]} )); then + rm -rf ${to_rm[@]} + fi + m ex $f + dir=${f%.tar.gz} + s install $dir/prometheus $dir/promtool /usr/local/bin + fi + ## end get latest upstream prometheus ### + ;; *) pi prometheus-node-exporter diff --git a/filesystem/etc/default/prometheus b/filesystem/etc/default/prometheus index 9ee91ab..cd109eb 100644 --- a/filesystem/etc/default/prometheus +++ b/filesystem/etc/default/prometheus @@ -1,117 +1,97 @@ # iank: initial file from 2.24, added to empty ARGS. +# overlapping-blocks is for backfil of recording rules, +# https://jessicagreben.medium.com/prometheus-fill-in-data-for-new-recording-rules-30a14ccb8467 -# Set the command-line arguments to pass to the server. +# config.file and tsdb.path are to configure the upstream version to use +# the default locations of the debian package. -ARGS="--web.listen-address=127.0.0.1:9090 --web.external-url=https://i.b8.nz:9091 --log.level=info" +# Sets the command-line arguments to pass to the server. +ARGS="--web.listen-address=127.0.0.1:9090 +--web.external-url=https://i.b8.nz:9091 +--log.level=info +--storage.tsdb.allow-overlapping-blocks +--config.file=/etc/prometheus/prometheus.yml +--storage.tsdb.path=/var/lib/prometheus/metrics2/" -# Prometheus supports the following options: -# --config.file="/etc/prometheus/prometheus.yml" -# Prometheus configuration file path. -# --web.listen-address="0.0.0.0:9090" -# Address to listen on for UI, API, and telemetry. -# --web.read-timeout=5m Maximum duration before timing out read of the -# request, and closing idle connections. -# --web.max-connections=512 Maximum number of simultaneous connections. -# --web.external-url= The URL under which Prometheus is externally -# reachable (for example, if Prometheus is served -# via a reverse proxy). Used for generating -# relative and absolute links back to Prometheus -# itself. If the URL has a path portion, it will -# be used to prefix all HTTP endpoints served by -# Prometheus. If omitted, relevant URL components -# will be derived automatically. -# --web.route-prefix= Prefix for the internal routes of web endpoints. -# Defaults to path of --web.external-url. -# --web.local-assets="/usr/share/prometheus/web/" -# Path to static asset/templates directory. -# --web.user-assets= Path to user asset directory, available at -# /user. -# --web.enable-lifecycle Enable shutdown and reload via HTTP request. -# --web.enable-admin-api Enable API endpoints for admin control actions. -# --web.console.templates="/etc/prometheus/consoles" -# Path to the console template directory, -# available at /consoles. -# --web.console.libraries="/etc/prometheus/console_libraries" -# Path to the console library directory. -# --web.page-title="Prometheus Time Series Collection and Processing Server" -# Document title of Prometheus instance. -# --web.cors.origin=".*" Regex for CORS origin. It is fully anchored. -# Example: 'https?://(domain1|domain2)\.com' -# --storage.tsdb.path="/var/lib/prometheus/metrics2/" -# Base path for metrics storage. -# --storage.tsdb.retention=15d -# [DEPRECATED] How long to retain samples in -# storage. This flag has been deprecated, use -# "storage.tsdb.retention.time" instead -# --storage.tsdb.retention.time=15d -# How long to retain samples in storage. When this -# flag is set it overrides -# "storage.tsdb.retention". -# If neither this flag nor "storage.tsdb.retention" -# nor "storage.tsdb.retention.size" is set, the -# retention time defaults to 15d. -# Units Supported: y, w, d, h, m, s, ms. -# --storage.tsdb.retention.size= -# [EXPERIMENTAL] Maximum number of bytes that can -# be stored for blocks. Units supported: KB, MB, -# GB, TB, PB. This flag is experimental and can be -# changed in future releases. -# --storage.tsdb.use-lockfile -# Create a lockfile in data directory. -# --storage.tsdb.allow-overlapping-blocks -# [EXPERIMENTAL] Allow overlapping blocks, which -# in turn enables vertical compaction and -# vertical query merge. -# --storage.tsdb.wal-compression -# Compress the tsdb WAL. -# --storage.remote.flush-deadline= -# How long to wait flushing sample on shutdown or -# config reload. -# --storage.remote.read-sample-limit=5e7 -# Maximum overall number of samples to return via -# the remote read interface, in a single query. 0 -# means no limit. This limit is ignored for -# streamed response types. -# --storage.remote.read-concurrent-limit=10 -# Maximum number of concurrent remote read calls. -# 0 means no limit. -# --storage.remote.read-max-bytes-in-frame=1048576 -# Maximum number of bytes in a single frame for -# streaming remote read response types before -# marshalling. Note that client might have limit on -# frame size as well. 1MB as recommended by -# protobuf by default. -# --rules.alert.for-outage-tolerance=1h -# Max time to tolerate prometheus outage for -# restoring "for" state of alert. -# --rules.alert.for-grace-period=10m -# Minimum duration between alert and restored "for" -# state. This is maintained only for alerts with -# configured "for" time greater than grace period. -# --rules.alert.resend-delay=1m -# Minimum amount of time to wait before resending -# an alert to Alertmanager. -# --alertmanager.notification-queue-capacity=10000 -# The capacity of the queue for pending -# Alertmanager notifications. -# --alertmanager.timeout=10s -# Timeout for sending alerts to Alertmanager. -# --query.lookback-delta=5m The maximum lookback duration for retrieving -# metrics during expression evaluations and -# federation. -# --query.timeout=2m Maximum time a query may take before being -# aborted. -# --query.max-concurrency=20 -# Maximum number of queries executed concurrently. -# --query.max-samples=50000000 -# Maximum number of samples a single query can load -# into memory. Note that queries will fail if they -# try to load more samples than this into memory, -# so this also limits the number of samples a query -# can return. -# --log.level=info Only log messages with the given severity or -# above. One of: [debug, info, warn, error] -# --log.format=logfmt Output format of log messages. One of: [logfmt, -# json] + +# --config.file="prometheus.yml" +# Prometheus configuration file path. +# --web.listen-address="0.0.0.0:9090" +# Address to listen on for UI, API, and telemetry. +# --web.config.file="" [EXPERIMENTAL] Path to configuration file that can enable TLS or authentication. +# --web.read-timeout=5m Maximum duration before timing out read of the request, and closing idle connections. +# --web.max-connections=512 Maximum number of simultaneous connections. +# --web.external-url= The URL under which Prometheus is externally reachable (for example, if Prometheus is served via a reverse proxy). Used for generating relative and absolute links back to +# Prometheus itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Prometheus. If omitted, relevant URL components will be derived +# automatically. +# --web.route-prefix= Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url. +# --web.user-assets= Path to static asset directory, available at /user. +# --web.enable-lifecycle Enable shutdown and reload via HTTP request. +# --web.enable-admin-api Enable API endpoints for admin control actions. +# --web.enable-remote-write-receiver +# Enable API endpoint accepting remote write requests. +# --web.console.templates="consoles" +# Path to the console template directory, available at /consoles. +# --web.console.libraries="console_libraries" +# Path to the console library directory. +# --web.page-title="Prometheus Time Series Collection and Processing Server" +# Document title of Prometheus instance. +# --web.cors.origin=".*" Regex for CORS origin. It is fully anchored. Example: 'https?://(domain1|domain2)\.com' +# --storage.tsdb.path="data/" +# Base path for metrics storage. Use with server mode only. +# --storage.tsdb.retention=STORAGE.TSDB.RETENTION +# [DEPRECATED] How long to retain samples in storage. This flag has been deprecated, use "storage.tsdb.retention.time" instead. Use with server mode only. +# --storage.tsdb.retention.time=STORAGE.TSDB.RETENTION.TIME +# How long to retain samples in storage. When this flag is set it overrides "storage.tsdb.retention". If neither this flag nor "storage.tsdb.retention" nor +# "storage.tsdb.retention.size" is set, the retention time defaults to 15d. Units Supported: y, w, d, h, m, s, ms. Use with server mode only. +# --storage.tsdb.retention.size=STORAGE.TSDB.RETENTION.SIZE +# Maximum number of bytes that can be stored for blocks. A unit is required, supported units: B, KB, MB, GB, TB, PB, EB. Ex: "512MB". Based on powers-of-2, so 1KB is 1024B. Use +# with server mode only. +# --storage.tsdb.no-lockfile +# Do not create lockfile in data directory. Use with server mode only. +# --storage.tsdb.allow-overlapping-blocks +# Allow overlapping blocks, which in turn enables vertical compaction and vertical query merge. Use with server mode only. +# --storage.tsdb.head-chunks-write-queue-size=0 +# Size of the queue through which head chunks are written to the disk to be m-mapped, 0 disables the queue completely. Experimental. Use with server mode only. +# --storage.agent.path="data-agent/" +# Base path for metrics storage. Use with agent mode only. +# --storage.agent.wal-compression +# Compress the agent WAL. Use with agent mode only. +# --storage.agent.retention.min-time=STORAGE.AGENT.RETENTION.MIN-TIME +# Minimum age samples may be before being considered for deletion when the WAL is truncated Use with agent mode only. +# --storage.agent.retention.max-time=STORAGE.AGENT.RETENTION.MAX-TIME +# Maximum age samples may be before being forcibly deleted when the WAL is truncated Use with agent mode only. +# --storage.agent.no-lockfile +# Do not create lockfile in data directory. Use with agent mode only. +# --storage.remote.flush-deadline= +# How long to wait flushing sample on shutdown or config reload. +# --storage.remote.read-sample-limit=5e7 +# Maximum overall number of samples to return via the remote read interface, in a single query. 0 means no limit. This limit is ignored for streamed response types. Use with +# server mode only. +# --storage.remote.read-concurrent-limit=10 +# Maximum number of concurrent remote read calls. 0 means no limit. Use with server mode only. +# --storage.remote.read-max-bytes-in-frame=1048576 +# Maximum number of bytes in a single frame for streaming remote read response types before marshalling. Note that client might have limit on frame size as well. 1MB as +# recommended by protobuf by default. Use with server mode only. +# --rules.alert.for-outage-tolerance=1h +# Max time to tolerate prometheus outage for restoring "for" state of alert. Use with server mode only. +# --rules.alert.for-grace-period=10m +# Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. Use with server mode only. +# --rules.alert.resend-delay=1m +# Minimum amount of time to wait before resending an alert to Alertmanager. Use with server mode only. +# --alertmanager.notification-queue-capacity=10000 +# The capacity of the queue for pending Alertmanager notifications. Use with server mode only. +# --query.lookback-delta=5m The maximum lookback duration for retrieving metrics during expression evaluations and federation. Use with server mode only. +# --query.timeout=2m Maximum time a query may take before being aborted. Use with server mode only. +# --query.max-concurrency=20 +# Maximum number of queries executed concurrently. Use with server mode only. +# --query.max-samples=50000000 +# Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the +# number of samples a query can return. Use with server mode only. +# --enable-feature= ... Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, +# promql-negative-offset, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager. See https://prometheus.io/docs/prometheus/latest/feature_flags/ +# for more details. +# --log.level=info Only log messages with the given severity or above. One of: [debug, info, warn, error] +# --log.format=logfmt Output format of log messages. One of: [logfmt, json] diff --git a/filesystem/etc/profile.d/environment.sh b/filesystem/etc/profile.d/environment.sh index df27665..d5c7e57 100644 --- a/filesystem/etc/profile.d/environment.sh +++ b/filesystem/etc/profile.d/environment.sh @@ -1,7 +1,11 @@ #!/bin/sh if [ -f $HOME/path-add-function ]; then . $HOME/path-add-function - path-add /usr/sbin /usr/local/sbin /sbin /a/exe /a/opt/bin + path-add /usr/sbin /usr/local/sbin /a/exe /a/opt/bin + # if usr merge, dont need it + if [[ ! -L /sbin ]]; then + path-add /sbin + fi path-add --end $HOME/.cabal/bin path-add --end /snap/bin diff --git a/filesystem/etc/prometheus/rules/iank.yml b/filesystem/etc/prometheus/rules/iank.yml index 0ee3335..971e392 100644 --- a/filesystem/etc/prometheus/rules/iank.yml +++ b/filesystem/etc/prometheus/rules/iank.yml @@ -44,7 +44,10 @@ groups: ###### END MISC NOTES ###### - +# various queries only look at increases, so invert the up metric so we +# can better query on down. + - record: down + expr: up == bool 0 # alerting on missing metrics: @@ -112,15 +115,6 @@ groups: annotations: summary: '12 minutes down' - # 42 mins: enough for a 30 min queue run plus 12 - - alert: mailtest_check_vps - expr: |- - time() - mailtest_check_last_usec{job="tlsnode"} >= 60 * 42 - labels: - severity: prod - annotations: - summary: '42 minutes down' - - alert: mailtest_check_mailhost expr: |- time() - max by (folder,from) (mailtest_check_last_usec{job="node"}) >= 60 * 12 @@ -130,9 +124,9 @@ groups: summary: '12 minutes down' # 42 mins: enough for a 30 min queue run plus 12 - - alert: mailtest_check_mailhost + - alert: mailtest_check_gnu_mailhost expr: |- - time() - max by (folder,from) (mailtest_check_last_usec{job="node"}) >= 60 * 42 + time() - max by (folder,from) (mailtest_check_last_usec{folder="/m/md/l/testignore", from="iank@gnu.org"}) >= 60 * 42 labels: severity: prod annotations: @@ -175,11 +169,11 @@ groups: # avg_over_time(node_systemd_unit_state{name="dynamicipupdate.service",state="active"}[1d]) < .95 - alert: up_resets expr: |- - resets(up[2d]) - changes(node_boot_time_seconds[2d]) > 12 + resets(up[1d]) - changes(node_boot_time_seconds[1d]) > 12 labels: severity: warn annotations: - summary: "Target has gone down {{ $value }} times in 2 days, > 12" + summary: "Target has gone down {{ $value }} times in 1 day, > 12" diff --git a/filesystem/etc/systemd/system/prometheus.d/restart.conf b/filesystem/etc/systemd/system/prometheus.service.d/override.conf similarity index 59% rename from filesystem/etc/systemd/system/prometheus.d/restart.conf rename to filesystem/etc/systemd/system/prometheus.service.d/override.conf index aa2ea84..a928b2a 100644 --- a/filesystem/etc/systemd/system/prometheus.d/restart.conf +++ b/filesystem/etc/systemd/system/prometheus.service.d/override.conf @@ -6,3 +6,7 @@ StartLimitIntervalSec=0 Restart=always # time to sleep before restarting a service RestartSec=600 + +# empty signifies to replace the existing value +ExecStart= +ExecStart=/usr/local/bin/prometheus $ARGS diff --git a/filesystem/usr/local/bin/myupgrade b/filesystem/usr/local/bin/myupgrade index fb8d1d0..32dd7ff 100755 --- a/filesystem/usr/local/bin/myupgrade +++ b/filesystem/usr/local/bin/myupgrade @@ -25,12 +25,13 @@ d() { if [[ $DEBUG ]]; then pee cat "wall -n" else - sed 's/^/myupgrade /' | pee logger "wall -n" + # 2>/dev/null hopefully gets rid of errors like: wall: /dev/pts/0: No such file or directory + sed 's/^/myupgrade /' | pee logger "wall -n" 2>/dev/null fi } myreboot() { for x in {30..1}; do - echo "pid $$. unattended upgrade, rebooting in $((x*10)) seconds" | wall -n + echo "pid $$. unattended upgrade, rebooting in $((x*10)) seconds" | wall -n 2>/dev/null sleep 10 done for x in {30..1}; do @@ -40,7 +41,7 @@ myreboot() { /sbin/reboot exit 0 fi - echo "pid $$. unattended upgrade reboot waiting 10 seconds for dpkg lock" | wall -n + echo "pid $$. unattended upgrade reboot waiting 10 seconds for dpkg lock" | wall -n 2>/dev/null sleep 10 done echo "pid $$. dpkg locked for 5 minutes, automatic reboot failed" | d diff --git a/i3-sway/common.conf b/i3-sway/common.conf index 5860ee4..ee678f8 100644 --- a/i3-sway/common.conf +++ b/i3-sway/common.conf @@ -18,7 +18,10 @@ bindsym $mod+6 exec "/a/bin/redshift.sh" # bindsym $mod+grave exec "t s lunch; t in; t out -a '45 minutes from now'" -bindsym $mod+w focus parent +bindsym $mod+equal focus parent +# move firefox to current workspace +bindsym $mod+w [class="abrowser"] move workspace current + bindsym $mod+e fullscreen toggle bindsym $mod+r exec "/a/bin/ds/xl" # todo, in newer i3, make this toggle split tabbed diff --git a/mail-setup b/mail-setup index 7da0524..a026f06 100755 --- a/mail-setup +++ b/mail-setup @@ -3,6 +3,12 @@ # Copyright (C) 2019 Ian Kelling # SPDX-License-Identifier: AGPL-3.0-or-later +# todo: setup a logrotate for /var/log/mymain and mypanic + +# todo: setup an alert for bouncing test emails. + +# todo: bounces to my fsf mail can come from fsf@iankelling.org, +# think about making bounces go from the original address. # todo: add a prometheus alert for dovecot. @@ -954,6 +960,14 @@ enabled = true port = 25,587 filter = exim banaction = iptables-exim + +# 209.51.188.13 = mail.fsf.org +# 2001:470:142::13 = mail.fsf.org +# 209.51.188.92 = eggs.gnu.org +# 2001:470:142:3::10 = eggs.gnu.org +# 72.14.176.105 2600:3c00:e000:280::2 = mail.iankelling.org +# 10.173.8.1 = non-nn net +ignoreip = 209.51.188.13 2001:470:142::13 209.51.188.92 2001:470:142:3::10 72.14.176.105 2600:3c00:e000:280::2 10.173.8.1 EOF if $ir; then m systemctl restart fail2ban @@ -2673,11 +2687,28 @@ deny EOF echo|i /etc/exim4/conf.d/router/880_universal_forward + + cat >>/etc/exim4/conf.d/main/000_local <>/etc/myexim4/conf.d/main/000_local-nn <<'EOF' +# this makes it easier to see which exim is doing what +log_file_path = /var/log/exim4/my%s +EOF # If we ever wanted to have a separate spool, # we could do it like this. # cat >>/etc/exim4/conf.d/main/000_local-nn <<'EOF' @@ -3154,7 +3188,7 @@ EOF ;;& $MAIL_HOST) test_froms=(ian@iankelling.org z@zroe.org iank@gnu.org) - test_to="testignore@expertpathologyreview.com, testignore@je.b8.nz, testignore@amnimal.ninja, jtuttle@gnu.org" + test_tos=(testignore@expertpathologyreview.com testignore@je.b8.nz testignore@amnimal.ninja jtuttle@gnu.org) cat >>/etc/cron.d/mailtest < /etc/exim4/ignore-sent + for t in ${test_tos[@]:1}; do + test_to+=", $t" + echo $t >> /etc/exim4/ignore-sent + done cat >/usr/local/bin/send-test-forward <<'EOF' #!/bin/bash olds=( diff --git a/subdir_files/.config/i3/config b/subdir_files/.config/i3/config index 3ed4cdd..938f67e 100644 --- a/subdir_files/.config/i3/config +++ b/subdir_files/.config/i3/config @@ -18,7 +18,10 @@ bindsym $mod+6 exec "/a/bin/redshift.sh" # bindsym $mod+grave exec "t s lunch; t in; t out -a '45 minutes from now'" -bindsym $mod+w focus parent +bindsym $mod+equal focus parent +# move firefox to current workspace +bindsym $mod+w [class="abrowser"] move workspace current + bindsym $mod+e fullscreen toggle bindsym $mod+r exec "/a/bin/ds/xl" # todo, in newer i3, make this toggle split tabbed diff --git a/subdir_files/.config/sway/config b/subdir_files/.config/sway/config index f0e45c4..0d56fef 100644 --- a/subdir_files/.config/sway/config +++ b/subdir_files/.config/sway/config @@ -18,7 +18,10 @@ bindsym $mod+6 exec "/a/bin/redshift.sh" # bindsym $mod+grave exec "t s lunch; t in; t out -a '45 minutes from now'" -bindsym $mod+w focus parent +bindsym $mod+equal focus parent +# move firefox to current workspace +bindsym $mod+w [class="abrowser"] move workspace current + bindsym $mod+e fullscreen toggle bindsym $mod+r exec "/a/bin/ds/xl" # todo, in newer i3, make this toggle split tabbed diff --git a/ziva-backup-check b/ziva-backup-check index 849453d..1d5e432 100755 --- a/ziva-backup-check +++ b/ziva-backup-check @@ -18,8 +18,15 @@ fi ## begin check on btrbk -age_limit_sec=$(( 60 * 60 * 50 )) # 50 hours +age_limit_sec=$(( 60 * 60 * 74 )) # 74 hours for prefix in root boot; do + if [[ $prefix == boot ]]; then + # its not uncommon for the /boot subvol to have no changes, and thus + # no new backups for 10 days or so. todo: instead of this error + # prone check, we should make it so the ziva computer will + # touch a file on our computer whenever btrbk succeeds + age_limit_sec=$(( age_limit_sec + 60* 60 * 24 * 15 )) + fi vol=${prefix}_ubuntubionic snaps=(/mnt/r7/amy/$prefix/btrbk/${vol}.20*) if [[ ! ${snaps[*]} ]]; then @@ -33,7 +40,7 @@ for prefix in root boot; do unix_time=$(date -d $(sed -r 's/(.{4})(..)(.{5})(..)(.*)/\1-\2-\3:\4:\5/' <<<${f#$vol.}) +%s) printf "%s %s\n" $unix_time $s # part of the pipeline done | sort -r | head -n 1 ||: - ) + ) if [[ ! $last_snap ]]; then # should not happen. err "could not find latest snapshot for $svp among ${snaps[*]}"