fix mistake from recent commit
[distro-setup] / btrbk-run
1 #!/bin/bash
2 # Copyright (C) 2016 Ian Kelling
3
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7
8 # http://www.apache.org/licenses/LICENSE-2.0
9
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16
17 # todo: if we cancel in the middle of a btrfs send, then run again immediately, the received subvolume doesn't get a Received UUID: field, and we won't mount it. Need to figure out a solution that will fix this.
18
19 set -eE -o pipefail
20 trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
21
22 [[ $EUID == 0 ]] || exec sudo -E "$BASH_SOURCE" "$@"
23
24 usage() {
25 cat <<'EOF'
26 btrbk-run [OPTIONS]
27 usually -t TARGET_HOST or -s SOURCE_HOST
28
29 Note, at source location, intentionally not executable, run and read
30 install-my-scripts.
31
32 EOF
33 echo "top of script file:"
34 sed -n '1,/^[# ]*end command line/{p;b};q' "$0"
35 exit $1
36 }
37
38 script_dir=$(dirname $(readlink -f "$BASH_SOURCE"))
39
40 # note q is owned by root:1000
41 # note p/m is owned 1000:1000 and chmod 700
42
43
44 mountpoints=()
45
46 rsync_mountpoint=/q
47
48 # default options
49 conf_only=false
50 dry_run=false # mostly for testing
51 rate_limit=no
52 verbose=true; verbose_arg=-v
53 progress_arg="--progress"
54
55 default_args_file=/etc/btrbk-run.conf
56 if [[ -s $default_args_file ]]; then
57 set -- $(< $default_args_file) "$@"
58 fi
59
60 temp=$(getopt -l help cl:m:nps:t:vh "$@") || usage 1
61 eval set -- "$temp"
62 while true; do
63 case $1 in
64 # only creates the config file, does not run btrbk
65 -c) conf_only=true; shift ;;
66 # bytes per second, suffix k m g
67 -l) rate_limit=$2; shift 2 ;;
68 # Comma separated mountpoints to backup. This has defaults set below.
69 -m) IFS=, mountpoints=($2); unset IFS; shift 2 ;;
70 -n) dry_run=true; dry_run_arg=-n; shift ;;
71 -p) progress_arg="--progress"; shift ;;
72 -q) verbose=false; verbose_arg=; progress_arg=; shift ;;
73 # source host to receive a backup from
74 -s) source=$2; shift 2 ;;
75 # target hosts to send to. empty is valid for just doing local
76 # snapshot. we have default hosts we will populate.
77 -t) IFS=, targets=($2); unset IFS; shift 2 ;;
78 -v) verbose=true; verbose_arg=-v; shift ;;
79 -h|--help) usage ;;
80 --) shift; break ;;
81 *) echo "$0: Internal error!" ; exit 1 ;;
82 esac
83 done
84
85 # usefull commands are resume and archive
86 cmd_arg=${1:-run}
87
88 if [[ -s $default_args_file ]]; then
89 echo "$0: warning: default btrbk-run options set in $default_args_file (sleeping 5 seconds):"
90 cat $default_args_file
91 sleep 5
92 fi
93
94 if [[ -v targets && $source ]]; then
95 echo "$0: error: -t and -s are mutually exclusive" >&2
96 exit 1
97 fi
98
99 echo -e "$0: options: conf_only=$conf_only\ndry_run=$dry_run\nrate_limit=$rate_limit\nverbose=$verbose\ncmd_arg=$cmd_arg"
100
101 # set default targets
102 if [[ ! -v targets && ! $source ]]; then
103 case $HOSTNAME in
104 x2|kw)
105 if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
106 targets=($HOME_DOMAIN)
107 fi
108 ;;
109 tp)
110 targets=(frodo)
111 if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
112 if timeout -s 9 10 ssh x2 :; then
113 targets+=(x2)
114 fi
115 fi
116 ;;
117 frodo)
118 targets=()
119 ;;
120 *)
121 echo "$0: error: no default targets for this host, use -t"
122 exit 1
123 ;;
124 esac
125 fi
126
127 if [[ -v targets ]]; then
128 echo "targets: ${targets[*]}"
129 fi
130
131 if [[ $source ]]; then
132 echo "source: $source"
133 fi
134
135
136
137 # set default mountpoints
138 case $HOSTNAME in
139 # no remote backups atm. note, if we do enable this, configuration below will need some changes.
140 # frodo)
141 # prospective_mps=(/i)
142 # ;;
143 *)
144 prospective_mps=(/a /q)
145 if [[ $HOSTNAME == "$MAIL_HOST" ]]; then
146 prospective_mps+=(/o)
147 fi
148 ;;
149 esac
150 case ${targets[0]} in
151 kw|kww)
152 prospective_mps=(/a)
153 ;;
154 esac
155
156 for mp in ${prospective_mps[@]}; do # default mountpoints to sync
157 if [[ -e /nocow/btrfs-stale/$mp ]]; then
158 echo "$0: warning: $mp stale, not adding to default mountpoints"
159 continue
160 fi
161 if awk '{print $2}' /etc/fstab | grep -xF $mp &>/dev/null; then
162 mountpoints+=($mp)
163 fi
164 done
165
166 echo "mountpoints: ${mountpoints[*]}"
167
168 ##### end command line parsing ########
169
170 rsync-dirs() {
171 local host=$1
172 local path=$2
173 m rsync $dry_run_arg -ahi --relative --delete "$path" "root@$host:/"
174 }
175
176
177 m() { printf "%s: %s\n" "${0##*/}" "$*"; "$@"; }
178
179
180 if ! which btrbk &>/dev/null; then
181 echo "$0: error: no btrbk binary found"
182 exit 1
183 fi
184 # if our mountpoints are from stale snapshots,
185 # it doesn't make sense to do a backup.
186 check-subvol-stale ${mountpoints[@]} || exit 1
187
188
189 cat >/etc/btrbk.conf <<EOF
190 ssh_identity /root/.ssh/home
191 # Just a guess that local7 is a good facility to pick.
192 # It's a bit odd that the transaction log has to be logged to
193 # a file or syslog, while other output is sent to std out.
194 # The man does not mention a way for them to be together, but
195 # I dunno if setting a log level like warn might also output
196 # transaction info.
197 transaction_syslog local7
198
199 # 20%ish speedup[]
200 stream_buffer 512m
201
202 # so we only run one at a time
203 lockfile /var/lock/btrbk.lock
204
205 # default format of short does not accomidate hourly preservation setting
206 timestamp_format long-iso
207
208 # only make a snapshot if things have changed
209 snapshot_create onchange
210 # I could make this different from target_preserve,
211 # if one disk had less space.
212 # for now, keeping them equal.
213 snapshot_preserve 36h 14d 8w 24m
214 snapshot_preserve_min 4h
215 snapshot_dir btrbk
216
217 # so, total backups = ~89
218 target_preserve 36h 14d 8w 24m
219 target_preserve_min 4h
220
221 # if something fails and it's not obvious, try doing
222 # btrbk -l debug -v dryrun
223
224 rate_limit $rate_limit
225 EOF
226
227
228
229 for tg in ${targets[@]:-$HOSTNAME}; do
230 # for an initial run, btrbk requires the dir to exist.
231 ssh root@$tg mkdir -p /mnt/root/btrbk
232 done
233
234
235
236 vol=/mnt/root
237 for m in ${mountpoints[@]}; do
238 sub=${m##*/}
239 if [[ $source ]]; then
240 cat >>/etc/btrbk.conf <<EOF
241 volume ssh://$source$vol
242 subvolume $sub
243 target send-receive $vol/btrbk
244 EOF
245 else
246 cat >>/etc/btrbk.conf <<EOF
247 volume $vol
248 subvolume $sub
249 EOF
250 for tg in ${targets[@]}; do
251 cat >>/etc/btrbk.conf <<EOF
252 target send-receive ssh://$tg$vol/btrbk
253 EOF
254 done
255 fi
256 done
257
258
259 # todo: umount first to ensure we don't have any errors
260 # todo: do some kill fuser stuff to make umount more reliable
261
262
263 if $conf_only; then
264 exit
265 fi
266
267
268
269 if $dry_run; then
270 m btrbk -v -n $cmd_arg
271 elif [[ $cmd_arg == archive ]]; then
272 if [[ $source ]]; then
273 m btrbk $verbose_arg $progress_arg $cmd_arg ssh://$source$vol $vol
274 else
275 for tg in ${targets[@]}; do
276 m btrbk $verbose_arg $progress_arg $cmd_arg $vol ssh://$tg$vol
277 done
278 fi
279 exit 0
280 fi
281 # -q and just using the syslog option seemed nice,
282 # but it doesn't show when a send has a parent and when it doesn't.
283 m btrbk $verbose_arg $progress_arg $cmd_arg
284
285 # if we have it, sync to systems which don't
286 if mountpoint $rsync_mountpoint >/dev/null; then
287 for tg in ${targets[@]}; do
288 case $tg in
289 li|lk)
290 for x in /p/c/machine_specific/*.hosts; do
291 if grep -qxF $tg $x; then
292 dir=${x%.hosts}
293 rsync-dirs $tg $dir
294 fi
295 done
296 ;;
297 esac
298 done
299 fi
300
301 /a/bin/distro-setup/install-my-scripts
302 if [[ $source ]]; then
303 m mount-latest-subvol
304 else
305 m mount-latest-remote ${targets[@]}
306 fi
307
308 # todo: move variable data we don't care about backing up
309 # to /nocow and symlink it.
310
311
312 # background on btrbk timezones. with short/long, timestamps use local time.
313 # for long, if your local time moves backwards, by moving timezones or
314 # for an hour when daylight savings changes it, you will temporarily get
315 # a more aggressive retention policy for the overlapping period, and
316 # vice versa for the opposite timezone move. The alternative is using
317 # long-iso, which puts timezone info into the timestamp, which means
318 # that instead of shifting time, you shift the start of day/week/month
319 # which is used for retention to your new local time, which means for
320 # example, if you moved forward by 8 hours, the daily/weekly/monthly
321 # retention will be 8 hours more aggressive since midnight is at a new
322 # time, unless you fake the timzeone using the TZ env variable.
323 # However, in the short term, there will be no inconsistencies.
324 # I don't see any problem with shifting when the day starts for
325 # retention, so I'm using long-iso.
326
327 # note to create a long-iso timestamp: date +%Y%m%dT%H%M%S%z