Multi-boot/distro btrfs provisioning
-Some things are specific to my home network. Uses PXE, designed for bare
-metal but also works for pxe booted VM.
-
-Features people may find useful: installs encrypted arch, debian stable,
-& debian testing all on the same btrfs filesystem, smartly utilizing
-multiple disks, with scripts to automatically decrypt on reboots. The
-partititioning and filesystem script is the biggest part and is at
-fai/config/hooks/partition.DEFAULT. Other debian based distros should
-work fine, and I'm planning to add Fedora support. Disks are grouped as
-ssd or hdd and raided in raid 1 or raid 0 per configuration. The base
-partitions are divided into boot, swap, and root, (only boot is
-unencrypted). There are scripts to resize those partitions
-post-provision and while the system is running.
-
-The repo name fai is copied from a project of the same name which this
-project uses for debian installs. It stands for "fully automated
-installer."
+Some things are specific to my home network. Uses pxe or pxe-kexec (for
+systems like libreboot with no pxe rom, we boot into a live cd or distro
+for bootsraping). Works for bare-metal or vms.
+
+Features people may find useful: installs encrypted trisquel belanos,
+arch, debian stable, & debian testing all on the same btrfs filesystem.
+Smartly utilizes multiple disks, with scripts to automatically decrypt
+on reboots. The partititioning and filesystem script is at
+fai/config/hooks/partition.DEFAULT. Other debian based distros at least
+as new as ubuntu 14.04 should work fine, and I'm planning to add Fedora
+support. Disks are grouped as ssd or hdd and raided in raid 1 or raid 0
+per configuration. The base partitions are divided into boot, swap, and
+root, (only boot is unencrypted). There are scripts to resize those
+partitions post-provision and while the system is running.
+
+The repo name fai is copied from the debian project of the same name,
+meaning "fully automated installer."
It also fully automates configuration of an openwrt router after manual
initial installation.
-Provisionining is done, I sync files using unison, then automate further
-setup using a different set of scripts,
+After provisionining is done, I sync files using unison, then automate
+further setup using a different set of scripts,
https://iankelling.org/git/?p=distro-setup;a=tree.
My network is a wndr3700v2 router with openwrt on it and a few pcs/laptops.
fresize # resize swap or boot partitions in a host
pxe-server # temporarily enable (usually) fai or arch boot server
wrt-setup-remote # setup my router
+ubuntu-xenial-live-fai-kexec # do fai install from xenial live cd using kexec
+myfai-chboot # use instead of pxe-server for fai kexec based install
License stuff:
The license for the project is GPLv2 or later, mostly because fai is
# The default settings in the installer expect to find the NFS at /run/archiso/bootmnt
-pxe-server arch
+pxe-server default arch
# background:
# great documentation at https://wiki.archlinux.org/index.php/PXE
cd /
e umount $mount_dir
-e $src/pxe-server -p plain # my script
+e $src/pxe-server default plain # my script
set -x
cleanup() { pxe-server :; }
-pxe-server fai $host
+pxe-server $host fai
if $reboot; then
ssh $host "touch /tmp/keyscript-off; sudo reboot" ||: &
fi
-pxe-server -a :
+pxe-server -a
cleanup() { :; }
error=true
[[ ${0##*/} == arch-revm ]]
}
-cleanup() { ./pxe-server :; }
+cleanup() { ./pxe-server; }
if is_arch_revm; then
- ./pxe-server arch
+ ./pxe-server demohost arch
sleep 2
# via osinfo-query os. guessing arch is closest to latest fedora.
variant=fedora22
else
- ./pxe-server fai
+ ./pxe-server demohost fai
sleep 2
# I don't think these variants actually make a diff for us, but I
# use the appropriate one when trying a new distro just in case.
e sleep 5
done
cleanup() { :; }
-e pxe-server :
+e pxe-server
if is_arch_revm; then
./arch-init-remote $name
fi
--- /dev/null
+#!/bin/sh
+# shebang is for editor file mode detection only
+
+function save_vars {
+ if [ -s $envfile ]; then
+ for var in $@; do
+ save_env --file $envfile $var
+ done
+ fi
+}
+
+function save_chosen {
+ last_boot=$CHOSEN
+ save_vars did_fai_check last_boot
+}
+
+# we don't set this to fai check so we can't get into
+# an infinite reboot cycle. We depend on the os to
+# create the initial grubenv file.
+set default=/debianstable_bootstrap # could use 0 here.
+set timeout=1
+
+for part in (ahci*4) (ata*4); do
+ envfile=$part/grubenv
+ if [ -s $envfile ]; then
+ load_env --file $envfile
+ if [ x$did_fai_check != xtrue -a x$last_boot != x$default ]; then
+ set default=fai-check
+ elif [ ! -z $last_boot ]; then
+ set default=$last_boot
+ fi
+ break
+ fi
+done
+
+did_fai_check=false
+
+bs_dir=/debianstable_bootstrap
+menuentry $bs_dir --id=$bs_dir {
+ save_chosen
+ configfile $bs_dir/boot/grub/grub.cfg
+}
+
+for dir in /boot_*; do
+ if [ $dir == '/boot_*' ]; then
+ break
+ fi
+ menuentry $dir --id=$dir {
+ save_chosen
+ configfile $1/grub/grub.cfg
+ }
+done
+
+menuentry fai-check --id=fai-check {
+ did_fai_check=true
+ save_vars did_fai_check
+ configfile $bs_dir/boot/grub/grub.cfg
+}
--- /dev/null
+[Unit]
+Description=check whether to kexec to fai, reboot, or do nothing
+
+[Service]
+Type=oneshot
+ExecStart=/root/fai-check
+
+[Install]
+WantedBy=multi-user.target
--- /dev/null
+#!/bin/bash
+
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
+
+fai_check=false
+check-fai() {
+ # we could just as well check if last_boot != /debianstable_boostrap
+ # the intent with this one is just a little clearer.
+ if [[ $did_fai_check == true ]]; then
+ fai_check=true
+ pxe-kexec -n --ignore-whitelist -l fai-generated faiserver ||:
+ else
+ return 0
+ fi
+}
+
+first=true
+for dev in $(btrfs fi show / | sed -rn 's#^\s*devid\s.*\s([^0-9 ]+)\S+$#\1#p' \
+ |sort); do
+ dev+=4
+ mount $dev /mnt
+ if $first; then
+ if [[ -e /mnt/grubenv ]]; then
+ source <(grub-editenv /mnt/grubenv list)
+ fi
+ first=false
+ check-fai
+ else
+ # we make sure there is only 1 grubenv,
+ # so grub can just find the first one, in whatever order
+ # if looks at them, which may not be the same as us.
+ # If the disk dies, we just lose the default boot option,
+ # we will have to do manual steps to replace it anyways.
+ rm -f /mnt/gruvenv
+ fi
+ umount /mnt
+done
+
+if $fai_check && [[ $last_boot != /debianstable_boostrap ]]; then
+ # no need to reboot if we actually want to boot into this os.
+ reboot
+fi
# # fai's setup-storage won't do btrfs on luks,
# # so we do it ourself :)
+# inspiration taken from files in fai-setup-storage package
+
skiptask partition || ! type skiptask # for running not in fai
rootn=1
swapn=2
bootn=3
-bios_grubn=4
+# ext partition so grub can write persistent variables,
+# so it can do a one time boot.
+grub_extn=4
+# bios boot partition,
+# https://wiki.archlinux.org/index.php/GRUB
+bios_grubn=5
+lastn=$bios_grubn
boot_mib=4000
bootdev() { add-part $@ $bootn; }
rootdev() { add-part $@ $rootn; }
swapdev() { add-part $@ $swapn; }
-bios_grubdev() { add-part $@ $bios_grubn; }
+grub_extdev() { add-part $@ $grub_extn; }
+# Commented because it's not used, but left because it
+# finishes the pattern and if we ever do need to use it, it's here.
+#bios_grubdev() { add-part $@ $bios_grubn; }
crypt-dev() { echo /dev/mapper/crypt_dev_${1##*/}; }
crypt-name() { echo crypt_dev_${1##*/}; }
partition=false # change to true to force a full wipe
fi
-lastn=$bios_grubn
hdds=()
# check if the partitions exist have the right filesystems
#blkid="$(blkid -s TYPE)"
for dev in ${short_devs[@]}; do
- ! $partition || break
+ if $partition; then break; fi
y=$(readlink -f $dev)
x=($y[0-9])
[[ ${#x[@]} == "${lastn}" ]] || partition=true
if $partition && ifclass PARTITION_PROMPT; then
echo "Press any key except ctrl-c to continue and partition these drives:"
- echo " ${short_devs[@]}"
- read
+ echo " ${short_devs[*]}"
+ read -r
fi
devs=()
done
-
+first=false
boot_devs=()
for dev in ${devs[@]}; do
if ifclass frodo; then
else
boot_devs+=(`bootdev`)
fi
+ if [[ $boot_devs && $first ]]; then
+ first_grub_extdev=`grub_extdev`
+ first=false
+ fi
done
if [[ ! $DISTRO ]]; then
exit 1
fi
fi
+first_boot_dev=${boot_devs[0]}
case ${#boot_devs[@]} in
esac
}
-first_boot_dev=${boot_devs[0]}
# keyfiles generated like:
# head -c 2048 /dev/urandom | od | s dd of=/q/root/luks/host-demohost
first_root_crypt=$(root-cryptdev ${devs[0]})
-bios_grubn=4
# 1.5 x based on https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Installation_Guide/sect-disk-partitioning-setup-x86.html#sect-custom-partitioning-x86
swap_mib=$(( $(grep ^MemTotal: /proc/meminfo | \
awk '{print $2}') * 3/(${#devs[@]} * 2 ) / 1024 ))
parted -s $dev mklabel gpt
# MiB because parted complains about alignment otherwise.
pcmd="parted -a optimal -s -- $dev"
- $pcmd mkpart primary "ext3" 4MiB ${root_end}MiB
+ $pcmd mkpart primary "ext3" 12MiB ${root_end}MiB
$pcmd mkpart primary "linux-swap" ${root_end}MiB ${swap_end}MiB
$pcmd mkpart primary "" ${swap_end}MiB ${disk_mib}MiB
- # gpt ubuntu cloud image uses ~4. fai uses 1 MiB.
- # I read something in the parted manual saying cheap flash media
- # likes to start at 4.
+ # i only need a few k, but googling min size,
+ # I found someone saying that gparted required
+ # required at least 8 because of their hard drive cylinder size.
+ # And 8 is still very tiny.
+ $pcmd mkpart primary "ext2" 4MiB 12MiB
+ # gpt ubuntu cloud image uses ~4 mb for this partition. fai uses 1 MiB.
+ # so, I use 3, whatever.
+ # note: parted manual saying cheap flash media
+ # should to start at 4.
$pcmd mkpart primary "" 1MiB 4MiB
$pcmd set $bios_grubn bios_grub on
$pcmd set $bootn boot on # generally not needed on modern systems
# the mkfs failed before on a vm, which prompted me to add
# sleep .1
- # then failed on a physical machine
+ # then it failed again on a physical machine
# with:
# Device /dev/disk/by-id/foo doesn't exist or access denied,
- # so here we wait.
+ # so I added a wait until it existed.
+ # Then I added the mkfs.ext2, which claimed to succeed,
+ # but then couldn't be found upon reboot. In that case we didn't
+ # wait at all. So I've added a 3 second minimum wait.
+ sleep 3
secs=0
while [[ ! -e `rootdev` ]] && (( secs < 10 )); do
sleep 1
secs=$((secs +1))
done
+ # Holds just a single file, rarely written, so
+ # use ext2, like was often used for the /boot partition.
+ # This exists because grub can only persist data to a non-cow fs.
+ # And we use persisting a var in grub to do a one time boot.
+ # We could pass the data on the kernel command line and persist it
+ # to grubenv after booting, but that relies on the boot always succeeding.
+ # This is just a bit more robust, and it could work for booting
+ # into ipxe which can't persist data, if we ever got that working.
+ mkfs.ext2 `grub_extdev`
yes YES | cryptsetup luksFormat `rootdev` $luks_dir/host-$HOSTNAME \
-c aes-cbc-essiv:sha256 -s 256 || [[ $? == 141 ]]
yes "$lukspw" | \
bpart ${boot_devs[@]}
else
for dev in ${devs[@]}; do
+ mkfs.ext2 `grub_extdev`
cryptsetup luksOpen `rootdev` `root-cryptname` \
--key-file $luks_dir/host-$HOSTNAME
done
mount -o subvolid=0 $first_boot_dev /mnt
cd /mnt
-btrfs subvolume set-default 0 /mnt
+btrfs subvolume set-default 0 /mnt # already default, just ensuring it.
-# for libreboot systems
+# for libreboot systems.
mkdir -p /mnt/grub2
-cat >/mnt/grub2/libreboot_grub.cfg <<'EOF'
-#!/bin/sh
-# shebang is for editor file mode detection only
-
-if [ -s $prefix/grubenv ]; then
- load_env
-fi
-
-set default="0"
-set timeout=1
-
-menuentry debianstable_bootstrap {
- configfile /$1/boot/grub/grub.cfg
-}
-
-for dir in /boot_*; do
- if [ x$dir = x$default_subvol ]; then
- default=default_id
- menuentry $dir --id=default_id {
- configfile $1/grub/grub.cfg
- }
- else
- menuentry $dir {
- configfile $1/grub/grub.cfg
- }
- fi
-done
-EOF
+cp $FAI/distro-install-common/libreboot_grub.cfg /mnt/grub2
if [[ $DISTRO == debianstable_bootstrap ]]; then
# this is just convenience for the libreboot_grub config
umount /mnt
## end create subvols ##
+dev=${boot_devs[0]}
+mount $first_grub_extdev /mnt
+grub-editenv /mnt/grubenv set did_fai_check=true
+grub-editenv /mnt/grubenv set last_boot=/$boot_vol
+umount /mnt
if [[ $DISTRO == debianstable_bootstrap ]]; then
cat > /tmp/fai/fstab <<EOF
exit 1
fi
-# note:
-# fcopy -i = ignore nonmatching class error, always return 0.
-fcopy -riM /root/.ssh
+
+if ! type -t fcopy &>/dev/null; then
+ sudo apt-get -y install fai-client
+fi
chroot $FAI_ROOT bash <<'EOFOUTER'
debconf-set-selections <<EOF
apt-get install -y pxe-kexec
EOFOUTER
+# note: # fcopy -i = ignore nonmatching class error, always return 0.
+# this is also done by FABASE/10-misc by default.
+fcopy -ir /root
if ifclass STABLE_BOOTSTRAP; then
+ fcopy -ri /etc/systemd/system
+ chroot $FAI_ROOT bash <<'EOFOUTER'
+systemctl enable fai_check.service
+EOFOUTER
exit 0
fi
-if ! type -t fcopy &>/dev/null; then
- sudo apt-get -y install fai-client
-fi
-
dir=/q/root/shadow
fai_shadow=$FAI/distro-install-common/shadow
if [[ ! -e $dir && -e $fai_shadow ]]; then
fi
$FAI/distro-install-common/end
-if ifclass STABLE || ifclass LINODESTABLE; then
- fcopy -M /etc/apt/preferences
-fi
-if ifclass DEBIAN; then
- fcopy -M /etc/apt/preferences.d/unstable
-fi
-fcopy -riM /etc/apt/sources.list.d
+# these get copied in an earlier stage by fai, but leaving it here since
+# I run this as a single post-fai script to update things that have changed.
+fcopy -ri /etc/apt/preferences.d
+fcopy -ri /etc/apt/sources.list.d
$ROOTCMD apt-get update
-
rm -f $FAI_ROOT/etc/apt/sources.list
chroot $FAI_ROOT bash <<'EOF'
for g in plugdev audio video cdrom; do
$ROOTCMD usermod -a -G $g traci
done
-
# note: with a vm, pxe boot is turned off in the bios after it's first reboot.
cleanup() { :; }
-./pxe-server :
+./pxe-server
ssh $opts root@faiserver ./faiserver-setup
# Add debug to -f flag for more verbose output.
-std_arg="-u nfs://faiserver/srv/fai/config"
-e fai-chboot -Iv $std_arg default # reset so we are idempotent
-kernel=$(fai-chboot -L '^default$' | awk '{print $3}')
-type -t host &>/dev/null || apt-get -y install dnsutils
-# resolve host using gateway address
-my_ip=$(host faiserver $(route -n | sed -rn 's/^(0\.){3}0\s+(\S+).*/\2/p') | \
- sed -rn 's/^\S+ has address //p')
-k_args=$(fai-chboot -L '^default$' | \
- sed -r "s/^(\S+\s+){3}(.*root=)(.*)/\2$my_ip:\3/")
-e fai-chboot -k "$k_args" -v -f verbose,sshd,createvt,reboot $std_arg $kernel default
-
# make the faiserver also the apt proxy server
apt-get -y install apt-cacher-ng
--- /dev/null
+#!/bin/bash
+
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
+
+
+usage() {
+ cat <<'EOF'
+usage $0 [hostname|ip|default]...
+
+Sets up tftp pxe config. No argument disables for all hosts.
+EOF
+ exit $1
+}
+
+cd "${BASH_SOURCE%/*}" # directory of the script
+
+ssh root@faiserver bash -s "$@" <myfai-chboot-local
--- /dev/null
+#!/bin/bash
+
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR
+
+[[ $EUID == 0 ]] || exec sudo "${BASH_SOURCE}" "$@"
+
+e() { echo "$@"; "$@"; }
+
+
+rm -f /srv/tftp/fai/pxelinux.cfg/*
+if [[ ! $1 ]]; then
+ exit 0
+fi
+
+std_arg="-u nfs://faiserver/srv/fai/config"
+e fai-chboot -Iv $std_arg default # set it to default to get a val out of it next
+kernel=$(fai-chboot -L '^default$' | awk '{print $3}')
+# man page doesn't explain this, but this deletes & thus disables
+# all chboot systems.
+rm -f /srv/tftp/fai/pxelinux.cfg/*
+type -t host &>/dev/null || apt-get -y install dnsutils
+gateway_ip=$(route -n | sed -rn 's/^(0\.){3}0\s+(\S+).*/\2/p')
+my_ip=$(host faiserver $gateway_ip | sed -rn 's/^\S+ has address //p')
+k_args=$(fai-chboot -L '^default$' | \
+ sed -r "s/^(\S+\s+){3}(.*root=)(.*)/\2$my_ip:\3/")
+e fai-chboot -k "$k_args" -v -f verbose,sshd,createvt,reboot $std_arg $kernel "$@"
+
+
+# todo, remove the nopxe script. adjust fai-check to reboot if it fails on the kexec.
usage() {
cat <<EOF
-Usage: ${0##*/} [OPTIONS] TYPE [HOST]
+Usage: ${0##*/} [OPTIONS] [HOST TYPE]
One line description
-TYPE is one of arch, plain, fai, or : for no pxe server.
-HOST makes the pxe server only for that specific host
+TYPE is one of arch, plain, fai.
+HOST is a hostname known to the dhcp server, or default for all, or none to disable
-r Don't redeploy fai config.
-a Wait for 2 dhcp acks, then disable the pxe server after a delay.
esac
done
-read type host <<<"$@"
+read host type <<<"$@"
-if [[ ! $type ]]; then
- echo "$0: error: exptected 1 argument of type"
- usage 1
-fi
+case $# in
+ 0|2);;
+ *)
+ echo "$0: error: expected 0 or 2 arguments"
+ echo
+ usage 1
+ ;;
+esac
if [[ $host ]]; then
host_tag="tag:$host,"
}
set-pxe() {
- ${1:-$type} | ssh wrt "cedit pxe-server /etc/dnsmasq.conf || /etc/init.d/dnsmasq restart
-if [[ $type == arch ]]; then arch-pxe-mount; fi"
+ ${type:-:}|ssh wrt "cedit pxe /etc/dnsmasq.conf || /etc/init.d/dnsmasq restart
+$([[ $type == arch ]] && echo arch-pxe-mount)"
}
if $set; then
set-pxe
-
if [[ $type == fai ]]; then
+ myfai-chboot $host
if $redep; then
fai-redep
fi
faiserver-enable
+ else
+ myfai-chboot
+ faiserver-disable
fi
fi
# bleh.
echo "waiting for $acks dhcp acks then disabling pxe"
ack-wait $acks
- set-pxe :
+ set-pxe
# previously tried waiting for one more ack then disabling faiserver,
# since it can contain sensitive info, so turn it off when not in use,
--- /dev/null
+#!/bin/bash
+# You can copy this to a http server, then curl url|sudo bash
+
+set -ex
+sed -ri '/^\s*deb\s+http/s/$/ universe/' /etc/apt/sources.list
+apt-get update
+apt-get install -y debconf
+debconf-set-selections <<EOF
+kexec-tools kexec-tools/load_kexec boolean false
+EOF
+apt-get install -y pxe-kexec
+sleep 5
+pxe-kexec -n --ignore-whitelist -l fai-generated faiserver
iankelling.org / git / automated-distro-installer / commitdiff