From: Ian Kelling Date: Tue, 11 Oct 2016 05:08:12 +0000 (-0700) Subject: better support for systems with no pxe X-Git-Url: https://iankelling.org/git/?p=automated-distro-installer;a=commitdiff_plain;h=d6df8985152a6c17523cedc3fee62694544eeaf2 better support for systems with no pxe --- diff --git a/README b/README index b535507..b7bfa78 100644 --- a/README +++ b/README @@ -1,28 +1,28 @@ Multi-boot/distro btrfs provisioning -Some things are specific to my home network. Uses PXE, designed for bare -metal but also works for pxe booted VM. - -Features people may find useful: installs encrypted arch, debian stable, -& debian testing all on the same btrfs filesystem, smartly utilizing -multiple disks, with scripts to automatically decrypt on reboots. The -partititioning and filesystem script is the biggest part and is at -fai/config/hooks/partition.DEFAULT. Other debian based distros should -work fine, and I'm planning to add Fedora support. Disks are grouped as -ssd or hdd and raided in raid 1 or raid 0 per configuration. The base -partitions are divided into boot, swap, and root, (only boot is -unencrypted). There are scripts to resize those partitions -post-provision and while the system is running. - -The repo name fai is copied from a project of the same name which this -project uses for debian installs. It stands for "fully automated -installer." +Some things are specific to my home network. Uses pxe or pxe-kexec (for +systems like libreboot with no pxe rom, we boot into a live cd or distro +for bootsraping). Works for bare-metal or vms. + +Features people may find useful: installs encrypted trisquel belanos, +arch, debian stable, & debian testing all on the same btrfs filesystem. +Smartly utilizes multiple disks, with scripts to automatically decrypt +on reboots. The partititioning and filesystem script is at +fai/config/hooks/partition.DEFAULT. Other debian based distros at least +as new as ubuntu 14.04 should work fine, and I'm planning to add Fedora +support. Disks are grouped as ssd or hdd and raided in raid 1 or raid 0 +per configuration. The base partitions are divided into boot, swap, and +root, (only boot is unencrypted). There are scripts to resize those +partitions post-provision and while the system is running. + +The repo name fai is copied from the debian project of the same name, +meaning "fully automated installer." It also fully automates configuration of an openwrt router after manual initial installation. -Provisionining is done, I sync files using unison, then automate further -setup using a different set of scripts, +After provisionining is done, I sync files using unison, then automate +further setup using a different set of scripts, https://iankelling.org/git/?p=distro-setup;a=tree. My network is a wndr3700v2 router with openwrt on it and a few pcs/laptops. @@ -47,6 +47,8 @@ faiserver-setup # install fai-server on the current machine fresize # resize swap or boot partitions in a host pxe-server # temporarily enable (usually) fai or arch boot server wrt-setup-remote # setup my router +ubuntu-xenial-live-fai-kexec # do fai install from xenial live cd using kexec +myfai-chboot # use instead of pxe-server for fai kexec based install License stuff: The license for the project is GPLv2 or later, mostly because fai is diff --git a/arch-pxe b/arch-pxe index 78c3106..5a26fd7 100755 --- a/arch-pxe +++ b/arch-pxe @@ -59,7 +59,7 @@ ssh wrt "cd /mnt/usb && rm -f tftpboot && ln -s $iso tftpboot" # The default settings in the installer expect to find the NFS at /run/archiso/bootmnt -pxe-server arch +pxe-server default arch # background: # great documentation at https://wiki.archlinux.org/index.php/PXE diff --git a/debian-pxe-preseed b/debian-pxe-preseed index f973194..c2a2087 100755 --- a/debian-pxe-preseed +++ b/debian-pxe-preseed @@ -44,4 +44,4 @@ e ln -s debian-wheezy tftpboot cd / e umount $mount_dir -e $src/pxe-server -p plain # my script +e $src/pxe-server default plain # my script diff --git a/dsfull b/dsfull index 814300e..51ecdda 100755 --- a/dsfull +++ b/dsfull @@ -26,7 +26,7 @@ fi set -x cleanup() { pxe-server :; } -pxe-server fai $host +pxe-server $host fai if $reboot; then @@ -34,7 +34,7 @@ if $reboot; then ssh $host "touch /tmp/keyscript-off; sudo reboot" ||: & fi -pxe-server -a : +pxe-server -a cleanup() { :; } error=true diff --git a/fai-revm b/fai-revm index fcb9c07..97b6338 100755 --- a/fai-revm +++ b/fai-revm @@ -79,14 +79,14 @@ is_arch_revm() { [[ ${0##*/} == arch-revm ]] } -cleanup() { ./pxe-server :; } +cleanup() { ./pxe-server; } if is_arch_revm; then - ./pxe-server arch + ./pxe-server demohost arch sleep 2 # via osinfo-query os. guessing arch is closest to latest fedora. variant=fedora22 else - ./pxe-server fai + ./pxe-server demohost fai sleep 2 # I don't think these variants actually make a diff for us, but I # use the appropriate one when trying a new distro just in case. @@ -132,7 +132,7 @@ while ! timeout -s 9 10 ssh root@$name /bin/true; do e sleep 5 done cleanup() { :; } -e pxe-server : +e pxe-server if is_arch_revm; then ./arch-init-remote $name fi diff --git a/fai/config/distro-install-common/libreboot_grub.cfg b/fai/config/distro-install-common/libreboot_grub.cfg new file mode 100644 index 0000000..b48ea52 --- /dev/null +++ b/fai/config/distro-install-common/libreboot_grub.cfg @@ -0,0 +1,58 @@ +#!/bin/sh +# shebang is for editor file mode detection only + +function save_vars { + if [ -s $envfile ]; then + for var in $@; do + save_env --file $envfile $var + done + fi +} + +function save_chosen { + last_boot=$CHOSEN + save_vars did_fai_check last_boot +} + +# we don't set this to fai check so we can't get into +# an infinite reboot cycle. We depend on the os to +# create the initial grubenv file. +set default=/debianstable_bootstrap # could use 0 here. +set timeout=1 + +for part in (ahci*4) (ata*4); do + envfile=$part/grubenv + if [ -s $envfile ]; then + load_env --file $envfile + if [ x$did_fai_check != xtrue -a x$last_boot != x$default ]; then + set default=fai-check + elif [ ! -z $last_boot ]; then + set default=$last_boot + fi + break + fi +done + +did_fai_check=false + +bs_dir=/debianstable_bootstrap +menuentry $bs_dir --id=$bs_dir { + save_chosen + configfile $bs_dir/boot/grub/grub.cfg +} + +for dir in /boot_*; do + if [ $dir == '/boot_*' ]; then + break + fi + menuentry $dir --id=$dir { + save_chosen + configfile $1/grub/grub.cfg + } +done + +menuentry fai-check --id=fai-check { + did_fai_check=true + save_vars did_fai_check + configfile $bs_dir/boot/grub/grub.cfg +} diff --git a/fai/config/files/etc/apt/preferences/LINODESTABLE b/fai/config/files/etc/apt/preferences.d/testing/LINODESTABLE similarity index 100% rename from fai/config/files/etc/apt/preferences/LINODESTABLE rename to fai/config/files/etc/apt/preferences.d/testing/LINODESTABLE diff --git a/fai/config/files/etc/apt/preferences/STABLE b/fai/config/files/etc/apt/preferences.d/testing/STABLE similarity index 100% rename from fai/config/files/etc/apt/preferences/STABLE rename to fai/config/files/etc/apt/preferences.d/testing/STABLE diff --git a/fai/config/files/etc/systemd/system/fai_check.service/STABLE_BOOTSTRAP b/fai/config/files/etc/systemd/system/fai_check.service/STABLE_BOOTSTRAP new file mode 100644 index 0000000..2328452 --- /dev/null +++ b/fai/config/files/etc/systemd/system/fai_check.service/STABLE_BOOTSTRAP @@ -0,0 +1,9 @@ +[Unit] +Description=check whether to kexec to fai, reboot, or do nothing + +[Service] +Type=oneshot +ExecStart=/root/fai-check + +[Install] +WantedBy=multi-user.target diff --git a/fai/config/files/root/fai-check/STABLE_BOOTSTRAP b/fai/config/files/root/fai-check/STABLE_BOOTSTRAP new file mode 100755 index 0000000..29bb1fb --- /dev/null +++ b/fai/config/files/root/fai-check/STABLE_BOOTSTRAP @@ -0,0 +1,43 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + +fai_check=false +check-fai() { + # we could just as well check if last_boot != /debianstable_boostrap + # the intent with this one is just a little clearer. + if [[ $did_fai_check == true ]]; then + fai_check=true + pxe-kexec -n --ignore-whitelist -l fai-generated faiserver ||: + else + return 0 + fi +} + +first=true +for dev in $(btrfs fi show / | sed -rn 's#^\s*devid\s.*\s([^0-9 ]+)\S+$#\1#p' \ + |sort); do + dev+=4 + mount $dev /mnt + if $first; then + if [[ -e /mnt/grubenv ]]; then + source <(grub-editenv /mnt/grubenv list) + fi + first=false + check-fai + else + # we make sure there is only 1 grubenv, + # so grub can just find the first one, in whatever order + # if looks at them, which may not be the same as us. + # If the disk dies, we just lose the default boot option, + # we will have to do manual steps to replace it anyways. + rm -f /mnt/gruvenv + fi + umount /mnt +done + +if $fai_check && [[ $last_boot != /debianstable_boostrap ]]; then + # no need to reboot if we actually want to boot into this os. + reboot +fi diff --git a/fai/config/hooks/partition.DEFAULT b/fai/config/hooks/partition.DEFAULT index f613226..9905602 100755 --- a/fai/config/hooks/partition.DEFAULT +++ b/fai/config/hooks/partition.DEFAULT @@ -20,6 +20,8 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR # # fai's setup-storage won't do btrfs on luks, # # so we do it ourself :) +# inspiration taken from files in fai-setup-storage package + skiptask partition || ! type skiptask # for running not in fai @@ -28,7 +30,13 @@ skiptask partition || ! type skiptask # for running not in fai rootn=1 swapn=2 bootn=3 -bios_grubn=4 +# ext partition so grub can write persistent variables, +# so it can do a one time boot. +grub_extn=4 +# bios boot partition, +# https://wiki.archlinux.org/index.php/GRUB +bios_grubn=5 +lastn=$bios_grubn boot_mib=4000 @@ -55,7 +63,10 @@ add-part() { # add partition suffix to $dev bootdev() { add-part $@ $bootn; } rootdev() { add-part $@ $rootn; } swapdev() { add-part $@ $swapn; } -bios_grubdev() { add-part $@ $bios_grubn; } +grub_extdev() { add-part $@ $grub_extn; } +# Commented because it's not used, but left because it +# finishes the pattern and if we ever do need to use it, it's here. +#bios_grubdev() { add-part $@ $bios_grubn; } crypt-dev() { echo /dev/mapper/crypt_dev_${1##*/}; } crypt-name() { echo crypt_dev_${1##*/}; } @@ -73,7 +84,6 @@ else partition=false # change to true to force a full wipe fi -lastn=$bios_grubn hdds=() @@ -98,7 +108,7 @@ fi # check if the partitions exist have the right filesystems #blkid="$(blkid -s TYPE)" for dev in ${short_devs[@]}; do - ! $partition || break + if $partition; then break; fi y=$(readlink -f $dev) x=($y[0-9]) [[ ${#x[@]} == "${lastn}" ]] || partition=true @@ -115,8 +125,8 @@ done if $partition && ifclass PARTITION_PROMPT; then echo "Press any key except ctrl-c to continue and partition these drives:" - echo " ${short_devs[@]}" - read + echo " ${short_devs[*]}" + read -r fi devs=() @@ -126,7 +136,7 @@ for short_dev in ${short_devs[@]}; do done - +first=false boot_devs=() for dev in ${devs[@]}; do if ifclass frodo; then @@ -157,6 +167,10 @@ for dev in ${devs[@]}; do else boot_devs+=(`bootdev`) fi + if [[ $boot_devs && $first ]]; then + first_grub_extdev=`grub_extdev` + first=false + fi done if [[ ! $DISTRO ]]; then @@ -175,6 +189,7 @@ if [[ ! $DISTRO ]]; then exit 1 fi fi +first_boot_dev=${boot_devs[0]} case ${#boot_devs[@]} in @@ -191,7 +206,6 @@ bpart() { # btrfs a partition esac } -first_boot_dev=${boot_devs[0]} # keyfiles generated like: # head -c 2048 /dev/urandom | od | s dd of=/q/root/luks/host-demohost @@ -214,7 +228,6 @@ fi first_root_crypt=$(root-cryptdev ${devs[0]}) -bios_grubn=4 # 1.5 x based on https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/7/html/Installation_Guide/sect-disk-partitioning-setup-x86.html#sect-custom-partitioning-x86 swap_mib=$(( $(grep ^MemTotal: /proc/meminfo | \ awk '{print $2}') * 3/(${#devs[@]} * 2 ) / 1024 )) @@ -252,26 +265,45 @@ if $partition; then parted -s $dev mklabel gpt # MiB because parted complains about alignment otherwise. pcmd="parted -a optimal -s -- $dev" - $pcmd mkpart primary "ext3" 4MiB ${root_end}MiB + $pcmd mkpart primary "ext3" 12MiB ${root_end}MiB $pcmd mkpart primary "linux-swap" ${root_end}MiB ${swap_end}MiB $pcmd mkpart primary "" ${swap_end}MiB ${disk_mib}MiB - # gpt ubuntu cloud image uses ~4. fai uses 1 MiB. - # I read something in the parted manual saying cheap flash media - # likes to start at 4. + # i only need a few k, but googling min size, + # I found someone saying that gparted required + # required at least 8 because of their hard drive cylinder size. + # And 8 is still very tiny. + $pcmd mkpart primary "ext2" 4MiB 12MiB + # gpt ubuntu cloud image uses ~4 mb for this partition. fai uses 1 MiB. + # so, I use 3, whatever. + # note: parted manual saying cheap flash media + # should to start at 4. $pcmd mkpart primary "" 1MiB 4MiB $pcmd set $bios_grubn bios_grub on $pcmd set $bootn boot on # generally not needed on modern systems # the mkfs failed before on a vm, which prompted me to add # sleep .1 - # then failed on a physical machine + # then it failed again on a physical machine # with: # Device /dev/disk/by-id/foo doesn't exist or access denied, - # so here we wait. + # so I added a wait until it existed. + # Then I added the mkfs.ext2, which claimed to succeed, + # but then couldn't be found upon reboot. In that case we didn't + # wait at all. So I've added a 3 second minimum wait. + sleep 3 secs=0 while [[ ! -e `rootdev` ]] && (( secs < 10 )); do sleep 1 secs=$((secs +1)) done + # Holds just a single file, rarely written, so + # use ext2, like was often used for the /boot partition. + # This exists because grub can only persist data to a non-cow fs. + # And we use persisting a var in grub to do a one time boot. + # We could pass the data on the kernel command line and persist it + # to grubenv after booting, but that relies on the boot always succeeding. + # This is just a bit more robust, and it could work for booting + # into ipxe which can't persist data, if we ever got that working. + mkfs.ext2 `grub_extdev` yes YES | cryptsetup luksFormat `rootdev` $luks_dir/host-$HOSTNAME \ -c aes-cbc-essiv:sha256 -s 256 || [[ $? == 141 ]] yes "$lukspw" | \ @@ -296,6 +328,7 @@ if $partition; then bpart ${boot_devs[@]} else for dev in ${devs[@]}; do + mkfs.ext2 `grub_extdev` cryptsetup luksOpen `rootdev` `root-cryptname` \ --key-file $luks_dir/host-$HOSTNAME done @@ -342,38 +375,11 @@ fi mount -o subvolid=0 $first_boot_dev /mnt cd /mnt -btrfs subvolume set-default 0 /mnt +btrfs subvolume set-default 0 /mnt # already default, just ensuring it. -# for libreboot systems +# for libreboot systems. mkdir -p /mnt/grub2 -cat >/mnt/grub2/libreboot_grub.cfg <<'EOF' -#!/bin/sh -# shebang is for editor file mode detection only - -if [ -s $prefix/grubenv ]; then - load_env -fi - -set default="0" -set timeout=1 - -menuentry debianstable_bootstrap { - configfile /$1/boot/grub/grub.cfg -} - -for dir in /boot_*; do - if [ x$dir = x$default_subvol ]; then - default=default_id - menuentry $dir --id=default_id { - configfile $1/grub/grub.cfg - } - else - menuentry $dir { - configfile $1/grub/grub.cfg - } - fi -done -EOF +cp $FAI/distro-install-common/libreboot_grub.cfg /mnt/grub2 if [[ $DISTRO == debianstable_bootstrap ]]; then # this is just convenience for the libreboot_grub config @@ -388,6 +394,11 @@ cd / umount /mnt ## end create subvols ## +dev=${boot_devs[0]} +mount $first_grub_extdev /mnt +grub-editenv /mnt/grubenv set did_fai_check=true +grub-editenv /mnt/grubenv set last_boot=/$boot_vol +umount /mnt if [[ $DISTRO == debianstable_bootstrap ]]; then cat > /tmp/fai/fstab </dev/null; then + sudo apt-get -y install fai-client +fi chroot $FAI_ROOT bash <<'EOFOUTER' debconf-set-selections </dev/null; then - sudo apt-get -y install fai-client -fi - dir=/q/root/shadow fai_shadow=$FAI/distro-install-common/shadow if [[ ! -e $dir && -e $fai_shadow ]]; then @@ -35,19 +39,15 @@ if [[ ! -e $dir && -e $fai_shadow ]]; then fi $FAI/distro-install-common/end -if ifclass STABLE || ifclass LINODESTABLE; then - fcopy -M /etc/apt/preferences -fi -if ifclass DEBIAN; then - fcopy -M /etc/apt/preferences.d/unstable -fi -fcopy -riM /etc/apt/sources.list.d +# these get copied in an earlier stage by fai, but leaving it here since +# I run this as a single post-fai script to update things that have changed. +fcopy -ri /etc/apt/preferences.d +fcopy -ri /etc/apt/sources.list.d $ROOTCMD apt-get update - rm -f $FAI_ROOT/etc/apt/sources.list chroot $FAI_ROOT bash <<'EOF' @@ -91,4 +91,3 @@ EOF for g in plugdev audio video cdrom; do $ROOTCMD usermod -a -G $g traci done - diff --git a/faiserver-revm b/faiserver-revm index 1065f3b..7842723 100755 --- a/faiserver-revm +++ b/faiserver-revm @@ -34,6 +34,6 @@ done # note: with a vm, pxe boot is turned off in the bios after it's first reboot. cleanup() { :; } -./pxe-server : +./pxe-server ssh $opts root@faiserver ./faiserver-setup diff --git a/faiserver-setup b/faiserver-setup index 18933bb..e43d019 100755 --- a/faiserver-setup +++ b/faiserver-setup @@ -108,17 +108,6 @@ e fai-setup -vf # Add debug to -f flag for more verbose output. -std_arg="-u nfs://faiserver/srv/fai/config" -e fai-chboot -Iv $std_arg default # reset so we are idempotent -kernel=$(fai-chboot -L '^default$' | awk '{print $3}') -type -t host &>/dev/null || apt-get -y install dnsutils -# resolve host using gateway address -my_ip=$(host faiserver $(route -n | sed -rn 's/^(0\.){3}0\s+(\S+).*/\2/p') | \ - sed -rn 's/^\S+ has address //p') -k_args=$(fai-chboot -L '^default$' | \ - sed -r "s/^(\S+\s+){3}(.*root=)(.*)/\2$my_ip:\3/") -e fai-chboot -k "$k_args" -v -f verbose,sshd,createvt,reboot $std_arg $kernel default - # make the faiserver also the apt proxy server apt-get -y install apt-cacher-ng diff --git a/myfai-chboot b/myfai-chboot new file mode 100755 index 0000000..e216aa0 --- /dev/null +++ b/myfai-chboot @@ -0,0 +1,18 @@ +#!/bin/bash + +set -eE -o pipefail +trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR + + +usage() { + cat <<'EOF' +usage $0 [hostname|ip|default]... + +Sets up tftp pxe config. No argument disables for all hosts. +EOF + exit $1 +} + +cd "${BASH_SOURCE%/*}" # directory of the script + +ssh root@faiserver bash -s "$@" &2' ERR + +[[ $EUID == 0 ]] || exec sudo "${BASH_SOURCE}" "$@" + +e() { echo "$@"; "$@"; } + + +rm -f /srv/tftp/fai/pxelinux.cfg/* +if [[ ! $1 ]]; then + exit 0 +fi + +std_arg="-u nfs://faiserver/srv/fai/config" +e fai-chboot -Iv $std_arg default # set it to default to get a val out of it next +kernel=$(fai-chboot -L '^default$' | awk '{print $3}') +# man page doesn't explain this, but this deletes & thus disables +# all chboot systems. +rm -f /srv/tftp/fai/pxelinux.cfg/* +type -t host &>/dev/null || apt-get -y install dnsutils +gateway_ip=$(route -n | sed -rn 's/^(0\.){3}0\s+(\S+).*/\2/p') +my_ip=$(host faiserver $gateway_ip | sed -rn 's/^\S+ has address //p') +k_args=$(fai-chboot -L '^default$' | \ + sed -r "s/^(\S+\s+){3}(.*root=)(.*)/\2$my_ip:\3/") +e fai-chboot -k "$k_args" -v -f verbose,sshd,createvt,reboot $std_arg $kernel "$@" + + +# todo, remove the nopxe script. adjust fai-check to reboot if it fails on the kexec. diff --git a/pxe-server b/pxe-server index d8cbeb8..6adbf19 100755 --- a/pxe-server +++ b/pxe-server @@ -28,11 +28,11 @@ trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?" >&2' ERR usage() { cat <