From: Ian Kelling Date: Wed, 18 Aug 2021 04:03:04 +0000 (-0400) Subject: Merge branch 'upstream' X-Git-Url: https://iankelling.org/git/?p=automated-distro-installer;a=commitdiff_plain;h=591c84435e22df12f2b5121c08578bff7c6bb047;hp=-c Merge branch 'upstream' --- 591c84435e22df12f2b5121c08578bff7c6bb047 diff --combined fai/config/basefiles/mk-basefile index e4fd05d,085b9a4..39471d8 --- a/fai/config/basefiles/mk-basefile +++ b/fai/config/basefiles/mk-basefile @@@ -1,7 -1,8 +1,7 @@@ #! /bin/bash - # mk-basefile, create basefiles for some distributions # - # Thomas Lange, Uni Koeln, 2011-2020 + # Thomas Lange, Uni Koeln, 2011-2021 # based on the Makefile implementation of Michael Goetze # # Usage example: mk-basefile -J STRETCH64 @@@ -21,7 -22,6 +21,7 @@@ # For the first stage, set the CentOS/SLC mirror in /etc/rinse/rinse.conf MIRROR_DEBIAN=http://deb.debian.org/debian/ MIRROR_UBUNTU=http://mirror.netcologne.de/ubuntu/ +MIRROR_TRISQUEL=http://mirror.fsf.org/trisquel/ MIRROR_CENTOS=http://mirror.netcologne.de/ EXCLUDE_SQUEEZE=isc-dhcp-client,isc-dhcp-common,info @@@ -30,21 -30,18 +30,23 @@@ EXCLUDE_JESSIE=inf EXCLUDE_STRETCH=info EXCLUDE_BUSTER= EXCLUDE_BULLSEYE= + EXCLUDE_BOOKWORM= + EXCLUDE_TRIXIE= EXCLUDE_SID= +EXCLUDE_BELENOS=dhcp3-client,dhcp3-common,info EXCLUDE_TRUSTY=dhcp3-client,dhcp3-common,info EXCLUDE_XENIAL=udhcpc,dibbler-client,info +EXCLUDE_FLIDAS=udhcpc,dibbler-client,info EXCLUDE_BIONIC=udhcpc,dibbler-client,info +EXCLUDE_ETIONA=udhcpc,dibbler-client,info EXCLUDE_FOCAL=udhcpc,dibbler-client,info +EXCLUDE_NABIA=udhcpc,dibbler-client,info # here you can add packages, that are needed very early INCLUDE_DEBIAN= - +INCLUDE_ETIONA=ifupdown +INCLUDE_NABIA=ifupdown setarch() { @@@ -111,14 -108,6 +113,14 @@@ EO cleanup-deb() { + if [[ $cmd ]]; then + if [[ -e $cmd ]]; then + cp "$cmd" $xtmp/tmp + chroot $xtmp "/tmp/$cmd" + else + chroot $xtmp $cmd + fi + fi chroot $xtmp apt-get clean rm -f $xtmp/etc/hostname $xtmp/etc/resolv.conf \ $xtmp/var/lib/apt/lists/*_* $xtmp/usr/bin/qemu-*-static \ @@@ -190,11 -179,12 +192,11 @@@ debgeneric() dist=${DIST%%[0-9][0-9]} local exc="EXCLUDE_$dist" [ -n "${!exc}" ] && exc="--exclude=${!exc}" || unset exc + local inc="INCLUDE_$dist" + [ -n "${!inc}" ] && inc="--include=${!inc}" || unset inc dist=${dist,,} check - if [ -n "$INCLUDE_DEBIAN" ]; then - local inc="--include=$INCLUDE_DEBIAN" - fi if [ -n "$arch" ]; then qemu-debootstrap --arch $arch ${exc} $inc $dist $xtmp $mirror @@@ -224,17 -214,16 +226,19 @@@ prtdists() SLC7_64 TRUSTY32 TRUSTY64 XENIAL32 XENIAL64 + FLIDAS64 BIONIC64 + ETIONA64 FOCAL64 + NABIA64 SQUEEZE32 SQUEEZE64 WHEEZY32 WHEEZY64 JESSIE32 JESSIE64 STRETCH32 STRETCH64 BUSTER32 BUSTER64 BULLSEYE32 BULLSEYE64 + BOOKWORM32 BOOKWORM64 + TRIXIE32 TRIXIE64 SID32 SID64 " } @@@ -254,8 -243,6 +258,8 @@@ Usage: mk-basefile [OPTION] ... DISTRIB -z Use gzip for compressing the tar file. -J Use xz for compressing the tar file. -k Keep the temporary subtree structure, do not remove it. + -x CMD Run CMD in chroot. If CMD exists as a file, copy it and run it. + Debian based only -h Print help. Usage example: mk-basefile -J STRETCH64 @@@ -273,7 -260,7 +277,7 @@@ attributes cleanup=1 attributes="--xattrs --selinux --acls" -while getopts ashzJd:kf: opt ; do +while getopts ashzJd:kf:x: opt ; do case "$opt" in a) echo "$0: Warning. -a is ignored, because xtattrs, acls and selinux are always added." ;; d) export TMPDIR=$OPTARG ;; @@@ -283,7 -270,6 +287,7 @@@ k) cleanup=0 ;; h) usage ;; s) prtdists ; exit 0;; + x) cmd="$OPTARG" ;; ?) exit 3 ;; # error in option parsing esac done @@@ -312,11 -298,9 +316,11 @@@ case "$target" i SLC6_32) slc i386 6 ;; SLC6_64) slc amd64 6 ;; SLC7_64) slc amd64 7 ;; + BELENOS*|FLIDAS*|ETIONA*|NABIA*) + debgeneric $target $MIRROR_TRISQUEL ;; TRUSTY*|XENIAL*|BIONIC*|FOCAL*) debgeneric $target $MIRROR_UBUNTU ;; - SQUEEZE*|WHEEZY*|JESSIE*|STRETCH*|BUSTER*|BULLSEYE*|SID*) + SQUEEZE*|WHEEZY*|JESSIE*|STRETCH*|BUSTER*|BULLSEYE*|BOOKWORM*|TRIXIE*|SID*) debgeneric $target $MIRROR_DEBIAN $ARCH;; *) echo "Unknown distribution. Aborting." prtdists diff --combined fai/config/class/DEBIAN.var index 367c2e3,dc133a5..a00d0f8 --- a/fai/config/class/DEBIAN.var +++ b/fai/config/class/DEBIAN.var @@@ -1,7 -1,13 +1,14 @@@ - # sources are set with fcopy - #release=buster - #apt_cdn=http://deb.debian.org - #security_cdn=http://security.debian.org -release=bullseye -apt_cdn=http://deb.debian.org -security_cdn=http://security.debian.org ++# ian, commented, sources are set with fcopy ++# release=bullseye ++# apt_cdn=http://deb.debian.org ++# security_cdn=http://security.debian.org + + # since bullseye Debian changed the suite name for security + if [ $release = buster ]; then + secsuite=buster/updates + else + secsuite=$release-security + fi CONSOLEFONT= KEYMAP=us-latin1 @@@ -14,8 -20,7 +21,8 @@@ MODULESLIST="usbhid psmouse # if you have enough RAM (>2GB) you may want to enable this line. It # also puts /var/cache into a ramdisk. -#FAI_RAMDISKS="$target/var/lib/dpkg $target/var/cache" +# ian: uncommented +FAI_RAMDISKS="$target/var/lib/dpkg $target/var/cache" # if you want to use the faiserver as APT proxy -#APTPROXY=http://faiserver:3142 +# APTPROXY=http://faiserver:3142 diff --combined fai/config/package_config/DEBIAN index 25e9d9e,fe7cff7..c1cb0bf --- a/fai/config/package_config/DEBIAN +++ b/fai/config/package_config/DEBIAN @@@ -1,11 -1,31 +1,18 @@@ -PACKAGES install-norec -apt-transport-https # is only needed for stretch -debconf-utils -file -less -linuxlogo -rsync -openssh-client openssh-server -time -procinfo -nullmailer -eject -locales -console-setup kbd -pciutils usbutils -unattended-upgrades +# otherwise sshd takes like 10 seconds to start +PACKAGES install BUSTER +haveged PACKAGES install NONFREE # you may want these non-free kernel drivers firmware-bnx2 firmware-bnx2x firmware-realtek firmware-linux-nonfree + # a list of firmware for wifi/wireless + atmel-firmware firmware-atheros firmware-brcm80211 + firmware-iwlwifi firmware-libertas firmware-ralink firmware-zd1211 + firmware-brcm80211 firmware-ti-connectivity + firmware-netronome firmware-netxen firmware-realtek + firmware-cavium + # firmware-ipw2x00 # needs a debconf question PACKAGES install I386 linux-image-686-pae @@@ -14,8 -34,6 +21,8 @@@ memtest86 PACKAGES install CHROOT linux-image-686-pae- linux-image-amd64- +initramfs-tools-core- +dropbear-initramfs- PACKAGES install AMD64 linux-image-amd64 @@@ -25,16 -43,11 +32,13 @@@ PACKAGES install ARM6 grub-efi-arm64 linux-image-arm64 -PACKAGES install GRUB_PC -grub-pc +# this is duplicate with STANDARD. +#PACKAGES install GRUB_PC +#grub-pc + +#PACKAGES install GRUB_EFI +#grub-efi -PACKAGES install GRUB_EFI -grub-efi PACKAGES install LVM lvm2 - - PACKAGES install CLOUD - unattended-upgrades diff --combined fai/config/package_config/STANDARD index 00376cb,de178ae..413254f --- a/fai/config/package_config/STANDARD +++ b/fai/config/package_config/STANDARD @@@ -18,66 -18,20 +18,64 @@@ man-d manpages mime-support ncurses-term -netcat-traditional openssh-client pciutils perl - # newer distros dont have python, it gets naturally removed ++# ian: newer distros dont have python, it gets naturally removed +python +python-minimal +python3 +python3-minimal reportbug telnet traceroute ucf xz-utils +# ian standard packages +# lsof is used in my btrfs util scritps. +# netcat is used for proxy. +keyutils +cryptsetup +btrfs-progs +sudo +bridge-utils +netcat-openbsd +lsof +debconf-utils +file +less +rsync +openssh-client openssh-server +time +procinfo +locales +console-setup kbd +pciutils usbutils +unattended-upgrades +initramfs-tools-core +dropbear-initramfs +apt-transport-https +# ifupdown because etiona doesnt have it by default +# and fai scripts want to call ifquery. +ifupdown +netplan.io- +libnss-resolve +publicsuffix +iso-codes +# new package buster/nabia+ +cryptsetup-initramfs +# https://wiki.debian.org/UsrMerge +usrmerge +# for btrbk +zstd -PACKAGES install-norec STRETCH BUSTER -python -python-minimal +# iank, copied from DEBIAN so it goes into ubuntu too +PACKAGES install GRUB_PC +grub-pc -PACKAGES install-norec BULLSEYE -python3 -python3-minimal +PACKAGES install GRUB_EFI +# normally would have just grub-efi +# but theres a dependency problem with it in nabia: for some reason it depends on +# a version in security, but theres a later version in updates that the system +# really wants to install. +grub-efi-amd64 - - PACKAGES install NABIA FOCAL diff --combined fai/config/scripts/DEBIAN/10-rootpw index 7a7f617,329e2f6..539f2eb --- a/fai/config/scripts/DEBIAN/10-rootpw +++ b/fai/config/scripts/DEBIAN/10-rootpw @@@ -4,9 -4,14 +4,9 @@@ error=0; trap 'error=$(($?>$error?$?:$e # set root password if [ -n "$ROOTPW" ]; then - $ROOTCMD usermod -p "$ROOTPW" root - else + $ROOTCMD chpasswd --encrypted <<< "root:${ROOTPW}" + elif [ -n "$username" ]; then $ROOTCMD usermod -L root - # enable sudo for user - ainsl /etc/sudoers "$username ALL = ALL" - if [ ! -f $target/usr/bin/sudo ]; then - echo "WARNING. Package sudo is not installed" - fi fi exit $error diff --combined fai/config/scripts/GRUB_PC/10-setup index 85e90d2,270b356..e26aa26 --- a/fai/config/scripts/GRUB_PC/10-setup +++ b/fai/config/scripts/GRUB_PC/10-setup @@@ -3,7 -3,6 +3,7 @@@ error=0; trap 'error=$(($?>$error?$?:$error))' ERR # save maximum error code +set -x set -a # do not set up grub during dirinstall @@@ -17,7 -16,7 +17,7 @@@ if [ -z "$BOOT_DEVICE" ]; the exit 189 fi - # disable os-prober because of #788062 + # disable os-prober because of #802717 ainsl /etc/default/grub 'GRUB_DISABLE_OS_PROBER=true' # skip the rest, if not an initial installation @@@ -26,7 -25,27 +26,27 @@@ if [ $FAI_ACTION != "install" ]; the exit $error fi - GROOT=$($ROOTCMD grub-probe -tdrive -d $BOOT_DEVICE) + get_stable_devname() { + + local _DEV="$1" + local i + declare -a _RES + + # prefer SCSI over ATA over WWN over path + # do not use by-path + + for i in $($ROOTCMD udevadm info -r --query=symlink "$_DEV"); do + if [[ "$i" =~ /by-id/scsi ]]; then + _RES[10]="$i" + elif [[ "$i" =~ /by-id/ata ]]; then + _RES[20]="$i" + elif [[ "$i" =~ /by-id/wwn ]]; then + _RES[99]="$i" + fi + done + + echo "${_RES[@]::1}" + } # handle /boot in lvm-on-md _bdev=$(readlink -f $BOOT_DEVICE) @@@ -36,22 -55,30 +56,33 @@@ f # Check if RAID is used for the boot device if [[ $BOOT_DEVICE =~ '/dev/md' ]]; then + GROOT=$($ROOTCMD grub-probe -tdrive -d $BOOT_DEVICE) raiddev=${BOOT_DEVICE#/dev/} # install grub on all members of RAID for device in $(LC_ALL=C perl -ne 'if(/^'$raiddev'\s.+raid\d+\s(.+)/){ $_=$1; s/\d+\[\d+\]//g; s/(nvme.+?)p/$1/g; print }' /proc/mdstat); do - echo Install grub on /dev/$device + pdevice=$(get_stable_devname /dev/$device) + if [ -z "$pdevice" ]; then + # if we cannot find a persistent name (for e.g. in a VM) use old name + pdevice="/dev/$device" + fi + mbrdevices+="$pdevice, " + echo Installing grub on /dev/$device = $pdevice $ROOTCMD grub-install --no-floppy "/dev/$device" done + # remove last , + mbrdevices=${mbrdevices%, } else - for dev in $BOOT_DEVICE; do - GROOT=$($ROOTCMD grub-probe -tdrive -d $dev) - $ROOTCMD grub-install --no-floppy "$GROOT" - if [ $? -eq 0 ]; then - echo "Grub installed on $dev = $GROOT" - fi - done - mbrdevices=$(get_stable_devname $BOOT_DEVICE) ++ for dev in $BOOT_DEVICE; do ++ mbrdevices=$(get_stable_devname $dev) + if [ -z "$mbrdevices" ]; then - # if we cannot find a persistent name (for e.g. in a VM) use old name - mbrdevices=$BOOT_DEVICE ++ # if we cannot find a persistent name (for e.g. in a VM) use old name ++ mbrdevices=$dev + fi - echo "Installing grub on $BOOT_DEVICE = $mbrdevices" ++ echo "Installing grub on $dev = $mbrdevices" + $ROOTCMD grub-install --no-floppy "$mbrdevices" ++ done fi - $ROOTCMD update-grub + echo "grub-pc grub-pc/install_devices multiselect $mbrdevices" | $ROOTCMD debconf-set-selections + $ROOTCMD dpkg-reconfigure grub-pc exit $error diff --combined fai/config/scripts/LAST/50-misc index 2b41683,a573f50..687700a --- a/fai/config/scripts/LAST/50-misc +++ b/fai/config/scripts/LAST/50-misc @@@ -15,15 -15,13 +15,15 @@@ els fi fi - usedm=$(dmsetup ls 2>/dev/null | egrep -v '^live-rw|^live-base|^No devices found' | wc -l) - if [ $usedm -ne 0 ]; then - if [ ! -d $target/etc/lvm ]; then - echo ERROR: Found lvm devices, but the lvm2 package was not installed - error=1 - fi - fi + # i use dm for crypt, not lvm, so this gives false positive. todo, send patch to remove this + # upstream. + # usedm=$(dmsetup ls 2>/dev/null | egrep -v '^live-rw|^live-base|^No devices found' | wc -l) + # if [ $usedm -ne 0 ]; then + # if [ ! -d $target/etc/lvm ]; then + # echo ERROR: Found lvm devices, but the lvm2 package was not installed + # error=1 + # fi + # fi fi # remove backup files from cfengine, but only if cfengine is installed @@@ -47,7 -45,7 +47,7 @@@ if [ $do_init_tasks -eq 1 ] ; the fi # Make sure everything is configured properly -if ifclass DEBIAN ; then +if ifclass DEBIAN || ifclass UBUNTU; then $ROOTCMD apt-get -f install -y fi @@@ -74,7 -72,7 +74,7 @@@ setrel() return fi - dists="jessie stretch buster bullseye bookworm focal bionic xenial trusty" + dists="jessie stretch buster bullseye bookworm trixie focal bionic xenial trusty" for d in $dists; do if grep -iq $d $target/etc/os-release; then release=$d @@@ -88,7 -86,7 +88,7 @@@ setre if [ -f $target/etc/apt/sources.list -a -n "$release" ]; then grep -q 'file generated by fai-cd' $target/etc/apt/sources.list && cat < $target/etc/apt/sources.list deb $apt_cdn/debian $release main contrib non-free - deb $apt_cdn/debian-security $release/updates main contrib non-free + deb $security_cdn/debian-security ${secsuite} main contrib non-free #deb [trusted=yes] http://fai-project.org/download $release koeln EOF # if the package fai-server was installed, enable the project's repository