From 80075767383c310e73cfcf1eb682bd3cc8475a20 Mon Sep 17 00:00:00 2001 From: gjb Date: Thu, 16 Oct 2014 23:25:38 +0000 Subject: [PATCH] MFstable10 r273198: MFC r273093, r273096: r273093: Merge the following from ^/projects/release-vmimage: r272436, r272437, r272792: r272436: Remove the first argument to panic(), which was initially intended to be the exit code, however when a non-zero exit code was returned to release/Makefile, this would prevent any remaining (and possibly successful) stages from being attempted. r272437: If the vm-base target fails, prevent the vm-image target from being run since it cannot possibly succeed. r272792: Add /usr/local/bin and /usr/local/sbin to PATH, needed if third-party software needs to use utilities outside of the base system during post-install stages (indexinfo is one culprit). r273096: Merge the following from ^/projects/release-vmimage: r273076, r273077, r273079, r273095: r273076: Add a separate make(1) target to release/Makefile to build FreeBSD virtual machine disk images for use on the Microsoft Azure service. For now, this target is not directly connected to the build, however can be manually invoked. The 'vm-azure' target invokes {amd64,i386}/mk-azure.sh, which does the heavy lifting to produce proper VHDs. mk-azure.sh uses a configuration file, defaulting to tools/azure.conf if otherwise unset. r273077: Clear VM_RC_LIST. r273079: Fix signal list to trigger umount(8). r273095: Output an informational message when mkimg(1) runs, so it does not appear that the process has stopped while waiting for a 'y/n' response when waagent is deprovisioned. Relnotes: yes Approved by: re (delphij) Sponsored by: The FreeBSD Foundation git-svn-id: svn://svn.freebsd.org/base/releng/10.1@273199 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- release/Makefile | 15 ++++ release/amd64/mk-azure.sh | 173 ++++++++++++++++++++++++++++++++++++ release/amd64/mk-vmimage.sh | 25 ++++-- release/i386/mk-azure.sh | 173 ++++++++++++++++++++++++++++++++++++ release/i386/mk-vmimage.sh | 25 ++++-- release/tools/azure.conf | 14 +++ 6 files changed, 409 insertions(+), 16 deletions(-) create mode 100755 release/amd64/mk-azure.sh create mode 100755 release/i386/mk-azure.sh create mode 100644 release/tools/azure.conf diff --git a/release/Makefile b/release/Makefile index a86e3687f..ca186672e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -117,6 +117,7 @@ VMTARGETS= vm-base vm-image VMFORMATS?= vhd vmdk qcow2 raw VMSIZE?= 20G VMBASE?= vm +AZURECONF?= ${.CURDIR}/tools/azure.conf CLEANFILES= packagesystem *.txz MANIFEST system ${IMAGES} .if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES) @@ -137,6 +138,12 @@ CLEANDIRS= dist ftp release bootonly dvd .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES) CLEANDIRS+= ${VMTARGETS} .endif +.if exists(${.CURDIR}/${TARGET}/mk-azure.sh) +CLEANFILES+= ${OSRELEASE}.vhd \ + ${OSRELEASE}.vhd.raw \ + azure.img +CLEANDIRS+= vm-azure +.endif beforeclean: chflags -R noschg . .include @@ -372,3 +379,11 @@ vm-image: vm-base . endif .endif touch ${.TARGET} + +vm-azure: +.if exists(${.CURDIR}/${TARGET}/mk-azure.sh) + env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \ + ${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \ + ${WORLDDIR} ${.TARGET} ${VMSIZE} ${OSRELEASE}.vhd +.endif + touch ${.TARGET} diff --git a/release/amd64/mk-azure.sh b/release/amd64/mk-azure.sh new file mode 100755 index 000000000..45b70ae73 --- /dev/null +++ b/release/amd64/mk-azure.sh @@ -0,0 +1,173 @@ +#!/bin/sh +#- +# Copyright (c) 2014 The FreeBSD Foundation +# All rights reserved. +# +# This software was developed by Glen Barber under sponsorship +# from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# mk-azure.sh: Create virtual machine disk images for Microsoft Azure +# +# $FreeBSD$ +# + +export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin" + +usage() { + echo "Usage:" + echo -n "$(basename ${0}) vm-azure " + echo " " + exit 1 +} + +panic() { + msg="${@}" + printf "${msg}\n" + if [ ! -z "${mddev}" ]; then + mdconfig -d -u ${mddev} + fi + # Do not allow one failure case to chain through any remaining image + # builds. + exit 0 +} + +vm_create_azure() { + # Arguments: + # vm-azure + + VMBASE="${1}" + WORLDDIR="${2}" + DESTDIR="${3}" + VMSIZE="${4}" + VMIMAGE="${5}" + + if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \ + -o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then + usage + fi + + trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM + + i=0 + mkdir -p ${DESTDIR} + truncate -s ${VMSIZE} ${VMBASE} + mddev=$(mdconfig -f ${VMBASE}) + newfs -j /dev/${mddev} + mkdir -p ${DESTDIR} + mount /dev/${mddev} ${DESTDIR} + make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \ + installworld installkernel distribution || \ + panic 1 "\n\nCannot install the base system to ${DESTDIR}." + mount -t devfs devfs ${DESTDIR}/dev + chroot ${DESTDIR} /usr/bin/newaliases + echo '# Custom /etc/fstab for FreeBSD VM images' \ + > ${DESTDIR}/etc/fstab + echo '/dev/gpt/rootfs / ufs rw 2 2' \ + >> ${DESTDIR}/etc/fstab + # Although a swap partition is created, it is not used in Azure. + echo '#/dev/gpt/swapfs none swap sw 0 0' \ + >> ${DESTDIR}/etc/fstab + + chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \ + python python2 python27 py27-asn1 sudo bash + if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \ + ${VM_EXTRA_PACKAGES} + fi + + fetch -o ${DESTDIR}/usr/sbin/waagent \ + http://people.freebsd.org/~gjb/waagent + chmod +x ${DESTDIR}/usr/sbin/waagent + rm -f ${DESTDIR}/etc/resolv.conf + chroot ${DESTDIR} /usr/sbin/waagent -verbose -install + yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision + echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf + echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf + echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf + + echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf + echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf + + if [ ! -z "${VM_RC_LIST}" ]; then + for _rcvar in ${VM_RC_LIST}; do + echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf + done + fi + + sync + + while ! umount ${DESTDIR}/dev ${DESTDIR}; do + i=$(( $i + 1 )) + if [ $i -ge 10 ]; then + # This should never happen. But, it has happened. + msg="Cannot umount(8) ${DESTDIR}\n" + msg="${msg}Something has gone horribly wrong." + panic 1 "${msg}" + fi + sleep 1 + done + + echo "Creating image... Please wait." + + mkimg -f vhdf -s gpt \ + -b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \ + -p freebsd-swap/swapfs::1G \ + -p freebsd-ufs/rootfs:=${VMBASE} \ + -o ${VMIMAGE}.raw + + if [ ! -x "/usr/local/bin/qemu-img" ]; then + env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel + fi + + size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',') + size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) )) + qemu-img resize ${VMIMAGE}.raw ${size} + qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE} + + return 0 +} + +main() { + cmd="${1}" + shift 1 + + if [ -e "${AZURECONF}" -a ! -c "${AZURECONF}" ]; then + . ${AZURECONF} + fi + + case ${cmd} in + vm-azure) + eval vm_create_azure "$@" || return 0 + ;; + *|\?) + usage + ;; + esac + + return 0 +} + +main "$@" diff --git a/release/amd64/mk-vmimage.sh b/release/amd64/mk-vmimage.sh index 6ee148248..b3ffd232f 100755 --- a/release/amd64/mk-vmimage.sh +++ b/release/amd64/mk-vmimage.sh @@ -32,7 +32,7 @@ # $FreeBSD$ # -PATH="/bin:/usr/bin:/sbin:/usr/sbin" +PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" export PATH usage_vm_base() { @@ -58,16 +58,25 @@ usage() { } panic() { - rc="${1}" - shift 1 msg="${@}" printf "${msg}\n" if [ ! -z "${mddev}" ]; then mdconfig -d -u ${mddev} fi + case ${cmd} in + vm-base) + # If the vm-base target fails, the vm-image target + # cannot possibly succeed. Touch the .TARGET file + # so it is not attempted. + touch vm-image + ;; + *) + # FALLTHROUGH + ;; + esac # Do not allow one failure case to chain through any remaining image # builds. - exit 0 + return 1 } vm_create_baseimage() { @@ -96,7 +105,7 @@ vm_create_baseimage() { cd ${WORLDDIR} && \ make DESTDIR=${DESTDIR} \ installworld installkernel distribution || \ - panic 1 "\n\nCannot install the base system to ${DESTDIR}." + panic "\n\nCannot install the base system to ${DESTDIR}." chroot ${DESTDIR} /usr/bin/newaliases echo '# Custom /etc/fstab for FreeBSD VM images' \ > ${DESTDIR}/etc/fstab @@ -111,7 +120,7 @@ vm_create_baseimage() { # This should never happen. But, it has happened. msg="Cannot umount(8) ${DESTDIR}\n" msg="${msg}Something has gone horribly wrong." - panic 1 "${msg}" + panic "${msg}" fi sleep 1 done @@ -141,11 +150,11 @@ vm_create_vmdisk() { if [ -z "${mkimg_version}" ]; then msg="Cannot determine mkimg(1) version.\n" msg="${msg}Cannot continue without a known mkimg(1) version." - panic 0 "${msg}" + panic "${msg}" fi if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then - panic 0 "'${FORMAT}' is not supported by this mkimg(1).\n" + panic "'${FORMAT}' is not supported by this mkimg(1).\n" fi case ${FORMAT} in diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh new file mode 100755 index 000000000..45b70ae73 --- /dev/null +++ b/release/i386/mk-azure.sh @@ -0,0 +1,173 @@ +#!/bin/sh +#- +# Copyright (c) 2014 The FreeBSD Foundation +# All rights reserved. +# +# This software was developed by Glen Barber under sponsorship +# from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# mk-azure.sh: Create virtual machine disk images for Microsoft Azure +# +# $FreeBSD$ +# + +export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin" + +usage() { + echo "Usage:" + echo -n "$(basename ${0}) vm-azure " + echo " " + exit 1 +} + +panic() { + msg="${@}" + printf "${msg}\n" + if [ ! -z "${mddev}" ]; then + mdconfig -d -u ${mddev} + fi + # Do not allow one failure case to chain through any remaining image + # builds. + exit 0 +} + +vm_create_azure() { + # Arguments: + # vm-azure + + VMBASE="${1}" + WORLDDIR="${2}" + DESTDIR="${3}" + VMSIZE="${4}" + VMIMAGE="${5}" + + if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \ + -o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then + usage + fi + + trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM + + i=0 + mkdir -p ${DESTDIR} + truncate -s ${VMSIZE} ${VMBASE} + mddev=$(mdconfig -f ${VMBASE}) + newfs -j /dev/${mddev} + mkdir -p ${DESTDIR} + mount /dev/${mddev} ${DESTDIR} + make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \ + installworld installkernel distribution || \ + panic 1 "\n\nCannot install the base system to ${DESTDIR}." + mount -t devfs devfs ${DESTDIR}/dev + chroot ${DESTDIR} /usr/bin/newaliases + echo '# Custom /etc/fstab for FreeBSD VM images' \ + > ${DESTDIR}/etc/fstab + echo '/dev/gpt/rootfs / ufs rw 2 2' \ + >> ${DESTDIR}/etc/fstab + # Although a swap partition is created, it is not used in Azure. + echo '#/dev/gpt/swapfs none swap sw 0 0' \ + >> ${DESTDIR}/etc/fstab + + chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \ + python python2 python27 py27-asn1 sudo bash + if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \ + ${VM_EXTRA_PACKAGES} + fi + + fetch -o ${DESTDIR}/usr/sbin/waagent \ + http://people.freebsd.org/~gjb/waagent + chmod +x ${DESTDIR}/usr/sbin/waagent + rm -f ${DESTDIR}/etc/resolv.conf + chroot ${DESTDIR} /usr/sbin/waagent -verbose -install + yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision + echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf + echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf + echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf + + echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf + echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf + + if [ ! -z "${VM_RC_LIST}" ]; then + for _rcvar in ${VM_RC_LIST}; do + echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf + done + fi + + sync + + while ! umount ${DESTDIR}/dev ${DESTDIR}; do + i=$(( $i + 1 )) + if [ $i -ge 10 ]; then + # This should never happen. But, it has happened. + msg="Cannot umount(8) ${DESTDIR}\n" + msg="${msg}Something has gone horribly wrong." + panic 1 "${msg}" + fi + sleep 1 + done + + echo "Creating image... Please wait." + + mkimg -f vhdf -s gpt \ + -b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \ + -p freebsd-swap/swapfs::1G \ + -p freebsd-ufs/rootfs:=${VMBASE} \ + -o ${VMIMAGE}.raw + + if [ ! -x "/usr/local/bin/qemu-img" ]; then + env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel + fi + + size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',') + size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) )) + qemu-img resize ${VMIMAGE}.raw ${size} + qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE} + + return 0 +} + +main() { + cmd="${1}" + shift 1 + + if [ -e "${AZURECONF}" -a ! -c "${AZURECONF}" ]; then + . ${AZURECONF} + fi + + case ${cmd} in + vm-azure) + eval vm_create_azure "$@" || return 0 + ;; + *|\?) + usage + ;; + esac + + return 0 +} + +main "$@" diff --git a/release/i386/mk-vmimage.sh b/release/i386/mk-vmimage.sh index 6ee148248..b3ffd232f 100755 --- a/release/i386/mk-vmimage.sh +++ b/release/i386/mk-vmimage.sh @@ -32,7 +32,7 @@ # $FreeBSD$ # -PATH="/bin:/usr/bin:/sbin:/usr/sbin" +PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" export PATH usage_vm_base() { @@ -58,16 +58,25 @@ usage() { } panic() { - rc="${1}" - shift 1 msg="${@}" printf "${msg}\n" if [ ! -z "${mddev}" ]; then mdconfig -d -u ${mddev} fi + case ${cmd} in + vm-base) + # If the vm-base target fails, the vm-image target + # cannot possibly succeed. Touch the .TARGET file + # so it is not attempted. + touch vm-image + ;; + *) + # FALLTHROUGH + ;; + esac # Do not allow one failure case to chain through any remaining image # builds. - exit 0 + return 1 } vm_create_baseimage() { @@ -96,7 +105,7 @@ vm_create_baseimage() { cd ${WORLDDIR} && \ make DESTDIR=${DESTDIR} \ installworld installkernel distribution || \ - panic 1 "\n\nCannot install the base system to ${DESTDIR}." + panic "\n\nCannot install the base system to ${DESTDIR}." chroot ${DESTDIR} /usr/bin/newaliases echo '# Custom /etc/fstab for FreeBSD VM images' \ > ${DESTDIR}/etc/fstab @@ -111,7 +120,7 @@ vm_create_baseimage() { # This should never happen. But, it has happened. msg="Cannot umount(8) ${DESTDIR}\n" msg="${msg}Something has gone horribly wrong." - panic 1 "${msg}" + panic "${msg}" fi sleep 1 done @@ -141,11 +150,11 @@ vm_create_vmdisk() { if [ -z "${mkimg_version}" ]; then msg="Cannot determine mkimg(1) version.\n" msg="${msg}Cannot continue without a known mkimg(1) version." - panic 0 "${msg}" + panic "${msg}" fi if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then - panic 0 "'${FORMAT}' is not supported by this mkimg(1).\n" + panic "'${FORMAT}' is not supported by this mkimg(1).\n" fi case ${FORMAT} in diff --git a/release/tools/azure.conf b/release/tools/azure.conf new file mode 100644 index 000000000..8f16acc0d --- /dev/null +++ b/release/tools/azure.conf @@ -0,0 +1,14 @@ +#!/bin/sh +# +# $FreeBSD$ +# + +# Set to a list of packages to install. +# Example: +#export VM_EXTRA_PACKAGES="www/apache24" +export VM_EXTRA_PACKAGES= + +# Set to a list of third-party software to enable in rc.conf(5). +# Example: +#export VM_RC_LIST="apache24" +export VM_RC_LIST= -- 2.42.0