From 8428e63f3f425b224cd3f6670eb20985163f7680 Mon Sep 17 00:00:00 2001 From: CyberLeo Date: Sun, 13 May 2012 23:26:44 -0500 Subject: [PATCH] Rework to support multiple overlays and require less hardcoding of zpool datasets --- apply | 206 ++++++++++++++++++ .../Mk_bsd.local.mk-strip-perl5-python.patch | 0 .../Mk_bsd.local.mk-strip-perl5-python.patch | 0 .../deprecated/devel_glib20_Makefile.patch | 0 {patch => cdn/patch}/deprecated/refactor | 0 .../deprecated/www_squid_Makefile.patch | 0 {ports => cdn/ports}/Makefile.local | 0 {ports => cdn/ports}/misc-cdn/Makefile | 0 .../ports}/misc-cdn/bash-config/Makefile | 0 .../ports}/misc-cdn/bash-config/distinfo | 0 .../ports}/misc-cdn/bash-config/pkg-descr | 0 .../ports}/misc-cdn/crc32sum/Makefile | 0 .../ports}/misc-cdn/crc32sum/distinfo | 0 .../ports}/misc-cdn/crc32sum/pkg-descr | 0 {ports => cdn/ports}/misc-cdn/flag/Makefile | 0 {ports => cdn/ports}/misc-cdn/flag/distinfo | 0 {ports => cdn/ports}/misc-cdn/flag/pkg-descr | 0 .../ports}/misc-cdn/freebsd-rc/Makefile | 0 .../ports}/misc-cdn/freebsd-rc/distinfo | 0 .../ports}/misc-cdn/freebsd-rc/pkg-descr | 0 {ports => cdn/ports}/misc-cdn/shlib/Makefile | 0 {ports => cdn/ports}/misc-cdn/shlib/distinfo | 0 {ports => cdn/ports}/misc-cdn/shlib/pkg-descr | 0 .../net/rsync/files/patch-server_pause.patch | 0 update | 121 ---------- 25 files changed, 206 insertions(+), 121 deletions(-) create mode 100755 apply rename {patch => cdn/patch}/Mk_bsd.local.mk-strip-perl5-python.patch (100%) rename {patch => cdn/patch}/deprecated/Mk_bsd.local.mk-strip-perl5-python.patch (100%) rename {patch => cdn/patch}/deprecated/devel_glib20_Makefile.patch (100%) rename {patch => cdn/patch}/deprecated/refactor (100%) rename {patch => cdn/patch}/deprecated/www_squid_Makefile.patch (100%) rename {ports => cdn/ports}/Makefile.local (100%) rename {ports => cdn/ports}/misc-cdn/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/bash-config/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/bash-config/distinfo (100%) rename {ports => cdn/ports}/misc-cdn/bash-config/pkg-descr (100%) rename {ports => cdn/ports}/misc-cdn/crc32sum/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/crc32sum/distinfo (100%) rename {ports => cdn/ports}/misc-cdn/crc32sum/pkg-descr (100%) rename {ports => cdn/ports}/misc-cdn/flag/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/flag/distinfo (100%) rename {ports => cdn/ports}/misc-cdn/flag/pkg-descr (100%) rename {ports => cdn/ports}/misc-cdn/freebsd-rc/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/freebsd-rc/distinfo (100%) rename {ports => cdn/ports}/misc-cdn/freebsd-rc/pkg-descr (100%) rename {ports => cdn/ports}/misc-cdn/shlib/Makefile (100%) rename {ports => cdn/ports}/misc-cdn/shlib/distinfo (100%) rename {ports => cdn/ports}/misc-cdn/shlib/pkg-descr (100%) rename {ports => cdn/ports}/net/rsync/files/patch-server_pause.patch (100%) delete mode 100755 update diff --git a/apply b/apply new file mode 100755 index 0000000..4634057 --- /dev/null +++ b/apply @@ -0,0 +1,206 @@ +#!/bin/sh -e + +# Run as root +[ "$(id -u)" -eq 0 ] || exec sudo "${0}" "${@}" +cd "$(dirname "${0}")" + +meh() { printf " \033[1;32m*\033[0m %s\n" "${*}"; } +omg() { printf " \033[1;33m*\033[0m %s\n" "${*}"; } +wtf() { printf " \033[1;31m*\033[0m %s\n" "${*}"; kill $$; exit 1; } + +# Return the zpool name for a given dataset +zfs_ds_pool() { + [ "${1}" ] || return 1 + echo "${1%%/*}" +} + +# Make sure the datasets have the same pool +zfs_is_same_pool() { + [ "${1}" -a "${2}" ] || return 1 + [ "$(zfs_ds_pool "${1}")" = "$(zfs_ds_pool "${2}")" ] +} + +# Translate a mountpoint into its containing dataset name +zfs_mount_to_ds() { + [ "${1}" ] || return 1 + zfs list -H -o name "${1}" +} + +# Returns the mountpoint for a given dataset +zfs_ds_mount() { + [ "${1}" ] || return 1 + zfs get -H -o value mountpoint "${1}" +} + +# Make sure the passed directory corresponds exactly +# with its containing dataset's mountpoint +zfs_is_mountpoint() { + [ "${1}" ] || return 1 + ds="$(zfs_mount_to_ds "${1}")" + [ "${ds}" ] || return 1 + mp="$(zfs_ds_mount "${ds}")" + [ "${mp}" = "${1}" ] +} + +# Fetch the origin snapshot for a given dataset +zfs_ds_origin() { + [ "${1}" ] || return 1 + zfs get -H -o value origin "${1}" +} + +# Fetch the origin dataset for a given dataset +zfs_ds_origin_ds() { + [ "${1}" ] || return 1 + sn="$(zfs_ds_origin "${1}")" + echo "${sn%%@*}" +} + +# List all snapshots for a given dataset, sorted by creation time +zfs_ds_snapshots() { + [ "${1}" ] || return 1 + zfs list -rHt snapshot -d 1 -o name -s creation "${1}" | sed -e 's/^[^@]*@//' +} + +# Get creation time in seconds since epoch for given dataset +zfs_ds_creation() { + [ "${1}" ] || return 1 + zfs get -H -p -o value creation "${1}" +} + +zfs_tests() { + set +e -x + zfs_is_same_pool "mtumishi/srv/ports/upstream/ports" "mtumishi/devel/ports"; echo $? + + zfs_mount_to_ds "/usr/ports" + zfs_mount_to_ds "/usr/ports/misc" + + zfs_ds_mount "mtumishi/devel/ports" + + zfs_is_mountpoint "/usr/ports"; echo $? + zfs_is_mountpoint "/usr/ports/misc"; echo $? + + zfs_ds_origin "mtumishi/devel/ports" + zfs_ds_origin_ds "mtumishi/devel/ports" + + zfs_ds_snapshots "mtumishi/srv/ports/upstream/ports" + + zfs_ds_creation "mtumishi/devel/ports" + set +x -e +} + +ports_tree_valid() { + [ "${1}" ] || return 1 + [ -f "${1}/COPYRIGHT" -a -f "${1}/Makefile" -a -d "${1}/Mk" ] +} + +overlays() { + ( cd "${overlay_fs}" + ls -1 | while read ovl + do + [ -d "${ovl}" -a -d "${ovl}/ports" -a -d "${ovl}/patch" ] || continue + echo "${ovl}" + done + ) +} + +#zfs_tests +#exit + +# Location of overlays (Defaults to directory containing this script) +overlay_fs="$(realpath "$(dirname "${0}")")" + +# Target ports tree (Defaults to /usr/ports) +ports_fs="/usr/ports" + +# Upstream ports tree (Defaults to ../upstream/ports) +upstream_fs="$(realpath "${overlay_fs}/../upstream/ports")" + +# Dataset to hold temporary datasets (defaults to dataset holding overlays) +temp_ds="$(zfs_mount_to_ds "${overlay_fs}")" +prep_ds="${temp_ds}/prepare" + +# Compute upstream dataset name from upstream filesystem +zfs_is_mountpoint "${upstream_fs}" || { echo "Upstream filesystem ${upstream_fs} is not a ZFS dataset" >&2; exit 1; } +upstream_ds="$(zfs_mount_to_ds "${upstream_fs}")" +[ "${upstream_ds}" ] || { echo "Unable to get dataset for ${upstream_fs}" >&2; exit 1; } + +# Compute target ports dataset name; synthesize a new one if nonexistent +if zfs_is_mountpoint "${ports_fs}" +then + ports_ds="$(zfs_mount_to_ds "${ports_fs}")" + [ "$(zfs_ds_origin_ds "${ports_ds}")" = "${upstream_ds}" ] || wtf "Ports dataset is not a descendent of upstream dataset" +else + # Compute a temporary dataset name for ports tree + ports_ds="${temp_ds}/liveports" + create_ports=true + omg "Unable to get dataset for ${ports_fs}; ports tree will be placed here instead:" + omg " ${ports_ds}" + omg "Make sure you move it to where you want it afterwards!" +fi + +# Get most recent upstream snapshot; whine if it's more than 24 hours old +upstream_snap="$(zfs_ds_snapshots "${upstream_ds}" | tail -n 1)" +[ "${upstream_snap}" ] || wtf "No snapshots available for ${upstream_ds}" +upstream_snap_age="$(( $(date +%s) - $(zfs_ds_creation "${upstream_ds}@${upstream_snap}") ))" +[ "${upstream_snap_age}" -le 86400 ] || omg "Snapshot is stale! (${upstream_snap_age} seconds)" + +# Make sure the snapshot actually looks like a ports tree +ports_tree_valid "${upstream_fs}/.zfs/snapshot/${upstream_snap}" || wtf "This sure is an odd-looking ports tree... ${upstream_fs}" + +# Make sure nothing is / +[ "${ports_fs}" -a "${ports_fs}" != '/' ] || wtf "ports_fs is / ?!" +[ "${upstream_fs}" -a "${upstream_fs}" != '/' ] || wtf "usptream_fs is / ?!" + +# All tests pass! Now perform the overlay + +meh Clone +zfs clone "${upstream_ds}@${upstream_snap}" "${prep_ds}" || wtf "Clone failed" +zfs set atime=off "${prep_ds}" || wtf "atimes failed" +prep_fs="$(zfs_ds_mount "${prep_ds}")" + +overlays | while read ovl +do + meh "=> Applying ${ovl} <=" + + meh Overlay + ( cd "${overlay_fs}/${ovl}/ports"; find * | cpio -pR root:wheel "${prep_fs}" ) || wtf "overlay failed" + + meh Patch + for patch in "${overlay_fs}/${ovl}/patch"/*.patch + do + meh "... ${patch##*/}" + ( cd "${prep_fs}"; patch -p0 ) < "${patch}" || wtf "patch failed" + done +done + +meh Install +zfs set readonly=on "${prep_ds}" || wtf "readonly failed" + +# Remember and unmount any null-mounts of the filesystem that is about to be replaced +umount_and_generate_mount_lines() { + mount | grep '(nullfs, ' | grep "^${ports_fs} " | while read null_src null_on null_dst null_opt + do + # Figure out if mount is read-only + mount_flags="" + case "${null_opt}" in + *read-only*) mount_flags="-r" + esac + # Craft a mount line + echo "echo '${null_dst}'; /sbin/mount -t nullfs ${mount_flags} '${null_src}' '${null_dst}';" + meh "Unmounting ${null_dst}" >&2 + /sbin/umount "${null_dst}" + done +} +remount_cmd="$(umount_and_generate_mount_lines)" + +# Half of this is conditional on the old ports tree's existence +[ "${create_ports}" ] || zfs set mountpoint=legacy "${ports_ds}" || wtf "failed hiding old ports" +zfs set mountpoint="${ports_fs}" "${prep_ds}" || wtf "failed mounting new ports" +[ "${create_ports}" ] || zfs destroy "${ports_ds}" || wtf "failed destroying old ports" +zfs rename "${prep_ds}" "${ports_ds}" || wtf "failed renaming ports" + +# Remount null-mounts of this filesystem +[ "${remount_cmd}" ] && meh "Remounting filesystems" +eval "${remount_cmd}" + +echo "All done!" diff --git a/patch/Mk_bsd.local.mk-strip-perl5-python.patch b/cdn/patch/Mk_bsd.local.mk-strip-perl5-python.patch similarity index 100% rename from patch/Mk_bsd.local.mk-strip-perl5-python.patch rename to cdn/patch/Mk_bsd.local.mk-strip-perl5-python.patch diff --git a/patch/deprecated/Mk_bsd.local.mk-strip-perl5-python.patch b/cdn/patch/deprecated/Mk_bsd.local.mk-strip-perl5-python.patch similarity index 100% rename from patch/deprecated/Mk_bsd.local.mk-strip-perl5-python.patch rename to cdn/patch/deprecated/Mk_bsd.local.mk-strip-perl5-python.patch diff --git a/patch/deprecated/devel_glib20_Makefile.patch b/cdn/patch/deprecated/devel_glib20_Makefile.patch similarity index 100% rename from patch/deprecated/devel_glib20_Makefile.patch rename to cdn/patch/deprecated/devel_glib20_Makefile.patch diff --git a/patch/deprecated/refactor b/cdn/patch/deprecated/refactor similarity index 100% rename from patch/deprecated/refactor rename to cdn/patch/deprecated/refactor diff --git a/patch/deprecated/www_squid_Makefile.patch b/cdn/patch/deprecated/www_squid_Makefile.patch similarity index 100% rename from patch/deprecated/www_squid_Makefile.patch rename to cdn/patch/deprecated/www_squid_Makefile.patch diff --git a/ports/Makefile.local b/cdn/ports/Makefile.local similarity index 100% rename from ports/Makefile.local rename to cdn/ports/Makefile.local diff --git a/ports/misc-cdn/Makefile b/cdn/ports/misc-cdn/Makefile similarity index 100% rename from ports/misc-cdn/Makefile rename to cdn/ports/misc-cdn/Makefile diff --git a/ports/misc-cdn/bash-config/Makefile b/cdn/ports/misc-cdn/bash-config/Makefile similarity index 100% rename from ports/misc-cdn/bash-config/Makefile rename to cdn/ports/misc-cdn/bash-config/Makefile diff --git a/ports/misc-cdn/bash-config/distinfo b/cdn/ports/misc-cdn/bash-config/distinfo similarity index 100% rename from ports/misc-cdn/bash-config/distinfo rename to cdn/ports/misc-cdn/bash-config/distinfo diff --git a/ports/misc-cdn/bash-config/pkg-descr b/cdn/ports/misc-cdn/bash-config/pkg-descr similarity index 100% rename from ports/misc-cdn/bash-config/pkg-descr rename to cdn/ports/misc-cdn/bash-config/pkg-descr diff --git a/ports/misc-cdn/crc32sum/Makefile b/cdn/ports/misc-cdn/crc32sum/Makefile similarity index 100% rename from ports/misc-cdn/crc32sum/Makefile rename to cdn/ports/misc-cdn/crc32sum/Makefile diff --git a/ports/misc-cdn/crc32sum/distinfo b/cdn/ports/misc-cdn/crc32sum/distinfo similarity index 100% rename from ports/misc-cdn/crc32sum/distinfo rename to cdn/ports/misc-cdn/crc32sum/distinfo diff --git a/ports/misc-cdn/crc32sum/pkg-descr b/cdn/ports/misc-cdn/crc32sum/pkg-descr similarity index 100% rename from ports/misc-cdn/crc32sum/pkg-descr rename to cdn/ports/misc-cdn/crc32sum/pkg-descr diff --git a/ports/misc-cdn/flag/Makefile b/cdn/ports/misc-cdn/flag/Makefile similarity index 100% rename from ports/misc-cdn/flag/Makefile rename to cdn/ports/misc-cdn/flag/Makefile diff --git a/ports/misc-cdn/flag/distinfo b/cdn/ports/misc-cdn/flag/distinfo similarity index 100% rename from ports/misc-cdn/flag/distinfo rename to cdn/ports/misc-cdn/flag/distinfo diff --git a/ports/misc-cdn/flag/pkg-descr b/cdn/ports/misc-cdn/flag/pkg-descr similarity index 100% rename from ports/misc-cdn/flag/pkg-descr rename to cdn/ports/misc-cdn/flag/pkg-descr diff --git a/ports/misc-cdn/freebsd-rc/Makefile b/cdn/ports/misc-cdn/freebsd-rc/Makefile similarity index 100% rename from ports/misc-cdn/freebsd-rc/Makefile rename to cdn/ports/misc-cdn/freebsd-rc/Makefile diff --git a/ports/misc-cdn/freebsd-rc/distinfo b/cdn/ports/misc-cdn/freebsd-rc/distinfo similarity index 100% rename from ports/misc-cdn/freebsd-rc/distinfo rename to cdn/ports/misc-cdn/freebsd-rc/distinfo diff --git a/ports/misc-cdn/freebsd-rc/pkg-descr b/cdn/ports/misc-cdn/freebsd-rc/pkg-descr similarity index 100% rename from ports/misc-cdn/freebsd-rc/pkg-descr rename to cdn/ports/misc-cdn/freebsd-rc/pkg-descr diff --git a/ports/misc-cdn/shlib/Makefile b/cdn/ports/misc-cdn/shlib/Makefile similarity index 100% rename from ports/misc-cdn/shlib/Makefile rename to cdn/ports/misc-cdn/shlib/Makefile diff --git a/ports/misc-cdn/shlib/distinfo b/cdn/ports/misc-cdn/shlib/distinfo similarity index 100% rename from ports/misc-cdn/shlib/distinfo rename to cdn/ports/misc-cdn/shlib/distinfo diff --git a/ports/misc-cdn/shlib/pkg-descr b/cdn/ports/misc-cdn/shlib/pkg-descr similarity index 100% rename from ports/misc-cdn/shlib/pkg-descr rename to cdn/ports/misc-cdn/shlib/pkg-descr diff --git a/ports/net/rsync/files/patch-server_pause.patch b/cdn/ports/net/rsync/files/patch-server_pause.patch similarity index 100% rename from ports/net/rsync/files/patch-server_pause.patch rename to cdn/ports/net/rsync/files/patch-server_pause.patch diff --git a/update b/update deleted file mode 100755 index fd38bc3..0000000 --- a/update +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/sh - -# Run as root -[ "$(id -u)" -eq 0 ] || exec sudo "${0}" "${@}" -cd "$(dirname "${0}")" - -[ "${1}" = "-n" ] && NO_SYNC=true - -meh() { printf " \033[1;32m*\033[0m %s\n" "${*}"; } -omg() { printf " \033[1;33m*\033[0m %s\n" "${*}"; } -wtf() { printf " \033[1;31m*\033[0m %s\n" "${*}"; kill $$; exit 1; } - -sync="rsync://paka/freebsd-ports" - -sync_opts="--archive --compress --delete --hard-links --sparse --stats --verbose" - -zpool="amani" - -ports=${zpool}/ports -upstream=${zpool}/srcs/freebsd/ports/upstream -prepare=${zpool}/srcs/freebsd/ports/prepare - -today="$(date +%Y-%m-%d)" -upstream_fs="$(zfs get -H -o value mountpoint "${upstream}")" - -# Check if target ports tree actually exists; otherwise, assume it's safe to replace -if zfs list "${ports}" >&- 2>&- -then - ports_exists="YES" - ports_origin="$(zfs get -H -o value origin "${ports}")" - ports_origin="${ports_origin%%@*}" - ports_fs="$(zfs get -H -o value mountpoint "${ports}")" - - # Make sure the ports tree is a descendent of the ports upstream - [ "${ports_origin}" = "${upstream}" ] || wtf "Target ${ports} is not a child of ${upstream} - (is ${ports_origin})" -fi -ports_fs="${ports_fs:-/usr/ports}" - -if [ -z "${NO_SYNC}" ] -then - # Compute the next available snapshot - last_snapshot="$(zfs list -rHt snapshot -o name "${upstream}" | grep "@${today}_" | sort | tail -n 1 | sed -e 's/^.*_\([0-9]\{2\}\)$/\1/')" - if [ "${last_snapshot}" ] - then - snapshot="$(printf "${today}_%02u" "$(( ${last_snapshot} + 1 ))")" - echo "${today}_${last_snapshot} -> ${snapshot}" - else - snapshot="${today}_00" - echo "None -> ${snapshot}" - fi -else - # Don't bother creating a new snapshot if the tree will not be synchronized - last_snapshot="$(zfs list -rHt snapshot -o name "${upstream}" | sort | tail -n 1)" - snapshot="${last_snapshot##*@}" - echo "Using existing snapshot ${snapshot}" -fi - -# Bail out if anything is / -[ -z "${ports_fs}" ] && wtf "ports_fs is / ?!" -[ -z "${upstream_fs}" ] && wtf "upstream_fs is / ?!" -[ -z "${ports_fs}" -o -z "${upstream_fs}" ] && wtf "Writing to /? Are you nuts?" - -if [ -z "${NO_SYNC}" ] -then - meh "Update" - rsync ${sync_opts} "${sync}/" "${upstream_fs}/" || wtf "update failed" - - meh "Snapshot" - zfs snapshot "${upstream}@${snapshot}" || wtf "snapshot failed" -fi - -meh "Clone" -zfs clone "${upstream}@${snapshot}" "${prepare}" || wtf "clone failed" -zfs set atime=off "${prepare}" || wtf "atime-off failed" - -# Resolve filesystem mountpoints, now that everything should exist -prepare_fs="$(zfs get -H -o value mountpoint "${prepare}")" - -meh "Overlay" -( cd ports; find * | cpio -pR root:wheel "${prepare_fs}" ) || wtf "overlay failed" - -meh "Patch" -for patch in patch/*.patch -do - meh "... ${patch}" - ( cd "${prepare_fs}"; patch -p0 ) < "${patch}" || wtf "patch failed" -done - -meh "Install" -zfs set readonly=on "${prepare}" || wtf "readonly failed" - -# Remember and unmount any null-mounts of the filesystem that is about to be replaced -umount_and_generate_mount_lines() { - mount | grep '(nullfs, ' | grep "^/usr/ports " | while read null_src null_on null_dst null_opt - do - # Figure out if mount is read-only - mount_flags="" - case "${null_opt}" in - *read-only*) mount_flags="-r" - esac - # Craft a mount line - echo "echo '${null_dst}'; /sbin/mount -t nullfs ${mount_flags} '${null_src}' '${null_dst}';" - meh "Unmounting ${null_dst}" >&2 - /sbin/umount "${null_dst}" - done -} -remount_cmd="$(umount_and_generate_mount_lines)" - -# Half of this is conditional on the old ports tree's existence -[ -z "${ports_exists}" ] || zfs rename "${ports}" "${ports}-bak" || wtf "backup rename failed" -zfs rename "${prepare}" "${ports}" || wtf "rename prepare -> ports failed" -[ -z "${ports_exists}" ] || zfs inherit mountpoint "${ports}-bak" || wtf "inherit failed" -zfs set mountpoint="${ports_fs}" "${ports}" || wtf "mountpoint failed" -[ -z "${ports_exists}" ] || zfs destroy "${ports}-bak" || wtf "destroy failed" - -# Remount null-mounts of this filesystem -[ "${remount_cmd}" ] && meh "Remounting filesystems" -eval "${remount_cmd}" - -meh "All done" -- 2.42.0