From cf25989f45dde8f51f658bb34610f632350b7b1e Mon Sep 17 00:00:00 2001 From: CyberLeo Date: Mon, 9 Jul 2012 02:32:28 -0500 Subject: [PATCH] script/makepkg: import and use kvs shlib for runtime caching --- script/lib/kvs.sh | 72 +++++++++++++++++++++++++++++++++++++++++++++++ script/makepkg | 61 +++++++++++++++++---------------------- 2 files changed, 98 insertions(+), 35 deletions(-) create mode 100644 script/lib/kvs.sh diff --git a/script/lib/kvs.sh b/script/lib/kvs.sh new file mode 100644 index 0000000..2b165ff --- /dev/null +++ b/script/lib/kvs.sh @@ -0,0 +1,72 @@ +# Simple bourne shell based key-value store + +if [ -z "${__kvs_sh_loaded}" ] +then + __kvs_sh_loaded=yes + +t="$(printf '\t')" + +[ "${kvs}" ] || { + cat < "${kvs}" + +# Add or replace a value stored for a given key and id in the kvs +kvs_set() { + [ "${1}" -a "${2}" ] || return 255 + local id="${1}" + local var="${2}" + local val="${3}" + printf "%s\t%s\t%s\n" "${id}" "${var}" "${val}" >> "${kvs}" +} + +# Fetch the most recent value stored for a given key and id from the kvs +kvs_get() { + [ "${1}" -a "${2}" ] || return 255 + local id="${1}" + local var="${2}" + grep "^${id}${t}${var}${t}" "${kvs}" | tail -n 1 | cut -d"${t}" -f3- +} + +# Does the kvs have any entries for a given ID? +kvs_has_id() { + [ "${1}" ] || return 255 + local id="${1}" + grep -q "^${id}${t}" "${kvs}" +} + +# Does the kvs have a given key for a given ID? +kvs_has_key() { + [ "${1}" -a "${2}" ] || return 255 + local id="${1}" + local var="${2}" + grep -q "^${id}${t}${var}${t}" "${kvs}" +} + +# Remove a given key with a given ID from the kvs +kvs_unset() { + [ "${1}" -a "${2}" ] || return 255 + local id="${1}" + local var="${2}" + grep -v "^${id}${t}${var}${t}" "${kvs}" > "${kvs}.tmp" && mv -f "${kvs}.tmp" "${kvs}" || rm -f "${kvs}.tmp" +} + +# Remove all keys with a given ID from the kvs +kvs_unset_all() { + [ "${1}" ] || return 255 + local id="${1}" + grep -v "^${id}${t}" "${kvs}" > "${kvs}.tmp" && mv -f "${kvs}.tmp" "${kvs}" || rm -f "${kvs}.tmp" +} + +fi diff --git a/script/makepkg b/script/makepkg index e8ab9f1..8ee3183 100755 --- a/script/makepkg +++ b/script/makepkg @@ -167,16 +167,15 @@ port_bdeps() { } port_all_bdeps() { # Clear cache if first value isn't '-r' - [ "${1}" = '-r' ] && shift || { : > "${_port_all_bdeps_cache}"; : > "${_port_all_rdeps_cache}"; } + [ "${1}" = '-r' ] && shift || { kvs_unset_all 'port_all_bdeps'; kvs_unset_all 'port_all_rdeps'; } # rdeps for rdeps are rdeps, bdeps for rdeps are bdeps, rdeps for bdeps are bdeps; thus: ( port_bdeps "${@}" | while read port do - [ "${VERBOSE_CACHE}" ] && logf "**** bdep cache %s: '%s'\n" "$(grep -q "${port}" \ - "${_port_all_bdeps_cache}" && echo "hit" || echo "miss")" "${port}" >&2 - if ! grep -q "${port}" "${_port_all_bdeps_cache}" + [ "${VERBOSE_CACHE}" ] && logf "**** bdep cache %s: '%s'\n" "$(kvs_has_key 'port_all_bdeps' "${port}" && echo "hit" || echo "miss")" "${port}" >&2 + if ! kvs_has_key 'port_all_bdeps' "${port}" then - echo "${port}" >> "${_port_all_bdeps_cache}" + kvs_set 'port_all_bdeps' "${port}" "" echo "${port}" port_all_bdeps -r "${port}" port_all_rdeps -r "${port}" @@ -184,11 +183,10 @@ port_all_bdeps() { done port_all_rdeps -r "${@}" | while read port do - [ "${VERBOSE_CACHE}" ] && logf "**** bdep cache %s: '%s'\n" "$(grep -q "${port}" \ - "${_port_all_bdeps_cache}" && echo "hit" || echo "miss")" "${port}" >&2 - if ! grep -q "${port}" "${_port_all_bdeps_cache}" + [ "${VERBOSE_CACHE}" ] && logf "**** bdep cache %s: '%s'\n" "$(kvs_has_key 'port_all_bdeps' "${port}" && echo "hit" || echo "miss")" "${port}" >&2 + if ! kvs_has_key 'port_all_bdeps' "${port}" then - echo "${port}" >> "${_port_all_bdeps_cache}" + kvs_set 'port_all_bdeps' "${port}" "" port_all_bdeps -r "${port}" fi done @@ -208,14 +206,13 @@ port_rdeps() { } port_all_rdeps() { # Clear cache if first value isn't '-r' - [ "${1}" = '-r' ] && shift || { : > "${_port_all_rdeps_cache}"; } + [ "${1}" = '-r' ] && shift || kvs_unset_all 'port_all_rdeps' port_rdeps "${@}" | while read port do - [ "${VERBOSE_CACHE}" ] && logf "**** rdep cache %s: '%s'\n" "$(grep -q "${port}" \ - "${_port_all_rdeps_cache}" && echo "hit" || echo "miss")" "${port}" >&2 - if ! grep -q "${port}" "${_port_all_rdeps_cache}" + [ "${VERBOSE_CACHE}" ] && logf "**** rdep cache %s: '%s'\n" "$(kvs_has_key 'port_all_rdeps' "${port}" && echo "hit" || echo "miss")" "${port}" >&2 + if ! kvs_has_key 'port_all_rdeps' "${port}" then - echo "${port}" >> "${_port_all_rdeps_cache}" + kvs_set 'port_all_rdeps' "${port}" echo "${port}" port_all_rdeps -r "${port}" fi @@ -264,22 +261,21 @@ port_bdep_tree() { # Clear cache if first value isn't '-r' [ "${1}" = '-r' ] && shift || { [ ${VERBOSE_CACHE} ] && logf "**** bdep_tree cache cleared\n" - : > "${_port_bdep_tree_cache}" + kvs_unset_all 'port_bdep_tree' } port="${1##/usr/ports/}" printf "%${2}s%s\n" "" "${port}" - [ "${VERBOSE_CACHE}" ] && logf "**** bdep_tree cache %s: '%s'\n" "$(grep -q "${port}" \ - "${_port_bdep_tree_cache}" && echo "hit" || echo "miss")" "${port}" >&2 + [ "${VERBOSE_CACHE}" ] && logf "**** bdep_tree cache %s: '%s'\n" "$(kvs_has_key 'port_bdep_tree' "${port}" && echo "hit" || echo "miss")" "${port}" >&2 - if ! grep -q "${port}" "${_port_bdep_tree_cache}" + if ! kvs_has_key 'port_bdep_tree' "${port}" then ( chmake "${port}" build-depends-list chmake "${port}" run-depends-list ) | sort -u | while read port do port_bdep_tree -r "${port}" $(( ${2:-0} + 1 )) - echo "${port}" >> "${_port_bdep_tree_cache}" + kvs_set 'port_bdep_tree' "${port}" done else printf "%${2}s %s\n" "" "..." @@ -295,20 +291,19 @@ port_rdep_tree() { # Clear cache if first value isn't '-r' [ "${1}" = '-r' ] && shift || { [ ${VERBOSE_CACHE} ] && logf "**** rdep_tree cache cleared\n" - : > "${_port_rdep_tree_cache}" + kvs_unset_all 'port_rdep_tree' } port="${1##/usr/ports/}" printf "%${2}s%s\n" "" "${port}" - [ "${VERBOSE_CACHE}" ] && logf "**** rdep_tree cache %s: '%s'\n" "$(grep -q "${port}" \ - "${_port_rdep_tree_cache}" && echo "hit" || echo "miss")" "${port}" >&2 + [ "${VERBOSE_CACHE}" ] && logf "**** rdep_tree cache %s: '%s'\n" "$(kvs_has_key 'port_rdep_tree' "${port}" && echo "hit" || echo "miss")" "${port}" >&2 - if ! grep -q "${port}" "${_port_rdep_tree_cache}" + if ! kvs_has_key 'port_rdep_tree' "${port}" then chmake "${port}" run-depends-list | sort -u | while read port do port_rdep_tree -r "${port}" $(( ${2:-0} + 1 )) - echo "${port}" >> "${_port_rdep_tree_cache}" + kvs_set 'port_rdep_tree' "${port}" done else printf "%${2}s %s\n" "" "..." @@ -334,7 +329,7 @@ port_config() { # Make config-conditional for a list of ports, and all dependencies port_config_recursive() { # Clear cache if first value isn't '-r' - [ "${1}" = '-r' ] && shift || _port_config_recursive_cache="" + [ "${1}" = '-r' ] && shift || kvs_unset_all 'port_config_recursive' while [ "${1}" ] do # Do not use config-recursive because it computes the depchain first; @@ -342,18 +337,16 @@ port_config_recursive() { # proper depchain is followed. Use config-conditional to avoid dialog # if the config is already complete. Also use a cache to avoid re-config # and re-recurse on previously handled port branches. - if echo "${_port_config_recursive_cache}" | grep -qv " ${1} " + if ! kvs_has_key 'port_config_recursive' "${1}" then meh "port config-recursive ${1}" chmake "${1}" config-conditional < /dev/tty port_config_recursive -r $(port_deps "${1}") - _port_config_recursive_cache="${_port_config_recursive_cache} ${1} " + kvs_set 'port_config_recursive' "${1}" fi shift done } -# Config cache -unset _port_config_recursive_cache # Remove saved config for a list of ports port_rmconfig() { @@ -619,12 +612,6 @@ bdeps_dir="${pkg_dir}/bdeps" chroot_pkg_dir="${pkg_dir##${chroot_dir}}" chroot_bdeps_dir="${bdeps_dir##${chroot_dir}}" -# Cache files to speed up recursive bdep/rdep scanning -_port_all_bdeps_cache="${chroot_dir}/tmp/_port_all_bdeps_cache" -_port_all_rdeps_cache="${chroot_dir}/tmp/_port_all_rdeps_cache" -_port_bdep_tree_cache="${chroot_dir}/tmp/_port_bdep_tree_cache" -_port_rdep_tree_cache="${chroot_dir}/tmp/_port_rdep_tree_cache" - # Chroot environment chroot_env=" USER=root @@ -635,6 +622,10 @@ SHELL=/bin/sh TERM=${TERM} " +# Configure and load kvs +kvs="${chroot_dir}/.makepkg.kvs" +want kvs + help() { cat <