usr.sbin/bsdconfig/share/packages/index.subr

   1 if [ ! "$_PACKAGES_INDEX_SUBR" ]; then _PACKAGES_INDEX_SUBR=1
   2 #
   3 # Copyright (c) 2013 Devin Teske
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions
   8 # are met:
   9 # 1. Redistributions of source code must retain the above copyright
  10 #    notice, this list of conditions and the following disclaimer.
  11 # 2. Redistributions in binary form must reproduce the above copyright
  12 #    notice, this list of conditions and the following disclaimer in the
  13 #    documentation and/or other materials provided with the distribution.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25 # SUCH DAMAGE.
  26 #
  27 # $FreeBSD$
  28 #
  29 ############################################################ INCLUDES
  30
  31 BSDCFG_SHARE="/usr/share/bsdconfig"
  32 . $BSDCFG_SHARE/common.subr || exit 1
  33 f_dprintf "%s: loading includes..." packages/index.subr
  34 f_include $BSDCFG_SHARE/device.subr
  35 f_include $BSDCFG_SHARE/media/common.subr
  36 f_include $BSDCFG_SHARE/strings.subr
  37
  38 BSDCFG_LIBE="/usr/libexec/bsdconfig"
  39 f_include_lang $BSDCFG_LIBE/include/messages.subr
  40
  41 ############################################################ GLOBALS
  42
  43 PACKAGE_INDEX=
  44 _INDEX_INITTED=
  45
  46 #
  47 # Default path to pkg(8) repo-packagesite.sqlite database
  48 #
  49 SQLITE_REPO="/var/db/pkg/repo-packagesite.sqlite"
  50
  51 #
  52 # Default path to on-disk cache INDEX file
  53 #
  54 PACKAGES_INDEX_CACHEFILE="/var/run/bsdconfig/packages_INDEX.cache"
  55
  56 #
  57 # INDEX format for FreeBSD-6.0 or higher:
  58 #
  59 #       package|port-origin|install-prefix|comment|port-desc-file|maintainer|
  60 #       categories|build-deps|run-deps|www-site|reserve|reserve|reserve|disc
  61 #
  62 INDEX_FORMAT="%n-%v" # package
  63 INDEX_FORMAT="$INDEX_FORMAT|/usr/ports/%o"      # port-origin
  64 INDEX_FORMAT="$INDEX_FORMAT|%p"                 # install-prefix
  65 INDEX_FORMAT="$INDEX_FORMAT|%c"                 # comment
  66 INDEX_FORMAT="$INDEX_FORMAT|/usr/ports/%o/pkg-descr"    # port-desc-file
  67 INDEX_FORMAT="$INDEX_FORMAT|%m"                 # maintainer
  68 INDEX_FORMAT="$INDEX_FORMAT|@CATEGORIES@"       # place-holder
  69 INDEX_FORMAT="$INDEX_FORMAT|"                   # build-deps
  70 INDEX_FORMAT="$INDEX_FORMAT|@RUNDEPS@"          # place-holder
  71 INDEX_FORMAT="$INDEX_FORMAT|%w"                 # www-site
  72 INDEX_FORMAT="$INDEX_FORMAT|"                   # reserved
  73 INDEX_FORMAT="$INDEX_FORMAT|"                   # reserved
  74 INDEX_FORMAT="$INDEX_FORMAT|"                   # reserved
  75 INDEX_FORMAT="$INDEX_FORMAT|"                   # disc
  76
  77 ############################################################ FUNCTIONS
  78
  79 # f_index_initialize [$var_to_set]
  80 #
  81 # Read and initialize the global index. Returns success unless media cannot be
  82 # initialized for any reason (e.g. user cancels media selection dialog or an
  83 # error occurs). The index is sorted before being loaded into $var_to_set.
  84 #
  85 # NOTE: The index is processed with f_index_read() [below] after being loaded.
  86 #
  87 f_index_initialize()
  88 {
  89         local __funcname=f_index_initialize
  90         local __var_to_set="${2:-PACKAGE_INDEX}"
  91
  92         [ "$_INDEX_INITTED" ] && return $SUCCESS
  93
  94         # Got any media?
  95         f_media_verify || return $FAILURE
  96
  97         # Does it move when you kick it?
  98         f_device_init device_media || return $FAILURE
  99
 100         f_show_info "$msg_attempting_to_update_repository_catalogue"
 101
 102         #
 103         # Generate $PACKAGESITE variable for pkg(8) based on media type
 104         #
 105         local __type __data __site
 106         device_media get type __type
 107         device_media get private __data
 108         case "$__type" in
 109         $DEVICE_TYPE_DIRECTORY)
 110                 __site="file://$__data/packages/$PKG_ABI" ;;
 111         $DEVICE_TYPE_FLOPPY)
 112                 __site="file://${__data:-$MOUNTPOINT}/packages/$PKG_ABI" ;;
 113         $DEVICE_TYPE_FTP)
 114                 f_getvar $VAR_FTP_PATH __site
 115                 __site="$__site/packages/$PKG_ABI" ;;
 116         $DEVICE_TYPE_HTTP)
 117                 f_getvar $VAR_HTTP_PATH __site
 118                 __site="$__site/$PKG_ABI/latest" ;;
 119         $DEVICE_TYPE_HTTP_PROXY)
 120                 f_getvar $VAR_HTTP_PROXY_PATH __site
 121                 __site="$__site/packages/$PKG_ABI" ;;
 122         $DEVICE_TYPE_CDROM)
 123                 __site="file://$MOUNTPOINT/packages/$PKG_ABI"
 124                 export REPOS_DIR="$MOUNTPOINT/packages/repos" ;;
 125         *) # UFS, DISK, CDROM, USB, DOS, NFS, etc.
 126                 __site="file://$MOUNTPOINT/packages/$PKG_ABI"
 127         esac
 128
 129         export PACKAGESITE="$__site"
 130         f_dprintf "PACKAGESITE=[%s]" "$PACKAGESITE"
 131         if ! f_eval_catch $__funcname pkg "pkg update"; then
 132                 f_show_err "$msg_unable_to_update_pkg_from_selected_media"
 133                 f_device_shutdown device_media
 134                 return $FAILURE
 135         fi
 136
 137         #
 138         # Try to get contents from validated on-disk cache
 139         #
 140
 141         #
 142         # Calculate digest used to determine if the on-disk persistant cache
 143         # INDEX (containing this digest on the first line) is valid and can be
 144         # used to quickly populate the environment.
 145         #
 146         local __sqlite_digest
 147         if ! __sqlite_digest=$( md5 < "$SQLITE_REPO" 2> /dev/null ); then
 148                 f_show_err "$msg_no_pkg_database_found"
 149                 f_device_shutdown device_media
 150                 return $FAILURE
 151         fi
 152
 153         #
 154         # Check to see if the persistant cache INDEX file exists
 155         #
 156         if [ -f "$PACKAGES_INDEX_CACHEFILE" ]; then
 157                 #
 158                 # Attempt to populate the environment with the (soon to be)
 159                 # validated on-disk cache. If validation fails, fall-back to
 160                 # generating a fresh cache.
 161                 #
 162                 if eval $__var_to_set='$(
 163                         (       # Get digest as the first word on first line
 164                                 read digest rest_ignored
 165
 166                                 #
 167                                 # If the stored digest matches the calculated-
 168                                 # one populate the environment from the on-disk
 169                                 # cache and provide success exit status.
 170                                 #
 171                                 if [ "$digest" = "$__sqlite_digest" ]; then
 172                                         cat
 173                                         exit $SUCCESS
 174                                 else
 175                                         # Otherwise, return the current value
 176                                         eval echo \"\$__var_to_set\"
 177                                         exit $FAILURE
 178                                 fi
 179                         ) < "$PACKAGES_INDEX_CACHEFILE" 2> /dev/null
 180                 )'; then
 181                         f_show_info \
 182                           "$msg_located_index_now_reading_package_data_from_it"
 183                         if ! f_index_read "$__var_to_set"; then
 184                                 f_show_err \
 185                                         "$msg_io_or_format_error_on_index_file"
 186                                 return $FAILURE
 187                         fi
 188                         _INDEX_INITTED=1
 189                         return $SUCCESS
 190                 fi
 191                 # Otherwise, fall-thru to create a fresh cache from scratch
 192         fi
 193
 194         #
 195         # If we reach this point, we need to generate the data from scratch
 196         #
 197
 198         f_show_info "$msg_getting_package_categories_via_pkg_rquery"
 199         if ! eval "$( pkg rquery "%n-%v %C" | awk '
 200         { categories[$1] = categories[$1] " " $2 }
 201         END {
 202                 for (package in categories)
 203                 {
 204                         cats = categories[package]
 205                         sub(/^ /, "", cats)
 206                         gsub(/[^[:alnum:]_]/, "_", package)
 207                         printf "local _%s_categories=\"%s\";\n", package, cats
 208                 }
 209         }' )"; then
 210                 f_show_err "$msg_unable_to_pkg_rquery_package_dependencies"
 211                 f_device_shutdown device_media
 212                 return $FAILURE
 213         fi
 214
 215         f_show_info "$msg_getting_package_dependencies_via_pkg_rquery"
 216         if ! eval "$( pkg rquery "%n-%v %dn-%dv" | awk '
 217         { rundeps[$1] = rundeps[$1] " " $2 }
 218         END {
 219                 for (package in rundeps)
 220                 {
 221                         deps = rundeps[package]
 222                         sub(/^ /, "", deps)
 223                         gsub(/[^[:alnum:]_]/, "_", package)
 224                         printf "local _%s_rundeps=\"%s\";\n", package, deps
 225                 }
 226         }' )"; then
 227                 f_show_err "$msg_unable_to_pkg_rquery_package_dependencies"
 228                 f_device_shutdown device_media
 229                 return $FAILURE
 230         fi
 231
 232         f_show_info "$msg_generating_index_from_pkg_database"
 233         eval "$__var_to_set"='$( pkg rquery "$INDEX_FORMAT" |
 234                 while read LINE; do
 235                         package="${LINE%%|*}";
 236                         f_str2varname "$package" varpkg;
 237                         eval f_replaceall \"\$LINE\" \"\|@CATEGORIES@\|\" \
 238                                 \"\|\$_${varpkg}_categories\|\" LINE
 239                         eval f_replaceall \"\$LINE\" \"\|@RUNDEPS@\|\" \
 240                                 \"\|\$_${varpkg}_rundeps\|\" LINE
 241                         echo "$LINE"
 242                 done
 243         )' # always returns true (status of last item in pipe-chain)
 244         eval "$__var_to_set"='$( debug= f_getvar "$__var_to_set" | sort )'
 245
 246         #
 247         # Attempt to create the persistant on-disk cache
 248         #
 249
 250         # Create a new temporary file to write to
 251         local __tmpfile
 252         if f_eval_catch -dk __tmpfile $__funcname mktemp \
 253                 'mktemp -t "%s"' "$pgm"
 254         then
 255                 # Write the temporary file contents
 256                 echo "$__sqlite_digest" > "$__tmpfile"
 257                 debug= f_getvar "$__var_to_set" >> "$__tmpfile"
 258
 259                 # Finally, move the temporary file into place
 260                 case "$PACKAGES_INDEX_CACHEFILE" in
 261                 */*) f_eval_catch -d $funcname mkdir \
 262                         'mkdir -p "%s"' "${PACKAGES_INDEX_CACHEFILE%/*}"
 263                 esac
 264                 f_eval_catch -d $__funcname mv 'mv -f "%s" "%s"' \
 265                         "$__tmpfile" "$PACKAGES_INDEX_CACHEFILE"
 266         fi
 267
 268         f_show_info "$msg_located_index_now_reading_package_data_from_it"
 269         if ! f_index_read "$__var_to_set"; then
 270                 f_show_err "$msg_io_or_format_error_on_index_file"
 271                 return $FAILURE
 272         fi
 273
 274         _INDEX_INITTED=1
 275         return $SUCCESS
 276 }
 277
 278 # f_index_read [$var_to_get]
 279 #
 280 # Process the INDEX file (contents contained in $var_to_get) and...
 281 #
 282 # 1. create a list ($CATEGORY_MENU_LIST) of categories with package counts
 283 # 2. For convenience, create $_npkgs holding the total number of all packages
 284 # 3. extract associative categories for each package into $_categories_$varpkg
 285 # 4. extract runtime dependencies for each package into $_rundeps_$varpkg
 286 # 5. extract a [sorted] list of categories into $PACKAGE_CATEGORIES
 287 # 6. create $_npkgs_$varcat holding the total number of packages in category
 288 #
 289 # NOTE: $varpkg is the product of f_str2varname $package varpkg
 290 # NOTE: $package is the name as it appears in the INDEX (no archive suffix)
 291 # NOTE: We only show categories for which there are at least one package.
 292 # NOTE: $varcat is the product of f_str2varname $category varcat
 293 #
 294 f_index_read()
 295 {
 296         local var_to_get="${1:-PACKAGE_INDEX}"
 297
 298         # Export variables required by awk(1) below
 299         export msg_no_description_provided
 300         export msg_all msg_all_desc
 301         export VALID_VARNAME_CHARS
 302         export msg_packages
 303
 304         eval "$( debug= f_getvar "$var_to_get" | awk -F'|' '
 305         function asorti(src, dest)
 306         {
 307                 # Copy src indices to dest and calculate array length
 308                 nitems = 0; for (i in src) dest[++nitems] = i
 309
 310                 # Sort the array of indices (dest) using insertion sort method
 311                 for (i = 1; i <= nitems; k = i++)
 312                 {
 313                         idx = dest[i]
 314                         while ((k > 0) && (dest[k] > idx))
 315                         {
 316                                 dest[k+1] = dest[k]
 317                                 k--
 318                         }
 319                         dest[k+1] = idx
 320                 }
 321
 322                 return nitems
 323         }
 324         function print_category(category, npkgs, desc)
 325         {
 326                 cat = category
 327                 # Accent the category if the first page has been
 328                 # cached (also acting as a visitation indicator)
 329                 if ( ENVIRON["_index_page_" varcat "_1"] )
 330                         cat = cat "*"
 331                 printf "'\''%s'\'' '\''%s " packages "'\'' '\''%s'\''\n",
 332                        cat, npkgs, desc
 333         }
 334         BEGIN {
 335                 valid_chars = ENVIRON["VALID_VARNAME_CHARS"]
 336                 default_desc = ENVIRON["msg_no_description_provided"]
 337                 packages = ENVIRON["msg_packages"]
 338                 tpkgs = 0
 339                 prefix = ""
 340         }
 341         {
 342                 tpkgs++
 343                 varpkg = $1
 344                 gsub("[^" valid_chars "]", "_", varpkg)
 345                 print "_categories_" varpkg "=\"" $7 "\""
 346                 split($7, pkg_categories, /[[:space:]]+/)
 347                 for (pkg_category in pkg_categories)
 348                         categories[pkg_categories[pkg_category]]++
 349                 print "_rundeps_" varpkg "=\"" $9 "\""
 350         }
 351         END {
 352                 print "_npkgs=" tpkgs # For convenience, total package count
 353
 354                 n = asorti(categories, categories_sorted)
 355
 356                 # Produce package counts for each category
 357                 for (i = 1; i <= n; i++)
 358                 {
 359                         cat = varcat = categories_sorted[i]
 360                         npkgs = categories[cat]
 361                         gsub("[^" valid_chars "]", "_", varcat)
 362                         print "_npkgs_" varcat "=\"" npkgs "\""
 363                 }
 364
 365                 # Create menu list and generate list of categories at same time
 366                 print "CATEGORY_MENU_LIST=\""
 367                 print_category(ENVIRON["msg_all"], tpkgs,
 368                                ENVIRON["msg_all_desc"])
 369                 category_list = ""
 370                 for (i = 1; i <= n; i++)
 371                 {
 372                         cat = varcat = categories_sorted[i]
 373                         npkgs = categories[cat]
 374                         cur_prefix = tolower(substr(cat, 1, 1))
 375                         if ( prefix != cur_prefix )
 376                                 prefix = cur_prefix
 377                         else
 378                                 cat = " " cat
 379                         gsub("[^" valid_chars "]", "_", varcat)
 380                         desc = ENVIRON["_category_" varcat]
 381                         if ( ! desc ) desc = default_desc
 382                         print_category(cat, npkgs, desc)
 383                         category_list = category_list " " cat
 384                 }
 385                 print "\""
 386
 387                 # Produce the list of categories (calculated in above block)
 388                 sub(/^ /, "", category_list)
 389                 print "PACKAGE_CATEGORIES=\"" category_list "\""
 390
 391         }' )" # End-Quote
 392 }
 393
 394 # f_index_extract_pages $var_to_get $var_basename $pagesize [$category]
 395 #
 396 # Extracts the package INDEX ($PACKAGE_INDEX by default if/when $var_to_get is
 397 # NULL; but should not be missing) into a series of sequential variables
 398 # corresponding to "pages" containing up to $pagesize packages. The package
 399 # INDEX data must be contained in the variable $var_to_get. The extracted pages
 400 # are stored in variables ${var_basename}_# -- where "#" is a the page number.
 401 # If $category is set, only packages for that category are extracted.
 402 # Otherwise, if $category is "All", missing, or NULL, all packages are
 403 # extracted and no filtering is done.
 404 #
 405 f_index_extract_pages()
 406 {
 407         local var_to_get="${1:-PACKAGE_INDEX}" var_basename="$2" pagesize="$3"
 408         local category="$4" # Optional
 409
 410         eval "$(
 411                 debug= f_getvar "$var_to_get" | awk -F'|' \
 412                         -v cat="$category" \
 413                         -v pagesize="$pagesize" \
 414                         -v var_basename="$var_basename" \
 415                         -v i18n_all="$msg_all" '
 416                 BEGIN { n = page = 0 }
 417                 /'\''/{ gsub(/'\''/, "'\''\\'\'\''") }
 418                 {
 419                         if ( cat !~ "(^$|^" i18n_all "$)" && $7 !~ \
 420                              "(^|[[:space:]])" cat "([[:space:]]|$)" ) next
 421                         starting_new_page = (n++ == (pagesize * page))
 422                         if ( starting_new_page )
 423                                 printf "%s%s", ( n > 1 ? "'\''\n" : "" ),
 424                                        var_basename "_" ++page "='\''"
 425                         printf "%s%s", ( starting_new_page ? "" : "\n" ), $0
 426                 }
 427                 END { if ( n > 0 ) print "'\''" }'
 428         )"
 429 }
 430
 431 # f_index_search $var_to_get $name [$var_to_set]
 432 #
 433 # Search the package INDEX ($PACKAGE_INDEX by default if/when $var_to_get is
 434 # NULL; but should not be missing) for $name, returning the first match.
 435 # Matches are strict (not regular expressions) and must match the beginning
 436 # portion of the package name to be considered a match. If $var_to_set is
 437 # missing or NULL, output is sent to standard output. If a match is found,
 438 # returns success; otherwise failure.
 439 #
 440 f_index_search()
 441 {
 442         local __var_to_get="${1:-PACKAGE_INDEX}" __pkg_basename="$2"
 443         local __var_to_set="$3"
 444
 445         f_dprintf "f_index_search: Searching package data (in %s) for %s" \
 446                   "$__var_to_get" "$__pkg_basename"
 447
 448         local __pkg=
 449         __pkg=$( debug= f_getvar "$__var_to_get" |
 450                         awk -F'|' -v basename="$__pkg_basename" '
 451                 BEGIN { n = length(basename) }
 452                 substr($1, 0, n) == basename { print $1; exit }
 453         ' )
 454         if [ ! "$__pkg" ]; then
 455                 f_dprintf "f_index_search: No packages matching %s found" \
 456                           "$__pkg_basename"
 457                 return $FAILURE
 458         fi
 459
 460         f_dprintf "f_index_search: Found package %s" "$__pkg"
 461         if [ "$__var_to_set" ]; then
 462                 setvar "$__var_to_set" "$__pkg"
 463         else
 464                 echo "$__pkg"
 465         fi
 466         return $SUCCESS
 467 }
 468
 469 ############################################################ MAIN
 470
 471 f_dprintf "%s: Successfully loaded." packages/index.subr
 472
 473 fi # ! $_PACKAGES_INDEX_SUBR