4 # SPDX-License-Identifier: BSD-2-Clause-FreeBSD
6 # Copyright 2004-2005 Colin Percival
9 # Redistribution and use in source and binary forms, with or without
10 # modification, are permitted providing that the following conditions
12 # 1. Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # 2. Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in the
16 # documentation and/or other materials provided with the distribution.
18 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
26 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
27 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28 # POSSIBILITY OF SUCH DAMAGE.
32 #### Usage function -- called from command-line handling code.
34 # Usage instructions. Options not listed:
35 # --debug -- don't filter output from utilities
36 # --no-stats -- don't show progress statistics while fetching files
39 usage: `basename $0` [options] command ... [path]
42 -d workdir -- Store working files in workdir
43 (default: /var/db/portsnap/)
44 -f conffile -- Read configuration options from conffile
45 (default: /etc/portsnap.conf)
46 -I -- Update INDEX only. (update command only)
47 -k KEY -- Trust an RSA key with SHA256 hash of KEY
48 -l descfile -- Merge the specified local describes file into the INDEX.
49 -p portsdir -- Location of uncompressed ports tree
50 (default: /usr/ports/)
51 -s server -- Server from which to fetch updates.
52 (default: portsnap.FreeBSD.org)
53 --interactive -- interactive: override auto-detection of calling process
54 (use this when calling portsnap from an interactive, non-
55 terminal application AND NEVER ELSE).
56 path -- Extract only parts of the tree starting with the given
57 string. (extract command only)
59 fetch -- Fetch a compressed snapshot of the ports tree,
60 or update an existing snapshot.
61 cron -- Sleep rand(3600) seconds, and then fetch updates.
62 extract -- Extract snapshot of ports tree, replacing existing
63 files and directories.
64 update -- Update ports tree to match current snapshot, replacing
65 files and directories which have changed.
66 auto -- Fetch updates, and either extract a new ports tree or
67 update an existing tree.
72 #### Parameter handling functions.
74 # Initialize parameters to null, just in case they're
75 # set in the environment.
97 # Parse the command line
99 while [ $# -gt 0 ]; do
102 if [ $# -eq 1 ]; then usage; fi
103 if [ ! -z "${WORKDIR}" ]; then usage; fi
107 QUIETREDIR="/dev/stderr"
108 STATSREDIR="/dev/stderr"
118 if [ $# -eq 1 ]; then usage; fi
119 if [ ! -z "${CONFFILE}" ]; then usage; fi
129 if [ $# -eq 1 ]; then usage; fi
130 if [ ! -z "${KEYPRINT}" ]; then usage; fi
134 if [ $# -eq 1 ]; then usage; fi
135 if [ ! -z "${LOCALDESC}" ]; then usage; fi
136 shift; LOCALDESC="$1"
139 if [ -z "${STATSREDIR}" ]; then
140 STATSREDIR="/dev/null"
145 if [ $# -eq 1 ]; then usage; fi
146 if [ ! -z "${PORTSDIR}" ]; then usage; fi
150 if [ $# -eq 1 ]; then usage; fi
151 if [ ! -z "${SERVERNAME}" ]; then usage; fi
152 shift; SERVERNAME="$1"
154 cron | extract | fetch | update | auto)
155 COMMANDS="${COMMANDS} $1"
158 COMMANDS="${COMMANDS} update"
161 COMMANDS="${COMMANDS} auto"
164 if [ $# -gt 1 ]; then usage; fi
165 if echo ${COMMANDS} | grep -vq extract; then
174 if [ -z "${COMMANDS}" ]; then
179 # If CONFFILE was specified at the command-line, make
180 # sure that it exists and is readable.
182 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
183 echo -n "File does not exist "
184 echo -n "or is not readable: "
190 # If a configuration file hasn't been specified, use
191 # the default value (/etc/portsnap.conf)
193 if [ -z "${CONFFILE}" ]; then
194 CONFFILE="/etc/portsnap.conf"
198 # Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration
199 # file if they haven't already been set. If the configuration
200 # file doesn't exist, do nothing.
201 # Also read REFUSE (which cannot be set via the command line) if it is
202 # present in the configuration file.
204 if [ -r "${CONFFILE}" ]; then
205 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do
207 if [ -z "${_}" ]; then
208 eval ${X}=`grep "^${X}=" "${CONFFILE}" |
209 cut -f 2- -d '=' | tail -1`
213 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then
215 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" |
216 cut -c 7- | xargs echo | tr ' ' '|'
220 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then
222 grep -E "^INDEX[[:space:]]" "${CONFFILE}" |
223 cut -c 7- | tr ' ' '|' | xargs echo`"
228 # If parameters have not been set, use default values
230 _QUIETREDIR="/dev/null"
232 _STATSREDIR="/dev/stdout"
233 _WORKDIR="/var/db/portsnap"
234 _PORTSDIR="/usr/ports"
236 _LOCALDESC="/dev/null"
237 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \
241 if [ -z "${_}" ]; then
245 if [ -z "${INTERACTIVE}" ]; then
254 # Perform sanity checks and set some final parameters
255 # in preparation for fetching files. Also chdir into
256 # the working directory.
257 fetch_check_params() {
258 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)"
261 "SERVERNAME must be given via command line or configuration file."
262 _KEYPRINT_z="Key must be given via -k option or configuration file."
263 _KEYPRINT_bad="Invalid key fingerprint: "
264 _WORKDIR_bad="Directory does not exist or is not writable: "
266 if [ -z "${SERVERNAME}" ]; then
267 echo -n "`basename $0`: "
268 echo "${_SERVERNAME_z}"
271 if [ -z "${KEYPRINT}" ]; then
272 echo -n "`basename $0`: "
273 echo "${_KEYPRINT_z}"
276 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
277 echo -n "`basename $0`: "
278 echo -n "${_KEYPRINT_bad}"
282 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
283 echo -n "`basename $0`: "
284 echo -n "${_WORKDIR_bad}"
288 cd ${WORKDIR} || exit 1
290 BSPATCH=/usr/bin/bspatch
292 PHTTPGET=/usr/libexec/phttpget
295 # Perform sanity checks and set some final parameters
296 # in preparation for extracting or updating ${PORTSDIR}
297 # Complain if ${PORTSDIR} exists but is not writable,
298 # but don't complain if ${PORTSDIR} doesn't exist.
299 extract_check_params() {
300 _WORKDIR_bad="Directory does not exist: "
301 _PORTSDIR_bad="Directory is not writable: "
303 if ! [ -d "${WORKDIR}" ]; then
304 echo -n "`basename $0`: "
305 echo -n "${_WORKDIR_bad}"
309 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then
310 echo -n "`basename $0`: "
311 echo -n "${_PORTSDIR_bad}"
316 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \
317 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then
318 echo "No snapshot available. Try running"
319 echo "# `basename $0` fetch"
323 MKINDEX=/usr/libexec/make_index
326 # Perform sanity checks and set some final parameters
327 # in preparation for updating ${PORTSDIR}
328 update_check_params() {
331 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then
332 echo "${PORTSDIR} was not created by portsnap."
333 echo -n "You must run '`basename $0` extract' before "
334 echo "running '`basename $0` update'."
340 #### Core functionality -- the actual work gets done here
342 # Use an SRV query to pick a server. If the SRV query doesn't provide
343 # a useful answer, use the server name specified by the user.
344 # Put another way... look up _http._tcp.${SERVERNAME} and pick a server
345 # from that; or if no servers are returned, use ${SERVERNAME}.
346 # This allows a user to specify "portsnap.freebsd.org" (in which case
347 # portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org"
348 # (in which case portsnap will use that particular server, since there
349 # won't be an SRV entry for that name).
351 # We ignore the Port field, since we are always going to use port 80.
353 # Fetch the mirror list, but do not pick a mirror yet. Returns 1 if
354 # no mirrors are available for any reason.
355 fetch_pick_server_init() {
358 # Check that host(1) exists (i.e., that the system wasn't built with the
359 # WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
360 if ! which -s host; then
365 echo -n "Looking up ${SERVERNAME} mirrors... "
367 # Issue the SRV query and pull out the Priority, Weight, and Target fields.
368 # BIND 9 prints "$name has SRV record ..." while BIND 8 prints
369 # "$name server selection ..."; we allow either format.
370 MLIST="_http._tcp.${SERVERNAME}"
371 host -t srv "${MLIST}" |
372 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
373 cut -f 1,2,4 -d ' ' |
375 sort > serverlist_full
377 # If no records, give up -- we'll just use the server name we were given.
378 if [ `wc -l < serverlist_full` -eq 0 ]; then
383 # Report how many mirrors we found.
384 echo `wc -l < serverlist_full` "mirrors found."
386 # Generate a random seed for use in picking mirrors. If HTTP_PROXY
387 # is set, this will be used to generate the seed; otherwise, the seed
389 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
390 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
394 RANDVALUE=`jot -r 1 0 999999999`
398 # Pick a mirror. Returns 1 if we have run out of mirrors to try.
399 fetch_pick_server() {
400 # Generate a list of not-yet-tried mirrors
401 sort serverlist_tried |
402 comm -23 serverlist_full - > serverlist
404 # Have we run out of mirrors?
405 if [ `wc -l < serverlist` -eq 0 ]; then
406 echo "No mirrors remaining, giving up."
410 # Find the highest priority level (lowest numeric value).
411 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
413 # Add up the weights of the response lines at that priority level.
418 SRV_W=`echo $X | cut -f 2 -d ' '`
419 SRV_WSUM=$(($SRV_WSUM + $SRV_W))
424 # If all the weights are 0, pretend that they are all 1 instead.
425 if [ ${SRV_WSUM} -eq 0 ]; then
426 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
432 # Pick a value between 0 and the sum of the weights - 1
433 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
435 # Read through the list of mirrors and set SERVERNAME. Write the line
436 # corresponding to the mirror we selected into serverlist_tried so that
437 # we won't try it again.
441 SRV_W=`echo $X | cut -f 2 -d ' '`
442 SRV_W=$(($SRV_W + $SRV_W_ADD))
443 if [ $SRV_RND -lt $SRV_W ]; then
444 SERVERNAME=`echo $X | cut -f 3 -d ' '`
445 echo "$X" >> serverlist_tried
448 SRV_RND=$(($SRV_RND - $SRV_W))
455 # Check that we have a public key with an appropriate hash, or
456 # fetch the key if it doesn't exist. Returns 1 if the key has
457 # not yet been fetched.
459 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
463 echo -n "Fetching public key from ${SERVERNAME}... "
465 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \
466 2>${QUIETREDIR} || true
467 if ! [ -r pub.ssl ]; then
471 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
472 echo "key has incorrect hash."
479 # Fetch a snapshot tag
481 rm -f snapshot.ssl tag.new
483 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... "
484 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \
485 2>${QUIETREDIR} || true
486 if ! [ -r $1.ssl ]; then
491 openssl rsautl -pubin -inkey pub.ssl -verify \
492 < $1.ssl > tag.new 2>${QUIETREDIR} || true
495 if ! [ `wc -l < tag.new` = 1 ] ||
496 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then
497 echo "invalid snapshot tag."
503 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new`
504 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new`
507 # Sanity-check the date on a snapshot tag
508 fetch_snapshot_tagsanity() {
509 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then
510 echo "Snapshot appears to be more than a year old!"
511 echo "(Is the system clock correct?)"
512 echo "Cowardly refusing to proceed any further."
515 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then
516 echo -n "Snapshot appears to have been created more than "
517 echo "one day into the future!"
518 echo "(Is the system clock correct?)"
519 echo "Cowardly refusing to proceed any further."
524 # Sanity-check the date on a snapshot update tag
525 fetch_update_tagsanity() {
526 fetch_snapshot_tagsanity || return 1
528 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then
529 echo -n "Latest snapshot on server is "
530 echo "older than what we already have!"
531 echo -n "Cowardly refusing to downgrade from "
532 date -r ${OLDSNAPSHOTDATE}
533 echo "to `date -r ${SNAPSHOTDATE}`."
538 # Compare old and new tags; return 1 if update is unnecessary
539 fetch_update_neededp() {
540 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then
541 echo -n "Latest snapshot on server matches "
542 echo "what we already have."
543 echo "No updates needed."
547 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then
548 echo -n "Ports tree hasn't changed since "
549 echo "last snapshot."
550 echo "No updates needed."
558 # Fetch snapshot metadata file
560 rm -f ${SNAPSHOTHASH} tINDEX.new
562 echo ${NDEBUG} "Fetching snapshot metadata... "
563 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \
564 2>${QUIETREDIR} || return
565 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then
566 echo "snapshot metadata corrupt."
569 mv ${SNAPSHOTHASH} tINDEX.new
573 # Warn user about bogus metadata
574 fetch_metadata_freakout() {
576 echo "Portsnap metadata is correctly signed, but contains"
577 echo "at least one line which appears bogus."
578 echo "Cowardly refusing to proceed any further."
581 # Sanity-check a snapshot metadata file
582 fetch_metadata_sanity() {
583 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then
584 fetch_metadata_freakout
587 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then
589 echo "Portsnap metadata appears bogus."
590 echo "Cowardly refusing to proceed any further."
595 # Take a list of ${oldhash}|${newhash} and output a list of needed patches
596 fetch_make_patchlist() {
598 echo "" 1>${QUIETREDIR}
599 grep -vE "^([0-9a-f]{64})\|\1$" |
601 printf "Processing: $X $Y ...\r" 1>${QUIETREDIR}
602 if [ -f "files/${Y}.gz" -o ! -f "files/${X}.gz" ]; then continue; fi
605 echo "" 1>${QUIETREDIR}
608 # Print user-friendly progress statistics
613 if [ $(($LNC % 10)) = 0 ]; then
615 elif [ $(($LNC % 2)) = 0 ]; then
624 if [ $TOTAL -gt 0 ]; then
626 printf "($1/$2) %02.2f%% " `echo "scale=4;$LNC / $TOTAL * 100"|bc`
630 fetch_progress_percent() {
636 if [ $(($LNC % 100)) = 0 ]; then
638 elif [ $(($LNC % 10)) = 0 ]; then
646 # Sanity-check an index file
647 fetch_index_sanity() {
648 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new ||
649 fgrep -q "./" INDEX.new; then
650 fetch_metadata_freakout
655 # Verify a list of files
656 fetch_snapshot_verify() {
658 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then
659 echo "snapshot corrupt."
666 # Fetch a snapshot tarball, extract, and verify.
668 while ! fetch_tag snapshot; do
669 fetch_pick_server || return 1
671 fetch_snapshot_tagsanity || return 1
672 fetch_metadata || return 1
673 fetch_metadata_sanity || return 1
677 # Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will
678 # probably take a while, so the progrees reports that fetch(1) generates
679 # will be useful for keeping the users' attention from drifting.
680 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:"
681 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1
683 echo -n "Extracting snapshot... "
684 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1
685 rm ${SNAPSHOTHASH}.tgz
688 echo -n "Verifying snapshot integrity... "
689 # Verify the metadata files
690 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1
693 gunzip -c < snap/`look INDEX tINDEX.new |
694 cut -f 2 -d '|'`.gz > INDEX.new
695 fetch_index_sanity || return 1
696 # Verify the snapshot contents
697 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1
698 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u |
699 lam -s 'snap/' - -s '.gz' > files.expected
700 find snap -mindepth 1 | sort > files.snap
701 if ! cmp -s files.expected files.snap; then
702 echo "unexpected files in snapshot."
705 rm files.expected files.snap
708 # Move files into their proper locations
709 rm -f tag INDEX tINDEX
719 # Update a compressed snapshot
721 rm -f patchlist diff OLD NEW filelist INDEX.new
723 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag`
724 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag`
726 while ! fetch_tag latest; do
727 fetch_pick_server || return 1
729 fetch_update_tagsanity || return 1
730 fetch_update_neededp || return 0
731 fetch_metadata || return 1
732 fetch_metadata_sanity || return 1
734 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` "
735 echo "to `date -r ${SNAPSHOTDATE}`."
737 # Generate a list of wanted metadata patches
738 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new |
739 fetch_make_patchlist > patchlist
741 # Attempt to fetch metadata patches
742 echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
743 echo ${NDEBUG} "metadata patches.${DDSTATS}"
744 tr '|' '-' < patchlist |
745 lam -s "tp/" - -s ".gz" |
746 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
747 2>${STATSREDIR} | fetch_progress
750 # Attempt to apply metadata patches
751 echo -n "Applying metadata patches... "
752 local oldifs="$IFS" IFS='|'
754 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
755 gunzip -c < ${X}-${Y}.gz > diff
756 gunzip -c < files/${X}.gz > OLD
757 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp
758 grep '^\+' diff | cut -c 2- |
759 sort -k 1,1 -t '|' -m - ptmp > NEW
760 if [ `${SHA256} -q NEW` = ${Y} ]; then
764 rm -f diff OLD NEW ${X}-${Y}.gz ptmp
765 done < patchlist 2>${QUIETREDIR}
769 # Update metadata without patches
770 join -t '|' -v 2 tINDEX tINDEX.new |
771 cut -f 2 -d '|' /dev/stdin patchlist |
773 if [ ! -f "files/${Y}.gz" ]; then
777 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
778 echo ${NDEBUG} "metadata files... "
779 lam -s "f/" - -s ".gz" < filelist |
780 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
784 echo -n "Verifying ${Y}... " 1>${QUIETREDIR}
785 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
786 mv ${Y}.gz files/${Y}.gz
788 echo "metadata is corrupt."
791 echo "ok." 1>${QUIETREDIR}
796 echo -n "Extracting index... " 1>${QUIETREDIR}
797 gunzip -c < files/`look INDEX tINDEX.new |
798 cut -f 2 -d '|'`.gz > INDEX.new
799 fetch_index_sanity || return 1
801 # If we have decided to refuse certain updates, construct a hybrid index which
802 # is equal to the old index for parts of the tree which we don't want to
803 # update, and equal to the new index for parts of the tree which gets updates.
804 # This means that we should always have a "complete snapshot" of the ports
805 # tree -- with the caveat that it isn't actually a snapshot.
806 if [ ! -z "${REFUSE}" ]; then
807 echo "Refusing to download updates for ${REFUSE}" \
810 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp
811 grep -E "${REFUSE}" INDEX |
812 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new
816 # Generate a list of wanted ports patches
817 echo -n "Generating list of wanted patches..." 1>${QUIETREDIR}
818 join -t '|' -o 1.2,2.2 INDEX INDEX.new |
819 fetch_make_patchlist > patchlist
820 echo " done." 1>${QUIETREDIR}
822 # Attempt to fetch ports patches
823 patchcnt=`wc -l < patchlist | tr -d ' '`
824 echo -n "Fetching $patchcnt "
825 echo ${NDEBUG} "patches.${DDSTATS}"
827 tr '|' '-' < patchlist | lam -s "bp/" - |
828 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
829 2>${STATSREDIR} | fetch_progress_percent $patchcnt
832 # Attempt to apply ports patches
833 PATCHCNT=`wc -l patchlist`
834 echo "Applying patches... "
835 local oldifs="$IFS" IFS='|'
840 if [ ! -f "${F}" ]; then
841 printf " Skipping ${F} (${I} of ${PATCHCNT}).\r"
844 echo " Processing ${F}..." 1>${QUIETREDIR}
845 gunzip -c < files/${X}.gz > OLD
846 ${BSPATCH} OLD NEW ${X}-${Y}
847 if [ `${SHA256} -q NEW` = ${Y} ]; then
851 rm -f diff OLD NEW ${X}-${Y}
852 done < patchlist 2>${QUIETREDIR}
856 # Update ports without patches
857 join -t '|' -v 2 INDEX INDEX.new |
858 cut -f 2 -d '|' /dev/stdin patchlist |
860 if [ ! -f "files/${Y}.gz" ]; then
864 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
865 echo ${NDEBUG} "new ports or files... "
866 lam -s "f/" - -s ".gz" < filelist |
867 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
873 printf " Processing ${Y} (${I} of ${PATCHCNT}).\r" 1>${QUIETREDIR}
874 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
875 mv ${Y}.gz files/${Y}.gz
877 echo "snapshot is corrupt."
883 # Remove files which are no longer needed
884 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles
885 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles |
886 lam -s "files/" - -s ".gz" | xargs rm -f
887 rm patchlist filelist oldfiles
897 # Do the actual work involved in "fetch" / "cron".
899 fetch_pick_server_init && fetch_pick_server
901 while ! fetch_key; do
902 fetch_pick_server || return 1
905 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then
906 fetch_snapshot || return 1
908 fetch_update || return 1
911 # Build a ports INDEX file
912 extract_make_index() {
913 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then
914 echo -n "$1 not provided by portsnap server; "
915 echo "$2 not being generated."
917 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX |
918 cut -f 2 -d '|'`.gz" |
920 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2
924 # Create INDEX, INDEX-5, INDEX-6
926 echo -n "Building new INDEX files... "
927 for PAIR in ${INDEXPAIRS}; do
928 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'`
929 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'`
930 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1
935 # Create .portsnap.INDEX; if we are REFUSEing to touch certain directories,
936 # merge the values from any exiting .portsnap.INDEX file.
938 if [ -z "${REFUSE}" ]; then
939 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX
940 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then
941 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \
942 > ${PORTSDIR}/.portsnap.INDEX.tmp
943 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort |
944 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \
945 > ${PORTSDIR}/.portsnap.INDEX
946 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp
948 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \
949 > ${PORTSDIR}/.portsnap.INDEX
953 # Do the actual work involved in "extract"
955 local oldifs="$IFS" IFS='|'
956 mkdir -p ${PORTSDIR} || return 1
959 if ! [ -z "${EXTRACTPATH}" ]; then
960 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX
961 elif ! [ -z "${REFUSE}" ]; then
962 grep -vE "${REFUSE}" ${WORKDIR}/INDEX
965 fi | while read FILE HASH; do
966 echo ${PORTSDIR}/${FILE}
967 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then
968 echo "files/${HASH}.gz not found -- snapshot corrupt."
973 rm -rf ${PORTSDIR}/${FILE%/}
974 mkdir -p ${PORTSDIR}/${FILE}
975 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
976 -C ${PORTSDIR}/${FILE}
979 rm -f ${PORTSDIR}/${FILE}
980 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
981 -C ${PORTSDIR} ${FILE}
987 if [ ! -z "${EXTRACTPATH}" ]; then
997 update_run_extract() {
1001 echo "Extracting new files:"
1003 if ! [ -z "${REFUSE}" ]; then
1004 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort
1006 sort ${WORKDIR}/INDEX
1008 comm -13 ${PORTSDIR}/.portsnap.INDEX - |
1009 while read FILE HASH; do
1010 echo ${PORTSDIR}/${FILE}
1011 if ! [ -s "${WORKDIR}/files/${HASH}.gz" ]; then
1012 echo "files/${HASH}.gz not found -- snapshot corrupt."
1017 mkdir -p ${PORTSDIR}/${FILE}
1018 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
1019 -C ${PORTSDIR}/${FILE}
1022 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
1023 -C ${PORTSDIR} ${FILE}
1031 # Do the actual work involved in "update"
1033 if ! [ -z "${INDEXONLY}" ]; then
1034 extract_indices >/dev/null || return 1
1038 if sort ${WORKDIR}/INDEX |
1039 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
1040 echo "Ports tree is already up to date."
1044 # If we are REFUSEing to touch certain directories, don't remove files
1045 # from those directories (even if they are out of date)
1046 echo -n "Removing old files and directories... "
1047 if ! [ -z "${REFUSE}" ]; then
1048 sort ${WORKDIR}/INDEX |
1049 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
1050 grep -vE "${REFUSE}" |
1051 lam -s "${PORTSDIR}/" - |
1052 sed -e 's|/$||' | xargs rm -rf
1054 sort ${WORKDIR}/INDEX |
1055 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
1056 lam -s "${PORTSDIR}/" - |
1057 sed -e 's|/$||' | xargs rm -rf
1061 update_run_extract || return 1
1066 #### Main functions -- call parameter-handling and core functions
1068 # Using the command line, configuration file, and defaults,
1069 # set all the parameters which are needed later.
1079 # Fetch command. Make sure that we're being called
1080 # interactively, then run fetch_check_params and fetch_run
1082 if [ "${INTERACTIVE}" != "YES" ]; then
1083 echo -n "`basename $0` fetch should not "
1084 echo "be run non-interactively."
1085 echo "Run `basename $0` cron instead"
1092 # Cron command. Make sure the parameters are sensible; wait
1093 # rand(3600) seconds; then fetch updates. While fetching updates,
1094 # send output to a temporary file; only print that file if the
1098 sleep `jot -r 1 0 3600`
1100 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1
1101 if ! fetch_run >> ${TMPFILE}; then
1110 # Extract command. Make sure the parameters are sensible,
1111 # then extract the ports tree (or part thereof).
1113 extract_check_params
1114 extract_run || exit 1
1117 # Update command. Make sure the parameters are sensible,
1118 # then update the ports tree.
1121 update_run || exit 1
1124 # Auto command. Run 'fetch' or 'cron' depending on
1125 # whether stdin is a terminal; then run 'update' or
1126 # 'extract' depending on whether ${PORTSDIR} exists.
1128 if [ "${INTERACTIVE}" = "YES" ]; then
1133 if [ -r ${PORTSDIR}/.portsnap.INDEX ]; then
1142 # Make sure we find utilities from the base system
1143 export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
1145 # Set LC_ALL in order to avoid problems with character ranges like [A-Z].
1149 for COMMAND in ${COMMANDS}; do