4 # Copyright 2004-2005 Colin Percival
7 # Redistribution and use in source and binary forms, with or without
8 # modification, are permitted providing that the following conditions
10 # 1. Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 # notice, this list of conditions and the following disclaimer in the
14 # documentation and/or other materials provided with the distribution.
16 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 # POSSIBILITY OF SUCH DAMAGE.
30 #### Usage function -- called from command-line handling code.
32 # Usage instructions. Options not listed:
33 # --debug -- don't filter output from utilities
34 # --no-stats -- don't show progress statistics while fetching files
37 usage: `basename $0` [options] command ... [path]
40 -d workdir -- Store working files in workdir
41 (default: /var/db/portsnap/)
42 -f conffile -- Read configuration options from conffile
43 (default: /etc/portsnap.conf)
44 -I -- Update INDEX only. (update command only)
45 -k KEY -- Trust an RSA key with SHA256 hash of KEY
46 -l descfile -- Merge the specified local describes file into the INDEX.
47 -p portsdir -- Location of uncompressed ports tree
48 (default: /usr/ports/)
49 -s server -- Server from which to fetch updates.
50 (default: portsnap.FreeBSD.org)
51 path -- Extract only parts of the tree starting with the given
52 string. (extract command only)
54 fetch -- Fetch a compressed snapshot of the ports tree,
55 or update an existing snapshot.
56 cron -- Sleep rand(3600) seconds, and then fetch updates.
57 extract -- Extract snapshot of ports tree, replacing existing
58 files and directories.
59 update -- Update ports tree to match current snapshot, replacing
60 files and directories which have changed.
65 #### Parameter handling functions.
67 # Initialize parameters to null, just in case they're
68 # set in the environment.
89 # Parse the command line
91 while [ $# -gt 0 ]; do
94 if [ $# -eq 1 ]; then usage; fi
95 if [ ! -z "${WORKDIR}" ]; then usage; fi
99 QUIETREDIR="/dev/stderr"
100 STATSREDIR="/dev/stderr"
107 if [ $# -eq 1 ]; then usage; fi
108 if [ ! -z "${CONFFILE}" ]; then usage; fi
118 if [ $# -eq 1 ]; then usage; fi
119 if [ ! -z "${KEYPRINT}" ]; then usage; fi
123 if [ $# -eq 1 ]; then usage; fi
124 if [ ! -z "${LOCALDESC}" ]; then usage; fi
125 shift; LOCALDESC="$1"
128 if [ -z "${STATSREDIR}" ]; then
129 STATSREDIR="/dev/null"
134 if [ $# -eq 1 ]; then usage; fi
135 if [ ! -z "${PORTSDIR}" ]; then usage; fi
139 if [ $# -eq 1 ]; then usage; fi
140 if [ ! -z "${SERVERNAME}" ]; then usage; fi
141 shift; SERVERNAME="$1"
143 cron | extract | fetch | update | alfred)
144 COMMANDS="${COMMANDS} $1"
147 COMMANDS="${COMMANDS} update"
150 if [ $# -gt 1 ]; then usage; fi
151 if echo ${COMMANDS} | grep -vq extract; then
160 if [ -z "${COMMANDS}" ]; then
165 # If CONFFILE was specified at the command-line, make
166 # sure that it exists and is readable.
168 if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
169 echo -n "File does not exist "
170 echo -n "or is not readable: "
176 # If a configuration file hasn't been specified, use
177 # the default value (/etc/portsnap.conf)
179 if [ -z "${CONFFILE}" ]; then
180 CONFFILE="/etc/portsnap.conf"
184 # Read {KEYPRINT, SERVERNAME, WORKDIR, PORTSDIR} from the configuration
185 # file if they haven't already been set. If the configuration
186 # file doesn't exist, do nothing.
187 # Also read REFUSE (which cannot be set via the command line) if it is
188 # present in the configuration file.
190 if [ -r "${CONFFILE}" ]; then
191 for X in KEYPRINT WORKDIR PORTSDIR SERVERNAME; do
193 if [ -z "${_}" ]; then
194 eval ${X}=`grep "^${X}=" "${CONFFILE}" |
195 cut -f 2- -d '=' | tail -1`
199 if grep -qE "^REFUSE[[:space:]]" ${CONFFILE}; then
201 grep -E "^REFUSE[[:space:]]" "${CONFFILE}" |
202 cut -c 7- | xargs echo | tr ' ' '|'
206 if grep -qE "^INDEX[[:space:]]" ${CONFFILE}; then
208 grep -E "^INDEX[[:space:]]" "${CONFFILE}" |
209 cut -c 7- | tr ' ' '|' | xargs echo`"
214 # If parameters have not been set, use default values
216 _QUIETREDIR="/dev/null"
218 _STATSREDIR="/dev/stdout"
219 _WORKDIR="/var/db/portsnap"
220 _PORTSDIR="/usr/ports"
222 _LOCALDESC="/dev/null"
223 for X in QUIETREDIR QUIETFLAG STATSREDIR WORKDIR PORTSDIR \
227 if [ -z "${_}" ]; then
233 # Perform sanity checks and set some final parameters
234 # in preparation for fetching files. Also chdir into
235 # the working directory.
236 fetch_check_params() {
237 export HTTP_USER_AGENT="portsnap (${COMMAND}, `uname -r`)"
240 "SERVERNAME must be given via command line or configuration file."
241 _KEYPRINT_z="Key must be given via -k option or configuration file."
242 _KEYPRINT_bad="Invalid key fingerprint: "
243 _WORKDIR_bad="Directory does not exist or is not writable: "
245 if [ -z "${SERVERNAME}" ]; then
246 echo -n "`basename $0`: "
247 echo "${_SERVERNAME_z}"
250 if [ -z "${KEYPRINT}" ]; then
251 echo -n "`basename $0`: "
252 echo "${_KEYPRINT_z}"
255 if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
256 echo -n "`basename $0`: "
257 echo -n "${_KEYPRINT_bad}"
261 if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
262 echo -n "`basename $0`: "
263 echo -n "${_WORKDIR_bad}"
267 cd ${WORKDIR} || exit 1
269 BSPATCH=/usr/bin/bspatch
271 PHTTPGET=/usr/libexec/phttpget
274 # Perform sanity checks and set some final parameters
275 # in preparation for extracting or updating ${PORTSDIR}
276 # Complain if ${PORTSDIR} exists but is not writable,
277 # but don't complain if ${PORTSDIR} doesn't exist.
278 extract_check_params() {
279 _WORKDIR_bad="Directory does not exist: "
280 _PORTSDIR_bad="Directory is not writable: "
282 if ! [ -d "${WORKDIR}" ]; then
283 echo -n "`basename $0`: "
284 echo -n "${_WORKDIR_bad}"
288 if [ -d "${PORTSDIR}" ] && ! [ -w "${PORTSDIR}" ]; then
289 echo -n "`basename $0`: "
290 echo -n "${_PORTSDIR_bad}"
295 if ! [ -d "${WORKDIR}/files" -a -r "${WORKDIR}/tag" \
296 -a -r "${WORKDIR}/INDEX" -a -r "${WORKDIR}/tINDEX" ]; then
297 echo "No snapshot available. Try running"
298 echo "# `basename $0` fetch"
302 MKINDEX=/usr/libexec/make_index
305 # Perform sanity checks and set some final parameters
306 # in preparation for updating ${PORTSDIR}
307 update_check_params() {
310 if ! [ -r ${PORTSDIR}/.portsnap.INDEX ]; then
311 echo "${PORTSDIR} was not created by portsnap."
312 echo -n "You must run '`basename $0` extract' before "
313 echo "running '`basename $0` update'."
319 #### Core functionality -- the actual work gets done here
321 # Use an SRV query to pick a server. If the SRV query doesn't provide
322 # a useful answer, use the server name specified by the user.
323 # Put another way... look up _http._tcp.${SERVERNAME} and pick a server
324 # from that; or if no servers are returned, use ${SERVERNAME}.
325 # This allows a user to specify "portsnap.freebsd.org" (in which case
326 # portsnap will select one of the mirrors) or "portsnap5.tld.freebsd.org"
327 # (in which case portsnap will use that particular server, since there
328 # won't be an SRV entry for that name).
330 # We ignore the Port field, since we are always going to use port 80.
332 # Fetch the mirror list, but do not pick a mirror yet. Returns 1 if
333 # no mirrors are available for any reason.
334 fetch_pick_server_init() {
337 # Check that host(1) exists (i.e., that the system wasn't built with the
338 # WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
339 if ! which -s host; then
344 echo -n "Looking up ${SERVERNAME} mirrors... "
346 # Issue the SRV query and pull out the Priority, Weight, and Target fields.
347 # BIND 9 prints "$name has SRV record ..." while BIND 8 prints
348 # "$name server selection ..."; we allow either format.
349 MLIST="_http._tcp.${SERVERNAME}"
350 host -t srv "${MLIST}" |
351 sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
352 cut -f 1,2,4 -d ' ' |
354 sort > serverlist_full
356 # If no records, give up -- we'll just use the server name we were given.
357 if [ `wc -l < serverlist_full` -eq 0 ]; then
362 # Report how many mirrors we found.
363 echo `wc -l < serverlist_full` "mirrors found."
365 # Generate a random seed for use in picking mirrors. If HTTP_PROXY
366 # is set, this will be used to generate the seed; otherwise, the seed
368 if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
369 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
373 RANDVALUE=`jot -r 1 0 999999999`
377 # Pick a mirror. Returns 1 if we have run out of mirrors to try.
378 fetch_pick_server() {
379 # Generate a list of not-yet-tried mirrors
380 sort serverlist_tried |
381 comm -23 serverlist_full - > serverlist
383 # Have we run out of mirrors?
384 if [ `wc -l < serverlist` -eq 0 ]; then
385 echo "No mirrors remaining, giving up."
389 # Find the highest priority level (lowest numeric value).
390 SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
392 # Add up the weights of the response lines at that priority level.
397 SRV_W=`echo $X | cut -f 2 -d ' '`
398 SRV_WSUM=$(($SRV_WSUM + $SRV_W))
403 # If all the weights are 0, pretend that they are all 1 instead.
404 if [ ${SRV_WSUM} -eq 0 ]; then
405 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
411 # Pick a value between 0 and the sum of the weights - 1
412 SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
414 # Read through the list of mirrors and set SERVERNAME. Write the line
415 # corresponding to the mirror we selected into serverlist_tried so that
416 # we won't try it again.
420 SRV_W=`echo $X | cut -f 2 -d ' '`
421 SRV_W=$(($SRV_W + $SRV_W_ADD))
422 if [ $SRV_RND -lt $SRV_W ]; then
423 SERVERNAME=`echo $X | cut -f 3 -d ' '`
424 echo "$X" >> serverlist_tried
427 SRV_RND=$(($SRV_RND - $SRV_W))
434 # Check that we have a public key with an appropriate hash, or
435 # fetch the key if it doesn't exist. Returns 1 if the key has
436 # not yet been fetched.
438 if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
442 echo -n "Fetching public key from ${SERVERNAME}... "
444 fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \
445 2>${QUIETREDIR} || true
446 if ! [ -r pub.ssl ]; then
450 if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
451 echo "key has incorrect hash."
458 # Fetch a snapshot tag
460 rm -f snapshot.ssl tag.new
462 echo ${NDEBUG} "Fetching snapshot tag from ${SERVERNAME}... "
463 fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl \
464 2>${QUIETREDIR} || true
465 if ! [ -r $1.ssl ]; then
470 openssl rsautl -pubin -inkey pub.ssl -verify \
471 < $1.ssl > tag.new 2>${QUIETREDIR} || true
474 if ! [ `wc -l < tag.new` = 1 ] ||
475 ! grep -qE "^portsnap\|[0-9]{10}\|[0-9a-f]{64}" tag.new; then
476 echo "invalid snapshot tag."
482 SNAPSHOTDATE=`cut -f 2 -d '|' < tag.new`
483 SNAPSHOTHASH=`cut -f 3 -d '|' < tag.new`
486 # Sanity-check the date on a snapshot tag
487 fetch_snapshot_tagsanity() {
488 if [ `date "+%s"` -gt `expr ${SNAPSHOTDATE} + 31536000` ]; then
489 echo "Snapshot appears to be more than a year old!"
490 echo "(Is the system clock correct?)"
491 echo "Cowardly refusing to proceed any further."
494 if [ `date "+%s"` -lt `expr ${SNAPSHOTDATE} - 86400` ]; then
495 echo -n "Snapshot appears to have been created more than "
496 echo "one day into the future!"
497 echo "(Is the system clock correct?)"
498 echo "Cowardly refusing to proceed any further."
503 # Sanity-check the date on a snapshot update tag
504 fetch_update_tagsanity() {
505 fetch_snapshot_tagsanity || return 1
507 if [ ${OLDSNAPSHOTDATE} -gt ${SNAPSHOTDATE} ]; then
508 echo -n "Latest snapshot on server is "
509 echo "older than what we already have!"
510 echo -n "Cowardly refusing to downgrade from "
511 date -r ${OLDSNAPSHOTDATE}
512 echo "to `date -r ${SNAPSHOTDATE}`."
517 # Compare old and new tags; return 1 if update is unnecessary
518 fetch_update_neededp() {
519 if [ ${OLDSNAPSHOTDATE} -eq ${SNAPSHOTDATE} ]; then
520 echo -n "Latest snapshot on server matches "
521 echo "what we already have."
522 echo "No updates needed."
526 if [ ${OLDSNAPSHOTHASH} = ${SNAPSHOTHASH} ]; then
527 echo -n "Ports tree hasn't changed since "
528 echo "last snapshot."
529 echo "No updates needed."
537 # Fetch snapshot metadata file
539 rm -f ${SNAPSHOTHASH} tINDEX.new
541 echo ${NDEBUG} "Fetching snapshot metadata... "
542 fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH} \
543 2>${QUIETREDIR} || return
544 if [ "`${SHA256} -q ${SNAPSHOTHASH}`" != ${SNAPSHOTHASH} ]; then
545 echo "snapshot metadata corrupt."
548 mv ${SNAPSHOTHASH} tINDEX.new
552 # Warn user about bogus metadata
553 fetch_metadata_freakout() {
555 echo "Portsnap metadata is correctly signed, but contains"
556 echo "at least one line which appears bogus."
557 echo "Cowardly refusing to proceed any further."
560 # Sanity-check a snapshot metadata file
561 fetch_metadata_sanity() {
562 if grep -qvE "^[0-9A-Z.]+\|[0-9a-f]{64}$" tINDEX.new; then
563 fetch_metadata_freakout
566 if [ `look INDEX tINDEX.new | wc -l` != 1 ]; then
568 echo "Portsnap metadata appears bogus."
569 echo "Cowardly refusing to proceed any further."
574 # Take a list of ${oldhash}|${newhash} and output a list of needed patches
575 fetch_make_patchlist() {
576 grep -vE "^([0-9a-f]{64})\|\1$" |
578 X=`echo ${LINE} | cut -f 1 -d '|'`
579 Y=`echo ${LINE} | cut -f 2 -d '|'`
580 if [ -f "files/${Y}.gz" ]; then continue; fi
581 if [ ! -f "files/${X}.gz" ]; then continue; fi
586 # Print user-friendly progress statistics
591 if [ $(($LNC % 10)) = 0 ]; then
593 elif [ $(($LNC % 2)) = 0 ]; then
600 # Sanity-check an index file
601 fetch_index_sanity() {
602 if grep -qvE "^[-_+./@0-9A-Za-z]+\|[0-9a-f]{64}$" INDEX.new ||
603 fgrep -q "./" INDEX.new; then
604 fetch_metadata_freakout
609 # Verify a list of files
610 fetch_snapshot_verify() {
612 if [ "`gunzip -c < snap/${F}.gz | ${SHA256} -q`" != ${F} ]; then
613 echo "snapshot corrupt."
620 # Fetch a snapshot tarball, extract, and verify.
622 while ! fetch_tag snapshot; do
623 fetch_pick_server || return 1
625 fetch_snapshot_tagsanity || return 1
626 fetch_metadata || return 1
627 fetch_metadata_sanity || return 1
631 # Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will
632 # probably take a while, so the progrees reports that fetch(1) generates
633 # will be useful for keeping the users' attention from drifting.
634 echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:"
635 fetch -r http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1
637 echo -n "Extracting snapshot... "
638 tar -xz --numeric-owner -f ${SNAPSHOTHASH}.tgz snap/ || return 1
639 rm ${SNAPSHOTHASH}.tgz
642 echo -n "Verifying snapshot integrity... "
643 # Verify the metadata files
644 cut -f 2 -d '|' tINDEX.new | fetch_snapshot_verify || return 1
647 gunzip -c < snap/`look INDEX tINDEX.new |
648 cut -f 2 -d '|'`.gz > INDEX.new
649 fetch_index_sanity || return 1
650 # Verify the snapshot contents
651 cut -f 2 -d '|' INDEX.new | fetch_snapshot_verify || return 1
652 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u > files.expected
653 find snap -mindepth 1 | sed -E 's^snap/(.*)\.gz^\1^' | sort > files.snap
654 if ! cmp -s files.expected files.snap; then
655 echo "unexpected files in snapshot."
658 rm files.expected files.snap
661 # Move files into their proper locations
662 rm -f tag INDEX tINDEX
672 # Update a compressed snapshot
674 rm -f patchlist diff OLD NEW filelist INDEX.new
676 OLDSNAPSHOTDATE=`cut -f 2 -d '|' < tag`
677 OLDSNAPSHOTHASH=`cut -f 3 -d '|' < tag`
679 while ! fetch_tag latest; do
680 fetch_pick_server || return 1
682 fetch_update_tagsanity || return 1
683 fetch_update_neededp || return 0
684 fetch_metadata || return 1
685 fetch_metadata_sanity || return 1
687 echo -n "Updating from `date -r ${OLDSNAPSHOTDATE}` "
688 echo "to `date -r ${SNAPSHOTDATE}`."
690 # Generate a list of wanted metadata patches
691 join -t '|' -o 1.2,2.2 tINDEX tINDEX.new |
692 fetch_make_patchlist > patchlist
694 # Attempt to fetch metadata patches
695 echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
696 echo ${NDEBUG} "metadata patches.${DDSTATS}"
697 tr '|' '-' < patchlist |
698 lam -s "tp/" - -s ".gz" |
699 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
700 2>${STATSREDIR} | fetch_progress
703 # Attempt to apply metadata patches
704 echo -n "Applying metadata patches... "
706 X=`echo ${LINE} | cut -f 1 -d '|'`
707 Y=`echo ${LINE} | cut -f 2 -d '|'`
708 if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
709 gunzip -c < ${X}-${Y}.gz > diff
710 gunzip -c < files/${X}.gz > OLD
711 cut -c 2- diff | join -t '|' -v 2 - OLD > ptmp
712 grep '^\+' diff | cut -c 2- |
713 sort -k 1,1 -t '|' -m - ptmp > NEW
714 if [ `${SHA256} -q NEW` = ${Y} ]; then
718 rm -f diff OLD NEW ${X}-${Y}.gz ptmp
719 done < patchlist 2>${QUIETREDIR}
722 # Update metadata without patches
723 join -t '|' -v 2 tINDEX tINDEX.new |
724 cut -f 2 -d '|' /dev/stdin patchlist |
726 if [ ! -f "files/${Y}.gz" ]; then
730 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
731 echo ${NDEBUG} "metadata files... "
732 lam -s "f/" - -s ".gz" < filelist |
733 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
737 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
738 mv ${Y}.gz files/${Y}.gz
740 echo "metadata is corrupt."
747 gunzip -c < files/`look INDEX tINDEX.new |
748 cut -f 2 -d '|'`.gz > INDEX.new
749 fetch_index_sanity || return 1
751 # If we have decided to refuse certain updates, construct a hybrid index which
752 # is equal to the old index for parts of the tree which we don't want to
753 # update, and equal to the new index for parts of the tree which gets updates.
754 # This means that we should always have a "complete snapshot" of the ports
755 # tree -- with the caveat that it isn't actually a snapshot.
756 if [ ! -z "${REFUSE}" ]; then
757 echo "Refusing to download updates for ${REFUSE}" \
760 grep -Ev "${REFUSE}" INDEX.new > INDEX.tmp
761 grep -E "${REFUSE}" INDEX |
762 sort -m -k 1,1 -t '|' - INDEX.tmp > INDEX.new
766 # Generate a list of wanted ports patches
767 join -t '|' -o 1.2,2.2 INDEX INDEX.new |
768 fetch_make_patchlist > patchlist
770 # Attempt to fetch ports patches
771 echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
772 echo ${NDEBUG} "patches.${DDSTATS}"
773 tr '|' '-' < patchlist | lam -s "bp/" - |
774 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
775 2>${STATSREDIR} | fetch_progress
778 # Attempt to apply ports patches
779 echo -n "Applying patches... "
781 X=`echo ${LINE} | cut -f 1 -d '|'`
782 Y=`echo ${LINE} | cut -f 2 -d '|'`
783 if [ ! -f "${X}-${Y}" ]; then continue; fi
784 gunzip -c < files/${X}.gz > OLD
785 ${BSPATCH} OLD NEW ${X}-${Y}
786 if [ `${SHA256} -q NEW` = ${Y} ]; then
790 rm -f diff OLD NEW ${X}-${Y}
791 done < patchlist 2>${QUIETREDIR}
794 # Update ports without patches
795 join -t '|' -v 2 INDEX INDEX.new |
796 cut -f 2 -d '|' /dev/stdin patchlist |
798 if [ ! -f "files/${Y}.gz" ]; then
802 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
803 echo ${NDEBUG} "new ports or files... "
804 lam -s "f/" - -s ".gz" < filelist |
805 xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
809 if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
810 mv ${Y}.gz files/${Y}.gz
812 echo "snapshot is corrupt."
818 # Remove files which are no longer needed
819 cut -f 2 -d '|' tINDEX INDEX | sort -u > oldfiles
820 cut -f 2 -d '|' tINDEX.new INDEX.new | sort -u | comm -13 - oldfiles |
821 lam -s "files/" - -s ".gz" | xargs rm -f
822 rm patchlist filelist oldfiles
832 # Do the actual work involved in "fetch" / "cron".
834 fetch_pick_server_init && fetch_pick_server
836 while ! fetch_key; do
837 fetch_pick_server || return 1
840 if ! [ -d files -a -r tag -a -r INDEX -a -r tINDEX ]; then
841 fetch_snapshot || return 1
843 fetch_update || return 1
846 # Build a ports INDEX file
847 extract_make_index() {
848 if ! look $1 ${WORKDIR}/tINDEX > /dev/null; then
849 echo -n "$1 not provided by portsnap server; "
850 echo "$2 not being generated."
852 gunzip -c < "${WORKDIR}/files/`look $1 ${WORKDIR}/tINDEX |
853 cut -f 2 -d '|'`.gz" |
855 ${MKINDEX} /dev/stdin > ${PORTSDIR}/$2
859 # Create INDEX, INDEX-5, INDEX-6
861 echo -n "Building new INDEX files... "
862 for PAIR in ${INDEXPAIRS}; do
863 INDEXFILE=`echo ${PAIR} | cut -f 1 -d '|'`
864 DESCRIBEFILE=`echo ${PAIR} | cut -f 2 -d '|'`
865 extract_make_index ${DESCRIBEFILE} ${INDEXFILE} || return 1
870 # Create .portsnap.INDEX; if we are REFUSEing to touch certain directories,
871 # merge the values from any exiting .portsnap.INDEX file.
873 if [ -z "${REFUSE}" ]; then
874 sort ${WORKDIR}/INDEX > ${PORTSDIR}/.portsnap.INDEX
875 elif [ -f ${PORTSDIR}/.portsnap.INDEX ]; then
876 grep -E "${REFUSE}" ${PORTSDIR}/.portsnap.INDEX \
877 > ${PORTSDIR}/.portsnap.INDEX.tmp
878 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort |
879 sort -m - ${PORTSDIR}/.portsnap.INDEX.tmp \
880 > ${PORTSDIR}/.portsnap.INDEX
881 rm -f ${PORTSDIR}/.portsnap.INDEX.tmp
883 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort \
884 > ${PORTSDIR}/.portsnap.INDEX
888 # Do the actual work involved in "extract"
890 mkdir -p ${PORTSDIR} || return 1
893 if ! [ -z "${EXTRACTPATH}" ]; then
894 grep "^${EXTRACTPATH}" ${WORKDIR}/INDEX
895 elif ! [ -z "${REFUSE}" ]; then
896 grep -vE "${REFUSE}" ${WORKDIR}/INDEX
899 fi | tr '|' ' ' | while read FILE HASH; do
900 echo ${PORTSDIR}/${FILE}
901 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
902 echo "files/${HASH}.gz not found -- snapshot corrupt."
907 rm -rf ${PORTSDIR}/${FILE%/}
908 mkdir -p ${PORTSDIR}/${FILE}
909 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
910 -C ${PORTSDIR}/${FILE}
913 rm -f ${PORTSDIR}/${FILE}
914 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
915 -C ${PORTSDIR} ${FILE}
921 if [ ! -z "${EXTRACTPATH}" ]; then
929 # Do the actual work involved in "update"
931 if ! [ -z "${INDEXONLY}" ]; then
932 extract_indices >/dev/null || return 1
936 if sort ${WORKDIR}/INDEX |
937 cmp -s ${PORTSDIR}/.portsnap.INDEX -; then
938 echo "Ports tree is already up to date."
942 # If we are REFUSEing to touch certain directories, don't remove files
943 # from those directories (even if they are out of date)
944 echo -n "Removing old files and directories... "
945 if ! [ -z "${REFUSE}" ]; then
946 sort ${WORKDIR}/INDEX |
947 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
948 grep -vE "${REFUSE}" |
949 lam -s "${PORTSDIR}/" - |
950 sed -e 's|/$||' | xargs rm -rf
952 sort ${WORKDIR}/INDEX |
953 comm -23 ${PORTSDIR}/.portsnap.INDEX - | cut -f 1 -d '|' |
954 lam -s "${PORTSDIR}/" - |
955 sed -e 's|/$||' | xargs rm -rf
960 echo "Extracting new files:"
962 if ! [ -z "${REFUSE}" ]; then
963 grep -vE "${REFUSE}" ${WORKDIR}/INDEX | sort
965 sort ${WORKDIR}/INDEX
967 comm -13 ${PORTSDIR}/.portsnap.INDEX - |
969 FILE=`echo ${LINE} | cut -f 1 -d '|'`
970 HASH=`echo ${LINE} | cut -f 2 -d '|'`
971 echo ${PORTSDIR}/${FILE}
972 if ! [ -r "${WORKDIR}/files/${HASH}.gz" ]; then
973 echo "files/${HASH}.gz not found -- snapshot corrupt."
978 mkdir -p ${PORTSDIR}/${FILE}
979 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
980 -C ${PORTSDIR}/${FILE}
983 tar -xz --numeric-owner -f ${WORKDIR}/files/${HASH}.gz \
984 -C ${PORTSDIR} ${FILE}
995 #### Main functions -- call parameter-handling and core functions
997 # Using the command line, configuration file, and defaults,
998 # set all the parameters which are needed later.
1008 # Fetch command. Make sure that we're being called
1009 # interactively, then run fetch_check_params and fetch_run
1012 echo -n "`basename $0` fetch should not "
1013 echo "be run non-interactively."
1014 echo "Run `basename $0` cron instead."
1021 # Cron command. Make sure the parameters are sensible; wait
1022 # rand(3600) seconds; then fetch updates. While fetching updates,
1023 # send output to a temporary file; only print that file if the
1027 sleep `jot -r 1 0 3600`
1029 TMPFILE=`mktemp /tmp/portsnap.XXXXXX` || exit 1
1030 if ! fetch_run >> ${TMPFILE}; then
1039 # Extract command. Make sure the parameters are sensible,
1040 # then extract the ports tree (or part thereof).
1042 extract_check_params
1043 extract_run || exit 1
1046 # Update command. Make sure the parameters are sensible,
1047 # then update the ports tree.
1050 update_run || exit 1
1053 # Alfred command. Run 'fetch' or 'cron' depending on
1054 # whether stdin is a terminal; then run 'update' or
1055 # 'extract' depending on whether ${PORTSDIR} exists.
1062 if [ -d ${PORTSDIR} ]; then
1071 # Make sure we find utilities from the base system
1072 export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
1074 # Set LC_ALL in order to avoid problems with character ranges like [A-Z].
1078 for COMMAND in ${COMMANDS}; do