tools/tools/sysbuild/sysbuild.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 1994-2009 Poul-Henning Kamp.
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions
   8 # are met:
   9 # 1. Redistributions of source code must retain the above copyright
  10 #    notice, this list of conditions and the following disclaimer.
  11 # 2. Redistributions in binary form must reproduce the above copyright
  12 #    notice, this list of conditions and the following disclaimer in the
  13 #    documentation and/or other materials provided with the distribution.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25 # SUCH DAMAGE.
  26 #
  27 # $FreeBSD$
  28 #
  29
  30 set -e
  31
  32 exec < /dev/null
  33
  34 if [ `uname -m` = "i386" -o `uname -m` = "amd64" ] ; then
  35         TARGET_PART=`df / | sed '
  36         1d
  37         s/[    ].*//
  38         s,/dev/,,
  39         s,s1a,s3a,
  40         s,s2a,s1a,
  41         s,s3a,s2a,
  42         '`
  43
  44         FREEBSD_PART=`sed -n    \
  45                 -e 's/#.*//'    \
  46                 -e '/[  ]\/freebsd[     ]/!d'   \
  47                 -e 's/[         ].*//p' \
  48                 /etc/fstab`
  49
  50         # Calculate a suggested gpart command
  51         TARGET_DISK=`expr ${TARGET_PART} : '\(.*\)s[12]a$' || true`
  52         TARGET_SLICE=`expr ${TARGET_PART} : '.*s\([12]\)a$' || true`
  53         GPART_SUGGESTION="gpart set -a active -i $TARGET_SLICE /dev/$TARGET_DISK"
  54         unset TARGET_DISK TARGET_SLICE
  55 else
  56         TARGET_PART=unknown
  57         FREEBSD_PART=unknown
  58         GPART_SUGGESTION=unknown
  59 fi
  60
  61
  62 # Relative to /freebsd
  63 PORTS_PATH=ports
  64 SRC_PATH=src
  65 # OBJ_PATH=obj
  66
  67 # Name of kernel
  68 KERNCONF=GENERIC
  69
  70 # srcconf
  71 #SRCCONF="SRCCONF=/usr/src/src.conf"
  72
  73 # -j arg to make(1)
  74
  75 ncpu=`sysctl -n kern.smp.cpus`
  76 if [ $ncpu -gt 1 ] ; then
  77         JARG="-j $ncpu"
  78 fi
  79
  80 # serial console ?
  81 SERCONS=false
  82
  83 PKG_DIR=/usr/ports/packages/All
  84
  85 # Remotely mounted distfiles
  86 # REMOTEDISTFILES=fs:/rdonly/distfiles
  87
  88 # Proxy
  89 #FTP_PROXY=http://127.0.0.1:3128/
  90 #HTTP_PROXY=http://127.0.0.1:3128/
  91 #export FTP_PROXY HTTP_PROXY
  92
  93 PORTS_WE_WANT='
  94 '
  95
  96 PORTS_OPTS="BATCH=YES A4=yes"
  97
  98 PORTS_WITHOUT=""
  99 PORTS_WITH=""
 100
 101 CONFIGFILES='
 102 '
 103
 104 SBMNT="/mnt.sysbuild"
 105
 106 cleanup() (
 107 )
 108
 109 before_ports() (
 110 )
 111
 112 before_ports_chroot() (
 113 )
 114
 115 final_root() (
 116 )
 117
 118 final_chroot() (
 119 )
 120
 121 #######################################################################
 122 # -P is a pretty neat way to clean junk out from your ports dist-files:
 123 #
 124 #       mkdir /freebsd/ports/distfiles.old
 125 #       mv /freebsd/ports/distfiles/* /freebsd/ports/distfiles.old
 126 #       sh sysbuild.sh -c $yourconfig -P /freebsd/ports/distfiles.old
 127 #       rm -rf /freebsd/ports/distfiles.old
 128 #
 129 # Unfortunately bsd.ports.mk does not attempt to use a hard-link so
 130 # while this runs you need diskspace for both your old and your "new"
 131 # distfiles.
 132 #
 133 #######################################################################
 134
 135 usage () {
 136         (
 137         echo "Usage: $0 [-b/-k/-w] [-c config_file]"
 138         echo "  -b      suppress builds (both kernel and world)"
 139         echo "  -k      suppress buildkernel"
 140         echo "  -w      suppress buildworld"
 141         echo "  -p      used cached packages"
 142         echo "  -P <dir> prefetch ports"
 143         echo "  -c      specify config file"
 144         ) 1>&2
 145         exit 2
 146 }
 147
 148 #######################################################################
 149 #######################################################################
 150
 151 if [ ! -f $0 ] ; then
 152         echo "Must be able to access self ($0)" 1>&2
 153         exit 1
 154 fi
 155
 156 if grep -q 'Magic String: 0`0nQT40W%l,CX&' $0 ; then
 157         true
 158 else
 159         echo "self ($0) does not contain magic string" 1>&2
 160         exit 1
 161 fi
 162
 163 #######################################################################
 164
 165 set -e
 166
 167 log_it() (
 168         a="$*"
 169         set `cat /tmp/_sb_log`
 170         TX=`date +%s`
 171         echo "$1 $TX" > /tmp/_sb_log
 172         DT=`expr $TX - $1 || true`
 173         DL=`expr $TX - $2 || true`
 174         echo -n "### `date +%H:%M:%S`"
 175         printf " ### %5d ### %5d ### %s\n" $DT $DL "$a"
 176 )
 177
 178 #######################################################################
 179
 180 ports_make() {
 181         make $* WITH="${PORTS_WITH}" WITHOUT="${PORTS_WITHOUT}" ${PORTS_OPTS}
 182 }
 183
 184 ports_recurse() (
 185         cd /usr/ports
 186         t=$1
 187         shift
 188         if [ "x$t" = "x." ] ; then
 189                 true > /tmp/_.plist
 190                 true > /tmp/_.plist.tdone
 191                 echo 'digraph {' > /tmp/_.plist.dot
 192         fi
 193         if grep -q "^$t\$" /tmp/_.plist.tdone ; then
 194                 return
 195         fi
 196         echo "$t" >> /tmp/_.plist.tdone
 197         for d
 198         do
 199                 if [ "x$d" == "xpatch" ] ; then
 200                         continue
 201                 fi
 202                 fl=""
 203                 if [ ! -d $d ] ; then
 204                         fl=FLAVOR=`expr $d : '.*@\(.*\)'`
 205                         bd=`expr $d : '\(.*\)@.*'`
 206                         if [ ! -d "$bd" ] ; then
 207                                 echo "Missing port $d ($t) (fl $fl) (bd $bd)" 1>&2
 208                                 continue
 209                         fi
 210                         # echo "Flavored port $d ($t) (fl $fl) (bd $bd)" 1>&2
 211                         d=$bd
 212                 fi
 213                 d=`cd /usr/ports && cd $d && /bin/pwd`
 214                 if [ ! -f $d/Makefile ] ; then
 215                         echo "Missing port (Makefile) $d" 1>&2
 216                         continue
 217                 fi
 218                 if [ "x$t" != "x." ] ; then
 219                         echo "\"$t\" -> \"$d\"" >> /tmp/_.plist.dot
 220                 fi
 221                 if grep -q "^$d\$" /tmp/_.plist ; then
 222                         true
 223                 elif grep -q "^$d\$" /tmp/_.plist.tdone ; then
 224                         true
 225                 else
 226                         (
 227                         cd $d
 228                         l=""
 229                         for a in `ports_make -V _UNIFIED_DEPENDS $fl`
 230                         do
 231                                 x=`expr "$a" : '.*:\(.*\)'`
 232                                 l="${l} ${x}"
 233                         done
 234                         ports_recurse $d $l
 235                         )
 236                         echo "$d" >> /tmp/_.plist
 237                 fi
 238         done
 239         if [ "x$t" = "x." ] ; then
 240                 echo '}' >> /tmp/_.plist.dot
 241         fi
 242 )
 243
 244 ports_build() (
 245
 246         ports_recurse . $PORTS_WE_WANT
 247
 248         if [ "x${PKG_DIR}" != "x" ] ; then
 249                 mkdir -p ${PKG_DIR}
 250         fi
 251
 252         pd=`cd /usr/ports && /bin/pwd`
 253         # Now build & install them
 254         for p in `cat /tmp/_.plist`
 255         do
 256                 b=`echo $p | tr / _`
 257                 t=`echo $p | sed "s,${pd},,"`
 258                 pn=`cd $p && ports_make package-name`
 259
 260                 if [ "x`basename $p`" == "xpkg" ] ; then
 261                         log_it "Very Special: $t ($pn)"
 262
 263                         (
 264                         cd $p
 265                         ports_make clean all install
 266                         ) > _.$b 2>&1 < /dev/null
 267                         continue
 268                 fi
 269
 270                 if pkg info $pn > /dev/null 2>&1 ; then
 271                         log_it "Already installed: $t ($pn)"
 272                         continue
 273                 fi
 274
 275                 if [ "x${PKG_DIR}" != "x" -a -f ${PKG_DIR}/$pn.txz ] ; then
 276                         if [ "x$use_pkg" = "x-p" ] ; then
 277                                 log_it "Install $t ($pn)"
 278                                 (
 279                                 set +e
 280                                 pkg add ${PKG_DIR}/$pn.txz || true
 281                                 ) > _.$b 2>&1 < /dev/null
 282                                 continue
 283                         fi
 284                 fi
 285
 286                 miss=`(cd $p ; ports_make missing) || true`
 287
 288                 if [ "x${miss}" != "x" ] ; then
 289                         log_it "NB: MISSING for $p:" $miss
 290                 fi
 291
 292                 log_it "build $pn ($p)"
 293                 (
 294                         set +e
 295                         cd $p
 296                         ports_make clean
 297                         if ports_make install ; then
 298                                 if [ "x${PKG_DIR}" != "x" ] ; then
 299                                         ports_make package
 300                                 fi
 301                         else
 302                                 log_it FAIL build $p
 303                         fi
 304                         ports_make clean
 305
 306                 ) > _.$b 2>&1 < /dev/null
 307         done
 308 )
 309
 310 ports_prefetch() (
 311         (
 312         set +x
 313         ldir=$1
 314         true > /${ldir}/_.prefetch
 315         echo "Building /tmp/_.plist" >> /${ldir}/_.prefetch
 316
 317         ports_recurse . $PORTS_WE_WANT
 318
 319         echo "Completed /tmp/_.plist" >> /${ldir}/_.prefetch
 320         # Now checksump/fetch them
 321         for p in `cat /tmp/_.plist`
 322         do
 323                 b=`echo $p | tr / _`
 324                 (
 325                         cd $p
 326                         if ports_make checksum ; then
 327                                 rm -f /${ldir}/_.prefetch.$b
 328                                 echo "OK $p" >> /${ldir}/_.prefetch
 329                                 exit 0
 330                         fi
 331                         ports_make distclean
 332                         ports_make checksum || true
 333
 334                         if ports_make checksum > /dev/null 2>&1 ; then
 335                                 rm -f /${ldir}/_.prefetch.$b
 336                                 echo "OK $p" >> /${ldir}/_.prefetch
 337                         else
 338                                 echo "BAD $p" >> /${ldir}/_.prefetch
 339                         fi
 340                 ) > /${ldir}/_.prefetch.$b 2>&1
 341         done
 342         echo "Done" >> /${ldir}/_.prefetch
 343         )
 344 )
 345
 346 #######################################################################
 347
 348 do_world=true
 349 do_kernel=true
 350 do_prefetch=false
 351 use_pkg=""
 352 c_arg=""
 353
 354 set +e
 355 args=`getopt bc:hkpP:w $*`
 356 if [ $? -ne 0 ] ; then
 357         usage
 358 fi
 359 set -e
 360
 361 set -- $args
 362 for i
 363 do
 364         case "$i"
 365         in
 366         -b)
 367                 shift;
 368                 do_world=false
 369                 do_kernel=false
 370                 ;;
 371         -c)
 372                 c_arg=$2
 373                 if [ ! -f "$c_arg" ] ; then
 374                         echo "Cannot read $c_arg" 1>&2
 375                         usage
 376                 fi
 377                 . "$2"
 378                 shift
 379                 shift
 380                 ;;
 381         -h)
 382                 usage
 383                 ;;
 384         -k)
 385                 shift;
 386                 do_kernel=false
 387                 ;;
 388         -p)
 389                 shift;
 390                 use_pkg="-p"
 391                 ;;
 392         -P)
 393                 shift;
 394                 do_prefetch=true
 395                 distfile_cache=$1
 396                 shift;
 397                 ;;
 398         -w)
 399                 shift;
 400                 do_world=false
 401                 ;;
 402         --)
 403                 shift
 404                 break;
 405                 ;;
 406         esac
 407 done
 408
 409 #######################################################################
 410
 411 if [ "x$1" = "xchroot_script" ] ; then
 412         set -e
 413
 414         shift
 415
 416         before_ports_chroot
 417
 418         ports_build
 419
 420         exit 0
 421 fi
 422
 423 if [ "x$1" = "xfinal_chroot" ] ; then
 424         final_chroot
 425         exit 0
 426 fi
 427
 428 if [ $# -gt 0 ] ; then
 429         echo "$0: Extraneous arguments supplied"
 430         usage
 431 fi
 432
 433 #######################################################################
 434
 435 T0=`date +%s`
 436 echo $T0 $T0 > /tmp/_sb_log
 437
 438 [ ! -d ${SBMNT} ] && mkdir -p ${SBMNT}
 439
 440 if $do_prefetch ; then
 441         rm -rf /tmp/sysbuild/ports
 442         mkdir -p /tmp/sysbuild/ports
 443         ln -s ${distfile_cache} /tmp/sysbuild/ports/distfiles
 444         export PORTS_OPTS=CD_MOUNTPTS=/tmp/sysbuild
 445         ports_prefetch /tmp
 446         exit 0
 447 fi
 448
 449 log_it Unmount everything
 450 (
 451         ( cleanup )
 452         umount /freebsd/distfiles || true
 453         umount ${SBMNT}/freebsd/distfiles || true
 454         umount ${FREEBSD_PART} || true
 455         umount ${SBMNT}/freebsd || true
 456         umount ${SBMNT}/dev || true
 457         umount ${SBMNT} || true
 458         umount /dev/${TARGET_PART} || true
 459 ) # > /dev/null 2>&1
 460
 461 log_it Prepare running image
 462 mkdir -p /freebsd
 463 mount ${FREEBSD_PART} /freebsd
 464
 465 #######################################################################
 466
 467 if [ ! -d /freebsd/${PORTS_PATH} ] ;  then
 468         echo PORTS_PATH does not exist 1>&2
 469         exit 1
 470 fi
 471
 472 if [ ! -d /freebsd/${SRC_PATH} ] ;  then
 473         echo SRC_PATH does not exist 1>&2
 474         exit 1
 475 fi
 476
 477 log_it TARGET_PART $TARGET_PART
 478 sleep 5
 479
 480 rm -rf /usr/ports
 481 ln -s /freebsd/${PORTS_PATH} /usr/ports
 482
 483 rm -rf /usr/src
 484 ln -s /freebsd/${SRC_PATH} /usr/src
 485
 486 if $do_world ; then
 487         if [ "x${OBJ_PATH}" != "x" ] ; then
 488                 rm -rf /usr/obj
 489                 (cd /freebsd && mkdir -p ${OBJ_PATH} && ln -s ${OBJ_PATH} /usr/obj)
 490         else
 491                 rm -rf /usr/obj/*
 492                 mkdir -p /usr/obj
 493         fi
 494 fi
 495
 496 #######################################################################
 497
 498 for i in ${PORTS_WE_WANT}
 499 do
 500         (
 501         cd /usr/ports
 502         if [ ! -d $i ] ; then
 503                 fl=FLAVOR=`expr $i : '.*@\(.*\)'`
 504                 i=`expr $i : '\(.*\)@.*'`
 505         fi
 506         if [ ! -d $i ]  ; then
 507                 echo "Port $i not found" 1>&2
 508                 exit 2
 509         fi
 510         )
 511 done
 512
 513 #export PORTS_WE_WANT
 514 #export PORTS_OPTS
 515
 516 #######################################################################
 517
 518 log_it Prepare destination partition
 519 newfs -t -E -O2 -U /dev/${TARGET_PART} > /dev/null
 520 mount /dev/${TARGET_PART} ${SBMNT}
 521 mkdir -p ${SBMNT}/dev
 522 mount -t devfs devfs ${SBMNT}/dev
 523
 524 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 525         rm -rf /freebsd/${PORTS_PATH}/distfiles
 526         ln -s /freebsd/distfiles /freebsd/${PORTS_PATH}/distfiles
 527         mkdir -p /freebsd/distfiles
 528         mount  ${REMOTEDISTFILES} /freebsd/distfiles
 529 fi
 530
 531 log_it copy ports config files
 532 (cd / ; find var/db/ports -print | cpio -dumpv ${SBMNT} > /dev/null 2>&1)
 533
 534 log_it "Start prefetch of ports distfiles"
 535 ports_prefetch ${SBMNT} &
 536
 537 if $do_world ; then
 538         (
 539         cd /usr/src
 540         log_it "Buildworld"
 541         make ${JARG} -s buildworld ${SRCCONF} > ${SBMNT}/_.bw 2>&1
 542         )
 543 fi
 544
 545 if $do_kernel ; then
 546         (
 547         cd /usr/src
 548         log_it "Buildkernel"
 549         make ${JARG} -s buildkernel KERNCONF=$KERNCONF > ${SBMNT}/_.bk 2>&1
 550         )
 551 fi
 552
 553
 554 log_it Installworld
 555 (cd /usr/src && make ${JARG} installworld DESTDIR=${SBMNT} ${SRCCONF} ) \
 556         > ${SBMNT}/_.iw 2>&1
 557
 558 log_it distribution
 559 (cd /usr/src && make -m /usr/src/share/mk distribution DESTDIR=${SBMNT} ${SRCCONF} ) \
 560         > ${SBMNT}/_.dist 2>&1
 561
 562 log_it Installkernel
 563 (cd /usr/src && make ${JARG} installkernel DESTDIR=${SBMNT} KERNCONF=$KERNCONF ) \
 564         > ${SBMNT}/_.ik 2>&1
 565
 566 if [ "x${OBJ_PATH}" != "x" ] ; then
 567         rmdir ${SBMNT}/usr/obj
 568         ( cd /freebsd && mkdir -p ${OBJ_PATH} && ln -s ${OBJ_PATH} ${SBMNT}/usr/obj )
 569 fi
 570
 571 log_it Wait for ports prefetch
 572 log_it "(Tail ${SBMNT}/_.prefetch for progress)"
 573 wait
 574
 575 log_it Move filesystems
 576
 577 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 578         umount /freebsd/distfiles
 579 fi
 580 umount ${FREEBSD_PART} || true
 581 mkdir -p ${SBMNT}/freebsd
 582 mount ${FREEBSD_PART} ${SBMNT}/freebsd
 583 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 584         mount  ${REMOTEDISTFILES} ${SBMNT}/freebsd/distfiles
 585 fi
 586
 587 rm -rf ${SBMNT}/usr/ports || true
 588 ln -s /freebsd/${PORTS_PATH} ${SBMNT}/usr/ports
 589
 590 rm -rf ${SBMNT}/usr/src || true
 591 ln -s /freebsd/${SRC_PATH} ${SBMNT}/usr/src
 592
 593 log_it Build and install ports
 594
 595 # Make sure fetching will work in the chroot
 596 if [ -f /etc/resolv.conf ] ; then
 597         log_it copy resolv.conf
 598         cp /etc/resolv.conf ${SBMNT}/etc
 599         chflags schg ${SBMNT}/etc/resolv.conf
 600 fi
 601
 602 if [ -f /etc/localtime ] ; then
 603         log_it copy localtime
 604         cp /etc/localtime ${SBMNT}/etc
 605         if [ -f /var/db/zoneinfo ] ; then
 606                 log_it copy zoneinfo
 607                 cp /var/db/zoneinfo ${SBMNT}/var/db
 608         fi
 609 fi
 610
 611 log_it ldconfig in chroot
 612 chroot ${SBMNT} sh /etc/rc.d/ldconfig start
 613
 614 log_it before_ports
 615 (
 616         before_ports
 617 )
 618
 619 log_it fixing fstab
 620 sed "/[         ]\/[    ]/s;^[^         ]*[     ];/dev/${TARGET_PART}   ;" \
 621         /etc/fstab > ${SBMNT}/etc/fstab
 622
 623 log_it build ports
 624
 625 cp $0 ${SBMNT}/root
 626 cp /tmp/_sb_log ${SBMNT}/tmp
 627 b=`basename $0`
 628 if [ "x$c_arg" != "x" ] ; then
 629         cp $c_arg ${SBMNT}/root
 630         chroot ${SBMNT} sh /root/$0 -c /root/`basename $c_arg` $use_pkg chroot_script
 631 else
 632         chroot ${SBMNT} sh /root/$0 $use_pkg chroot_script
 633 fi
 634 cp ${SBMNT}/tmp/_sb_log /tmp
 635
 636 log_it create all mountpoints
 637 grep -v '^[     ]*#' ${SBMNT}/etc/fstab |
 638 while read a b c
 639 do
 640         mkdir -p ${SBMNT}/$b
 641 done
 642
 643 if [ "x$SERCONS" != "xfalse" ] ; then
 644         log_it serial console
 645         echo " -h" > ${SBMNT}/boot.config
 646         sed -i "" -e /ttyd0/s/off/on/ ${SBMNT}/etc/ttys
 647         sed -i "" -e /ttyu0/s/off/on/ ${SBMNT}/etc/ttys
 648         sed -i "" -e '/^ttyv[0-8]/s/    on/     off/' ${SBMNT}/etc/ttys
 649 fi
 650
 651 log_it move dist config files "(expect warnings)"
 652 (
 653         cd ${SBMNT}
 654         mkdir root/configfiles_dist
 655         find ${CONFIGFILES} -print | cpio -dumpv root/configfiles_dist
 656 )
 657
 658 log_it copy live config files
 659 (cd / && find ${CONFIGFILES} -print | cpio -dumpv ${SBMNT})
 660
 661 log_it final_root
 662 ( final_root )
 663 log_it final_chroot
 664 cp /tmp/_sb_log ${SBMNT}/tmp
 665 if [ "x$c_arg" != "x" ] ; then
 666         chroot ${SBMNT} sh /root/$0 -c /root/`basename $c_arg` final_chroot
 667 else
 668         chroot ${SBMNT} sh /root/$0 final_chroot
 669 fi
 670 cp ${SBMNT}/tmp/_sb_log /tmp
 671 log_it "Check these messages (if any):"
 672 grep '^Stop' ${SBMNT}/_* || true
 673 log_it DONE
 674 echo "Now you probably want to:"
 675 echo "    $GPART_SUGGESTION"
 676 echo "    shutdown -r now"