tools/tools/sysbuild/sysbuild.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 1994-2009 Poul-Henning Kamp.
   4 # All rights reserved.
   5 #
   6 # Redistribution and use in source and binary forms, with or without
   7 # modification, are permitted provided that the following conditions
   8 # are met:
   9 # 1. Redistributions of source code must retain the above copyright
  10 #    notice, this list of conditions and the following disclaimer.
  11 # 2. Redistributions in binary form must reproduce the above copyright
  12 #    notice, this list of conditions and the following disclaimer in the
  13 #    documentation and/or other materials provided with the distribution.
  14 #
  15 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25 # SUCH DAMAGE.
  26 #
  27 # $FreeBSD$
  28 #
  29
  30 set -e
  31
  32 exec < /dev/null
  33
  34 if [ `uname -m` = "i386" -o `uname -m` = "amd64" ] ; then
  35         TARGET_PART=`df / | sed '
  36         1d
  37         s/[    ].*//
  38         s,/dev/,,
  39         s,s1a,s3a,
  40         s,s2a,s1a,
  41         s,s3a,s2a,
  42         '`
  43
  44         FREEBSD_PART=`sed -n    \
  45                 -e 's/#.*//'    \
  46                 -e '/[  ]\/freebsd[     ]/!d'   \
  47                 -e 's/[         ].*//p' \
  48                 /etc/fstab`
  49
  50         # Calculate a suggested gpart command
  51         TARGET_DISK=`expr ${TARGET_PART} : '\(.*\)s[12]a$' || true`
  52         TARGET_SLICE=`expr ${TARGET_PART} : '.*s\([12]\)a$' || true`
  53         GPART_SUGGESTION="gpart set -a active -i $TARGET_SLICE /dev/$TARGET_DISK"
  54         unset TARGET_DISK TARGET_SLICE
  55 else
  56         TARGET_PART=unknown
  57         FREEBSD_PART=unknown
  58         GPART_SUGGESTION=unknown
  59 fi
  60
  61
  62 # Relative to /freebsd
  63 PORTS_PATH=ports
  64 SRC_PATH=src
  65 # OBJ_PATH=obj
  66
  67 # Name of kernel
  68 KERNCONF=GENERIC
  69
  70 # srcconf
  71 #SRCCONF="SRCCONF=/usr/src/src.conf"
  72
  73 # -j arg to make(1)
  74
  75 ncpu=`sysctl -n kern.smp.cpus`
  76 if [ $ncpu -gt 1 ] ; then
  77         JARG="-j $ncpu"
  78 fi
  79
  80 # serial console ?
  81 SERCONS=false
  82
  83 PKG_DIR=/usr/ports/packages/All
  84
  85 # Remotely mounted distfiles
  86 # REMOTEDISTFILES=fs:/rdonly/distfiles
  87
  88 # Proxy
  89 #FTP_PROXY=http://127.0.0.1:3128/
  90 #HTTP_PROXY=http://127.0.0.1:3128/
  91 #export FTP_PROXY HTTP_PROXY
  92
  93 PORTS_WE_WANT='
  94 '
  95
  96 PORTS_OPTS="BATCH=YES A4=yes"
  97
  98 CONFIGFILES='
  99 '
 100
 101 SBMNT="/mnt.sysbuild"
 102
 103 cleanup() (
 104 )
 105
 106 before_ports() (
 107 )
 108
 109 before_ports_chroot() (
 110 )
 111
 112 final_root() (
 113 )
 114
 115 final_chroot() (
 116 )
 117
 118 #######################################################################
 119 # -P is a pretty neat way to clean junk out from your ports dist-files:
 120 #
 121 #       mkdir /freebsd/ports/distfiles.old
 122 #       mv /freebsd/ports/distfiles/* /freebsd/ports/distfiles.old
 123 #       sh sysbuild.sh -c $yourconfig -P /freebsd/ports/distfiles.old
 124 #       rm -rf /freebsd/ports/distfiles.old
 125 #
 126 # Unfortunately bsd.ports.mk does not attempt to use a hard-link so
 127 # while this runs you need diskspace for both your old and your "new"
 128 # distfiles.
 129 #
 130 #######################################################################
 131
 132 usage () {
 133         (
 134         echo "Usage: $0 [-b/-k/-w] [-c config_file]"
 135         echo "  -b      suppress builds (both kernel and world)"
 136         echo "  -k      suppress buildkernel"
 137         echo "  -w      suppress buildworld"
 138         echo "  -p      used cached packages"
 139         echo "  -P <dir> prefetch ports"
 140         echo "  -c      specify config file"
 141         ) 1>&2
 142         exit 2
 143 }
 144
 145 #######################################################################
 146 #######################################################################
 147
 148 if [ ! -f $0 ] ; then
 149         echo "Must be able to access self ($0)" 1>&2
 150         exit 1
 151 fi
 152
 153 if grep -q 'Magic String: 0`0nQT40W%l,CX&' $0 ; then
 154         true
 155 else
 156         echo "self ($0) does not contain magic string" 1>&2
 157         exit 1
 158 fi
 159
 160 #######################################################################
 161
 162 set -e
 163
 164 log_it() (
 165         a="$*"
 166         set `cat /tmp/_sb_log`
 167         TX=`date +%s`
 168         echo "$1 $TX" > /tmp/_sb_log
 169         DT=`expr $TX - $1 || true`
 170         DL=`expr $TX - $2 || true`
 171         echo -n "### `date +%H:%M:%S`"
 172         printf " ### %5d ### %5d ### %s\n" $DT $DL "$a"
 173 )
 174
 175 #######################################################################
 176
 177
 178 ports_recurse() (
 179         cd /usr/ports
 180         t=$1
 181         shift
 182         if [ "x$t" = "x." ] ; then
 183                 true > /tmp/_.plist
 184                 true > /tmp/_.plist.tdone
 185                 echo 'digraph {' > /tmp/_.plist.dot
 186         fi
 187         if grep -q "^$t\$" /tmp/_.plist.tdone ; then
 188                 return
 189         fi
 190         echo "$t" >> /tmp/_.plist.tdone
 191         for d
 192         do
 193                 if [ ! -d $d ] ; then
 194                         echo "Missing port $d" 1>&2
 195                         continue
 196                 fi
 197                 d=`cd /usr/ports && cd $d && /bin/pwd`
 198                 if [ ! -f $d/Makefile ] ; then
 199                         echo "Missing port $d" 1>&2
 200                         continue
 201                 fi
 202                 if [ "x$t" != "x." ] ; then
 203                         echo "\"$t\" -> \"$d\"" >> /tmp/_.plist.dot
 204                 fi
 205                 if grep -q "^$d\$" /tmp/_.plist ; then
 206                         true
 207                 elif grep -q "^$d\$" /tmp/_.plist.tdone ; then
 208                         true
 209                 else
 210                         (
 211                         cd $d
 212                         l=""
 213                         for a in `make -V _UNIFIED_DEPENDS ${PORTS_OPTS}`
 214                         do
 215                                 x=`expr "$a" : '.*:\(.*\)'`
 216                                 l="${l} ${x}"
 217                         done
 218                         ports_recurse $d $l
 219                         # -> _UNIFIED_DEPENDS
 220                         #ports_recurse $d `make -V _DEPEND_DIRS ${PORTS_OPTS}`
 221                         #ports_recurse $d `make all-depends-list`
 222                         )
 223                         echo "$d" >> /tmp/_.plist
 224                 fi
 225         done
 226         if [ "x$t" = "x." ] ; then
 227                 echo '}' >> /tmp/_.plist.dot
 228         fi
 229 )
 230
 231 ports_build() (
 232
 233         ports_recurse . $PORTS_WE_WANT
 234
 235         if [ "x${PKG_DIR}" != "x" ] ; then
 236                 mkdir -p ${PKG_DIR}
 237         fi
 238
 239         pd=`cd /usr/ports && /bin/pwd`
 240         # Now build & install them
 241         for p in `cat /tmp/_.plist`
 242         do
 243                 b=`echo $p | tr / _`
 244                 t=`echo $p | sed "s,${pd},,"`
 245                 pn=`cd $p && make package-name ${PORTS_OPTS}`
 246
 247                 if [ "x`basename $p`" == "xpkg" ] ; then
 248                         log_it "Very Special: $t ($pn)"
 249
 250                         (
 251                         cd $p
 252                         make clean all install ${PORTS_OPTS}
 253                         ) > _.$b 2>&1 < /dev/null
 254                         continue
 255                 fi
 256
 257                 if pkg info $pn > /dev/null 2>&1 ; then
 258                         log_it "Already installed: $t ($pn)"
 259                         continue
 260                 fi
 261
 262                 if [ "x${PKG_DIR}" != "x" -a -f ${PKG_DIR}/$pn.txz ] ; then
 263                         if [ "x$use_pkg" = "x-p" ] ; then
 264                                 log_it "Install $t ($pn)"
 265                                 (
 266                                 set +e
 267                                 pkg add ${PKG_DIR}/$pn.txz || true
 268                                 ) > _.$b 2>&1 < /dev/null
 269                                 continue
 270                         fi
 271                 fi
 272
 273                 miss=`(cd $p ; make missing ${PORTS_OPTS}) || true`
 274
 275                 if [ "x${miss}" != "x" ] ; then
 276                         log_it "MISSING for $p:" $miss
 277                         continue
 278                 fi
 279
 280                 log_it "build $pn ($p)"
 281                 (
 282                         set +e
 283                         cd $p
 284                         make clean ${PORTS_OPTS}
 285                         if make install ${PORTS_OPTS} ; then
 286                                 if [ "x${PKG_DIR}" != "x" ] ; then
 287                                         make package ${PORTS_OPTS}
 288                                 fi
 289                         else
 290                                 log_it FAIL build $p
 291                         fi
 292                         make clean ${PORTS_OPTS}
 293                 ) > _.$b 2>&1 < /dev/null
 294         done
 295 )
 296
 297 ports_prefetch() (
 298         (
 299         set +x
 300         ldir=$1
 301         true > /${ldir}/_.prefetch
 302         echo "Building /tmp/_.plist" >> /${ldir}/_.prefetch
 303
 304         ports_recurse . $PORTS_WE_WANT
 305
 306         echo "Completed /tmp/_.plist" >> /${ldir}/_.prefetch
 307         # Now checksump/fetch them
 308         for p in `cat /tmp/_.plist`
 309         do
 310                 b=`echo $p | tr / _`
 311                 (
 312                         cd $p
 313                         if make checksum $PORTS_OPTS ; then
 314                                 rm -f /${ldir}/_.prefetch.$b
 315                                 echo "OK $p" >> /${ldir}/_.prefetch
 316                                 exit 0
 317                         fi
 318                         make distclean
 319                         make checksum $PORTS_OPTS || true
 320
 321                         if make checksum $PORTS_OPTS > /dev/null 2>&1 ; then
 322                                 rm -f /${ldir}/_.prefetch.$b
 323                                 echo "OK $p" >> /${ldir}/_.prefetch
 324                         else
 325                                 echo "BAD $p" >> /${ldir}/_.prefetch
 326                         fi
 327                 ) > /${ldir}/_.prefetch.$b 2>&1
 328         done
 329         echo "Done" >> /${ldir}/_.prefetch
 330         )
 331 )
 332
 333 #######################################################################
 334
 335 do_world=true
 336 do_kernel=true
 337 do_prefetch=false
 338 use_pkg=""
 339 c_arg=""
 340
 341 set +e
 342 args=`getopt bc:hkpP:w $*`
 343 if [ $? -ne 0 ] ; then
 344         usage
 345 fi
 346 set -e
 347
 348 set -- $args
 349 for i
 350 do
 351         case "$i"
 352         in
 353         -b)
 354                 shift;
 355                 do_world=false
 356                 do_kernel=false
 357                 ;;
 358         -c)
 359                 c_arg=$2
 360                 if [ ! -f "$c_arg" ] ; then
 361                         echo "Cannot read $c_arg" 1>&2
 362                         usage
 363                 fi
 364                 . "$2"
 365                 shift
 366                 shift
 367                 ;;
 368         -h)
 369                 usage
 370                 ;;
 371         -k)
 372                 shift;
 373                 do_kernel=false
 374                 ;;
 375         -p)
 376                 shift;
 377                 use_pkg="-p"
 378                 ;;
 379         -P)
 380                 shift;
 381                 do_prefetch=true
 382                 distfile_cache=$1
 383                 shift;
 384                 ;;
 385         -w)
 386                 shift;
 387                 do_world=false
 388                 ;;
 389         --)
 390                 shift
 391                 break;
 392                 ;;
 393         esac
 394 done
 395
 396 #######################################################################
 397
 398 if [ "x$1" = "xchroot_script" ] ; then
 399         set -e
 400
 401         shift
 402
 403         before_ports_chroot
 404
 405         ports_build
 406
 407         exit 0
 408 fi
 409
 410 if [ "x$1" = "xfinal_chroot" ] ; then
 411         final_chroot
 412         exit 0
 413 fi
 414
 415 if [ $# -gt 0 ] ; then
 416         echo "$0: Extraneous arguments supplied"
 417         usage
 418 fi
 419
 420 #######################################################################
 421
 422 T0=`date +%s`
 423 echo $T0 $T0 > /tmp/_sb_log
 424
 425 [ ! -d ${SBMNT} ] && mkdir -p ${SBMNT}
 426
 427 if $do_prefetch ; then
 428         rm -rf /tmp/sysbuild/ports
 429         mkdir -p /tmp/sysbuild/ports
 430         ln -s ${distfile_cache} /tmp/sysbuild/ports/distfiles
 431         export PORTS_OPTS=CD_MOUNTPTS=/tmp/sysbuild
 432         ports_prefetch /tmp
 433         exit 0
 434 fi
 435
 436 log_it Unmount everything
 437 (
 438         ( cleanup )
 439         umount /freebsd/distfiles || true
 440         umount ${SBMNT}/freebsd/distfiles || true
 441         umount ${FREEBSD_PART} || true
 442         umount ${SBMNT}/freebsd || true
 443         umount ${SBMNT}/dev || true
 444         umount ${SBMNT} || true
 445         umount /dev/${TARGET_PART} || true
 446 ) # > /dev/null 2>&1
 447
 448 log_it Prepare running image
 449 mkdir -p /freebsd
 450 mount ${FREEBSD_PART} /freebsd
 451
 452 #######################################################################
 453
 454 if [ ! -d /freebsd/${PORTS_PATH} ] ;  then
 455         echo PORTS_PATH does not exist 1>&2
 456         exit 1
 457 fi
 458
 459 if [ ! -d /freebsd/${SRC_PATH} ] ;  then
 460         echo SRC_PATH does not exist 1>&2
 461         exit 1
 462 fi
 463
 464 log_it TARGET_PART $TARGET_PART
 465 sleep 5
 466
 467 rm -rf /usr/ports
 468 ln -s /freebsd/${PORTS_PATH} /usr/ports
 469
 470 rm -rf /usr/src
 471 ln -s /freebsd/${SRC_PATH} /usr/src
 472
 473 if $do_world ; then
 474         if [ "x${OBJ_PATH}" != "x" ] ; then
 475                 rm -rf /usr/obj
 476                 mkdir -p /freebsd/${OBJ_PATH}
 477                 ln -s /freebsd/${OBJ_PATH} /usr/obj
 478         else
 479                 rm -rf /usr/obj
 480                 mkdir -p /usr/obj
 481         fi
 482 fi
 483
 484 #######################################################################
 485
 486 for i in ${PORTS_WE_WANT}
 487 do
 488         (
 489         cd /usr/ports
 490         if [ ! -d $i ]  ; then
 491                 echo "Port $i not found" 1>&2
 492                 exit 2
 493         fi
 494         )
 495 done
 496
 497 export PORTS_WE_WANT
 498 export PORTS_OPTS
 499
 500 #######################################################################
 501
 502 log_it Prepare destination partition
 503 newfs -t -E -O2 -U /dev/${TARGET_PART} > /dev/null
 504 mount /dev/${TARGET_PART} ${SBMNT}
 505 mkdir -p ${SBMNT}/dev
 506 mount -t devfs devfs ${SBMNT}/dev
 507
 508 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 509         rm -rf /freebsd/${PORTS_PATH}/distfiles
 510         ln -s /freebsd/distfiles /freebsd/${PORTS_PATH}/distfiles
 511         mkdir -p /freebsd/distfiles
 512         mount  ${REMOTEDISTFILES} /freebsd/distfiles
 513 fi
 514
 515 log_it copy ports config files
 516 (cd / ; find var/db/ports -print | cpio -dumpv ${SBMNT} > /dev/null 2>&1)
 517
 518 log_it "Start prefetch of ports distfiles"
 519 ports_prefetch ${SBMNT} &
 520
 521 if $do_world ; then
 522         (
 523         cd /usr/src
 524         log_it "Buildworld"
 525         make ${JARG} -s buildworld ${SRCCONF} > ${SBMNT}/_.bw 2>&1
 526         )
 527 fi
 528
 529 if $do_kernel ; then
 530         (
 531         cd /usr/src
 532         log_it "Buildkernel"
 533         make ${JARG} -s buildkernel KERNCONF=$KERNCONF > ${SBMNT}/_.bk 2>&1
 534         )
 535 fi
 536
 537
 538 log_it Installworld
 539 (cd /usr/src && make ${JARG} installworld DESTDIR=${SBMNT} ${SRCCONF} ) \
 540         > ${SBMNT}/_.iw 2>&1
 541
 542 log_it distribution
 543 (cd /usr/src/etc && make -m /usr/src/share/mk distribution DESTDIR=${SBMNT} ${SRCCONF} ) \
 544         > ${SBMNT}/_.dist 2>&1
 545
 546 log_it Installkernel
 547 (cd /usr/src && make ${JARG} installkernel DESTDIR=${SBMNT} KERNCONF=$KERNCONF ) \
 548         > ${SBMNT}/_.ik 2>&1
 549
 550 if [ "x${OBJ_PATH}" != "x" ] ; then
 551         rmdir ${SBMNT}/usr/obj
 552         ln -s /freebsd/${OBJ_PATH} ${SBMNT}/usr/obj
 553 fi
 554
 555 log_it Wait for ports prefetch
 556 log_it "(Tail ${SBMNT}/_.prefetch for progress)"
 557 wait
 558
 559 log_it Move filesystems
 560
 561 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 562         umount /freebsd/distfiles
 563 fi
 564 umount ${FREEBSD_PART} || true
 565 mkdir -p ${SBMNT}/freebsd
 566 mount ${FREEBSD_PART} ${SBMNT}/freebsd
 567 if [ "x${REMOTEDISTFILES}" != "x" ] ; then
 568         mount  ${REMOTEDISTFILES} ${SBMNT}/freebsd/distfiles
 569 fi
 570
 571 rm -rf ${SBMNT}/usr/ports || true
 572 ln -s /freebsd/${PORTS_PATH} ${SBMNT}/usr/ports
 573
 574 rm -rf ${SBMNT}/usr/src || true
 575 ln -s /freebsd/${SRC_PATH} ${SBMNT}/usr/src
 576
 577 log_it Build and install ports
 578
 579 # Make sure fetching will work in the chroot
 580 if [ -f /etc/resolv.conf ] ; then
 581         log_it copy resolv.conf
 582         cp /etc/resolv.conf ${SBMNT}/etc
 583         chflags schg ${SBMNT}/etc/resolv.conf
 584 fi
 585
 586 if [ -f /etc/localtime ] ; then
 587         log_it copy localtime
 588         cp /etc/localtime ${SBMNT}/etc
 589 fi
 590
 591 log_it ldconfig in chroot
 592 chroot ${SBMNT} sh /etc/rc.d/ldconfig start
 593
 594 log_it before_ports
 595 (
 596         before_ports
 597 )
 598
 599 log_it fixing fstab
 600 sed "/[         ]\/[    ]/s;^[^         ]*[     ];/dev/${TARGET_PART}   ;" \
 601         /etc/fstab > ${SBMNT}/etc/fstab
 602
 603 log_it build ports
 604
 605 cp $0 ${SBMNT}/root
 606 cp /tmp/_sb_log ${SBMNT}/tmp
 607 b=`basename $0`
 608 if [ "x$c_arg" != "x" ] ; then
 609         cp $c_arg ${SBMNT}/root
 610         chroot ${SBMNT} sh /root/$0 -c /root/`basename $c_arg` $use_pkg chroot_script
 611 else
 612         chroot ${SBMNT} sh /root/$0 $use_pkg chroot_script
 613 fi
 614 cp ${SBMNT}/tmp/_sb_log /tmp
 615
 616 log_it create all mountpoints
 617 grep -v '^[     ]*#' ${SBMNT}/etc/fstab |
 618 while read a b c
 619 do
 620         mkdir -p ${SBMNT}/$b
 621 done
 622
 623 if [ "x$SERCONS" != "xfalse" ] ; then
 624         log_it serial console
 625         echo " -h" > ${SBMNT}/boot.config
 626         sed -i "" -e /ttyd0/s/off/on/ ${SBMNT}/etc/ttys
 627         sed -i "" -e /ttyu0/s/off/on/ ${SBMNT}/etc/ttys
 628         sed -i "" -e '/^ttyv[0-8]/s/    on/     off/' ${SBMNT}/etc/ttys
 629 fi
 630
 631 log_it move dist config files "(expect warnings)"
 632 (
 633         cd ${SBMNT}
 634         mkdir root/configfiles_dist
 635         find ${CONFIGFILES} -print | cpio -dumpv root/configfiles_dist
 636 )
 637
 638 log_it copy live config files
 639 (cd / && find ${CONFIGFILES} -print | cpio -dumpv ${SBMNT})
 640
 641 log_it final_root
 642 ( final_root )
 643 log_it final_chroot
 644 cp /tmp/_sb_log ${SBMNT}/tmp
 645 if [ "x$c_arg" != "x" ] ; then
 646         chroot ${SBMNT} sh /root/$0 -c /root/`basename $c_arg` final_chroot
 647 else
 648         chroot ${SBMNT} sh /root/$0 final_chroot
 649 fi
 650 cp ${SBMNT}/tmp/_sb_log /tmp
 651 log_it "Check these messages (if any):"
 652 grep '^Stop' ${SBMNT}/_* || true
 653 log_it DONE
 654 echo "Now you probably want to:"
 655 echo "    $GPART_SUGGESTION"
 656 echo "    shutdown -r now"