Merge branch 'joyent/feature/pbulk/trunk' into joyent/release/trunk

mk/pbulk/pbulk-index.mk

@@ -117,7 +117,7 @@ pbulk-index-item:
 	@echo "USE_DESTDIR="${_USE_DESTDIR:Q}
 	@echo "BOOTSTRAP_PKG="${BOOTSTRAP_PKG}
 	@echo "USERGROUP_PHASE="${USERGROUP_PHASE:Q}
-	@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:O:u:Q}
+	@echo "SCAN_DEPENDS="${_PBULK_SCAN_DEPENDS:N${PKGBUILD_BASEDIR:U/nonexistent}/*:O:u:Q}
 .if defined(_PBULK_MULTI_NEEDED)
 	@printf "MULTI_VERSION="
 .for _t in ${_PBULK_MULTI_NEEDED}

pkgtools/pbulk/Makefile

@@ -6,7 +6,7 @@ COMMENT=	Modular bulk build framework
 .include "../../pkgtools/pbulk/Makefile.common"
 
 USE_TOOLS+=	awk:run bzip2:run digest:run gzip:run make:run \
-		mail:run sed:run tar:run
+		mail:run sed:run tar:run xz:run
 DEPENDS+=	rsync-[0-9]*:../../net/rsync
 DEPENDS+=	pbulk-base>=0.38:../../pkgtools/pbulk-base
 
@@ -25,7 +25,7 @@ SUBST_STAGE.tools=	pre-configure
 SUBST_MESSAGE.tools=	Fixing references to tools
 SUBST_FILES.tools=	pbulk.conf scripts/build scripts/build-client-start \
 	scripts/bulkbuild scripts/bulkbuild-rebuild scripts/bulkbuild-restart \
-	scripts/client-clean scripts/client-prepare \
+	scripts/chroot scripts/client-clean scripts/client-prepare \
 	scripts/pkg-build scripts/pkg-up-to-date scripts/pre-build \
 	scripts/report scripts/scan scripts/scan-client-start scripts/upload \
 	scripts/compute-packages.awk scripts/create-broken-graph.awk \
@@ -34,7 +34,7 @@ SUBST_FILES.tools=	pbulk.conf scripts/build scripts/build-client-start \
 SUBST_VARS.tools=	AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
 			PBULK_CONFIG PBULK_CONFIG_VERSION PKG_ADD_CMD \
 			PKG_ADMIN_CMD PKG_DELETE_CMD PKG_INFO_CMD PREFIX \
-			SED SH TAR TARGET_MAKE
+			SED SH TAR TARGET_MAKE XZ
 
 CONF_FILES+=	share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf
 

pkgtools/pbulk/PLIST

@@ -4,6 +4,7 @@ bin/bulkbuild-rebuild
 bin/bulkbuild-restart
 libexec/pbulk/build
 libexec/pbulk/build-client-start
+libexec/pbulk/chroot
 libexec/pbulk/client-clean
 libexec/pbulk/client-prepare
 libexec/pbulk/compute-packages

pkgtools/pbulk/files/pbulk/pbulk.conf

@@ -26,6 +26,23 @@ build_clients="192.168.75.21 192.168.75.22 192.168.75.23 192.168.75.24"
 master_port_scan=${master_ip}:2001
 master_port_build=${master_ip}:2002
 
+# Create chroots for each phase.  This allows concurrent build/scan on the
+# same host.  chroot_create/chroot_delete are scripts which take a single
+# argument which is the chroot directory to create/delete, and chroot_dir is
+# used as the prefix for the name of the chroot directory.
+#
+# To use chroots on a single host, set master_mode=yes and leave both
+# build_clients and scan_clients unset or empty.
+#
+#chroot_create=/path/to/mksandbox
+#chroot_delete=/path/to/rmsandbox
+#chroot_dir=/chroot/pkgsrc
+
+# To enable concurrent chroots, set build_chroots/scan_chroots > 1.
+#
+#build_chroots=4
+#scan_chroots=8
+
 # Some flags for the programs that publish the binary packages and the
 # build report. If you want to disable the publishing at all, see below.
 #
@@ -36,6 +53,15 @@ report_rsync_target="pkgsrc@192.168.75.1:/public/reports/current/DragonFly-1.8"
 report_subject_prefix="pkgsrc"
 report_recipients="pkgsrc-bulk@netbsd.org"
 
+# Example variables used by an optional pre-upload script to ensure certain
+# criteria are met before publishing.
+#
+# Set a minimum number of successful packages.
+#pkg_rsync_min=10000
+#
+# Ensure all packages listed in the file built successfully.
+#pkg_rsync_required_pkgs=/required_pkgs
+
 # An archive containing the pkgsrc binary tree after bootstrapping.
 #
 bootstrapkit=/usr/pkgsrc/bootstrap/bootstrap.tar.gz
@@ -102,6 +128,7 @@ neato=@NEATO@
 rsync=@PREFIX@/bin/rsync
 sed=@SED@
 tar=@TAR@
+xz=@XZ@
 
 loc=${bulklog}/meta
 
@@ -119,6 +146,7 @@ keep_prefix=no
 client_prepare=@PREFIX@/libexec/pbulk/client-prepare
 client_prepare_action=@PREFIX@/libexec/pbulk/client-clean
 client_prepare_chroot=:
+chroot_script=@PREFIX@/libexec/pbulk/chroot
 pkg_up_to_date_script=@PREFIX@/libexec/pbulk/pkg-up-to-date
 pbuild_script=@PREFIX@/libexec/pbulk/pkg-build
 pbuild_start_script=@PREFIX@/libexec/pbulk/build-client-start
@@ -133,6 +161,8 @@ script_phase_pre_build=@PREFIX@/libexec/pbulk/pre-build
 script_phase_build=@PREFIX@/libexec/pbulk/build
 script_phase_report=@PREFIX@/libexec/pbulk/report
 script_phase_scan=@PREFIX@/libexec/pbulk/scan
+# Optional pre-upload script called from upload.
+#script_phase_pre_upload=/pre-upload
 script_phase_upload=@PREFIX@/libexec/pbulk/upload
 
 # Arguments are buildlog and pkgname
@@ -142,3 +172,7 @@ sync_buildlog=:
 sync_package=default_sync_package
 # PKG_SUFX as set in mk.conf
 pkg_sufx=.tgz
+
+# Optional local override scripts.
+#
+#client_clean_local=/client_clean_local

pkgtools/pbulk/files/pbulk/pscan/client.c

@@ -80,6 +80,30 @@ find_full_tree_client(int fd, const char *bmake_path)
 		if (cat_len == 0)
 			break;
 
+		/*
+		 * Check for SUBDIR including a '/', if so then it has been
+		 * passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
+		 * and should be added as a package path directly.
+		 */
+		if (memchr(cat, '/', cat_len) != NULL) {
+			if (len_pkgs == allocated_pkgs) {
+				if (allocated_pkgs == 0) {
+					allocated_pkgs = 1024;
+					pkgs = xmalloc(sizeof(*pkgs) *
+							allocated_pkgs);
+				} else {
+					allocated_pkgs *= 2;
+					pkgs = xrealloc(pkgs,
+							sizeof(*pkgs) * allocated_pkgs);
+				}
+			}
+			pkgs[len_pkgs] = xasprintf("%.*s", (int)cat_len, cat);
+			len_pkgs_data += strlen(pkgs[len_pkgs]) + 1;
+			++len_pkgs;
+			cat += cat_len;
+			continue;
+		}
+
 		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
 		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
 		free(cat_path);

pkgtools/pbulk/files/pbulk/pscan/jobs.c

@@ -278,6 +278,16 @@ add_job(const char *cat, size_t cat_len, const char *dir, size_t dir_len)
 	free(location);
 }
 
+void
+add_job_pkgpath(const char *pkgpath, size_t pkgpath_len)
+{
+	char *location;
+
+	location = xasprintf("%.*s", (int)pkgpath_len, pkgpath);
+	add_job_full(location);
+	free(location);
+}
+
 void
 add_job_full(const char *location)
 {

pkgtools/pbulk/files/pbulk/pscan/pscan.c

@@ -225,6 +225,17 @@ find_full_tree(void)
 		if (cat_len == 0)
 			break;
 
+		/*
+		 * Check for SUBDIR including a '/', if so then it has been
+		 * passed via USER_ADDITIONAL_PKGS from the top-level Makefile,
+		 * and should be added as a package path directly.
+		 */
+		if (memchr(cat, '/', cat_len) != NULL) {
+			add_job_pkgpath(cat, cat_len);
+			cat += cat_len;
+			continue;
+		}
+
 		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
 		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
 		free(cat_path);

pkgtools/pbulk/files/pbulk/pscan/pscan.h

@@ -56,6 +56,7 @@ void		 client_mode(const char *, const char *);
 void		 master_mode(const char *, const char *);
 
 void		 add_job(const char *, size_t, const char *, size_t);
+void		 add_job_pkgpath(const char *, size_t);
 void		 add_job_full(const char *);
 struct scan_job	*get_job(void);
 void		 process_job(struct scan_job *, enum job_state);

pkgtools/pbulk/files/pbulk/scripts/Makefile

@@ -2,7 +2,7 @@
 
 SCRIPTS=	build build-client-start bulkbuild bulkbuild-rebuild \
 		bulkbuild-restart \
-		client-clean client-prepare \
+		chroot client-clean client-prepare \
 		pkg-build pkg-up-to-date pre-build report \
 		scan scan-client-start upload \
 		compute-packages.awk create-broken-graph.awk \

pkgtools/pbulk/files/pbulk/scripts/build

@@ -30,7 +30,7 @@
 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
@@ -46,7 +46,9 @@ case "${master_mode}" in
 	${pbuild} -r ${loc}/pbuild -v -b ${pbuild_script} ${loc}/presolve ${loc}/success ${loc}/error
 	;;
 [yY][eE][sS])
+	[ -n "${chroot_dir}" ] && ${chroot_script} build create ${chroot_dir}
 	${pbuild} -r ${loc}/pbuild -I ${pbuild_start_script} -m ${master_port_build} -v ${loc}/presolve ${loc}/success ${loc}/error
+	[ -n "${chroot_dir}" ] && ${chroot_script} build delete ${chroot_dir}
 	;;
 *)
 	echo "master_mode must be either yes or no."
@@ -59,16 +61,37 @@ BUILD_END_EPOCHE=%s' >> ${loc}/status
 
 echo "Building pkg_summary..."
 cd ${packages}/All
-sed 's/$/.tgz/' < ${loc}/success | sort | xargs ${pkg_info} -X | ${gzip} -c > pkg_summary.gz
-${gzip} -dc < pkg_summary.gz | ${bzip2} -c > pkg_summary.bz2
+{
+	sed -e 's,^,../pkginfo/,' -e 's/$/.pkglist/' < ${loc}/success \
+	    | sort | xargs cat 2>/dev/null > pkg_files || true
+	cat pkg_files | ${gzip} -c > pkg_files.gz &
+	cat pkg_files | ${bzip2} -c > pkg_files.bz2 &
+	cat pkg_files | ${xz} -zc > pkg_files.xz &
+	wait
+	rm -f pkg_files
+} &
+{
+	sed -e 's,^,../pkginfo/,' -e 's/$/.pkginfo/' < ${loc}/success \
+	    | sort | xargs cat 2>/dev/null > pkg_summary || true
+	cat pkg_summary | ${gzip} -c > pkg_summary.gz &
+	cat pkg_summary | ${bzip2} -c > pkg_summary.bz2 &
+	cat pkg_summary | ${xz} -zc > pkg_summary.xz &
+	wait
+	rm -f pkg_summary
+} &
+wait
 
 if [ "${checksum_packages}" != "no" ] && \
    [ "${checksum_packages}" != "NO" ]; then
 	echo "Building SHA512..."
 	cd ${packages}
 	{
-		echo "All/pkg_summary.bz2"
-		echo "All/pkg_summary.gz"
-		sed 's|^\(.*\)$|All/\1.tgz|' < ${loc}/success
-	} | sort | xargs ${digest} SHA512 | ${bzip2} -c > SHA512.bz2
+		${digest} SHA512 All/pkg_files.bz2
+		${digest} SHA512 All/pkg_files.gz
+		${digest} SHA512 All/pkg_files.xz
+		${digest} SHA512 All/pkg_summary.bz2
+		${digest} SHA512 All/pkg_summary.gz
+		${digest} SHA512 All/pkg_summary.xz
+		sed -e 's,^,checksums/,' -e 's/$/.sha512/' < ${loc}/success | xargs cat 2>/dev/null
+	} | sort | ${bzip2} -c > SHA512.bz2
 fi

pkgtools/pbulk/files/pbulk/scripts/build-client-start

@@ -1,7 +1,7 @@
 #!@SH@
 # $NetBSD: build-client-start,v 1.5 2016/12/18 22:59:35 joerg Exp $
 
-. @PBULK_CONFIG@
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
@@ -29,7 +29,13 @@ for client in ${build_clients}; do
 		;;
 	esac
 	if [ -z "$path" ]; then
-		ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
+		if [ -n "${chroot_create}" -a "${chroot_dir}" ]; then
+			ssh $port $client "for dir in ${chroot_dir}-build*; do
+				PBULK_CONF=${PBULK_CONF} chroot \${dir} @SH@ -c \"${pbuild} -c ${master_port_build} -b ${pbuild_script}\" &
+			done" &
+		else
+			ssh $port $client "${pbuild} -c ${master_port_build} -b ${pbuild_script}" &
+		fi
 	else
 		chroot "$path" ${pbuild} -c ${master_port_build} -b ${pbuild_script} &
 	fi

pkgtools/pbulk/files/pbulk/scripts/bulkbuild

@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild,v 1.1.1.1 2007/06/19 19:49:59 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "usage: $0 [-h] [<config file>]"
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 

pkgtools/pbulk/files/pbulk/scripts/bulkbuild-rebuild

@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild-rebuild,v 1.3 2008/09/16 18:21:30 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "Usage: $0 [-h] [<config file>] package ..."
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e
 
@@ -33,7 +47,7 @@ build_pkg() {
 }
 
 if [ $# = 0 ]; then
-	echo "Usage: $0 package ..."
+	echo "Usage: $0 [-h] [<config file>] package ..."
 fi
 
 while [ $# != 0 ]; do

pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart

@@ -1,7 +1,21 @@
 #!@SH@
 # $NetBSD: bulkbuild-restart,v 1.4 2016/12/18 22:38:16 joerg Exp $
 
-. @PBULK_CONFIG@
+if [ "$1" = "-h" ]; then
+	echo "usage: $0 [-h] [<config file>]"
+	exit 2
+elif [ -f "$1" ]; then
+	case "$1" in
+	/*)
+		export PBULK_CONF="$1"; shift
+		;;
+	*)
+		export PBULK_CONF="$(pwd)/$1"; shift
+		;;
+	esac
+fi
+
+. ${PBULK_CONF:-@PBULK_CONFIG@}
 
 set -e