configure

#!/usr/bin/bash
#
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright 2019 Joyent, Inc.
#
shopt -s xpg_echo
unalias -a

PATH=/usr/bin:/usr/sbin:/sbin:/opt/local/bin:/opt/local/sbin
export PATH

conf_arg0=$(basename $0)
conf_arg0_dir=$(dirname $0)
conf_root=$PWD
conf_ips=
conf_priv="pfexec"
conf_pkgsrcurl="https://download.joyent.com/pub/build/pkgsrc"

#
# Build defaults which are unlikely to change, but may be overridden by
# configure-build.
#
source default.configure-build
if [[ -f configure-build ]]; then
	source configure-build || fatal "failed to source configure-build"
fi

#
# We configure the projects framework using a 'configure-projects' file. This
# is a plain text file which contains the project name, the branch we wish to
# build it at, and either a URL to a git repository or a keyword to denote
# which git repository to use, either 'cr' or 'origin'. See create_projects.
#
read -r -d '' configure_projects < default.configure-projects

function warn
{
	local msg="$*"
	[[ -z "$msg" ]] && msg="failed"
	echo "$conf_arg0: $msg" >&2
}

function fatal
{
	local msg="$*"
	[[ -z "$msg" ]] && msg="failed"
	echo "$conf_arg0: $msg" >&2
	exit 1
}

function check_loc
{
	[[ "$(readlink -f "$PWD")" != "$(readlink -f "$conf_arg0_dir")" ]] && \
	    fatal "configure must be run from inside $conf_arg0_dir"
	[[ "$(uname -s)" != "SunOS" ]] && fatal "build only works on SunOS"
	if [[ "$(uname -v)" =~ "joyent_" ]]; then
		[[ "$(zonename)" == "global" ]] && fatal \
		    "building in a GZ on SmartOS is not supported"
	else
		echo "Not on SmartOS, assuming IPS and sudo"
		conf_ips=1
		conf_priv="sudo"
	fi
}

#
# Checkout a project at the specific branch if the repository doesn't exist,
# or isn't already at the required branch. Warn if the repository contains
# uncommitted contents.
#
function checkout_project
{
	local dir="$1"
	local branch="$2"
	local url="$3"

	echo "Using $url at $branch in $dir"

	if [[ ! -d "$dir" ]]; then
		git clone "$url" "${dir}" || fatal "unable to clone $dir"
	fi

	local cur=$(git -C ${dir} branch | grep "^*" | \
	    awk 'BEGIN{ FS=" " } { print $2 }')

	# If our repository is already at the desired branch, then do nothing,
	# intentionally preserving whatever's in the repository.
	if [[ "$cur " != "$branch" ]]; then
		git -C ${dir} checkout ${branch}
		if [[ $? -ne 0 ]]; then

			# If this is a remote branch, say 'refs/changes/34/1234/1',
			# then we might need to fetch it before being able to check it out.
			echo "unable to checkout $branch, attempting to fetch it ..."

			# Use a local branch name without '/' chars so a detached head
			# doesn't confuse our standard parsing of 'git describe' output
			local branch_desc=$(echo $branch | sed -e 's/\//_/g')

			# We assume the remote is named 'origin', regardless as to whether
			# it's from cr.joyent.us or github. Our Jenkins instance doesn't
			# have multi-SCM support, so ./projects won't have 'cr' remotes.
			git -C ${dir} fetch origin ${branch}:${branch_desc} || \
				fatal "unable to fetch ${branch} from origin"

			git -C ${dir} checkout ${branch_desc} || \
				fatal "unable to checkout ${branch_desc}"
		fi
	fi

	git -C ${dir} describe --all --long --dirty | egrep -e '-dirty$' &&
	    echo "Warning: ${dir} contains uncommitted changes!"
	return 0
}

#
# Read a flat file called 'configure-projects' of the form:
# <project name>:<project branch>:[project git repo URL or path]
#
# The special tokens 'cr' or 'origin' can be used in place of a full git
# repo URL to denote either standard github.com or gerrit URLs for that
# project. If no URL is given, we default to github.com.
#
# Once the configure-projects file has been read (or we've parsed the
# defaults), invoke checkout_project for each one.
#
function create_projects
{
	local dir="projects/local"
	local git_stem="https://github.com/joyent"

	[[ -d "$dir" ]] || mkdir -p "$dir" || fatal "failed to create $dir"

	if [[ -f "configure-projects" ]]; then
		# note we're not attempting to merge configure-projects with the default
		read -r -d '' configure_projects  < configure-projects
		echo "Info: using the following configure-projects file:"
		echo "$configure_projects"
	else
		echo "Info: no configure-projects file found, using defaults."
	fi
	echo "$configure_projects" | grep -v '^#' | while read -r line; do
		local project_dir=$(echo $line | cut -d: -f 1 | sed -e 's/ //g')
		local project_name=$(basename $project_dir)
		local git_name="${project_name}.git"
		local project_branch=$(echo $line | cut -d: -f 2 | sed -e 's/ //g')
		local project_url=$(echo $line | cut -d: -f 3- | sed -e 's/ //g')

		# special cases for some projects
		case ${project_name} in
		'kvm'|'kvm-cmd')
			git_name="illumos-${project_name}.git"
			;;
		'illumos')
			git_name="illumos-joyent.git"
			;;
		'ur-agent')
			git_name="sdc-ur-agent.git"
			;;
		esac

		# process URL keywords
		case $project_url in
		'cr')
			project_url="https://cr.joyent.us/joyent/$git_name"
			;;
		'origin'|'')
			project_url="$git_stem/$git_name"
			;;
		esac
		checkout_project "projects/$project_dir" "$project_branch" \
		    "$project_url"
	done || fatal "Error encountered when creating projects directory"
}

function install_pkgin
{
	local pkglist=

	#
	# Newer images have a pkgin repo with a 'build-essential' package
	# that pulls in several of the packages we need.  This is useful since
	# it allows us to not worry about things like gcc46 vs. gcc47 etc.
	#
	$conf_priv pkgin -f update || fatal "failed to update pkgsrc repository"
	if pkgin av build-essential | grep ^build-essential >/dev/null; then
		pkglist="build-essential"
	else
		pkglist="gmake binutils autoconf automake bison m4"
		pkglist="$pkglist libtool-base gcc-compiler"
	fi

	pkglist="$pkglist flex libxslt openjdk7 nodejs"
	pkglist="$pkglist p5-XML-Parser gettext python27 py27-expat"
	pkglist="$pkglist coreutils gsed pkg_alternatives cdrtools"
	pkglist="$pkglist py27-sqlite3 nasm pigz gcc49"

	for pkg in $pkglist; do
		if ! pkg_info -qe $pkg; then
			$conf_priv pkgin -y install $pkg || fatal \
		    "failed to install $pkg"
		fi
	done

	#
	# Packages not in pkgsrc
	#
	for pkg in dmake sgstools rpcgen astmsgtools; do
		if ! pkg_info -qe $pkg; then
			curl -k "$conf_pkgsrcurl/$pkg.tgz" -o \
			    /var/tmp/$pkg.tgz || fatal \
			    "failed to fetch $pkg.tgz"

			#
			# We set the machine to be i386 to force pkg_add
			# to install the package even if we are on x86_64.
			#
			$conf_priv pkg_add -m i386 -C /dev/null \
			    /var/tmp/$pkg.tgz || fatal \
			    "failed to pkg_add $pkg"
			rm -f /var/tmp/$pkg.tgz
		fi
	done

	#
	# There is a bug in pkgsrc (PKGSRC-1338) in that installing a package
	# as i386 on x86_64 results in some temporary confusion in the
	# database.  To workaround this, we perform a meaningless pkgin
	# operation to get the database back into a correct state with respect
	# to our true machine architecture.
	#
	pkgin -y list > /dev/null
}

function install_ips
{
	fatal "Building on ips based systems has yet to be implemented"
}

function install_packages
{
	if [[ -z "$conf_ips" ]]; then
		install_pkgin
	else
		install_ips
	fi
	[[ $? -eq 0 ]] || fatal "failed to install packages"
}

function fetch_studio
{
	[[ -f "/opt/SUNWspro/prod/bin/cc" ]] && return
	[[ -z "$SUNW_SPRO12_URL" ]] && fatal \
	    "SUNW_SPRO12_URL not set in configure.*"
	curl -k "$SUNW_SPRO12_URL" | bzcat | $conf_priv tar xf - -C /opt
	[[ $? -eq 0 ]] || fatal "failed to fetch Studio"
	[[ -e /opt/SUNspro/sunstudio12.1 ]] && return
	$conf_priv ln -s /opt/SUNWspro /opt/SUNWspro/sunstudio12.1 || fatal \
	    "cannot create Studio symlink"
}

function fetch_adjuncts
{
	local tgz
	[[ -z "$ILLUMOS_ADJUNCT_TARBALL_URL" ]] && fatal \
	    "ILLUMOS_ADJUNCT_TARBALL_URL missing from configure"
	tgz=$(curl -k "$ILLUMOS_ADJUNCT_TARBALL_URL" | grep href | tail -n1 | \
	    cut -d '"' -f2)
	[[ -z "$tgz" ]] && fatal \
	    "Unable to get adjuncts from ILLUMOS_ADJUNCT_TARBALL_URL"
	curl -kO $ILLUMOS_ADJUNCT_TARBALL_URL/$tgz
	[[ $? -eq 0 ]] || fatal "failed to fetch adjuncts tarball"
}

function setup_overlays
{
	local of="overlay/order"
	[[ -z "$OVERLAYS" ]] && fatal \
	    "OVERLAYS variable missing from configure.*"
	rm -f $of
	touch $of || fail "failed to create $of"
	for o in $OVERLAYS; do
		echo -n "$conf_root/overlay/$o " >> $of || fatal \
		    "failed to append to $of"
	done
}

function fetch_closed
{
	local ildir="projects/illumos"
	local cld="on-closed-bin.i386.tar.bz2"
	local clnd="on-closed-bin-nd.i386.tar.bz2"
	[[ -z "$ON_CLOSED_BINS_URL" ]] && fatal \
	    "missing ON_CLOSED_BINS_URL from configure"
	[[ -z "$ON_CLOSED_BINS_ND_URL" ]] && fatal \
	    "missing ON_CLOSED_BINS_ND_URL from configure"
	if [[ ! -f $ildir/$cld ]]; then
		curl -k $ON_CLOSED_BINS_URL -o $ildir/$cld || fatal \
		"failed to fetch closed bins (debug)"
	fi
	if [[ ! -f $ildir/$clnd ]]; then
		curl -k $ON_CLOSED_BINS_ND_URL -o $ildir/$clnd || fatal \
		"failed to fetch closed bins (non-debug)"
	fi
	cd $ildir >/dev/null 2>&1 || fatal "failed to cd into $ildir"
	tar xpjf $cld || fatal \
	    "failed to extract closed bins (debug)"
	tar xpjf $clnd || fatal \
	    "failed to extract closed bins (non-debug)"
	cd - >/dev/null 2>&1
}

function generate_env
{
	#
	# Note: since a nightly clobber build removes the whole proto, and we're
	# installing non-Illumos things there, we always use the -i option, and
	# ILLUMOS_CLOBBER is handled separately in Makefile.
	#
	local nopts="-CimMNnt"
	local lprefix

	case $ILLUMOS_ENABLE_DEBUG in
	exclusive)
		nopts="${nopts}DF" ;;
	yes)
		nopts="${nopts}D" ;;
	no|"")
		;;
	*)
		echo "unknown debug option $ILLUMOS_ENABLE_DEBUG" >&2
		exit 2 ;;
	esac

	GNUC_ROOT="$conf_root/proto.strap/usr/gcc/$PRIMARY_COMPILER_VER"
	PRIMARY_CC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/gcc,gnu"
	PRIMARY_CCC="gcc$PRIMARY_COMPILER_VER,$GNUC_ROOT/bin/g++,gnu"

	SHADOW_CCS=
	SHADOW_CCCS=

	IFS=,
	for cc in $SHADOW_COMPILERS; do
		gcc_ver=$(echo $cc | sed 's/^gcc//')
		root="$conf_root/proto.strap/usr/gcc/$gcc_ver"
		SHADOW_CCS+=" gcc${gcc_ver},$root/bin/gcc,gnu"
		SHADOW_CCCS+=" gcc${gcc_ver},$root/bin/g++,gnu"
	done
	unset IFS

	lprefix=$(echo $conf_root | tr / _)
	[[ $? -eq 0 ]] || fatal "failed to create lock prefix"

cat > "projects/illumos/illumos.sh" <<EOF
NIGHTLY_OPTIONS="$nopts";			export NIGHTLY_OPTIONS
GATE="${RELEASE_VER}";				export GATE
CODEMGR_WS="$conf_root/projects/illumos";	export CODEMGR_WS
MAX_JOBS=128
maxjobs() {
        ncpu=\`kstat -p cpu_info:::state | grep -c on-line\`
        if [[ \$(( \$ncpu + 2 )) -lt \${MAX_JOBS} ]]; then
                expr \$ncpu + 2
        else
                printf "%d\n" \${MAX_JOBS}
        fi
}
DMAKE_MAX_JOBS=\`maxjobs\`;			export DMAKE_MAX_JOBS

PARENT_WS="";					export PARENT_WS
CLONE_WS="http://hg.illumos.org/illumos-gate"	export CLONE_WS
STAFFER="nobody";				export STAFFER
BUILD_PROJECT="";				export BUILD_PROJECT
LOCKNAME="\`whoami\`_${lprefix}_nightly.lock";	export LOCKNAME
ATLOG="\$CODEMGR_WS/log";			export ATLOG
LOGFILE="\$ATLOG/nightly.log";			export LOGFILE
MACH=\`uname -p\`;				export MACH
ON_CLOSED_BINS="\$CODEMGR_WS/closed";		export ON_CLOSED_BINS
REF_PROTO_LIST="\$PARENT_WS/usr/src/proto_list_\${MACH}";
						export REF_PROTO_LIST
ROOT="$conf_root/proto";			export ROOT
ADJUNCT_PROTO="$conf_root/proto.strap";		export ADJUNCT_PROTO
NATIVE_ADJUNCT="/opt/local";			export NATIVE_ADJUNCT
SRC="\$CODEMGR_WS/usr/src";			export SRC
VERSION="\$GATE";				export VERSION
PARENT_ROOT="$conf_root/proto";			export PARENT_ROOT
PARENT_TOOLS_ROOT="\$PARENT_WS/usr/src/tools/proto/root_\$MACH-nd"
						export PARENT_TOOLS_ROOT
PKGARCHIVE="\${CODEMGR_WS}/packages/\${MACH}/nightly";
						export PKGARCHIVE
PKGPUBLISHER_REDIST="${PUBLISHER}";		export PKGPUBLISHER_REDIST
MAKEFLAGS=k;					export MAKEFLAGS
UT_NO_USAGE_TRACKING="1";			export UT_NO_USAGE_TRACKING
MULTI_PROTO="no";				export MULTI_PROTO
BUILD_TOOLS="\$SRC/tools/proto/root_\${MACH}-nd/opt";
						export BUILD_TOOLS
SPRO_ROOT=/opt/SUNWspro;			export SPRO_ROOT
SPRO_VROOT=\$SPRO_ROOT;				export SPRO_VROOT
GNU_ROOT="$conf_root/proto.strap/usr/gnu"	export GNU_ROOT
# Use GCC as the primary compiler
__GNUC="";					export __GNUC
# Use GCC4 or later specific flags
__GNUC4="";					export __GNUC4
# root of \$PRIMARY_CC
GNUC_ROOT="$GNUC_ROOT";				export GNUC_ROOT
PRIMARY_CC="$PRIMARY_CC";			export PRIMARY_CC
PRIMARY_CCC="$PRIMARY_CCC";			export PRIMARY_CCC
SHADOW_CCS="$SHADOW_CCS";			export SHADOW_CCS
SHADOW_CCCS="$SHADOW_CCCS";			export SHADOW_CCCS
JAVA_ROOT=/opt/local/java/openjdk7;		export JAVA_ROOT
FLEX=/opt/local/bin/flex;			export FLEX
GNUXGETTEXT=/opt/local/bin/xgettext;		export GNUXGETTEXT
PYTHON=/opt/local/bin/python2.7;		export PYTHON
BUILDPY3='#';					export BUILDPY3
BUILDPY3TOOLS='#';				export BUILDPY3TOOLS
#
# Note that this isn't the strap perl: that only really exists for the benefit
# of projects/illumos-extra/openssl
#
PERL=/opt/local/bin/perl;			export PERL
ELFDUMP=/usr/bin/elfdump;			export ELFDUMP
LORDER=/usr/bin/lorder;				export LORDER
MCS=/usr/bin/mcs;				export MCS
NM=/usr/bin/nm;					export NM
STRIP=/usr/bin/strip;				export STRIP
TSORT=/usr/bin/tsort;				export TSORT
AR=/usr/bin/ar;					export AR
#
# We override $MAKE in ./tools/build_illumos so we can properly
# bootstrap the tools build.
#
if [[ -z "\$MAKE" ]]; then
MAKE="\$SRC/tools/proto/root_i386-nd/opt/onbld/bin/i386/dmake";	export MAKE
fi
LEX=/opt/local/bin/lex;				export LEX
YACC=/opt/local/bin/yacc;			export YACC
BISON=/opt/local/bin/bison;			export BISON
GM4=/opt/local/bin/gm4;				export GM4
RPCGEN=/opt/local/bin/rpcgen;			export RPCGEN
ASTBINDIR=/opt/local/ast/bin;			export ASTBINDIR
LD_TOXIC_PATH="\$ROOT/lib:\$ROOT/usr/lib";	export LD_TOXIC_PATH
EOF
	[[ $? -eq 0 ]] || fatal "failed to write illumos nightly env file"
}

#
# This construction lets us place a block of text verbatim into $usage.
#
read -r -d '' usage <<EOF
./configure [options]:
	-c
		clobber Illumos before each build [default: no]
	-d
		build Illumos in DEBUG mode only [default: no]
	-h
		this message
	-p gcc4
		primary compiler version [default: gcc4]
	-P password
		platform root password [default: randomly chosen]
	-r
		full strap build (no cache) [default: no]
	-s gcc7
		shadow compilers, comma delimited (gcc7,gcc#) [default: none]
EOF

while getopts "cdhp:P:rs:" arg; do
	case $arg in
	c)
		ILLUMOS_CLOBBER=yes ;;

	#
	# We have only one proto area: we'll skip building non-debug here, as
	# the results are over-written by nightly.  If the user really wants a
	# sanity check build, they can specify =yes via the env var, or by
	# directly editing illumos.sh.
	#
	d)
		ILLUMOS_ENABLE_DEBUG=exclusive ;;
	h)
		echo "$usage"
		exit 0 ;;
	p)
		PRIMARY_COMPILER=$OPTARG
		FORCE_STRAP_REBUILD=yes ;;
	P)
		PLATFORM_PASSWORD="$OPTARG" ;;
	r)
		FORCE_STRAP_REBUILD=yes ;;
	s)
		SHADOW_COMPILERS=$OPTARG
		FORCE_STRAP_REBUILD=yes ;;
	?)
		echo "$usage" >&2
		exit 2 ;;
	esac
done

[[ -n "$ILLUMOS_ENABLE_DEBUG" ]] || ILLUMOS_ENABLE_DEBUG=no
[[ -n "$ILLUMOS_CLOBBER" ]] || ILLUMOS_CLOBBER=no
[[ -n "$PRIMARY_COMPILER" ]] || PRIMARY_COMPILER=gcc4

PRIMARY_COMPILER_VER=$(echo $PRIMARY_COMPILER | sed 's/^gcc//')

# To allow building on base-64-lts 18.4 systems where
# /opt/local/bin/gcc is incapable of building our local copy
# of gcc 4, specify where to find gcc49 which gets installed
# during 'install_pkgin'.
HOST_GCC4_PREFIX=/opt/local/gcc49/bin

cat >build.env <<EOF
FORCE_STRAP_REBUILD=$FORCE_STRAP_REBUILD
ILLUMOS_CLOBBER=$ILLUMOS_CLOBBER
ILLUMOS_ENABLE_DEBUG=$ILLUMOS_ENABLE_DEBUG
PRIMARY_COMPILER=$PRIMARY_COMPILER
PRIMARY_COMPILER_VER=$PRIMARY_COMPILER_VER
SHADOW_COMPILERS=$SHADOW_COMPILERS
HOST_GCC4_PREFIX=$HOST_GCC4_PREFIX
EOF

[[ -n "$PLATFORM_PASSWORD" ]] && \
	echo "PLATFORM_PASSWORD=$PLATFORM_PASSWORD" >> build.env

echo "Doing pre-flight checks... \c "
check_loc
echo "done."
echo "Creating and updating projects directory ..."
create_projects
echo "done."
echo "Installing packages ... \c "
install_packages
echo "done."
echo "Fetching studio... \c "
fetch_studio
echo "done."
echo "Fetching adjuncts tgz... \c "
fetch_adjuncts
echo "done."
echo "Configuring overlays... \c "
setup_overlays
echo "done."
echo "Fetching and extracting closed bins ... \c "
fetch_closed
echo "done."
echo "Generating illumos environment file... \c "
generate_env
echo "done."

cat <<EOF
Configuration complete. To build the live image run:
gmake world && gmake live
EOF

# Congrats, we made it
exit 0