build-cmssw-ib-with-patch

#!/bin/sh -ex

function GetCmsBuildConfig ()
{
  pkg_tag=$1
  type=$2
  shift 2
  case $pkg_tag in
    V00-16-*)
      echo -e "[bootstrap]\npriority: 10\ndoNotBootstrap: True" > cmsBuild.cfg
      for p in  $@ ; do
        echo -e "[$type $p]\npriority: 20" >> cmsBuild.cfg
      done
      echo "--cfg cmsBuild.cfg"
      ;;
    *)
    echo "$type $@"
    ;;
  esac
}

ARCHITECTURE=$1
RELEASE_FORMAT=$2
REPOSITORY=$3
FORCE_FULL_IB=$4
PACKAGE_NAME=cmssw
# Workspace is usually defined by jenkins. If not running in
# jenkins it will assume the current directory is the workspace.
WORKSPACE="${WORKSPACE-$PWD}"

export USER_HOOK_PACKAGE=$WORKSPACE/cms-bot/scram-package-monitor
export USER_HOOK_PROJECT=$WORKSPACE/cms-bot/scram-package-monitor

# WORKDIR is where the build actually happens. In order to recycle things
# already downloaded, we go one level up (i.e. in build-any-ib) and create
# a directory using $REPOSITORY to distinguish between weeks.
# Notice we need to remove completely workdirs older than a week because they
# actually refer to an obsolete repository. We also do a cleanup if we are using
# too much disk space.
WORKDIR=$WORKSPACE/../build-cache/$REPOSITORY
THIS_WEEK=`date +%W`
if [ -e "$WORKDIR/current_week" ] && [ ! X`cat "$WORKDIR/current_week"` = X$THIS_WEEK ]; then
  mv $WORKDIR $WORKSPACE/../build-cache/deleteme-$REPOSITORY
  rm -rf $WORKSPACE/../build-cache/deleteme-* &
elif [ `df -k $WORKSPACE | tail -1 | awk '{print $4}'` -lt 100000000 ]; then
  rm -rf $WORKSPACE/../build-cache
fi
mkdir -p $WORKDIR
echo $THIS_WEEK > $WORKDIR/current_week
cd $WORKDIR
rm -rf w/SPECS

BUILD_JOBS=2
BUILD_NPROC=$(getconf _NPROCESSORS_ONLN)

case $ARCHITECTURE in
  *_mic_*) export BUILD_ARCH=`echo $ARCHITECTURE | cut -f1,2 -d_` ;;
  *_armv7hl_*) BUILD_JOBS=1; BUILD_NPROC=3 ;;
esac

# If you find a squid process, use it, assuming it has the default port.
if pgrep squid >/dev/null ; then
  export http_proxy=http://localhost:3128
fi

RELEASE_NAME=`date +$RELEASE_FORMAT`
DAY=`date +%Y-%m-%d`
HOUR=`echo $RELEASE_NAME | sed -e's|.*-\([0-9][0-9]\)00|\1|'`
# RELEASE_QUEUE=`echo $RELEASE_NAME | sed -e 's/\(CMSSW_[0-9][0-9]*_[0-9][0-9]*\).*/\1_X/'`
RELEASE_QUEUE=`echo $RELEASE_NAME | sed -e 's/_X.*/_X/'`

# Hardcode special queues like SLHC one.
case $RELEASE_NAME in
  *SLHCDEV*) ;;
  *SLHC*) RELEASE_QUEUE=${RELEASE_QUEUE}_SLHC ;;
esac

# Hardoce XROOTD special queue XROOTD
case $RELEASE_NAME in
  *XROOTD*) RELEASE_QUEUE=${RELEASE_QUEUE}ROOTD_X ;;
esac

CMSDIST_REPO=cms-sw
PKGTOOLS_REPO=cms-sw

CONFIG_MAP_URL="https://raw.githubusercontent.com/cms-sw/cms-bot/HEAD/config.map"
eval `curl -s -k -f $CONFIG_MAP_URL | grep "SCRAM_ARCH=$ARCHITECTURE;" | grep "RELEASE_QUEUE=$RELEASE_QUEUE;"`

# eval `curl https://raw.github.com/cms-sw/cms-bot/HEAD/config.map | grep $ARCHITECTURE`
# We use git clone -b because this forces git to think that $CMSDIST_TAG, PKGTOOLS_TAG,
# are both branches. See below to handle the case in which they are actually tags.

# In case github is down.
# git clone -b $CMSDIST_TAG https://:@git.cern.ch/kerberos/CMSDIST.git CMSDIST
# git clone -b $PKGTOOLS_TAG https://:@git.cern.ch/kerberos/PKGTOOLS.git PKGTOOLS

for x in $PKGTOOLS_REPO/pkgtools@$PKGTOOLS_TAG $CMSDIST_REPO/cmsdist@$CMSDIST_TAG; do
  REPO=`echo $x | cut -f1 -d@`
  T=`echo $x | cut -f2 -d@`
  P=`echo $REPO | cut -f2 -d/ | tr "[:lower:]" "[:upper:]"`
  if [ ! -d $P ]; then
    git clone -b $T git://github.com/$REPO.git $P
    pushd $P
      git remote add originrw git@github.com:$REPO.git
    popd
  fi
  pushd $P
    git fetch
    git clean -fxd ; git reset --hard HEAD
    git checkout $T
    git merge --ff-only origin/$T
    eval "${P}_HASH=`git rev-parse HEAD`"
  popd
done


# If we use a tag, rather than a branch, we need to check-it out explicitly.
# We also store the HASH of both PKGTOOLS and CMSDIST to reuse them later on.
# ALL_PREVIOUS_RELEASES contains both IBs and fill releases and we use it
# to eventually skip the build if the same exact IB is already available.
# We disable the whole logic and build a full IB if FORCE_FULL_IB is not empty.
pushd CMSDIST
  if [ ! "X$FORCE_FULL_IB" = Xtrue ]; then
    PREVIOUS_RELEASES=`git show --pretty='%d' HEAD | tr '[ ,()]' '[\n   ]' | grep -v $RELEASE_NAME | { grep "^\(IB\|REL\)/${RELEASE_QUEUE}_[0-9][^/]*/$ARCHITECTURE" || true; }`
    ALL_PREVIOUS_RELEASES=`git show --pretty='%d' HEAD | tr '[ ,()]' '[\n   ]' | grep -v $RELEASE_NAME | { grep "^ALL/${RELEASE_QUEUE}_[0-9][^/]*/$ARCHITECTURE" || true; }`
  fi
  # We checkout a few files from the default branch in any case.
  DEFAULT_CMSDIST_URL="https://raw.githubusercontent.com/cms-sw/cmsdist/HEAD"
  curl -L -s -k $DEFAULT_CMSDIST_URL/cmssw-validation.spec > cmssw-validation.spec
  curl -L -s -k $DEFAULT_CMSDIST_URL/cmssw-ib.spec > cmssw-ib.spec
  curl -L -s -k $DEFAULT_CMSDIST_URL/das-cache.file > das-cache.file
popd

CMS_TAG=""
if [ "X`PKGTOOLS/cmsBuild --help | grep tag=NAME`" != "X" ] ; then
  case `uname -s` in
    Darwin ) MD5_CMD=md5;;
    * )      MD5_CMD=md5sum;;
  esac
  CMS_TAG="--tag `echo $RELEASE_QUEUE | $MD5_CMD |  tr '0123456789' 'ghijklmnop' | cut -b1-6`"
fi
CMSBUILD_CMD="PKGTOOLS/cmsBuild --debug --builders $BUILD_JOBS -j $BUILD_NPROC --repository $REPOSITORY --architecture $ARCHITECTURE --work-dir w $CMS_TAG"
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG build SCRAMV1`

# Decide if we already have a release which uses the same CMSDIST.
pushd w
for previous in $PREVIOUS_RELEASES; do
  previous=`echo $previous | cut -f2 -d/`
  echo "Checking if we can install $previous"
  set +e
  CMSSW_RELEASE_BASE=`source $ARCHITECTURE/apt-init.sh; apt-cache search cms\\\+cmssw\\\+$previous | grep 'cms[+]cmssw[+]' | cut -f1 -d\  | cut -f3 -d+ | grep -e "^$previous\$"`
  set -e
  if [ ! "X$CMSSW_RELEASE_BASE" = X ]; then
    PACKAGE_NAME=cmssw-patch
    perl -p -i -e "s/BuildRequires: cmssw/BuildRequires: cmssw-patch/" ../CMSDIST/cmssw-ib.spec
    perl -p -i -e "s/BuildRequires: cmssw/BuildRequires: cmssw-patch/" ../CMSDIST/cmssw-validation.spec
    perl -p -i -e "s/%define baserelver[\s].*/%define baserelver $CMSSW_RELEASE_BASE/" ../CMSDIST/cmssw-patch-build.file
    perl -p -i -e "s/%define baserelqueue[\s].*/%define baserelqueue $CMSSW_RELEASE_BASE/" ../CMSDIST/cmssw-patch-build.file
    perl -p -i -e 's/CMSSW_ROOT/CMSSW_PATCH_ROOT/' ../CMSDIST/cmssw-ib.spec
    perl -p -i -e 's/CMSSW_VERSION/CMSSW_PATCH_VERSION/' ../CMSDIST/cmssw-ib.spec
    break
  fi
done

# This is done to decide wether or not we actually need to build a release or
# the same CMSDIST / PKGTOOLS / CMSSW tags were used to build an IB which
# is already available in the apt repository.
if [ "X$PACKAGE_NAME" = Xcmssw-patch ]; then
  AVAILABLE_RELEASES=`source $ARCHITECTURE/apt-init.sh; apt-cache search cms\\\+cmssw | grep ${RELEASE_QUEUE} | grep 'cms[+]cmssw\(-patch\|\)[+]' | cut -f1 -d\  | cut -f3 -d+`
  for same_tag in $RELEASE_LIST; do
    if echo $ALL_PREVIOUS_RELEASES | grep $same_tag; then
      if echo $AVAILABLE_RELEASES | grep $same_tag; then
        echo $same_tag uses the same CMSDIST, PKGTOOLS and CMSSW tag. Not building.
        exit 0
      fi
    fi
  done
fi
popd

case $ARCHITECTURE in
 *mic*) CMSSW_RELEASE_BASE=""; PACKAGE_NAME=cmssw; CMSSW_TOOL_CONF=cmssw-mic-tool-conf ;;
 *) CMSSW_TOOL_CONF=$PACKAGE_NAME-tool-conf ;;
esac

pushd CMSDIST
  DEBUG_SUBPACKS="cmssw.spec coral.spec cmssw-patch.spec"
  if [ "X$ENABLE_DEBUG" = X ]; then
    perl -p -i -e 's/^[\s]*%define[\s]+subpackageDebug[\s]+./#subpackage debug disabled/' $DEBUG_SUBPACKS
  fi
  if [ ! "X$PACKAGE_NAME" = Xcmssw-patch ]; then
    CMSSW_RELEASE_BASE=$RELEASE_NAME
  fi
  echo "### RPM cms dummy `date +%%s`\n%prep\n%build\n%install\n" > dummy.spec
  perl -p -i -e "s/### RPM cms cmssw .*/### RPM cms cmssw $CMSSW_RELEASE_BASE/" cmssw.spec
  perl -p -i -e "s/### RPM cms cmssw-ib .*/### RPM cms cmssw-ib $RELEASE_NAME/" cmssw-ib.spec
  perl -p -i -e "s/### RPM cms cmssw-validation .*/### RPM cms cmssw-validation $RELEASE_NAME/" cmssw-validation.spec
  perl -p -i -e "s/### RPM cms cmssw-patch.*/### RPM cms cmssw-patch $RELEASE_NAME/" cmssw-patch.spec
popd

#FIXME: Fix --specs-only option which is not available for PKGTOOLS V00-16.
$CMSBUILD_CMD --specs-only `GetCmsBuildConfig $PKGTOOLS_TAG build $PACKAGE_NAME`
TOOL_CONF_PACKAGES="local-cern-siteconf `grep '^%define \(build\|\)directpkgreqs' w/SPECS/cms/$PACKAGE_NAME/*/spec | grep -v '%{nil}' | sed 's|.*directpkgreqs[ \t]*||' | tr ' ' '\n' | cut -f2 -d/ | sort | uniq | tr '\n' ' '`"
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG build coral`
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG build $TOOL_CONF_PACKAGES`
$CMSBUILD_CMD --sync-back `GetCmsBuildConfig $PKGTOOLS_TAG upload $TOOL_CONF_PACKAGES`
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG deprecate-local $TOOL_CONF_PACKAGES`
$CMSBUILD_CMD -k -j `echo "$BUILD_NPROC * 2" | bc` `GetCmsBuildConfig $PKGTOOLS_TAG build cmssw-ib`
$CMSBUILD_CMD --sync-back `GetCmsBuildConfig $PKGTOOLS_TAG upload cmssw-ib`

pushd CMSDIST
# In case we do not have errors in the full release, or in case we are forcing the
# tagging we create an ALL tag and use IB if we discover it's a full release later on.
# In case we have errors, we use an ERR tag in case we discover it's a full release later on.
if [ "X$ALWAYS_TAG_CMSSW" != "X" ] || [ ! -e $WORKDIR/w/$ARCHITECTURE/cms/cmssw/$RELEASE_NAME/build-errors ]; then
  git tag ALL/$RELEASE_NAME/$ARCHITECTURE $CMSDIST_HASH
  git push originrw ALL/$RELEASE_NAME/$ARCHITECTURE || true
  TAG_TYPE="IB"
else
  TAG_TYPE="ERR"
fi

# Check if it is a full release and tag depending on wether it had ERRors or not.
if [ X$CMSSW_RELEASE_BASE = X$RELEASE_NAME ]; then
  git tag $TAG_TYPE/$RELEASE_NAME/$ARCHITECTURE $CMSDIST_HASH
  git push originrw $TAG_TYPE/$RELEASE_NAME/$ARCHITECTURE || true
fi
popd

$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG deprecate-local cmssw-ib`

echo CMSDIST_HASH=$CMSDIST_HASH > $WORKSPACE/buildprops
echo PKGTOOLS_HASH=$PKGTOOLS_HASH >> $WORKSPACE/buildprops
echo PACKAGE_NAME=$PACKAGE_NAME >> $WORKSPACE/buildprops
echo CMSSW_RELEASE_BASE=$CMSSW_RELEASE_BASE >> $WORKSPACE/buildprops
echo TRIGGERING_RELEASE=${RELEASE_QUEUE}_${DATE_FORMAT} >> $WORKSPACE/buildprops
echo TRIGGERING_ARCHITECTURE=${ARCHITECTURE} >> $WORKSPACE/buildprops
echo RELEASE_QUEUE=${RELEASE_QUEUE} >> $WORKSPACE/buildprops
echo ENABLE_DEBUG=${ENABLE_DEBUG} >> $WORKSPACE/buildprops

# This is used to guide the creation of additional files which drive the
# spawning of extra tests.
CONFIG_MAP_URL="https://raw.githubusercontent.com/cms-sw/cms-bot/HEAD/config.map"
for x in `curl -k -s $CONFIG_MAP_URL | grep ADDITIONAL_TESTS= | grep "SCRAM_ARCH=$ARCHITECTURE;" | grep "RELEASE_QUEUE=$RELEASE_QUEUE;" | sed -e's/.*ADDITIONAL_TESTS=//;s/;.*//' | tr , '\n'`; do
  echo "Creating property file $WORKSPACE/$x-properties."
  echo "RELEASE_FORMAT=${RELEASE_QUEUE}_${DATE_FORMAT}" > $WORKSPACE/$x-properties
  echo "ARCHITECTURE=${ARCHITECTURE}" >> $WORKSPACE/$x-properties
  echo "RELEASE_QUEUE=${RELEASE_QUEUE}" >> $WORKSPACE/$x-properties
done

# This part is needed to drive the installation on afs for different OS.
OS=`echo $ARCHITECTURE | cut -d"_" -f 1`
echo TRIGGERING_RELEASE=${RELEASE_QUEUE}_${DATE_FORMAT} > $WORKSPACE/install_$OS
echo TRIGGERING_ARCHITECTURE=${ARCHITECTURE} >> $WORKSPACE/install_$OS
echo ARCHITECTURE=${ARCHITECTURE} >> $WORKSPACE/install_$OS