build-cmssw-ib-with-patch

#!/bin/sh -ex

function GetCmsBuildConfig ()
{
  pkg_tag=$1
  type=$2
  shift 2
  case $pkg_tag in
    V00-16-*)
      echo -e "[bootstrap]\npriority: 10\ndoNotBootstrap: True" > cmsBuild.cfg
      for p in  $@ ; do
        echo -e "[$type $p]\npriority: 20" >> cmsBuild.cfg
      done
      echo "--cfg cmsBuild.cfg"
      ;;
    *)
    echo "$type $@"
    ;;
  esac
}
CMS_BOT_DIR=$(dirname $0)
case $CMS_BOT_DIR in /*) ;; *) CMS_BOT_DIR=$(pwd)/${CMS_BOT_DIR} ;; esac

echo DATE=`date`
ARCHITECTURE=$1
RELEASE_FORMAT=$2
REPOSITORY=$3
FORCE_FULL_IB=$4
RELEASE_LIST=$5
PACKAGE_NAME=cmssw
FORCE_PKGTOOLS_TAG=$6
USE_DEV_CMSPKG=$7
FORCE_BUILD_PATCH_RELEASE=$8
# Workspace is usually defined by jenkins. If not running in
# jenkins it will assume the current directory is the workspace.
WORKSPACE="${WORKSPACE-$PWD}"

rm -rf $WORKSPACE/../build-cache &
cd $WORKSPACE
rm -rf w/SPECS

BUILD_JOBS=4
BUILD_NPROC=$(getconf _NPROCESSORS_ONLN)

case $ARCHITECTURE in
  *_mic_*) export BUILD_ARCH=`echo $ARCHITECTURE | cut -f1,2 -d_` ;;
  *_armv7hl_*) BUILD_JOBS=1; BUILD_NPROC=3 ;;
esac

# If you find a squid process, use it, assuming it has the default port.
if pgrep squid >/dev/null ; then
  export http_proxy=http://localhost:3128
fi

RELEASE_NAME=`date +$RELEASE_FORMAT`
DAY=`date +%Y-%m-%d`
HOUR=`echo $RELEASE_NAME | sed -e's|.*-\([0-9][0-9]\)00|\1|'`
# RELEASE_QUEUE=`echo $RELEASE_NAME | sed -e 's/\(CMSSW_[0-9][0-9]*_[0-9][0-9]*\).*/\1_X/'`
RELEASE_QUEUE=`echo $RELEASE_NAME | sed -e 's/_X.*/_X/'`

# Hardcode special queues like SLHC one.
case $RELEASE_NAME in
  *SLHCDEV*) ;;
  *SLHC*) RELEASE_QUEUE=${RELEASE_QUEUE}_SLHC ;;
esac

# Hardoce XROOTD special queue XROOTD
case $RELEASE_NAME in
  *XROOTD*) RELEASE_QUEUE=${RELEASE_QUEUE}ROOTD_X ;;
esac

CMSDIST_REPO=cms-sw
PKGTOOLS_REPO=cms-sw
unset BUILD_PATCH_RELEASE
eval `cat $CMS_BOT_DIR/config.map | grep "SCRAM_ARCH=$ARCHITECTURE;" | grep "RELEASE_QUEUE=$RELEASE_QUEUE;"`
[ "X$BUILD_PATCH_RELEASE" = "X1" ] && FORCE_BUILD_PATCH_RELEASE=true

# We use git clone -b because this forces git to think that $CMSDIST_TAG, PKGTOOLS_TAG,
# are both branches. See below to handle the case in which they are actually tags.

# In case github is down.
# git clone -b $CMSDIST_TAG https://:@git.cern.ch/kerberos/CMSDIST.git CMSDIST
# git clone -b $PKGTOOLS_TAG https://:@git.cern.ch/kerberos/PKGTOOLS.git PKGTOOLS

#For now force V00-30 tags for all IBs unless it is overridden by FORCE_PKGTOOLS_TAG
PKGTOOLS_TAG=V00-30-XX
[ "$FORCE_PKGTOOLS_TAG" != "" ] && PKGTOOLS_TAG=$FORCE_PKGTOOLS_TAG
USE_DEV=""
[ "$USE_DEV_CMSPKG" = "true" ] && USE_DEV="--use-dev"

for x in $PKGTOOLS_REPO/pkgtools@$PKGTOOLS_TAG $CMSDIST_REPO/cmsdist@$CMSDIST_TAG; do
  REPO=`echo $x | cut -f1 -d@`
  T=`echo $x | cut -f2 -d@`
  P=`echo $REPO | cut -f2 -d/ | tr "[:lower:]" "[:upper:]"`
  if [ ! -d $P ]; then
    git clone -b $T git://github.com/$REPO.git $P
    pushd $P
      git remote add originrw git@github.com:$REPO.git
    popd
  fi
  pushd $P
    git fetch
    git clean -fxd ; git reset --hard HEAD
    git checkout $T
    git merge --ff-only origin/$T
    eval "${P}_HASH=`git rev-parse HEAD`"
  popd
done
echo DATE=`date`

# If we use a tag, rather than a branch, we need to check-it out explicitly.
# We also store the HASH of both PKGTOOLS and CMSDIST to reuse them later on.
# ALL_PREVIOUS_RELEASES contains both IBs and fill releases and we use it
# to eventually skip the build if the same exact IB is already available.
# We disable the whole logic and build a full IB if FORCE_FULL_IB is not empty.
pushd CMSDIST
  if [ ! "X$FORCE_FULL_IB" = Xtrue ]; then
    PREVIOUS_RELEASES=`git show --pretty='%d' HEAD | tr '[ ,()]' '[\n   ]' | grep -v $RELEASE_NAME | { grep "^\(IB\|REL\)/${RELEASE_QUEUE}_[0-9][^/]*/$ARCHITECTURE" || true; }`
    ALL_PREVIOUS_RELEASES=`git show --pretty='%d' HEAD | tr '[ ,()]' '[\n   ]' | grep -v $RELEASE_NAME | { grep "^ALL/${RELEASE_QUEUE}_[0-9][^/]*/$ARCHITECTURE" || true; }`
  fi
  # We checkout a few files from the default branch in any case.
  DEFAULT_CMSDIST_URL="https://raw.githubusercontent.com/cms-sw/cmsdist/HEAD"
  curl -L -s -k $DEFAULT_CMSDIST_URL/cmssw-validation.spec > cmssw-validation.spec
  curl -L -s -k $DEFAULT_CMSDIST_URL/cmssw-ib.spec > cmssw-ib.spec
  curl -L -s -k $DEFAULT_CMSDIST_URL/das-cache.file > das-cache.file
popd

CMS_TAG=""
if [ "X`PKGTOOLS/cmsBuild --help | grep tag=NAME`" != "X" ] ; then
  case `uname -s` in
    Darwin ) MD5_CMD=md5;;
    * )      MD5_CMD=md5sum;;
  esac
  CMS_TAG="--tag `echo $RELEASE_QUEUE | $MD5_CMD |  tr '0123456789' 'ghijklmnop' | cut -b1-6`"
fi
CMSBUILD_CMD="PKGTOOLS/cmsBuild $USE_DEV --debug --builders $BUILD_JOBS -j $BUILD_NPROC --repository $REPOSITORY --architecture $ARCHITECTURE --work-dir w $CMS_TAG"
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG build SCRAMV1`

# Decide if we already have a release which uses the same CMSDIST.
pushd w
for previous in $PREVIOUS_RELEASES; do
  previous=`echo $previous | cut -f2 -d/`
  echo "Checking if we can install $previous"
  set +e
  #CMSSW_RELEASE_BASE=`source $ARCHITECTURE/apt-init.sh; apt-cache search cms\\\+cmssw\\\+$previous | grep 'cms[+]cmssw[+]' | cut -f1 -d\  | cut -f3 -d+ | grep -e "^$previous\$"`
  CMSSW_RELEASE_BASE=`./common/cmspkg -a $ARCHITECTURE -f search cms+cmssw+$previous | grep 'cms[+]cmssw[+]' | cut -f1 -d\  | cut -f3 -d+ | grep -e "^$previous\$"`
  set -e
  if [ ! "X$CMSSW_RELEASE_BASE" = X ]; then
    PACKAGE_NAME=cmssw-patch
    perl -p -i -e "s/BuildRequires: cmssw/BuildRequires: cmssw-patch/" ../CMSDIST/cmssw-ib.spec
    perl -p -i -e "s/BuildRequires: cmssw/BuildRequires: cmssw-patch/" ../CMSDIST/cmssw-validation.spec
    perl -p -i -e "s/%define baserelver[\s].*/%define baserelver $CMSSW_RELEASE_BASE/" ../CMSDIST/cmssw-patch-build.file
    perl -p -i -e "s/%define baserelqueue[\s].*/%define baserelqueue $CMSSW_RELEASE_BASE/" ../CMSDIST/cmssw-patch-build.file
    perl -p -i -e 's/CMSSW_ROOT/CMSSW_PATCH_ROOT/' ../CMSDIST/cmssw-ib.spec
    perl -p -i -e 's/CMSSW_VERSION/CMSSW_PATCH_VERSION/' ../CMSDIST/cmssw-ib.spec
    break
  fi
done

# This is done to decide wether or not we actually need to build a release or
# the same CMSDIST / PKGTOOLS / CMSSW tags were used to build an IB which
# is already available in the apt repository.
if [ "X$PACKAGE_NAME" = Xcmssw-patch -a "X$FORCE_BUILD_PATCH_RELEASE" != "Xtrue" ]; then
  #AVAILABLE_RELEASES=`source $ARCHITECTURE/apt-init.sh; apt-cache search cms\\\+cmssw | grep ${RELEASE_QUEUE} | grep 'cms[+]cmssw\(-patch\|\)[+]' | cut -f1 -d\  | cut -f3 -d+`
  AVAILABLE_RELEASES=`./common/cmspkg -a $ARCHITECTURE search cms+cmssw | grep ${RELEASE_QUEUE} | grep 'cms[+]cmssw\(-patch\|\)[+]' | cut -f1 -d\  | cut -f3 -d+`
  for same_tag in $RELEASE_LIST; do
    if [ "X`echo $ALL_PREVIOUS_RELEASES | grep $same_tag`" != X ] ; then
      if [ "X`echo $AVAILABLE_RELEASES | grep $same_tag`" != "X" ] ; then
        echo $same_tag uses the same CMSDIST, PKGTOOLS and CMSSW tag. Not building.
        exit 0
      fi
    fi
  done
fi
popd
echo DATE=`date`

case $ARCHITECTURE in
 *mic*) CMSSW_RELEASE_BASE=""; PACKAGE_NAME=cmssw; CMSSW_TOOL_CONF=cmssw-mic-tool-conf ;;
 *) CMSSW_TOOL_CONF=$PACKAGE_NAME-tool-conf ;;
esac

pushd CMSDIST
  DEBUG_SUBPACKS="cmssw.spec coral.spec cmssw-patch.spec root.spec"
  if [ "X$ENABLE_DEBUG" = X ]; then
    perl -p -i -e 's/^[\s]*%define[\s]+subpackageDebug[\s]+./#subpackage debug disabled/' $DEBUG_SUBPACKS
  fi
  if [ ! "X$PACKAGE_NAME" = Xcmssw-patch ]; then
    CMSSW_RELEASE_BASE=$RELEASE_NAME
  fi
  echo "### RPM cms dummy `date +%%s`\n%prep\n%build\n%install\n" > dummy.spec
  perl -p -i -e "s/### RPM cms cmssw .*/### RPM cms cmssw $CMSSW_RELEASE_BASE/" cmssw.spec
  perl -p -i -e "s/### RPM cms cmssw-ib .*/### RPM cms cmssw-ib $RELEASE_NAME/" cmssw-ib.spec
  perl -p -i -e "s/### RPM cms cmssw-validation .*/### RPM cms cmssw-validation $RELEASE_NAME/" cmssw-validation.spec
  perl -p -i -e "s/### RPM cms cmssw-patch.*/### RPM cms cmssw-patch $RELEASE_NAME/" cmssw-patch.spec
popd

#FIXME: Fix --specs-only option which is not available for PKGTOOLS V00-16.
$CMSBUILD_CMD --specs-only `GetCmsBuildConfig $PKGTOOLS_TAG build $PACKAGE_NAME`
TOOL_CONF_PACKAGES="local-cern-siteconf `grep '^%define \(build\|\)directpkgreqs' w/SPECS/cms/$PACKAGE_NAME/*/spec | grep -v '%{nil}' | sed 's|.*directpkgreqs[ \t]*||' | tr ' ' '\n' | cut -f2 -d/ | sort | uniq | tr '\n' ' '`"
echo DATE=`date`
$CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG build $TOOL_CONF_PACKAGES`
echo DATE=`date`
if [ `ls w/RPMS/${ARCHITECTURE} | wc -l` -gt 0 ] ; then
  $CMSBUILD_CMD --sync-back `GetCmsBuildConfig $PKGTOOLS_TAG upload $TOOL_CONF_PACKAGES`
  echo DATE=`date`
  $CMSBUILD_CMD `GetCmsBuildConfig $PKGTOOLS_TAG deprecate-local $TOOL_CONF_PACKAGES`
  echo DATE=`date`
fi
$CMSBUILD_CMD -k -j `echo "$BUILD_NPROC + 1" | bc` `GetCmsBuildConfig $PKGTOOLS_TAG build cmssw-ib`
echo DATE=`date`
$CMSBUILD_CMD --sync-back `GetCmsBuildConfig $PKGTOOLS_TAG upload cmssw-ib`
echo DATE=`date`
pushd CMSDIST
# In case we do not have errors in the full release, or in case we are forcing the
# tagging we create an ALL tag and use IB if we discover it's a full release later on.
# In case we have errors, we use an ERR tag in case we discover it's a full release later on.
if [ "X$ALWAYS_TAG_CMSSW" != "X" ] || [ ! -e $WORKSPACE/w/$ARCHITECTURE/cms/cmssw/$RELEASE_NAME/build-errors ]; then
  git tag ALL/$RELEASE_NAME/$ARCHITECTURE $CMSDIST_HASH
  git push originrw ALL/$RELEASE_NAME/$ARCHITECTURE || true
  TAG_TYPE="IB"
else
  TAG_TYPE="ERR"
fi

# Check if it is a full release and tag depending on wether it had ERRors or not.
if [ X$CMSSW_RELEASE_BASE = X$RELEASE_NAME ]; then
  git tag $TAG_TYPE/$RELEASE_NAME/$ARCHITECTURE $CMSDIST_HASH
  git push originrw $TAG_TYPE/$RELEASE_NAME/$ARCHITECTURE || true
fi
popd

if [ "X$DO_NOT_INSTALL" != "X" ] ; then exit 0 ; fi
echo CMSDIST_HASH=$CMSDIST_HASH > $WORKSPACE/buildprops
echo PKGTOOLS_HASH=$PKGTOOLS_HASH >> $WORKSPACE/buildprops
echo PACKAGE_NAME=$PACKAGE_NAME >> $WORKSPACE/buildprops
echo RELEASE_FORMAT=${RELEASE_FORMAT} >> $WORKSPACE/buildprops
echo CMSSW_RELEASE_BASE=$CMSSW_RELEASE_BASE >> $WORKSPACE/buildprops
echo RELEASE_QUEUE=${RELEASE_QUEUE} >> $WORKSPACE/buildprops
echo ENABLE_DEBUG=${ENABLE_DEBUG} >> $WORKSPACE/buildprops
echo DOCKER_IMG=${DOCKER_IMG} >> $WORKSPACE/buildprops
echo ARCHITECTURE=${ARCHITECTURE} >> $WORKSPACE/buildprops

for x in `echo $ADDITIONAL_TESTS | tr , '\n'`; do
  echo "Creating property file $WORKSPACE/$x-properties."
  cp $WORKSPACE/buildprops $WORKSPACE/$x-properties
done

echo DATE=`date`