Two examples of submission scripts to PSI's T3 batch system.
#!/bin/bash
#################################
# PSI Tier-3 example batch Job #
#################################
##### CONFIGURATION ##############################################
# Output files to be copied back to the User Interface
# (the file path must be given relative to the working directory)
OUTFILES="myout.txt myerr.txt"
# Output files to be copied to the SE
# (as above the file path must be given relative to the working directory)
SEOUTFILES="mybigfile"
#
# By default, the files will be copied to $USER_SRM_HOME/$username/$JOBDIR,
# but here you can define the subdirectory under your SE storage path
# to which files will be copied (uncomment line)
#SEUSERSUBDIR="mytestsubdir/somedir"
#
# User's CMS hypernews name (needed for user's SE storage home path
# USER_SRM_HOME below)
HN_NAME=your-hn-name
# set DBG=1 for additional debug output in the job report files
# DBG=2 will also give detailed output on SRM operations
DBG=0
#### The following configurations you should not need to change
# The SE's user home area (SRMv2 URL)
USER_SRM_HOME="srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/trivcat/store/user/"
# Top working directory on worker node's local disk. The batch
# job working directory will be created below this
TOPWORKDIR=/scratch/`whoami`
# Basename of job sandbox (job workdir will be $TOPWORKDIR/$JOBDIR)
JOBDIR=sgejob-$JOB_ID
##################################################################
############ BATCH QUEUE DIRECTIVES ##############################
# Lines beginning with #$ are used to set options for the SGE
# queueing system (same as specifying these options to the qsub
# command
# Job name (defines name seen in monitoring by qstat and the
# job script's stderr/stdout names)
#$ -N example_job
### Specify the queue on which to run
#$ -q all.q
# Change to the current working directory from which the job got
# submitted (will also result in the job report stdout/stderr being
# written to this directory)
#$ -cwd
# here you could change location of the job report stdout/stderr files
# if you did not want them in the submission directory
# #$ -o /shome/username/mydir/
# #$ -e /shome/username/mydir/
##################################################################
##### MONITORING/DEBUG INFORMATION ###############################
DATE_START=`date +%s`
echo "Job started at " `date`
cat <<EOF
################################################################
## QUEUEING SYSTEM SETTINGS:
HOME=$HOME
USER=$USER
JOB_ID=$JOB_ID
JOB_NAME=$JOB_NAME
HOSTNAME=$HOSTNAME
TASK_ID=$TASK_ID
QUEUE=$QUEUE
EOF
echo "################################################################"
if test 0"$DBG" -gt 0; then
echo "######## Environment Variables ##########"
env
echo "################################################################"
fi
##### SET UP WORKDIR AND ENVIRONMENT ######################################
STARTDIR=`pwd`
WORKDIR=$TOPWORKDIR/$JOBDIR
RESULTDIR=$STARTDIR/$JOBDIR
if test x"$SEUSERSUBDIR" = x; then
SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$JOBDIR
else
SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$SEUSERSUBDIR
fi
if test -e "$WORKDIR"; then
echo "ERROR: WORKDIR ($WORKDIR) already exists! Aborting..." >&2
exit 1
fi
mkdir -p $WORKDIR
if test ! -d "$WORKDIR"; then
echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
exit 1
fi
cd $WORKDIR
cat <<EOF
################################################################
## JOB SETTINGS:
STARTDIR=$STARTDIR
WORKDIR=$WORKDIR
RESULTDIR=$RESULTDIR
SERESULTDIR=$SERESULTDIR
EOF
###########################################################################
## YOUR FUNCTIONALITY CODE GOES HERE
# set up CMS environment
source $VO_CMS_SW_DIR/cmsset_default.sh
# Here we produce some output for the files that are copied back to
# our shared home
scramv1 list > myout.txt 2>myerr.txt
lcg-ls -b -D srmv2 -l srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/ >> myout.txt 2>>myerr.txt
# create a dummy file for copying back to the SE
dd if=/dev/urandom of=mybigfile count=100 &>/dev/null
CMSSW_DIR=/shome/feichtinger/cmssw_test/CMSSW_2_1_6
CMSSW_CONFIG_FILE=$CMSSW_DIR/demoanalyzer-classic.cfg
cd $CMSSW_DIR/src
eval `scramv1 runtime -sh`
if test $? -ne 0; then
echo "ERROR: Failed to source scram environment" >&2
exit 1
fi
cd $WORKDIR
cmsRun $CMSSW_CONFIG_FILE maxEvents=$1 firstEvent=$2 inputFiles=$3 seed=$4> myout.txt 2>myerr.txt
#### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ############################
cd $WORKDIR
if test 0"$DBG" -gt 0; then
echo "########################################################"
echo "############# Working directory contents ###############"
echo "pwd: " `pwd`
ls -Rl
echo "########################################################"
echo "YOUR OUTPUT WILL BE MOVED TO $RESULTDIR"
echo "########################################################"
fi
if test x"$OUTFILES" != x; then
mkdir -p $RESULTDIR
if test ! -e "$RESULTDIR"; then
echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
exit 1
fi
for n in $OUTFILES; do
if test ! -e $WORKDIR/$n; then
echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
else
cp -a $WORKDIR/$n $RESULTDIR/$n
if test $? -ne 0; then
echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
fi
fi
done
fi
if test x"$SEOUTFILES" != x; then
if test 0"$DBG" -ge 2; then
srmdebug="-v"
fi
for n in $SEOUTFILES; do
if test ! -e $WORKDIR/$n; then
echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
else
lcg-cp $srmdebug -b -D srmv2 file:$WORKDIR/$n $SERESULTDIR/$n
if test $? -ne 0; then
echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
fi
fi
done
fi
echo "Cleaning up $WORKDIR"
rm -rf $WORKDIR
###########################################################################
DATE_END=`date +%s`
RUNTIME=$((DATE_END-DATE_START))
echo "################################################################"
echo "Job finished at " `date`
echo "Wallclock running time: $RUNTIME s"
exit 0
Trimmed version.
#! /bin/bash
# https://wiki.chipp.ch/twiki/bin/view/CmsTier3/HowToSubmitJobs
# run with qsub -q all.q submitAnalysis.sh
printf "###################################################\n"
printf "## Run Analysis with input %-20s##\n" $1
printf "###################################################\n"
DBG=2 # debugging level
INPUT=$1;
JOBLOGFILES="myout.txt myerr.txt"
OUTFILES="*.png"
TOPWORKDIR=/scratch/`whoami`
JOBDIR="analysis/$1"
BASEDIR="/shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder"
CMSSW_DIR="/shome/myusername/CMSSW_5_3_24"
WORKDIR=$TOPWORKDIR/$JOBDIR
OUTDIR=$BASEDIR/$JOBDIR # for output files
RESULTDIR=$BASEDIR/$JOBDIR # for log files
# to save large output files on storage element
SEUSERSUBDIR="analysis/$1"
SEOUTFILES="*.root"
USER_SE_HOME="root://t3dcachedb.psi.ch:1094/pnfs/psi.ch/cms/trivcat/store/user/$USER"
SERESULTDIR=$USER_SE_HOME/$SEUSERSUBDIR
##### MONITORING/DEBUG INFORMATION ########################################
# write job report (stdout of this script)
mkdir -p /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -o /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -e /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
DATE_START=`date +%s`
echo "Job started at " `date`
cat <<EOF
###########################################
## QUEUEING SYSTEM SETTINGS: ##
###########################################
HOME=$HOME
USER=$USER
JOB_ID=$JOB_ID
JOB_NAME=$JOB_NAME
HOSTNAME=$HOSTNAME
TASK_ID=$TASK_ID
QUEUE=$QUEUE
EOF
if test 0"$DBG" -gt 0; then
echo " "
echo "###########################################"
echo "## Environment Variables ##"
echo "###########################################"
env
fi
##### SET ENVIRONMENT #####################################################
mkdir -p $WORKDIR
if test ! -d "$WORKDIR"; then
echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
exit 1
fi
cd $WORKDIR
cat <<EOF
###########################################
## JOB SETTINGS: ##
###########################################
BASEDIR=$BASEDIR
WORKDIR=$WORKDIR
RESULTDIR=$RESULTDIR
SERESULTDIR=$SERESULTDIR
EOF
echo " "
echo "###########################################"
echo "## MY FUNCTIONALITY CODE ##"
echo "###########################################"
source $VO_CMS_SW_DIR/cmsset_default.sh 2>> myerr.txt
cd $CMSSW_DIR/src
eval `scramv1 runtime -sh` >> myout.txt 2>> myerr.txt
if test $? -ne 0; then
echo "ERROR: Failed to source scram environment" >&2
exit 1
fi
cd $WORKDIR
echo "python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt"
python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt
ls >> myout.txt 2>> myerr.txt
##### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ###########################
cd $WORKDIR
if test 0"$DBG" -gt 0; then
echo " "
echo "###########################################################"
echo "## MY OUTPUT WILL BE MOVED TO \$RESULTDIR and \$OUTDIR ##"
echo "###########################################################"
echo " \$RESULTDIR=$RESULTDIR"
echo " \$OUTDIR=$OUTDIR"
echo "pwd: " `pwd`
ls -Rl
fi
# copy log files to $RESULTDIR
if test x"$JOBLOGFILES" != x; then
mkdir -p $RESULTDIR
if test ! -e "$RESULTDIR"; then
echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
fi
for n in $JOBLOGFILES; do
if test ! -e $WORKDIR/$n; then
echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
else
echo ">>> copying $n from $WORKDIR to $RESULTDIR"
cp -a $WORKDIR/$n $RESULTDIR/$n
if test $? -ne 0; then
echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
fi
fi
done
fi
# copy output files to $OUTDIR
if test x"$OUTFILES" != x; then
mkdir -p $OUTDIR
if test ! -e "$OUTDIR"; then
echo "ERROR: Failed to create $OUTDIR ...Aborting..." >&2
fi
for n in $OUTFILES; do
if test ! -e $WORKDIR/$n; then
echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
else
echo ">>> copying $n from $WORKDIR to $OUTDIR"
cp -a $WORKDIR/$n $OUTDIR/$n
if test $? -ne 0; then
echo "ERROR: Failed to copy $WORKDIR/$n to $OUTDIR/$n" >&2
fi
fi
done
fi
# copy large files to $SERESULTDIR
if test x"$SEOUTFILES" != x; then
if test ! -e "$SERESULTDIR"; then
echo "ERROR: Failed to create $SERESULTDIR ...Aborting..." >&2
fi
if test 0"$DBG" -ge 2; then
srmdebug="-v"
fi
for n in $SEOUTFILES; do
if test ! -e $WORKDIR/$n; then
echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
else
echo "xrdcp -f $WORKDIR/$n $SERESULTDIR/$n" >&2
xrdcp $srmdebug -d $DBG -f $WORKDIR/$n $SERESULTDIR/$n
if test $? -ne 0; then
echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
fi
fi
done
fi
# cleaning working directory
echo ">>> Cleaning up $WORKDIR"
rm -rf $WORKDIR
###########################################################################
DATE_END=`date +%s`
RUNTIME=$((DATE_END-DATE_START))
echo " "
echo "#####################################################"
echo " Job finished at " `date`
echo " Wallclock running time: $(( $RUNTIME / 3600 )):$(( $RUNTIME % 3600 /60 )):$(( $RUNTIME % 60 )) "
echo "#####################################################"
echo " "
exit 0