Two examples of submission scripts to PSI's T3 batch system.


 ===== Example 1 =====

<file text submitExample.sh>

  #!/bin/bash
  #################################
  # PSI Tier-3 example batch Job  #
  #################################
  
  ##### CONFIGURATION ##############################################
  # Output files to be copied back to the User Interface
  # (the file path must be given relative to the working directory)
  OUTFILES="myout.txt myerr.txt"
  
  # Output files to be copied to the SE
  # (as above the file path must be given relative to the working directory)
  SEOUTFILES="mybigfile" 
  #
  # By default, the files will be copied to $USER_SRM_HOME/$username/$JOBDIR,
  # but here you can define the subdirectory under your SE storage path
  # to which files will be copied (uncomment line)
  #SEUSERSUBDIR="mytestsubdir/somedir"
  #
  # User's CMS hypernews name (needed for user's SE storage home path
  # USER_SRM_HOME below)
  HN_NAME=your-hn-name
  
  # set DBG=1 for additional debug output in the job report files
  # DBG=2 will also give detailed output on SRM operations
  DBG=0
  
  #### The following configurations you should not need to change
  # The SE's user home area (SRMv2 URL)
  USER_SRM_HOME="srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/trivcat/store/user/"
  
  # Top working directory on worker node's local disk. The batch
  # job working directory will be created below this
  TOPWORKDIR=/scratch/`whoami`
  
  # Basename of job sandbox (job workdir will be $TOPWORKDIR/$JOBDIR)
  JOBDIR=sgejob-$JOB_ID
  ##################################################################
  
  
  ############ BATCH QUEUE DIRECTIVES ##############################
  # Lines beginning with #$ are used to set options for the SGE
  # queueing system (same as specifying these options to the qsub
  # command
  
  # Job name (defines name seen in monitoring by qstat and the
  #     job script's stderr/stdout names)
  #$ -N example_job
  
  ### Specify the queue on which to run
  #$ -q all.q
  
  # Change to the current working directory from which the job got
  # submitted (will also result in the job report stdout/stderr being
  # written to this directory)
  #$ -cwd
  
  # here you could change location of the job report stdout/stderr files
  #  if you did not want them in the submission directory
  #  #$ -o /shome/username/mydir/
  #  #$ -e /shome/username/mydir/
  
  ##################################################################
  
  
  
  ##### MONITORING/DEBUG INFORMATION ###############################
  DATE_START=`date +%s`
  echo "Job started at " `date`
  cat <<EOF
  ################################################################
  ## QUEUEING SYSTEM SETTINGS:
  HOME=$HOME
  USER=$USER
  JOB_ID=$JOB_ID
  JOB_NAME=$JOB_NAME
  HOSTNAME=$HOSTNAME
  TASK_ID=$TASK_ID
  QUEUE=$QUEUE
  
  EOF
  
  echo "################################################################"
  
  if test 0"$DBG" -gt 0; then
     echo "######## Environment Variables ##########"
     env
     echo "################################################################"
  fi
  
  
  ##### SET UP WORKDIR AND ENVIRONMENT ######################################
  STARTDIR=`pwd`
  WORKDIR=$TOPWORKDIR/$JOBDIR
  RESULTDIR=$STARTDIR/$JOBDIR
  if test x"$SEUSERSUBDIR" = x; then
     SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$JOBDIR
  else
     SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$SEUSERSUBDIR
  fi
  if test -e "$WORKDIR"; then
     echo "ERROR: WORKDIR ($WORKDIR) already exists! Aborting..." >&2
     exit 1
  fi
  mkdir -p $WORKDIR
  if test ! -d "$WORKDIR"; then
     echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
     exit 1
  fi
  
  cd $WORKDIR
  cat <<EOF
  ################################################################
  ## JOB SETTINGS:
  STARTDIR=$STARTDIR
  WORKDIR=$WORKDIR
  RESULTDIR=$RESULTDIR
  SERESULTDIR=$SERESULTDIR
  EOF
  
  ###########################################################################
  ## YOUR FUNCTIONALITY CODE GOES HERE
  
  # set up CMS environment
  source $VO_CMS_SW_DIR/cmsset_default.sh
  
  # Here we produce some output for the files that are copied back to
  #  our shared home
  scramv1 list > myout.txt 2>myerr.txt
  
  lcg-ls -b -D srmv2 -l srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/ >> myout.txt 2>>myerr.txt
  
  # create a dummy file for copying back to the SE
  dd if=/dev/urandom of=mybigfile count=100 &>/dev/null
  
  CMSSW_DIR=/shome/feichtinger/cmssw_test/CMSSW_2_1_6
  CMSSW_CONFIG_FILE=$CMSSW_DIR/demoanalyzer-classic.cfg
  
  cd $CMSSW_DIR/src
  eval `scramv1 runtime -sh`
  if test $? -ne 0; then
     echo "ERROR: Failed to source scram environment" >&2
     exit 1
  fi
  
  cd $WORKDIR
  cmsRun $CMSSW_CONFIG_FILE maxEvents=$1 firstEvent=$2 inputFiles=$3 seed=$4> myout.txt 2>myerr.txt
  
  #### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ############################
  cd $WORKDIR
  if test 0"$DBG" -gt 0; then
      echo "########################################################"
      echo "############# Working directory contents ###############"
      echo "pwd: " `pwd`
      ls -Rl
      echo "########################################################"
      echo "YOUR OUTPUT WILL BE MOVED TO $RESULTDIR"
      echo "########################################################"
  fi
  
  if test x"$OUTFILES" != x; then
     mkdir -p $RESULTDIR
     if test ! -e "$RESULTDIR"; then
            echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
            exit 1
     fi
     for n in $OUTFILES; do
         if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
         else
            cp -a $WORKDIR/$n $RESULTDIR/$n
            if test $? -ne 0; then
               echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
            fi
     fi
     done
  fi
  
  if test x"$SEOUTFILES" != x; then
     if test 0"$DBG" -ge 2; then
        srmdebug="-v"
     fi
     for n in $SEOUTFILES; do
         if test ! -e $WORKDIR/$n; then
          echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
         else
            lcg-cp $srmdebug -b -D srmv2 file:$WORKDIR/$n $SERESULTDIR/$n
            if test $? -ne 0; then
               echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
            fi
     fi
     done
  fi
  
  echo "Cleaning up $WORKDIR"
  rm -rf $WORKDIR
  
  ###########################################################################
  DATE_END=`date +%s`
  RUNTIME=$((DATE_END-DATE_START))
  echo "################################################################"
  echo "Job finished at " `date`
  echo "Wallclock running time: $RUNTIME s"
  exit 0
  
</file>


 ===== Example 2 =====

Trimmed version.


<file text submitExample2.sh>

#! /bin/bash
# https://wiki.chipp.ch/twiki/bin/view/CmsTier3/HowToSubmitJobs
# run with qsub -q all.q submitAnalysis.sh

printf "###################################################\n"
printf "##   Run Analysis with input %-20s##\n" $1
printf "###################################################\n"

DBG=2 # debugging level
INPUT=$1;
JOBLOGFILES="myout.txt myerr.txt"
OUTFILES="*.png"
TOPWORKDIR=/scratch/`whoami`
JOBDIR="analysis/$1"
BASEDIR="/shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder"
CMSSW_DIR="/shome/myusername/CMSSW_5_3_24"

WORKDIR=$TOPWORKDIR/$JOBDIR
OUTDIR=$BASEDIR/$JOBDIR     # for output files
RESULTDIR=$BASEDIR/$JOBDIR  # for log files

# to save large output files on storage element
SEUSERSUBDIR="analysis/$1"
SEOUTFILES="*.root"
USER_SE_HOME="root://t3dcachedb.psi.ch:1094/pnfs/psi.ch/cms/trivcat/store/user/$USER"
SERESULTDIR=$USER_SE_HOME/$SEUSERSUBDIR



##### MONITORING/DEBUG INFORMATION ########################################

# write job report (stdout of this script)
mkdir -p /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -o /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -e /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis

DATE_START=`date +%s`
echo "Job started at " `date`
cat <<EOF

###########################################
##       QUEUEING SYSTEM SETTINGS:       ##
###########################################
  HOME=$HOME
  USER=$USER
  JOB_ID=$JOB_ID
  JOB_NAME=$JOB_NAME
  HOSTNAME=$HOSTNAME
  TASK_ID=$TASK_ID
  QUEUE=$QUEUE
EOF

if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################"
    echo "##         Environment Variables         ##"
    echo "###########################################"
    env
fi



##### SET ENVIRONMENT #####################################################

mkdir -p $WORKDIR
if test ! -d "$WORKDIR"; then
    echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
    exit 1
fi
cd $WORKDIR

cat <<EOF

###########################################
##             JOB SETTINGS:             ##
###########################################
  BASEDIR=$BASEDIR
  WORKDIR=$WORKDIR
  RESULTDIR=$RESULTDIR
  SERESULTDIR=$SERESULTDIR
EOF



echo " "
echo "###########################################"
echo "##         MY FUNCTIONALITY CODE         ##"
echo "###########################################"

source $VO_CMS_SW_DIR/cmsset_default.sh 2>> myerr.txt
cd $CMSSW_DIR/src
eval `scramv1 runtime -sh` >> myout.txt 2>> myerr.txt
if test $? -ne 0; then
    echo "ERROR: Failed to source scram environment" >&2
    exit 1
fi

cd $WORKDIR
echo "python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt"
python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt
ls >> myout.txt 2>> myerr.txt



##### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ###########################

cd $WORKDIR
if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################################"
    echo "##   MY OUTPUT WILL BE MOVED TO \$RESULTDIR and \$OUTDIR   ##"
    echo "###########################################################"
    echo "  \$RESULTDIR=$RESULTDIR"
    echo "  \$OUTDIR=$OUTDIR"
    echo "pwd: " `pwd`
    ls -Rl
fi

# copy log files to $RESULTDIR
if test x"$JOBLOGFILES" != x; then
    mkdir -p $RESULTDIR
    if test ! -e "$RESULTDIR"; then
        echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
    fi
    for n in $JOBLOGFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $RESULTDIR"
            cp -a $WORKDIR/$n $RESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
            fi
    fi
    done
fi

# copy output files to $OUTDIR
if test x"$OUTFILES" != x; then
    mkdir -p $OUTDIR
    if test ! -e "$OUTDIR"; then
          echo "ERROR: Failed to create $OUTDIR ...Aborting..." >&2
    fi
    for n in $OUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $OUTDIR"
	        cp -a $WORKDIR/$n $OUTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $OUTDIR/$n" >&2
            fi
        fi
    done
fi


# copy large files to $SERESULTDIR
if test x"$SEOUTFILES" != x; then
    if test ! -e "$SERESULTDIR"; then
          echo "ERROR: Failed to create $SERESULTDIR ...Aborting..." >&2
    fi
    if test 0"$DBG" -ge 2; then
        srmdebug="-v"
    fi
    for n in $SEOUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo "xrdcp -f $WORKDIR/$n $SERESULTDIR/$n" >&2
            xrdcp $srmdebug -d $DBG -f $WORKDIR/$n $SERESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
            fi
        fi
    done
fi

# cleaning working directory
echo ">>> Cleaning up $WORKDIR"
rm -rf $WORKDIR



###########################################################################

DATE_END=`date +%s`
RUNTIME=$((DATE_END-DATE_START))
echo " "
echo "#####################################################"
echo "    Job finished at " `date`
echo "    Wallclock running time: $(( $RUNTIME / 3600 )):$(( $RUNTIME % 3600 /60 )):$(( $RUNTIME % 60 )) "
echo "#####################################################"
echo " "
exit 0

</file>