CMS Wiki Pages

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
Two examples of submission scripts to PSI's T3 batch system.


 ===== Example 1 =====

<file text submitExample.sh>

  #!/bin/bash
  #################################
  # PSI Tier-3 example batch Job  #
  #################################
  
  ##### CONFIGURATION ##############################################
  # Output files to be copied back to the User Interface
  # (the file path must be given relative to the working directory)
  OUTFILES="myout.txt myerr.txt"
  
  # Output files to be copied to the SE
  # (as above the file path must be given relative to the working directory)
  SEOUTFILES="mybigfile" 
  #
  # By default, the files will be copied to $USER_SRM_HOME/$username/$JOBDIR,
  # but here you can define the subdirectory under your SE storage path
  # to which files will be copied (uncomment line)
  #SEUSERSUBDIR="mytestsubdir/somedir"
  #
  # User's CMS hypernews name (needed for user's SE storage home path
  # USER_SRM_HOME below)
  HN_NAME=your-hn-name
  
  # set DBG=1 for additional debug output in the job report files
  # DBG=2 will also give detailed output on SRM operations
  DBG=0
  
  #### The following configurations you should not need to change
  # The SE's user home area (SRMv2 URL)
  USER_SRM_HOME="srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/trivcat/store/user/"
  
  # Top working directory on worker node's local disk. The batch
  # job working directory will be created below this
  TOPWORKDIR=/scratch/`whoami`
  
  # Basename of job sandbox (job workdir will be $TOPWORKDIR/$JOBDIR)
  JOBDIR=sgejob-$JOB_ID
  ##################################################################
  
  
  ############ BATCH QUEUE DIRECTIVES ##############################
  # Lines beginning with #$ are used to set options for the SGE
  # queueing system (same as specifying these options to the qsub
  # command
  
  # Job name (defines name seen in monitoring by qstat and the
  #     job script's stderr/stdout names)
  #$ -N example_job
  
  ### Specify the queue on which to run
  #$ -q all.q
  
  # Change to the current working directory from which the job got
  # submitted (will also result in the job report stdout/stderr being
  # written to this directory)
  #$ -cwd
  
  # here you could change location of the job report stdout/stderr files
  #  if you did not want them in the submission directory
  #  #$ -o /shome/username/mydir/
  #  #$ -e /shome/username/mydir/
  
  ##################################################################
  
  
  
  ##### MONITORING/DEBUG INFORMATION ###############################
  DATE_START=`date +%s`
  echo "Job started at " `date`
  cat <<EOF
  ################################################################
  ## QUEUEING SYSTEM SETTINGS:
  HOME=$HOME
  USER=$USER
  JOB_ID=$JOB_ID
  JOB_NAME=$JOB_NAME
  HOSTNAME=$HOSTNAME
  TASK_ID=$TASK_ID
  QUEUE=$QUEUE
  
  EOF
  
  echo "################################################################"
  
  if test 0"$DBG" -gt 0; then
     echo "######## Environment Variables ##########"
     env
     echo "################################################################"
  fi
  
  
  ##### SET UP WORKDIR AND ENVIRONMENT ######################################
  STARTDIR=`pwd`
  WORKDIR=$TOPWORKDIR/$JOBDIR
  RESULTDIR=$STARTDIR/$JOBDIR
  if test x"$SEUSERSUBDIR" = x; then
     SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$JOBDIR
  else
     SERESULTDIR=$USER_SRM_HOME/$HN_NAME/$SEUSERSUBDIR
  fi
  if test -e "$WORKDIR"; then
     echo "ERROR: WORKDIR ($WORKDIR) already exists! Aborting..." >&2
     exit 1
  fi
  mkdir -p $WORKDIR
  if test ! -d "$WORKDIR"; then
     echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
     exit 1
  fi
  
  cd $WORKDIR
  cat <<EOF
  ################################################################
  ## JOB SETTINGS:
  STARTDIR=$STARTDIR
  WORKDIR=$WORKDIR
  RESULTDIR=$RESULTDIR
  SERESULTDIR=$SERESULTDIR
  EOF
  
  ###########################################################################
  ## YOUR FUNCTIONALITY CODE GOES HERE
  
  # set up CMS environment
  source $VO_CMS_SW_DIR/cmsset_default.sh
  
  # Here we produce some output for the files that are copied back to
  #  our shared home
  scramv1 list > myout.txt 2>myerr.txt
  
  lcg-ls -b -D srmv2 -l srm://t3se01.psi.ch:8443/srm/managerv2?SFN=/pnfs/psi.ch/cms/ >> myout.txt 2>>myerr.txt
  
  # create a dummy file for copying back to the SE
  dd if=/dev/urandom of=mybigfile count=100 &>/dev/null
  
  CMSSW_DIR=/shome/feichtinger/cmssw_test/CMSSW_2_1_6
  CMSSW_CONFIG_FILE=$CMSSW_DIR/demoanalyzer-classic.cfg
  
  cd $CMSSW_DIR/src
  eval `scramv1 runtime -sh`
  if test $? -ne 0; then
     echo "ERROR: Failed to source scram environment" >&2
     exit 1
  fi
  
  cd $WORKDIR
  cmsRun $CMSSW_CONFIG_FILE maxEvents=$1 firstEvent=$2 inputFiles=$3 seed=$4> myout.txt 2>myerr.txt
  
  #### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ############################
  cd $WORKDIR
  if test 0"$DBG" -gt 0; then
      echo "########################################################"
      echo "############# Working directory contents ###############"
      echo "pwd: " `pwd`
      ls -Rl
      echo "########################################################"
      echo "YOUR OUTPUT WILL BE MOVED TO $RESULTDIR"
      echo "########################################################"
  fi
  
  if test x"$OUTFILES" != x; then
     mkdir -p $RESULTDIR
     if test ! -e "$RESULTDIR"; then
            echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
            exit 1
     fi
     for n in $OUTFILES; do
         if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
         else
            cp -a $WORKDIR/$n $RESULTDIR/$n
            if test $? -ne 0; then
               echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
            fi
     fi
     done
  fi
  
  if test x"$SEOUTFILES" != x; then
     if test 0"$DBG" -ge 2; then
        srmdebug="-v"
     fi
     for n in $SEOUTFILES; do
         if test ! -e $WORKDIR/$n; then
          echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
         else
            lcg-cp $srmdebug -b -D srmv2 file:$WORKDIR/$n $SERESULTDIR/$n
            if test $? -ne 0; then
               echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
            fi
     fi
     done
  fi
  
  echo "Cleaning up $WORKDIR"
  rm -rf $WORKDIR
  
  ###########################################################################
  DATE_END=`date +%s`
  RUNTIME=$((DATE_END-DATE_START))
  echo "################################################################"
  echo "Job finished at " `date`
  echo "Wallclock running time: $RUNTIME s"
  exit 0
  
</file>


 ===== Example 2 =====

Trimmed version.


<file text submitExample2.sh>

#! /bin/bash
# https://wiki.chipp.ch/twiki/bin/view/CmsTier3/HowToSubmitJobs
# run with qsub -q all.q submitAnalysis.sh

printf "###################################################\n"
printf "##   Run Analysis with input %-20s##\n" $1
printf "###################################################\n"

DBG=2 # debugging level
INPUT=$1;
JOBLOGFILES="myout.txt myerr.txt"
OUTFILES="*.png"
TOPWORKDIR=/scratch/`whoami`
JOBDIR="analysis/$1"
BASEDIR="/shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder"
CMSSW_DIR="/shome/myusername/CMSSW_5_3_24"

WORKDIR=$TOPWORKDIR/$JOBDIR
OUTDIR=$BASEDIR/$JOBDIR     # for output files
RESULTDIR=$BASEDIR/$JOBDIR  # for log files

# to save large output files on storage element
SEUSERSUBDIR="analysis/$1"
SEOUTFILES="*.root"
USER_SE_HOME="root://t3dcachedb.psi.ch:1094/pnfs/psi.ch/cms/trivcat/store/user/$USER"
SERESULTDIR=$USER_SE_HOME/$SEUSERSUBDIR



##### MONITORING/DEBUG INFORMATION ########################################

# write job report (stdout of this script)
mkdir -p /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -o /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -e /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis

DATE_START=`date +%s`
echo "Job started at " `date`
cat <<EOF

###########################################
##       QUEUEING SYSTEM SETTINGS:       ##
###########################################
  HOME=$HOME
  USER=$USER
  JOB_ID=$JOB_ID
  JOB_NAME=$JOB_NAME
  HOSTNAME=$HOSTNAME
  TASK_ID=$TASK_ID
  QUEUE=$QUEUE
EOF

if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################"
    echo "##         Environment Variables         ##"
    echo "###########################################"
    env
fi



##### SET ENVIRONMENT #####################################################

mkdir -p $WORKDIR
if test ! -d "$WORKDIR"; then
    echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
    exit 1
fi
cd $WORKDIR

cat <<EOF

###########################################
##             JOB SETTINGS:             ##
###########################################
  BASEDIR=$BASEDIR
  WORKDIR=$WORKDIR
  RESULTDIR=$RESULTDIR
  SERESULTDIR=$SERESULTDIR
EOF



echo " "
echo "###########################################"
echo "##         MY FUNCTIONALITY CODE         ##"
echo "###########################################"

source $VO_CMS_SW_DIR/cmsset_default.sh 2>> myerr.txt
cd $CMSSW_DIR/src
eval `scramv1 runtime -sh` >> myout.txt 2>> myerr.txt
if test $? -ne 0; then
    echo "ERROR: Failed to source scram environment" >&2
    exit 1
fi

cd $WORKDIR
echo "python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt"
python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt
ls >> myout.txt 2>> myerr.txt



##### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ###########################

cd $WORKDIR
if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################################"
    echo "##   MY OUTPUT WILL BE MOVED TO \$RESULTDIR and \$OUTDIR   ##"
    echo "###########################################################"
    echo "  \$RESULTDIR=$RESULTDIR"
    echo "  \$OUTDIR=$OUTDIR"
    echo "pwd: " `pwd`
    ls -Rl
fi

# copy log files to $RESULTDIR
if test x"$JOBLOGFILES" != x; then
    mkdir -p $RESULTDIR
    if test ! -e "$RESULTDIR"; then
        echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
    fi
    for n in $JOBLOGFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $RESULTDIR"
            cp -a $WORKDIR/$n $RESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
            fi
    fi
    done
fi

# copy output files to $OUTDIR
if test x"$OUTFILES" != x; then
    mkdir -p $OUTDIR
    if test ! -e "$OUTDIR"; then
          echo "ERROR: Failed to create $OUTDIR ...Aborting..." >&2
    fi
    for n in $OUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $OUTDIR"
	        cp -a $WORKDIR/$n $OUTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $OUTDIR/$n" >&2
            fi
        fi
    done
fi


# copy large files to $SERESULTDIR
if test x"$SEOUTFILES" != x; then
    if test ! -e "$SERESULTDIR"; then
          echo "ERROR: Failed to create $SERESULTDIR ...Aborting..." >&2
    fi
    if test 0"$DBG" -ge 2; then
        srmdebug="-v"
    fi
    for n in $SEOUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo "xrdcp -f $WORKDIR/$n $SERESULTDIR/$n" >&2
            xrdcp $srmdebug -d $DBG -f $WORKDIR/$n $SERESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
            fi
        fi
    done
fi

# cleaning working directory
echo ">>> Cleaning up $WORKDIR"
rm -rf $WORKDIR



###########################################################################

DATE_END=`date +%s`
RUNTIME=$((DATE_END-DATE_START))
echo " "
echo "#####################################################"
echo "    Job finished at " `date`
echo "    Wallclock running time: $(( $RUNTIME / 3600 )):$(( $RUNTIME % 3600 /60 )):$(( $RUNTIME % 60 )) "
echo "#####################################################"
echo " "
exit 0

</file>
Physik-Institut

CMS Wiki Pages

User Tools

Site Tools

Page Tools
