#! /bin/bash
# https://wiki.chipp.ch/twiki/bin/view/CmsTier3/HowToSubmitJobs
# run with qsub -q all.q submitAnalysis.sh

printf "###################################################\n"
printf "##   Run Analysis with input %-20s##\n" $1
printf "###################################################\n"

DBG=2 # debugging level
INPUT=$1;
JOBLOGFILES="myout.txt myerr.txt"
OUTFILES="*.png"
TOPWORKDIR=/scratch/`whoami`
JOBDIR="analysis/$1"
BASEDIR="/shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder"
CMSSW_DIR="/shome/myusername/CMSSW_5_3_24"

WORKDIR=$TOPWORKDIR/$JOBDIR
OUTDIR=$BASEDIR/$JOBDIR     # for output files
RESULTDIR=$BASEDIR/$JOBDIR  # for log files

# to save large output files on storage element
SEUSERSUBDIR="analysis/$1"
SEOUTFILES="*.root"
USER_SE_HOME="root://t3dcachedb.psi.ch:1094/pnfs/psi.ch/cms/trivcat/store/user/$USER"
SERESULTDIR=$USER_SE_HOME/$SEUSERSUBDIR



##### MONITORING/DEBUG INFORMATION ########################################

# write job report (stdout of this script)
mkdir -p /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -o /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis
#$ -e /shome/myusername/CMSSW_5_3_24/src/myAnalysisFolder/analysis

DATE_START=`date +%s`
echo "Job started at " `date`
cat <<EOF

###########################################
##       QUEUEING SYSTEM SETTINGS:       ##
###########################################
  HOME=$HOME
  USER=$USER
  JOB_ID=$JOB_ID
  JOB_NAME=$JOB_NAME
  HOSTNAME=$HOSTNAME
  TASK_ID=$TASK_ID
  QUEUE=$QUEUE
EOF

if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################"
    echo "##         Environment Variables         ##"
    echo "###########################################"
    env
fi



##### SET ENVIRONMENT #####################################################

mkdir -p $WORKDIR
if test ! -d "$WORKDIR"; then
    echo "ERROR: Failed to create workdir ($WORKDIR)! Aborting..." >&2
    exit 1
fi
cd $WORKDIR

cat <<EOF

###########################################
##             JOB SETTINGS:             ##
###########################################
  BASEDIR=$BASEDIR
  WORKDIR=$WORKDIR
  RESULTDIR=$RESULTDIR
  SERESULTDIR=$SERESULTDIR
EOF



echo " "
echo "###########################################"
echo "##         MY FUNCTIONALITY CODE         ##"
echo "###########################################"

source $VO_CMS_SW_DIR/cmsset_default.sh 2>> myerr.txt
cd $CMSSW_DIR/src
eval `scramv1 runtime -sh` >> myout.txt 2>> myerr.txt
if test $? -ne 0; then
    echo "ERROR: Failed to source scram environment" >&2
    exit 1
fi

cd $WORKDIR
echo "python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt"
python $BASEDIR/myAnalysis.py $INPUT >> myout.txt 2>> myerr.txt
ls >> myout.txt 2>> myerr.txt



##### RETRIEVAL OF OUTPUT FILES AND CLEANING UP ###########################

cd $WORKDIR
if test 0"$DBG" -gt 0; then
    echo " "
    echo "###########################################################"
    echo "##   MY OUTPUT WILL BE MOVED TO \$RESULTDIR and \$OUTDIR   ##"
    echo "###########################################################"
    echo "  \$RESULTDIR=$RESULTDIR"
    echo "  \$OUTDIR=$OUTDIR"
    echo "pwd: " `pwd`
    ls -Rl
fi

# copy log files to $RESULTDIR
if test x"$JOBLOGFILES" != x; then
    mkdir -p $RESULTDIR
    if test ! -e "$RESULTDIR"; then
        echo "ERROR: Failed to create $RESULTDIR ...Aborting..." >&2
    fi
    for n in $JOBLOGFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $RESULTDIR"
            cp -a $WORKDIR/$n $RESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $RESULTDIR/$n" >&2
            fi
    fi
    done
fi

# copy output files to $OUTDIR
if test x"$OUTFILES" != x; then
    mkdir -p $OUTDIR
    if test ! -e "$OUTDIR"; then
          echo "ERROR: Failed to create $OUTDIR ...Aborting..." >&2
    fi
    for n in $OUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo ">>> copying $n from $WORKDIR to $OUTDIR"
	        cp -a $WORKDIR/$n $OUTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $OUTDIR/$n" >&2
            fi
        fi
    done
fi


# copy large files to $SERESULTDIR
if test x"$SEOUTFILES" != x; then
    if test ! -e "$SERESULTDIR"; then
          echo "ERROR: Failed to create $SERESULTDIR ...Aborting..." >&2
    fi
    if test 0"$DBG" -ge 2; then
        srmdebug="-v"
    fi
    for n in $SEOUTFILES; do
        if test ! -e $WORKDIR/$n; then
            echo "WARNING: Cannot find output file $WORKDIR/$n. Ignoring it" >&2
        else
            echo "xrdcp -f $WORKDIR/$n $SERESULTDIR/$n" >&2
            xrdcp $srmdebug -d $DBG -f $WORKDIR/$n $SERESULTDIR/$n
            if test $? -ne 0; then
                echo "ERROR: Failed to copy $WORKDIR/$n to $SERESULTDIR/$n" >&2
            fi
        fi
    done
fi

# cleaning working directory
echo ">>> Cleaning up $WORKDIR"
rm -rf $WORKDIR



###########################################################################

DATE_END=`date +%s`
RUNTIME=$((DATE_END-DATE_START))
echo " "
echo "#####################################################"
echo "    Job finished at " `date`
echo "    Wallclock running time: $(( $RUNTIME / 3600 )):$(( $RUNTIME % 3600 /60 )):$(( $RUNTIME % 60 )) "
echo "#####################################################"
echo " "
exit 0