Workflow 5116, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4193255424 (3999 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UConn-HPC, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/duneproshift/fdvd_marley_es/05116/1/1	*reco.root
2	https://fndcadoor.fnal.gov:2880/dune/scratch/users/duneproshift/fdvd_marley_es/05116/1/2	*reco.root.json

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/dcde153bc779174c7c1d23f07ef8a1bdab9eba68

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10	0	1	0	0	0	0	9

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
125	0	0	0	0	62	0	0	15	0	48	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	59	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
US_NotreDame	3	2
UK_RAL-Tier1	1	6
UK_Manchester	1	11
NL_NIKHEF	0	21
CZ_FZU	0	4
UK_RAL-PPD	0	1

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d00}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#env

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '` 

if [ -z ${LINENUMBER} ] ; then
  LINENUMBER=$pfn
fi

# echo "linenumber " $LINENUMBER
ex_code=0
pstep=10
tstep=0

now=$(date -u +"%Y%m%dT%H%M%SZ")
namespace=${JUSTIN_SCOPE:-"usertests"} 

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER

# define run number and set number of events
firstsubrun=0
#nevts=100
nevts=2
e_pfn=$(echo "$pfn" | sed 's/^0*//')
start_e_pfn=$((e_pfn -1))
firstevent=$((start_e_pfn * nevts + 1))

echo "======checking===="
echo ${DID_PFN_RSE}
echo ${pfn}
echo $runid
echo $firstsubrun
echo $nevts
echo $firstevent


  
# Run Generator
echo "============generator=========================" 
prodname="prodmarley_nue_es_flat_dunevd10kt_1x8x14_3view_30deg_${now}_gen_${pfn}" 
istep=1
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
tcode=$((nstep+istep))
tcode=$((tcode*1000)) 
 

#lar -c prodmarley_nue_es_flat_dunevd10kt_1x8x14_3view_30deg.fcl -o ${prodname}.root -n 100
lar -c prodmarley_nue_es_flat_dunevd10kt_1x8x14_3view_30deg.fcl -o ${prodname}.root -n $nevts -e ${runid}:${firstsubrun}:${firstevent}

exit_code=$?  
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
 echo "ERROR: lar (generation) exit code: $ex_code " 
 echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi
 
# Stage 1 G4
echo "============G4 stage1=========================" 
 
g4stage1_name="${prodname}_supernova_g4stage1"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000))


lar -c supernova_g4stage1_dunevd10kt_1x8x14_3view_30deg.fcl   ${prodname}.root -o ${g4stage1_name}.root -n -1

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (geant4 step1) exit code: $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi
 

# Stage 2 G4  
echo "============G4 stage2=========================" 
g4stage2_name="${g4stage1_name}_g4stage2"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000))

lar -c  standard_g4stage2_dunevd10kt_1x8x14_3view_30deg.fcl ${g4stage1_name}.root -o ${g4stage2_name}.root -n -1

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (geant4 step2) exit code: $ex_code " 
    echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi
 
# Detsim
echo "============detsim========================="  
detsim_name="${g4stage2_name}_detsim"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1  
tcode=$((nstep+istep))
tcode=$((tcode*1000))

lar -c standard_detsim_dunevd10kt_1x8x14_3view_30deg.fcl ${g4stage2_name}.root  -o ${detsim_name}.root -n -1

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then 
  echo "ERROR: lar (detsim) exit code: $ex_code " 
  echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi 
 
 
# Reco
echo "============reco ========================="  
reco_name="${detsim_name}_reco"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000)) 

lar -c standard_reco1_dunevd10kt_1x8x14_3view_30deg.fcl ${detsim_name}.root -o ${reco_name}.root -n -1

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (reco) exit code: $ex_code " 
  echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi 

istep=$((istep+1))
tcode=$((nstep+istep))
tcode=$((tcode*1000))  
extractor_prod.py --infile ${reco_name}.root --no_crc    --appfamily art --appname reco --appversion  v09_91_04d00  --requestid ritm2205749  --strip_parents --input_json ${INPUT_DIR}/le_input.json> ${reco_name}.root.ext.json  && sed -i -e 's/stepfcl/standard_reco1_dunevd10kt_1x8x14_3view_30deg.fcl/g' ${reco_name}.root.ext.json 

exit_code=$?   
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -gt 1 ]; then 
  echo "ERROR: metadata generation   $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi 
 

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt 
python ${INPUT_DIR}/pdjson2metadata ${reco_name}.root.ext.json  all-input-dids.txt usertests > ${reco_name}.root.temp.json 
sed '/DUNE/ s/.*/\L&/' ${reco_name}.root.temp.json > ${reco_name}.root.json 
exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*` 
if [ $exit_code -ne 0 ]; then
  echo "ERROR: metadata writing  $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi  
 



if [ $? -ne 0 ]
then
  echo "Exiting with error"
  return 1
else
  files=`ls *_${now}_*` 
   for f in $files 
      do
       size=`stat -c %s $f`
       echo "written output file: $f $size"
      done  

  echo "$pfn" > justin-processed-pfns.txt
fi