justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 5214, Stage 1

Priority50
Processors1
Wall seconds80000
RSS bytes8388608000 (8000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UConn-HPC, US_Wisconsin
Scopefardet-vd
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1Rucio fardet-vd:fardet-vd-reco_5214-w5214s1p1*reco.root10368000FalseDUNE_US_FNAL_DISK_STAGE

Environment variables

NameValue
INPUT_DIR/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/9227c25e1635d18779282cfab801fcdc7cd063b1

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
50000000500000

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
599400005715025254000
Files processed00100100200200300300400400500500600600700700800800900900100010001100110012001200Feb-19 11:00Feb-19 12:00Feb-19 13:00Feb-19 14:00Feb-19 15:00Feb-19 16:00Feb-19 17:00Feb-19 18:00Feb-19 19:00Feb-19 20:00Feb-19 21:00Feb-19 22:00Feb-19 23:00Feb-20 00:00Feb-20 01:00Feb-20 02:00Feb-20 03:00Feb-20 04:00Feb-20 05:00Files processedBin start timesNumber per binCERNFR_CCIN2P3ES_PICUS_WisconsinCZ_FZUNL_NIKHEFUK_RAL-Tier1US_FNAL-T1US_FNAL-FermiG…US_FNAL-FermiGridUK_DurhamUS_ColoradoUK_RAL-PPDNL_SURFsaraUK_ManchesterUK_BristolUK_BrunelUK_QMULCA_VictoriaUS_NotreDameUK_Liverpool

RSEs used

NameInputsOutputs
MONTECARLO52740
DUNE_US_FNAL_DISK_STAGE04469
RAL_ECHO011
DUNE_CERN_EOS03
RAL-PP03
NIKHEF02
DUNE_US_BNL_SDCC01
SURFSARA01
DUNE_UK_MANCHESTER_CEPH01

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
US_NotreDame1460
UK_RAL-Tier1239
UK_Bristol110
ES_PIC101
UK_Manchester102
US_FNAL-FermiGrid101
CZ_FZU90
UK_RAL-PPD56
CERN51
NL_SURFsara43
US_Colorado30
FR_CCIN2P321
UK_Liverpool20
NL_NIKHEF14
UK_Durham10
UK_Brunel10
US_FNAL-T102
UK_QMUL01

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d00}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#env

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '` 

if [ -z ${LINENUMBER} ] ; then
  LINENUMBER=$pfn
fi

# echo "linenumber " $LINENUMBER
ex_code=0
pstep=10
tstep=0

now=$(date -u +"%Y%m%dT%H%M%SZ")
namespace=${JUSTIN_SCOPE:-"usertests"} 
echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER

# define run number and set number of events
firstsubrun=0
#nevts=1
nevts=10
e_pfn=$(echo "$pfn" | sed 's/^0*//')
start_e_pfn=$((e_pfn -1))
firstevent=$((start_e_pfn * nevts + 1))

echo "======checking===="
echo ${DID_PFN_RSE}
echo ${pfn}
echo $runid
echo $firstsubrun
echo $nevts
echo $firstevent



# Run Generator
echo "============generator=========================" 
prodname="prodbackground_radiological_decay0_dunevd10kt_1x8x14_${now}_gen_${pfn}" 
istep=1
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
tcode=$((nstep+istep))
tcode=$((tcode*1000)) 
 
#lar -c prodbackground_radiological_decay0_dunevd10kt_1x8x14.fcl -o ${prodname}.root -n 10 
lar -c prodbackground_radiological_decay0_dunevd10kt_1x8x14.fcl -o ${prodname}.root -n $nevts -e ${runid}:${firstsubrun}:${firstevent}

exit_code=$?  
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
 echo "ERROR: lar (generation) exit code: $ex_code " 
 echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi
 
# Stage 1 G4
echo "============G4 stage1=========================" 
g4stage1_name="${prodname}_supernova_g4stage1"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000))

lar -c supernova_g4stage1_dunevd10kt_1x8x14_3view_30deg.fcl   ${prodname}.root -o ${g4stage1_name}.root

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (geant4 step1) exit code: $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi
 

# Stage 2 G4  
echo "============G4 stage2=========================" 
g4stage2_name="${g4stage1_name}_g4stage2"
tstep=$((tstep+1))	 
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000))

lar -c  standard_g4stage2_dunevd10kt_1x8x14_3view_30deg.fcl ${g4stage1_name}.root -o ${g4stage2_name}.root

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (geant4 step2) exit code: $ex_code " 
    echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi
 
# Detsim
echo "============detsim========================="  
detsim_name="${g4stage2_name}_detsim"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1

lar -c standard_detsim_dunevd10kt_1x8x14_3view_30deg.fcl ${g4stage2_name}.root  -o ${detsim_name}.root 

exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then 
  echo "ERROR: lar (detsim) exit code: $ex_code " 
  echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi  
 
# Reco
echo "============reco =========================" 
reco_name="${detsim_name}_reco"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep)) 
istep=1 
tcode=$((nstep+istep))
tcode=$((tcode*1000)) 

lar -c standard_reco1_dunevd10kt_1x8x14_3view_30deg.fcl ${detsim_name}.root -o ${reco_name}.root


exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -ne 0 ]; then
  echo "ERROR: lar (reco) exit code: $ex_code " 
  echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi 

istep=$((istep+1))
tcode=$((nstep+istep))
tcode=$((tcode*1000))  
extractor_prod.py --infile ${reco_name}.root --no_crc    --appfamily art --appname reco --appversion  v09_91_04d00  --requestid ritm2205749  --strip_parents --input_json ${INPUT_DIR}/le_input.json> ${reco_name}.root.ext.json  && sed -i -e 's/stepfcl/standard_reco1_dunevd10kt_1x8x14_3view_30deg.fcl/g' ${reco_name}.root.ext.json 

exit_code=$?   
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*`  
if [ $exit_code -gt 1 ]; then 
  echo "ERROR: metadata generation   $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
 return $ex_code 
fi 
 

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt 
python ${INPUT_DIR}/pdjson2metadata ${reco_name}.root.ext.json  all-input-dids.txt usertests > ${reco_name}.root.temp.json 
sed '/DUNE/ s/.*/\L&/' ${reco_name}.root.temp.json > ${reco_name}.root.json

 
exit_code=$?    
ex_code=$((exit_code+tcode))
files=`ls *_${now}_*` 
if [ $exit_code -ne 0 ]; then
  echo "ERROR: metadata writing  $ex_code " 
   echo "output files size: "
     for f in $files 
      do
       size=`stat -c %s $f`
       echo   $f $size 
      done   
  return $ex_code 
fi  
 
 

if [ $? -ne 0 ]
then
  echo "Exiting with error"
  return 1
else
  files=`ls *_${now}_*` 
   for f in $files 
      do
       size=`stat -c %s $f`
       echo "written output file: $f $size"
      done  

  echo "$pfn" > justin-processed-pfns.txt
fi
justIN time: 2025-02-23 21:09:57 UTC       justIN version: 01.02.01