justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 2363, Stage 1

Priority	50
Processors	1
Wall seconds	80000
RSS bytes	4193255424 (3999 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_LANCASTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_LANCASTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	vd-protodune
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
1	Rucio vd-protodune:vd-protodune-reco_2363	*reco.root	5184000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/1eda8a4ae6da0378b3150f8da76b587f12b53fe8

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
500	0	0	0	0	500	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
717	0	0	0	0	667	0	4	9	0	0	37

RSEs used

Name	Inputs	Outputs
MONTECARLO	508	0
DUNE_US_FNAL_DISK_STAGE	0	251
DUNE_CERN_EOS	0	84
RAL-PP	0	78
NIKHEF	0	40
DUNE_FR_CCIN2P3_DISK	0	15
QMUL	0	13
DUNE_ES_PIC	0	7
RAL_ECHO	0	6
DUNE_US_BNL_SDCC	0	4
SURFSARA	0	2

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
CERN	8	0

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_88_00d00}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

env

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '` 

if [ -z ${LINENUMBER} ] ; then
  LINENUMBER=$pfn
fi

# echo "linenumber " $LINENUMBER
ex_code=0
wstep=0 
i_pstep=10000
i_tstep=1000 

now=$(date -u +"%Y%m%dT%H%M%SZ")
namespace=${JUSTIN_SCOPE:-"usertests"}
  

# Run Generator
echo "generator" 
#prodname="prod_cosmics_protodunevd_${now}_gen_${pfn}"
prodname="prod_beam_protodunevd_${now}_${pfn}_gen"
wstep=$((wstep+1))  
istep=1

pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep)) 
 lar -c gen_protodunevd_electron_1GeV.fcl -o ${prodname}.root -n 20
#  lar -c gen_protodunevd_electron_1GeV.fcl -o ${prodname}.root -n 1
   ex_code=$?
if [ $ex_code -ne 0 ]; then 
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (generation) exit code: $c_ex_code " 
  files=`ls *_${now}_*`
  for f in $files 
    do
      size=`stat -c %s $f`
      echo "written output file: $f $size"
    done 
else
  istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt

extractor_prod.py --infile ${prodname}.root  --appfamily art --appname gen --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json ${INPUT_DIR}/pdvd_input_nop.json >    ${prodname}.root.ext.json && sed -i -e 's/stepfcl/gen_protodunevd_cosmics/g' ${prodname}.root.ext.json
 
python ${INPUT_DIR}/pdjson2metadata ${prodname}.root.ext.json all-input-dids.txt usertests > ${prodname}.root.json
 
    ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
      c_ex_code=$(($pstep+$tstep+$ex_code))
      echo "ERROR: metadata extraction (generation) exit code: $c_ex_code"
      files=`ls *_${now}_*`
      for f in $files 
         do
           size=`stat -c %s $f`
           echo "written output file: $f $size"
       done 
   fi
fi 


echo "=====checking generation MD"

jq . ${prodname}.root.json


  awk '/"core.runs/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json > run.json
  awk '/"core.runs_subruns/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json >> run.json
  awk 'NR==1 {print; exit}' ${INPUT_DIR}/pdvd_input.json > pdvd_run_input.json
  cat run.json >> pdvd_run_input.json
  tail -n +2 ${INPUT_DIR}/pdvd_input.json  >> pdvd_run_input.json 
#echo "generation exit code " $ex_code   
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi
 
# Stage 1 G4
echo "G4 stage 1" 
g4stage1_name="${prodname}_g4_stage1"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c protodunevd_refactored_g4_stage1.fcl ${prodname}.root -o ${g4stage1_name}.root
 
   ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (geant step1) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
       do
         size=`stat -c %s $f`
         echo "written output file: $f $size"
       done 
else
  istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt


extractor_prod.py --infile ${g4stage1_name}.root --appfamily art --appname g4-stage1 --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json pdvd_run_input.json > ${g4stage1_name}.root.ext.json && sed -i -e 's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage1/g' ${g4stage1_name}.root.ext.json    

python ${INPUT_DIR}/pdjson2metadata ${g4stage1_name}.root.ext.json all-input-dids.txt usertests > ${g4stage1_name}.root.json 

     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
     echo "ERROR: metadata extraction (geant step1) exit code: $c_ex_code" 
     files=`ls *_${now}_*`
     for f in $files 
        do
          size=`stat -c %s $f`
          echo "written output file: $f $size"
        done 
    fi	
fi 
 
 
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi

echo "=============checking g4stage1 MD"

jq . ${g4stage1_name}.root.json


# Stage 2 G4  
echo "G4 stage 2"
g4stage2_name="${g4stage1_name}_g4_stage2"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c  protodunevd_refactored_g4_stage2_pureAr.fcl ${g4stage1_name}.root -o ${g4stage2_name}.root

   ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (geant step2) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
      do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done 
else
   istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt 
echo ${namespace}":"${g4stage1_name}.root > all-input-dids.txt

extractor_prod.py --infile ${g4stage2_name}.root  --appfamily art --appname g4-stage2 --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json pdvd_run_input.json > ${g4stage2_name}.root.ext.json && sed -i -e's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage2_pureAr/g' ${g4stage2_name}.root.ext.json 

python ${INPUT_DIR}/pdjson2metadata ${g4stage2_name}.root.ext.json all-input-dids.txt usertests > ${g4stage2_name}.root.json 


     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (geant step2) exit code: $c_ex_code"
       files=`ls *_${now}_*`
        for f in $files 
          do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done 
    fi
fi 
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi


echo "========== checking g4stage2 MD"

jq . ${g4stage2_name}.root.json 
 
# Detsim
echo "detsim"
detsim_name="${g4stage2_name}_detsim"
wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c protodunevd_refactored_detsim_pureAr.fcl ${g4stage2_name}.root  -o ${detsim_name}.root 

  ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (detsim) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
      do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done   
else    
 istep=$((istep+1))
   tstep=$(($tstep*$istep))
    

rm -fr all-input-dids.txt 
echo ${namespace}":"${g4stage2_name}.root > all-input-dids.txt

extractor_prod.py --infile ${detsim_name}.root  --appfamily art --appname detsim --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json ${INPUT_DIR}/pdvd_input.json > ${detsim_name}.root.ext.json  && sed -i -e 's/stepfcl/protodunevd_refactored_detsim_pureAr/g' ${detsim_name}.root.ext.json
python ${INPUT_DIR}/pdjson2metadata ${detsim_name}.root.ext.json all-input-dids.txt usertests >  ${detsim_name}.root.json

     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (detsim) exit code: $c_ex_code" 
       files=`ls *_${now}_*` 
         for f in $files 
           do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done         
    fi
fi 

if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi


echo "============= checking detsim MD "

jq . ${detsim_name}.root.json 
 


# Reco
echo "reco"
reco_name="${detsim_name}_reco"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  


 lar -c protodunevd_reco.fcl ${detsim_name}.root -o ${reco_name}.root

 ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code)) 
   echo "ERROR: lar (reco) exit code: $c_ex_code"
   files=`ls *_${now}_*` 
     for f in $files 
       do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done   
else     
   istep=$((istep+1))
   tstep=$(($tstep*$istep))     
 
rm -fr all-input-dids.txt  
# echo ${namespace}":"${detsim_name}.root > all-input-dids.txt 
echo "noparents:noparents.root" > all-input-dids.txt


 
extractor_prod.py --infile ${reco_name}.root  --appfamily art --appname reco --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --strip_parents --input_json ${INPUT_DIR}/pdvd_input_nop.json > ${reco_name}.root.ext.json  && sed -i -e 's/stepfcl/protodunevd_reco/g' ${reco_name}.root.ext.json   
python ${INPUT_DIR}/pdjson2metadata ${reco_name}.root.ext.json all-input-dids.txt usertests > ${reco_name}.root.json 

    ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (reco) exit code: $c_ex_code" 
       files=`ls *_${now}_*` 
         for f in $files 
           do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done                 
    fi
fi 
#overrides=""
#python ${INPUT_DIR}/pdhd_meta_writer.py \
#       --json ${INPUT_DIR}/pdhd_base_meta.json \
#       --overrides ${overrides} \
#       --parent ${prodname}.root \
#       -o ${reco_name}.root.ext.json

echo "===========checking reco MD"

jq . ${reco_name}.root.json 
 


if [ $? -ne 0 ]
then
  echo "Exiting with error"
  exit 1
else
  files=`ls *_${now}_*` 
   for f in $files 
      do
       size=`stat -c %s $f`
       echo "written output file: $f $size"
      done  

  echo "$pfn" > justin-processed-pfns.txt
fi

justIN time: 2024-09-29 09:27:03 UTC       justIN version: 01.01.08