justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 3349, Stage 1

Priority	50
Processors	1
Wall seconds	80000
RSS bytes	4193255424 (3999 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
1	Rucio usertests:vd-protodune-reco_3349	*reco.root	86400	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/a3f8177290472c19071e45c30c818d43c63c2238

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
1	0	0	0	0	1	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
2	0	0	0	0	2	0	0	0	0	0	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	1	0
DUNE_US_FNAL_DISK_STAGE	0	1

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_88_00d00}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

env

echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '` 

if [ -z ${LINENUMBER} ] ; then
  LINENUMBER=$pfn
fi

# echo "linenumber " $LINENUMBER
ex_code=0
wstep=0 
i_pstep=10000
i_tstep=1000 

now=$(date -u +"%Y%m%dT%H%M%SZ")
namespace=${JUSTIN_SCOPE:-"usertests"}
  

# Run Generator
echo "generator" 
#prodname="prod_cosmics_protodunevd_${now}_gen_${pfn}"
prodname="prod_beam_protodunevd_${now}_${pfn}_gen"
wstep=$((wstep+1))  
istep=1

pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep)) 
 lar -c gen_protodunevd_electron_1GeV.fcl -o ${prodname}.root -n 20
#  lar -c gen_protodunevd_electron_1GeV.fcl -o ${prodname}.root -n 1
   ex_code=$?
if [ $ex_code -ne 0 ]; then 
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (generation) exit code: $c_ex_code " 
  files=`ls *_${now}_*`
  for f in $files 
    do
      size=`stat -c %s $f`
      echo "written output file: $f $size"
    done 
else
  istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt

extractor_prod.py --infile ${prodname}.root  --appfamily art --appname gen --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json ${INPUT_DIR}/pdvd_input_nop.json >    ${prodname}.root.ext.json && sed -i -e 's/stepfcl/gen_protodunevd_cosmics/g' ${prodname}.root.ext.json
 
python ${INPUT_DIR}/pdjson2metadata ${prodname}.root.ext.json all-input-dids.txt usertests > ${prodname}.root.json
 
    ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
      c_ex_code=$(($pstep+$tstep+$ex_code))
      echo "ERROR: metadata extraction (generation) exit code: $c_ex_code"
      files=`ls *_${now}_*`
      for f in $files 
         do
           size=`stat -c %s $f`
           echo "written output file: $f $size"
       done 
   fi
fi 


echo "=====checking generation MD"

jq . ${prodname}.root.json


  awk '/"core.runs/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json > run.json
  awk '/"core.runs_subruns/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json >> run.json
  awk 'NR==1 {print; exit}' ${INPUT_DIR}/pdvd_input.json > pdvd_run_input.json
  cat run.json >> pdvd_run_input.json
  tail -n +2 ${INPUT_DIR}/pdvd_input.json  >> pdvd_run_input.json 
#echo "generation exit code " $ex_code   
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi
 
# Stage 1 G4
echo "G4 stage 1" 
g4stage1_name="${prodname}_g4_stage1"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c protodunevd_refactored_g4_stage1.fcl ${prodname}.root -o ${g4stage1_name}.root
 
   ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (geant step1) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
       do
         size=`stat -c %s $f`
         echo "written output file: $f $size"
       done 
else
  istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt
echo "noparents:noparents.root" > all-input-dids.txt


extractor_prod.py --infile ${g4stage1_name}.root --appfamily art --appname g4-stage1 --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json pdvd_run_input.json > ${g4stage1_name}.root.ext.json && sed -i -e 's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage1/g' ${g4stage1_name}.root.ext.json    

python ${INPUT_DIR}/pdjson2metadata ${g4stage1_name}.root.ext.json all-input-dids.txt usertests > ${g4stage1_name}.root.json 

     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
     echo "ERROR: metadata extraction (geant step1) exit code: $c_ex_code" 
     files=`ls *_${now}_*`
     for f in $files 
        do
          size=`stat -c %s $f`
          echo "written output file: $f $size"
        done 
    fi	
fi 
 
 
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi

echo "=============checking g4stage1 MD"

jq . ${g4stage1_name}.root.json


# Stage 2 G4  
echo "G4 stage 2"
g4stage2_name="${g4stage1_name}_g4_stage2"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c  protodunevd_refactored_g4_stage2_pureAr.fcl ${g4stage1_name}.root -o ${g4stage2_name}.root

   ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (geant step2) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
      do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done 
else
   istep=$((istep+1))
   tstep=$(($tstep*$istep))

rm -fr all-input-dids.txt 
echo ${namespace}":"${g4stage1_name}.root > all-input-dids.txt

extractor_prod.py --infile ${g4stage2_name}.root  --appfamily art --appname g4-stage2 --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json pdvd_run_input.json > ${g4stage2_name}.root.ext.json && sed -i -e's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage2_pureAr/g' ${g4stage2_name}.root.ext.json 

python ${INPUT_DIR}/pdjson2metadata ${g4stage2_name}.root.ext.json all-input-dids.txt usertests > ${g4stage2_name}.root.json 


     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (geant step2) exit code: $c_ex_code"
       files=`ls *_${now}_*`
        for f in $files 
          do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done 
    fi
fi 
if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi


echo "========== checking g4stage2 MD"

jq . ${g4stage2_name}.root.json 
 
# Detsim
echo "detsim"
detsim_name="${g4stage2_name}_detsim"
wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  

 lar -c protodunevd_refactored_detsim_pureAr.fcl ${g4stage2_name}.root  -o ${detsim_name}.root 

  ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code))
  echo "ERROR: lar (detsim) exit code: $c_ex_code"
  files=`ls *_${now}_*`
    for f in $files 
      do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done   
else    
 istep=$((istep+1))
   tstep=$(($tstep*$istep))
    

rm -fr all-input-dids.txt 
echo ${namespace}":"${g4stage2_name}.root > all-input-dids.txt

extractor_prod.py --infile ${detsim_name}.root  --appfamily art --appname detsim --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --input_json ${INPUT_DIR}/pdvd_input.json > ${detsim_name}.root.ext.json  && sed -i -e 's/stepfcl/protodunevd_refactored_detsim_pureAr/g' ${detsim_name}.root.ext.json
python ${INPUT_DIR}/pdjson2metadata ${detsim_name}.root.ext.json all-input-dids.txt usertests >  ${detsim_name}.root.json

     ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (detsim) exit code: $c_ex_code" 
       files=`ls *_${now}_*` 
         for f in $files 
           do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done         
    fi
fi 

if [ $ex_code -ne 0 ]; then exit $c_ex_code ; fi


echo "============= checking detsim MD "

jq . ${detsim_name}.root.json 
 


# Reco
echo "reco"
reco_name="${detsim_name}_reco"

wstep=$((wstep+1))  
istep=1
pstep=$(($wstep*$i_pstep)) 
tstep=$(($istep*$i_tstep))  


 lar -c protodunevd_reco.fcl ${detsim_name}.root -o ${reco_name}.root

 ex_code=$?

if [ $ex_code -ne 0 ]; then
  c_ex_code=$(($pstep+$tstep+$ex_code)) 
   echo "ERROR: lar (reco) exit code: $c_ex_code"
   files=`ls *_${now}_*` 
     for f in $files 
       do
        size=`stat -c %s $f`
        echo "written output file: $f $size"
      done   
else     
   istep=$((istep+1))
   tstep=$(($tstep*$istep))     
 
rm -fr all-input-dids.txt  
# echo ${namespace}":"${detsim_name}.root > all-input-dids.txt 
echo "noparents:noparents.root" > all-input-dids.txt


 
extractor_prod.py --infile ${reco_name}.root  --appfamily art --appname reco --appversion  v09_88_00d00  --no_crc --requestid ritm1998918  --strip_parents --input_json ${INPUT_DIR}/pdvd_input_nop.json > ${reco_name}.root.ext.json  && sed -i -e 's/stepfcl/protodunevd_reco/g' ${reco_name}.root.ext.json   
python ${INPUT_DIR}/pdjson2metadata ${reco_name}.root.ext.json all-input-dids.txt usertests > ${reco_name}.root.json 

    ex_code=$?
  if [ $ex_code -ne 0 ] ; then 
     c_ex_code=$(($pstep+$tstep+$ex_code)) 
       echo "ERROR: metadata extraction (reco) exit code: $c_ex_code" 
       files=`ls *_${now}_*` 
         for f in $files 
           do
            size=`stat -c %s $f`
            echo "written output file: $f $size"
          done                 
    fi
fi 
#overrides=""
#python ${INPUT_DIR}/pdhd_meta_writer.py \
#       --json ${INPUT_DIR}/pdhd_base_meta.json \
#       --overrides ${overrides} \
#       --parent ${prodname}.root \
#       -o ${reco_name}.root.ext.json

echo "===========checking reco MD"

jq . ${reco_name}.root.json 
 


if [ $? -ne 0 ]
then
  echo "Exiting with error"
  exit 1
else
  files=`ls *_${now}_*` 
   for f in $files 
      do
       size=`stat -c %s $f`
       echo "written output file: $f $size"
      done  

  echo "$pfn" > justin-processed-pfns.txt
fi

justIN time: 2024-11-25 03:05:05 UTC       justIN version: 01.01.09