Workflow 1747, Stage 1
| Priority | 50 |
| Processors | 1 |
| Wall seconds | 80000 |
| RSS bytes | 4193255424 (3999 MiB) |
| Max distance for inputs | 100.0 |
| Enabled input RSEs |
|
| Enabled output RSEs |
|
| Enabled sites |
|
| Scope | usertests |
| Events for this stage |
Output patterns
| | Destination | Pattern | Lifetime | For next stage |
|---|
| 0 | Rucio usertests:vd-protodune-reco_1747 | *reco.root | 86400 | False |
Environment variables
| Name | Value |
|---|
| INPUT_DIR | /cvmfs/fifeuser2.opensciencegrid.org/sw/dune/f0e5150ef6f472c5af268f13e5cd533aa152c7d7 |
File states
| Total files | Finding | Unallocated | Allocated | Outputting | Processed | Not found | Failed |
|---|
|
| 2 | 0 | 0 | 0 | 0 | 0 | 0 | 2 |
Job states
| Total | Submitted | Started | Processing | Outputting | Finished | Notused | Aborted | Stalled | Jobscript error | Outputting failed | None processed |
|---|
| 389 | 0 | 0 | 0 | 0 | 10 | 367 | 12 | 0 | 0 | 0 | 0 |
RSEs used
| Name | Inputs | Outputs |
|---|
| MONTECARLO | 12 | 0 |
Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)
File reset events, by site
| Site | Allocated | Outputting |
|---|
| UK_RAL-Tier1 | 0 | 5 |
| CA_SFU | 0 | 4 |
| NL_NIKHEF | 0 | 1 |
Jobscript
#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune
#Setup recent lar software suite
setup dunesw \
"${DUNE_VERSION:-v09_82_02d01}" \
-q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
if [ -z ${JUSTIN_PROCESSORS} ]; then
JUSTIN_PROCESSORS=1
fi
echo "Justin processors: ${JUSTIN_PROCESSORS}"
export TF_NUM_THREADS=${JUSTIN_PROCESSORS}
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS}
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS}
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS}
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS}
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}
env
echo "Will use justin-get-file"
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
if [ "${DID_PFN_RSE}" == "" ] ; then
echo "Could not get file"
exit 0
fi
pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
if [ -z ${LINENUMBER} ] ; then
LINENUMBER=$pfn
fi
# echo "linenumber " $LINENUMBER
ex_code=0
pstep=10
tstep=0
now=$(date -u +"%Y%m%dT%H%M%SZ")
namespace=${JUSTIN_SCOPE:-"usertests"}
# Run Generator
echo "generator"
prodname="prod_cosmics_protodunevd_${now}_gen_${pfn}"
istep=1
tstep=$((tstep+1))
nstep=$(($pstep*$tstep))
lar -c gen_protodunevd_cosmics.fcl -o ${prodname}.root -n 1
if [ $? -ne 0 ]; then
ex_code=$(($nstep+$istep))
echo "ERROR: lar (generation) exit code: $ex_code "
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
else
istep=$((istep+1))
extractor_prod.py --infile ${prodname}.root --appfamily art --appname gen --appversion v09_82_02d01 --no_crc --requestid ritm1998918 --input_json ${INPUT_DIR}/pdvd_input.json > ${prodname}.root.ext.json && sed -i -e 's/stepfcl/gen_protodunevd_cosmics/g' ${prodname}.root.ext.json
touch all-input-dids.txt
# /cvmfs/dune.opensciencegrid.org/products/dune/justin/pro/NULL/jobutils/pdjson2metadata ${prodname}.root.ext.json all-input-dids.txt > ${prodname}.root.json
python ${INPUT_DIR}/pdjson2metadata ${prodname}.root.ext.json all-input-dids.txt usertests > ${prodname}.root.json
if [ $? -ne 0 ] ; then
ex_code=$((nstep+istep))
echo "ERROR: metadata extraction (generation) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
fi
fi
echo "=====checking generation MD"
jq . ${prodname}.root.json
awk '/"core.runs/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json > run.json
awk '/"core.runs_subruns/,/],/ { print;f=1} f&&/],/{ exit}' ${prodname}.root.json >> run.json
awk 'NR==1 {print; exit}' ${INPUT_DIR}/pdvd_input.json > pdvd_run_input.json
cat run.json >> pdvd_run_input.json
tail -n +2 ${INPUT_DIR}/pdvd_input.json >> pdvd_run_input.json
#echo "generation exit code " $ex_code
if [ $ex_code -ne 0 ]; then exit $ex_code ; fi
# Stage 1 G4
echo "G4 stage 1"
g4stage1_name="${prodname}_g4_stage1"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep))
istep=1
lar -c protodunevd_refactored_g4_stage1.fcl ${prodname}.root -o ${g4stage1_name}.root
if [ $? -ne 0 ]; then
ex_code=$((nstep+istep))
echo "ERROR: lar (geant step1) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
else
istep=$((istep+1))
rm -fr all-input-dids.txt
echo ${namespace}":"${prodname}.root > all-input-dids.txt
extractor_prod.py --infile ${g4stage1_name}.root --appfamily art --appname g4-stage1 --appversion v09_82_02d01 --no_crc --requestid ritm1998918 --input_json pdvd_run_input.json > ${g4stage1_name}.root.ext.json && sed -i -e 's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage1/g' ${g4stage1_name}.root.ext.json
# /cvmfs/dune.opensciencegrid.org/products/dune/justin/pro/NULL/jobutils/pdjson2metadata ${g4stage1_name}.root.ext.json all-input-dids.txt > ${g4stage1_name}.root.json
python ${INPUT_DIR}/pdjson2metadata ${g4stage1_name}.root.ext.json all-input-dids.txt usertests > ${g4stage1_name}.root.json
if [ $? -ne 0 ] ;then
ex_code=$((nstep+istep))
echo "ERROR: metadata extraction (geant step1) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
fi
fi
if [ $ex_code -ne 0 ]; then exit $ex_code ; fi
echo "=============checking g4stage1 MD"
jq . ${g4stage1_name}.root.json
# Stage 2 G4
echo "G4 stage 2"
g4stage2_name="${g4stage1_name}_g4_stage2"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep))
istep=1
lar -c protodunevd_refactored_g4_stage2_pureAr.fcl ${g4stage1_name}.root -o ${g4stage2_name}.root
if [ $? -ne 0 ]; then
ex_code=$((nstep+istep))
echo "ERROR: lar (geant step2) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
else
istep=$((istep+1))
rm -fr all-input-dids.txt
echo ${namespace}":"${g4stage1_name}.root > all-input-dids.txt
extractor_prod.py --infile ${g4stage2_name}.root --appfamily art --appname g4-stage2 --appversion v09_82_02d01 --no_crc --requestid ritm1998918 --input_json pdvd_run_input.json > ${g4stage2_name}.root.ext.json && sed -i -e's/unknown/mc/g;s/stepfcl/protodunevd_refactored_g4_stage2_pureAr/g' ${g4stage2_name}.root.ext.json
# /cvmfs/dune.opensciencegrid.org/products/dune/justin/pro/NULL/jobutils/pdjson2metadata ${g4stage2_name}.root.ext.json all-input-dids.txt > ${g4stage2_name}.root.json
python ${INPUT_DIR}/pdjson2metadata ${g4stage2_name}.root.ext.json all-input-dids.txt usertests > ${g4stage2_name}.root.json
if [ $? -ne 0 ] ;then
ex_code=$((nstep+istep))
echo "ERROR: metadata extraction (geant step2) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
fi
fi
if [ $ex_code -ne 0 ]; then exit $ex_code ; fi
echo "========== checking g4stage2 MD"
jq . ${g4stage2_name}.root.json
# Detsim
echo "detsim"
detsim_name="${g4stage2_name}_detsim"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep))
istep=1
lar -c protodunevd_refactored_detsim_pureAr.fcl ${g4stage2_name}.root -o ${detsim_name}.root
if [ $? -ne 0 ]; then
ex_code=$((nstep+istep))
echo "ERROR: lar (detsim) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
else
istep=$((istep+1))
rm -fr all-input-dids.txt
echo ${namespace}":"${g4stage2_name}.root > all-input-dids.txt
extractor_prod.py --infile ${detsim_name}.root --appfamily art --appname detsim --appversion v09_82_02d01 --no_crc --requestid ritm1998918 --input_json ${INPUT_DIR}/pdvd_input.json > ${detsim_name}.root.ext.json && sed -i -e 's/stepfcl/protodunevd_refactored_detsim_pureAr/g' ${detsim_name}.root.ext.json
# /cvmfs/dune.opensciencegrid.org/products/dune/justin/pro/NULL/jobutils/pdjson2metadata ${detsim_name}.root.ext.json all-input-dids.txt > ${detsim_name}.root.json
python ${INPUT_DIR}/pdjson2metadata ${detsim_name}.root.ext.json all-input-dids.txt usertests > ${detsim_name}.root.json
if [ $? -ne 0 ] ;then
ex_code=$((nstep+istep))
echo "ERROR: metadata extraction (detsim) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
fi
fi
if [ $ex_code -ne 0 ]; then exit $ex_code ; fi
echo "============= checking detsim MD "
jq . ${detsim_name}.root.json
# Reco
echo "reco"
reco_name="${detsim_name}_reco"
tstep=$((tstep+1))
nstep=$(($pstep*$tstep))
istep=1
lar -c protodunevd_reco.fcl ${detsim_name}.root -o ${reco_name}.root
if [ $? -ne 0 ]; then
ex_code=$((nstep+istep))
echo "ERROR: lar (reco) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
else
istep=$((istep+1))
rm -fr all-input-dids.txt
echo ${namespace}":"${detsim_name}.root > all-input-dids.txt
extractor_prod.py --infile ${reco_name}.root --appfamily art --appname reco --appversion v09_82_02d01 --no_crc --requestid ritm1998918 --input_json ${INPUT_DIR}/pdvd_input.json > ${reco_name}.root.ext.json && sed -i -e 's/stepfcl/protodunevd_reco/g' ${reco_name}.root.ext.json
# /cvmfs/dune.opensciencegrid.org/products/dune/justin/pro/NULL/jobutils/pdjson2metadata ${reco_name}.root.ext.json all-input-dids.txt > ${reco_name}.root.json
python ${INPUT_DIR}/pdjson2metadata ${reco_name}.root.ext.json all-input-dids.txt usertests > ${reco_name}.root.json
if [ $? -ne 0 ] ;then
ex_code=$((nstep+istep))
echo "ERROR: metadata extraction (reco) exit code: $ex_code"
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
fi
fi
#overrides=""
#python ${INPUT_DIR}/pdhd_meta_writer.py \
# --json ${INPUT_DIR}/pdhd_base_meta.json \
# --overrides ${overrides} \
# --parent ${prodname}.root \
# -o ${reco_name}.root.ext.json
echo "===========checking reco MD"
jq . ${reco_name}.root.json
if [ $? -ne 0 ]
then
echo "Exiting with error"
exit 1
else
files=`ls *_${now}_*`
for f in $files
do
size=`stat -c %s $f`
echo "written output file: $f $size"
done
echo "$pfn" > justin-processed-pfns.txt
fi
