Workflow 1630, Stage 1
| Priority | 50 |
| Processors | 1 |
| Wall seconds | 80000 |
| RSS bytes | 2621440000 (2500 MiB) |
| Max distance for inputs | 30.0 |
| Enabled input RSEs |
|
| Enabled output RSEs |
|
| Enabled sites |
|
| Scope | fardet-hd |
| Events for this stage |
Output patterns
| | Destination | Pattern | Lifetime | For next stage |
|---|
| 0 | Rucio fardet-hd:fardet-hd-reco2_ritm1780305_nu_fhc_skip7000_limit5000_1630 | *reco2.root | 2592000 | False |
| 0 | Rucio fardet-hd:fardet-hd-reco2ana_ritm1780305_nu_fhc_skip7000_limit5000_1630 | *reco2_ana.root | 2592000 | False |
Environment variables
| Name | Value |
|---|
| CONVERT_DIR | /cvmfs/fifeuser2.opensciencegrid.org/sw/dune/075c0e56c49d2ffca32bc443c9c2fcc3efb66d49 |
| DETPROD | HD |
| HCPROD | FHC |
File states
| Total files | Finding | Unallocated | Allocated | Outputting | Processed | Not found | Failed |
|---|
|
| 5000 | 0 | 0 | 0 | 0 | 4691 | 1 | 308 |
Job states
| Total | Submitted | Started | Processing | Outputting | Finished | Notused | Aborted | Stalled | Jobscript error | Outputting failed | None processed |
|---|
| 10957 | 0 | 0 | 0 | 0 | 5570 | 0 | 4055 | 857 | 376 | 99 | 0 |
RSEs used
| Name | Inputs | Outputs |
|---|
| DUNE_ES_PIC | 6730 | 497 |
| DUNE_US_FNAL_DISK_STAGE | 2734 | 367 |
| DUNE_US_BNL_SDCC | 0 | 2847 |
| RAL_ECHO | 0 | 2334 |
| RAL-PP | 0 | 1217 |
| PRAGUE | 0 | 761 |
| DUNE_CERN_EOS | 0 | 733 |
| NIKHEF | 0 | 679 |
| SURFSARA | 0 | 129 |
| DUNE_UK_LANCASTER_CEPH | 0 | 27 |
| DUNE_FR_CCIN2P3_DISK | 0 | 25 |
Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)
File reset events, by site
| Site | Allocated | Outputting |
|---|
| UK_RAL-Tier1 | 115 | 90 |
| US_FNAL-FermiGrid | 93 | 117 |
| UK_Edinburgh | 46 | 8 |
| US_UChicago | 35 | 56 |
| UK_Manchester | 32 | 49 |
| UK_RAL-PPD | 27 | 24 |
| CZ_FZU | 24 | 27 |
| CERN | 15 | 20 |
| ES_PIC | 14 | 15 |
| US_BNL | 13 | 25 |
| UK_QMUL | 11 | 14 |
| US_SU-ITS | 10 | 12 |
| US_PuertoRico | 10 | 0 |
| US_Colorado | 10 | 11 |
| UK_Brunel | 7 | 3 |
| IT_CNAF | 7 | 15 |
| UK_Sheffield | 7 | 2 |
| NL_NIKHEF | 7 | 10 |
| UK_Oxford | 6 | 4 |
| US_NotreDame | 6 | 0 |
| US_Wisconsin | 5 | 11 |
| NL_SURFsara | 5 | 4 |
| UK_Imperial | 3 | 2 |
| CA_SFU | 3 | 1 |
| UK_Bristol | 3 | 2 |
| CA_Victoria | 2 | 6 |
| US_UConn-HPC | 2 | 5 |
| UK_Liverpool | 2 | 3 |
| ES_CIEMAT | 2 | 4 |
| CH_UNIBE-LHEP | 2 | 3 |
| US_FNAL-T1 | 1 | 8 |
| FR_CCIN2P3 | 0 | 1 |
Jobscript
#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune
##TODO -- write usage
DETECTOR="${DETPROD:-HD}"
HC="${HCPROD:-FHC}"
## Build up the fcls to use
if [ "${DETECTOR}" = "HD" ]; then
ANAFCL="standard_ana_dune10kt_1x2x6.fcl"
MDDETTYPE="fardet-hd"
if [ "$HC" = "FHC" ]; then
RECOFCL="standard_reco2_dune10kt_nu_1x2x6.fcl"
elif [ "$HC" = "RHC" ]; then
RECOFCL="standard_reco2_dune10kt_anu_1x2x6.fcl"
fi
elif [ "${DETECTOR}" = "VD" ]; then
ANAFCL="anatree_dunevd10kt_1x8x6_3view_30deg_geov3.fcl"
MDDETTYPE="fardet-vd"
if [ "$HC" = "FHC" ]; then
RECOFCL="reco2_dunevd10kt_nu_1x8x6_3view_30deg_geov3.fcl"
elif [ "$HC" = "RHC" ]; then
RECOFCL="reco2_dunevd10kt_anu_1x8x6_3view_30deg_geov3.fcl"
fi
elif [ "${DETECTOR}" = "TEST" ]; then
RECOFCL="${INPUT_TAR_DIR_LOCAL}/test.fcl"
elif [ "${DETECTOR}" = "TEST2" ]; then
RECOFCL="${INPUT_TAR_DIR_LOCAL}/test2.fcl"
else
echo "NEED TO REQUEST EITHER HD OR VD. USER REQUESTED ${DETECTOR}"
exit 1
fi
echo "Using Reco fcl: ${RECOFCL}"
echo "and Ana fcl: ${ANAFCL}"
#Setup recent lar software suite
setup dunesw \
"${DUNE_VERSION:-v09_81_00d02}" \
-q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
env
##Force certain env vars to force good behavior
#export TF_NUM_THREADS=$JUSTIN_PROCESSORS
#export OPENBLAS_NUM_THREADS=$JUSTIN_PROCESSORS
#export JULIA_NUM_THREADS=$JUSTIN_PROCESSORS
#export MKL_NUM_THREADS=$JUSTIN_PROCESSORS
#export NUMEXPR_NUM_THREADS=$JUSTIN_PROCESSORS
#export OMP_NUM_THREADS=$JUSTIN_PROCESSORS
if [ -z ${JUSTIN_PROCESSORS} ]; then
JUSTIN_PROCESSORS=1
fi
echo "Justin processors: ${JUSTIN_PROCESSORS}"
export TF_NUM_THREADS=${JUSTIN_PROCESSORS}
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS}
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS}
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS}
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS}
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}
#
echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
#getfileExit=$?
#if [ $getfileExit -ne 0 ] ; then
# # Success !
# # Error -- exit immediately
# jobscriptExit=1
# echo "Error in justin-get-file"
# exit $jobscriptExit
#fi
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
echo "Could not get file"
exit 0
fi
FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"
metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
echo "old metadata:"
cat old_md.json
else
echo "Could not retrieve old metadata"
exit 1
fi
#
#
now=$(date -u +"%Y%m%dT%H%M%SZ")
#OUTFILE="${OUTPREFIX:-fd_mc_prod_test}.${JUSTIN_REQUEST_ID}"
#OUTFILE="$OUTFILE.$JUSTIN_JOBSUB_ID.${now}"
#OUTFILE=`echo $OUTFILE | sed -e 's/@/./'`
#
####Run reco2
lar -c "${RECOFCL}" \
-n "${NEVENTS:--1}" \
"$FILE" >reco2.log 2>&1
#-o "$OUTFILE.root" \
#-n "${NEVENTS:--1}" \
#"$FILE" >$OUTFILE.log 2>&1
larExit=$?
echo "Reco step lar exit code $larExit"
if [ $larExit -eq 0 ] ; then
# Success !
# Log the file for justin and move on to the next step
#echo "$FILE" > justin-processed-pfns.txt
echo "Moving on to analysis "
else
# Error -- exit immediately
#jobscriptExit=1
tail -100 reco2.log
exit $larExit
fi
if [ "${DETECTOR}" = "TEST" ] || [ "${DETECTOR}" = "TEST2" ]; then
echo "Done testing. Exiting"
exit 0
fi
ORIG_OUTFILE=`ls *reco2.root`
OUTFILE=`echo "${ORIG_OUTFILE}_${now}_reco2" | sed -e 's/reco2.root//'`
mv ${ORIG_OUTFILE} ${OUTFILE}.root
### Get the metadata
##TODO -- convert this is the right file extensions
extractor_prod.py --infile ${OUTFILE}.root --campaign ${CAMPAIGN:-fd_mc_2023a_reco2} \
--requestid ritm1780305 --no_crc > ${OUTFILE}.root.json
extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
# Success !
echo "Extracted metadata"
else
# Error -- exit immediately
jobscriptExit=1
echo "Failed to extract md"
exit $extractorExit
fi
### Convert the metadata to metacat
##TODO -- make sure the RCDS behavior is correct
python $CONVERT_DIR/convert_metadata.py -i ${OUTFILE}.root.json \
-c ${RECOFCL} -j old_md.json \
--app "art.reco2" \
--app_ver "${DUNE_VERSION:-v09_81_00d02}" \
--det "${MDDETTYPE}" \
--parent ${DID}
converterExit=$?
if [ $converterExit -eq 0 ] ; then
# Success !
echo "Converted metadata"
else
# Error -- exit immediately
jobscriptExit=1
echo "Failed to convert md"
exit $converterExit
fi
##If running VD, there will also be pandora files. Rename these
if [ "${DETECTOR}" = "VD" ]; then
all_good=true
mv Validation.root "${OUTFILE}_Validation.root" || all_good=false
mv Pandora_Events.pndr "${OUTFILE}_Pandora_Events.pndr" || all_good=false
##Copy over the metadata for pndr
##Edit the fields corresepondingly
sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Pandora_Events.pndr.json || all_good=false
sed -i ${OUTFILE}_Pandora_Events.pndr.json -e 's/artroot/binary/' || all_good=false
sed -i ${OUTFILE}_Pandora_Events.pndr.json -e '/art.file_format/d' || all_good=false
##Copy over the metadata for validation.root
##Edit the fields corresepondingly
sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Validation.root.json || all_good=false
sed -i ${OUTFILE}_Validation.root.json -e 's/artroot/root/' || all_good=false
sed -i ${OUTFILE}_Validation.root.json -e '/art.file_format/d' || all_good=false
if [ $all_good = false ]; then
echo "Something failed when editing pandora metadata"
jobscriptExit=1
exit $jobscriptExit
fi
fi
ANAOUTFILE="${OUTFILE}_ana"
echo "Will output ana to $ANAOUTFILE"
###Run ana
lar -c "${ANAFCL}" \
-n "${NEVENTS:--1}" \
-T "$ANAOUTFILE.root" \
"$OUTFILE.root" >ana.log 2>&1
larExit=$?
echo "Ana step lar exit code $larExit"
### TODO: Handle this correctly
if [ $larExit -eq 0 ] ; then
# Success !
# Log the file for justin and move on to the next step
echo "$FILE" > justin-processed-pfns.txt
else
# Error -- exit immediately
jobscriptExit=1
tail -100 ana.log
exit $larExit
fi
sed_good=true
sed ${OUTFILE}.root.json -e 's/full-reconstructed/root-tuple-virtual/' > ${ANAOUTFILE}.root.json || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e 's/artroot/root/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.file_format/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.process/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${RECOFCL}/${ANAFCL}/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/campaign/!s/reco2/anatree/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${DID}/${OUTFILE}.root/" || sed_good=false
if [ $sed_good = false ]; then
echo "Something failed when editing ana metadata"
jobscriptExit=1
exit $jobscriptExit
fi
ls
exit 0
