Workflow 1535, Stage 1

Priority	50
Processors	1
Wall seconds	3600
Image
RSS bytes	2621440000 (2500 MiB)
Max distance for inputs	30.0
Enabled input RSEs
Enabled output RSEs
Enabled sites
Scope	fardet-hd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
0	Rucio fardet-hd:fardet-hd-reco2_ritm1780305_anu_rhc_skip0_limit5000	*reco2.root	86400	False
0	Rucio fardet-hd:fardet-hd-reco2ana_ritm1780305_anu_rhc_skip0_limit5000	*reco2_ana.root	86400	False

Environment variables

Name	Value
CONVERT_DIR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/523f4bf9052aa4030d4cad66a2e9f6f70d3dbe19
DETPROD	HD
HCPROD	RHC

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
5000	1	3271	0	0	1694	0	34

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
14374	0	0	0	0	1792	0	11296	947	31	308	0

RSEs used

Name	Inputs	Outputs
DUNE_ES_PIC	9961	205
DUNE_US_FNAL_DISK_STAGE	3684	711
NIKHEF	0	379
DUNE_US_BNL_SDCC	0	322
DUNE_FR_CCIN2P3_DISK	0	319
RAL_ECHO	0	272
MANCHESTER	0	242
RAL-PP	0	233
SURFSARA	0	184
DUNE_CERN_EOS	0	175
PRAGUE	0	130
DUNE_UK_LANCASTER_CEPH	0	114
QMUL	0	97

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
US_PuertoRico	154	102
UK_QMUL	110	297
NL_NIKHEF	24	931
US_SU-ITS	21	650
CERN	21	405
UK_Imperial	19	285
UK_RAL-Tier1	18	1397
NL_SURFsara	17	460
CA_SFU	17	214
ES_PIC	15	88
UK_Manchester	13	825
UK_RAL-PPD	13	923
CZ_FZU	12	1010
US_Wisconsin	12	201
US_UChicago	11	1262
UK_Sheffield	8	141
US_NotreDame	8	0
FR_CCIN2P3	8	570
US_FNAL-FermiGrid	8	90
ES_CIEMAT	7	224
UK_Lancaster	7	283
UK_Bristol	7	459
US_Michigan	6	23
US_BNL	3	184
UK_Oxford	2	142
US_Colorado	0	48
US_FNAL-T1	0	43
US_UCSD	0	41
US_UConn-HPC	0	17
UK_Brunel	0	15
CA_Victoria	0	15

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


##TODO -- write usage



DETECTOR="${DETPROD:-HD}"
HC="${HCPROD:-FHC}"

## Build up the fcls to use
if [ "${DETECTOR}" = "HD" ]; then
  ANAFCL="standard_ana_dune10kt_1x2x6.fcl"
  MDDETTYPE="fardet-hd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="standard_reco2_dune10kt_nu_1x2x6.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="standard_reco2_dune10kt_anu_1x2x6.fcl"
  fi

elif [ "${DETECTOR}" = "VD" ]; then
  ANAFCL="anatree_dunevd10kt_1x8x6_3view_30deg_geov3.fcl"
  MDDETTYPE="fardet-vd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="reco2_dunevd10kt_nu_1x8x6_3view_30deg_geov3.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="reco2_dunevd10kt_anu_1x8x6_3view_30deg_geov3.fcl"
  fi

elif [ "${DETECTOR}" = "TEST" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test.fcl"

elif [ "${DETECTOR}" = "TEST2" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test2.fcl"

else
  echo "NEED TO REQUEST EITHER HD OR VD. USER REQUESTED ${DETECTOR}"
  exit 1
fi

echo "Using Reco fcl: ${RECOFCL}"
echo "and Ana fcl: ${ANAFCL}"


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_81_00d02}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
env


##Force certain env vars to force good behavior
#export TF_NUM_THREADS=$JUSTIN_PROCESSORS    
#export OPENBLAS_NUM_THREADS=$JUSTIN_PROCESSORS  
#export JULIA_NUM_THREADS=$JUSTIN_PROCESSORS  
#export MKL_NUM_THREADS=$JUSTIN_PROCESSORS  
#export NUMEXPR_NUM_THREADS=$JUSTIN_PROCESSORS  
#export OMP_NUM_THREADS=$JUSTIN_PROCESSORS   

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
#getfileExit=$?

#if [ $getfileExit -ne 0 ] ; then
#  # Success !
#  # Error -- exit immediately 
#  jobscriptExit=1
#  echo "Error in justin-get-file"
#  exit $jobscriptExit
#fi

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"

metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

#
#
now=$(date -u +"%Y%m%dT%H%M%SZ")
#OUTFILE="${OUTPREFIX:-fd_mc_prod_test}.${JUSTIN_REQUEST_ID}"
#OUTFILE="$OUTFILE.$JUSTIN_JOBSUB_ID.${now}"
#OUTFILE=`echo $OUTFILE | sed -e 's/@/./'`
#
####Run reco2
lar -c "${RECOFCL}" \
    -n "${NEVENTS:--1}" \
    "$FILE" >reco2.log 2>&1
    #-o "$OUTFILE.root" \
    #-n "${NEVENTS:--1}" \
    #"$FILE" >$OUTFILE.log 2>&1

larExit=$?
echo "Reco step lar exit code $larExit"

if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  #echo "$FILE" > justin-processed-pfns.txt
  echo "Moving on to analysis "
else
  # Error -- exit immediately 
  #jobscriptExit=1
  tail -100 reco2.log
  exit $larExit 
fi

if [ "${DETECTOR}" = "TEST" ] || [ "${DETECTOR}" = "TEST2" ]; then
  echo "Done testing. Exiting"
  exit 0
fi
ORIG_OUTFILE=`ls *reco2.root`
OUTFILE=`echo "${ORIG_OUTFILE}_${now}_reco2" | sed -e 's/reco2.root//'`

mv ${ORIG_OUTFILE} ${OUTFILE}.root

### Get the metadata
##TODO -- convert this is the right file extensions
extractor_prod.py --infile ${OUTFILE}.root --campaign ${CAMPAIGN:-fd_mc_2023a_reco2} \
                  --requestid ritm1780305 --no_crc > ${OUTFILE}.root.json
extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

### Convert the metadata to metacat
##TODO -- make sure the RCDS behavior is correct
python $CONVERT_DIR/convert_metadata.py -i ${OUTFILE}.root.json \
                                                -c ${RECOFCL} -j old_md.json \
                                                --app "art.reco2" \
                                                --app_ver "${DUNE_VERSION:-v09_81_00d02}" \
                                                --det "${MDDETTYPE}" \
                                                --parent ${DID}
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "Converted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to convert md"
  exit $converterExit
fi

##If running VD, there will also be pandora files. Rename these
if [ "${DETECTOR}" = "VD" ]; then
  all_good=true
  mv Validation.root "${OUTFILE}_Validation.root" || all_good=false
  mv Pandora_Events.pndr "${OUTFILE}_Pandora_Events.pndr" || all_good=false

  ##Copy over the metadata for pndr
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Pandora_Events.pndr.json || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e 's/artroot/binary/' || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e '/art.file_format/d' || all_good=false
  
  ##Copy over the metadata for validation.root 
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Validation.root.json || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e 's/artroot/root/' || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e '/art.file_format/d' || all_good=false

  if [ $all_good = false ]; then
    echo "Something failed when editing pandora metadata"
    jobscriptExit=1
    exit $jobscriptExit
  fi

fi

ANAOUTFILE="${OUTFILE}_ana"
echo "Will output ana to $ANAOUTFILE"

###Run ana 
lar -c "${ANAFCL}" \
    -n "${NEVENTS:--1}" \
    -T "$ANAOUTFILE.root" \
    "$OUTFILE.root" >ana.log 2>&1


larExit=$?
echo "Ana step lar exit code $larExit"

### TODO: Handle this correctly
if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  echo "$FILE" > justin-processed-pfns.txt
else
  # Error -- exit immediately 
  jobscriptExit=1
  tail -100 ana.log
  exit $larExit
fi

sed_good=true
sed ${OUTFILE}.root.json -e 's/full-reconstructed/root-tuple-virtual/' > ${ANAOUTFILE}.root.json || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e 's/artroot/root/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.file_format/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.process/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${RECOFCL}/${ANAFCL}/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/campaign/!s/reco2/anatree/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${DID}/${OUTFILE}.root/" || sed_good=false

if [ $sed_good = false ]; then
  echo "Something failed when editing ana metadata"
  jobscriptExit=1
  exit $jobscriptExit
fi

ls
exit 0