Workflow 1604, Stage 1

Priority	50
Processors	1
Wall seconds	80000
RSS bytes	2621440000 (2500 MiB)
Max distance for inputs	30.0
Enabled input RSEs
Enabled output RSEs
Enabled sites
Scope	fardet-hd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
0	Rucio fardet-hd:fardet-hd-reco2_ritm1780305_anue_rhc_skip0_limit5000_1604	*reco2.root	2592000	False
0	Rucio fardet-hd:fardet-hd-reco2ana_ritm1780305_anue_rhc_skip0_limit5000_1604	*reco2_ana.root	2592000	False

Environment variables

Name	Value
CONVERT_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/075c0e56c49d2ffca32bc443c9c2fcc3efb66d49
DETPROD	HD
HCPROD	RHC

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
5000	0	0	0	0	4709	2	289

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
17250	0	0	0	0	7048	0	3158	3403	2921	720	0

RSEs used

Name	Inputs	Outputs
DUNE_ES_PIC	9073	291
DUNE_US_FNAL_DISK_STAGE	5589	2519
RAL_ECHO	0	3410
DUNE_US_BNL_SDCC	0	1546
RAL-PP	0	1472
PRAGUE	0	1012
DUNE_UK_LANCASTER_CEPH	0	804
NIKHEF	0	520
DUNE_CERN_EOS	0	440
SURFSARA	0	202
DUNE_FR_CCIN2P3_DISK	0	66

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_RAL-Tier1	851	679
UK_Manchester	375	348
US_FNAL-FermiGrid	292	489
UK_RAL-PPD	194	245
CERN	164	181
US_SU-ITS	150	228
CZ_FZU	149	210
IT_CNAF	133	110
UK_QMUL	103	87
US_BNL	94	178
UK_Imperial	93	54
US_UChicago	84	240
US_NotreDame	76	13
UK_Oxford	57	56
US_Colorado	55	183
UK_Brunel	54	48
ES_PIC	51	75
ES_CIEMAT	41	22
NL_SURFsara	41	21
US_Wisconsin	39	151
UK_Bristol	34	36
CH_UNIBE-LHEP	33	10
UK_Liverpool	28	30
NL_NIKHEF	26	44
US_PuertoRico	25	4
UK_Sheffield	23	35
UK_Edinburgh	22	14
CA_Victoria	12	28
US_UConn-HPC	9	40
US_FNAL-T1	9	50
FR_CCIN2P3	3	3
CA_SFU	3	14
US_UCSD	0	7

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


##TODO -- write usage



DETECTOR="${DETPROD:-HD}"
HC="${HCPROD:-FHC}"

## Build up the fcls to use
if [ "${DETECTOR}" = "HD" ]; then
  ANAFCL="standard_ana_dune10kt_1x2x6.fcl"
  MDDETTYPE="fardet-hd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="standard_reco2_dune10kt_nu_1x2x6.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="standard_reco2_dune10kt_anu_1x2x6.fcl"
  fi

elif [ "${DETECTOR}" = "VD" ]; then
  ANAFCL="anatree_dunevd10kt_1x8x6_3view_30deg_geov3.fcl"
  MDDETTYPE="fardet-vd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="reco2_dunevd10kt_nu_1x8x6_3view_30deg_geov3.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="reco2_dunevd10kt_anu_1x8x6_3view_30deg_geov3.fcl"
  fi

elif [ "${DETECTOR}" = "TEST" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test.fcl"

elif [ "${DETECTOR}" = "TEST2" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test2.fcl"

else
  echo "NEED TO REQUEST EITHER HD OR VD. USER REQUESTED ${DETECTOR}"
  exit 1
fi

echo "Using Reco fcl: ${RECOFCL}"
echo "and Ana fcl: ${ANAFCL}"


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_81_00d02}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
env


##Force certain env vars to force good behavior
#export TF_NUM_THREADS=$JUSTIN_PROCESSORS    
#export OPENBLAS_NUM_THREADS=$JUSTIN_PROCESSORS  
#export JULIA_NUM_THREADS=$JUSTIN_PROCESSORS  
#export MKL_NUM_THREADS=$JUSTIN_PROCESSORS  
#export NUMEXPR_NUM_THREADS=$JUSTIN_PROCESSORS  
#export OMP_NUM_THREADS=$JUSTIN_PROCESSORS   

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
#getfileExit=$?

#if [ $getfileExit -ne 0 ] ; then
#  # Success !
#  # Error -- exit immediately 
#  jobscriptExit=1
#  echo "Error in justin-get-file"
#  exit $jobscriptExit
#fi

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"

metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

#
#
now=$(date -u +"%Y%m%dT%H%M%SZ")
#OUTFILE="${OUTPREFIX:-fd_mc_prod_test}.${JUSTIN_REQUEST_ID}"
#OUTFILE="$OUTFILE.$JUSTIN_JOBSUB_ID.${now}"
#OUTFILE=`echo $OUTFILE | sed -e 's/@/./'`
#
####Run reco2
lar -c "${RECOFCL}" \
    -n "${NEVENTS:--1}" \
    "$FILE" >reco2.log 2>&1
    #-o "$OUTFILE.root" \
    #-n "${NEVENTS:--1}" \
    #"$FILE" >$OUTFILE.log 2>&1

larExit=$?
echo "Reco step lar exit code $larExit"

if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  #echo "$FILE" > justin-processed-pfns.txt
  echo "Moving on to analysis "
else
  # Error -- exit immediately 
  #jobscriptExit=1
  tail -100 reco2.log
  exit $larExit 
fi

if [ "${DETECTOR}" = "TEST" ] || [ "${DETECTOR}" = "TEST2" ]; then
  echo "Done testing. Exiting"
  exit 0
fi
ORIG_OUTFILE=`ls *reco2.root`
OUTFILE=`echo "${ORIG_OUTFILE}_${now}_reco2" | sed -e 's/reco2.root//'`

mv ${ORIG_OUTFILE} ${OUTFILE}.root

### Get the metadata
##TODO -- convert this is the right file extensions
extractor_prod.py --infile ${OUTFILE}.root --campaign ${CAMPAIGN:-fd_mc_2023a_reco2} \
                  --requestid ritm1780305 --no_crc > ${OUTFILE}.root.json
extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

### Convert the metadata to metacat
##TODO -- make sure the RCDS behavior is correct
python $CONVERT_DIR/convert_metadata.py -i ${OUTFILE}.root.json \
                                                -c ${RECOFCL} -j old_md.json \
                                                --app "art.reco2" \
                                                --app_ver "${DUNE_VERSION:-v09_81_00d02}" \
                                                --det "${MDDETTYPE}" \
                                                --parent ${DID}
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "Converted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to convert md"
  exit $converterExit
fi

##If running VD, there will also be pandora files. Rename these
if [ "${DETECTOR}" = "VD" ]; then
  all_good=true
  mv Validation.root "${OUTFILE}_Validation.root" || all_good=false
  mv Pandora_Events.pndr "${OUTFILE}_Pandora_Events.pndr" || all_good=false

  ##Copy over the metadata for pndr
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Pandora_Events.pndr.json || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e 's/artroot/binary/' || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e '/art.file_format/d' || all_good=false
  
  ##Copy over the metadata for validation.root 
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Validation.root.json || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e 's/artroot/root/' || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e '/art.file_format/d' || all_good=false

  if [ $all_good = false ]; then
    echo "Something failed when editing pandora metadata"
    jobscriptExit=1
    exit $jobscriptExit
  fi

fi

ANAOUTFILE="${OUTFILE}_ana"
echo "Will output ana to $ANAOUTFILE"

###Run ana 
lar -c "${ANAFCL}" \
    -n "${NEVENTS:--1}" \
    -T "$ANAOUTFILE.root" \
    "$OUTFILE.root" >ana.log 2>&1


larExit=$?
echo "Ana step lar exit code $larExit"

### TODO: Handle this correctly
if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  echo "$FILE" > justin-processed-pfns.txt
else
  # Error -- exit immediately 
  jobscriptExit=1
  tail -100 ana.log
  exit $larExit
fi

sed_good=true
sed ${OUTFILE}.root.json -e 's/full-reconstructed/root-tuple-virtual/' > ${ANAOUTFILE}.root.json || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e 's/artroot/root/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.file_format/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.process/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${RECOFCL}/${ANAFCL}/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/campaign/!s/reco2/anatree/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${DID}/${OUTFILE}.root/" || sed_good=false

if [ $sed_good = false ]; then
  echo "Something failed when editing ana metadata"
  jobscriptExit=1
  exit $jobscriptExit
fi

ls
exit 0