Workflow 1533, Stage 1

Priority	50
Processors	1
Wall seconds	3600
RSS bytes	2621440000 (2500 MiB)
Max distance for inputs	30.0
Enabled input RSEs
Enabled output RSEs
Enabled sites
Scope	fardet-hd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
0	Rucio fardet-hd:fardet-hd-reco2_ritm1780305_nu_rhc_skip0_limit5000	*reco2.root	86400	False
0	Rucio fardet-hd:fardet-hd-reco2ana_ritm1780305_nu_rhc_skip0_limit5000	*reco2_ana.root	86400	False

Environment variables

Name	Value
CONVERT_DIR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/523f4bf9052aa4030d4cad66a2e9f6f70d3dbe19
DETPROD	HD
HCPROD	RHC

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
5000	0	0	0	0	0	2	4998

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
40420	0	0	0	0	37	2112	29988	7977	83	223	0

RSEs used

Name	Inputs	Outputs
DUNE_ES_PIC	20814	0
DUNE_US_FNAL_DISK_STAGE	9174	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
CERN	155	3905
US_UChicago	112	2787
UK_Edinburgh	85	223
UK_RAL-Tier1	80	1878
US_PuertoRico	73	166
NL_NIKHEF	70	2033
US_SU-ITS	68	1097
US_FNAL-FermiGrid	54	978
UK_QMUL	50	986
CZ_FZU	44	1462
UK_Manchester	43	998
US_Michigan	41	159
UK_RAL-PPD	38	980
UK_Oxford	34	588
UK_Brunel	29	261
UK_Sheffield	27	277
NL_SURFsara	25	319
US_Colorado	21	314
ES_CIEMAT	20	372
ES_PIC	20	270
US_Wisconsin	20	675
FR_CCIN2P3	17	924
UK_Bristol	16	555
UK_Imperial	13	350
CA_SFU	13	294
US_BNL	12	142
CA_Victoria	10	313
UK_Lancaster	10	157
US_UCSD	6	79
US_FNAL-T1	5	57
UK_Liverpool	5	96
US_UConn-HPC	0	3

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


##TODO -- write usage



DETECTOR="${DETPROD:-HD}"
HC="${HCPROD:-FHC}"

## Build up the fcls to use
if [ "${DETECTOR}" = "HD" ]; then
  ANAFCL="standard_ana_dune10kt_1x2x6.fcl"
  MDDETTYPE="fardet-hd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="standard_reco2_dune10kt_nu_1x2x6.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="standard_reco2_dune10kt_anu_1x2x6.fcl"
  fi

elif [ "${DETECTOR}" = "VD" ]; then
  ANAFCL="anatree_dunevd10kt_1x8x6_3view_30deg_geov3.fcl"
  MDDETTYPE="fardet-vd"

  if [ "$HC" = "FHC" ]; then
    RECOFCL="reco2_dunevd10kt_nu_1x8x6_3view_30deg_geov3.fcl"
  elif [ "$HC" = "RHC" ]; then
    RECOFCL="reco2_dunevd10kt_anu_1x8x6_3view_30deg_geov3.fcl"
  fi

elif [ "${DETECTOR}" = "TEST" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test.fcl"

elif [ "${DETECTOR}" = "TEST2" ]; then
  RECOFCL="${INPUT_TAR_DIR_LOCAL}/test2.fcl"

else
  echo "NEED TO REQUEST EITHER HD OR VD. USER REQUESTED ${DETECTOR}"
  exit 1
fi

echo "Using Reco fcl: ${RECOFCL}"
echo "and Ana fcl: ${ANAFCL}"


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_81_00d02}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
echo "printing env"
env


##Force certain env vars to force good behavior
#export TF_NUM_THREADS=$JUSTIN_PROCESSORS    
#export OPENBLAS_NUM_THREADS=$JUSTIN_PROCESSORS  
#export JULIA_NUM_THREADS=$JUSTIN_PROCESSORS  
#export MKL_NUM_THREADS=$JUSTIN_PROCESSORS  
#export NUMEXPR_NUM_THREADS=$JUSTIN_PROCESSORS  
#export OMP_NUM_THREADS=$JUSTIN_PROCESSORS   

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file"
#
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
#getfileExit=$?

#if [ $getfileExit -ne 0 ] ; then
#  # Success !
#  # Error -- exit immediately 
#  jobscriptExit=1
#  echo "Error in justin-get-file"
#  exit $jobscriptExit
#fi

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"

metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

#
#
now=$(date -u +"%Y%m%dT%H%M%SZ")
#OUTFILE="${OUTPREFIX:-fd_mc_prod_test}.${JUSTIN_REQUEST_ID}"
#OUTFILE="$OUTFILE.$JUSTIN_JOBSUB_ID.${now}"
#OUTFILE=`echo $OUTFILE | sed -e 's/@/./'`
#
####Run reco2
lar -c "${RECOFCL}" \
    -n "${NEVENTS:--1}" \
    "$FILE" >reco2.log 2>&1
    #-o "$OUTFILE.root" \
    #-n "${NEVENTS:--1}" \
    #"$FILE" >$OUTFILE.log 2>&1

larExit=$?
echo "Reco step lar exit code $larExit"

if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  #echo "$FILE" > justin-processed-pfns.txt
  echo "Moving on to analysis "
else
  # Error -- exit immediately 
  #jobscriptExit=1
  tail -100 reco2.log
  exit $larExit 
fi

if [ "${DETECTOR}" = "TEST" ] || [ "${DETECTOR}" = "TEST2" ]; then
  echo "Done testing. Exiting"
  exit 0
fi
ORIG_OUTFILE=`ls *reco2.root`
OUTFILE=`echo "${ORIG_OUTFILE}_${now}_reco2" | sed -e 's/reco2.root//'`

mv ${ORIG_OUTFILE} ${OUTFILE}.root

### Get the metadata
##TODO -- convert this is the right file extensions
extractor_prod.py --infile ${OUTFILE}.root --campaign ${CAMPAIGN:-fd_mc_2023a_reco2} \
                  --requestid ritm1780305 --no_crc > ${OUTFILE}.root.json
extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

### Convert the metadata to metacat
##TODO -- make sure the RCDS behavior is correct
python $CONVERT_DIR/convert_metadata.py -i ${OUTFILE}.root.json \
                                                -c ${RECOFCL} -j old_md.json \
                                                --app "art.reco2" \
                                                --app_ver "${DUNE_VERSION:-v09_81_00d02}" \
                                                --det "${MDDETTYPE}" \
                                                --parent ${DID}
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "Converted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to convert md"
  exit $converterExit
fi

##If running VD, there will also be pandora files. Rename these
if [ "${DETECTOR}" = "VD" ]; then
  all_good=true
  mv Validation.root "${OUTFILE}_Validation.root" || all_good=false
  mv Pandora_Events.pndr "${OUTFILE}_Pandora_Events.pndr" || all_good=false

  ##Copy over the metadata for pndr
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Pandora_Events.pndr.json || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e 's/artroot/binary/' || all_good=false
  sed -i ${OUTFILE}_Pandora_Events.pndr.json -e '/art.file_format/d' || all_good=false
  
  ##Copy over the metadata for validation.root 
  ##Edit the fields corresepondingly
  sed ${OUTFILE}.root.json -e 's/full-reconstructed/pandora-info/' > ${OUTFILE}_Validation.root.json || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e 's/artroot/root/' || all_good=false
  sed -i ${OUTFILE}_Validation.root.json -e '/art.file_format/d' || all_good=false

  if [ $all_good = false ]; then
    echo "Something failed when editing pandora metadata"
    jobscriptExit=1
    exit $jobscriptExit
  fi

fi

ANAOUTFILE="${OUTFILE}_ana"
echo "Will output ana to $ANAOUTFILE"

###Run ana 
lar -c "${ANAFCL}" \
    -n "${NEVENTS:--1}" \
    -T "$ANAOUTFILE.root" \
    "$OUTFILE.root" >ana.log 2>&1


larExit=$?
echo "Ana step lar exit code $larExit"

### TODO: Handle this correctly
if [ $larExit -eq 0 ] ; then
  # Success !
  # Log the file for justin and move on to the next step
  echo "$FILE" > justin-processed-pfns.txt
else
  # Error -- exit immediately 
  jobscriptExit=1
  tail -100 ana.log
  exit $larExit
fi

sed_good=true
sed ${OUTFILE}.root.json -e 's/full-reconstructed/root-tuple-virtual/' > ${ANAOUTFILE}.root.json || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e 's/artroot/root/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.file_format/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/art.process/d' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${RECOFCL}/${ANAFCL}/" || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e '/campaign/!s/reco2/anatree/' || sed_good=false
sed -i ${ANAOUTFILE}.root.json -e "s/${DID}/${OUTFILE}.root/" || sed_good=false

if [ $sed_good = false ]; then
  echo "Something failed when editing ana metadata"
  jobscriptExit=1
  exit $jobscriptExit
fi

ls
exit 0