Workflow 2173, Stage 1

Priority	50
Processors	1
Wall seconds	80000
RSS bytes	5767168000 (5500 MiB)
Max distance for inputs	30.0
Enabled input RSEs
Enabled output RSEs
Enabled sites
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
0	Rucio usertests:pdhd_1gev_rerun_test_sce_plus_reco_skip0_2173	*reco.root	604800	False
0	Rucio usertests:pdhd_1gev_rerun_test_sce_plus_pdspana_skip0_2173	*pdspana.root	604800	False

Environment variables

Name	Value
DUNESIM_TAR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/5aa1c030e818b0176375b02e7ca9d1d1c99f27fa
FCL_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/120c67294d536d3c4cc0fbab7129d1deeffd2a82
INPUT_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/99ff67fc99d00f6015d396607593e600a84e7b63
TYPE	plus

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
5025	0	0	0	0	5025	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
6959	0	0	0	0	5508	0	1418	31	2	0	0

RSEs used

Name	Inputs	Outputs
MANCHESTER	6475	0
SURFSARA	0	3516
RAL_ECHO	0	1806
DUNE_CERN_EOS	0	1596
QMUL	0	1272
NIKHEF	0	870
RAL-PP	0	670
DUNE_ES_PIC	0	152
PRAGUE	0	106
DUNE_FR_CCIN2P3_DISK	0	62

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
NL_SURFsara	466	0
CERN	210	0
UK_RAL-Tier1	188	0
UK_Manchester	139	1
UK_QMUL	101	0
IT_CNAF	95	0
NL_NIKHEF	88	0
UK_Brunel	39	0
ES_PIC	28	0
UK_Oxford	21	0
UK_Sheffield	19	0
UK_Lancaster	13	0
UK_Edinburgh	11	0
CZ_FZU	9	0
UK_Liverpool	9	0
UK_Imperial	8	0
UK_Durham	3	0

Jobscript

#!/bin/bash
#

check_tar() {
  stat ${1}
  if [ $? -ne 0 ]; then
    echo "Failed to stat ${1}. Exiting safely"
    exit 0
  fi
}


#These must be defined
if [ -z $FCL_DIR ]; then
  echo "Fatal Must provide FCL_DIR env var"
  exit 1
fi

if [ -z $DUNESIM_TAR ]; then
  echo "Fatal Must provide DUNESIM_TAR env var"
  exit 1
fi

if [ -z $INPUT_DIR ]; then
  echo "Fatal Must provide INPUT_DIR env var"
  exit 1
fi

check_tar ${FCL_DIR}
check_tar ${DUNESIM_TAR}
check_tar ${INPUT_DIR}

#stat ${FCL_DIR}
#if [ $? -ne 0 ]; then
#  echo "Failed to stat $FCL_DIR. Exiting safely"
#  exit 0
#fi
#
#stat ${DUNESIM_TAR}
#if [ $? -ne 0 ]; then
#  echo "Failed to stat $DUNESIM_TAR. Exiting safely"
#  exit 0
#fi

TYPE=${TYPE:-"nominal"}
if [ $TYPE == "nominal" ]; then
g4_stage2_fcl=${FCL_DIR}/filtered_g4_stage2.fcl
reco_fcl=${FCL_DIR}/filtered_reco.fcl
base_reco_name=filtered_reco.fcl
base_g4_name=filtered_g4_stage2.fcl
elif [ $TYPE == "plus" ]; then
g4_stage2_fcl=${FCL_DIR}/plus_sigma_sce_g4_stage2.fcl
reco_fcl=${FCL_DIR}/plus_sigma_sce_reco.fcl
base_reco_name=plus_sigma_sce_reco.fcl
base_g4_name=plus_sigma_sce_g4_stage2.fcl
elif [ $TYPE == "minus" ]; then
g4_stage2_fcl=${FCL_DIR}/minus_sigma_sce_g4_stage2.fcl
reco_fcl=${FCL_DIR}/minus_sigma_sce_reco.fcl
base_reco_name=minus_sigma_sce_reco.fcl
base_reco_name=minus_sigma_sce_reco.fcl
base_g4_name=minus_sigma_sce_g4_stage2.fcl
fi

echo "$TYPE sce"
echo "g4_stage2: $g4_stage2_fcl"
echo "reco: $reco_fcl"

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

export PRODUCTS=$DUNESIM_TAR:$PRODUCTS

#Setup recent lar software suite
#Set up local
DUNE_VERSION=${DUNE_VERSION:-v09_85_00d00}
setup dunesw \
   "${DUNE_VERSION}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"

setup_exit=$?
if [ $? -ne 0 ]; then
  echo "Failed to setup dunesw $DUNE_VERSION $DUNE_QUALIFIER"
  exit $setup_exit
fi


echo "DUNESW loc:"
ups active | grep dunesw

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

# Set fhicl file path to include the RCDS fcls
export FHICL_FILE_PATH=${FCL_DIR}:${FHICL_FILE_PATH}

echo "printing env"
env

echo "Justin specific env vars"
env | grep JUSTIN

echo "Will use justin-get-file"
#

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi


pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
now=$(date -u +"%Y%m%dT%H%M%SZ")

##TODO -- edit this
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | awk -F '.' '{print $1}'`
G4Stage1File="PDSP_RerunG4_${jobid}_${JUSTIN_STAGE_ID}_${JUSTIN_WORKFLOW_ID}_${now}.root"


nevents=${NEVENTS:--1}

##Only doing ntuple on input
if [ -n "$ANAONLY" ]; then
  AnaFile=`echo ${did} | cut -f2 -d':' | sed -e "s/.root/_pdspana.root/"`
  echo "Running pdspana only. Output: $AnaFile"
  recostart=`date +"%s"`.0
  lar -c ${FCL_DIR}/pduneana_filtered.fcl \
      -T ${AnaFile} \
       -n ${nevents} \
      ${pfn} >ana.log 2>&1
  larExit=$?
  recoend=`date +"%s"`.0
  
  if [ $larExit -ne 0 ]; then
    echo "Error in ana production"
    cat ana.log
    exit $larExit
  fi
  echo "Ran successfully"

  echo "Forming ana metadata"
  
  echo "${did}" > input_dids.list
  
  ### Get the metadata
  python ${INPUT_DIR}/ntuple_prod_utils.py \
    metadata \
    --root_file ${AnaFile} \
    --dids input_dids.list \
    --version ${DUNE_VERSION} \
    --fcl_name pduneana_filtered.fcl \
    --log_file ana.log \
    -o $AnaFile.json
  mdExit=$?
  if [ $mdExit -ne 0 ]; then
    echo "Error in ntuple metdata production"
    exit $mdExit
  fi
  
  echo "formed"
  cat ${AnaFile}.json




  echo "$pfn" > justin-processed-pfns.txt
  exit 0
fi



### FILTERED G4 STAGE 1 ###
echo "Running rerun g4"
#touch g4.log
starttime=`date +"%s"`.0
lar -c ${FCL_DIR}/rerun_g4_stage1.fcl \
    -n ${nevents} \
    -o ${G4Stage1File} \
    ${pfn} >g4.log 2>&1
larExit=$?
endtime=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in stage 1 production"
  cat g4.log
  exit $larExit
fi
echo "Ran successfully"
##########################

#------------ Generic Path -------------------#
G4Stage2File=`echo ${G4Stage1File} | sed -e "s/.root/_${TYPE}_g4_stage2.root/"`
echo "Running ${TYPE} G4 Stage 2"
lar -c $g4_stage2_fcl \
    -o ${G4Stage2File} \
    ${G4Stage1File} >g4_s2.log 2>&1
larExit=$?

if [ $larExit -ne 0 ]; then
  echo "Error in stage 2 production"
  cat g4_s2.log
  exit $larExit
fi
echo "Ran successfully"

DetsimFile=`echo ${G4Stage2File} | sed -e "s/.root/_detsim.root/"`
echo "Running Detsim"
lar -c ${FCL_DIR}/filtered_detsim.fcl \
    -o ${DetsimFile} \
    ${G4Stage2File} >detsim.log 2>&1
larExit=$?

if [ $larExit -ne 0 ]; then
  echo "Error in detsim production"
  cat detsim.log
  exit $larExit
fi
echo "Ran successfully"

RecoFile=`echo ${DetsimFile} | sed -e "s/.root/_reco.root/"`
echo "Running ${TYPE} SCE Reco"
lar -c $reco_fcl \
    -o ${RecoFile} \
    ${DetsimFile} >reco.log 2>&1
larExit=$?

if [ $larExit -ne 0 ]; then
  echo "Error in reco production"
  cat reco.log
  exit $larExit
fi
echo "Ran successfully"

AnaFile=`echo ${RecoFile} | sed -e "s/.root/_pdspana.root/"`
echo "Running sigma pdspana"
recostart=`date +"%s"`.0
lar -c ${FCL_DIR}/pduneana_filtered.fcl \
    -T ${AnaFile} \
    ${RecoFile} >ana.log 2>&1
larExit=$?
recoend=`date +"%s"`.0

if [ $larExit -ne 0 ]; then
  echo "Error in sigma ana production"
  cat ana.log
  exit $larExit
fi
echo "Ran successfully"

# Make metadata
overrides="core.data_tier=full-reconstructed \
 core.application.version=${DUNE_VERSION} \
 dune.config_file=${base_reco_name} \
 core.start_time=${recostart} \
 core.end_time=${recoend} \
 core.application.name=reco \
 core.application=art.reco \
 dune_mc.space_charge=yes \

"

namespace=${JUSTIN_SCOPE:-"usertests"}

echo "Forming reco metadata"
#-- ${filenum} \
python ${INPUT_DIR}/pdhd_meta_writer.py \
       --json ${INPUT_DIR}/pdhd_base_meta.json \
       --overrides ${overrides} \
       --nevents ${nevents} \
       --jobid ${JUSTIN_JOBSUB_ID} \
       --parent ${did} \
       --past_fcls rerun_g4_stage1.fcl \
                   $base_g4_name \
                   filtered_detsim.fcl \
       --past_apps g4_stage1 g4_stage2 detsim \
       --inherit_run \
       --exclude "dune.requestid" \
       -o ${RecoFile}.json


if [ $? -ne 0 ]; then
  echo "Error writing reco json"
  exit 1
fi

cat ${RecoFile}.json
echo "formed"

echo "Forming ana metadata"

echo "${namespace}:$RecoFile" > input_dids.list

### Get the metadata
python ${INPUT_DIR}/ntuple_prod_utils.py \
  metadata \
  --root_file ${AnaFile} \
  --dids input_dids.list \
  --version ${DUNE_VERSION} \
  --fcl_name pduneana_filtered.fcl \
  --log_file ana.log \
  --parent_metas ${RecoFile}.json \
  -o $AnaFile.json
mdExit=$?
if [ $mdExit -ne 0 ]; then
  echo "Error in ntuple metdata production"
  exit $mdExit
fi

echo "formed"
cat ${AnaFile}.json


###############################################
echo "$pfn" > justin-processed-pfns.txt