Workflow 6967, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	30.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/ichong/06967/1	training1_CaloHitListW*.data

Environment variables

Name	Value
CODE_TAR_DIR_LOCAL	/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/c976ce192292f2f9835b2b96cb18eba74fad6274
DUNE_QUALIFIER	e26:prof
DUNE_VERSION	v10_04_06d00
FCL_FILE	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/4cda2edbbc6539500fbeb1e9883d004cece9df43/atm-training-extract.fcl
NUM_EVENTS	10
XML_MASTER	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/4cda2edbbc6539500fbeb1e9883d004cece9df43/PandoraSettings_Master_Training_CSV.xml
XML_NEUTRINO	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/4cda2edbbc6539500fbeb1e9883d004cece9df43/PandoraSettings_Neutrino_Training_CSV.xml

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
50	0	0	0	0	47	0	3

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
126	0	0	0	0	53	0	0	50	23	0	0

RSEs used

Name	Inputs	Outputs
PRAGUE	26	0
QMUL	17	0
NIKHEF	10	0
SURFSARA	9	0
DUNE_ES_PIC	3	0
RAL_ECHO	2	0
RAL-PP	2	0
DUNE_FR_CCIN2P3_DISK	1	0

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash
:<<'EOF'
This jobscript is designed to generate CaloHitList-based graph data 
from input reco2 ROOT files using your custom LArSoft setup.

Environment variables used:
  - FCL_FILE
  - CODE_TAR_DIR_LOCAL
  - DUNE_VERSION
  - DUNE_QUALIFIER
  - XML_MASTER
  - XML_NEUTRINO
  - NUM_EVENTS (optional)
EOF

# === Setup FCL and version info ===
FCL_FILE=${FCL_FILE:-atm-training-extract.fcl}
DUNE_VERSION=${DUNE_VERSION:-v10_04_06d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
HAS_ART_OUTPUT=false

# === Number of events ===
if [ -n "$NUM_EVENTS" ]; then
  events_option="-n $NUM_EVENTS"
fi

# === Get a file from justIN ===
did_pfn_rse=$($JUSTIN_PATH/justin-get-file)
if [ -z "$did_pfn_rse" ]; then
  echo "No file assigned. Exiting jobscript."
  exit 0
fi

# === Parse PFN from did_pfn_rse ===
pfn=$(echo "$did_pfn_rse" | cut -d' ' -f2)
echo "Input PFN = $pfn"

# === DUNE software setup ===
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
export PRODUCTS=/cvmfs/dune.opensciencegrid.org/products/dune/testproducts:${PRODUCTS}
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# === Custom local products ===
if [ -n "$CODE_TAR_DIR_LOCAL" ]; then
  echo "Loading local LArSoft from $CODE_TAR_DIR_LOCAL"
  source ${CODE_TAR_DIR_LOCAL}/*/localProducts*/setup
  mrbslp
fi

# === Output file naming ===
now=$(date -u +"%Y-%m-%dT_%H%M%SZ")
fname=$(basename "$pfn" .root)

# === Set FW search path to include both XML directories ===
XML_DIR_MASTER=$(dirname "$XML_MASTER")
XML_DIR_NEUTRINO=$(dirname "$XML_NEUTRINO")
export FW_SEARCH_PATH="$XML_DIR_MASTER:$XML_DIR_NEUTRINO:$FW_SEARCH_PATH"

# === Run lar ===
export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
echo "Running LArSoft with FCL: $FCL_FILE"
lar -c "$FCL_FILE" $events_option "$pfn" > ${fname}_graph_${now}.log 2>&1
larExit=$?
# === Mark processed if successful ===
if [ $larExit -eq 0 ]; then
  echo "$pfn" > justin-processed-pfns.txt
  jobscriptExit=0
else
  jobscriptExit=1
fi

# === Package logs ===
tar zcf "${JUSTIN_JOBSUB_ID}.logs.tgz" *.log

# === Rename .data files ===
data_files=$(ls -1 *.data 2>/dev/null)
if [ -n "$data_files" ]; then
  for f in $data_files; do
    base=$(basename "$f" .data)
    new_name="${base}__${fname}__${JUSTIN_JOBSUB_ID}.data"
    mv "$f" "$new_name"
  done
else
  echo "No .data files to rename."
fi


# === Show generated .data files ===
echo "=== Generated .data files ==="
data_files=$(ls -1 *.data 2>/dev/null)
if [ -n "$data_files" ]; then
  echo "$data_files"
else
  echo "No .data files found."
fi

# === Export log content for Justin dashboard ===
echo "=== Last 100 lines of log (for dashboard display) ==="
tail -n 100 ${fname}_graph_${now}.log > jobscript.log
cat jobscript.log

ls -lh *.data *.log || true

exit $jobscriptExit