21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 8051, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	90.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	fardet-vd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio fardet-vd:fardet-vd-reco2-ritm2440929_8051-w8051s1p1	*reco2.root	10368000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/d8dadc139bff0f0047b9fbf848af00bb7bad89eb

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10000	0	699	0	0	8985	23	293

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
27013	0	0	0	0	9106	0	1	17834	15	2	55

RSEs used

Name	Inputs	Outputs
DUNE_US_FNAL_DISK_STAGE	17593	8796
DUNE_US_BNL_SDCC	0	31
RAL_ECHO	0	31
SURFSARA	0	10
DUNE_UK_GLASGOW	0	9
DUNE_CERN_EOS	0	8
PRAGUE	0	7
DUNE_FR_CCIN2P3_DISK	0	6
RAL-PP	0	5
DUNE_UK_MANCHESTER_CEPH	0	4
DUNE_UK_LANCASTER_CEPH	0	1
NIKHEF	0	1
QMUL	0	1

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_RAL-Tier1	2632	66
NL_SURFsara	1188	19
UK_Manchester	617	4
ES_PIC	589	11
US_FNAL-FermiGrid	537	34
UK_Lancaster	386	2
CZ_FZU	278	9
US_NotreDame	240	0
UK_QMUL	237	2
UK_RAL-PPD	211	4
US_UCSD	208	3
FR_CCIN2P3	195	16
US_UChicago	165	2
NL_NIKHEF	153	1
CERN	120	10
IT_CNAF	97	1
US_Wisconsin	52	1
US_Colorado	50	0
UK_Durham	46	0
US_FNAL-T1	35	0
UK_Oxford	32	0
BR_CBPF	16	0
CA_Victoria	12	0
UK_Sheffield	7	2
UK_Bristol	6	0
US_PuertoRico	4	0
UK_Brunel	1	0

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_04d01}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
#echo "printing env"
#env


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file" 

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 

##Check that any file was returned
if [ "${DID_PFN_RSE}" == "" ] ; then
  echo "Could not get file"
  exit 0
fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
echo "DID: ${DID}"
 
metacat file show -mj ${DID} > old_md.json
mcExit=$?
if [ $mcExit -eq 0 ] ; then
  echo "old metadata:"
  cat old_md.json
else
  echo "Could not retrieve old metadata"
  exit 1 
fi

now=$(date -u +"%Y%m%dT%H%M%SZ")  
namespace=${JUSTIN_SCOPE:-"usertests"}

echo "===============JUSTIN_JOBSUB_ID"
runid=$JUSTIN_WORKFLOW_ID
CLUSTER=`echo $JUSTIN_JOBSUB_ID | awk '{split($0,a,"."); print a[1]}'`
echo $CLUSTER
 
# reco2
echo "============reco2=========================" 


infile=`echo $DID | awk -F ':' '{print $2}'`
reco2="${infile%.root}" 
reco2_name=${reco2}_${now}_reco2 

echo "check  output filename "  ${reco2_name}.root

#lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n 1 #-n -1 
lar -c reco2_supernova_dunevd10kt_1x8x14_3view_30deg_prod2024.fcl $FILE -o ${reco2_name}.root -n -1 
larExit=$? 


#echo "Reco step lar exit code $larExit"


if [ $larExit -eq 0 ] ; then 
  echo "Moving on to metadata extractor "
else
  exit $larExit 
fi
 
OUTFILE=`ls *reco2.root`
echo "============OUTPUT FILE: " $OUTFILE  
 
extractor_prod.py --infile ${OUTFILE}  --no_crc    --appfamily art --appname reco2 --appversion  v09_91_04d01  --requestid ritm2440929  --input_json ${INPUT_DIR}/le_input.json> ${OUTFILE}.ext.json  && sed -i -e 's/physics/fardet-vd/g' ${OUTFILE}.ext.json

extractorExit=$?
if [ $extractorExit -eq 0 ] ; then
  # Success !
  echo "Extracted metadata"
else
  # Error -- exit immediately 
  jobscriptExit=1
  echo "Failed to extract md"
  exit $extractorExit
fi

sed -i 's/Reco2/reco2/g'  ${OUTFILE}.ext.json

echo "checking file======" $DID
echo $DID > all-input-dids.txt
python ${INPUT_DIR}/pdjson2metadata ${OUTFILE}.ext.json  all-input-dids.txt usertests > ${OUTFILE}.json 
		  
converterExit=$?
if [ $converterExit -eq 0 ] ; then
  # Success !
  echo "MD conversion to MetaCat OK"
  echo "$FILE" > justin-processed-pfns.txt
else 
  jobscriptExit=1
  echo "Failed to convert md to MetacaCat"
  exit $converterExit
fi

 
ls
exit 0