Workflow 3239, Stage 1

Priority	50
Processors	1
Wall seconds	80000
RSS bytes	4194304000 (4000 MiB)
Max distance for inputs	0.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MANCHESTER, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	fardet-hd
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage
1	Rucio fardet-hd:fardet-hd-caf_3239	caf_*.root	10368000	False

Environment variables

Name	Value
INPUT_DIR	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/833941fb2dc223b19c3aa1ebf3715428401daab1

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
6000	0	0	0	0	6000	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
4676	0	0	0	0	4227	0	234	207	0	8	0

RSEs used

Name	Inputs	Outputs
PRAGUE	7308	371
RAL_ECHO	838	0
DUNE_US_BNL_SDCC	184	1
DUNE_ES_PIC	50	1
DUNE_CERN_EOS	0	49
SURFSARA	0	6
DUNE_US_FNAL_DISK_STAGE	0	1
RAL-PP	0	1

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
CZ_FZU	1230	764
UK_RAL-Tier1	230	100
US_BNL	33	0
ES_PIC	23	0

Jobscript

#!/bin/bash
#
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup metacat
export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune


##TODO -- write usage

#CAF_FCL="cafmaker_dunevd10kt_1x8x6_3view_30deg_runreco-nuenergy_geov3.fcl"
CAF_FCL="cafmaker_atmos_dune10kt_1x2x6_runreco-nuenergy-nuangular_geov5.fcl"
echo "Using CAF fcl: ${CAF_FCL}"

#Setup recent lar software suite
setup dunesw \
   "${DUNE_VERSION:-v09_91_01d00}" \
   -q "${DUNE_QUALIFIER:-e26:prof}"
#echo "printing env"
#env
 

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

#
echo "Will use justin-get-file"

now=$(date -u +"%Y%m%dT%H%M%SZ")


for nf in {1..15}
  do
  DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file` 
 ##Check that any file was returned
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
#   exit 0
   continue
 fi

FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 echo ${DID} >> did.list
 echo ${FILE} >> file.list 
done



now=$(date -u +"%Y%m%dT%H%M%SZ")
 
####Run cafmaker
 
echo "now run lar on these files"

cat file.list
cat did.list
 
# echo $FILE 
# lar -c  "${CAF_FCL}" "$FILE"  
lar -c  "${CAF_FCL}" -S file.list > caf_$now.log

# python $INPUT_DIR/mergeMetaCat.py --fileName caf.root --nameSpace usertest --fileList did.list --config ${CAF_FCL} --campaign caf_fd_2024
python $INPUT_DIR/mergeMetaCat.py --fileName caf.root --nameSpace fardet-hd --fileList did.list --config ${CAF_FCL} --campaign caf_fd_2024

# echo "$FILE" > justin-processed-pfns.txt
  cat file.list > justin-processed-pfns.txt

  echo "processed files"
cat justin-processed-pfns.txt 

  cp justin-processed-pfns.txt caf_$now.pfns 
  cat did.list >> caf_$now.did
  cat file.list >> caf_$now.file
  cp caf.root caf_fd_hd_atmo_${JUSTIN_WORKFLOW_ID}_$now.root
  cp caf.root.json caf_fd_hd_atmo_${JUSTIN_WORKFLOW_ID}_$now.root.json
#  cp caf.root caf_$now.root
#  cp caf.root.json caf_$now.root.json

ls