justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 7441, Stage 1

Priority	50
Processors	1
Wall seconds	50000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	4193255424 (3999 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	ehn1-beam-np04
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	Rucio ehn1-beam-np04:calcuttj_g4beamline-w7441s1p1	*root	7776000	False

Environment variables

Name	Value
G4BL_DIR	/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/023b964308029fd1755f5b4a2ae2fc05d5107859/
G4DATA_DIR	/cvmfs/fifeuser4.opensciencegrid.org/sw/dune/754ab2932db4c5a30549905327154f5c7a9f083c/
INPUT_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/40a9891e2a7c1f62d618824f8f8588fe2ebec5fe/
NPART	100000
PACK_DIR	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/3851f3036b8fdf366c8f0dd3fe8fea81c1d30f87/

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10000	0	0	0	0	9496	0	504

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
31158	0	0	0	0	10541	0	191	20198	0	1	227

RSEs used

Name	Inputs	Outputs
MONTECARLO	27288	0
DUNE_US_FNAL_DISK_STAGE	0	2413
DUNE_UK_GLASGOW	0	1686
SURFSARA	0	1304
DUNE_UK_LANCASTER_CEPH	0	936
DUNE_CERN_EOS	0	756
RAL_ECHO	0	675
DUNE_UK_MANCHESTER_CEPH	0	553
RAL-PP	0	487
QMUL	0	302
DUNE_FR_CCIN2P3_DISK	0	241
DUNE_US_BNL_SDCC	0	100
DUNE_CA_SFU	0	37

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
NL_SURFsara	3487	14
UK_Manchester	2038	2
US_FNAL-FermiGrid	1976	42
US_NotreDame	1935	0
UK_RAL-PPD	974	2
UK_RAL-Tier1	903	21
ES_PIC	900	8
CERN	868	8
UK_Lancaster	778	17
FR_CCIN2P3	652	5
UK_QMUL	613	2
US_UChicago	579	5
UK_Sheffield	234	3
UK_Bristol	181	2
UK_Oxford	166	3
US_UCSD	160	3
US_Colorado	126	0
UK_Durham	125	0
IT_CNAF	112	0
CA_SFU	100	1
US_FNAL-T1	60	0
US_Wisconsin	59	0
UK_Brunel	38	0
CA_Victoria	23	0
UK_Edinburgh	21	0
BR_CBPF	11	0
US_PuertoRico	6	0

Jobscript

#!/bin/bash

SECONDS=0

## Set the poolarity
export POLARITY="${POLARITY:-+}"
if [ $POLARITY != "+" ] && [ ${POLARITY} != "-" ]; then
  echo "ERROR MUST SUPPLY + OR - TO POLARITY"
  exit 1
fi

##Set the upstream momentum
export PMOMENTUM="${POLARITY:-+}80000"
echo "PMOMENTUM: ${PMOMENTUM}"

export BEAMLINE="${BEAMLINE:-H4}"
export INFILE=${BEAMLINE}.in
export CENTRALP="${CENTRALP:-1}" #Set the momentum going into protodune
export MOMENTUMVLE="${POLARITY}${CENTRALP}" #"3"
echo "MOMENTUMVLE: ${MOMENTUMVLE}"

##Number of POT to run
export PARTPERJOB=${NPART:-100}
export ADDPARAM="momentumVLE=$MOMENTUMVLE pMomentum=$PMOMENTUM"
export ADDFILES=""


if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  


##Get the MC number from this to bookkeep for justin
DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
pfn_exit=$?
if [ $pfn_exit -ne 0 ]; then
  echo "Error in justin-get-file. Exiting safely"
  exit 0
fi
echo "did_pfn_rse $DID_PFN_RSE"
pfn=`echo $DID_PFN_RSE | cut -f2 -d' '`
JOBID=$pfn
echo "JOBID: ${JOBID}"

echo $INPUT_DIR
ls $INPUT_DIR

echo $G4DATA_DIR
ls $G4DATA_DIR
#cp -rs $G4DATA_DIR ./Geant4Data/
ln -s $G4DATA_DIR/Geant4Data ./Geant4Data

echo $G4BL_DIR
ls $G4BL_DIR
cp -rs $G4BL_DIR/g4bl ./g4bl
export G4BL_DIR=$PWD/g4bl

echo $PACK_DIR
ls $PACK_DIR
for i in $PACK_DIR/*; do
  ln -s $i .
done

#Unpack all the tars -- TODO: put on cvmfs as a single tar
#echo "Unpacking g4bl"
#tar -xzf $INPUT_DIR/g4bl.tar.gz --checkpoint=1000
#if [ $? -ne 0 ]
#then
#  echo "Exiting with error"
#  exit 1
#fi
#
#echo "Unpacking Geant4Data"
#tar -xzf $INPUT_DIR/Geant4Data.tar.gz --checkpoint=1000
#if [ $? -ne 0 ]
#then
#  echo "Exiting with error"
#  exit 1
#fi
CURDIR=$(pwd)
echo $CURDIR/Geant4Data > g4bl/.data


#echo "Unpacking Inputfiles Pack"
#tar -xzf $INPUT_DIR/pack.tar.gz --checkpoint=1000
#if [ $? -ne 0 ]
#then
#  echo "Exiting with error"
#  exit 1
#fi

#Run
echo "running"
#$CURDIR
./g4bl/bin/g4bl $INFILE jobID=$JOBID totNumEv=$PARTPERJOB $ADDPARAM 2>&1 | tee g4bloutput.txt
g4bl_res=$?
if [ $g4bl_res -ne 0 ]
then
  echo "Failed running g4bl"
  exit $g4bl_res
fi
echo "ran"

#Clean up
unlink Geant4Data
rm -rf g4bl
for i in *.in *.map; do
  unlink ${i}
done



#Add timestamp to the output
now=$(date -u +"%Y%m%dT%H%M%SZ")
oldname=`ls ${BEAMLINE}*.root`
newname=`echo ${oldname} | sed -e "s/.root/_${now}_${pfn}.root/"`
mv ${oldname} ${newname}
if [ $? -ne 0 ]
then
  echo "Failed renaming ${oldname} ${newname}"
  exit 1
fi


if [ $POLARITY != "+" ]; then
  polar_str="neg"
else
  polar_str="pos"
fi

subrun=`echo $JUSTIN_JOBSUB_ID  | cut -f1 -d@ | cut -f2 -d.`
run=`echo $JUSTIN_JOBSUB_ID  | cut -f1 -d@ | cut -f1 -d.`

python $INPUT_DIR/make_g4bl_metadata.py \
  --bl "${BEAMLINE}" --polarity $polar_str --momentum $CENTRALP \
  --run $run --subrun $subrun --name $newname \
  --namespace ${JUSTIN_SCOPE:-dummy}

if [ $? -ne 0 ]
then
  echo "Exiting with error"
  exit 1
else
  echo "$pfn" > justin-processed-pfns.txt
fi

#errorsSaving=$((`cat g4bloutput.txt | grep "Error in <T" | wc -l`))
#if [ $errorsSaving -ne 0 ]
#then
#  echo "Exiting with error"
#  exit 1
#fi
#
#echo "RUNTIME: $SECONDS seconds elapsed."

justIN time: 2025-08-14 16:31:52 UTC       justIN version: 01.03.02