justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 7321, Stage 1

Priority50
Processors4
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes15728640000 (15000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1https://fndcadoor.fnal.gov:2880/dune/scratch/users/sergeym/07321/1*.h5

Environment variables

NameValue
WIRECELL_CONFIGS/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/87d73d6e16cfd7e90331489dd0d3ffe569058dc8

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
10000001

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
1700001008260

RSEs used

NameInputsOutputs
MONTECARLO60
None02

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
UK_Manchester10
ES_PIC10
UK_QMUL10
US_NotreDame10
CERN01

Jobscript

#!/bin/bash
#

#These must be defined
# if [ -z $UTIL_TAR ]; then
#   echo "Fatal must provide UTIL_TAR env var"
#   exit 1
# fi
# export PYTHONPATH=$UTIL_TAR:$PYTHONPATH

source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
# setup metacat
# export METACAT_SERVER_URL=https://metacat.fnal.gov:9443/dune_meta_prod/app
# export METACAT_AUTH_SERVER_URL=https://metacat.fnal.gov:8143/auth/dune

if [ -z ${JUSTIN_PROCESSORS} ]; then
  JUSTIN_PROCESSORS=1
fi

echo "Justin processors: ${JUSTIN_PROCESSORS}"

export TF_NUM_THREADS=${JUSTIN_PROCESSORS}   
export OPENBLAS_NUM_THREADS=${JUSTIN_PROCESSORS} 
export JULIA_NUM_THREADS=${JUSTIN_PROCESSORS} 
export MKL_NUM_THREADS=${JUSTIN_PROCESSORS} 
export NUMEXPR_NUM_THREADS=${JUSTIN_PROCESSORS} 
export OMP_NUM_THREADS=${JUSTIN_PROCESSORS}  

echo "Justin specific env vars"
env | grep JUSTIN
now=$(date -u +"%Y%m%dT%H%M%SZ")
jobid=`echo "${JUSTIN_JOBSUB_ID:-1}" | cut -f1 -d'@' | sed -e "s/\./_/"`
stageid=${JUSTIN_STAGE_ID:-1}
a=1
 if [ -n "$DUNESW_TAR" ]; then
   stat ${DUNESW_TAR}
   if [ $? -ne 0 ]; then
     echo "Failed to stat $DUNESW_TAR. Exiting safely"
     exit 0
   fi
 
   export PRODUCTS=$DUNESW_TAR:$PRODUCTS 
   echo "Set dunesw to $DUNESW_TAR"
 fi

 if [ -n "$WIRECELL_TAR" ]; then
   stat ${WIRECELL_TAR}
   if [ $? -ne 0 ]; then
     echo "Failed to stat $WIRECELL_TAR. Exiting safely"
     exit 0
   fi
 
   export PRODUCTS=$WIRECELL_TAR:$PRODUCTS 
   echo "Set wirecell to $WIRECELL_TAR"
 fi

 #Setup recent lar software suite
 DUNESW_VERSION=${DUNESW_VERSION:-v10_04_07d01}
 setup dunesw \
    "${DUNESW_VERSION}" \
    -q "${DUNE_QUALIFIER:-e26:prof}"
 
 setup_exit=$?
 if [ $? -ne 0 ]; then
   echo "Failed to setup dunesw $DUNESW_VERSION $DUNE_QUALIFIER"
   exit $setup_exit
 fi
 
 echo "DUNESW loc:"
 ups active | grep dunesw

 echo "WIRECELL loc:"
 ups active | grep wirecell
 
 
 echo "printing env"
 env
 echo "DUNE specific env vars"
 env | grep DUNE

# # User can specify to pip install pyyaml
# if [ -n "$pipyaml" ]; then
#   echo "Installing yaml"
#   pip install --user pyyaml
#   echo "Done"
# # elif [ -n "${YAML_TAR}" ]; then
# #   echo "Using venv at ${YAML_TAR}"
# #   source ${YAML_TAR}/yaml_venv/bin/activate
# fi


 echo "Will use justin-get-file"
 DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
 if [ "${DID_PFN_RSE}" == "" ] ; then
   echo "Could not get file"
 #  exit 0
 fi
 pfn=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
 did=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
 
 input_filename=`echo $did | cut -f2 -d':'`
 echo "input file: $input_filename"

 echo "jobsub_id: ${JUSTIN_JOBSUB_ID:-1}"
 
#  #Get run/subrun from parent if using input -- else from justin job ID/process"
#  if [ -z "$MCJob" ]; then
#    echo "Getting run subrun from $did"
#    run_subrun=`python -m beam_job_utils get_run_subrun -i $did`
#    get_run_ret=$?
#    if [ $get_run_ret -ne 0 ]; then
#      echo "error in get_run_subrun"
#    fi
 
#    echo $run_subrun
#    run=`echo $run_subrun | cut -f1 -d' '`
#    subrun=`echo $run_subrun | cut -f2 -d' '`
#  else
#    run=`echo $jobid | cut -f1 -d'_'`  ##TODO -- check order here
#    subrun=`echo $jobid | cut -f2 -d'_'`
#  fi
 nevents=${NEVENTS:--1} #Set from env override or -1 
 NSKIP=${NSKIP:-0} 
 echo "nevents: $nevents"
 
#  overrides="core.data_tier=full-reconstructed \
#   core.application.version=${DUNESW_VERSION} \
#   core.application.name=reco \
#   core.application.family=dunesw \
#   core.application=dunesw.reco \
#  "

# #User specifies if they'd like to use an input file or not 
# if [ -n "$MCJob" ]; then
#   inputflag=""
# else
#   inputflag="-i $pfn --parent $did"
# fi

echo "input flag: $inputflag"

# if [[ $pfn == *.hdf5 ]]; then
#   echo "Setting LD_PRELOAD"  
#   export LD_PRELOAD=$XROOTD_LIB/libXrdPosixPreload.so
# fi

echo "TESTING DIRECTORIES"
echo ${PWD}
cp -r ${WIRECELL_CONFIGS}/ts-model .

cp ${WIRECELL_CONFIGS}/wirecell_dune.fcl .


ls -lh

export WIRECELL_PATH=${WIRECELL_CONFIGS}/wire-cell-cfg:$WIRECELL_PATH

echo $WIRECELL_PATH

ups active

lar -n50 -c prod_cosmics_protodunehd.fcl -o gen.root
exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running generation. Exiting with ${exitcode}"
  exit $exitcode
fi

lar -n50 -c standard_g4_protodunehd.fcl gen.root -o g4.root
exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running g4. Exiting with ${exitcode}"
  exit $exitcode
fi

lar -n50 -c pdhd_wirecell_sim_deposplat.fcl g4.root
exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error running wirecell. Exiting with ${exitcode}"
  exit $exitcode
fi

run=`echo $jobid | cut -f1 -d'_'`  ##TODO -- check order here
subrun=`echo $jobid | cut -f2 -d'_'`

mv g4-rec-0.h5 g4-rec-0-${run}-${subrun}-${now}.h5
exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error moving reco. Exiting with ${exitcode}"
  exit $exitcode
fi

mv g4-tru-0.h5 g4-tru-0-${run}-${subrun}-${now}.h5
exitcode=$?
if [ $exitcode -ne 0 ]; then
  echo "Error moving true. Exiting with ${exitcode}"
  exit $exitcode
fi


echo "$pfn" > justin-processed-pfns.txt
justIN time: 2025-08-14 16:31:55 UTC       justIN version: 01.03.02