justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Workflow 4111, Stage 1

Priority50
Processors1
Wall seconds80000
RSS bytes4194304000 (4000 MiB)
Max distance for inputs30.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_SFU, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Imperial, UK_Lancaster, UK_Liverpool, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_BNL, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stage
1https://fndcadoor.fnal.gov:2880/dune/scratch/users/lavaut/04111/1*singleHit*.*

Environment variables

NameValue
INPUT_TAR_DIR_LOCAL/cvmfs/fifeuser3.opensciencegrid.org/sw/dune/b45eb38553651f97b6c3d6135486f05ed5eb726f

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
10000790000921

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
601500000000060150
Replicas per RSE998470.2009095816593179.01262498594315998360.3925372606365322.14521290931697237279.42524999862775223.2757520089874162289.30874705796145180.0209560270332114306.7361942289307154.5961148487327868322.12390560389986141.2461956961642567335.11083370690994133.4983701563615561348.3666321376465128.0292680310608460361.4934599492966124.6423316057286729371.3770445028029123.2953331251814424377.30934293629264122.96099766973255Replicas per RSEDUNE_ES_PIC (35%)FNAL_DCACHE (35%)SURFSARA (8%)DUNE_US_BNL_SDCC (5…DUNE_US_BNL_SDCC (5%)QMUL (4%)RAL-PP (2%)RAL_ECHO (2%)NIKHEF (2%)DUNE_FR_CCIN2P3_DIS…DUNE_FR_CCIN2P3_DISK (2%)PRAGUE (1%)DUNE_CERN_EOS (0%)

RSEs used

NameInputsOutputs
DUNE_ES_PIC21130
SURFSARA12980
QMUL4520
RAL-PP3720
DUNE_US_BNL_SDCC3670
RAL_ECHO3660
NIKHEF3380
DUNE_FR_CCIN2P3_DISK3140
PRAGUE1570
DUNE_CERN_EOS1290

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
NL_NIKHEF075
CERN069
NL_SURFsara056
UK_RAL-PPD040
CZ_FZU027
UK_Manchester024
UK_QMUL018
UK_RAL-Tier1012
US_UChicago09
UK_Imperial08
ES_PIC05
US_FNAL-T104
US_FNAL-FermiGrid04
UK_Edinburgh01

Jobscript

#!/bin/bash
:<<'EOF'

To use this jobscript to process 10 files from the dc4-vd-coldbox-bottom
data and put the output in the usertests namespace (MetaCat) and 
scope (Rucio), and in the usertests:output-test-01 dataset in MetaCat and
Rucio, use this command to create the workflow:

justin simple-workflow \
--mql \
"files from dune:all where core.run_type='dc4-vd-coldbox-bottom' and dune.campaign='dc4' limit 10" \
--jobscript dc4-vd-coldbox-bottom.jobscript --max-distance 30 --rss-mb 4000 \
--scope usertests --output-pattern '*_reco_data_*.root:output-test-01' 

The following optional environment variables can be set when creating the
workflow/stage: FCL_FILE, NUM_EVENTS, DUNE_VERSION, DUNE_QUALIFIER 

EOF

# fcl file and DUNE software version/qualifier to be used
FCL_FILE=MyrunSingleHit_PDHD.fcl
DUNE_VERSION=${DUNE_VERSION:-v10_00_02d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}

# number of events to process from the input file
if [ "$NUM_EVENTS" != "" ] ; then
 events_option="-n $NUM_EVENTS"
fi

# First get an unprocessed file from this stage
for nf in {1..10}
do
   DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
   ##Check that any file was returned
   if [ "${DID_PFN_RSE}" == "" ] ; then
     echo "Could not get file"
     #exit 0
     continue
   fi
   FILE=`echo ${DID_PFN_RSE} | cut -f2 -d' '`
   DID=`echo ${DID_PFN_RSE} | cut -f1 -d' '`
   echo ${DID}  >> did.list
   echo ${FILE} >> file.list

done

did_pfn_rse=`$JUSTIN_PATH/justin-get-file`

#if [ "$did_pfn_rse" = "" ] ; then
 # echo "Nothing to process - exit jobscript"
 # exit 0
#fi

# Keep a record of all input DIDs, for pdjson2meta file -> DID mapping
echo "$did_pfn_rse" | cut -f1 -d' ' >>all-input-dids.txt

# pfn is also needed when creating justin-processed-pfns.txt
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
echo "Input PFN = $pfn"

# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh

export PRODUCTS=$INPUT_TAR_DIR_LOCAL:${PRODUCTS}
echo "PRODUCTS: ${PRODUCTS}"
echo "DUNESW_DIR: ${DUNESW_DIR}"


# Properly setup custom code with INPUT_TAR_DIR_LOCAL

export DUNESW_DIR=${INPUT_TAR_DIR_LOCAL}/dunesw/${DUNE_VERSION}
export DUNESW_FQ_DIR=${DUNESW_DIR}/slf7.x86_64.e26.prof
export DUNESW_LIB=${DUNESW_FQ_DIR}/lib

export PROTODUNEANA_DIR=${INPUT_TAR_DIR_LOCAL}/protoduneana/${DUNE_VERSION}
export PROTODUNEANA_LIB=${PROTODUNEANA_DIR}/include
export PROTODUNEANA_FQ_DIR=${PROTODUNEANA_DIR}/slf7.x86_64.e26.prof
export PROTODUNEANA_LIB=${PROTODUNEANA_FQ_DIR}/lib

#export WIRECELL_PATH=${DUNE_PARDATA_DIR}/WireCellData
#export WIRECELL_PATH=${WIRECELL_DIR}/Linux64bit+3.10-2.17-e26-prof/share/wirecell:${WIRECELL_PATH}
#export WIRECELL_PATH=${PROTODUNEANADIR}/wire-cell-cfg:${WIRECELL_PATH}

export DUNEPROTOTYPES_DIR=${INPUT_TAR_DIR_LOCAL}/duneprototypes/${DUNE_VERSION}
export DUNEPROTOTYPES_INC=${DUNEPROTOTYPES_DIR}/include
export DUNEPROTOTYPES_FQ_DIR=${DUNEPROTOTYPES_DIR}/slf7.x86_64.e26.prof
export DUNEPROTOTYPES_LIB=${DUNEPROTOTYPES_FQ_DIR}/lib

export FHICL_FILE_PATH=${DUNESW_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${DUNEPROTOTYPES_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${PROTODUNEANA_DIR}/fcl:${FHICL_FILE_PATH}
export FHICL_FILE_PATH=${INPUT_TAR_DIR_LOCAL}/myfcl:${FHICL_FILE_PATH}

echo "DUNESW_DIR: ${DUNESW_DIR}"
echo "PROTODUNEANA_DIR: ${PROTODUNEANA_DIR}"
echo "DUNEPROTOTYPES_DIR: ${DUNEPROTOTYPES_DIR}"

#echo "WIRECELL_PATH: ${WIRECELL_PATH}"
#echo "FHICL_FILE_PATH: ${FHICL_FILE_PATH}"


export OMP_NUM_THREADS=${JUSTIN_PROCESSORS} 

source "${INPUT_TAR_DIR_LOCAL}/localProducts_larsoft_v10_00_02d00_prof_e26/setup2"
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

echo " VERSION ACTIVE =  VERSION ACTIVE =  VERSION ACTIVE = "
ups active | grep "dunesw\|duneprototypes"
mrbslp

# Construct outFile from input $pfn 
now=$(date -u +"%Y-%m-%dT_%H%M%SZ")
Ffname=`echo $pfn | awk -F/ '{print $NF}'`
fname=`echo $Ffname | awk -F. '{print $1}'`
outFile=${fname}_singleHit_${now}.root

campaign="justIN.r${JUSTIN_WORKFLOW_ID}s${JUSTIN_STAGE_ID}"

(
# Do the scary preload stuff in a subshell!
#export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
#echo "$LD_PRELOAD"
lar -c $events_option ${INPUT_TAR_DIR_LOCAL}/myfcl/$FCL_FILE  -T $outFile -S file.list  > ${fname}_singleHit${now}.log 2>&1
)

# Subshell exits with exit code of last command
larExit=$?
echo "lar exit code $larExit"
echo "$pfn" > justin-processed-pfns.txt

ls -lRS

# Create compressed tar file with all log files 
tar zcf `echo "$JUSTIN_JOBSUB_ID.logs.tgz" | sed 's/@/_/g'` *.log
#exit $jobscriptExit
exit $larExit

justIN time: 2024-11-24 01:58:12 UTC       justIN version: 01.01.09
</