justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 7871, Stage 1

Priority50
Processors1
Wall seconds80000
Image/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes2097152000 (2000 MiB)
Max distance for inputs100.0
Enabled input RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Caltech, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_MIT, US_Nebraska, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scopeusertests
Events for this stage

Output patterns

 DestinationPatternLifetimeFor next stageRSE expression
1https://fndcadoor.fnal.gov:2880/dune/scratch/users/ahimmel/justin/07871/1*.*

Environment variables

NameValue
INPUT_TAR_DIR/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/184bb80486b0553a1fe3a909c5796c3d641d2d95
OUTPUT_FILENAMElarsmc_smalltest.root

File states

Total filesFindingUnallocatedAllocatedOutputtingProcessedNot foundFailed
60200103

Job states

TotalSubmittedStartedProcessingOutputtingFinishedNotusedAbortedStalledJobscript errorOutputting failedNone processed
400000102601300
Files processed000.10.10.20.20.30.30.40.40.50.50.60.60.70.70.80.80.90.911Jun-23 18:00Jun-23 19:00Jun-23 20:00Files processedBin start timesNumber per binUK_RAL-Tier1

RSEs used

NameInputsOutputs
MONTECARLO270
None05

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

SiteAllocatedOutputting
UK_RAL-Tier1140
ES_CIEMAT60
UK_QMUL30

Jobscript

#!/bin/bash
# lar_smc.jobscript  run one tiny MC job

#set -euo pipefail
echo "Node $(hostname)    start $(date)"

# --- Check that INPUT_TAR_DIR is set -----------------------------------
if [[ -z "${INPUT_TAR_DIR:-}" ]]; then
    echo "ERROR: INPUT_TAR_DIR environment variable is not set" >&2
    exit 2
fi

echo "INPUT_TAR_DIR: $INPUT_TAR_DIR"

if [[ ! -d "$INPUT_TAR_DIR" ]]; then
    echo "ERROR: INPUT_TAR_DIR is set but directory does not exist" >&2
    exit 3
fi

export UPS_OVERRIDE="-H Linux64bit+3.10-2.17"
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup -B geant4 v4_10_6_p01c -q+e20:prof
setup -B root v6_20_08a -q+e20:p383b:prof
setup -B cmake v3_19_6
export CXX=`which g++`
export CC=`which gcc`
export G4INSTALL=/cvmfs/larsoft.opensciencegrid.org/products/geant4/v4_10_6_p01/Linux64bit+2.6-2.12-e19-prof
export G4DIR=$G4INSTALL/lib64

export LARSMC_HOME="$INPUT_TAR_DIR"
export LD_LIBRARY_PATH="$LARSMC_HOME/lib:$LD_LIBRARY_PATH"

EXE="$LARSMC_HOME/TBMC"

# --- Determine which MC job this is ---------------------------------------
did_pfn_rse=`$JUSTIN_PATH/justin-get-file`
did=`echo $did_pfn_rse | cut -f1 -d' '`
pfn=`echo $did_pfn_rse | cut -f2 -d' '`
rse=`echo $did_pfn_rse | cut -f3 -d' '`

echo "did=$did"
echo "pfn=$pfn"
echo "rse=$rse"

# --- Set the GDML based on the number in PFN -----------------------------
# Check if GDML is already set
if [[ -n "${GDML:-}" ]]; then
    echo "GDML already set to: $GDML"
else
    # Extract the number from pfn (assuming pfn is just an integer)
    pfn_number=$pfn

    # Hard-coded list of GDML files in order
    declare -a gdml_files=(
        "lars_tallbo_true_h2d22_v2.gdml"
        "lars_tallbo_true_h25d34_v2.gdml"
        "lars_tallbo_true_h50d97_v2.gdml"
        "lars_tallbo_true_h76d33_v2.gdml"
        "lars_tallbo_true_h101d83_v2.gdml"
        "lars_tallbo_true_h127d22_v2.gdml"
    )

    # Check if PFN number is valid
    if [[ $pfn_number -lt 1 || $pfn_number -gt ${#gdml_files[@]} ]]; then
        echo "ERROR: PFN number $pfn_number is out of range. Must be between 1 and ${#gdml_files[@]}." >&2
        echo "Available GDML files: ${#gdml_files[@]}" >&2
        for i in "${!gdml_files[@]}"; do
            echo "  PFN $((i+1)): ${gdml_files[$i]}" >&2
        done
        exit 4
    fi

    # Select GDML file (array is 0-indexed, pfn starts at 1)
    gdml_index=$((pfn_number - 1))
    export GDML=${gdml_files[$gdml_index]}

    echo "PFN number: $pfn_number"
    echo "Selected GDML: $GDML"
fi


echo "Completed setup"

# --- Set output filename from environment variable -------------------------
if [[ -z "${OUTPUT_FILENAME:-}" ]]; then
    OUTPUT_FILENAME="larsmc_test.root"
fi

echo "Output filename: $OUTPUT_FILENAME"

# --- Check that LARSMC_HOME has TBMC executable --------------------------------
if [[ ! -x "$EXE" ]]; then
    echo "ERROR: LARSMC_HOME does not contain LArSMC executable" >&2
    echo "Expected executable at: $EXE" >&2
    echo "Current contents of LARSMC_HOME:" >&2
    ls -l $LARSMC_HOME >&2
    exit 5
fi

# --- Check that GDML file exists -----------------------------------------------
if [[ -z "${GDML:-}" ]]; then
    echo "ERROR: GDML environment variable is not set" >&2
    exit 6
fi
if [[ ! -f "$LARSMC_HOME/gdml/$GDML" ]]; then
    echo "ERROR: GDML file $GDML not found in LARSMC_HOME/gdml/" >&2
    echo "Current contents of $LARSMC_HOME/gdml:" >&2
    ls -l $LARSMC_HOME/gdml >&2
    exit 7
fi

echo "Running TBMC"
echo $EXE -g $LARSMC_HOME/gdml/$GDML -m $LARSMC_HOME/vis.mac -o $OUTPUT_FILENAME

# --- Run the actual simulation -------------------------------------------------
$EXE -g $LARSMC_HOME/gdml/$GDML -m $LARSMC_HOME/vis.mac -o $OUTPUT_FILENAME
exit_code=$?

# --- Check if simulation succeeded ---------------------------------------------
if [[ $exit_code -ne 0 ]]; then
    echo "ERROR: TBMC simulation failed with exit code $exit_code" >&2
    exit $exit_code
fi

# Check if the output file was created
if [[ ! -f "$OUTPUT_FILENAME" ]]; then
    echo "ERROR: Output file $OUTPUT_FILENAME was not created" >&2
    echo "Current directory contents:" >&2
    ls -l >&2
    exit 8
fi

# Mark the Monte Carlo counter file as successfully processed
echo "$did" >> justin-processed-dids.txt

echo "Successfully processed file: $did"
echo "Created output file: $OUTPUT_FILENAME"

echo "Node $(hostname)    end $(date)"
justIN time: 2025-08-14 16:31:50 UTC       justIN version: 01.03.02