21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 8014, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	41943040000 (40000 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/gabrielb/08014/1	cosmics_detsim_*.root

Environment variables

Name	Value
FCL_FILE	standard_g4_protodunehd.fcl
INPUT_TAR_DIR_LOCAL	/cvmfs/fifeuser2.opensciencegrid.org/sw/dune/4ba9ae46eedb12aa71db8bc56526e4c1dcceea0e
NUM_EVENTS	10

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10	0	0	0	0	10	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
15	0	0	0	0	11	0	2	0	2	0	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	10	0
None	0	11

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

Jobscript

#!/bin/bash

: <<'EOF'

To submit to tarball

1) tar cvz coisa.tar coisa_original
2) setup justin
3) justin time
4) justin get-token
5) INPUT_TAR_DIR_LOCAL=`justin-cvmfs-upload coisa.tar`---> wait a little
6) ls -l $INPUT_TAR_DIR_LOCAL

To run this script with justin:

USERF=$USER
FNALURL='https://fndcadoor.fnal.gov:2880/dune/scratch/users'

justin simple-workflow \
  --monte-carlo 10 \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_detsim_*.root:${FNALURL}/${USERF}" 

or with gpu

justin simple-workflow \
  --monte-carlo 1 --gpu \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_g4.root:${FNALURL}/${USERF}" 
  
To test:

justin-test-jobscript \
  --monte-carlo 1 \
  --jobscript jobscript_local.jobscript  \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \

To stop:
justin finish-workflow --workflow-id <ID_DO_WORKFLOW>


EOF

# Set defaults
FCL_FILE_GEN=$INPUT_TAR_DIR_LOCAL/gen_cosmics.fcl
FCL_FILE2=$INPUT_TAR_DIR_LOCAL/gen_cosmics_g4.fcl
DUNE_VERSION=${DUNE_VERSION:-v10_08_00d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
#INPUT_FILE=${INPUT_FILE:-$INPUT_TAR_DIR_LOCAL/cosmics_gen.root}

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
pfn=`echo $DID_PFN_RSE | cut -f2 -d' '`

first_file="cosmics_gen_${pfn}.root"
second_file="cosmics_g4_${pfn}.root"
third_file="cosmics_detsim_${pfn}.root"


# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# Define number of events if set
if [ -n "$NUM_EVENTS" ]; then
  events_option="-n $NUM_EVENTS"
fi

# Run LArSoft
#lar -c prod_cosmics_protodunehd.fcl -o data_gen.root -n 10
#lar -c "$FCL_FILE" -s "$INPUT_FILE" -o data_g4.root $events_option 
(
export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
echo "$LD_PRELOAD"

lar -c $FCL_FILE_GEN -o "$first_file"
lar -c $FCL_FILE -s "$first_file" -o "$second_file"
lar -c standard_detsim_protodunehd.fcl -s "$second_file" -o "$third_file"
  
)

larExit=$?
echo "lar exit code $larExit"
echo "pfn $pfn"
echo "outfile $third_file"

if [ $larExit -eq 0 ]; then
  echo "$pfn" > justin-processed-pfns.txt
  jobscriptExit=0
else
  jobscriptExit=1
fi

#tar -zcf "$(echo "${JUSTIN_JOBSUB_ID}.logs.tgz" | sed 's/@/_/g')" *.log
exit $jobscriptExit