21 July 2025: This instance at RAL is read-only. Please do not try submitting new workflows for now.

Workflow 8009, Stage 1

Priority	50
Processors	1
Wall seconds	80000
Image	/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest
RSS bytes	25165824000 (24000 MiB)
Max distance for inputs	100.0
Enabled input RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, MONTECARLO, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled output RSEs	CERN_PDUNE_EOS, DUNE_CA_SFU, DUNE_CERN_EOS, DUNE_ES_PIC, DUNE_FR_CCIN2P3_DISK, DUNE_IN_TIFR, DUNE_IT_INFN_CNAF, DUNE_UK_GLASGOW, DUNE_UK_LANCASTER_CEPH, DUNE_UK_MANCHESTER_CEPH, DUNE_US_BNL_SDCC, DUNE_US_FNAL_DISK_STAGE, FNAL_DCACHE, FNAL_DCACHE_STAGING, FNAL_DCACHE_TEST, NIKHEF, PRAGUE, QMUL, RAL-PP, RAL_ECHO, SURFSARA, T3_US_NERSC
Enabled sites	BR_CBPF, CA_Victoria, CERN, CH_UNIBE-LHEP, CZ_FZU, ES_CIEMAT, ES_PIC, FR_CCIN2P3, IN_TIFR, IT_CNAF, NL_NIKHEF, NL_SURFsara, UK_Bristol, UK_Brunel, UK_Durham, UK_Edinburgh, UK_Lancaster, UK_Manchester, UK_Oxford, UK_QMUL, UK_RAL-PPD, UK_RAL-Tier1, UK_Sheffield, US_Colorado, US_FNAL-FermiGrid, US_FNAL-T1, US_Michigan, US_NotreDame, US_PuertoRico, US_SU-ITS, US_Swan, US_UChicago, US_UConn-HPC, US_UCSD, US_Wisconsin
Scope	usertests
Events for this stage

Output patterns

	Destination	Pattern	Lifetime	For next stage	RSE expression
1	https://fndcadoor.fnal.gov:2880/dune/scratch/users/gabrielb/08009/1	cosmics_g4_*.root

Environment variables

Name	Value
FCL_FILE	standard_g4_protodunehd.fcl
INPUT_TAR_DIR_LOCAL	/cvmfs/fifeuser1.opensciencegrid.org/sw/dune/ab9ee1d42d690b109e06b1879e78d8c9a0123d9b
NUM_EVENTS	10

File states

Total files	Finding	Unallocated	Allocated	Outputting	Processed	Not found	Failed
10	0	1	0	0	9	0	0

Job states

Total	Submitted	Started	Processing	Outputting	Finished	Notused	Aborted	Stalled	Jobscript error	Outputting failed	None processed
15	0	0	0	0	10	0	1	4	0	0	0

RSEs used

Name	Inputs	Outputs
MONTECARLO	10	0
None	0	10

Stats of processed input files as CSV or JSON, and of uploaded output files as CSV or JSON (up to 10000 files included)

File reset events, by site

Site	Allocated	Outputting
UK_Manchester	1	0

Jobscript

#!/bin/bash

: <<'EOF'

To submit to tarball

1) tar cvz coisa.tar coisa_original
2) setup justin
3) justin time
4) justin get-token
5) INPUT_TAR_DIR_LOCAL=`justin-cvmfs-upload coisa.tar`---> wait a little
6) ls -l $INPUT_TAR_DIR_LOCAL

To run this script with justin:

USERF=$USER
FNALURL='https://fndcadoor.fnal.gov:2880/dune/scratch/users'

justin simple-workflow \
  --monte-carlo 10 \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_g4_*.root:${FNALURL}/${USERF}" 

or with gpu

justin simple-workflow \
  --monte-carlo 1 --gpu \
  --jobscript-git GBotogoske/my_grid/my_submit.jobscript:main \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \
  --scope usertests --lifetime-days 2 \
  --rss-mib 24000 \
  --output-pattern "cosmics_g4.root:${FNALURL}/${USERF}" 
  
To test:

justin-test-jobscript \
  --monte-carlo 1 \
  --jobscript jobscript_local.jobscript  \
  --env NUM_EVENTS=10 \
  --env FCL_FILE="standard_g4_protodunehd.fcl" \
  --env INPUT_TAR_DIR_LOCAL="$INPUT_TAR_DIR_LOCAL" \

To stop:
justin finish-workflow --workflow-id <ID_DO_WORKFLOW>


EOF

# Set defaults
FCL_FILE=$INPUT_TAR_DIR_LOCAL/gen_cosmics.fcl
FCL_FILE2=$INPUT_TAR_DIR_LOCAL/gen_cosmics_g4.fcl
DUNE_VERSION=${DUNE_VERSION:-v10_08_00d00}
DUNE_QUALIFIER=${DUNE_QUALIFIER:-e26:prof}
#INPUT_FILE=${INPUT_FILE:-$INPUT_TAR_DIR_LOCAL/cosmics_gen.root}

DID_PFN_RSE=`$JUSTIN_PATH/justin-get-file`
pfn=`echo $DID_PFN_RSE | cut -f2 -d' '`


outFile="cosmics_g4_${pfn}.root"

# Setup DUNE environment
source /cvmfs/dune.opensciencegrid.org/products/dune/setup_dune.sh
setup dunesw "$DUNE_VERSION" -q "$DUNE_QUALIFIER"

# Define number of events if set
if [ -n "$NUM_EVENTS" ]; then
  events_option="-n $NUM_EVENTS"
fi

# Run LArSoft
#lar -c prod_cosmics_protodunehd.fcl -o data_gen.root -n 10
#lar -c "$FCL_FILE" -s "$INPUT_FILE" -o data_g4.root $events_option 
(
export LD_PRELOAD=${XROOTD_LIB}/libXrdPosixPreload.so
echo "$LD_PRELOAD"

lar -c $FCL_FILE -o data_gen.root
lar -c $FCL_FILE2 -s data_gen.root -o "$outFile"
  
)

larExit=$?
echo "lar exit code $larExit"
echo "pfn $pfn"
echo "outfile $outFile"

if [ $larExit -eq 0 ]; then
  echo "$pfn" > justin-processed-pfns.txt
  jobscriptExit=0
else
  jobscriptExit=1
fi

#tar -zcf "$(echo "${JUSTIN_JOBSUB_ID}.logs.tgz" | sed 's/@/_/g')" *.log
exit $jobscriptExit