justIN           Dashboard       Workflows       Jobs       AWT       Sites       Storages       Docs       Login

Jobsub ID 265113.7@justin-prod-sched01.dune.hep.ac.uk

Jobsub ID265113.7@justin-prod-sched01.dune.hep.ac.uk
Workflow ID3493
Stage ID1
User namecalcuttj@fnal.gov
HTCondor Groupgroup_dune.prod_mcsim
RequestedProcessors1
RSS bytes4193255424 (3999 MiB)
Wall seconds limit18000 (5 hours)
Submitted time2024-09-27 16:40:37
SiteUS_BNL
EntryDUNE_US_BNL_sp02
Last heartbeat2024-09-27 16:43:52
From worker nodeHostnamespool0813.sdcc.bnl.gov
cpuinfoIntel(R) Xeon(R) Gold 6252 CPU @ 2.10GHz
OS releaseScientific Linux release 7.9 (Nitrogen)
Processors1
RSS bytes4193255424 (3999 MiB)
Wall seconds limit171000 (47 hours)
Inner Apptainer?False
Job statestalled
Allocator namejustin-allocator-pro.dune.hep.ac.uk
Started2024-09-27 16:43:18
Input files
Outputting started2024-09-27 16:43:52
Output files
Finished2024-09-27 17:36:31
List job events     Wrapper job log

Jobscript log (last 10,000 characters)

sciencegrid.org/products/g4particlexs/v2_1/G4NDL4.6
XERCES_C_VERSION=v3_2_3e
GEANT4_FQ_DIR=/cvmfs/larsoft.opensciencegrid.org/products/geant4/v4_10_6_p01g/Linux64bit+3.10-2.17-e26-prof
CETLIB_EXCEPT_DIR=/cvmfs/larsoft.opensciencegrid.org/products/cetlib_except/v1_09_01
CATCH_FQ=Linux64bit+3.10-2.17-e26
LIBXML2_VERSION=v2_9_12
MRB_FQ_DIR=/cvmfs/larsoft.opensciencegrid.org/products/mrb/v6_09_06/slf7.x86_64
SETUP_MRB=mrb v6_09_06 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products
CVMFS_larsoft_opensciencegrid_org_REVISION=900
CVMFS_singularity_opensciencegrid_org_REVISION=167434
GLIDEIN_PROVIDES=singularity/privileged
GWMS_SINGULARITY_LIB_VERSION=2
OMP_THREAD_LIMIT=1
JUSTIN_RSS_MIB=3999
VALGRIND_INC=/cvmfs/larsoft.opensciencegrid.org/products/valgrind/v3_17_0/Linux64bit+3.10-2.17/include
LARSOFTOBJ_LIB=/cvmfs/larsoft.opensciencegrid.org/products/larsoftobj/v09_35_05/slf7.x86_64.e26.prof/lib
BXDECAY0_DIR=/cvmfs/larsoft.opensciencegrid.org/products/bxdecay0/v1_1_0
DELAUNATOR_VERSION=v0_4_0
NUEVDB_DIR=/cvmfs/larsoft.opensciencegrid.org/products/nuevdb/v1_09_08
LARANA_INC=/cvmfs/larsoft.opensciencegrid.org/products/larana/v09_15_09/include
SETUP_RSTARTREE=rstartree v2016_07 -f NULL -z /cvmfs/larsoft.opensciencegrid.org/products
IFDHC_LIB=/cvmfs/larsoft.opensciencegrid.org/products/ifdhc/v2_7/Linux64bit+3.10-2.17-e26-p3915-prof/lib
GENIE=/cvmfs/larsoft.opensciencegrid.org/products/genie/v3_04_02/Linux64bit+3.10-2.17-e26-prof/GENIE-Generator
LHAPDF_VERSION=v6_5_4
LARDATA_INC=/cvmfs/larsoft.opensciencegrid.org/products/lardata/v09_16_05/include
LARDATAALG_INC=/cvmfs/larsoft.opensciencegrid.org/products/lardataalg/v09_17_05/include
SETUP_LARCOREALG=larcorealg v09_13_02 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
SETUP_G4TENDL=g4tendl v1_3_2 -f NULL -z /cvmfs/larsoft.opensciencegrid.org/products
CATCH_VERSION=v3_3_2
CLHEP_LIB=CLHEP-2.4.7.1
SETUP_POSTGRESQL=postgresql v15_3 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q p3915
SQLITE_FQ=Linux64bit+3.10-2.17
PROTOBUF_VERSION=v3_21_12a
GMP_DIR=/cvmfs/larsoft.opensciencegrid.org/products/gmp/v6_2_1
PRODUCTS=/cvmfs/dune.opensciencegrid.org/products/dune:/cvmfs/larsoft.opensciencegrid.org/products:/cvmfs/larsoft.opensciencegrid.org/packages:/cvmfs/fermilab.opensciencegrid.org/products/common/db/
GLIDEIN_LOCAL_TMP_DIR=/tmp/glide_fermipro_USgCUr
JUSTIN_PRO_Rank=isUndefined(Target.JUSTIN_PRO_Stage)?0:My.JUSTIN_PRO_Stage_Ranks[Target.JUSTIN_PRO_Stage]
LMOD_DIR=/cvmfs/connect.opensciencegrid.org/modules/packages/linux-rhel7-x86_64/gcc-4.8.5spack/lmod-7.8-gd44e35n2z5a6zbe54bpjtc55wrvjity/lmod/lmod/libexec
GENIE_XSEC_TUNE=AR23_20i_00_000
SETUP_DUNECALIB=dunecalib v09_91_02d01 -f Linux64bit+3.10-2.17 -z /cvmfs/dune.opensciencegrid.org/products/dune -q e26:prof
CRYDATAPATH=/cvmfs/larsoft.opensciencegrid.org/products/cry/v1_7q/Linux64bit+3.10-2.17-e26-prof/cry_v1.7/data
LHAPDF_LIB=/cvmfs/larsoft.opensciencegrid.org/products/lhapdf/v6_5_4/Linux64bit+3.10-2.17-e26-p3915-prof/lib
G4INCLDATA=/cvmfs/larsoft.opensciencegrid.org/products/g4incl/v1_0/G4INCL1.0
HEP_CONCURRENCY_FQ_DIR=/cvmfs/larsoft.opensciencegrid.org/products/hep_concurrency/v1_09_02/slf7.x86_64.e26.prof
CETLIB_DIR=/cvmfs/larsoft.opensciencegrid.org/products/cetlib/v3_18_02
CATCH_LIB=/cvmfs/larsoft.opensciencegrid.org/products/catch/v3_3_2/Linux64bit+3.10-2.17-e26/lib
SETUP_SCITOKENS_CPP=scitokens_cpp v1_0_1a -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:sql34001
PROTOBUF_LIB=/cvmfs/larsoft.opensciencegrid.org/products/protobuf/v3_21_12a/Linux64bit+3.10-2.17-e26/lib
ROOT_INC=/cvmfs/larsoft.opensciencegrid.org/products/root/v6_28_12/Linux64bit+3.10-2.17-e26-p3915-prof/include
HUB_DIR=/cvmfs/larsoft.opensciencegrid.org/products/hub/v2_14_2
CVMFS_annie_opensciencegrid_org_REVISION=72
_CONDOR_JOB_IWD=/srv
GEANT4REWEIGHT_DIR=/cvmfs/larsoft.opensciencegrid.org/products/geant4reweight/v01_20_08
DUNERECO_VERSION=v09_91_02d01
LARPANDORA_LIB=/cvmfs/larsoft.opensciencegrid.org/products/larpandora/v09_22_09/slf7.x86_64.e26.prof/lib
IFDH_ART_INC=/cvmfs/larsoft.opensciencegrid.org/products/ifdh_art/v2_17_05/slf7.x86_64.e26.prof/inc
MARLEY_LIB=/cvmfs/larsoft.opensciencegrid.org/products/marley/v1_2_1d/Linux64bit+3.10-2.17-e26-prof/lib
SETUP_LAREVT=larevt v09_10_05 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
HDF5_INC=/cvmfs/larsoft.opensciencegrid.org/products/hdf5/v1_12_2a/Linux64bit+3.10-2.17-e26-prof/include
SETUP_CANVAS=canvas v3_16_04 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
DUNEDETDATAFORMATS_INC=/cvmfs/dune.opensciencegrid.org/products/dune/dunedetdataformats/v4_4_1/include
SETUP_TBB=tbb v2021_9_0 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26
PYTHIA_INC=/cvmfs/larsoft.opensciencegrid.org/products/pythia/v6_4_28x/Linux64bit+3.10-2.17-e26-prof/include
SETUP_DUNESW=dunesw v09_91_02d01 -f Linux64bit+3.10-2.17 -z /cvmfs/dune.opensciencegrid.org/products/dune -q e26:prof
PERL_MM_OPT=INSTALL_BASE=/cvmfs/larsoft.opensciencegrid.org/products/mrb/v6_09_06/slf7.x86_64/CPAN
GIT_EXEC_PATH=/cvmfs/larsoft.opensciencegrid.org/products/git/v2_45_1/Linux64bit+3.10-2.17/libexec/git-core
GWMS_SINGULARITY_IMAGE_RESTRICTIONS=cvmfs
GWMS_SINGULARITY_OUTSIDE_PWD=/home/condor/local/sdcc/execute/dir_148549/glide_hPussk/execute/dir_83044
TF_LOOP_PARALLEL_ITERATIONS=1
DK_NODE=/cvmfs/connect.opensciencegrid.org/modules/spack/share/spack/dotkit/linux-rhel7-x86_64
PROTODUNEANA_INC=/cvmfs/dune.opensciencegrid.org/products/dune/protoduneana/v09_91_02d01/include
DUNEDATAPREP_LIB=/cvmfs/dune.opensciencegrid.org/products/dune/dunedataprep/v09_91_02d01/slf7.x86_64.e26.prof/lib
DUNERECO_DIR=/cvmfs/dune.opensciencegrid.org/products/dune/dunereco/v09_91_02d01
WIRECELL_LIB=/cvmfs/larsoft.opensciencegrid.org/products/wirecell/v0_27_1/Linux64bit+3.10-2.17-e26-prof/lib64
TRITON_INC=/cvmfs/larsoft.opensciencegrid.org/products/triton/v2_25_0d/Linux64bit+3.10-2.17-e26/include
LIBJPEG_TURBO_INC=/cvmfs/larsoft.opensciencegrid.org/products/libjpeg_turbo/v2_1_0/Linux64bit+3.10-2.17/include
LAREVENTDISPLAY_LIB=/cvmfs/larsoft.opensciencegrid.org/products/lareventdisplay/v09_11_09/slf7.x86_64.e26.prof/lib
PPFX_VERSION=v02_20_02
NUG4_VERSION=v1_16_06
SETUP_LARDATAALG=lardataalg v09_17_05 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
LARCOREALG_INC=/cvmfs/larsoft.opensciencegrid.org/products/larcorealg/v09_13_02/include
G4PARTICLEHPDATA=/cvmfs/larsoft.opensciencegrid.org/products/g4tendl/v1_3_2/G4TENDL1.3.2
SETUP_LARANA=larana v09_15_09 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
SETUP_LARSOFT=larsoft v09_91_02 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
SETUP_CETLIB=cetlib v3_18_02 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
SETUP_FHICLCPP=fhiclcpp v4_18_04 -f Linux64bit+3.10-2.17 -z /cvmfs/larsoft.opensciencegrid.org/products -q e26:prof
OPENBLAS_INC=/cvmfs/larsoft.opensciencegrid.org/products/openblas/v0_3_23/Linux64bit+3.10-2.17-e26/include
PYTHON_INCLUDE=/cvmfs/larsoft.opensciencegrid.org/products/python/v3_9_15/Linux64bit+3.10-2.17/include/python3.9
CONDOR_PROCD_ADDRESS=/home/condor/local/sdcc/execute/dir_148549/glide_hPussk/log/procd_address
BASH_FUNC_module()=() {  eval $($LMOD_CMD bash "$@") && eval $(${LMOD_SETTARG_CMD:-:} -s sh)
}
BASH_FUNC_spack()=() {  if [ -n "${ZSH_VERSION:-}" ]; then
 emulate -L sh;
 fi;
 args=("$@");
 _sp_flags="";
 while [[ "$1" =~ ^- ]]; do
 _sp_flags="$_sp_flags $1";
 shift;
 done;
 if [[ ( ! -z "$_sp_flags" ) && ( "$_sp_flags" =~ '.*h.*' || "$_sp_flags" =~ '.*V.*' ) ]]; then
 command spack $_sp_flags "$@";
 return;
 fi;
 _sp_subcommand="";
 if [ -n "$1" ]; then
 _sp_subcommand="$1";
 shift;
 fi;
 _sp_spec=("$@");
 case $_sp_subcommand in 
 "cd")
 _sp_arg="";
 if [ -n "$1" ]; then
 _sp_arg="$1";
 shift;
 fi;
 if [ "$_sp_arg" = "-h" ]; then
 command spack cd -h;
 else
 LOC="$(spack location $_sp_arg "$@")";
 if [[ -d "$LOC" ]]; then
 cd "$LOC";
 else
 return 1;
 fi;
 fi;
 return
 ;;
 "env")
 _sp_arg="";
 if [ -n "$1" ]; then
 _sp_arg="$1";
 shift;
 fi;
 if [ "$_sp_arg" = "-h" ]; then
 command spack env -h;
 else
 case $_sp_arg in 
 activate)
 _a="$@";
 if [ -z "$1" -o "${_a#*--sh}" != "$_a" -o "${_a#*--csh}" != "$_a" -o "${_a#*-h}" != "$_a" ]; then
 command spack "${args[@]}";
 else
 eval $(command spack $_sp_flags env activate --sh "$@");
 fi
 ;;
 deactivate)
 if [ -n "$1" ]; then
 command spack "${args[@]}";
 else
 eval $(command spack $_sp_flags env deactivate --sh);
 fi
 ;;
 *)
 command spack "${args[@]}"
 ;;
 esac;
 fi;
 return
 ;;
 "use" | "unuse" | "load" | "unload")
 _sp_subcommand_args="";
 _sp_module_args="";
 while [[ "$1" =~ ^- ]]; do
 if [ "$1" = "-r" -o "$1" = "--dependencies" ]; then
 _sp_subcommand_args="$_sp_subcommand_args $1";
 else
 _sp_module_args="$_sp_module_args $1";
 fi;
 shift;
 done;
 _sp_spec=("$@");
 case $_sp_subcommand in 
 "use")
 if _sp_full_spec=$(command spack $_sp_flags module dotkit find $_sp_subcommand_args "${_sp_spec[@]}"); then
 use $_sp_module_args $_sp_full_spec;
 else
 $(exit 1);
 fi
 ;;
 "unuse")
 if _sp_full_spec=$(command spack $_sp_flags module dotkit find $_sp_subcommand_args "${_sp_spec[@]}"); then
 unuse $_sp_module_args $_sp_full_spec;
 else
 $(exit 1);
 fi
 ;;
 "load")
 if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "${_sp_spec[@]}"); then
 module load $_sp_module_args $_sp_full_spec;
 else
 $(exit 1);
 fi
 ;;
 "unload")
 if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "${_sp_spec[@]}"); then
 module unload $_sp_module_args $_sp_full_spec;
 else
 $(exit 1);
 fi
 ;;
 esac
 ;;
 *)
 command spack "${args[@]}"
 ;;
 esac
}
BASH_FUNC_ml()=() {  eval $($LMOD_DIR/ml_cmd "$@")
}
_=/usr/bin/env
Will use justin-get-file
No files available from this stage
Could not get file
justIN time: 2024-11-17 04:03:45 UTC       justIN version: 01.01.09