From 70ddfa1a77879279f2397a8218249498dd6cfbee Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 07:56:18 -0700 Subject: [PATCH 01/25] use absolute URL for submodules to allow mirroring --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 71c162e1..21228d09 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "cmake/blt"] path = cmake/blt - url = ../../LLNL/blt.git + url = https://github.com/LLNL/blt.git + From bd96ccfed3aee1792c7c0951cf77fd0e0d9cb87c Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 08:44:23 -0700 Subject: [PATCH 02/25] adding gitlab ci yaml to automate testing the current geos develop head against tpl branches, and automate deployment --- .gitlab-ci.yml | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..20824f1a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,78 @@ +timeout: 4h + +stages: + - test + - deploy + +variables: + GIT_SUBMODULE_STRATEGY: recursive + GEOS_DIR: "geos" + GEOS_BRANCH: "feature/wrtobin/tpl-gitlab-ci-cd" # Default branch, can be overridden when CI is run manually + GEOS_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/llnl-tpls-base.cmake + INSTALL_SYSTEM: "false" + +test_lc_tpls: + stage: test + variables: + - INSTALL_DIR=builds + script: + - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} + - cd ${GEOS_DIR} + - git submodule init src/cmake/blt + - git submodule init src/coreComponents/LvArray + - git submodule init src/coreComponents/constitutive/PVTPackage + - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface + - git submodule update + - cd .. + - mkdir -p ${INSTALL_DIR} + - ./scripts/setupLC-TPL.bash ${GEOS_DIR} ${INSTALL_DIR} + - echo "set(GEOS_TPL_ROOT_DIR ${TPL_ROOT_DIR} CACHE PATH \"\")" > ${GEOS_TPL_BASE} + - echo "set(GEOS_TPL_DIR \$\{GEOS_TPL_ROOT_DIR\}/${TPL_DIR}/install-\$\{CONFIG_NAME\}-release CACHE PATH \"\")" >> ${GEOS_TPL_BASE} + - cd ${GEOS_DIR} + - scripts/setupLC.bash + tags: + - quartz + - shell + when: manual + +deploy_lc_tpls: + stage: deploy + dependencies: + - test_lc_tpls + variables: + - XSU: xsu mdgadmin + - TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs + - DATE: $(date +'%Y-%m-%d') + - INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE} + - BRANCH: feature/update-tpls-${DATE} + script: + - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} + - cd ${GEOS_DIR} + - git submodule init src/cmake/blt + - git submodule init src/coreComponents/LvArray + - git submodule init src/coreComponents/constitutive/PVTPackage + - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface + - git submodule update + - cd .. + - ${XSU} mkdir -p ${INSTALL_DIR} + - ${XSU} ./scripts/setupLC-TPL.bash ${GEOS_DIR} ${INSTALL_DIR} + - echo "set(GEOS_TPL_ROOT_DIR ${TPL_ROOT_DIR} CACHE PATH \"\")" > ${GEOS_TPL_BASE} + - echo "set(GEOS_TPL_DIR \$\{GEOS_TPL_ROOT_DIR\}/${TPL_DIR}/install-\$\{CONFIG_NAME\}-release CACHE PATH \"\")" >> ${GEOS_TPL_BASE} + - cd ${GEOS_DIR} + - scripts/setupLC.bash + - cd ${GEOS_DIR}/src/coreComponents/LvArray + - git checkout -b ${BRANCH} + - git add -A + - git commit -m "Update lc tpl install location." + - git push origin ${BRANCH} + - cd ${GEOS_DIR} + - git checkout -b ${BRANCH} + - git add -A + - git commit -m "Update lc tpl install location." + - git push origin ${BRANCH} + tags: + - quartz + - shell + when: manual + rules: + - if: '$CI_COMMIT_BRANCH == "master" && $INSTALL_SYSTEM == "true"' From 656ea31d09ada33e54872fe17b4f6d7bc9dd24ab Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 08:46:42 -0700 Subject: [PATCH 03/25] wrong service user --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 20824f1a..3c63c6ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,7 +40,7 @@ deploy_lc_tpls: dependencies: - test_lc_tpls variables: - - XSU: xsu mdgadmin + - XSU: xsu geosadmn - TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs - DATE: $(date +'%Y-%m-%d') - INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE} From 7e0eb24e858047626d73cc67109d5683dcc404d2 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 09:15:40 -0700 Subject: [PATCH 04/25] adding WIP spack environments, tioga only known working for base AMD build --- .../lassen-blueos_4_ppc64le_ib_p9/spack.yaml | 165 +++ spack/environments/lassen-toss_4_ppc64le_ib | 1 + .../quartz-toss_4_x86_64_ib/spack.yaml | 260 ++++ .../tioga-toss_4_x86_64_ib_cray/spack.yaml | 1088 +++++++++++++++++ 4 files changed, 1514 insertions(+) create mode 100644 spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml create mode 120000 spack/environments/lassen-toss_4_ppc64le_ib create mode 100644 spack/environments/quartz-toss_4_x86_64_ib/spack.yaml create mode 100644 spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml diff --git a/spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml b/spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml new file mode 100644 index 00000000..6162eb8a --- /dev/null +++ b/spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml @@ -0,0 +1,165 @@ +spack: + config: + build_stage: + - $spack/var/spack/spack-stage/build-$arch/ + install_missing_compilers: false + misc_cache: $spack/.cache + install_tree: + root: $spack/opt/spack/ + #root: /usr/WS1/GEOS/GEOSX/TPLs_$date/ + projections: + all: install-{arch}-{compiler.name}-{compiler.version}/{name}-{version} + compilers: + - compiler: + spec: clang@14.0.6 + paths: + cc: /usr/tce/packages/clang/clang-14.0.6/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6/bin/clang++ + f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + flags: {} + operating_system: rhel8 + target: ppc64le + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: gcc@11.2 + paths: + cc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-11.2.1/bin/g++ + f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran + fc: /usr/tce/packages/gcc/ggc-11.2.1/bin/gfortran + flags: {} + operating_system: rhel8 + target: ppc64le + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: xl@2023.06.28 + paths: + cc: /usr/tce/packages/xl/xl-2023.06.28/bin/xlc + cxx: /usr/tce/packages/xl/xl-2023.06.28/bin/xlc++ + f77: /usr/tce/packages/xl/xl-2023.06.28/bin/xlf + fc: /usr/tce/packages/xl/ggc-2023.06.28/bin/xlf90 + flags: {} + operating_system: rhel8 + target: ppc64le + modules: [] + environment: {} + extra_rpaths: [] + packages: + all: + target: [ppc64le] + compiler: [gcc, clang, xl] + providers: + blas: [essl] + lapack: [essl] + mpi: [spectrum-mpi] + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + buildable: false + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + buildable: false + cuda: + buildable: false + version: [11.8.0] + externals: + - spec: cuda@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ + cub: + buildable: false + version: [11.8.0] + externals: + - spec: cub@11.8.0 + prefix: /usr/tce/packages/cuda/cuda-11.8.0/ + mpi: + buildable: false + spectrum-mpi: + externals: + - spec: spectrum-mpi@rolling-release + modules: [ spectrum-mpi/rolling-release ] + essl: + buildable: false + externals: + - spec: essl@6.3.0.2 +lapackforessl + modules: [ essl/6.3.0.2 ] + lapack: + buildable: false + zlib: + buildable: false + version: [1.2.11] + externals: + - spec: zlib@1.2.11 + prefix: / + autoconf-archive: + buildable: false + externals: + - spec: autoconf-archive@6.6.3 + prefix: /usr + # Build tools + cmake: + buildable: false + version: [3.23.1] + externals: + - spec: cmake@3.23.1 + prefix: /usr/tce/packages/cmake/cmake-3.23.1 + pkgconf: + buildable: false + version: [1.6.3] + externals: + - spec: pkgconf@1.6.3 + prefix: /usr/tce/packages/python/python-3.8.2 + gmake: + buildable: false + version: [4.2.1] + externals: + - spec: gmake@4.2.1 + prefix: /usr/tcetmp/ + gmp: + buildable: false + version: [10.3.2] + externals: + - spec: gmp@10.3.2 + prefix: / + libtool: + buildable: false + version: [2.4.6] + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + buildable: false + version: [1.4.18] + externals: + - spec: m4@1.4.18 + prefix: /usr + python: + buildable: false + version: [3.8.2] + externals: + - spec: python@3.8.2 + prefix: /usr/tce/packages/python/python-3.8.2 + specs: + - conduit%clang@14.0.6 ~fortran ~hdf5_compat ^hdf5+hl+mpi~fortran ^spectrum-mpi@rolling-release + - raja@develop%clang@14.0.6 +cuda cuda_arch=70 ^cuda@11.8.0 ^cmake@3.23.1 + - chai@develop%clang@14.0.6~examples~openmp+raja+cuda cuda_arch=70 ^raja@develop+cuda + cuda_arch=70 ^umpire@develop~examples+cuda cuda_arch=70 ^cuda@11.8.0 ^blt@0.5.2 ^camp+cuda cuda_arch=70 + ^cmake@3.23.1 ^cub@11.8.0 + - hypre@develop%clang@14.0.6~int64+cuda+unified-memory cuda_arch=70 ^cuda@11.8.0 + ^spectrum-mpi@rolling-release ^essl@6.3.0.2 + - pugixml%clang@14.0.6 + - fmt%clang@14.0.6 + - suite-sparse@5.10.1%clang@14.0.6~cuda~graphblas~openmp+pic~tbb + - caliper@2.8.0%clang@14.0.6~adiak~libdw~papi+cuda cuda_arch=70 ^cuda@11.8.0 ^essl@6.3.0.2 \ No newline at end of file diff --git a/spack/environments/lassen-toss_4_ppc64le_ib b/spack/environments/lassen-toss_4_ppc64le_ib new file mode 120000 index 00000000..e93d0a51 --- /dev/null +++ b/spack/environments/lassen-toss_4_ppc64le_ib @@ -0,0 +1 @@ +lassen-blueos_4_ppc64le_ib_p9 \ No newline at end of file diff --git a/spack/environments/quartz-toss_4_x86_64_ib/spack.yaml b/spack/environments/quartz-toss_4_x86_64_ib/spack.yaml new file mode 100644 index 00000000..d872120f --- /dev/null +++ b/spack/environments/quartz-toss_4_x86_64_ib/spack.yaml @@ -0,0 +1,260 @@ +spack: + config: + build_stage: + - $spack/var/spack/spack-stage/build-$arch/ + install_missing_compilers: false + misc_cache: $spack/.cache + install_tree: + root: $spack/opt/spack/ + #root: /usr/WS1/GEOS/GEOSX/TPLs_$date/ + projections: + all: install-{arch}-{compiler.name}-{compiler.version}/{name}-{version} + compilers: + - compiler: + spec: clang@=14.0.6 + paths: + cc: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang + cxx: /usr/tce/packages/clang/clang-14.0.6-magic/bin/clang++ + f77: /usr/tce/packages/clang/clang-14.0.6-magic/bin/flang + fc: /usr/tce/packages/clang/clang-14.0.6-magic/bin/flang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: gcc@=12.1.1 + paths: + cc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/g++ + f77: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-12.1.1-magic/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: intel@=2021.6.0 + paths: + cc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icc + cxx: /usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/icpc + f77: /usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/ifort + fc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0-magic/bin/ifort + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + packages: + all: + target: [x86_64] + compiler: [gcc, clang, intel] + providers: + mpi: [mvapich2] + blas: [intel-oneapi-mkl] + lapack: [intel-oneapi-mkl] + blas: + buildable: false + lapack: + buildable: false + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + buildable: false + zlib: + buildable: false + version: [1.2.11] + externals: + - spec: zlib@1.2.11 + prefix: / + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + buildable: false + git: + externals: + - spec: git@2.31.1+tcltk + prefix: /usr + - spec: git@2.29.1+tcltk + prefix: /usr/tce + buildable: false + doxygen: + externals: + - spec: doxygen@1.8.14+graphviz~mscgen + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + buildable: false + swig: + externals: + - spec: swig@3.0.12 + prefix: /usr + buildable: false + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + buildable: false + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + buildable: false + libunwind: + externals: + - spec: libunwind@1.3.1 + prefix: / + buildable: false + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + buildable: false + mpi: + buildable: false + mvapich2: + externals: + - spec: mvapich2@2.3.7 %clang@=14.0.6 + modules: [ clang/14.0.6-magic, mvapich2/2.3.7 ] + - spec: mvapich2@2.3.7 %gcc@=12.1.1 + modules: [ gcc/12.1.1-magic, mvapich2/2.3.7 ] + buildable: false + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + buildable: false + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + buildable: false + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + buildable: false + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + - spec: pkgconf@1.8.0 + prefix: /usr/tce/packages/python/python-3.9.12 + buildable: false + git-lfs: + externals: + - spec: git-lfs@2.11.0 + prefix: /usr/tce + buildable: false + ninja: + externals: + - spec: ninja@1.10.2 + prefix: /usr/tce/packages/python/python-3.9.12 + buildable: false + meson: + externals: + - spec: meson@0.61.2 + prefix: /usr/tce/packages/python/python-3.9.12 + buildable: false + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + buildable: false + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + buildable: false + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + buildable: false + ccache: + externals: + - spec: ccache@3.7.7 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.20.2 + prefix: /usr + - spec: cmake@3.23.1 + prefix: /usr/tce + buildable: false + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + buildable: false + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + buildable: false + curl: + externals: + - spec: curl@7.61.1+gssapi+ldap+nghttp2 + prefix: /usr + buildable: false + binutils: + externals: + - spec: binutils@2.30.117 + prefix: /usr + buildable: false + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.26.3 + prefix: /usr + buildable: false + python: + externals: + - spec: python@3.9.12 + prefix: /usr/tce/packages/python/python-3.9.12/ + - spec: python@3.10.8 + prefix: /usr/tce/packages/python/python-3.10.8/ + buildable: false + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 + buildable: false + specs: + - conduit%clang@14.0.6 ~fortran ~hdf5_compat ^hdf5+hl+mpi~fortran ^mvapich2 + - raja@develop%clang@14.0.6 ^cmake@3.23.1 + - chai@develop%clang@14.0.6~examples+openmp+raja ^raja@develop+openmp ^umpire@develop~examples^blt@0.5.2 ^camp+openmp ^cmake@3.23.1 + - hypre@develop%clang@14.0.6~int64+openmp ^intel-oneapi-mkl mpi_family=mpich + - pugixml%clang@14.0.6 + - fmt%clang@14.0.6 + - suite-sparse@5.10.1%clang@14.0.6~graphblas~openmp+pic~tbb ^intel-oneapi-mkl +cluster mpi_family=mpich threads=none + - caliper@2.8.0%clang@14.0.6~adiak~libdw~papi \ No newline at end of file diff --git a/spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml new file mode 100644 index 00000000..2a173553 --- /dev/null +++ b/spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml @@ -0,0 +1,1088 @@ +spack: + config: + build_stage: + - $spack/var/spack/spack-stage/build-$arch-$date/ + install_missing_compilers: false + misc_cache: $spack/.cache + install_tree: + root: $spack/opt/spack/ + #root: /usr/WS1/GEOS/GEOSX/TPLs_$date/ + projections: + all: install-{os}-{arch}-{compiler.name}-{compiler.version}/{name}-{version} + compilers: + - compiler: + spec: gcc@8.3.1 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: gcc@10.2.1 + paths: + cc: /opt/rh/gcc-toolset-10/root/bin/gcc + cxx: /opt/rh/gcc-toolset-10/root/bin/g++ + f77: /opt/rh/gcc-toolset-10/root/bin/gfortran + fc: /opt/rh/gcc-toolset-10/root/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: cce@16.0.0 + paths: + cc: /opt/cray/pe/cce/16.0.0/bin/craycc + cxx: /opt/cray/pe/cce/16.0.0/bin/crayCC + f77: /opt/cray/pe/cce/16.0.0/bin/crayftn + fc: /opt/cray/pe/cce/16.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.6.0 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/16.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@15.0.0a + paths: + cc: /opt/cray/pe/cce/15.0.0/bin/craycc + cxx: /opt/cray/pe/cce/15.0.0/bin/crayCC + f77: /opt/cray/pe/cce/15.0.0/bin/crayftn + fc: /opt/cray/pe/cce/15.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.3.3 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/15.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@15.0.0c + paths: + cc: /opt/cray/pe/cce/15.0.0/bin/craycc + cxx: /opt/cray/pe/cce/15.0.0/bin/crayCC + f77: /opt/cray/pe/cce/15.0.0/bin/crayftn + fc: /opt/cray/pe/cce/15.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.4.1 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/15.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@15.0.0b + paths: + cc: /opt/cray/pe/cce/15.0.0/bin/craycc + cxx: /opt/cray/pe/cce/15.0.0/bin/crayCC + f77: /opt/cray/pe/cce/15.0.0/bin/crayftn + fc: /opt/cray/pe/cce/15.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.4.0 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/15.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@15.0.0 + paths: + cc: /opt/cray/pe/cce/15.0.0/bin/craycc + cxx: /opt/cray/pe/cce/15.0.0/bin/crayCC + f77: /opt/cray/pe/cce/15.0.0/bin/crayftn + fc: /opt/cray/pe/cce/15.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.3.0 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/15.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@14.0.4 + paths: + cc: /opt/cray/pe/cce/14.0.4/bin/craycc + cxx: /opt/cray/pe/cce/14.0.4/bin/crayCC + f77: /opt/cray/pe/cce/14.0.4/bin/crayftn + fc: /opt/cray/pe/cce/14.0.4/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.2.3 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/14.0.4/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@14.0.3 + paths: + cc: /opt/cray/pe/cce/14.0.3/bin/craycc + cxx: /opt/cray/pe/cce/14.0.3/bin/crayCC + f77: /opt/cray/pe/cce/14.0.3/bin/crayftn + fc: /opt/cray/pe/cce/14.0.3/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.2.3 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/14.0.3/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@14.0.2 + paths: + cc: /opt/cray/pe/cce/14.0.2/bin/craycc + cxx: /opt/cray/pe/cce/14.0.2/bin/crayCC + f77: /opt/cray/pe/cce/14.0.2/bin/crayftn + fc: /opt/cray/pe/cce/14.0.2/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.2.1 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/14.0.2/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@14.0.1 + paths: + cc: /opt/cray/pe/cce/14.0.1/bin/craycc + cxx: /opt/cray/pe/cce/14.0.1/bin/crayCC + f77: /opt/cray/pe/cce/14.0.1/bin/crayftn + fc: /opt/cray/pe/cce/14.0.1/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.1.1 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/14.0.1/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@14.0.0 + paths: + cc: /opt/cray/pe/cce/14.0.0/bin/craycc + cxx: /opt/cray/pe/cce/14.0.0/bin/crayCC + f77: /opt/cray/pe/cce/14.0.0/bin/crayftn + fc: /opt/cray/pe/cce/14.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-5.1.1 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/14.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@13.0.2 + paths: + cc: /opt/cray/pe/cce/13.0.2/bin/craycc + cxx: /opt/cray/pe/cce/13.0.2/bin/crayCC + f77: /opt/cray/pe/cce/13.0.2/bin/crayftn + fc: /opt/cray/pe/cce/13.0.2/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-4.5.2 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/13.0.2/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@13.0.1 + paths: + cc: /opt/cray/pe/cce/13.0.1/bin/craycc + cxx: /opt/cray/pe/cce/13.0.1/bin/crayCC + f77: /opt/cray/pe/cce/13.0.1/bin/crayftn + fc: /opt/cray/pe/cce/13.0.1/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-4.5.2 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/13.0.1/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@13.0.0 + paths: + cc: /opt/cray/pe/cce/13.0.0/bin/craycc + cxx: /opt/cray/pe/cce/13.0.0/bin/crayCC + f77: /opt/cray/pe/cce/13.0.0/bin/crayftn + fc: /opt/cray/pe/cce/13.0.0/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-4.3.1 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/13.0.0/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@12.0.3 + paths: + cc: /opt/cray/pe/cce/12.0.3/bin/craycc + cxx: /opt/cray/pe/cce/12.0.3/bin/crayCC + f77: /opt/cray/pe/cce/12.0.3/bin/crayftn + fc: /opt/cray/pe/cce/12.0.3/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-4.2.0 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/12.0.3/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: cce@12.0.2 + paths: + cc: /opt/cray/pe/cce/12.0.2/bin/craycc + cxx: /opt/cray/pe/cce/12.0.2/bin/crayCC + f77: /opt/cray/pe/cce/12.0.2/bin/crayftn + fc: /opt/cray/pe/cce/12.0.2/bin/crayftn + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + set: + TCE_ROCM_PATH: /opt/rocm-4.2.0 + CRAY_PE_USE_CLANG: /opt/cray/pe/cce/12.0.2/cce-clang/x86_64/bin/clang + extra_rpaths: [] + - compiler: + spec: rocmcc@4.2.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-4.2.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-4.2.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-4.2.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-4.2.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@4.3.1 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-4.3.1/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-4.3.1/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-4.3.1/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-4.3.1/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@4.5.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@4.5.2 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.2/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.2/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.2/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-4.5.2/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.0.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.0.2 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.2/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.2/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.2/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.0.2/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.6.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.6.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.6.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.6.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.6.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.1.1 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.1.1/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.1.1/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.1.1/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.1.1/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.2.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.2.1 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.1/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.1/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.1/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.1/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.2.3 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.3/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.3/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.3/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.2.3/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.3.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.3.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.3.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.3.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.3.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.4.0 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.0/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.0/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.0/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.0/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: + - rocmcc/5.4.0 + environment: {} + extra_rpaths: [] + - compiler: + spec: rocmcc@5.4.1 + paths: + cc: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.1/bin/amdclang + cxx: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.1/bin/amdclang++ + f77: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.1/bin/amdflang + fc: /usr/tce/packages/rocmcc-tce/rocmcc-5.4.1/bin/amdflang + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + packages: + all: + compiler: [gcc@8.3.1] + target: [x86_64] + providers: + iconv: [libc] + cce: + externals: + - spec: cce@16.0.0 + preifx: /opt/cray/pe/cce/16.0.0 + - spec: cce@15.0.0c + prefix: /opt/cray/pe/cce/15.0.0 + - spec: cce@15.0.0b + prefix: /opt/cray/pe/cce/15.0.0 + - spec: cce@15.0.0a + prefix: /opt/cray/pe/cce/15.0.0 + - spec: cce@15.0.0 + prefix: /opt/cray/pe/cce/15.0.0 + - spec: cce@14.0.4 + prefix: /opt/cray/pe/cce/14.0.4 + - spec: cce@14.0.3 + prefix: /opt/cray/pe/cce/14.0.3 + - spec: cce@14.0.2 + prefix: /opt/cray/pe/cce/14.0.2 + - spec: cce@14.0.1 + prefix: /opt/cray/pe/cce/14.0.1 + - spec: cce@14.0.0 + prefix: /opt/cray/pe/cce/14.0.0 + - spec: cce@13.0.2 + prefix: /opt/cray/pe/cce/13.0.2 + - spec: cce@13.0.1 + prefix: /opt/cray/pe/cce/13.0.1 + - spec: cce@13.0.0 + prefix: /opt/cray/pe/cce/13.0.0 + - spec: cce@12.0.3 + prefix: /opt/cray/pe/cce/12.0.3 + - spec: cce@12.0.2 + prefix: /opt/cray/pe/cce/12.0.2 + buildable: false + cray-mpich: + version: [8.1.12, 8.1.16, 8.1.18, 8.1.21, 8.1.26] + externals: + - spec: cray-mpich@8.1.12 + modules: + - cray-mpich/8.1.12 + - spec: cray-mpich@8.1.16 + modules: + - cray-mpich/8.1.16 + - spec: cray-mpich@8.1.18 + modules: + - cray-mpich/8.1.18 + - spec: cray-mpich@8.1.21 + modules: + - cray-mpich/8.1.21 + - spec: cray-mpich@8.1.26 + modules: + - cray-mpich/8.1.26 + target: [] + compiler: [] + buildable: false + providers: {} + gcc: + externals: + - spec: gcc@10.2.1%gcc@8.3.1 + prefix: /opt/rh/gcc-toolset-10/root + version: [10.2.1] + libc: #libc only has dummy 1.0 versions in spack + externals: + - spec: libc@1.0+iconv + prefix: / + variants: +iconv + version: [1.0] + ncurses: + externals: + - spec: ncurses@6.1~symlinks+termlib + prefix: /usr + version: [6.1] + variants: ~symlinks+termlib + pcre2: + externals: + - spec: pcre2@10.32 + prefix: /usr + version: [10.32] + perl: + externals: + - spec: perl@5.26.3+cpanm+shared+threads + prefix: /usr + variants: +cpanm+shared+threads + version: [5.26.3] + libidn2: + externals: + - spec: libidn2@2.2.0 + prefix: /usr + version: [2.2.0] + expat: + externals: + - spec: expat@2.2.5~libbsd + prefix: /usr + version: [2.2.5] + variants: ~libbsd + autoconf: + buildable: false + version: [2.69] + externals: + - spec: autoconf@2.69 + prefix: /usr + autoconf-archive: + buildable: false + version: [2022.02.01] + externals: + - spec: autoconf-archive@2022.02.01 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + version: [1.16.1] + m4: + buildable: false + version: [1.4.18] + externals: + - spec: m4@1.4.18 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + version: [2.4.6] + gettext: + externals: + - spec: gettext@0.19.8.1+curses+libunistring+libxml2~git+xz~tar~bzip2 + prefix: /usr + version: [0.19.8.1] + variants: +curses+libunistring+libxml2~git+xz~tar~bzip2 + tk: + externals: + - spec: tk@8.6+xft~xss + prefix: /usr/share/tk8.6/ + version: [8.6] + variants: +xft~xss + openssl: + buildable: false + version: [1.1.1] + target: [] + providers: {} + externals: + - spec: openssl@1.1.1 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + buildable: false + curl: + externals: + - spec: curl@7.61.1 ~libssh+libssh2+gssapi+nghttp2 + prefix: /usr + buildable: false + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + buildable: false + libxml2: + externals: + - spec: libxml2@2.9.7 + prefix: /usr + buildable: false + libpciaccess: + externals: + - spec: libpciaccess@0.11.1 + prefix: /usr + buildable: false + slurm: + externals: + - spec: slurm@20.11.5 + prefix: /usr + buildable: false + libfabric: + version: [1.7.2, 2.0] + buildable: false + externals: + - spec: libfabric@1.7.2 + modules: + - libfabric/1.7.2-llnl + - spec: libfabric@2.0 + modules: + - libfabric/2.0 + ucx: + externals: + - spec: ucx@1.9.0 + prefix: /usr + buildable: false + hwloc: + externals: + - spec: hwloc@2.4.1 + prefix: /usr + buildable: false + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + buildable: false + libedit: + externals: + - spec: libedit@3.1 + prefix: /usr + buildable: false + krb5: + externals: + - spec: krb5@1.18.2 + prefix: /usr + buildable: false + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + buildable: false + munge: + externals: + - spec: munge@0.5.3 + prefix: /usr + buildable: false + go: + externals: + - spec: go@1.15.13 + prefix: /usr + buildable: false + python: + buildable: false + version: [3.9.12] + externals: + - spec: python@3.9.12 + modules: + - python/3.9.12 + readline: + buildable: false + version: [7.0] + externals: + - spec: readline@7.0 + prefix: /usr/lib64 + gdbm: + buildable: false + version: [1.18.1] + externals: + - spec: gdbm@1.18.1 + prefix: /usr/lib64 + autoreconf: + buildable: false + version: [2.69] + externals: + - spec: autoreconf@2.69 + prefix: /usr + cmake: + version: [3.14.5, 3.19.2, 3.21.1, 3.22.2, 3.23.1, 3.24.2] + externals: + - spec: cmake@3.14.5 + modules: + - cmake/3.14.5 + - spec: cmake@3.19.2 + modules: + - cmake/3.19.2 + - spec: cmake@3.21.1 + modules: + - cmake/3.21.1 + - spec: cmake@3.22.2 + modules: + - cmake/3.22.2 + - spec: cmake@3.23.1 + modules: + - cmake/3.23.1 + - spec: cmake@3.24.2 + modules: + - cmake/3.24.2 + openmpi: + buildable: false + mpich: + buildable: false + cray-libsci: + buildable: false + version: [21.08.1.2, 22.08.1.1, 23.05.1.4] + externals: + - spec: cray-libsci@21.08.1.2 + modules: + - cray-libsci/21.08.1.2 + - cray-mpich/8.1.16 + - spec: cray-libsci@22.08.1.1 + modules: + - cray-libsci/22.08.1.1 + - cray-mpich/8.1.18 + - spec: cray-libsci@23.05.1.4 + modules: + - cray-libsci/23.05.1.4 + - cray-mpich/8.1.26 + tcsh: + buildable: false + version: [6.20.00] + externals: + - spec: tcsh@6.20.00 + prefix: /usr + hip: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hip@5.4.3 + prefix: /opt/rocm-5.4.3/hip + extra_attributes: + compilers: + hip: /opt/rocm-5.4.3/hip/bin/hipcc + - spec: hip@5.6.0 + prefix: /opt/rocm-5.6.0/ + extra_attributes: + compilers: + hip: /opt/rocm-5.6.0/bin/hipcc + - spec: hip@5.4.0 + prefix: /opt/rocm-5.4.0/hip + extra_attributes: + compilers: + hip: /opt/rocm-5.4.0/hip/bin/hipcc + llvm-amdgpu: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: llvm-amdgpu@5.4.3 + prefix: /opt/rocm-5.4.3/llvm + - spec: llvm-amdgpu@5.6.0 + prefix: /opt/rocm-5.6.0/llvm + - spec: llvm-amdgpu@5.4.0 + prefix: /opt/rocm-5.4.0/llvm + rocm-device-libs: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-device-libs@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocm-device-libs@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocm-device-libs@5.4.0 + prefix: /opt/rocm-5.4.0 + hsa-rocr-dev: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hsa-rocr-dev@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hsa-rocr-dev@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hsa-rocr-dev@5.4.0 + prefix: /opt/rocm-5.4.0 + hsakmt-roct: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hsakmt-roct@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hsakmt-roct@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hsakmt-roct@5.4.0 + prefix: /opt/rocm-5.4.0 + rocminfo: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocminfo@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocminfo@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocminfo@5.4.0 + prefix: /opt/rocm-5.4.0 + rccl: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rccl@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rccl@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rccl@5.4.0 + prefix: /opt/rocm-5.4.0 + hipify-clang: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipify-clang@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipify-clang@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hipify-clang@5.4.0 + prefix: /opt/rocm-5.4.0 + hipblas: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipblas@5.4.3 + prefix: /opt/rocm-5.4.3/hipblas + - spec: hipblas@5.6.0 + prefix: /opt/rocm-5.6.0/hipblas + - spec: hipblas@5.4.0 + prefix: /opt/rocm-5.4.0/hipblas + hipcub: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipcub@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipcub@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hipcub@5.4.0 + prefix: /opt/rocm-5.4.0 + hipsparse: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipsparse@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipsparse@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hipsparse@5.4.0 + prefix: /opt/rocm-5.4.0 + hipfort: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipfort@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipfort@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hipfort@5.4.0 + prefix: /opt/rocm-5.4.0 + hipfft: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: hipfft@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: hipfft@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: hipfft@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-opencl: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-opencl@5.4.3 + prefix: /opt/rocm-5.4.3/opencl + - spec: rocm-opencl@5.6.0 + prefix: /opt/rocm-5.6.0/opencl + - spec: rocm-opencl@5.4.0 + prefix: /opt/rocm-5.4.0/opencl + rocm-clang-ocl: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-clang-ocl@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocm-clang-ocl@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocm-clang-ocl@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-opencl-runtime: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-opencl-runtime@5.4.3 + prefix: /opt/rocm-5.4.3/opencl + - spec: rocm-opencl-runtime@5.6.0 + prefix: /opt/rocm-5.6.0/opencl + - spec: rocm-opencl-runtime@5.4.0 + prefix: /opt/rocm-5.4.0/opencl + rocm-openmp-extras: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-openmp-extras@5.4.3 + prefix: /opt/rocm-5.4.3/llvm + - spec: rocm-openmp-extras@5.6.0 + prefix: /opt/rocm-5.6.0/llvm + - spec: rocm-openmp-extras@5.4.0 + prefix: /opt/rocm-5.4.0/llvm + rocblas: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocblas@5.4.3 + prefix: /opt/rocm-5.4.3/rocblas + - spec: rocblas@5.6.0 + prefix: /opt/rocm-5.6.0/rocblas + - spec: rocblas@5.4.0 + prefix: /opt/rocm-5.4.0/rocblas + rocfft: + variants: amdgpu_target=gfx90a amdgpu_target_sram_ecc=gfx90a + rocrand: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocrand@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocrand@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocrand@5.4.0 + prefix: /opt/rocm-5.4.0 + rocthrust: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocthrust@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocthrust@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocthrust@5.4.0 + prefix: /opt/rocm-5.4.0 + rocsolver: + variants: amdgpu_target=gfx90a + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocsolver@5.4.3 + prefix: /opt/rocm-5.4.3/rocsolver + - spec: rocsolver@5.6.0 + prefix: /opt/rocm-5.6.0/rocsolver + - spec: rocsolver@5.4.0 + prefix: /opt/rocm-5.4.0/rocsolver + rocsparse: + variants: amdgpu_target=gfx90a + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocsparse@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocsparse@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocsparse@5.4.0 + prefix: /opt/rocm-5.4.0 + roctracer-dev: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: roctracer-dev@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: roctracer-dev@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: roctracer-dev@5.4.0 + prefix: /opt/rocm-5.4.0 + rocprofiler-dev: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocprofiler-dev@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocprofiler-dev@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocprofiler-dev@5.4.0 + prefix: /opt/rocm-5.4.0 + rocprim: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocprim@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocprim@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocprim@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-smi: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocmsmi@5.4.3 + prefix: /opt/rocm-5.4.3/rocm_smi + - spec: rocm-smi@5.6.0 + prefix: /opt/rocm-5.6.0/rocm_smi + - spec: rocm-smi@5.4.0 + prefix: /opt/rocm-5.6.0/rocm.2.0 + rocm-smi-lib: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-smi-lib@5.4.3 + prefix: /opt/rocm-5.4.3/rocm_smi + - spec: rocm-smi-lib@5.6.0 + prefix: /opt/rocm-5.6.0/rocm_smi + - spec: rocm-smi-lib@5.4.0 + prefix: /opt/rocm-5.6.0/rocm.2.0 + miopen-hip: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: miopen-hip@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: miopen-hip@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: miopen-hip@5.4.0 + prefix: /opt/rocm-5.4.0 + rocalution: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocalution@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocalution@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocalution@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-gdb: + version: [5.4.3, 5.6.0, 5.4.0] + buildable: false + externals: + - spec: rocm-gdb@5.4.3 + prefix: /opt/rocm-5.4.3 + - spec: rocm-gdb@5.6.0 + prefix: /opt/rocm-5.6.0 + - spec: rocm-gdb@5.4.0 + prefix: /opt/rocm-5.4.0 + specs: + - conduit%cce@16.0.0 ~fortran ~hdf5_compat generator=make ^hdf5+hl+mpi~fortran ^cray-mpich@8.1.26 + - raja@develop%cce@16.0.0 +rocm amdgpu_target=gfx90a ^hip@5.6.0 ^cmake@3.24.2 + - chai@develop%cce@16.0.0~examples~openmp+raja+rocm amdgpu_target=gfx90a generator=make ^raja@develop+rocm + amdgpu_target=gfx90a ^umpire@develop~examples+rocm amdgpu_target=gfx90a ^hip@5.6.0 + ^blt@0.5.2 ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 ^camp+rocm amdgpu_target=gfx90a + ^cmake@3.24.2 + - hypre@develop%cce@16.0.0~int64~fortran+rocm+unified-memory amdgpu_target=gfx90a ^hip@5.6.0 + ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 ^cray-mpich@8.1.26 ^cray-libsci@23.05.1.4 + - pugixml%cce@16.0.0 generator=make + - fmt%cce@16.0.0 generator=make + - suite-sparse@5.10.1%cce@16.0.0~cuda~graphblas~openmp+pic~tbb + - caliper@master%cce@16.0.0~adiak~libdw~papi+rocm amdgpu_target=gfx90a generator=make ^hip@5.6.0 + ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 \ No newline at end of file From 5f4351f7faa2bed097fcc58974bb62d711ea845c Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 09:23:49 -0700 Subject: [PATCH 05/25] moving environments to avoid potential for name conflicts in ci --- .../lassen-blueos_4_ppc64le_ib_p9/spack.yaml | 0 .../environments => spack-environments}/lassen-toss_4_ppc64le_ib | 0 .../quartz-toss_4_x86_64_ib/spack.yaml | 0 .../tioga-toss_4_x86_64_ib_cray/spack.yaml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {spack/environments => spack-environments}/lassen-blueos_4_ppc64le_ib_p9/spack.yaml (100%) rename {spack/environments => spack-environments}/lassen-toss_4_ppc64le_ib (100%) rename {spack/environments => spack-environments}/quartz-toss_4_x86_64_ib/spack.yaml (100%) rename {spack/environments => spack-environments}/tioga-toss_4_x86_64_ib_cray/spack.yaml (100%) diff --git a/spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml b/spack-environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml similarity index 100% rename from spack/environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml rename to spack-environments/lassen-blueos_4_ppc64le_ib_p9/spack.yaml diff --git a/spack/environments/lassen-toss_4_ppc64le_ib b/spack-environments/lassen-toss_4_ppc64le_ib similarity index 100% rename from spack/environments/lassen-toss_4_ppc64le_ib rename to spack-environments/lassen-toss_4_ppc64le_ib diff --git a/spack/environments/quartz-toss_4_x86_64_ib/spack.yaml b/spack-environments/quartz-toss_4_x86_64_ib/spack.yaml similarity index 100% rename from spack/environments/quartz-toss_4_x86_64_ib/spack.yaml rename to spack-environments/quartz-toss_4_x86_64_ib/spack.yaml diff --git a/spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml similarity index 100% rename from spack/environments/tioga-toss_4_x86_64_ib_cray/spack.yaml rename to spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml From adb5892cad81c0d2ec2aa96b70ce9295e8fb6486 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:24:21 -0700 Subject: [PATCH 06/25] working on spack testing --- .gitlab-ci.yml | 35 +++++++++++++++++++ .../tioga-toss_4_x86_64_ib_cray/spack.yaml | 1 - 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c63c6ea..bc8673d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,41 @@ variables: GEOS_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/llnl-tpls-base.cmake INSTALL_SYSTEM: "false" + +.test_spack: + script: + - git clone git@github.com:spack/spack.git + - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64 +# - source spack/share/spack/setup-env.sh +# - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml +# - spack env activate ${CLUSTER}-${SYS_TYPE} +# - spack install + +.install_spack: + script: + - git clone git@github.com:spack/spack.git + - source spack/share/spack/setup-env.sh + - modifyYaml.py -i spack-environment/${CLUSTER-${SYSTEM}/spack.yaml -o spack-environments/${CLUSTER}-${SYSTEM}/spack.yaml -y "spack:config:install_tree:root:/usr/WS1/GEOS/GEOSX/TPLs_$(date +'%Y-%m-%d')" + - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml + - spack env activate ${CLUSTER}-${SYS_TYPE} + - spack install + +test_spack_tioga: + stage: test + tags: + - tioga + - shell + variables: + NNODES: 1 + NPROCS: 64 + QUEUE: pdebug + WALL_TIME: 90 + LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" + CLUSTER: tioga + SYS_TYPE: toss_4_x86_64_ib_cray + extends: [.test_spack] + + test_lc_tpls: stage: test variables: diff --git a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml index 2a173553..0ce498ec 100644 --- a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml +++ b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml @@ -6,7 +6,6 @@ spack: misc_cache: $spack/.cache install_tree: root: $spack/opt/spack/ - #root: /usr/WS1/GEOS/GEOSX/TPLs_$date/ projections: all: install-{os}-{arch}-{compiler.name}-{compiler.version}/{name}-{version} compilers: From 859f4f0279b8eebcfe83a215623f5a357b370a25 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:27:15 -0700 Subject: [PATCH 07/25] drop incorrect timeout syntax --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc8673d8..91645984 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,3 @@ -timeout: 4h - stages: - test - deploy From 07bb10fe85632261d116f64b3d6f123f8bdffc5e Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:28:57 -0700 Subject: [PATCH 08/25] incorrect variable syntax --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91645984..6eff8e5f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ test_spack_tioga: test_lc_tpls: stage: test variables: - - INSTALL_DIR=builds + INSTALL_DIR=builds script: - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} - cd ${GEOS_DIR} @@ -73,11 +73,11 @@ deploy_lc_tpls: dependencies: - test_lc_tpls variables: - - XSU: xsu geosadmn - - TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs - - DATE: $(date +'%Y-%m-%d') - - INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE} - - BRANCH: feature/update-tpls-${DATE} + XSU: xsu geosadmn + TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs + DATE: $(date +'%Y-%m-%d') + INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE} + BRANCH: feature/update-tpls-${DATE} script: - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} - cd ${GEOS_DIR} From eaaf7c1423c7a32e6cba2a5c9f68b1ad4286c450 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:29:58 -0700 Subject: [PATCH 09/25] incorrect variable syntax (again) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6eff8e5f..bd2c238d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,7 +47,7 @@ test_spack_tioga: test_lc_tpls: stage: test variables: - INSTALL_DIR=builds + INSTALL_DIR: builds script: - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} - cd ${GEOS_DIR} From 68baf8c1a75c5dc5532fbc4d33bdc1d53d308840 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:32:56 -0700 Subject: [PATCH 10/25] gitlab-runner vs cloud runner acts differently yay --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd2c238d..b5a3538f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,8 +41,10 @@ test_spack_tioga: LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" CLUSTER: tioga SYS_TYPE: toss_4_x86_64_ib_cray - extends: [.test_spack] - + extends: + - .test_spack + script: + - !reference [ .test_spack, script ] test_lc_tpls: stage: test From eb92a4c8eb46d6c56ecd05bd45482f160418d114 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:33:35 -0700 Subject: [PATCH 11/25] gitlab is garbage --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5a3538f..4181702a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,10 +41,9 @@ test_spack_tioga: LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" CLUSTER: tioga SYS_TYPE: toss_4_x86_64_ib_cray - extends: - - .test_spack script: - - !reference [ .test_spack, script ] + - git clone git@github.com:spack/spack.git + - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64 test_lc_tpls: stage: test From 20ce7bd30f9efbf01ac15d1aede5b5f8783a2365 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:34:12 -0700 Subject: [PATCH 12/25] gitlab is garbage --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4181702a..45a727e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,8 +37,8 @@ test_spack_tioga: NNODES: 1 NPROCS: 64 QUEUE: pdebug - WALL_TIME: 90 - LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" +# WALL_TIME: 90 +# LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" CLUSTER: tioga SYS_TYPE: toss_4_x86_64_ib_cray script: From 1535fe9d40cc5b6a548367ae0567cf653d9ef74a Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:34:29 -0700 Subject: [PATCH 13/25] gitlab is garbage --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 45a727e8..c8eca85d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,8 +34,8 @@ test_spack_tioga: - tioga - shell variables: - NNODES: 1 - NPROCS: 64 +# NNODES: 1 +# NPROCS: 64 QUEUE: pdebug # WALL_TIME: 90 # LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" From b8ad38be3d562d239d53342aa801c47d7af4e02b Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 11:52:38 -0700 Subject: [PATCH 14/25] adding job names --- scripts/setupLC-TPL.bash | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/setupLC-TPL.bash b/scripts/setupLC-TPL.bash index 52c11973..2b900a8e 100755 --- a/scripts/setupLC-TPL.bash +++ b/scripts/setupLC-TPL.bash @@ -27,11 +27,11 @@ rm -rf toBeDeleted & echo "Building all LC TPLs from $GEOSX_DIR to be installed at $INSTALL_DIR" chmod -R g+rx $INSTALL_DIR chgrp -R GEOS $INSTALL_DIR -./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz clang-14 "srun -N 1 -t 90 -n 1 -A geosecp" $@ & -./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz gcc-12 "srun -N 1 -t 90 -n 1 -A geosecp" $@ & -./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen gcc-8-cuda-11 "lalloc 1 -qpdebug" $@ & -./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-13-cuda-11 "lalloc 1 -qpdebug" $@ & -./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-10-cuda-11 "lalloc 1 -qpdebug" $@ & +./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz clang-14 "srun -N 1 -t 90 -n 1 -A geosecp -J quartz-clang-14-tpls" $@ & +./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz gcc-12 "srun -N 1 -t 90 -n 1 -A geosecp -J quartz-gcc-12-tpls" $@ & +./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen gcc-8-cuda-11 "bsub -J lassen-gcc-8-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ & +./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-13-cuda-11 "bsub -J lassen-clang-13-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ & +./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-10-cuda-11 "bsub -J lassen-clang-10-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ & wait echo "Complete" From f9d8959eab0de9c7697b303df70d65453dc343a8 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 12:34:01 -0700 Subject: [PATCH 15/25] adding spack installation test script --- scripts/spack-install.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 scripts/spack-install.sh diff --git a/scripts/spack-install.sh b/scripts/spack-install.sh new file mode 100644 index 00000000..1662f221 --- /dev/null +++ b/scripts/spack-install.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +CLUSTER=$1 +SYS_TYPE=$2 +NPROCS=$3 + +source spack/share/spack/setup-env.sh +spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml +spack env activate ${CLUSTER}-${SYS_TYPE} +spack install -j $NPROCS From 28672579496f14280ec8a997bef3a268a985cc24 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 12:37:17 -0700 Subject: [PATCH 16/25] +x --- scripts/spack-install.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/spack-install.sh diff --git a/scripts/spack-install.sh b/scripts/spack-install.sh old mode 100644 new mode 100755 From 2bd1f92da31064b2f751a722784ab1ae9e04079f Mon Sep 17 00:00:00 2001 From: wrtobin Date: Wed, 11 Oct 2023 13:06:40 -0700 Subject: [PATCH 17/25] correct install directory spec to prevent redundancy --- spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml index 0ce498ec..f80c38d3 100644 --- a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml +++ b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml @@ -7,7 +7,7 @@ spack: install_tree: root: $spack/opt/spack/ projections: - all: install-{os}-{arch}-{compiler.name}-{compiler.version}/{name}-{version} + all: install-{arch}-{compiler.name}-{compiler.version}/{name}-{version} compilers: - compiler: spec: gcc@8.3.1 From cfffaf14b29217d39f78d4acff3a9ea671938672 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 08:55:06 -0700 Subject: [PATCH 18/25] repairing tioga spack environment fully --- .../tioga-toss_4_x86_64_ib_cray/spack.yaml | 63 ++++++++----------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml index f80c38d3..f6877b27 100644 --- a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml +++ b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml @@ -7,7 +7,7 @@ spack: install_tree: root: $spack/opt/spack/ projections: - all: install-{arch}-{compiler.name}-{compiler.version}/{name}-{version} + all: install-{target}-{compiler.name}-{compiler.version}/{name}-{version} compilers: - compiler: spec: gcc@8.3.1 @@ -61,10 +61,13 @@ spack: flags: {} operating_system: rhel8 target: x86_64 - modules: [] + modules: + - PrgEnv-cray/8.4.0 + - rocm/5.4.3 + - cce/15.0.0 environment: set: - TCE_ROCM_PATH: /opt/rocm-5.3.3 + TCE_ROCM_PATH: /opt/rocm-5.4.3 CRAY_PE_USE_CLANG: /opt/cray/pe/cce/15.0.0/cce-clang/x86_64/bin/clang extra_rpaths: [] - compiler: @@ -498,23 +501,16 @@ spack: prefix: /opt/cray/pe/cce/12.0.2 buildable: false cray-mpich: - version: [8.1.12, 8.1.16, 8.1.18, 8.1.21, 8.1.26] + version: [8.1.24,8.1.26] externals: - - spec: cray-mpich@8.1.12 - modules: - - cray-mpich/8.1.12 - - spec: cray-mpich@8.1.16 - modules: - - cray-mpich/8.1.16 - - spec: cray-mpich@8.1.18 + - spec: cray-mpich@8.1.24 modules: - - cray-mpich/8.1.18 - - spec: cray-mpich@8.1.21 - modules: - - cray-mpich/8.1.21 + - cray-mpich/8.1.24 + prefix: /opt/cray/pe/mpich/8.1.24/ofi/crayclang/10.0 - spec: cray-mpich@8.1.26 modules: - cray-mpich/8.1.26 + prefix: /opt/cray/pe/mpich/8.1.26/ofi/crayclang/16.0 target: [] compiler: [] buildable: false @@ -738,20 +734,12 @@ spack: buildable: false cray-libsci: buildable: false - version: [21.08.1.2, 22.08.1.1, 23.05.1.4] + version: [23.05.1.4] externals: - - spec: cray-libsci@21.08.1.2 - modules: - - cray-libsci/21.08.1.2 - - cray-mpich/8.1.16 - - spec: cray-libsci@22.08.1.1 - modules: - - cray-libsci/22.08.1.1 - - cray-mpich/8.1.18 - spec: cray-libsci@23.05.1.4 modules: - cray-libsci/23.05.1.4 - - cray-mpich/8.1.26 + - cray-mpich/8.1.24 tcsh: buildable: false version: [6.20.00] @@ -1072,16 +1060,17 @@ spack: - spec: rocm-gdb@5.4.0 prefix: /opt/rocm-5.4.0 specs: - - conduit%cce@16.0.0 ~fortran ~hdf5_compat generator=make ^hdf5+hl+mpi~fortran ^cray-mpich@8.1.26 - - raja@develop%cce@16.0.0 +rocm amdgpu_target=gfx90a ^hip@5.6.0 ^cmake@3.24.2 - - chai@develop%cce@16.0.0~examples~openmp+raja+rocm amdgpu_target=gfx90a generator=make ^raja@develop+rocm - amdgpu_target=gfx90a ^umpire@develop~examples+rocm amdgpu_target=gfx90a ^hip@5.6.0 - ^blt@0.5.2 ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 ^camp+rocm amdgpu_target=gfx90a + - conduit%cce@15.0.0a ~fortran ~hdf5_compat generator=make ^hdf5+hl+mpi~fortran ^cray-mpich@8.1.24 + - raja@develop%cce@15.0.0a +rocm amdgpu_target=gfx90a ^hip@5.4.3 ^cmake@3.24.2 + - chai@develop%cce@15.0.0a~examples~openmp+raja+rocm amdgpu_target=gfx90a generator=make ^raja@develop+rocm + amdgpu_target=gfx90a ^umpire@develop~examples+rocm amdgpu_target=gfx90a ^hip@5.4.3 + ^blt@0.5.2 ^hsa-rocr-dev@5.4.3 ^llvm-amdgpu@5.4.3 ^camp+rocm amdgpu_target=gfx90a ^cmake@3.24.2 - - hypre@develop%cce@16.0.0~int64~fortran+rocm+unified-memory amdgpu_target=gfx90a ^hip@5.6.0 - ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 ^cray-mpich@8.1.26 ^cray-libsci@23.05.1.4 - - pugixml%cce@16.0.0 generator=make - - fmt%cce@16.0.0 generator=make - - suite-sparse@5.10.1%cce@16.0.0~cuda~graphblas~openmp+pic~tbb - - caliper@master%cce@16.0.0~adiak~libdw~papi+rocm amdgpu_target=gfx90a generator=make ^hip@5.6.0 - ^hsa-rocr-dev@5.6.0 ^llvm-amdgpu@5.6.0 \ No newline at end of file + - hypre@develop%cce@15.0.0a~int64~fortran+rocm+unified-memory amdgpu_target=gfx90a ^hip@5.4.3 + ^hsa-rocr-dev@5.4.3 ^llvm-amdgpu@5.4.3 ^cray-mpich@8.1.24 ^cray-libsci@23.05.1.4 + ^rocprim@5.4.3 ^rocrand@5.4.3 ^rocsparse@5.4.3 ^rocthrust@5.4.3 + - pugixml%cce@15.0.0a generator=make + - fmt%cce@15.0.0a generator=make + - suite-sparse@5.10.1%cce@15.0.0a~cuda~graphblas~openmp+pic~tbb + - caliper@master%cce@15.0.0a~adiak~libdw~papi+rocm amdgpu_target=gfx90a generator=make ^hip@5.4.3 + ^hsa-rocr-dev@5.4.3 ^llvm-amdgpu@5.4.3 \ No newline at end of file From 0aa5275bd38e958f335a183cf1ca562c714e8a92 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 11:43:17 -0700 Subject: [PATCH 19/25] working on tioga testing --- .gitlab-ci.yml | 87 ++++++++++++++----- scripts/spack-install.sh | 10 ++- .../tioga-toss_4_x86_64_ib_cray/spack.yaml | 2 +- 3 files changed, 71 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c8eca85d..553485eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,9 +6,39 @@ variables: GIT_SUBMODULE_STRATEGY: recursive GEOS_DIR: "geos" GEOS_BRANCH: "feature/wrtobin/tpl-gitlab-ci-cd" # Default branch, can be overridden when CI is run manually + SPACK_DIR: "spack" + SPACK_BRANCH: "develop" + DEPLOY: "false" # default, override when launching manually to actually deploy GEOS_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/llnl-tpls-base.cmake - INSTALL_SYSTEM: "false" + TIOGA_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/tioga-base.cmake + TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs + DATE: $(date +'%Y-%m-%d') + HOST_CONFIG_INSTALL_SCHEME: install-\$\{CONFIG_NAME\}-release + SPACK_INSTALL_SCHEME: install-\$\{CLUSTER\}-\$\{SYS_TYPE\}-release +.clone_geos: + script: + - git clone -b ${GEOS_BRANCH} git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} + - cd ${GEOS_DIR} + - git submodule init src/cmake/blt + - git submodule init src/coreComponents/LvArray + - git submodule init src/coreComponents/constitutive/PVTPackage + - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface + - git submodule update + - cd .. + +.update_geos: + script: + - cd ${GEOS_DIR}/src/coreComponents/LvArray + - git checkout -b ${BRANCH} + - git add -A + - git commit -m "Update lc tpl install location." + - git push origin ${BRANCH} + - cd ${GEOS_DIR} + - git checkout -b ${BRANCH} + - git add -A + - git commit -m "Update lc tpl install location." + - git push origin ${BRANCH} .test_spack: script: @@ -24,29 +54,45 @@ variables: - git clone git@github.com:spack/spack.git - source spack/share/spack/setup-env.sh - modifyYaml.py -i spack-environment/${CLUSTER-${SYSTEM}/spack.yaml -o spack-environments/${CLUSTER}-${SYSTEM}/spack.yaml -y "spack:config:install_tree:root:/usr/WS1/GEOS/GEOSX/TPLs_$(date +'%Y-%m-%d')" - - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml - - spack env activate ${CLUSTER}-${SYS_TYPE} - - spack install + - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64 + # - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml + # - spack env activate ${CLUSTER}-${SYS_TYPE} + # - spack install + # - INSTALL_SCHEME=${SPACK_INSTALL_SCHEME} test_spack_tioga: stage: test + when: manual tags: - tioga - shell variables: -# NNODES: 1 -# NPROCS: 64 - QUEUE: pdebug -# WALL_TIME: 90 -# LLNL_SLURM_SCHEDULER_PARAMETERS: "-N $NNODES -n $NPROCS -t $WALL_TIME" CLUSTER: tioga SYS_TYPE: toss_4_x86_64_ib_cray script: - - git clone git@github.com:spack/spack.git - - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64 + - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} + - cd ${GEOS_DIR} + - git submodule init src/cmake/blt + - git submodule init src/coreComponents/LvArray + - git submodule init src/coreComponents/constitutive/PVTPackage + - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface + - git submodule update + - cd .. + - git clone git@github.com:spack/spack.git ${SPACK_DIR} + - SPACK_DIR_ABS=$(realpath "${SPACK_DIR}") + - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}"" 64 + - sed -i '7s|.*|set(GEOSX_TPL_DIR "${SPACK_DIR}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/" CACHE PATH "")|' "${TIOGA_TPL_BASE}" + - cd ${GEOS_DIR} + - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.camke -bt Release + - cd build-tioga-cce-15-release + - srun -N 1 -t 90 -n 1 make -j 64 test_lc_tpls: stage: test + when: manual + tags: + - quartz + - shell variables: INSTALL_DIR: builds script: @@ -64,19 +110,19 @@ test_lc_tpls: - echo "set(GEOS_TPL_DIR \$\{GEOS_TPL_ROOT_DIR\}/${TPL_DIR}/install-\$\{CONFIG_NAME\}-release CACHE PATH \"\")" >> ${GEOS_TPL_BASE} - cd ${GEOS_DIR} - scripts/setupLC.bash - tags: - - quartz - - shell - when: manual deploy_lc_tpls: + rules: + - if: '$CI_COMMIT_BRANCH == "master" && $DEPLOY == "true"' stage: deploy + when: manual + tags: + - quartz + - shell dependencies: - test_lc_tpls variables: XSU: xsu geosadmn - TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs - DATE: $(date +'%Y-%m-%d') INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE} BRANCH: feature/update-tpls-${DATE} script: @@ -104,9 +150,4 @@ deploy_lc_tpls: - git add -A - git commit -m "Update lc tpl install location." - git push origin ${BRANCH} - tags: - - quartz - - shell - when: manual - rules: - - if: '$CI_COMMIT_BRANCH == "master" && $INSTALL_SYSTEM == "true"' + diff --git a/scripts/spack-install.sh b/scripts/spack-install.sh index 1662f221..74cac6f8 100755 --- a/scripts/spack-install.sh +++ b/scripts/spack-install.sh @@ -1,10 +1,12 @@ #!/bin/bash -CLUSTER=$1 -SYS_TYPE=$2 -NPROCS=$3 +SPACK_DIR=$1 +CLUSTER=$2 +SYS_TYPE=$3 +NPROCS=$4 source spack/share/spack/setup-env.sh spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml spack env activate ${CLUSTER}-${SYS_TYPE} -spack install -j $NPROCS +spack install -j ${NPROCS} +despacktivate \ No newline at end of file diff --git a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml index f6877b27..2409a8d1 100644 --- a/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml +++ b/spack-environments/tioga-toss_4_x86_64_ib_cray/spack.yaml @@ -7,7 +7,7 @@ spack: install_tree: root: $spack/opt/spack/ projections: - all: install-{target}-{compiler.name}-{compiler.version}/{name}-{version} + all: install-{os}-{target}-{compiler.name}-{compiler.version}/{name} compilers: - compiler: spec: gcc@8.3.1 From 75a7ae62b73f3172a409333e1d4667ce1179d112 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 11:46:37 -0700 Subject: [PATCH 20/25] working on tioga testing --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 553485eb..b50258ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,7 +81,7 @@ test_spack_tioga: - git clone git@github.com:spack/spack.git ${SPACK_DIR} - SPACK_DIR_ABS=$(realpath "${SPACK_DIR}") - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}"" 64 - - sed -i '7s|.*|set(GEOSX_TPL_DIR "${SPACK_DIR}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/" CACHE PATH "")|' "${TIOGA_TPL_BASE}" + - sed -i '7s|.*|set(GEOSX_TPL_DIR "${SPACK_DIR_ABS}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/" CACHE PATH "")|' "${TIOGA_TPL_BASE}" - cd ${GEOS_DIR} - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.camke -bt Release - cd build-tioga-cce-15-release From 62d1ccb1f13cef22b0c21402cd07e120a937d911 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 11:50:13 -0700 Subject: [PATCH 21/25] working on tioga testing --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b50258ed..2f0d5407 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,12 +80,12 @@ test_spack_tioga: - cd .. - git clone git@github.com:spack/spack.git ${SPACK_DIR} - SPACK_DIR_ABS=$(realpath "${SPACK_DIR}") - - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}"" 64 + - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}" 64 - sed -i '7s|.*|set(GEOSX_TPL_DIR "${SPACK_DIR_ABS}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/" CACHE PATH "")|' "${TIOGA_TPL_BASE}" - cd ${GEOS_DIR} - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.camke -bt Release - cd build-tioga-cce-15-release - - srun -N 1 -t 90 -n 1 make -j 64 + - srun -N 1 -t 90 -n 1 make -j 64 geosx test_lc_tpls: stage: test From e9017f7b46c61ce7c3d8665177ebada1433a2274 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 13:10:58 -0700 Subject: [PATCH 22/25] working on tioga testing --- scripts/spack-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack-install.sh b/scripts/spack-install.sh index 74cac6f8..cb4190b0 100755 --- a/scripts/spack-install.sh +++ b/scripts/spack-install.sh @@ -9,4 +9,4 @@ source spack/share/spack/setup-env.sh spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml spack env activate ${CLUSTER}-${SYS_TYPE} spack install -j ${NPROCS} -despacktivate \ No newline at end of file +spack env deactivate \ No newline at end of file From 555d7d3b5f7986840107e8e586f2e3141a1ec4ed Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 13:55:13 -0700 Subject: [PATCH 23/25] working on tioga testing --- .gitlab-ci.yml | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f0d5407..2c45b4f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: HOST_CONFIG_INSTALL_SCHEME: install-\$\{CONFIG_NAME\}-release SPACK_INSTALL_SCHEME: install-\$\{CLUSTER\}-\$\{SYS_TYPE\}-release -.clone_geos: +.clone_geos_no_integrated: script: - git clone -b ${GEOS_BRANCH} git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} - cd ${GEOS_DIR} @@ -32,12 +32,12 @@ variables: - cd ${GEOS_DIR}/src/coreComponents/LvArray - git checkout -b ${BRANCH} - git add -A - - git commit -m "Update lc tpl install location." + - git commit -m "Update lc tpl installation." - git push origin ${BRANCH} - cd ${GEOS_DIR} - git checkout -b ${BRANCH} - git add -A - - git commit -m "Update lc tpl install location." + - git commit -m "Update lc tpl installation." - git push origin ${BRANCH} .test_spack: @@ -81,12 +81,37 @@ test_spack_tioga: - git clone git@github.com:spack/spack.git ${SPACK_DIR} - SPACK_DIR_ABS=$(realpath "${SPACK_DIR}") - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}" 64 - - sed -i '7s|.*|set(GEOSX_TPL_DIR "${SPACK_DIR_ABS}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/" CACHE PATH "")|' "${TIOGA_TPL_BASE}" + - sed -i "7s|.*|set(GEOSX_TPL_DIR \"${SPACK_DIR_ABS}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/\" CACHE PATH \"\")|" "${TIOGA_TPL_BASE}" - cd ${GEOS_DIR} - - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.camke -bt Release + - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.cmake -bt Release - cd build-tioga-cce-15-release - srun -N 1 -t 90 -n 1 make -j 64 geosx +test_uberenv_tioga: + stage: test + when: manual + tags: + - tioga + - shell + variables: + CLUSTER: tioga + SYS_TYPE: toss_4_x86_64_ib_cray + script: + - TPL_ROOT=${PWD} + - git clone -b $GEOS_BRANCH git@github.com:GEOS-DEV/GEOS.git ${GEOS_DIR} + - cd ${GEOS_DIR} + - git submodule init src/cmake/blt + - git submodule init src/coreComponents/LvArray + - git submodule init src/coreComponents/constitutive/PVTPackage + - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface + - git submodule update + - cd scripts/uberenv + - srun -N 1 -t 90 -n 1 uberenv.py --spack-env-file ${TPL_ROOT}/spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml --spec "%cce@15.0.0a ++rocm amdgpu_targets==gfx90a +hypre lai=hypre" + # - cd ../.. + # - cp spack-build-*/tioga-rhel8-x85_64_ib.cmake host-configs/LLNL/ + # - cp spack-build-*/tioga-rhel8-x85_64_ib.cmake src/coreComponents/LvArray/host-configs/LLNL/ + + test_lc_tpls: stage: test when: manual From 413c31a820bbcf055c4b14717a235ca44f733f78 Mon Sep 17 00:00:00 2001 From: wrtobin Date: Thu, 12 Oct 2023 15:53:20 -0700 Subject: [PATCH 24/25] wip --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c45b4f4..523d531f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,10 +44,11 @@ variables: script: - git clone git@github.com:spack/spack.git - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64 -# - source spack/share/spack/setup-env.sh -# - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml -# - spack env activate ${CLUSTER}-${SYS_TYPE} -# - spack install + # - source spack/share/spack/setup-env.sh + # - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml + # - spack env activate ${CLUSTER}-${SYS_TYPE} + # - spack install + # - spack env deactivate .install_spack: script: @@ -59,6 +60,7 @@ variables: # - spack env activate ${CLUSTER}-${SYS_TYPE} # - spack install # - INSTALL_SCHEME=${SPACK_INSTALL_SCHEME} + # - spack env deactivate test_spack_tioga: stage: test From 46566fa489b30b6b86809c6184a916e95a7e444b Mon Sep 17 00:00:00 2001 From: wrtobin Date: Tue, 12 Dec 2023 07:49:56 -0800 Subject: [PATCH 25/25] adding frontier spack env --- .gitignore | 1 + .../frontier-sles15_x86_64_cray/spack.yaml | 456 ++++++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 spack-environments/frontier-sles15_x86_64_cray/spack.yaml diff --git a/.gitignore b/.gitignore index a4915200..079a72de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +builds/* build-* install-* .cproject diff --git a/spack-environments/frontier-sles15_x86_64_cray/spack.yaml b/spack-environments/frontier-sles15_x86_64_cray/spack.yaml new file mode 100644 index 00000000..6130a11b --- /dev/null +++ b/spack-environments/frontier-sles15_x86_64_cray/spack.yaml @@ -0,0 +1,456 @@ +# This is a Spack Environment file. +# +# It describes a set of packages to be installed, along with +# configuration settings. + +spack: + modules: + default: + lmod: + core_compilers: + - gcc@7.5.0 + - gcc@7-os + packages: + mesa: + buildable: false + version: [21.2.1] + externals: + - spec: mesa@21.2.1 + prefix: /sw/crusher/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/mesa-21.2.1-cbfrg4yfizu6ouirpoql7j7l62emid5c/ + modules: + - mesa/21.2.1 + curl: + buildable: false + version: [4.6.0] + externals: + - spec: curl@4.6.0 + prefix: /usr + gnutls: + buildable: false + version: [30.23.2] + externals: + - spec: gnutls@30.23.2 + prefix: /usr + gettext: + buildable: false + version: [0.20.2] + externals: + - spec: gettext@0.20.2 + prefix: /usr + libxml2: + buildable: false + version: [2.9.7] + externals: + - spec: libxml2@2.9.7 + prefix: /usr + ncurses: + buildable: false + version: [5.9] + externals: + - spec: ncurses@5.9 + prefix: /usr + automake: + buildable: false + version: [1.15.1] + externals: + - spec: automake@1.15.1 + prefix: /usr/bin + python: + buildable: false + version: [3.9.13] + externals: + - spec: python@3.9.13 + prefix: /sw/crusher/spack-envs/base/opt/cray-sles15-zen3/cce-14.0.0/python-3.9.13-psyaekfxbe4oz5wwcqqoh5l6cfwbtbhc + perl: + buildable: false + version: [5.26.1] + externals: + - spec: perl@5.26.1 + prefix: /usr/bin + readline: + buildable: false + version: [7.0] + externals: + - spec: readline@7.0 + prefix: /usr/lib64 + gdbm: + buildable: false + version: [1.12] + externals: + - spec: gdbm@1.12 + prefix: /usr/lib64 + m4: + buildable: false + version: [1.4.18] + externals: + - spec: m4@1.4.18 + prefix: /usr + autoconf: + buildable: false + version: [2.69] + externals: + - spec: autoconf@2.69 + modules: + - autoconf/2.69 + autoreconf: + buildable: false + version: [2.69] + externals: + - spec: autoreconf@2.69 + prefix: /usr/bin + libiconv: + buildable: false + version: [2.31] + externals: + - spec: libiconv@2.31 + prefix: /usr/ + cmake: + buildable: false + version: [3.20.4, 3.21.3, 3.23.2] + externals: + - spec: cmake@3.20.4 + prefix: /usr/ + - spec: cmake@3.21.3 + modules: + - cmake/3.21.3 + - spec: cmake@3.23.2 + modules: + - cmake/3.23.2 + openmpi: + buildable: false + mpich: + buildable: false + cray-mpich: + version: [8.1.12, 8.1.16, 8.1.18, 8.1.23] + externals: + - spec: cray-mpich@8.1.12 + prefix: /opt/cray/pe/mpich/8.1.12/ofi/crayclang/10.0 + modules: + - cray-mpich/8.1.12 + - spec: cray-mpich@8.1.16 + prefix: /opt/cray/pe/mpich/8.1.16/ofi/crayclang/10.0 + modules: + - cray-mpich/8.1.16 + - spec: cray-mpich@8.1.18 + prefix: /opt/cray/pe/mpich/8.1.18/ofi/crayclang/10.0 + modules: + - cray-mpich/8.1.18 + - spec: cray-mpich@8.1.23 + prefix: /opt/cray/pe/mpich/8.1.23/ofi/crayclang/10.0 + modules: + - cray-mpich/8.1.23 + target: [] + compiler: [] + buildable: false + providers: {} + cray-libsci: + buildable: false + version: [21.08.1.2, 22.08.1.1] + externals: + - spec: cray-libsci@21.08.1.2 + modules: + - cray-libsci/21.08.1.2 + - cray-mpich/8.1.16 + - spec: cray-libsci@22.08.1.1 + modules: + - cray-libsci/22.08.1.1 + - cray-mpich/8.1.18 + tcsh: + buildable: false + version: [6.20.00] + externals: + - spec: tcsh@6.20.00 + prefix: /usr + hip: + version: [5.4.0] + buildable: false + externals: + - spec: hip@5.4.0 + prefix: /opt/rocm-5.4.0/hip + extra_attributes: + compilers: + hip: /opt/rocm-5.4.0/hip/bin/hipcc + hip-rocclr: + version: [5.4.0] + buildable: false + externals: + - spec: hip-rocclr@5.4.0 + prefix: /opt/rocm-5.4.0/rocclr + comgr: + version: [5.4.0] + buildable: false + externals: + - spec: comgr@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-cmake: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-cmake@5.4.0 + prefix: /opt/rocm-5.4.0 + llvm-amdgpu: + version: [5.4.0] + buildable: false + externals: + - spec: llvm-amdgpu@5.4.0 + prefix: /opt/rocm-5.4.0/llvm + rocm-device-libs: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-device-libs@5.4.0 + prefix: /opt/rocm-5.4.0 + hsa-rocr-dev: + version: [5.4.0] + buildable: false + externals: + - spec: hsa-rocr-dev@5.4.0 + prefix: /opt/rocm-5.4.0 + hsakmt-roct: + version: [5.4.0] + buildable: false + externals: + - spec: hsakmt-roct@5.4.0 + prefix: /opt/rocm-5.4.0 + rocminfo: + version: [5.4.0] + buildable: false + externals: + - spec: rocminfo@5.4.0 + prefix: /opt/rocm-5.4.0 + rccl: + version: [5.4.0] + buildable: false + externals: + - spec: rccl@5.4.0 + prefix: /opt/rocm-5.4.0 + hipify-clang: + version: [5.4.0] + buildable: false + externals: + - spec: hipify-clang@5.4.0 + prefix: /opt/rocm-5.4.0 + hipblas: + version: [5.4.0] + buildable: false + externals: + - spec: hipblas@5.4.0 + prefix: /opt/rocm-5.4.0/hipblas + hipcub: + version: [5.4.0] + buildable: false + externals: + - spec: hipcub@5.4.0 + prefix: /opt/rocm-5.4.0 + hipsparse: + version: [5.4.0] + buildable: false + externals: + - spec: hipsparse@5.4.0 + prefix: /opt/rocm-5.4.0 + hipfort: + version: [5.4.0] + buildable: false + externals: + - spec: hipfort@5.4.0 + prefix: /opt/rocm-5.4.0 + hipfft: + version: [5.4.0] + buildable: false + externals: + - spec: hipfft@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-opencl: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-opencl@5.4.0 + prefix: /opt/rocm-5.4.0/opencl + rocm-clang-ocl: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-clang-ocl@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-opencl-runtime: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-opencl-runtime@5.4.0 + prefix: /opt/rocm-5.4.0/opencl + rocm-openmp-extras: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-openmp-extras@5.4.0 + prefix: /opt/rocm-5.4.0/llvm + rocblas: + version: [5.4.0] + buildable: false + externals: + - spec: rocblas@5.4.0 + prefix: /opt/rocm-5.4.0/rocblas + rocfft: + variants: amdgpu_target=gfx90a amdgpu_target_sram_ecc=gfx90a + rocrand: + version: [5.4.0] + buildable: false + externals: + - spec: rocrand@5.4.0 + prefix: /opt/rocm-5.4.0 + rocthrust: + version: [5.4.0] + buildable: false + externals: + - spec: rocthrust@5.4.0 + prefix: /opt/rocm-5.4.0 + rocsolver: + variants: amdgpu_target=gfx90a + version: [5.4.0] + buildable: false + externals: + - spec: rocsolver@5.4.0 + prefix: /opt/rocm-5.4.0/rocsolver + rocsparse: + variants: amdgpu_target=gfx90a + version: [5.4.0] + buildable: false + externals: + - spec: rocsparse@5.4.0 + prefix: /opt/rocm-5.4.0 + roctracer-dev: + version: [5.4.0] + buildable: false + externals: + - spec: roctracer-dev@5.4.0 + prefix: /opt/rocm-4.5.2 + rocprofiler-dev: + version: [5.4.0] + buildable: false + externals: + - spec: rocprofiler-dev@5.4.0 + prefix: /opt/rocm-5.4.0 + rocprim: + version: [5.4.0] + buildable: false + externals: + - spec: rocprim@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-smi: + version: [5.4.0] + buildable: false + externals: + - spec: rocmsmi@5.4.0 + prefix: /opt/rocm-5.4.0/rocm_smi + rocm-smi-lib: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-smi-lib@5.4.0 + prefix: /opt/rocm-5.4.0/rocm_smi + miopen-hip: + version: [5.4.0] + buildable: false + externals: + - spec: miopen-hip@5.4.0 + prefix: /opt/rocm-5.4.0 + rocalution: + version: [5.4.0] + buildable: false + externals: + - spec: rocalution@5.4.0 + prefix: /opt/rocm-5.4.0 + rocm-gdb: + version: [5.4.0] + buildable: false + externals: + - spec: rocm-gdb@5.4.0 + prefix: /opt/rocm-5.4.0 + slurm: + buildable: false + version: [20.11.3] + target: [] + compiler: [] + providers: {} + externals: + - spec: slurm@20.11.3 + prefix: /usr + openssl: + buildable: false + version: [1.1.1] + target: [] + providers: {} + externals: + - spec: openssl@1.1.1 + prefix: /usr + libfabric: + version: [1.13.0.0] + buildable: false + externals: + - spec: libfabric@1.13.0.0 + prefix: /opt/cray/libfabric/1.13.0.0 + llvm: + version: [12.0.1] + buildable: false + externals: + - spec: llvm@12.0.1 + prefix: /opt/cray/pe/craype/2.7.13/ + all: + buildable: true + version: [] + target: [zen3] + providers: + mpi: [cray-mpich@8.1.16, cray-mpich@8.1.18, cray-mpich@8.1.23] + lapack: [cray-libsci@21.08.1.2, cray-libsci@22.08.1.1] + blas: [cray-libsci@21.08.1.2, cray-libsci@22.08.1.1] + scalapack: [netlib-scalapack] + compiler: [cce@15.0.0] + view: false + upstreams: + olcf: + install_tree: /sw/crusher/spack-envs/base/opt + modules: + lmod: /sw/crusher/spack-envs/base/modules/spack + tcl: /sw/crusher/spack-envs/base/modules/flat + config: + install_tree: + root: /lustre/orion/geo127/world-shared/tpls/2023-04-04/ + projections: + all: install-frontier-{compiler.name}@{compiler.version}/{name}-{version} + build_stage: + - $spack/var/spack/spack-stage/cce-15.0.0/ + compilers: + - compiler: + spec: cce@15.0.0 + paths: + cc: /opt/cray/pe/craype/2.7.16/bin/cc + cxx: /opt/cray/pe/craype/2.7.16/bin/CC + f77: /opt/cray/pe/craype/2.7.16/bin/ftn + fc: /opt/cray/pe/craype/2.7.16/bin/ftn + operating_system: sles15 + target: any + modules: + - PrgEnv-cray + - cce/15.0.0 + - cray-mpich/8.1.23 + - craype-accel-amd-gfx90a + environment: + append_path: + PKG_CONFIG_PATH: /usr/lib64/pkgconfig + set: + MPICH_GPU_SUPPORT_ENABLED: 1 + # PE_MPICH_GTL_DIR_amd_gfx90a: "-L${CRAY_MPICH_ROOTDIR}/gtl/lib" + # PE_MPICH_GTL_LIBS_amd_gfx90a: "-lmpi_gtl_hsa" + specs: + - conduit%cce@15.0.0 ~blt_find_mpi ~fortran ~hdf5_compat ^hdf5@1.12.2+hl+mpi~fortran + - camp%cce@15.0.0 +rocm amdgpu_target=gfx90a ^hip@5.4.0 + - raja@2022.03.0%cce@15.0.0 +rocm amdgpu_target=gfx90a ^hip@5.4.0 + - umpire@2022.03.0%cce@15.0.0 +rocm amdgpu_target=gfx90a ^hip@5.4.0 + - chai@2022.03.0%cce@15.0.0~examples~openmp+raja+rocm amdgpu_target=gfx90a ^raja@2022.03.0+rocm + amdgpu_target=gfx90a ^umpire@2022.03.0~examples+rocm amdgpu_target=gfx90a ^hip@5.4.0 + ^blt@0.5.2 ^hsa-rocr-dev@5.4.0 ^llvm-amdgpu@5.4.0 ^camp+rocm amdgpu_target=gfx90a + - pugixml%cce@15.0.0 + - fmt@8.0.1%cce@15.0.0 + - suite-sparse@5.10.1%cce@15.0.0~cuda~graphblas~openmp+pic~tbb + - caliper@2.8.0%cce@15.0.0~adiak~libdw+rocm amdgpu_target=gfx90a ^hip@5.4.0 ^hsa-rocr-dev@5.4.0 + ^llvm-amdgpu@5.4.0 \ No newline at end of file