Add rocm CI

.github/workflows/docker_build_tpls.yml

@@ -81,6 +81,11 @@ jobs:
           #   SPEC: "~pygeosx~openmp cuda_arch=70 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers"
           #   INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU
           #   RUNS_ON: ubuntu-latest
+          - name: Ubuntu (24.04, amdclang 19 + ROCm 6.4.3)
+            DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3
+            TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile
+            DOCKER_ROOT_IMAGE: rocm/dev-ubuntu-24.04:6.4.3
+            RUNS_ON: ubuntu-latest
 
     steps:
     - name: Checkout

docker/spack.yaml

@@ -6,8 +6,7 @@ spack:
         all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
     misc_cache: $spack/../misc_cache
     test_stage: $spack/../test_stage
-    build_stage::
-    - $spack/../build_stage
+    build_stage: $spack/../build_stage
 
   # Regular TPLs do not need views
   view: false
@@ -18,6 +17,15 @@ spack:
   - ../versions.yaml
 
   toolchains:
+    amdclang-19:
+      - spec: '%[virtuals=c]llvm-amdgpu@6.4.3'
+        when: '%c'
+      - spec: '%[virtuals=cxx]llvm-amdgpu@6.4.3'
+        when: '%cxx'
+      - spec: '%[virtuals=fortran]gcc@13.3.0'
+        when: '%fortran'
+      - spec: '%openmpi@4.1.2'
+        when: '%mpi'
     clang-15:
       - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb'
         when: '%c'
@@ -80,22 +88,34 @@ spack:
         when: '%fortran'
       - spec: '%openmpi@4.1.2'
         when: '%mpi'
+    gcc-13:
+      - spec: cxxflags='-pthread'
+      - spec: cflags='-pthread'
+      - spec: '%c=gcc@13.3.0'
+        when: '%c'
+      - spec: '%cxx=gcc@13.3.0'
+        when: '%cxx'
+      - spec: '%fortran=gcc@13.3.0'
+        when: '%fortran'
+      - spec: '%openmpi@4.1.2'
+        when: '%mpi'
 
   packages:
     all:
       target: [x86_64]
 
+    # Require
     mpi:
       require:
       - openmpi
-
     blas:
       require:
       - "netlib-lapack"
     lapack:
       require:
       - "netlib-lapack"
 
+    # Compiler packages
     llvm:
       externals:
       - spec: llvm@10.0.0+clang~flang~lld~lldb
@@ -110,7 +130,15 @@ spack:
           compilers:
             c: /usr/bin/clang-15
             cxx: /usr/bin/clang++-15
-
+    llvm-amdgpu:
+      buildable: false
+      externals:
+      - spec: llvm-amdgpu@6.4.3
+        prefix: /usr
+        extra_attributes:
+          compilers:
+            c: /usr/bin/amdclang
+            cxx: /usr/bin/amdclang++
     gcc:
       externals:
       - spec: gcc@9.5.0 languages:='c,c++,fortran'
@@ -141,7 +169,118 @@ spack:
             c: /usr/bin/gcc-12
             cxx: /usr/bin/g++-12
             fortran: /usr/bin/gfortran-12
+      - spec: gcc@13.3.0 languages:='c,c++,fortran'
+        prefix: /usr
+        extra_attributes:
+          compilers:
+            c: /usr/bin/gcc-13
+            cxx: /usr/bin/g++-13
+            fortran: /usr/bin/gfortran-13
+
+    # MPI
+    openmpi:
+      externals:
+      - spec: openmpi@4.1.2 %llvm
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+        modules: [mpi]
+      - spec: openmpi@4.1.2 %llvm-amdgpu@6.4.3
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+      - spec: openmpi@4.1.2 %gcc@9.5.0
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+      - spec: openmpi@4.1.2 %gcc@10.5.0
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+      - spec: openmpi@4.1.2 %gcc@11.4.0
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+      - spec: openmpi@4.1.2 %gcc@12.3.0
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+      - spec: openmpi@4.1.2 %gcc@13.3.0
+        prefix: /usr/lib/x86_64-linux-gnu/openmpi
+
+    # CUDA packages
+    cuda:
+      buildable: False
+      externals:
+      - spec: cuda@11.8.0 +allow-unsupported-compilers
+        prefix: /usr/local/cuda
+
+    # ROCm packages
+    hip:
+      buildable: false
+      externals:
+      - spec: hip@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocprim:
+      buildable: false
+      externals:
+      - spec: rocprim@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocsparse:
+      buildable: false
+      externals:
+      - spec: rocsparse@6.4.3
+        prefix: /opt/rocm-6.4.3
+    roctracer:
+      buildable: false
+      externals:
+      - spec: roctracer@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocblas:
+      buildable: false
+      externals:
+      - spec: rocblas@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocrand:
+      buildable: false
+      externals:
+      - spec: rocrand@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocsolver:
+      buildable: false
+      externals:
+      - spec: rocsolver@6.4.3
+        prefix: /opt/rocm-6.4.3
+    rocthrust:
+      buildable: false
+      externals:
+      - spec: rocthrust@6.4.3
+        prefix: /opt/rocm-6.4.3
+    hipblas:
+      buildable: false
+      externals:
+      - spec: hipblas@6.4.3 +rocm
+        prefix: /opt/rocm-6.4.3
+    hipsparse:
+      buildable: false
+      externals:
+      - spec: hipsparse@6.4.3 +rocm
+        prefix: /opt/rocm-6.4.3
+    hipfft:
+      buildable: false
+      externals:
+      - spec: hipfft@6.4.3 +rocm
+        prefix: /opt/rocm-6.4.3
+    hipsolver:
+      buildable: false
+      externals:
+      - spec: hipsolver@6.4.3 +rocm
+        prefix: /opt/rocm-6.4.3
+    hiprand:
+      buildable: false
+      externals:
+      - spec: hiprand@6.4.3 +rocm
+        prefix: /opt/rocm-6.4.3
+    rocm-device-libs:
+      buildable: false
+      externals:
+      - spec: rocm-device-libs@6.4.3
+        prefix: /opt/rocm-6.4.3
+    hsa-rocr-dev:
+      buildable: false
+      externals:
+      - spec: hsa-rocr-dev@6.4.3
+        prefix: /opt/rocm-6.4.3
 
+    # System packages
     autoconf:
       version: [2.71]
       buildable: false
@@ -160,11 +299,6 @@ spack:
       externals:
       - spec: cmake@3.28.3
         prefix: /usr/local
-    cuda:
-      buildable: False
-      externals:
-      - spec: cuda@11.8.0 +allow-unsupported-compilers
-        prefix: /usr/local/cuda
     findutils:
       version: [4.7.0]
       buildable: false
@@ -181,29 +315,10 @@ spack:
       externals:
       - spec: mpfr@6.0.2
         prefix: /usr
-
     netlib-lapack:
       buildable: false
       externals:
-      - spec: netlib-lapack@3.10.0
-        prefix: /usr
-    netlib-blas:
-      buildable: false
-      externals:
-      - spec: netlib-blas@3.10.0
-        prefix: /usr
-    openmpi:
-      externals:
-      - spec: openmpi@4.1.2 %llvm
-        prefix: /usr
-        modules: [mpi]
-      - spec: openmpi@4.1.2 %gcc@9.5.0
-        prefix: /usr
-      - spec: openmpi@4.1.2 %gcc@10.5.0
-        prefix: /usr
-      - spec: openmpi@4.1.2 %gcc@11.4.0
-        prefix: /usr
-      - spec: openmpi@4.1.2 %gcc@12.3.0
+      - spec: netlib-lapack@3.12.0
         prefix: /usr
     perl:
       buildable: false
@@ -250,3 +365,59 @@ spack:
       externals:
       - spec: zlib@1.2.11
         prefix: /usr
+    z3:
+      buildable: false
+      externals:
+      - spec: z3@4.8.12
+        prefix: /usr
+    gettext:
+      buildable: false
+      externals:
+      - spec: gettext@0.21
+        prefix: /usr
+    bison:
+      buildable: false
+      externals:
+      - spec: bison@3.8.2
+        prefix: /usr
+    # Rely on spack to build a newer flex for scotch
+    # flex:
+    #   buildable: false
+    #   externals:
+    #   - spec: flex@2.6.4
+    #     prefix: /usr
+    bzip2:
+      buildable: false
+      externals:
+      - spec: bzip2@1.0.8
+        prefix: /usr
+    help2man:
+      buildable: false
+      externals:
+      - spec: help2man@1.49.3
+        prefix: /usr
+    libtool:
+      buildable: false
+      externals:
+      - spec: libtool@2.4.7
+        prefix: /usr
+    gmp:
+      buildable: false
+      externals:
+      - spec: gmp@6.3.0
+        prefix: /usr
+    pugixml:
+      buildable: false
+      externals:
+      - spec: pugixml@1.13.0
+        prefix: /usr
+    gmake:
+      buildable: false
+      externals:
+      - spec: gmake@4.4.1
+        prefix: /usr
+    ninja:
+      buildable: false
+      externals:
+      - spec: ninja@1.11.1
+        prefix: /usr

docker/tpl-centos-gcc-cuda.Dockerfile

@@ -1,5 +1,5 @@
 # NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments
-ARG TMP_DIR=/tmp 
+ARG TMP_DIR=/tmp
 ARG SRC_DIR=$TMP_DIR/thirdPartyLibs
 ARG BLD_DIR=$TMP_DIR/build
 
@@ -11,12 +11,12 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR
 
 RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
     sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo 
+    sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo
 
 # Using gcc 8.3.1 provided by the Software Collections (SCL).
 RUN yum install -y \
     centos-release-scl
-    
+
 # Modify the SCLo repository configuration
 RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \
     sed -i 's|^baseurl=http://mirror.centos.org/centos/\$releasever/sclo/\$basearch/rh|baseurl=http://vault.centos.org/7.9.2009/sclo/x86_64/rh|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \
@@ -25,7 +25,7 @@ RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo
 
 # Install necessary tools and update the system
 RUN yum -y update && \
-    yum -y install yum-utils    
+    yum -y install yum-utils
 
 RUN yum install -y \
     devtoolset-8-gcc \
@@ -48,6 +48,10 @@ RUN yum -y install \
     # xz \
     unzip \
     bzip2 \
+    flex \
+    bison \
+    gettext \
+    help2man \
     gnupg \
     && pip3 install virtualenv
 
@@ -65,6 +69,8 @@ ARG BLD_DIR
 RUN yum install -y \
     tbb-devel \
     make \
+    autopoint \
+    autotools-dev \
     bc \
     file \
     patch \

docker/tpl-rockylinux-clang-cuda-12.Dockerfile

@@ -13,7 +13,7 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR
 RUN dnf clean all && \
     dnf -y update && \
     dnf -y install \
-        which \ 
+        which \
         clang-17.0.6 \
         gcc-toolset-13 \
         python3 \
@@ -28,7 +28,11 @@ RUN dnf clean all && \
         unzip \
         mpfr-devel \
         bzip2 \
+        flex \
+        bison \
+        gettext \
         gnupg \
+        libtool \
         xz \
         python3-virtualenv
 

docker/tpl-rockylinux-gcc-cuda-12.Dockerfile

@@ -26,7 +26,11 @@ RUN dnf clean all && \
         unzip \
         mpfr-devel \
         bzip2 \
+        flex \
+        bison \
+        gettext \
         gnupg \
+        libtool \
         xz \
         python3-virtualenv