diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 20529b41..6b7a1531 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -61,10 +61,11 @@ jobs: DOCKER_REPOSITORY: geosx/rockylinux8-clang17-cuda12.5 TPL_DOCKERFILE: docker/tpl-rockylinux-clang-cuda-12.Dockerfile RUNS_ON: Runner_4core_16GB - - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10) - DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18 - TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile - DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4 + - name: Pangea 3 (AlmaLinux 8.8, gcc 11.4.0, open-mpi 4.1.6, cuda 11.8.0, openblas 0.3.18) + DOCKER_REPOSITORY: geosx/pangea3-gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0 + TPL_DOCKERFILE: docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile + DOCKER_ROOT_IMAGE: onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 + INSTALL_DIR_ROOT: /workrd/SCR/NUM/GEOS_environment/p3/install/tpl HOST_ARCH: ppc64le RUNS_ON: streak2 - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10, zlib 1.2.11) diff --git a/docker/TotalEnergies/Pangea3-base.Dockerfile b/docker/TotalEnergies/Pangea3-base.Dockerfile deleted file mode 100644 index d3510d82..00000000 --- a/docker/TotalEnergies/Pangea3-base.Dockerfile +++ /dev/null @@ -1,191 +0,0 @@ -# This Dockerfile is used to build a docker image reproducing the Pangea installation over a ppc64le architecture: -# It is not directly callable by the TPL ci but the built image is. - -# syntax=docker/dockerfile:1 -FROM ppc64le/almalinux:8 - -# Install other needed packages -RUN dnf install -y \ - # gcc deps \ - libmpc-devel.ppc64le glibc-devel \ - # mpirun deps \ - librdmacm hwloc \ - git git-lfs \ - python38-devel python38-numpy \ - zlib-devel \ - make \ - bc \ - file \ - # Scotch deps \ - bison \ - flex \ - # vtk deps \ - patch && \ - git-lfs install #&& alternatives --set python /usr/bin/python3 - -# copy pangea tree for modules needed by TPLs and export environment variables - -## Temporary local variables needed buy several modules -ARG MODULE_PATH="/data_local/sw" - -ARG SPACK_PATH="spack/0.17.0/opt/spack/linux-rhel8-power9le" - -ARG COMPILER="gcc" -ARG DEFAULT_COMPILER_VER="8.4.1" -ARG SPACK_COMPILER_VER=$DEFAULT_COMPILER_VER -ARG MODULE_COMPILER_VER="9.4.0" - -ARG SPACK_COMPILER=$COMPILER-$SPACK_COMPILER_VER - -## liblustre -COPY ./tarball/liblustreapi.so.1 /lib64/ - -## CMake -ADD ./tarball/cmake-*.tgz / - -### Environment variables to export -ENV PATH="/data_local/appli_local/MTS/GEOSX/cmake/3.26.4/bin:${PATH}" - -## gcc -ADD ./tarball/gcc-*.tgz / - -### Temporary local variables -ARG GCC_VER=$COMPILER-$MODULE_COMPILER_VER -ARG GCC_DIR="$GCC_VER-xe5cqnyajaqz75up3gflln5zlj2rue5v" - -### Environment variables to export -ENV CPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/include:${CPATH}" \ - LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib:\ -$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_LIBRARY_PATH}" \ - LD_RUN_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib:\ -$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/lib64:${LD_RUN_PATH}" \ - MANPATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/share/man:${MANPATH}" \ - PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR/bin:${PATH}" \ - CC=gcc \ - CXX=g++ \ - F77=gfortran \ - F90=gfortran \ - FC=gfortran \ - GCC_ROOT=$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$GCC_DIR - -## ompi -ADD ./tarball/ompi-*.tgz / - -### Temporary local variables -ARG MPI_VER="4.1.2" -ARG MPI_DIR="openmpi/$MPI_VER" -ARG MPI_PREFIX="OMPI" - -### Environment variables to export -ENV CPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/include:${CPATH}" \ - LD_LIBRARY_PATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/lib:${LD_LIBRARY_PATH}" \ - MANPATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/share/man:${MANPATH}" \ - PATH="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER/bin:${PATH}" \ - MPI_ROOT="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER" \ - ${MPI_PREFIX}_CC="gcc" \ - ${MPI_PREFIX}_CXX="g++" \ - ${MPI_PREFIX}_F77="gfortran" \ - ${MPI_PREFIX}_F90="gfortran" \ - ${MPI_PREFIX}_FC="gfortran" \ - OPAL_PREFIX="$MODULE_PATH/$MPI_DIR/env/$SPACK_COMPILER" \ - OMPI_MCA_btl="self,vader,openib" \ - OMPI_MCA_btl_openib_allow_ib="true" \ - OMPI_MCA_btl_openib_warn_default_gid_prefix="0" - -## Cuda -ADD ./tarball/cuda-*.tgz / - -### Temporary local variables -ARG LIBCUDA_VER=450.156.00 - -ARG CUDA_VER_MAJ="11" -ARG CUDA_VER_MIN="5" -ARG CUDA_VER_PATCH="0" -ARG CUDA_VER="$CUDA_VER_MAJ.$CUDA_VER_MIN.$CUDA_VER_PATCH" -ARG CUDA_DIR="cuda/$CUDA_VER" - -ARG CUPTI_DIR="$CUDA_DIR/extras/CUPTI" - -COPY ./tarball/libcuda.so.$LIBCUDA_VER /usr/lib64 -RUN ln -s /usr/lib64/libcuda.so.$LIBCUDA_VER /usr/lib64/libcuda.so.1 - -### Environment variables to export -ENV CPATH="$MODULE_PATH/$CUDA_DIR/include:\ -$MODULE_PATH/$CUPTI_DIR/include:${CPATH}" \ - LIBRARY_PATH="$MODULE_PATH/$CUDA_DIR/lib64/stubs:\ -$MODULE_PATH/$CUDA_DIR/lib64:\ -$MODULE_PATH/$CUPTI_DIR/lib64:${LIBRARY_PATH}" \ - LD_LIBRARY_PATH="$MODULE_PATH/$CUDA_DIR/lib64:\ -$MODULE_PATH/$CUPTI_DIR/lib64:${LD_LIBRARY_PATH}" \ - LD_RUN_PATH="$MODULE_PATH/$CUDA_DIR/lib64:${LD_RUN_PATH}" \ - MANPATH="$MODULE_PATH/$CUDA_DIR/doc/man:${MANPATH}" \ - PATH="$MODULE_PATH/$CUDA_DIR/bin:\ -$MODULE_PATH/$CUDA_DIR/samples/bin/ppc64le/linux/release:${PATH}" \ - CUBLAS_ROOT="$MODULE_PATH/$CUDA_DIR" \ - CUDA_ROOT="$MODULE_PATH/$CUDA_DIR" \ - CUDA_HOME="$MODULE_PATH/$CUDA_DIR" \ - CUDA_PATH="$MODULE_PATH/$CUDA_DIR" \ - CUDA_VERSION="$CUDA_VER_MAJ.$CUDA_VER_MIN" \ - NVHPC_CUDA_HOME="$MODULE_PATH/$CUDA_DIR" - -## Openblas -ADD ./tarball/openblas-*.tgz / - -### Temporary local variables -ARG SPACK_COMPILER=$COMPILER-$MODULE_COMPILER_VER - -ARG BLAS_VER_MAJ="0" -ARG BLAS_VER_MIN="3" -ARG BLAS_VER_PATCH="18" -ARG BLAS_DISTRIB="openblas" -ARG BLAS_VER="$BLAS_VER_MAJ.$BLAS_VER_MIN.$BLAS_VER_PATCH" -ARG BLAS_DIR="$BLAS_DISTRIB-$BLAS_VER-vk36pzksytuhylqesg4cca7667np5sjp" - -### Environment variables to export -ENV LD_LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LD_LIBRARY_PATH}" \ - LIBRARY_PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/lib:${LIBRARY_PATH}" \ - PATH="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR/bin:${PATH}" \ - OPENBLAS_ROOT="$MODULE_PATH/$SPACK_PATH/$SPACK_COMPILER/$BLAS_DIR" - -## lsf -ADD ./tarball/lsf-*.tgz / - -### Temporary local variables -ARG GLIBC_DIR="linux3.10-glibc2.17-ppc64le" - -ARG LSF_VER_MAJ="10" -ARG LSF_VER_MIN="1" -ARG LSF_VER="$LSF_VER_MAJ.$LSF_VER_MIN" -ARG LSF_DIR="lsf/$LSF_VER/$GLIBC_DIR" - -### Environment variables to export -ENV LD_LIBRARY_PATH="$MODULE_PATH/$LSF_DIR/lib:${LD_LIBRARY_PATH}" \ - PATH="$MODULE_PATH/$LSF_DIR/etc:$MODULE_PATH/$LSF_DIR/bin:${PATH}" - -# Uncomment the two following lines to test the TPLs build -# -#WORKDIR /root -#RUN git clone --depth=1 https://github.com/GEOS-DEV/thirdPartyLibs.git && \ -# cd thirdPartyLibs && git submodule init && git submodule update - -#RUN cd thirdPartyLibs && python3 scripts/config-build.py \ -# --hostconfig=host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake \ -# --buildtype=Release --installpath=/opt/tpls -DNUM_PROC=32 && \ -# make -C build-pangea3-gcc8.4.1-openmpi-4.1.2-release -j && \ -# cd .. && rm -rf thirdPartyLibs - -# Install tools needed by geos ci -RUN dnf -y --enablerepo=powertools install \ - ninja-build \ - openssh-clients \ - texlive \ - graphviz \ - libxml2 - -# build sccache from source as prebuilt binary is not available for current archi / OS couple -RUN dnf makecache --refresh && dnf -y install cargo openssl-devel -RUN cargo install sccache --locked && mkdir -p /opt/sccache/ && cp -r /root/.cargo/bin /opt/sccache/ -RUN dnf remove -y cargo openssl-devel - -### Environment variables to export -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/Pangea3.Dockerfile b/docker/TotalEnergies/Pangea3.Dockerfile deleted file mode 100644 index bfad0260..00000000 --- a/docker/TotalEnergies/Pangea3.Dockerfile +++ /dev/null @@ -1,89 +0,0 @@ -# Temporary local variables dedicated to the TPL build -ARG TMP_DIR=/tmp -ARG SRC_DIR=$TMP_DIR/thirdPartyLibs -ARG BLD_DIR=$TMP_DIR/build - -# The docker base image has to be pangea3-almalinux8-* -ARG DOCKER_ROOT_IMAGE - -FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain -ARG SRC_DIR -ARG BLD_DIR - -# Install additional required packages for spack -RUN dnf clean all && \ - dnf -y update && \ - dnf -y install \ - autoconf \ - automake \ - libtool \ - bzip2 \ - unzip - -# All the environment variables defined in this Dockerfile -# (GEOSX_TPL_DIR but also compiler information like CC, CXX...) -# are part of the image contract (otherwise ARG is used). -# GEOSX use them so consider modifying their names with care. -# -# The installation directory is provided as a docker build argument. -# We forward it using an environment variable. -ARG INSTALL_DIR -ENV GEOSX_TPL_DIR=$INSTALL_DIR - -# Get host config file from docker build arguments -#ARG HOST_CONFIG - -FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain - -# Run uberenv -# Have to create install directory first for uberenv -# -k flag is to ignore SSL errors -RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ - mkdir -p ${GEOSX_TPL_DIR} && \ -# Create symlink to existing libraries sp spack can find - ln -s /usr/lib64/libhwloc.so.15 /usr/lib64/libhwloc.so && \ - ln -s /usr/lib64/libibverbs.so.1.14.48.0 /usr/lib64/libibverbs.so && \ - ln -s /usr/lib64/libnsl.so.2.0.0 /usr/lib64/libnsl.so && \ - ln -s /usr/lib64/librdmacm.so.1.3.48.0 /usr/lib64/librdmacm.so && \ - ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so && \ -# Run uberenv - ./scripts/uberenv/uberenv.py \ - --spec "%gcc@9.4.0+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers ^caliper~gotcha~sampler~libunwind~libdw~papi" \ - --spack-env-file=${SRC_DIR}/docker/pangea-spack.yaml \ - --project-json=.uberenv_config.json \ - --prefix ${GEOSX_TPL_DIR} \ - -k && \ -# Remove host-config generated for LvArray - rm lvarray* && \ -# Rename and copy spack-generated host-config to root directory - cp *.cmake /spack-generated-wave-solver-only.cmake && \ -# Remove extraneous spack files - cd ${GEOSX_TPL_DIR} && \ - rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ - -# Build only the wave solver for Pangea 3 -RUN echo 'set ( GEOS_ENABLE_CONTACT OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_FLUIDFLOW OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_INDUCEDSEISMICITY OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_MULTIPHYSICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_SIMPLEPDE OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_SOLIDMECHANICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ - echo 'set ( GEOS_ENABLE_SURFACEGENERATION OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake - - -# Extract only TPL's from previous stage -FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain - -COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR - -# Extract the generated host-config -COPY --from=tpl_toolchain /spack-generated-wave-solver-only.cmake / - -# Regenerate symlinks for existing libraries -RUN ln -s /usr/lib64/libhwloc.so.15 /usr/lib64/libhwloc.so && \ - ln -s /usr/lib64/libibverbs.so.1.14.48.0 /usr/lib64/libibverbs.so && \ - ln -s /usr/lib64/libnsl.so.2.0.0 /usr/lib64/libnsl.so && \ - ln -s /usr/lib64/librdmacm.so.1.3.48.0 /usr/lib64/librdmacm.so && \ - ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so - -ENV SCCACHE=/opt/sccache/bin/sccache diff --git a/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile new file mode 100644 index 00000000..bc3b7dda --- /dev/null +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc-openmpi-openblas.Dockerfile @@ -0,0 +1,82 @@ +####################################### +# Pangea 3 image : gcc - openmpi - openblas +####################################### +# +# Installs : +# - openmpi = 4.1.6 +# - openblas = 0.3.18 +# - cuda = 11.8.0 +# +####################################### +# +# Description : +# - the image is based on onetechssc/pangea3:gcc11.4.0_v1.0 built from pangea3-gcc.Dockerfile +# - this image is deployed as onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# - openblas is installed via spack (as done for Pangea 3) +# - openmpi and cuda are installed via spack (part of system image on Pangea 3) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 -f pangea3-gcc-openmpi-openblas.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea3_gcc_openmpi_openblas -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# - podman exec -it pangea3_gcc_openmpi_openblas /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 docker://docker.io/onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA3 - GCC-HPCX-MKL +FROM docker.io/onetechssc/pangea3:gcc11.4.0_v1.0 AS pangea3 +# ------ +# LABELS +LABEL description="Pangea 3 image : gcc - cmake - python - openmpi - openblas - cuda" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG OPENBLAS_VERSION=0.3.18 +ARG OPENMPI_VERSION=4.1.6 +ARG CUDA_VERSION=11.8.0 +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# openblas (with openmp) +RUN spack install openblas@$OPENBLAS_VERSION threads=openmp %gcc@$GCC_VERSION +# cuda +RUN spack install cuda@$CUDA_VERSION %gcc@$GCC_VERSION +# openmpi +RUN spack install openmpi@$OPENMPI_VERSION +cuda cuda_arch=70 %gcc@$GCC_VERSION +# numpy with openblas +RUN spack install py-numpy %gcc@$GCC_VERSION ^python@$PYTHON_VERSION ^openblas@$OPENBLAS_VERSION +# virtualenv (openssl error when using curl) +RUN spack config --scope defaults add config:url_fetch_method:urllib &&\ + spack install py-virtualenv %gcc@$GCC_VERSION ^python@$PYTHON_VERSION &&\ + spack config --scope defaults add config:url_fetch_method:curl +# sccache +RUN dnf makecache --refresh && dnf -y install cargo openssl-devel &&\ + cargo install sccache --locked && mkdir -p /opt/sccache/ && cp -r /root/.cargo/bin /opt/sccache/ &&\ + dnf remove -y cargo openssl-devel && dnf clean all +# flex +RUN spack install flex %gcc@$GCC_VERSION +# zlib +RUN spack install zlib %gcc@$GCC_VERSION +# pugixml +RUN spack install pugixml %gcc@$GCC_VERSION +# ------ +# ENV +# sccache +ENV SCCACHE=/opt/sccache/bin/sccache +# create env script +RUN < /root/.setup_env.sh +. /opt/spack/share/spack/setup-env.sh +spack load gcc@$GCC_VERSION python@$PYTHON_VERSION cmake@$CMAKE_VERSION \ + openblas@$OPENBLAS_VERSION openmpi@$OPENMPI_VERSION cuda@$CUDA_VERSION \ + py-numpy py-virtualenv \ + flex zlib +EOF +RUN chmod +x /root/.setup_env.sh \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-3/pangea3-gcc.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-gcc.Dockerfile new file mode 100644 index 00000000..ed8b808b --- /dev/null +++ b/docker/TotalEnergies/pangea-3/pangea3-gcc.Dockerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.4 + +####################################### +# Pangea 3 image : gcc - cmake - python +####################################### +# +# Installs : +# - gcc = 11.4.0 +# - cmake = 3.27.9 +# - python = 3.11.7 +# +####################################### +# +# Description : +# - the image is based on spack/almalinux8:latest (Almalinux 8 image with spack installed) +# - this image is deployed as onetechssc/pangea3:gcc11.4.0_v1.0 +# - gcc, cmake, python and wget are installed via spack (as done for Pangea 3) +# +# Usage : +# build the image: +# - podman build --format docker -t onetechssc/pangea3:gcc11.4.0_v1.0 -f pangea3-gcc.Dockerfile -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem . +# run the image: +# - podman run -it --detach --privileged --name pangea3_gcc -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/pangea3:gcc11.4.0_v1.0 +# - podman exec -it pangea3_gcc /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/pangea3:gcc11.4.0_v1.0 docker://docker.io/onetechssc/pangea3:gcc11.4.0_v1.0 +# +####################################### + +# ------------------------------------- +# PANGEA3 - BASE [GCC - CMAKE - PYTHON] +# install tools via spack for linux/ppc64le architecture and almalinux platform +FROM --platform=linux/ppc64le docker.io/spack/almalinux8:latest AS pangea3-base +# ------ +# LABELS +LABEL description="Pangea 3 image : gcc - cmake - python" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG GCC_VERSION=11.4.0 +ARG CMAKE_VERSION=3.27.9 +ARG PYTHON_VERSION=3.11.7 +# ------ +# INSTALL +# spack config (make sure you survive our wonderfull proxy) +RUN spack config --scope defaults add config:connect_timeout:120 +RUN spack config --scope defaults add config:url_fetch_method:curl +# gcc +RUN spack install gcc@$GCC_VERSION +RUN . /opt/spack/share/spack/setup-env.sh &&\ + spack load gcc@$GCC_VERSION &&\ + spack compiler find +# python +RUN spack install python@$PYTHON_VERSION %gcc@$GCC_VERSION +# cmake (openssl error when using curl) +RUN spack config --scope defaults add config:url_fetch_method:urllib &&\ + spack install cmake@$CMAKE_VERSION %gcc@$GCC_VERSION &&\ + spack config --scope defaults add config:url_fetch_method:curl +# wget +RUN spack install wget %gcc@$GCC_VERSION +# ------ +# ENV +ENV GCC_VERSION=${GCC_VERSION} +ENV CMAKE_VERSION=${CMAKE_VERSION} +ENV PYTHON_VERSION=${PYTHON_VERSION} \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-3/pangea3-spack.yaml b/docker/TotalEnergies/pangea-3/pangea3-spack.yaml new file mode 100644 index 00000000..4e0139aa --- /dev/null +++ b/docker/TotalEnergies/pangea-3/pangea3-spack.yaml @@ -0,0 +1,220 @@ +# This is a Spack Environment file for Pangea-3 docker image. +# - replicate Pangea-3 environement as close as possible +# while targeting ppc64le for ci/cd. +# - used with onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# docker image +# +# It describes a set of packages to be installed, along with +# configuration settings. +# +# Run command from the top-level of the repository: +# ./scripts/uberenv/uberenv.py \ +# --spec "%gcc@11.4.0 +cuda+pygeosx+docs \ +# cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" \ +# --spack-env-file=scripts/spack_configs/pangea-3/spack.yaml \ +# --project-json=.uberenv_config.json \ +# --prefix ${GEOS_TPL_DIR} +spack: + config: + # make sure you survive our wonderfull proxy when testing on prem + connect_timeout: 120 + url_fetch_method: curl + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage:: + - $spack/../build_stage + + # Regular TPLs do not need views + view: false + + ############# + # COMPILERS # + ############# + + compilers: + - compiler: + spec: gcc@=11.4.0 + paths: + cc: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gcc-11.4.0-e2xjn3x3bppfveg6qawy43js2j2pyo7x/bin/gcc + cxx: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gcc-11.4.0-e2xjn3x3bppfveg6qawy43js2j2pyo7x/bin/g++ + f77: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gcc-11.4.0-e2xjn3x3bppfveg6qawy43js2j2pyo7x/bin/gfortran + fc: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gcc-11.4.0-e2xjn3x3bppfveg6qawy43js2j2pyo7x/bin/gfortran + flags: + cflags: -mcpu=power9 -mtune=power9 + cxxflags: -mcpu=power9 -mtune=power9 + operating_system: almalinux8 + target: ppc64le + modules: [] + environment: + set: # Needed for scotch + BISON: bison + FLEX: flex + extra_rpaths: [] + + ############# + # PACKAGES # + ############# + + packages: + all: + target: [ppc64le] + compiler: [gcc] + providers: + blas: [openblas] + lapack: [openblas] + mpi: [openmpi] + + #### + # make sure spack doesn't rebuild mpi, blas and lapack libs + # without this it will try to build mpich and openblas + mpi: + buildable: false + blas: + buildable: false + lapack: + buildable: false + + #### + # spec of packages to build for + + # v0.6.2 + blt: + require: "@git.9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35=develop" + # master - 10/18/24 + hypre: + require: "@git.c893886d15eb57e87dd36efec23693ece3ddc88e" + # v2024.07.0 + chai: + require: "@git.df7741f1dbbdc5fff5f7d626151fdf1904e62b19=develop" + # v2024.07.0 + umpire: + require: "@git.abd729f40064175e999a83d11d6b073dac4c01d2=develop" + # v2024.07.0 + raja: + require: "@git.4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1=develop" + # v2024.07.0 + camp: + require: "@git.0f07de4240c42e0b38a8d872a20440cb4b33d9f5=main" + # v2.12.0 + caliper: + require: "@git.287b7f3ad2d12f520aad04268d44f353cd05403c" + # v0.9.2 + conduit: + require: "@git.ad86e316ad56a75c099d30ca5ce75cff275b5924=develop" + # master - 04/12/20 + uncrustify: + require: "@git.401a4098bce9dcc47e024987403f2d59d9ba7bd2" + # master - 04/26/20 + superlu-dist: + require: "@git.0f6efc377df2440c235452d13d28d2c717f832a1" + + #### + # spec of spack packages to reuse + berkeley-db: + externals: + - spec: berkeley-db@18.1.40 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/berkeley-db-18.1.40-tx2xdea6z7gfdixr2rf6ux2s5kx46h4h + buildable: false + bison: + externals: + - spec: bison@3.8.2 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/bison-3.8.2-f6gtrzvi6642zjjjyc5nf5bwridswoos + buildable: false + cmake: + externals: + - spec: cmake@3.27.9 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/cmake-3.27.9-bznyrfsdyeybk44omm73ngz4i7hounaw + buildable: false + cuda: + externals: + - spec: cuda@11.8.0 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/cuda-11.8.0-tgu7vemuuuoajzsme3ychblz2dh43kmn + buildable: false + flex: + externals: + - spec: flex@2.6.3 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/flex-2.6.3-224ispwu2mu25zs4da2kkohy6zv4q65e + buildable: false + gcc-runtime: + externals: + - spec: gcc-runtime@11.4.0 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/gcc-runtime-11.4.0-dsxnezmi7bkgqao5dzxswvuouqknvobc + buildable: false + git: + externals: + - spec: git@2.45.1 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/git-2.45.1-7ft725l4mxetptri357qstumq7tcnejl + buildable: false + gmake: + externals: + - spec: gmake@4.4.1 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gmake-4.4.1-4ggqhci25tf33hjuoexjdvfumfkblame + buildable: false + gmp: + externals: + - spec: gmp@6.2.1 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/gmp-6.2.1-ug5lnjc3nggwzvlng4uivobi445ar4rz + buildable: false + mpfr: + externals: + - spec: mpfr@4.2.1 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/mpfr-4.2.1-d7qo5yi7c6li4sd7c6cpeyi6mp5ecff2 + m4: + externals: + - spec: m4@1.4.19 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/m4-1.4.19-bai5hhnbhyjx5c3hz6zkcyhugt7pucqw + buildable: false + openblas: + externals: + - spec: openblas@0.3.18 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/openblas-0.3.18-kh32dndiocjx2tbzqbzvwmxqj6h4xguz + buildable: false + openmpi: + externals: + - spec: openmpi@4.1.6 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/openmpi-4.1.6-hwhj3mmb3casoch5syfrgjfvfbwnnvb6 + buildable: false + perl: + externals: + - spec: perl@5.38.0 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/perl-5.38.0-thuwpjf45wfwqhgsh7ulpprrw7ln57nh + buildable: false + pkgconf: + externals: + - spec: pkgconf@2.2.0 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/pkgconf-2.2.0-pcnjsu5fbkx342z24qzkqjweyqokk7aa + buildable: false + pugixml: + externals: + - spec: pugixml@1.13 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/pugixml-1.13-nrhgzg4inoooc3e5q75ixvthhcxsvygt + buildable: false + python: + externals: + - spec: python@3.11.7 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/python-3.11.7-pfzktkmgta3ofr6pucgmhvucrn3izuzc + buildable: false + readline: + externals: + - spec: readline@8.2 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-8.5.0/readline-8.2-ebrc2f2ym2ughkwgfoawyiwsb3v5axq5 + buildable: false + zlib: + externals: + - spec: zlib@1.3.1 + prefix: /opt/spack/opt/spack/linux-almalinux8-power9le/gcc-11.4.0/zlib-1.3.1-dvatkvor6qnerz4mv4aktjeklajcjb4o + buildable: false + + #### + # spec of system packages to reuse + + # libs + glibc: + externals: + - spec: glibc@2.28 + prefix: /usr + buildable: false \ No newline at end of file diff --git a/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile new file mode 100644 index 00000000..7ff78270 --- /dev/null +++ b/docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile @@ -0,0 +1,105 @@ +####################################### +# Pangea 3 tpl image +# +# Description : +# - generic image for building geos tpls on Pangea 3 environments +# - the docker base image must match the spack environment +# - tools of the base image are expected to be sourced in the set_env.sh script +# +# Usage : +# build the image (from the root of the repository!): +# - podman build --format docker --progress=plain \ +# --build-arg DOCKER_ROOT_IMAGE=onetechssc/pangea3:gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 \ +# --build-arg INSTALL_DIR=/workrd/SCR/NUM/GEOS_environment/p3/install/tpl/install \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -t onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 \ +# -f docker/TotalEnergies/pangea-3/pangea3-tpl.Dockerfile . +# run the image: +# - podman run -it --detach --privileged --name pangea3_tpl -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# - podman exec -it pangea3_tpl /bin/bash +# push the image (requires to be part of the onetechssc docker organization): +# - podman login docker.io +# - podman push localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 docker://docker.io/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 +# build geos from the image (from the root of geos reopsitory!): +# - podman run --name pangea3_geos \ +# -v /etc/pki/ca-trust/extracted/pem/:/etc/pki/ca-trust/extracted/pem \ +# -v .:/tmp/geos \ +# localhost/onetechssc/geos:tpl_gcc11.4.0-openmpi4.1.6-openblas0.3.18-cuda11.8.0_v1.0 \ +# /tmp/geos/scripts/ci_build_and_test_in_container.sh \ +# --host-config /spack-generated-wave-solver-only.cmake --nproc=16 \ +# --repository /tmp/geos --cmake-build-type Release --install-dir /tmp/install \ +# --enable-hypre-device CUDA --build-exe-only +####################################### + +# ------------------------------------- +# pangea3 - TPL BASE +ARG DOCKER_ROOT_IMAGE +FROM $DOCKER_ROOT_IMAGE AS pangea3_tpl_base +# ------ +# LABELS +LABEL description="Pangea 3 image : geos_tpl" +LABEL version="1.0" +LABEL maintainer="TotalEnergies HPC Team" +# ------ +# ARGS +ARG INSTALL_DIR +# ------ +# ENV +ENV GEOS_TPL_DIR=$INSTALL_DIR +ENV GEOSX_TPL_DIR=$GEOS_TPL_DIR + +# ------------------------------------- +# pangea3 - TPL BUILDER +FROM pangea3_tpl_base AS pangea3_tpl_builder +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOS_TPL_DIR} && \ +# - run uberenv + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@11.4.0 +cuda+pygeosx cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers" \ + --spack-env-file=${SRC_DIR}/docker/TotalEnergies/pangea-3/pangea3-spack.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOS_TPL_DIR} \ + -k && \ +# - remove host-config generated for LvArray + rm lvarray* && \ +# - rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated-wave-solver-only.cmake && \ +# - remove extraneous spack files + cd ${GEOS_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Build only the wave solver for Pangea 3 +RUN echo 'set ( GEOS_ENABLE_CONTACT OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_FLUIDFLOW OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_INDUCEDSEISMICITY OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_MULTIPHYSICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_SIMPLEPDE OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_SOLIDMECHANICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \ + echo 'set ( GEOS_ENABLE_SURFACEGENERATION OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake + +# ------------------------------------- +# pangea3 - TPL TOOLCHAIN +FROM pangea3_tpl_base AS pangea3_tpl_toolchain +# ------ +# ARGS +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/src +# ------ +# INSTALL +# - copy TPLs install directory +COPY --from=pangea3_tpl_builder $GEOS_TPL_DIR $GEOS_TPL_DIR +# - copy the generated host-config +COPY --from=pangea3_tpl_builder /spack-generated-wave-solver-only.cmake / +# ------ +# ENTRYPOINT +# set entry point for geos ci build script +ENTRYPOINT ["/bin/bash", "-c", "source /root/.setup_env.sh && exec \"$@\"", "--"] \ No newline at end of file diff --git a/docker/pangea-spack.yaml b/docker/pangea-spack.yaml deleted file mode 100644 index 748d4ea6..00000000 --- a/docker/pangea-spack.yaml +++ /dev/null @@ -1,214 +0,0 @@ -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - compilers:: - - compiler: - spec: gcc@=9.4.0 - paths: - cc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gcc - cxx: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/g++ - f77: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - fc: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-8.4.1/gcc-9.4.0-xe5cqnyajaqz75up3gflln5zlj2rue5v/bin/gfortran - flags: {} - operating_system: almalinux8 - target: ppc64le - modules: [] - environment: - set: # Needed for scotch - BISON: bison - FLEX: flex - prepend_path: - LD_LIBRARY_PATH: /data_local/sw/lsf/10.1/linux3.10-glibc2.17-ppc64le/lib - extra_rpaths: [] - - packages: - all: - target: [x86_64] - compiler: [gcc] - providers: - blas: [openblas] - lapack: [openblas] - mpi: [openmpi] - - - # v0.6.2 - blt: - require: "@git.9ff77344f0b2a6ee345e452bddd6bfd46cbbfa35=develop" - - # master - 10/18/24 - hypre: - require: "@git.c893886d15eb57e87dd36efec23693ece3ddc88e" - - # v2024.07.0 - chai: - require: "@git.df7741f1dbbdc5fff5f7d626151fdf1904e62b19=develop" - - # v2024.07.0 - umpire: - require: "@git.abd729f40064175e999a83d11d6b073dac4c01d2=develop" - - # v2024.07.0 - raja: - require: "@git.4d7fcba55ebc7cb972b7cc9f6778b48e43792ea1=develop" - - # v2024.07.0 - camp: - require: "@git.0f07de4240c42e0b38a8d872a20440cb4b33d9f5=main" - - # v2.12.0 - caliper: - require: "@git.287b7f3ad2d12f520aad04268d44f353cd05403c" - - # v0.9.2 - conduit: - require: "@git.ad86e316ad56a75c099d30ca5ce75cff275b5924=develop" - - uncrustify: - require: "@git.401a4098bce9dcc47e024987403f2d59d9ba7bd2" - - superlu-dist: - require: "@git.0f6efc377df2440c235452d13d28d2c717f832a1" - - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - buildable: false - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - buildable: false - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - buildable: false - bzip2: - externals: - - spec: bzip2@1.0.6 - prefix: /usr - buildable: false - cmake: - externals: - - spec: cmake@3.26.4 - prefix: /data_local/appli_local/MTS/GEOSX/cmake/3.26.4 - buildable: false - cuda: - externals: - - spec: cuda@11.5.0 +allow-unsupported-compilers - prefix: /data_local/sw/cuda/11.5.0 - buildable: false - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - buildable: false - gettext: - externals: - - spec: gettext@0.19.8.1 - prefix: /usr - buildable: false - glibc: - externals: - - spec: glibc@2.28 - prefix: /usr - buildable: false - gmp: - externals: - - spec: gmp@6.1.2 - prefix: /usr - buildable: false - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - buildable: false - - openblas: - externals: - - spec: openblas@0.3.18 - prefix: /data_local/sw/spack/0.17.0/opt/spack/linux-rhel8-power9le/gcc-9.4.0/openblas-0.3.18-vk36pzksytuhylqesg4cca7667np5sjp - buildable: false - blas: - buildable: false - lapack: - buildable: false - - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - buildable: false - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - buildable: false - - # Spack may grab for mpi & we don't want to use them - mpi: - buildable: false - - openmpi: - externals: - - spec: openmpi@4.1.2%gcc@9.4.0 +cuda~cxx~cxx_exceptions~java+lustre~memchecker+pmi+static~wrapper-rpath schedulers=lsf - prefix: /data_local/sw/openmpi/4.1.2/env/gcc-8.4.1 - - # Built-in ncurses causes spack failure - # ncurses: - # externals: - # - spec: ncurses@6.1 - # prefix: /usr - # buildable: false - perl: - externals: - - spec: perl@5.26.3~cpanm+opcode+open+shared+threads - prefix: /usr - buildable: false - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - buildable: false - python: - externals: - - spec: python@3.8.17+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib - prefix: /usr - buildable: false - readline: - externals: - - spec: readline@7.0 - prefix: /usr - buildable: false - tar: - externals: - - spec: tar@1.30 - prefix: /usr - buildable: false - unzip: - buildable: false - externals: - - spec: unzip@6.0 - prefix: /usr - xz: - externals: - - spec: xz@5.2.4 - prefix: /usr - buildable: false - zlib: - externals: - - spec: zlib@1.2.11 - prefix: /usr - buildable: false