Skip to content

Commit c92adfc

Browse files
klevzoffklevzoff
committed
Trilinos build: enable CUDA and Tpetra-based solver packages (Belos, Amesos2, Ifpack2, MueLu)
1 parent c7b3f03 commit c92adfc

File tree

4 files changed

+52
-17
lines changed

4 files changed

+52
-17
lines changed

‎CMakeLists.txt‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -618,8 +618,10 @@ if (ENABLE_TRILINOS)
618618

619619
message(STATUS "Building TRILINOS found at ${TRILINOS_URL}")
620620

621-
if (NOT DEFINED TRILINOS_BUILD_COMMAND)
621+
if (NOT DEFINED TRILINOS_NUM_PROC)
622622
set(TRILINOS_BUILD_COMMAND ${TPL_BUILD_COMMAND})
623+
else()
624+
set(TRILINOS_BUILD_COMMAND make -j ${TRILINOS_NUM_PROC})
623625
endif()
624626

625627
set(TRILINOS_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS}")
@@ -649,7 +651,7 @@ if (ENABLE_TRILINOS)
649651

650652
message(STATUS "TRILINOS_EXTRA_ARGS = ${TRILINOS_EXTRA_ARGS}")
651653

652-
if( ${ENABLE_MPI} )
654+
if( ENABLE_MPI )
653655
set( TRILINOS_C_COMPILER ${MPI_C_COMPILER} )
654656
set( TRILINOS_CXX_COMPILER ${MPI_CXX_COMPILER} )
655657
set( TRILINOS_Fortran_COMPILER ${MPI_Fortran_COMPILER} )
@@ -659,6 +661,16 @@ if (ENABLE_TRILINOS)
659661
set( TRILINOS_Fortran_COMPILER ${CMAKE_Fortran_COMPILER} )
660662
endif()
661663

664+
if( ENABLE_CUDA )
665+
set( TRILINOS_CXX_FLAGS "${TRILINOS_CXX_FLAGS} -ccbin ${TRILINOS_CXX_COMPILER} -arch=${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr" )
666+
set( TRILINOS_CXX_COMPILER ${CMAKE_CURRENT_BINARY_DIR}/trilinos/src/trilinos/packages/kokkos/bin/nvcc_wrapper )
667+
endif()
668+
669+
set( TRILINOS_ENABLE_SERIAL ON )
670+
if( ENABLE_OPENMP OR ENABLE_CUDA )
671+
set( TRILINOS_ENABLE_SERIAL OFF )
672+
endif()
673+
662674
ExternalProject_Add( trilinos
663675
PREFIX ${PROJECT_BINARY_DIR}/trilinos
664676
URL ${TRILINOS_URL}
@@ -678,13 +690,15 @@ if (ENABLE_TRILINOS)
678690
-D CMAKE_Fortran_FLAGS_RELEASE:STRING=${CMAKE_Fortran_FLAGS_RELEASE}
679691
-D CMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
680692
-D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
681-
-D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI}
682-
-D BUILD_SHARED_LIBS:BOOL=ON
693+
-D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE}
683694
-D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
695+
-D BUILD_SHARED_LIBS:BOOL=ON
696+
-D CUDA_ROOT:PATH=${CUDA_TOOLKIT_ROOT_DIR}
697+
-D TPL_ENABLE_MPI:BOOL=${ENABLE_MPI}
698+
-D TPL_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
684699
-D Trilinos_ENABLE_OpenMP:BOOL=${ENABLE_OPENMP}
685700
-D Trilinos_ENABLE_Fortran:BOOL=ON
686701
-D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING=""
687-
-D CMAKE_VERBOSE_MAKEFILE:BOOL=${CMAKE_VERBOSE_MAKEFILE}
688702
-D Trilinos_ENABLE_TESTS:BOOL=OFF
689703
-D Trilinos_ENABLE_Gtest:BOOL=OFF
690704
-D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF
@@ -695,14 +709,33 @@ if (ENABLE_TRILINOS)
695709
-D Trilinos_ENABLE_Kokkos:BOOL=ON
696710
-D Trilinos_ENABLE_Stratimikos:BOOL=ON
697711
-D Trilinos_ENABLE_Amesos:BOOL=ON
712+
-D Trilinos_ENABLE_Amesos2:BOOL=ON
698713
-D Trilinos_ENABLE_AztecOO:BOOL=ON
714+
-D Trilinos_ENABLE_Belos:BOOL=ON
699715
-D Trilinos_ENABLE_Ifpack:BOOL=ON
716+
-D Trilinos_ENABLE_Ifpack2:BOOL=ON
700717
-D Trilinos_ENABLE_Teuchos:BOOL=ON
701718
-D Trilinos_ENABLE_ML:BOOL=ON
719+
-D Trilinos_ENABLE_MueLu:BOOL=ON
702720
-D Trilinos_ENABLE_Thyra:BOOL=ON
703721
-D Trilinos_ENABLE_STK:BOOL=OFF
704-
-D Trilinos_ENABLE_TrilinosCouplings:BOOL=ON
722+
-D Trilinos_ENABLE_TrilinosCouplings:BOOL=OFF
705723
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON
724+
-D Trilinos_ENABLE_COMPLEX_DOUBLE:BOOL=OFF
725+
-D KOKKOS_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
726+
-D KOKKOS_ENABLE_CUDA_LAMBDA:BOOL=${ENABLE_CUDA}
727+
-D KOKKOS_ENABLE_CUDA_UVM:BOOL=${ENABLE_CUDA}
728+
-D Tpetra_ENABLE_CUDA:BOOL=${ENABLE_CUDA}
729+
-D Tpetra_INST_SERIAL:BOOL=${TRILINOS_ENABLE_SERIAL}
730+
-D Tpetra_INST_INT_LONG_LONG:BOOL=ON
731+
-D Tpetra_INST_INT_INT:BOOL=OFF
732+
-D Tpetra_INST_DOUBLE:BOOL=ON
733+
-D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF
734+
-D Belos_ENABLE_Tpetra:BOOL=ON
735+
-D Belos_ENABLE_Epetra:BOOL=OFF
736+
-D Belos_ENABLE_Xpetra:BOOL=OFF
737+
-D Ifpack2_ENABLE_ThyraTpetraAdapters:BOOL=OFF
738+
-D Amesos2_ENABLE_TIMERS:BOOL=ON
706739
${TRILINOS_EXTRA_ARGS}
707740
)
708741

@@ -913,9 +946,10 @@ if( ENABLE_PETSC )
913946

914947
message(STATUS "Building PETSC found at ${PETSC_URL}")
915948

916-
set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_RELEASE}")
917-
set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_RELEASE}")
918-
set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_RELEASE}")
949+
string(TOUPPER "${CMAKE_BUILD_TYPE}" bt_uc)
950+
set(PETSC_C_FLAGS "-fPIC ${C_FLAGS_NO_WARNINGS} ${CMAKE_C_FLAGS_${bt_uc}}")
951+
set(PETSC_CXX_FLAGS "-fPIC ${CXX_FLAGS_NO_WARNINGS} ${CMAKE_CXX_FLAGS_${bt_uc}}")
952+
set(PETSC_Fortran_FLAGS "${CMAKE_Fortran_FLAGS_${bt_uc}}")
919953

920954
if (ENABLE_OPENMP)
921955
set(PETSC_OPENMP_FLAG --with-openmp=1)

‎docker/clang-cuda/Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,9 @@ ARG TMP_DIR
5353
ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs
5454
ARG TPL_BUILD_DIR=${TMP_DIR}/build
5555

56-
ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
57-
ARG CUDA_ARCH=sm_70
58-
ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
56+
ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
57+
ENV CUDA_ARCH=sm_70
58+
ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
5959

6060
COPY . ${TPL_SRC_DIR}
6161
RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \

‎docker/configure_tpl_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ python ${TPL_SRC_DIR}/scripts/config-build.py \
2323
--buildtype Release \
2424
--buildpath ${TPL_BUILD_DIR} \
2525
--installpath ${GEOSX_TPL_DIR} \
26-
-DTRILINOS_BUILD_COMMAND="make -j1" \
26+
-DTRILINOS_NUM_PROC=2 \
2727
-DNUM_PROC=$(nproc) \
2828
$*
2929
# Note that since docker is not used for mac,\

‎docker/gcc-cuda/Dockerfile‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,15 +108,16 @@ RUN yum install -y \
108108
file \
109109
bison \
110110
flex \
111-
patch
111+
patch \
112+
which
112113

113114
ARG TMP_DIR
114115
ARG TPL_SRC_DIR=${TMP_DIR}/thirdPartyLibs
115116
ARG TPL_BUILD_DIR=${TMP_DIR}/build
116117

117-
ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
118-
ARG CUDA_ARCH=sm_70
119-
ARG CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
118+
ENV CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
119+
ENV CUDA_ARCH=sm_70
120+
ENV CMAKE_CUDA_COMPILER=${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc
120121

121122
COPY . ${TPL_SRC_DIR}
122123
RUN ${TPL_SRC_DIR}/docker/configure_tpl_build.sh \

0 commit comments

Comments
 (0)