From 1ac86c4a5213b1547ceeafbdecb3da81eb31a0ed Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Wed, 18 Jan 2023 19:59:09 +0100 Subject: [PATCH 01/17] build: add catalyst optional dependency --- host-configs/tpls.cmake | 4 ++ src/cmake/GeosxOptions.cmake | 1 + .../thirdparty/SetupGeosxThirdParty.cmake | 43 +++++++++++++++++++ src/docs/sphinx/buildGuide/Dependencies.rst | 2 + 4 files changed, 50 insertions(+) diff --git a/host-configs/tpls.cmake b/host-configs/tpls.cmake index 08e605d79bf..15afc73d12b 100644 --- a/host-configs/tpls.cmake +++ b/host-configs/tpls.cmake @@ -24,6 +24,10 @@ if(EXISTS ${GEOSX_TPL_DIR}/conduit) set(CONDUIT_DIR ${GEOSX_TPL_DIR}/conduit CACHE PATH "" FORCE) endif() +if(EXISTS ${GEOSX_TPL_DIR}/catalyst) + set(CATALYST_DIR ${GEOSX_TPL_DIR}/catalyst CACHE PATH "" FORCE) +endif() + if(EXISTS ${GEOSX_TPL_DIR}/silo) set(SILO_DIR ${GEOSX_TPL_DIR}/silo CACHE PATH "" FORCE) endif() diff --git a/src/cmake/GeosxOptions.cmake b/src/cmake/GeosxOptions.cmake index ca0c999bd57..06d8daa95c5 100644 --- a/src/cmake/GeosxOptions.cmake +++ b/src/cmake/GeosxOptions.cmake @@ -36,6 +36,7 @@ option( ENABLE_SCOTCH "Enables SCOTCH" ON ) option( ENABLE_SILO "Enables SILO output" ON ) option( ENABLE_VTK "Enables VTK" ON ) +option( ENABLE_CATALYST "Enables catalyst" ON ) option( ENABLE_TOTALVIEW_OUTPUT "Enables Totalview custom view" OFF ) diff --git a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake index 81023551889..54f49a3775e 100644 --- a/src/cmake/thirdparty/SetupGeosxThirdParty.cmake +++ b/src/cmake/thirdparty/SetupGeosxThirdParty.cmake @@ -743,6 +743,49 @@ else() message(STATUS "Not using VTK") endif() +################################ +# Catalyst +################################ +if(DEFINED CATALYST_DIR) + message(STATUS "CATALYST_DIR = ${CATALYST_DIR}") + + set(CATALYST_CMAKE_PREFIX ${CATALYST_DIR}/lib/cmake/catalyst-2.0) + if(NOT EXISTS ${CATALYST_CMAKE_PREFIX}) + set(CATALYST_CMAKE_PREFIX ${CATALYST_DIR}/lib64/cmake/catalyst-2.0) + endif() + + find_package(catalyst REQUIRED + PATHS ${CATALYST_CMAKE_PREFIX} + NO_DEFAULT_PATH) + + if (NOT CATALYST_ABI_VERSION STREQUAL "2") + message(WARNING "CATALYST_DIR was set but catalyst with ABI version 2 was not found") + endif() + + set(CATALYST_TARGETS catalyst::catalyst) + foreach( targetName ${CATALYST_TARGETS} ) + + get_target_property( includeDirs ${targetName} INTERFACE_INCLUDE_DIRECTORIES) + + set_property(TARGET ${targetName} + APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES + ${includeDirs}) + endforeach() + + set(ENABLE_CATALYST ON CACHE BOOL "Boolean defining whether or not to build with catalyst") + set(thirdPartyLibs ${thirdPartyLibs} catalyst) + +else() + + if(ENABLE_CATALYST) + message(WARNING "ENABLE_CATALYST is ON but CATALYST_DIR is not defined") + endif() + + set(ENABLE_CATALYST OFF CACHE BOOL "Boolean defining whether or not to build with catalyst") + message(STATUS "Not using catalyst") + +endif() + ################################ # FMT ################################ diff --git a/src/docs/sphinx/buildGuide/Dependencies.rst b/src/docs/sphinx/buildGuide/Dependencies.rst index 03824bb29a4..a921c5835cb 100644 --- a/src/docs/sphinx/buildGuide/Dependencies.rst +++ b/src/docs/sphinx/buildGuide/Dependencies.rst @@ -23,6 +23,7 @@ Name Version Enable option Path variable ============= ========== =========================== ============================= ===================================== Adiak_ 0.2.0 :code:`ENABLE_CALIPER` :code:`ADIAK_DIR` Library for collecting metadata from HPC application runs, and distributing that metadata to subscriber tools. Caliper_ 2.4.0 :code:`ENABLE_CALIPER` :code:`CALIPER_DIR` Instrumentation and performance profiling library. +catalyst_ 2.0..0-rc3 :code:`ENABLE_CATALYST` :code:`CATALYST_DIR` Open source API specification developed for simulations to analyze and visualize data in situ. conduit_ 0.5.0 *mandatory* :code:`CONDUIT_DIR` Simplified Data Exchange for HPC Simulations. CHAI_ 2.2.2 *mandatory* :code:`CHAI_DIR` Copy-hiding array abstraction to automatically migrate data between memory spaces. RAJA_ 0.12.1 *mandatory* :code:`RAJA_DIR` Collection of C++ software abstractions that enable architecture portability for HPC applications. @@ -54,6 +55,7 @@ uncrustify_ 401a409 :code:`ENABLE_UNCRUSTIFY` :code:`UNCRUSTIFY_EXECUTABL .. _Adiak : https://github.com/LLNL/Adiak .. _Caliper: https://github.com/LLNL/Caliper +.. _catalyst: https://gitlab.kitware.com/paraview/catalyst .. _conduit: https://github.com/LLNL/conduit .. _CHAI : https://github.com/LLNL/CHAI .. _RAJA : https://github.com/LLNL/RAJA From f2cea4ca4e2080903492f24b9730d5a2500ed8c4 Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Thu, 19 Jan 2023 19:07:08 +0100 Subject: [PATCH 02/17] BlueprintOutput: refactor conduit blueprint creation for reuse --- .../fileIO/Outputs/BlueprintOutput.cpp | 49 ++++++++++++------- .../fileIO/Outputs/BlueprintOutput.hpp | 14 ++++++ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp b/src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp index 9423c1ae4b7..0fc8eb0bcb1 100644 --- a/src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/BlueprintOutput.cpp @@ -35,7 +35,7 @@ namespace internal { /** - * @brief @return The Blueprint shape from the GEOSX element type string. + * @brief @return The Blueprint shape from the GEOS element type string. * @param elementType the elementType to look up. */ string toBlueprintShape( ElementType const elementType ) @@ -136,26 +136,9 @@ bool BlueprintOutput::execute( real64 const time, { GEOS_MARK_FUNCTION; - MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization(); - conduit::Node meshRoot; conduit::Node & mesh = meshRoot[ "mesh" ]; - conduit::Node & coordset = mesh[ "coordsets/nodes" ]; - conduit::Node & topologies = mesh[ "topologies" ]; - - mesh[ "state/time" ] = time; - mesh[ "state/cycle" ] = cycle; - - addNodalData( meshLevel.getNodeManager(), coordset, topologies, mesh[ "fields" ] ); - - dataRepository::Group averagedElementData( "averagedElementData", this ); - addElementData( meshLevel.getElemManager(), coordset, topologies, mesh[ "fields" ], averagedElementData ); - - /// The Blueprint will complain if the fields node is present but empty. - if( mesh[ "fields" ].number_of_children() == 0 ) - { - mesh.remove( "fields" ); - } + this->mapMesh(time, cycle, domain, mesh); /// Verify that the mesh conforms to the Blueprint. conduit::Node info; @@ -178,6 +161,34 @@ bool BlueprintOutput::execute( real64 const time, return false; } +/////////////////////////////////////////////////////////////////////////////////////////////////// +void BlueprintOutput::mapMesh( real64 const time, + integer const cycle, + DomainPartition & domain, + conduit::Node & mesh ) +{ + GEOS_MARK_FUNCTION; + + MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization(); + + conduit::Node & coordset = mesh[ "coordsets/nodes" ]; + conduit::Node & topologies = mesh[ "topologies" ]; + + mesh[ "state/time" ] = time; + mesh[ "state/cycle" ] = cycle; + + addNodalData( meshLevel.getNodeManager(), coordset, topologies, mesh[ "fields" ] ); + + dataRepository::Group averagedElementData( "averagedElementData", this ); + addElementData( meshLevel.getElemManager(), coordset, topologies, mesh[ "fields" ], averagedElementData ); + + /// The Blueprint will complain if the fields node is present but empty. + if( mesh[ "fields" ].number_of_children() == 0 ) + { + mesh.remove( "fields" ); + } +} + /////////////////////////////////////////////////////////////////////////////////////////////////// void BlueprintOutput::addNodalData( NodeManager const & nodeManager, conduit::Node & coordset, diff --git a/src/coreComponents/fileIO/Outputs/BlueprintOutput.hpp b/src/coreComponents/fileIO/Outputs/BlueprintOutput.hpp index 4c4a26c52fd..49aa94bd4b5 100644 --- a/src/coreComponents/fileIO/Outputs/BlueprintOutput.hpp +++ b/src/coreComponents/fileIO/Outputs/BlueprintOutput.hpp @@ -79,6 +79,20 @@ class BlueprintOutput : public OutputBase DomainPartition & domain ) override { execute( time_n, 0, cycleNumber, eventCounter, eventProgress, domain ); } +protected: + + /** + * @brief Map the mesh to the conduit blueprint. + * @param time The time value associated with the mesh + * @param cycle The cycle from the execute method. + * @param domain the domain from the execute method + * @param meshRoot the conduit node to map the mesh to. + */ + virtual void mapMesh( real64 const time, + integer const cycle, + DomainPartition & domain, + conduit::Node & meshRoot ); + private: /** From ae895f76324df754dc1e680bdf83186fa69005d0 Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Thu, 19 Jan 2023 19:09:00 +0100 Subject: [PATCH 03/17] CatalystOutput: new output for in-situ capability This commit introduces a new output class for running in-situ co-processing using the catalyst API. It also introduces a new `Catalyst` module in the fileIO folder with specific code for dealing with `conduit` collisions. --- src/coreComponents/fileIO/CMakeLists.txt | 7 + .../fileIO/Catalyst/CMakeLists.txt | 29 ++ .../fileIO/Catalyst/CatalystActions.cpp | 174 ++++++++++ .../fileIO/Catalyst/CatalystActions.hpp | 46 +++ .../fileIO/Catalyst/ConduitCapsule.cpp | 19 ++ .../fileIO/Catalyst/ConduitCapsule.hpp | 151 +++++++++ .../fileIO/Catalyst/GenericConduitCapsule.hpp | 162 +++++++++ .../fileIO/Catalyst/GenericConduitCapsule.tpp | 314 ++++++++++++++++++ src/coreComponents/fileIO/Catalyst/README.md | 11 + .../fileIO/Outputs/CatalystOutput.cpp | 298 +++++++++++++++++ .../fileIO/Outputs/CatalystOutput.hpp | 88 +++++ .../schema/docs/BiotPorosity_other.rst | 29 +- src/coreComponents/schema/docs/Catalyst.rst | 18 + .../schema/docs/Catalyst_other.rst | 9 + src/coreComponents/schema/docs/Outputs.rst | 1 + .../schema/docs/Outputs_other.rst | 1 + src/coreComponents/schema/schema.xsd | 27 ++ src/coreComponents/schema/schema.xsd.other | 8 +- src/docs/doxygen/GeosxConfig.hpp | 14 +- src/docs/sphinx/CompleteXMLSchema.rst | 14 + 20 files changed, 1397 insertions(+), 23 deletions(-) create mode 100644 src/coreComponents/fileIO/Catalyst/CMakeLists.txt create mode 100644 src/coreComponents/fileIO/Catalyst/CatalystActions.cpp create mode 100644 src/coreComponents/fileIO/Catalyst/CatalystActions.hpp create mode 100644 src/coreComponents/fileIO/Catalyst/ConduitCapsule.cpp create mode 100644 src/coreComponents/fileIO/Catalyst/ConduitCapsule.hpp create mode 100644 src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.hpp create mode 100644 src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.tpp create mode 100644 src/coreComponents/fileIO/Catalyst/README.md create mode 100644 src/coreComponents/fileIO/Outputs/CatalystOutput.cpp create mode 100644 src/coreComponents/fileIO/Outputs/CatalystOutput.hpp create mode 100644 src/coreComponents/schema/docs/Catalyst.rst create mode 100644 src/coreComponents/schema/docs/Catalyst_other.rst diff --git a/src/coreComponents/fileIO/CMakeLists.txt b/src/coreComponents/fileIO/CMakeLists.txt index e99ee3bad42..40e492c099b 100644 --- a/src/coreComponents/fileIO/CMakeLists.txt +++ b/src/coreComponents/fileIO/CMakeLists.txt @@ -87,6 +87,13 @@ if( ENABLE_CUDA_NVTOOLSEXT ) set( dependencyList ${dependencyList} CUDA::nvToolsExt ) endif() +if ( ENABLE_CATALYST ) + add_subdirectory(Catalyst) + list( APPEND fileIO_headers Outputs/CatalystOutput.hpp ) + list( APPEND fileIO_sources Outputs/CatalystOutput.cpp ) + list( APPEND dependencyList fileIOCatalyst ) +endif() + blt_add_library( NAME fileIO SOURCES ${fileIO_sources} HEADERS ${fileIO_headers} diff --git a/src/coreComponents/fileIO/Catalyst/CMakeLists.txt b/src/coreComponents/fileIO/Catalyst/CMakeLists.txt new file mode 100644 index 00000000000..3c9f1be716e --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/CMakeLists.txt @@ -0,0 +1,29 @@ +set(catalyst_headers + CatalystActions.hpp + ConduitCapsule.hpp + GenericConduitCapsule.hpp +) + +set(catalyst_sources + CatalystActions.cpp + ConduitCapsule.cpp +) + +set(catalyst_template_sources + GenericConduitCapsule.tpp +) + +set(catalyst_dependencies + catalyst::catalyst +) + +list(APPEND ${catalyst_headers} ${catalyst_template_sources}) + +blt_add_library( NAME fileIOCatalyst + SOURCES ${catalyst_sources} + HEADERS ${catalyst_headers} + DEPENDS_ON ${catalyst_dependencies} + OBJECT ${GEOS_BUILD_OBJ_LIBS} + ) + +geosx_add_code_checks(PREFIX fileIOCatalyst ) diff --git a/src/coreComponents/fileIO/Catalyst/CatalystActions.cpp b/src/coreComponents/fileIO/Catalyst/CatalystActions.cpp new file mode 100644 index 00000000000..75fb45d8615 --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/CatalystActions.cpp @@ -0,0 +1,174 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file CatalystActions.cpp + */ + +// source includes +#include "CatalystActions.hpp" + +// external includes +#include + +#include + +namespace geos +{ + +namespace internal +{ + +/* + * Convert the encapsulated simulation conduit node into a catalyst conduit node + */ +conduit_cpp::Node Convert(ConduitCapsule* simulationNode) +{ + conduit_cpp::Node convertedNode; + + if (!simulationNode) + { + return convertedNode; + } + + if(simulationNode->IsList() && simulationNode->IsInt8()) + { + convertedNode.set_int8_vector(simulationNode->GetInt8Array()); + } + else if(simulationNode->IsList() && simulationNode->IsInt16()) + { + convertedNode.set_int16_vector(simulationNode->GetInt16Array()); + } + else if(simulationNode->IsList() && simulationNode->IsInt32()) + { + convertedNode.set_int32_vector(simulationNode->GetInt32Array()); + } + else if(simulationNode->IsList() && simulationNode->IsInt64()) + { + convertedNode.set_int64_vector(simulationNode->GetInt64Array()); + } + else if(simulationNode->IsList() && simulationNode->IsUInt8()) + { + convertedNode.set_uint8_vector(simulationNode->GetUInt8Array()); + } + else if(simulationNode->IsList() && simulationNode->IsUInt16()) + { + convertedNode.set_uint16_vector(simulationNode->GetUInt16Array()); + } + else if(simulationNode->IsList() && simulationNode->IsUInt32()) + { + convertedNode.set_uint32_vector(simulationNode->GetUInt32Array()); + } + else if(simulationNode->IsList() && simulationNode->IsUInt64()) + { + convertedNode.set_uint64_vector(simulationNode->GetUInt64Array()); + } + else if(simulationNode->IsList() && simulationNode->IsFloat32()) + { + convertedNode.set_float32_vector(simulationNode->GetFloat32Array()); + } + else if(simulationNode->IsList() && simulationNode->IsFloat64()) + { + convertedNode.set_float64_vector(simulationNode->GetFloat64Array()); + } + else if(simulationNode->IsInt8()) + { + convertedNode.set_int8(simulationNode->GetInt8Value()); + } + else if(simulationNode->IsInt16()) + { + convertedNode.set_int16(simulationNode->GetInt16Value()); + } + else if(simulationNode->IsInt32()) + { + convertedNode.set_int32(simulationNode->GetInt32Value()); + } + else if(simulationNode->IsInt64()) + { + convertedNode.set_int64(simulationNode->GetInt64Value()); + } + else if(simulationNode->IsUInt8()) + { + convertedNode.set_uint8(simulationNode->GetUInt8Value()); + } + else if(simulationNode->IsUInt16()) + { + convertedNode.set_uint16(simulationNode->GetUInt16Value()); + } + else if(simulationNode->IsUInt32()) + { + convertedNode.set_uint32(simulationNode->GetUInt32Value()); + } + else if(simulationNode->IsUInt64()) + { + convertedNode.set_uint64(simulationNode->GetUInt64Value()); + } + else if(simulationNode->IsFloat32()) + { + convertedNode.set_float32(simulationNode->GetFloat32Value()); + } + else if(simulationNode->IsFloat64()) + { + convertedNode.set_float64(simulationNode->GetFloat64Value()); + } + else if(simulationNode->IsString()) + { + convertedNode.set_string(simulationNode->GetStringValue()); + } + + // Recursively convert children + for(std::size_t iChild = 0; iChild < simulationNode->GetNumberOfChildren(); ++iChild) + { + auto childConduitNode = simulationNode->GetChild(iChild); + auto name = childConduitNode->GetName(); + if(name.empty()) + { + name="root"; + } + convertedNode.set_path_node(name, Convert(childConduitNode.get())); + } + + return convertedNode; +} + +bool CatalystAction(ConduitCapsule* simulationNode, std::function action) +{ + if(!simulationNode) + { + return false; + } + + conduit_cpp::Node convertedNode = Convert(simulationNode); + + return (action(conduit_cpp::c_node(&convertedNode)) == catalyst_status::catalyst_status_ok); +} + +} + +bool CatalystInitialize(ConduitCapsule* simulationNode) +{ + return internal::CatalystAction(simulationNode, catalyst_initialize); +} + +bool CatalystExecute(ConduitCapsule* simulationNode) +{ + return internal::CatalystAction(simulationNode, catalyst_execute); +} + +bool CatalystFinalize(ConduitCapsule* simulationNode) +{ + return internal::CatalystAction(simulationNode, catalyst_finalize); +} + +} diff --git a/src/coreComponents/fileIO/Catalyst/CatalystActions.hpp b/src/coreComponents/fileIO/Catalyst/CatalystActions.hpp new file mode 100644 index 00000000000..36204c3dcc1 --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/CatalystActions.hpp @@ -0,0 +1,46 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file CatalystActions.hpp + */ + +// source includes +#include "ConduitCapsule.hpp" + +namespace geos +{ + +/** + * @brief Wrapped call to catalyst_initialize + * @param the encapsulated conduit node to convert and send to initialize method + * @returns true on success + */ +bool CatalystInitialize(ConduitCapsule* simulationNode); + +/** + * @brief Wrapped call to catalyst_execute + * @param the encapsulated conduit node to convert and send to execute method + * @returns true on success + */ +bool CatalystExecute(ConduitCapsule* simulationNode); + +/** + * @brief Wrapped call to catalyst_finalize + * @param the encapsulated conduit node to convert and send to finalize method + * @returns true on success + */ +bool CatalystFinalize(ConduitCapsule* simulationNode); + +} diff --git a/src/coreComponents/fileIO/Catalyst/ConduitCapsule.cpp b/src/coreComponents/fileIO/Catalyst/ConduitCapsule.cpp new file mode 100644 index 00000000000..9ec4185bc97 --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/ConduitCapsule.cpp @@ -0,0 +1,19 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file ConduitCapsule.cpp + */ + +#include "ConduitCapsule.hpp" diff --git a/src/coreComponents/fileIO/Catalyst/ConduitCapsule.hpp b/src/coreComponents/fileIO/Catalyst/ConduitCapsule.hpp new file mode 100644 index 00000000000..f80730d712a --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/ConduitCapsule.hpp @@ -0,0 +1,151 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file ConduitCapsule.hpp + */ + +#ifndef GEOS_FILEIO_CATALYST_CONDUITCAPSULE_HPP_ +#define GEOS_FILEIO_CATALYST_CONDUITCAPSULE_HPP_ + +// External includes +#include +#include +#include + +namespace geos +{ + +/** + * @class ConduitCapsule + * @brief Pure interface for encapsulating a conduit node with the goal of coverting it to a catalyst conduit node + */ +class ConduitCapsule +{ +public: + + /** + * @brief No explicit memory deallocation here + */ + virtual ~ConduitCapsule() = default; + + /** + * @brief Get the number of children this node element has in the hiearchy + * @returns the number of children nodes this node has + */ + virtual std::size_t GetNumberOfChildren() const = 0; + + /** + * @brief Get the iChild'th node below this node + * @param iChild the index of the child node to get + * @returns A shared pointer to the iChild node already encapsulated + */ + virtual std::shared_ptr GetChild(std::size_t iChild) = 0; + + /** + * @brief Get the name of the underlying node + * @returns A string with the name of the node + */ + virtual std::string GetName() const = 0; + + ///@{ + /** + * @brief Type checkers for the data held by the node + * @returns true if the data type corresponds to the call + */ + virtual bool IsList() const = 0; + virtual bool IsInt8() const = 0; + virtual bool IsInt16() const = 0; + virtual bool IsInt32() const = 0; + virtual bool IsInt64() const = 0; + virtual bool IsUInt8() const = 0; + virtual bool IsUInt16() const = 0; + virtual bool IsUInt32() const = 0; + virtual bool IsUInt64() const = 0; + virtual bool IsFloat32() const = 0; + virtual bool IsFloat64() const = 0; + virtual bool IsString() const = 0; + ///@} + + ///@{ + /** + * @brief Check if the data held by the node is external + */ + virtual bool IsDataExternal() const = 0; + /** + * @brief Get a void* pointer to the external data when applicable + * @returns pointer to external data when relevant and nullptr if not + */ + virtual void* GetExternalDataPointer() const = 0; + /** + * @brief Get the offset with which to read the external data + * @retuns the offset meta-data + */ + virtual long GetExternalDataOffset() const = 0; + /** + * @brief Get the stride with which to read the external data + * @retuns the stride meta-data + */ + virtual long GetExternalDataStride() const = 0; + /** + * @brief Get the number of bytes one element of external data occupies + * @retuns the element size in bytes meta-data + */ + virtual long GetExternalDataElementBytes() const = 0; + /** + * @brief Get the endianness with which to read the external data + * @retuns the endianness meta-data + */ + virtual long GetExternalDataEndianness() const = 0; + ///@} + + + ///@{ + /** + * Get the values of data for conventional data types + */ + virtual int8_t GetInt8Value() const = 0; + virtual int16_t GetInt16Value() const = 0; + virtual int32_t GetInt32Value() const = 0; + virtual int64_t GetInt64Value() const = 0; + virtual uint8_t GetUInt8Value() const = 0; + virtual uint16_t GetUInt16Value() const = 0; + virtual uint32_t GetUInt32Value() const = 0; + virtual uint64_t GetUInt64Value() const = 0; + virtual float GetFloat32Value() const = 0; + virtual double GetFloat64Value() const = 0; + virtual std::string GetStringValue() const = 0; + ///@} + + ///@{ + /** + * Get the value arrays of data for conventional data types + */ + virtual std::vector GetInt8Array() const = 0; + virtual std::vector GetInt16Array() const = 0; + virtual std::vector GetInt32Array() const = 0; + virtual std::vector GetInt64Array() const = 0; + virtual std::vector GetUInt8Array() const = 0; + virtual std::vector GetUInt16Array() const = 0; + virtual std::vector GetUInt32Array() const = 0; + virtual std::vector GetUInt64Array() const = 0; + virtual std::vector GetFloat32Array() const = 0; + virtual std::vector GetFloat64Array() const = 0; + ///@} + +}; + +} + +#endif diff --git a/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.hpp b/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.hpp new file mode 100644 index 00000000000..71a55a252c2 --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.hpp @@ -0,0 +1,162 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file GenericConduitCapsule.hpp + */ + +#ifndef GEOS_FILEIO_CATALYST_GENERICCONDUITCAPSULE_HPP_ +#define GEOS_FILEIO_CATALYST_GENERICCONDUITCAPSULE_HPP_ + +// source includes +#include "ConduitCapsule.hpp" + +namespace geos +{ + +/** + * @class GenericConduitCapsule + * @brief Helper template class for streamlining capsule creation for classic conduit nodes + */ +template +class GenericConduitCapsule : public ConduitCapsule +{ +public: + + /** + * @brief Constructor following RAII principles + */ + GenericConduitCapsule(ConduitNodeT* node) : Node(node) {} + + /** + * @brief Default destructor since there is no explicit memory responsibility + */ + virtual ~GenericConduitCapsule() = default; + + /** + * @brief Get the number of children this node element has in the hiearchy + * @returns the number of children nodes this node has + */ + virtual std::size_t GetNumberOfChildren() const; + + /** + * @brief Get the iChild'th node below this node + * @param iChild the index of the child node to get + * @returns A shared pointer to the iChild node already encapsulated + */ + virtual std::shared_ptr GetChild(std::size_t iChild); + + /** + * @brief Get the name of the underlying node + * @returns A string with the name of the node + */ + virtual std::string GetName() const; + + ///@{ + /** + * @brief Type checkers for the data held by the node + * @returns true if the data type corresponds to the call + */ + virtual bool IsList() const; + virtual bool IsInt8() const; + virtual bool IsInt16() const; + virtual bool IsInt32() const; + virtual bool IsInt64() const; + virtual bool IsUInt8() const; + virtual bool IsUInt16() const; + virtual bool IsUInt32() const; + virtual bool IsUInt64() const; + virtual bool IsFloat32() const; + virtual bool IsFloat64() const; + virtual bool IsString() const; + ///@} + + ///@{ + /** + * @brief Check if the data held by the node is external + */ + virtual bool IsDataExternal() const; + /** + * @brief Get a void* pointer to the external data when applicable + * @returns pointer to external data when relevant and nullptr if not + */ + virtual void* GetExternalDataPointer() const; + /** + * @brief Get the offset with which to read the external data + * @retuns the offset meta-data + */ + virtual long GetExternalDataOffset() const; + /** + * @brief Get the stride with which to read the external data + * @retuns the stride meta-data + */ + virtual long GetExternalDataStride() const; + /** + * @brief Get the number of bytes one element of external data occupies + * @retuns the element size in bytes meta-data + */ + virtual long GetExternalDataElementBytes() const; + /** + * @brief Get the endianness with which to read the external data + * @retuns the endianness meta-data + */ + virtual long GetExternalDataEndianness() const; + ///@} + + + ///@{ + /** + * Get the values of data for conventional data types + */ + virtual int8_t GetInt8Value() const; + virtual int16_t GetInt16Value() const; + virtual int32_t GetInt32Value() const; + virtual int64_t GetInt64Value() const; + virtual uint8_t GetUInt8Value() const; + virtual uint16_t GetUInt16Value() const; + virtual uint32_t GetUInt32Value() const; + virtual uint64_t GetUInt64Value() const; + virtual float GetFloat32Value() const; + virtual double GetFloat64Value() const; + virtual std::string GetStringValue() const; + ///@} + + ///@{ + /** + * Get the value arrays of data for conventional data types + */ + virtual std::vector GetInt8Array() const; + virtual std::vector GetInt16Array() const; + virtual std::vector GetInt32Array() const; + virtual std::vector GetInt64Array() const; + virtual std::vector GetUInt8Array() const; + virtual std::vector GetUInt16Array() const; + virtual std::vector GetUInt32Array() const; + virtual std::vector GetUInt64Array() const; + virtual std::vector GetFloat32Array() const; + virtual std::vector GetFloat64Array() const; + ///@} + +protected: + + /** + * @brief the pointer to the node being encapsulated + */ + ConduitNodeT* Node; + +}; + +} + +#endif diff --git a/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.tpp b/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.tpp new file mode 100644 index 00000000000..4b0b90b553a --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/GenericConduitCapsule.tpp @@ -0,0 +1,314 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file GenericConduitCapsule.tpp + */ + +// source includes +#include "GenericConduitCapsule.hpp" + +// external includes +#include + +namespace geos +{ + +template +std::size_t GenericConduitCapsule::GetNumberOfChildren() const +{ + return this->Node->number_of_children(); +} + +template +std::shared_ptr GenericConduitCapsule::GetChild(std::size_t iChild) +{ + if (iChild >= this->GetNumberOfChildren()) + { + return nullptr; + } + return std::make_shared>(this->Node->child_ptr(iChild)); +} + +template +std::string GenericConduitCapsule::GetName() const +{ + return this->Node->name(); +} + +template +bool GenericConduitCapsule::IsList() const +{ + return this->Node->dtype().number_of_elements() > 1; +} + +template +bool GenericConduitCapsule::IsInt8() const +{ + return this->Node->dtype().is_int8(); +} + +template +bool GenericConduitCapsule::IsInt16() const +{ + return this->Node->dtype().is_int16(); +} + +template +bool GenericConduitCapsule::IsInt32() const +{ + return this->Node->dtype().is_int32(); +} + +template +bool GenericConduitCapsule::IsInt64() const +{ + return this->Node->dtype().is_int64(); +} + +template +bool GenericConduitCapsule::IsUInt8() const +{ + return this->Node->dtype().is_uint8(); +} + +template +bool GenericConduitCapsule::IsUInt16() const +{ + return this->Node->dtype().is_uint16(); +} + +template +bool GenericConduitCapsule::IsUInt32() const +{ + return this->Node->dtype().is_uint32(); +} + +template +bool GenericConduitCapsule::IsUInt64() const +{ + return this->Node->dtype().is_uint64(); +} + +template +bool GenericConduitCapsule::IsFloat32() const +{ + return this->Node->dtype().is_float32(); +} + +template +bool GenericConduitCapsule::IsFloat64() const +{ + return this->Node->dtype().is_float64(); +} + +template +bool GenericConduitCapsule::IsString() const +{ + return this->Node->dtype().is_string(); +} + +template +bool GenericConduitCapsule::IsDataExternal() const +{ + return this->Node->is_data_external(); +} + +template +void* GenericConduitCapsule::GetExternalDataPointer() const +{ + return this->Node->data_ptr(); +} + +template +long GenericConduitCapsule::GetExternalDataOffset() const +{ + return this->Node->dtype().offset(); +} + +template +long GenericConduitCapsule::GetExternalDataStride() const +{ + return this->Node->dtype().stride(); +} + +template +long GenericConduitCapsule::GetExternalDataElementBytes() const +{ + return this->Node->dtype().element_bytes(); +} + +template +long GenericConduitCapsule::GetExternalDataEndianness() const +{ + return this->Node->dtype().endianness(); +} + +template +int8_t GenericConduitCapsule::GetInt8Value() const +{ + return this->Node->as_int8(); +} + +template +int16_t GenericConduitCapsule::GetInt16Value() const +{ + return this->Node->as_int16(); +} + +template +int32_t GenericConduitCapsule::GetInt32Value() const +{ + return this->Node->as_int32(); +} + +template +int64_t GenericConduitCapsule::GetInt64Value() const +{ + return this->Node->as_int64(); +} + +template +uint8_t GenericConduitCapsule::GetUInt8Value() const +{ + return this->Node->as_uint8(); +} + +template +uint16_t GenericConduitCapsule::GetUInt16Value() const +{ + return this->Node->as_uint16(); +} + +template +uint32_t GenericConduitCapsule::GetUInt32Value() const +{ + return this->Node->as_uint32(); +} + +template +uint64_t GenericConduitCapsule::GetUInt64Value() const +{ + return this->Node->as_uint64(); +} + +template +float GenericConduitCapsule::GetFloat32Value() const +{ + return this->Node->as_float32(); +} + +template +double GenericConduitCapsule::GetFloat64Value() const +{ + return this->Node->as_float64(); +} + +template +std::string GenericConduitCapsule::GetStringValue() const +{ + return this->Node->as_char8_str(); +} + +template +std::vector GenericConduitCapsule::GetInt8Array() const +{ + auto array = this->Node->as_int8_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetInt16Array() const +{ + auto array = this->Node->as_int16_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetInt32Array() const +{ + auto array = this->Node->as_int32_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetInt64Array() const +{ + auto array = this->Node->as_int64_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetUInt8Array() const +{ + auto array = this->Node->as_uint8_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetUInt16Array() const +{ + auto array = this->Node->as_uint16_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetUInt32Array() const +{ + auto array = this->Node->as_uint32_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetUInt64Array() const +{ + auto array = this->Node->as_uint64_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetFloat32Array() const +{ + auto array = this->Node->as_float32_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +template +std::vector GenericConduitCapsule::GetFloat64Array() const +{ + auto array = this->Node->as_float64_array(); + std::vector v(array.number_of_elements()); + std::generate(v.begin(), v.end(), [i=-1, array] () mutable {i++;return array.element(i);}); + return v; +} + +} diff --git a/src/coreComponents/fileIO/Catalyst/README.md b/src/coreComponents/fileIO/Catalyst/README.md new file mode 100644 index 00000000000..f05c331bc59 --- /dev/null +++ b/src/coreComponents/fileIO/Catalyst/README.md @@ -0,0 +1,11 @@ +## Reason for this module + +When trying to input the `conduit::Node` used by the simulation into the `conduit_cpp::Node` needed by `catalyst` we run into an incompatibility between the two conduit libraries which cannot coexist in the same translation unit. As of the time of writing this README, `catalyst` does not have any ready-made solution for this type of collision with simulation codes. + +The solution here is to create a library in this module with the following specifications: +- Provides an interface class defining an API for interacting with generic `conduit` nodes +- Provides utilities to convert any implementation of the above interface into a `conduit_cpp::Node` for passing to `catalyst` +- Links to the `conduit` provided by `catalyst` in its own translation unit +- Provides its own `initialize`, `execute` and `finalize` methods taking care of the conversion from the simulation code's `conduit` and `catalyst`'s version + +As such, for any simulation code already instrumented with `conduit` blueprints, this design pattern enables coexistance and conversion between the two `conduit` versions. The simulation code has but to provide an implementation of the interface in this module (in a separate library that links to this one) and pass it to the specific initialization, execution and finalization methods implemented in this library. The interface node acts as a buffer layer between the two `conduit` libraries and separates the translation units cleanly. diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp new file mode 100644 index 00000000000..57baf63de7f --- /dev/null +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -0,0 +1,298 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file CatalystOutput.cpp + */ + +//GEOS +#include "CatalystOutput.hpp" + +#include "fileIO/Catalyst/CatalystActions.hpp" +#include "fileIO/Catalyst/GenericConduitCapsule.tpp" +#include "mesh/DomainPartition.hpp" +#include "mesh/MeshLevel.hpp" +#include "mesh/mpiCommunications/CommunicationTools.hpp" + +//TPL +#include +#include + +//std +#include +#include +#include +#include +#include + +namespace +{ + +// Change all the node associated fields to have vertex association and delete the "nodes" element entry +void SanitizeNode(conduit::Node& blueprintNode) +{ + if (blueprintNode["topologies"].number_of_children() > 2) + { + GEOS_ERROR("CatalystOutput: Not yet support for multiple regions"); + return; + } + blueprintNode.remove("topologies/nodes"); + std::string topoName = blueprintNode["topologies"].child(0).name(); + auto& fields = blueprintNode["fields"]; + for ( conduit::index_t iField = 0; iField < fields.number_of_children(); ++iField ) + { + auto& field = fields.child(iField); + if (field["topology"].as_char8_str() == std::string("nodes")) + { + field["topology"] = topoName; + field["association"] = "vertex"; + } + } +} + +} + +namespace geos +{ + +struct CatalystOutput::CatalystInternals +{ + bool initialized = false; + std::string scripts = ""; + std::string implementation = ""; + std::string implementationPath = ""; + std::string adiosConfig = ""; + std::string channelName = "fullfield"; + + void initializeCatalyst() + { + + std::string scriptsCopy = this->scripts; + std::vector scriptList; + std::string delimiter = ":"; + std::size_t pos = scriptsCopy.find(delimiter); + scriptList.emplace_back(scriptsCopy.substr(0, pos)); + scriptsCopy.erase(0, pos + delimiter.length()); + while ((pos = scriptsCopy.find(delimiter)) != std::string::npos) { + scriptList.emplace_back(scriptsCopy.substr(0, pos)); + scriptsCopy.erase(0, pos + delimiter.length()); + } + + if ( scriptList.empty() || scriptList[0] == "" ) + { + GEOS_ERROR("CatalystOutput: Constructor: no catalyst scripts found."); + } + + conduit::Node initializer; + for (std::size_t iScr = 0; iScr < scriptList.size(); ++iScr) + { + initializer["catalyst/scripts/script" + std::to_string(iScr)] = scriptList[iScr]; + } + + if ( !this->implementation.empty() ) + { + initializer["catalyst_load/implementation"] = this->implementation; + if ( !this->implementationPath.empty() ) + { + initializer["catalyst_load/search_paths/" + this->implementation] = this->implementationPath; + } + } + + if ( this->implementation == "adios" || + (this->implementation.empty() && + std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios")) + { + if ( this->adiosConfig == "" ) + { + GEOS_ERROR("CatalystOutput: Constructor: Cannot use the catalyst/adios2 implementation without providing an adios2 configuration file."); + } + initializer["adios/config"] = this->adiosConfig; + } + + auto capsule = GenericConduitCapsule(&initializer); + if( !CatalystInitialize(&capsule) ) + { + GEOS_ERROR("CatalystOutput: Constructor: catalyst failed to initialize."); + } + + this->initialized = true; + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// +CatalystOutput::CatalystOutput( std::string const & name, + Group * const parent ) + : BlueprintOutput( name, + parent ) + , internal( std::unique_ptr( new CatalystInternals() ) ) +{ + + this->registerWrapper( "scripts", &this->internal->scripts ). + setApplyDefaultValue( "" ). + setInputFlag( dataRepository::InputFlags::REQUIRED ). + setDescription( "Column separated paths to the catalyst scripts." ); + + this->registerWrapper( "implementation", &this->internal->implementation ). + setApplyDefaultValue( "" ). + setInputFlag( dataRepository::InputFlags::OPTIONAL ). + setDescription( "Name of the catalyst implementation to use." ); + + this->registerWrapper( "implementationPath", &this->internal->implementationPath ). + setApplyDefaultValue( "" ). + setInputFlag( dataRepository::InputFlags::OPTIONAL ). + setDescription( "Path to the catalyst the implementation to use." ); + + this->registerWrapper( "adiosConfig", &this->internal->adiosConfig ). + setApplyDefaultValue( "" ). + setInputFlag( dataRepository::InputFlags::OPTIONAL ). + setDescription( "Path to the adios configuration file when using the catalyst-adios implementation." ); + + this->registerWrapper( "fullFieldChannelName", &this->internal->channelName ). + setApplyDefaultValue( "" ). + setInputFlag( dataRepository::InputFlags::OPTIONAL ). + setDescription( "Name to give to the channel passing the full field data." ); + +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// +CatalystOutput::~CatalystOutput() = default; + +/////////////////////////////////////////////////////////////////////////////////////////////////// +bool CatalystOutput::execute( real64 const time_n, + real64 const /*dt*/, + integer const cycleNumber, + integer const /*eventCounter*/, + real64 const /*eventProgress*/, + DomainPartition & domain ) +{ + GEOS_MARK_FUNCTION; + + if ( !this->internal->initialized ) + { + this->internal->initializeCatalyst(); + } + + conduit::Node executeRoot; + auto& catalystState = executeRoot["catalyst/state"]; + catalystState["timestep"].set(cycleNumber); + catalystState["time"].set(time_n); + + auto& channel = executeRoot["catalyst/channels/" + this->internal->channelName]; + channel["type"] = "mesh"; + + auto& meshGEOSRoot = channel["data"]; + this->mapMesh(time_n, cycleNumber, domain, meshGEOSRoot); + ::SanitizeNode(meshGEOSRoot); + + // Add additional info if the implementation is adios based + if ( this->internal->implementation == "adios" || + (this->internal->implementation.empty() && + std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios") ) + { + int const thisRank = MpiWrapper::commRank( MPI_COMM_GEOSX ); + int const commSize = MpiWrapper::commSize( MPI_COMM_GEOSX ); + + MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization(); + + localIndex_array numberOfNodesPerRank( commSize ); + + MpiWrapper::allGather( + meshLevel.getNodeManager().size(), + numberOfNodesPerRank ); + + globalIndex offsetPoints = std::accumulate( + numberOfNodesPerRank.begin(), + numberOfNodesPerRank.begin() + thisRank, + 0 ); + + globalIndex totalNumberOfNodes = std::accumulate( + numberOfNodesPerRank.begin() + thisRank, + numberOfNodesPerRank.end(), + offsetPoints ); + + auto& mesh = channel[ "data" ]; + mesh[ "coordsets/coords/points_shape" ].set_uint64(totalNumberOfNodes); + mesh[ "coordsets/coords/points_start" ].set_uint64(offsetPoints); + mesh[ "coordsets/coords/points_count" ].set_uint64(numberOfNodesPerRank[ thisRank ]); + + // WARNING + // what follows on the elements only works if all the MPI partitions have the same number + // of corresponding subRegions. I do not know if this is the case. If it is not, we expose + // ourselves to hanging MPI communication here. + auto& topologies = mesh[ "topologies" ]; + meshLevel.getElemManager().forElementSubRegionsComplete< CellElementSubRegion >( + [&] ( localIndex, localIndex, ElementRegionBase const & region, CellElementSubRegion const & subRegion ) + { + std::string const topologyName = region.getName() + "-" + subRegion.getName(); + auto& topology = topologies[ topologyName ]; + + localIndex_array numberOfElementsPerRank( commSize ); + + MpiWrapper::allGather( + subRegion.size(), + numberOfElementsPerRank ); + + globalIndex offsetElements = std::accumulate( + numberOfElementsPerRank.begin(), + numberOfElementsPerRank.begin() + thisRank, + 0 ); + + globalIndex totalNumberOfElements = std::accumulate( + numberOfElementsPerRank.begin() + thisRank, + numberOfElementsPerRank.end(), + offsetElements ); + + topology["cells_shape"].set_uint64(totalNumberOfElements); + topology["cells_start"].set_uint64(offsetElements); + topology["cells_count"].set_uint64(numberOfElementsPerRank[ thisRank ]); + }); + } + + auto capsule = GenericConduitCapsule(&executeRoot); + if ( !CatalystExecute(&capsule) ) + { + GEOS_ERROR("CatalystOutput: execute: catalyst failed to execute."); + return true; + } + + return false; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// +void CatalystOutput::cleanup( real64 const time_n, + integer const cycleNumber, + integer const eventCounter, + real64 const eventProgress, + DomainPartition & domain ) +{ + GEOS_MARK_FUNCTION; + + if ( this->execute( time_n, 0, cycleNumber, eventCounter, eventProgress, domain ) ) + { + GEOS_ERROR("CatalystOutput: cleanup: last execute failed."); + } + + conduit::Node emptyNode; + auto capsule = GenericConduitCapsule(&emptyNode); + if ( !CatalystFinalize(&capsule) ) + { + GEOS_ERROR("CatalystOutput: cleanup: finalize catalyst failed"); + } + + this->internal->initialized = false; +} + +REGISTER_CATALOG_ENTRY( OutputBase, CatalystOutput, string const &, dataRepository::Group * const ) + +} diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp new file mode 100644 index 00000000000..2f52cd97d6f --- /dev/null +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp @@ -0,0 +1,88 @@ +/* + * ------------------------------------------------------------------------------------------------------------ + * SPDX-License-Identifier: LGPL-2.1-only + * + * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University + * Copyright (c) 2018-2020 TotalEnergies + * Copyright (c) 2019- GEOSX Contributors + * All rights reserved + * + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * ------------------------------------------------------------------------------------------------------------ + */ + +/** + * @file CatalystOutput.hpp + */ + +#ifndef GEOSX_FILEIO_OUTPUTS_CATALYSTOUTPUT_HPP_ +#define GEOSX_FILEIO_OUTPUTS_CATALYSTOUTPUT_HPP_ + +#include "BlueprintOutput.hpp" + +#include // for unique_ptr + +namespace geos +{ + +/** + * @class CatalystOutput + * @brief A class for outputing to a catalyst based in-situ pipeline. + */ +class CatalystOutput : public BlueprintOutput +{ +public: + + /** + * @brief Construct a new CatalystOutput object. + * @param name The name of the CatalystObject in the data repository. + * @param parent The parent Group. + */ + CatalystOutput( string const & name, + Group * const parent ); + + /** + * @brief Destructor. + */ + virtual ~CatalystOutput() override; + + /** + * @brief Get the name used to register this object in an XML file. + * @return The string "Catalyst". + */ + static string catalogName() { return "Catalyst"; } + + /** + * @brief Launches a catalyst execution + * @copydetails EventBase::execute() + */ + virtual bool execute( real64 const time_n, + real64 const dt, + integer const cycleNumber, + integer const eventCounter, + real64 const eventProgress, + DomainPartition & domain ) override; + + /** + * @brief Launches a last catalyst execution for the data present. + * @copydetails ExecutableGroup::cleanup() + */ + virtual void cleanup( real64 const time_n, + integer const cycleNumber, + integer const eventCounter, + real64 const eventProgress, + DomainPartition & domain ) override; + +private: + ///@{ + /** + */ + struct CatalystInternals; + std::unique_ptr internal; + ///@} +}; + +} + +#endif // GEOSX_FILEIO_OUTPUTS_CATALYSTOUTPUT_HPP_ diff --git a/src/coreComponents/schema/docs/BiotPorosity_other.rst b/src/coreComponents/schema/docs/BiotPorosity_other.rst index 8512163fc1c..962677d8251 100644 --- a/src/coreComponents/schema/docs/BiotPorosity_other.rst +++ b/src/coreComponents/schema/docs/BiotPorosity_other.rst @@ -1,18 +1,19 @@ -=========================== ============== ======================================================= -Name Type Description -=========================== ============== ======================================================= -biotCoefficient real64_array Biot coefficient -dPorosity_dPressure real64_array2d Derivative of rock porosity with respect to pressure -dPorosity_dTemperature real64_array2d Derivative of rock porosity with respect to temperature -initialPorosity real64_array2d Initial porosity -meanStressIncrement real64_array2d Volumetric stress increment -porosity real64_array2d Rock porosity -porosity_n real64_array2d Rock porosity at the previous converged time step -referencePorosity real64_array Reference porosity -solidBulkModulus real64_array Solid bulk modulus -thermalExpansionCoefficient real64_array Thermal expansion coefficient -=========================== ============== ======================================================= +===================================== ============== ==================================================================================================== +Name Type Description +===================================== ============== ==================================================================================================== +averageMeanEffectiveStressIncrement_k real64_array Mean effective stress increment averaged over quadrature points at the previous sequential iteration +biotCoefficient real64_array Biot coefficient +dPorosity_dPressure real64_array2d Derivative of rock porosity with respect to pressure +dPorosity_dTemperature real64_array2d Derivative of rock porosity with respect to temperature +initialPorosity real64_array2d Initial porosity +meanEffectiveStressIncrement_k real64_array2d Mean effective stress increment at quadrature points at the previous sequential iteration +porosity real64_array2d Rock porosity +porosity_n real64_array2d Rock porosity at the previous converged time step +referencePorosity real64_array Reference porosity +solidBulkModulus real64_array Solid bulk modulus +thermalExpansionCoefficient real64_array Thermal expansion coefficient +===================================== ============== ==================================================================================================== diff --git a/src/coreComponents/schema/docs/Catalyst.rst b/src/coreComponents/schema/docs/Catalyst.rst new file mode 100644 index 00000000000..a8369f04372 --- /dev/null +++ b/src/coreComponents/schema/docs/Catalyst.rst @@ -0,0 +1,18 @@ + + +======================== ============================= ======== ================================================================================== +Name Type Default Description +======================== ============================= ======== ================================================================================== +adiosConfig string Path to the adios configuration file when using the catalyst-adios implementation. +childDirectory string Child directory path +fullFieldChannelName string Name to give to the channel passing the full field data. +implementation string Name of the catalyst implementation to use. +implementationPath string Path to the catalyst the implementation to use. +name string required A name is required for any non-unique nodes +outputFullQuadratureData integer 0 If true writes out data associated with every quadrature point. +parallelThreads integer 1 Number of plot files. +plotLevel geos_dataRepository_PlotLevel 1 Determines which fields to write. +scripts string required Column separated paths to the catalyst scripts. +======================== ============================= ======== ================================================================================== + + diff --git a/src/coreComponents/schema/docs/Catalyst_other.rst b/src/coreComponents/schema/docs/Catalyst_other.rst new file mode 100644 index 00000000000..adf1c1b8aec --- /dev/null +++ b/src/coreComponents/schema/docs/Catalyst_other.rst @@ -0,0 +1,9 @@ + + +==== ==== ============================ +Name Type Description +==== ==== ============================ + (no documentation available) +==== ==== ============================ + + diff --git a/src/coreComponents/schema/docs/Outputs.rst b/src/coreComponents/schema/docs/Outputs.rst index 9950b014151..d7f59c47e9f 100644 --- a/src/coreComponents/schema/docs/Outputs.rst +++ b/src/coreComponents/schema/docs/Outputs.rst @@ -4,6 +4,7 @@ Name Type Default Description =========== ==== ======= ====================== Blueprint node :ref:`XML_Blueprint` +Catalyst node :ref:`XML_Catalyst` ChomboIO node :ref:`XML_ChomboIO` Python node :ref:`XML_Python` Restart node :ref:`XML_Restart` diff --git a/src/coreComponents/schema/docs/Outputs_other.rst b/src/coreComponents/schema/docs/Outputs_other.rst index cbcc1b80401..b2ee8317624 100644 --- a/src/coreComponents/schema/docs/Outputs_other.rst +++ b/src/coreComponents/schema/docs/Outputs_other.rst @@ -4,6 +4,7 @@ Name Type Description =========== ==== ================================ Blueprint node :ref:`DATASTRUCTURE_Blueprint` +Catalyst node :ref:`DATASTRUCTURE_Catalyst` ChomboIO node :ref:`DATASTRUCTURE_ChomboIO` Python node :ref:`DATASTRUCTURE_Python` Restart node :ref:`DATASTRUCTURE_Restart` diff --git a/src/coreComponents/schema/schema.xsd b/src/coreComponents/schema/schema.xsd index b3fc3d76ef9..067994b5312 100644 --- a/src/coreComponents/schema/schema.xsd +++ b/src/coreComponents/schema/schema.xsd @@ -264,6 +264,10 @@ + + + + @@ -1784,6 +1788,7 @@ the relative residual norm satisfies: + @@ -1804,6 +1809,28 @@ the relative residual norm satisfies: + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/coreComponents/schema/schema.xsd.other b/src/coreComponents/schema/schema.xsd.other index 890a518d197..6ad71a93261 100644 --- a/src/coreComponents/schema/schema.xsd.other +++ b/src/coreComponents/schema/schema.xsd.other @@ -446,6 +446,7 @@ + @@ -455,6 +456,7 @@ + @@ -1184,6 +1186,8 @@ + + @@ -1192,8 +1196,8 @@ - - + + diff --git a/src/docs/doxygen/GeosxConfig.hpp b/src/docs/doxygen/GeosxConfig.hpp index a0000940e9a..198305cfceb 100644 --- a/src/docs/doxygen/GeosxConfig.hpp +++ b/src/docs/doxygen/GeosxConfig.hpp @@ -30,10 +30,10 @@ #define GEOSX_USE_MPI /// Enables use of OpenMP (CMake option ENABLE_OPENMP) -#define GEOSX_USE_OPENMP +/* #undef GEOSX_USE_OPENMP */ /// Enables use of CUDA (CMake option ENABLE_CUDA) -#define GEOS_USE_CUDA +/* #undef GEOS_USE_CUDA */ /// Enables use of CUDA NVToolsExt (CMake option ENABLE_CUDA_NVTOOLSEXT) /* #undef GEOS_USE_CUDA_NVTOOLSEXT */ @@ -81,7 +81,7 @@ /// Denotes HYPRE using HIP #define GEOS_USE_HYPRE_HIP 2 /// Macro determining what parellel interface hypre is using -#define GEOS_USE_HYPRE_DEVICE GEOS_USE_HYPRE_CUDA +#define GEOS_USE_HYPRE_DEVICE GEOS_USE_HYPRE_CPU /// Enables use of SuperLU_dist library through HYPRE (CMake option ENABLE_SUPERLU_DIST) #define GEOSX_USE_SUPERLU_DIST @@ -129,7 +129,7 @@ #define HDF5_VERSION 1.12.1 /// Version information for Conduit -#define Conduit_VERSION 0.8.2 +#define Conduit_VERSION 0.8.6 /// Version information for RAJA #define RAJA_VERSION 2022.10.5 @@ -162,13 +162,13 @@ #define suitesparse_VERSION 5.7.9 /// Version information for VTK -#define VTK_VERSION 9.2.6 +#define VTK_VERSION 9.2.20230719 /// Version information for fmt -#define fmt_VERSION 10.0.0 +#define fmt_VERSION 9.1.0 /// Version information for python -#define Python3_VERSION 3.10.8 +#define Python3_VERSION 3.11.3 /// Version information for CUDAToolkit /* #undef CUDAToolkit_VERSION */ diff --git a/src/docs/sphinx/CompleteXMLSchema.rst b/src/docs/sphinx/CompleteXMLSchema.rst index eb52d4dbff6..41dd097aa0b 100644 --- a/src/docs/sphinx/CompleteXMLSchema.rst +++ b/src/docs/sphinx/CompleteXMLSchema.rst @@ -120,6 +120,13 @@ Element: CarmanKozenyPermeability .. include:: ../../coreComponents/schema/docs/CarmanKozenyPermeability.rst +.. _XML_Catalyst: + +Element: Catalyst +================= +.. include:: ../../coreComponents/schema/docs/Catalyst.rst + + .. _XML_CellElementRegion: Element: CellElementRegion @@ -1349,6 +1356,13 @@ Datastructure: CarmanKozenyPermeability .. include:: ../../coreComponents/schema/docs/CarmanKozenyPermeability_other.rst +.. _DATASTRUCTURE_Catalyst: + +Datastructure: Catalyst +======================= +.. include:: ../../coreComponents/schema/docs/Catalyst_other.rst + + .. _DATASTRUCTURE_CellElementRegion: Datastructure: CellElementRegion From 69912fff7c0c64f3e70654c24326f82e3e3a1cb9 Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Mon, 13 Mar 2023 18:12:18 +0100 Subject: [PATCH 04/17] VTK: add mangling compatibility This commit onward should be compatible with `thirdPartyLibrary` commit acf1e3836ec1cc89996222fabe9458f1a56c1f52 and onward --- .../fileIO/vtk/VTKPolyDataWriterInterface.hpp | 4 ++ src/docs/doxygen/GeosxConfig.hpp | 62 +++++-------------- 2 files changed, 18 insertions(+), 48 deletions(-) diff --git a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp index 6990cfec159..2768333630d 100644 --- a/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp +++ b/src/coreComponents/fileIO/vtk/VTKPolyDataWriterInterface.hpp @@ -22,9 +22,13 @@ #include "fileIO/vtk/VTKVTMWriter.hpp" #include "codingUtilities/EnumStrings.hpp" +#include // for mangled namespace + +VTK_ABI_NAMESPACE_BEGIN class vtkUnstructuredGrid; class vtkPointData; class vtkCellData; +VTK_ABI_NAMESPACE_END namespace geos { diff --git a/src/docs/doxygen/GeosxConfig.hpp b/src/docs/doxygen/GeosxConfig.hpp index 198305cfceb..affc50351e6 100644 --- a/src/docs/doxygen/GeosxConfig.hpp +++ b/src/docs/doxygen/GeosxConfig.hpp @@ -5,8 +5,8 @@ * Contains a CMake-generated list of macros that define a particular build configuration. */ -#ifndef GEOS_COMMON_CONFIG_HPP -#define GEOS_COMMON_CONFIG_HPP +#ifndef GEOSX_COMMON_CONFIG_HPP +#define GEOSX_COMMON_CONFIG_HPP /// Enables floating point exceptions #define GEOSX_USE_FPE @@ -17,9 +17,6 @@ /// Enables use of Caliper (CMake option ENABLE_CALIPER) #define GEOSX_USE_CALIPER -/// Enables use of Caliper (CMake option ENABLE_ADIAK) -/* #undef GEOSX_USE_ADIAK */ - /// Enables use of CHAI (CMake option ENABLE_CHAI) #define GEOSX_USE_CHAI @@ -30,16 +27,13 @@ #define GEOSX_USE_MPI /// Enables use of OpenMP (CMake option ENABLE_OPENMP) -/* #undef GEOSX_USE_OPENMP */ +#define GEOSX_USE_OPENMP /// Enables use of CUDA (CMake option ENABLE_CUDA) -/* #undef GEOS_USE_CUDA */ +#define GEOSX_USE_CUDA /// Enables use of CUDA NVToolsExt (CMake option ENABLE_CUDA_NVTOOLSEXT) -/* #undef GEOS_USE_CUDA_NVTOOLSEXT */ - -/// Enables use of HIP (CMake option ENABLE_HIP) -/* #undef GEOS_USE_HIP */ +#define GEOSX_USE_CUDA_NVTOOLSEXT /// Enables use of PVTPackage (CMake option ENABLE_PVTPackage) #define GEOSX_USE_PVTPackage @@ -65,26 +59,8 @@ /// Enables use of Hypre library (CMake option ENABLE_HYPRE) #define GEOSX_USE_HYPRE -#if defined( GEOSX_USE_HYPRE ) - /// Parsed hypre version information - #define HYPRE_VERSION_MAJOR 2 - /// Parsed hypre version information - #define HYPRE_VERSION_MINOR 28 - /// Parsed hypre version information - #define HYPRE_VERSION_PATCH 0 -#endif - -/// Denotes HYPRE using CPU -#define GEOS_USE_HYPRE_CPU 0 -/// Denotes HYPRE using CUDA -#define GEOS_USE_HYPRE_CUDA 1 -/// Denotes HYPRE using HIP -#define GEOS_USE_HYPRE_HIP 2 -/// Macro determining what parellel interface hypre is using -#define GEOS_USE_HYPRE_DEVICE GEOS_USE_HYPRE_CPU - -/// Enables use of SuperLU_dist library through HYPRE (CMake option ENABLE_SUPERLU_DIST) -#define GEOSX_USE_SUPERLU_DIST +/// Macro defined when using cuda in HYPRE (CMake option ENABLE_HYPRE_CUDA) +#define GEOSX_USE_HYPRE_CUDA /// Enables use of PETSc library (CMake option ENABLE_PETSC) #define GEOSX_USE_PETSC @@ -122,9 +98,6 @@ /// An integer flag representing the type that globalIndex will be aliased to. #define GEOSX_GLOBALINDEX_TYPE_FLAG 2 -/// The default block size for GEOSX on this platform -#define GEOSX_BLOCK_SIZE 32 - /// Version information for HDF5 #define HDF5_VERSION 1.12.1 @@ -132,10 +105,10 @@ #define Conduit_VERSION 0.8.6 /// Version information for RAJA -#define RAJA_VERSION 2022.10.5 +#define RAJA_VERSION 2022.3.0 /// Version information for umpire -#define umpire_VERSION 2022.10.0 +#define umpire_VERSION 2022.3.0 /// Version information for chai /* #undef chai_VERSION */ @@ -152,7 +125,7 @@ /// Version information for ParMetis #define PARAMETIS_VERSION 4.0.3 -/// Version information for scotch +/// Version information for scotch #define scotch_VERSION 6.0.9 /// Version information for superlu_dist @@ -162,24 +135,17 @@ #define suitesparse_VERSION 5.7.9 /// Version information for VTK -#define VTK_VERSION 9.2.20230719 +#define VTK_VERSION 9.2.20230310 /// Version information for fmt -#define fmt_VERSION 9.1.0 +#define fmt_VERSION 8.0.1 /// Version information for python -#define Python3_VERSION 3.11.3 +/* #undef Python3_VERSION */ /// Version information for CUDAToolkit /* #undef CUDAToolkit_VERSION */ -#if defined(GEOS_USE_CUDA) || defined(GEOS_USE_HIP) -// This needs to be placed into this header in order to appropriately replace -// the old usage of GEOS_USE_CUDA, since we detect whether it is defined -// rather than a value, not having it in the *same* header can cauase nebulous -// compilation problems including the USD of arrays changing depending the scope -#define GEOS_USE_DEVICE -#endif -#endif /* GEOS_CONFIG_HPP */ +#endif /* GEOSX_CONFIG_HPP */ From e9bcd67173e4de55a08ea4d61903c2d2057a1ad4 Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Thu, 20 Jul 2023 18:03:54 +0200 Subject: [PATCH 05/17] GCC13: fixes for warnings and errors --- .../mesh/generators/VTKFaceBlockUtilities.cpp | 2 +- .../constitutiveTests/testMultiFluid.cpp | 8 +-- src/docs/doxygen/GeosxConfig.hpp | 62 ++++++++++++++----- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/coreComponents/mesh/generators/VTKFaceBlockUtilities.cpp b/src/coreComponents/mesh/generators/VTKFaceBlockUtilities.cpp index 6bf7b886a43..dbf071a0f2b 100644 --- a/src/coreComponents/mesh/generators/VTKFaceBlockUtilities.cpp +++ b/src/coreComponents/mesh/generators/VTKFaceBlockUtilities.cpp @@ -583,7 +583,7 @@ ArrayOfArrays< localIndex > computeElem2dToNodes( vtkIdType num2dElements, std::set< localIndex > tmp; for( auto j = 0; j < faceToNodes[faceIndex].size(); ++j ) { - localIndex const & nodeIndex = faceToNodes[faceIndex][j]; + localIndex const nodeIndex = faceToNodes[faceIndex][j]; tmp.insert( nodeIndex ); } for( localIndex const & nodeIndex: tmp ) diff --git a/src/coreComponents/unitTests/constitutiveTests/testMultiFluid.cpp b/src/coreComponents/unitTests/constitutiveTests/testMultiFluid.cpp index 16c9ef5d22e..ee15dc8e160 100644 --- a/src/coreComponents/unitTests/constitutiveTests/testMultiFluid.cpp +++ b/src/coreComponents/unitTests/constitutiveTests/testMultiFluid.cpp @@ -177,7 +177,7 @@ void testNumericalDerivatives( MultiFluidBase & fluid, auto const & phaseDensCopy = GET_FLUID_DATA( fluidCopy, fields::multifluid::phaseDensity ); auto const & phaseViscCopy = GET_FLUID_DATA( fluidCopy, fields::multifluid::phaseViscosity ); auto const & phaseCompFracCopy = GET_FLUID_DATA( fluidCopy, fields::multifluid::phaseCompFraction ); - auto const & totalDensCopy = GET_FLUID_DATA( fluidCopy, fields::multifluid::totalDensity ); + auto totalDensCopy = [&]() { return GET_FLUID_DATA( fluidCopy, fields::multifluid::totalDensity ); }; #undef GET_FLUID_DATA @@ -211,7 +211,7 @@ void testNumericalDerivatives( MultiFluidBase & fluid, dP, relTol, absTol, "phaseDens", "Pres", phases ); checkDerivative( phaseViscCopy.toSliceConst(), phaseVisc.value.toSliceConst(), dPhaseVisc[Deriv::dP].toSliceConst(), dP, relTol, absTol, "phaseVisc", "Pres", phases ); - checkDerivative( totalDensCopy, totalDens.value, dTotalDens[Deriv::dP], + checkDerivative( totalDensCopy(), totalDens.value, dTotalDens[Deriv::dP], dP, relTol, absTol, "totalDens", "Pres" ); checkDerivative( phaseCompFracCopy.toSliceConst(), phaseCompFrac.value.toSliceConst(), dPhaseCompFrac[Deriv::dP].toSliceConst(), dP, relTol, absTol, "phaseCompFrac", "Pres", phases, components ); @@ -228,7 +228,7 @@ void testNumericalDerivatives( MultiFluidBase & fluid, dT, relTol, absTol, "phaseDens", "Temp", phases ); checkDerivative( phaseViscCopy.toSliceConst(), phaseVisc.value.toSliceConst(), dPhaseVisc[Deriv::dT].toSliceConst(), dT, relTol, absTol, "phaseVisc", "Temp", phases ); - checkDerivative( totalDensCopy, totalDens.value, dTotalDens[Deriv::dT], + checkDerivative( totalDensCopy(), totalDens.value, dTotalDens[Deriv::dT], dT, relTol, absTol, "totalDens", "Temp" ); checkDerivative( phaseCompFracCopy.toSliceConst(), phaseCompFrac.value.toSliceConst(), dPhaseCompFrac[Deriv::dT].toSliceConst(), dT, relTol, absTol, "phaseCompFrac", "Temp", phases, components ); @@ -277,7 +277,7 @@ void testNumericalDerivatives( MultiFluidBase & fluid, dC, relTol, absTol, "phaseDens", var, phases ); checkDerivative( phaseViscCopy.toSliceConst(), phaseVisc.value.toSliceConst(), dPhaseVisc[Deriv::dC+jc].toSliceConst(), dC, relTol, absTol, "phaseVisc", var, phases ); - checkDerivative( totalDensCopy, totalDens.value, dTotalDens[Deriv::dC+jc], + checkDerivative( totalDensCopy(), totalDens.value, dTotalDens[Deriv::dC+jc], dC, relTol, absTol, "totalDens", var ); checkDerivative( phaseCompFracCopy.toSliceConst(), phaseCompFrac.value.toSliceConst(), dPhaseCompFrac[Deriv::dC+jc].toSliceConst(), dC, relTol, absTol, "phaseCompFrac", var, phases, components ); diff --git a/src/docs/doxygen/GeosxConfig.hpp b/src/docs/doxygen/GeosxConfig.hpp index affc50351e6..198305cfceb 100644 --- a/src/docs/doxygen/GeosxConfig.hpp +++ b/src/docs/doxygen/GeosxConfig.hpp @@ -5,8 +5,8 @@ * Contains a CMake-generated list of macros that define a particular build configuration. */ -#ifndef GEOSX_COMMON_CONFIG_HPP -#define GEOSX_COMMON_CONFIG_HPP +#ifndef GEOS_COMMON_CONFIG_HPP +#define GEOS_COMMON_CONFIG_HPP /// Enables floating point exceptions #define GEOSX_USE_FPE @@ -17,6 +17,9 @@ /// Enables use of Caliper (CMake option ENABLE_CALIPER) #define GEOSX_USE_CALIPER +/// Enables use of Caliper (CMake option ENABLE_ADIAK) +/* #undef GEOSX_USE_ADIAK */ + /// Enables use of CHAI (CMake option ENABLE_CHAI) #define GEOSX_USE_CHAI @@ -27,13 +30,16 @@ #define GEOSX_USE_MPI /// Enables use of OpenMP (CMake option ENABLE_OPENMP) -#define GEOSX_USE_OPENMP +/* #undef GEOSX_USE_OPENMP */ /// Enables use of CUDA (CMake option ENABLE_CUDA) -#define GEOSX_USE_CUDA +/* #undef GEOS_USE_CUDA */ /// Enables use of CUDA NVToolsExt (CMake option ENABLE_CUDA_NVTOOLSEXT) -#define GEOSX_USE_CUDA_NVTOOLSEXT +/* #undef GEOS_USE_CUDA_NVTOOLSEXT */ + +/// Enables use of HIP (CMake option ENABLE_HIP) +/* #undef GEOS_USE_HIP */ /// Enables use of PVTPackage (CMake option ENABLE_PVTPackage) #define GEOSX_USE_PVTPackage @@ -59,8 +65,26 @@ /// Enables use of Hypre library (CMake option ENABLE_HYPRE) #define GEOSX_USE_HYPRE -/// Macro defined when using cuda in HYPRE (CMake option ENABLE_HYPRE_CUDA) -#define GEOSX_USE_HYPRE_CUDA +#if defined( GEOSX_USE_HYPRE ) + /// Parsed hypre version information + #define HYPRE_VERSION_MAJOR 2 + /// Parsed hypre version information + #define HYPRE_VERSION_MINOR 28 + /// Parsed hypre version information + #define HYPRE_VERSION_PATCH 0 +#endif + +/// Denotes HYPRE using CPU +#define GEOS_USE_HYPRE_CPU 0 +/// Denotes HYPRE using CUDA +#define GEOS_USE_HYPRE_CUDA 1 +/// Denotes HYPRE using HIP +#define GEOS_USE_HYPRE_HIP 2 +/// Macro determining what parellel interface hypre is using +#define GEOS_USE_HYPRE_DEVICE GEOS_USE_HYPRE_CPU + +/// Enables use of SuperLU_dist library through HYPRE (CMake option ENABLE_SUPERLU_DIST) +#define GEOSX_USE_SUPERLU_DIST /// Enables use of PETSc library (CMake option ENABLE_PETSC) #define GEOSX_USE_PETSC @@ -98,6 +122,9 @@ /// An integer flag representing the type that globalIndex will be aliased to. #define GEOSX_GLOBALINDEX_TYPE_FLAG 2 +/// The default block size for GEOSX on this platform +#define GEOSX_BLOCK_SIZE 32 + /// Version information for HDF5 #define HDF5_VERSION 1.12.1 @@ -105,10 +132,10 @@ #define Conduit_VERSION 0.8.6 /// Version information for RAJA -#define RAJA_VERSION 2022.3.0 +#define RAJA_VERSION 2022.10.5 /// Version information for umpire -#define umpire_VERSION 2022.3.0 +#define umpire_VERSION 2022.10.0 /// Version information for chai /* #undef chai_VERSION */ @@ -125,7 +152,7 @@ /// Version information for ParMetis #define PARAMETIS_VERSION 4.0.3 -/// Version information for scotch +/// Version information for scotch #define scotch_VERSION 6.0.9 /// Version information for superlu_dist @@ -135,17 +162,24 @@ #define suitesparse_VERSION 5.7.9 /// Version information for VTK -#define VTK_VERSION 9.2.20230310 +#define VTK_VERSION 9.2.20230719 /// Version information for fmt -#define fmt_VERSION 8.0.1 +#define fmt_VERSION 9.1.0 /// Version information for python -/* #undef Python3_VERSION */ +#define Python3_VERSION 3.11.3 /// Version information for CUDAToolkit /* #undef CUDAToolkit_VERSION */ +#if defined(GEOS_USE_CUDA) || defined(GEOS_USE_HIP) +// This needs to be placed into this header in order to appropriately replace +// the old usage of GEOS_USE_CUDA, since we detect whether it is defined +// rather than a value, not having it in the *same* header can cauase nebulous +// compilation problems including the USD of arrays changing depending the scope +#define GEOS_USE_DEVICE +#endif -#endif /* GEOSX_CONFIG_HPP */ +#endif /* GEOS_CONFIG_HPP */ From 9c9eccfe94de5667adf6045c63fcbda4a8640d7f Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Tue, 8 Aug 2023 18:17:57 +0200 Subject: [PATCH 06/17] Catalyst: improve blueprint formatting for multi-region support --- .../fileIO/Outputs/CatalystOutput.cpp | 628 +++++++++++++----- 1 file changed, 463 insertions(+), 165 deletions(-) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 57baf63de7f..cd1972c92de 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -3,12 +3,12 @@ * SPDX-License-Identifier: LGPL-2.1-only * * Copyright (c) 2018-2020 Lawrence Livermore National Security LLC - * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University - * Copyright (c) 2018-2020 TotalEnergies - * Copyright (c) 2019- GEOSX Contributors - * All rights reserved + * Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior + * University Copyright (c) 2018-2020 TotalEnergies Copyright (c) 2019- GEOSX + * Contributors All rights reserved * - * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. + * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS + * files for details. * ------------------------------------------------------------------------------------------------------------ */ @@ -16,7 +16,7 @@ * @file CatalystOutput.cpp */ -//GEOS +// GEOS #include "CatalystOutput.hpp" #include "fileIO/Catalyst/CatalystActions.hpp" @@ -25,49 +25,337 @@ #include "mesh/MeshLevel.hpp" #include "mesh/mpiCommunications/CommunicationTools.hpp" -//TPL +// TPL #include #include -//std +// std +#include #include +#include +#include #include +#include #include #include #include -namespace -{ +namespace { -// Change all the node associated fields to have vertex association and delete the "nodes" element entry -void SanitizeNode(conduit::Node& blueprintNode) -{ - if (blueprintNode["topologies"].number_of_children() > 2) - { - GEOS_ERROR("CatalystOutput: Not yet support for multiple regions"); +void SanitizeSimpleNode(conduit::Node& channel) { + auto& blueprintNode = channel["data"]; + if (blueprintNode["topologies"].number_of_children() > 1) { + GEOS_ERROR( + "CatalystOutput: multi region passed to simple region code path"); return; } - blueprintNode.remove("topologies/nodes"); std::string topoName = blueprintNode["topologies"].child(0).name(); auto& fields = blueprintNode["fields"]; - for ( conduit::index_t iField = 0; iField < fields.number_of_children(); ++iField ) - { + for (conduit::index_t iField = 0; iField < fields.number_of_children(); + ++iField) { auto& field = fields.child(iField); - if (field["topology"].as_char8_str() == std::string("nodes")) - { + if (field["topology"].as_char8_str() == std::string("nodes")) { field["topology"] = topoName; field["association"] = "vertex"; } } } +void SanitizeMultiNodeInSitu(conduit::Node& channel) { + auto& blueprintNode = channel["data"]; + for (conduit::index_t iTopo = 0; + iTopo < blueprintNode["topologies"].number_of_children(); ++iTopo) { + auto& topology = blueprintNode["topologies"].child(iTopo); + std::string topoName = topology.name(); + auto& iMesh = blueprintNode[topoName]; + + iMesh["state"].update(blueprintNode["state"]); + + iMesh["coordsets"].update(blueprintNode["coordsets"]); + iMesh["topologies/" + topoName].update(topology); + + auto& dstFields = iMesh["fields"]; + const auto& srcFields = blueprintNode["fields"]; + for (conduit::index_t iField = 0; iField < srcFields.number_of_children(); + ++iField) { + const auto& srcField = srcFields.child(iField); + if (srcField["topology"].as_char8_str() == std::string("nodes")) { + auto& dstField = dstFields[srcField.name()]; + dstField.update(srcField); + dstField["topology"] = topoName; + dstField["association"] = "vertex"; + } else if (srcField["topology"].as_char8_str() == topoName) { + auto& dstField = dstFields[srcField.name()]; + dstField.update(srcField); + } + } + if (dstFields.number_of_children() == 0) { + iMesh.remove("fields"); + } + } + blueprintNode.remove("state"); + blueprintNode.remove("topologies"); + blueprintNode.remove("coordsets"); + blueprintNode.remove("fields"); +} + +struct ScopedData {}; + +template +struct ScopedTypedData : public ScopedData { + public: + ScopedTypedData(T data) : Data(data) {} + + private: + T Data; +}; + +struct ScopedDataContainer : public std::vector> { + public: + template + void Add(T data) { + this->emplace_back( + std::unique_ptr(new ScopedTypedData(data))); + } +}; + +template +std::set ConstructNodeMapping(const IndexT* connectivity, + conduit::index_t size) { + std::set nodeMapping; + for (conduit::index_t iConnect = 0; iConnect < size; ++iConnect) { + nodeMapping.emplace( + static_cast(connectivity[iConnect])); + } + return nodeMapping; +} + +std::set ConstructNodeMapping( + const conduit::Node& connectivity) { + std::string dtype = connectivity.dtype().name(); + conduit::index_t numValues = connectivity.dtype().number_of_elements(); +#define DispatchNameToType(name, type) \ + do { \ + if (dtype == name) { \ + auto res = ConstructNodeMapping( \ + static_cast(connectivity.data_ptr()), numValues); \ + return res; \ + } \ + } while (0) + DispatchNameToType("int8", signed char); + DispatchNameToType("int16", short); + DispatchNameToType("int32", std::int32_t); + DispatchNameToType("int64", std::int64_t); + DispatchNameToType("uint8", unsigned char); + DispatchNameToType("uint16", unsigned short); + DispatchNameToType("uint32", std::uint32_t); + DispatchNameToType("uint64", std::uint64_t); + DispatchNameToType("float32", float); + DispatchNameToType("float64", double); +#undef DispatchNameToType + GEOS_ERROR("Could not dispatch connectivity with type " + dtype); + return std::set(); +} + +template +std::shared_ptr> ConstructConnectivity( + const std::set& nodeMapping, const DataT* src, + conduit_index_t size) { + auto res = std::make_shared>(size); + for (conduit::index_t iConnect = 0; iConnect < size; ++iConnect) { + (*res)[iConnect] = std::distance( + nodeMapping.begin(), + nodeMapping.find(static_cast(src[iConnect]))); + } + return res; +} + +bool DispatchConnectivity(const std::set& nodeMapping, + const conduit::Node& src, conduit::Node& dst, + ScopedDataContainer& dataContainer) { + std::string dtype = src.dtype().name(); + conduit_index_t size = src.dtype().number_of_elements(); +#define DispatchNameToType(name, type) \ + do { \ + if (dtype == name) { \ + auto res = ConstructConnectivity( \ + nodeMapping, static_cast(src.data_ptr()), size); \ + dataContainer.Add(res); \ + dst.set_external(res->data(), res->size()); \ + return true; \ + } \ + } while (0) + DispatchNameToType("int8", signed char); + DispatchNameToType("int16", short); + DispatchNameToType("int32", std::int32_t); + DispatchNameToType("int64", std::int64_t); + DispatchNameToType("uint8", unsigned char); + DispatchNameToType("uint16", unsigned short); + DispatchNameToType("uint32", std::uint32_t); + DispatchNameToType("uint64", std::uint64_t); + DispatchNameToType("float32", float); + DispatchNameToType("float64", double); +#undef DispatchNameToType + GEOS_ERROR("Could not dispatch connectivity type " + dtype); + return false; +} + +template +std::shared_ptr> ConstructReducedField( + const std::set& nodeMapping, const DataT* src, + std::size_t offset, std::size_t stride) { + auto res = std::make_shared>(nodeMapping.size(), 0); + auto resIt = res->begin(); + for (const auto& nodeIndex : nodeMapping) { + *resIt = src[static_cast(nodeIndex) * stride + offset]; + resIt++; + } + return res; +} + +bool DispatchFieldToContainers(const std::set& nodeMapping, + const conduit::Node& src, conduit::Node& dst, + ScopedDataContainer& dataContainer, + std::size_t offset = 0, std::size_t stride = 0) { + std::string dtype = src.dtype().name(); +#define DispatchNameToType(name, type) \ + do { \ + if (dtype == name) { \ + auto res = ConstructReducedField( \ + nodeMapping, static_cast(src.data_ptr()), offset, \ + stride); \ + dataContainer.Add(res); \ + dst.set_external(res->data(), res->size()); \ + return true; \ + } \ + } while (0) + DispatchNameToType("int8", signed char); + DispatchNameToType("int16", short); + DispatchNameToType("int32", std::int32_t); + DispatchNameToType("int64", std::int64_t); + DispatchNameToType("uint8", unsigned char); + DispatchNameToType("uint16", unsigned short); + DispatchNameToType("uint32", std::uint32_t); + DispatchNameToType("uint64", std::uint64_t); + DispatchNameToType("float32", float); + DispatchNameToType("float64", double); +#undef DispatchNameToType + GEOS_ERROR("Could not dispatch field type " + dtype); + return false; } -namespace geos -{ +void SanitizeMultiNodeInTransit(conduit::Node& channel, + ScopedDataContainer& dataContainer) { + auto& blueprintNode = channel["data"]; + for (conduit::index_t iTopo = 0; + iTopo < blueprintNode["topologies"].number_of_children(); ++iTopo) { + const auto& topology = blueprintNode["topologies"].child(iTopo); + + std::string topoName = topology.name(); + auto& iMesh = blueprintNode[topoName]; + + iMesh["state"].update(blueprintNode["state"]); + + const auto& srcConnectivity = topology["elements/connectivity"]; + std::set nodeMapping = ConstructNodeMapping(srcConnectivity); + + iMesh["topologies/" + topoName + "/type"].update(topology["type"]); + iMesh["topologies/" + topoName + "/elements/shape"].update( + topology["elements/shape"]); + iMesh["topologies/" + topoName + "/coordset"].update(topology["coordset"]); + + auto& dstConnectivity = + iMesh["topologies/" + topoName + "/elements/connectivity"]; + if (!DispatchConnectivity(nodeMapping, srcConnectivity, dstConnectivity, + dataContainer)) { + GEOS_ERROR( + "CatalystOutput: SanitizeMultiNodeInTransit: Dispatch of " + "connecitivities failed"); + } + + std::string coordsetName = + std::string("coordsets/") + topology["coordset"].as_char8_str(); + auto& coords = blueprintNode[coordsetName]; + iMesh[coordsetName + "/type"].update(coords["type"]); + std::map xyz = { + {"x", coords["values/x"]}, + {"y", coords["values/y"]}, + {"z", coords["values/z"]}}; + std::size_t offset = 0; + std::size_t stride = 1; + if (!coords.is_contiguous()) { + stride = 3; + } + for (const auto& component : xyz) { + auto& dst = iMesh[coordsetName + "/values/" + component.first]; + if (!DispatchFieldToContainers(nodeMapping, component.second, dst, + dataContainer, offset, stride)) { + GEOS_ERROR( + "CatalystOutput: SanitizeMultiNodeInTransit: Could not " + "dispatch " + + component.second.name() + " field."); + } + } + + const auto& srcFields = blueprintNode["fields"]; + auto& dstFields = iMesh["fields"]; + for (conduit::index_t iField = 0; iField < srcFields.number_of_children(); + ++iField) { + const auto& srcField = srcFields.child(iField); + bool isPointData = + srcField["topology"].as_char8_str() == std::string("nodes"); + bool isCellData = (srcField["topology"].as_char8_str() == topoName); + if (!isPointData && !isCellData) { + continue; + } + auto& dstField = dstFields[srcField.name()]; + if (isCellData) { + dstField.update(srcField); + continue; + } + dstField["topology"] = topoName; + dstField["volume_dependent"].update(srcField["volume_dependent"]); + dstField["association"] = "vertex"; + if (!DispatchFieldToContainers(nodeMapping, srcField["values"], + dstField["values"], dataContainer)) { + GEOS_ERROR( + "CatalystOutput: SanitizeMultiNodeInTransit: Could not " + "dispatch " + + srcField.name() + " field."); + } + } + if (dstFields.number_of_children() == 0) { + iMesh.remove("fields"); + } + } + blueprintNode.remove("state"); + blueprintNode.remove("topologies"); + blueprintNode.remove("coordsets"); + blueprintNode.remove("fields"); +} + +void SanitizeNode(conduit::Node& channel, bool isInTransit, + ScopedDataContainer& dataContainer) { + channel.remove("data/topologies/nodes"); + if (channel["data/topologies"].number_of_children() > 1) { + channel["type"] = "multimesh"; + if (isInTransit) { + SanitizeMultiNodeInTransit(channel, dataContainer); + return; + } + SanitizeMultiNodeInSitu(channel); + return; + } + + SanitizeSimpleNode(channel); + return; +} + +} // namespace + +namespace geos { -struct CatalystOutput::CatalystInternals -{ +struct CatalystOutput::CatalystInternals { bool initialized = false; std::string scripts = ""; std::string implementation = ""; @@ -75,9 +363,7 @@ struct CatalystOutput::CatalystInternals std::string adiosConfig = ""; std::string channelName = "fullfield"; - void initializeCatalyst() - { - + void initializeCatalyst() { std::string scriptsCopy = this->scripts; std::vector scriptList; std::string delimiter = ":"; @@ -89,40 +375,37 @@ struct CatalystOutput::CatalystInternals scriptsCopy.erase(0, pos + delimiter.length()); } - if ( scriptList.empty() || scriptList[0] == "" ) - { + if (scriptList.empty() || scriptList[0] == "") { GEOS_ERROR("CatalystOutput: Constructor: no catalyst scripts found."); } conduit::Node initializer; - for (std::size_t iScr = 0; iScr < scriptList.size(); ++iScr) - { - initializer["catalyst/scripts/script" + std::to_string(iScr)] = scriptList[iScr]; + for (std::size_t iScr = 0; iScr < scriptList.size(); ++iScr) { + initializer["catalyst/scripts/script" + std::to_string(iScr)] = + scriptList[iScr]; } - if ( !this->implementation.empty() ) - { + if (!this->implementation.empty()) { initializer["catalyst_load/implementation"] = this->implementation; - if ( !this->implementationPath.empty() ) - { - initializer["catalyst_load/search_paths/" + this->implementation] = this->implementationPath; + if (!this->implementationPath.empty()) { + initializer["catalyst_load/search_paths/" + this->implementation] = + this->implementationPath; } } - if ( this->implementation == "adios" || - (this->implementation.empty() && - std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios")) - { - if ( this->adiosConfig == "" ) - { - GEOS_ERROR("CatalystOutput: Constructor: Cannot use the catalyst/adios2 implementation without providing an adios2 configuration file."); + if (this->implementation == "adios" || + (this->implementation.empty() && + std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios")) { + if (this->adiosConfig == "") { + GEOS_ERROR( + "CatalystOutput: Constructor: Cannot use the catalyst/adios2 " + "implementation without providing an adios2 configuration file."); } initializer["adios/config"] = this->adiosConfig; } auto capsule = GenericConduitCapsule(&initializer); - if( !CatalystInitialize(&capsule) ) - { + if (!CatalystInitialize(&capsule)) { GEOS_ERROR("CatalystOutput: Constructor: catalyst failed to initialize."); } @@ -131,55 +414,51 @@ struct CatalystOutput::CatalystInternals }; /////////////////////////////////////////////////////////////////////////////////////////////////// -CatalystOutput::CatalystOutput( std::string const & name, - Group * const parent ) - : BlueprintOutput( name, - parent ) - , internal( std::unique_ptr( new CatalystInternals() ) ) -{ - - this->registerWrapper( "scripts", &this->internal->scripts ). - setApplyDefaultValue( "" ). - setInputFlag( dataRepository::InputFlags::REQUIRED ). - setDescription( "Column separated paths to the catalyst scripts." ); - - this->registerWrapper( "implementation", &this->internal->implementation ). - setApplyDefaultValue( "" ). - setInputFlag( dataRepository::InputFlags::OPTIONAL ). - setDescription( "Name of the catalyst implementation to use." ); - - this->registerWrapper( "implementationPath", &this->internal->implementationPath ). - setApplyDefaultValue( "" ). - setInputFlag( dataRepository::InputFlags::OPTIONAL ). - setDescription( "Path to the catalyst the implementation to use." ); - - this->registerWrapper( "adiosConfig", &this->internal->adiosConfig ). - setApplyDefaultValue( "" ). - setInputFlag( dataRepository::InputFlags::OPTIONAL ). - setDescription( "Path to the adios configuration file when using the catalyst-adios implementation." ); - - this->registerWrapper( "fullFieldChannelName", &this->internal->channelName ). - setApplyDefaultValue( "" ). - setInputFlag( dataRepository::InputFlags::OPTIONAL ). - setDescription( "Name to give to the channel passing the full field data." ); - +CatalystOutput::CatalystOutput(std::string const& name, Group* const parent) + : BlueprintOutput(name, parent), + internal(std::unique_ptr(new CatalystInternals())) { + this->registerWrapper("scripts", &this->internal->scripts) + .setApplyDefaultValue("") + .setInputFlag(dataRepository::InputFlags::REQUIRED) + .setDescription("Column separated paths to the catalyst scripts."); + + this->registerWrapper("implementation", &this->internal->implementation) + .setApplyDefaultValue("") + .setInputFlag(dataRepository::InputFlags::OPTIONAL) + .setDescription("Name of the catalyst implementation to use."); + + this->registerWrapper("implementationPath", + &this->internal->implementationPath) + .setApplyDefaultValue("") + .setInputFlag(dataRepository::InputFlags::OPTIONAL) + .setDescription("Path to the catalyst the implementation to use."); + + this->registerWrapper("adiosConfig", &this->internal->adiosConfig) + .setApplyDefaultValue("") + .setInputFlag(dataRepository::InputFlags::OPTIONAL) + .setDescription( + "Path to the adios configuration file when using the catalyst-adios " + "implementation."); + + this->registerWrapper("fullFieldChannelName", &this->internal->channelName) + .setApplyDefaultValue("") + .setInputFlag(dataRepository::InputFlags::OPTIONAL) + .setDescription( + "Name to give to the channel passing the full field data."); } /////////////////////////////////////////////////////////////////////////////////////////////////// CatalystOutput::~CatalystOutput() = default; /////////////////////////////////////////////////////////////////////////////////////////////////// -bool CatalystOutput::execute( real64 const time_n, - real64 const /*dt*/, - integer const cycleNumber, - integer const /*eventCounter*/, - real64 const /*eventProgress*/, - DomainPartition & domain ) -{ +bool CatalystOutput::execute(real64 const time_n, real64 const /*dt*/, + integer const cycleNumber, + integer const /*eventCounter*/, + real64 const /*eventProgress*/, + DomainPartition& domain) { GEOS_MARK_FUNCTION; - if ( !this->internal->initialized ) - { + if (!this->internal->initialized) { this->internal->initializeCatalyst(); } @@ -188,80 +467,101 @@ bool CatalystOutput::execute( real64 const time_n, catalystState["timestep"].set(cycleNumber); catalystState["time"].set(time_n); - auto& channel = executeRoot["catalyst/channels/" + this->internal->channelName]; + auto& channel = + executeRoot["catalyst/channels/" + this->internal->channelName]; channel["type"] = "mesh"; auto& meshGEOSRoot = channel["data"]; this->mapMesh(time_n, cycleNumber, domain, meshGEOSRoot); - ::SanitizeNode(meshGEOSRoot); + + bool isInTransit = + this->internal->implementation == "adios" || + (this->internal->implementation.empty() && + std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios"); + + ScopedDataContainer dataScoping; + ::SanitizeNode(channel, isInTransit, dataScoping); // Add additional info if the implementation is adios based - if ( this->internal->implementation == "adios" || - (this->internal->implementation.empty() && - std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios") ) - { - int const thisRank = MpiWrapper::commRank( MPI_COMM_GEOSX ); - int const commSize = MpiWrapper::commSize( MPI_COMM_GEOSX ); - - MeshLevel const & meshLevel = domain.getMeshBody( 0 ).getBaseDiscretization(); - - localIndex_array numberOfNodesPerRank( commSize ); - - MpiWrapper::allGather( - meshLevel.getNodeManager().size(), - numberOfNodesPerRank ); - - globalIndex offsetPoints = std::accumulate( - numberOfNodesPerRank.begin(), - numberOfNodesPerRank.begin() + thisRank, - 0 ); - - globalIndex totalNumberOfNodes = std::accumulate( - numberOfNodesPerRank.begin() + thisRank, - numberOfNodesPerRank.end(), - offsetPoints ); - - auto& mesh = channel[ "data" ]; - mesh[ "coordsets/coords/points_shape" ].set_uint64(totalNumberOfNodes); - mesh[ "coordsets/coords/points_start" ].set_uint64(offsetPoints); - mesh[ "coordsets/coords/points_count" ].set_uint64(numberOfNodesPerRank[ thisRank ]); - - // WARNING - // what follows on the elements only works if all the MPI partitions have the same number - // of corresponding subRegions. I do not know if this is the case. If it is not, we expose - // ourselves to hanging MPI communication here. - auto& topologies = mesh[ "topologies" ]; - meshLevel.getElemManager().forElementSubRegionsComplete< CellElementSubRegion >( - [&] ( localIndex, localIndex, ElementRegionBase const & region, CellElementSubRegion const & subRegion ) - { - std::string const topologyName = region.getName() + "-" + subRegion.getName(); - auto& topology = topologies[ topologyName ]; - - localIndex_array numberOfElementsPerRank( commSize ); - - MpiWrapper::allGather( - subRegion.size(), - numberOfElementsPerRank ); - - globalIndex offsetElements = std::accumulate( - numberOfElementsPerRank.begin(), - numberOfElementsPerRank.begin() + thisRank, - 0 ); - - globalIndex totalNumberOfElements = std::accumulate( - numberOfElementsPerRank.begin() + thisRank, - numberOfElementsPerRank.end(), - offsetElements ); - - topology["cells_shape"].set_uint64(totalNumberOfElements); - topology["cells_start"].set_uint64(offsetElements); - topology["cells_count"].set_uint64(numberOfElementsPerRank[ thisRank ]); - }); + if (isInTransit) { + int const thisRank = MpiWrapper::commRank(MPI_COMM_GEOSX); + int const commSize = MpiWrapper::commSize(MPI_COMM_GEOSX); + + MeshLevel const& meshLevel = domain.getMeshBody(0).getBaseDiscretization(); + + bool isMultiMesh = + (channel["type"].as_char8_str() == std::string("multimesh")); + conduit::index_t nMeshes = + (isMultiMesh ? channel["data"].number_of_children() : 1); + + for (conduit::index_t iMesh = 0; iMesh < nMeshes; ++iMesh) { + auto& mesh = + (isMultiMesh ? channel["data"].child(iMesh) : channel["data"]); + localIndex_array numberOfNodesPerRank(commSize); + + MpiWrapper::allGather(static_cast(mesh["coordsets"] + .child(0)["values"] + .child(0) + .dtype() + .number_of_elements()), + numberOfNodesPerRank); + + globalIndex offsetPoints = + std::accumulate(numberOfNodesPerRank.begin(), + numberOfNodesPerRank.begin() + thisRank, 0); + + globalIndex totalNumberOfNodes = + std::accumulate(numberOfNodesPerRank.begin() + thisRank, + numberOfNodesPerRank.end(), offsetPoints); + + mesh["coordsets/coords/points_shape"].set_uint64(totalNumberOfNodes); + mesh["coordsets/coords/points_start"].set_uint64(offsetPoints); + mesh["coordsets/coords/points_count"].set_uint64( + numberOfNodesPerRank[thisRank]); + + // WARNING + // what follows on the elements only works if all the MPI partitions have + // the same number of corresponding subRegions. I do not know if this is + // the case. If it is not, we expose ourselves to hanging MPI + // communication here. + auto& topologies = mesh["topologies"]; + if (topologies.number_of_children() != 1) { + GEOS_ERROR( + "CatalystOutput: execute: can only accept one topology per mesh"); + } + meshLevel.getElemManager() + .forElementSubRegionsComplete( + [&](localIndex, localIndex, ElementRegionBase const& region, + CellElementSubRegion const& subRegion) { + std::string const topologyName = + region.getName() + "-" + subRegion.getName(); + if (topologyName == topologies.child(0).name()) { + auto& topology = topologies[topologyName]; + + localIndex_array numberOfElementsPerRank(commSize); + + MpiWrapper::allGather(subRegion.size(), + numberOfElementsPerRank); + + globalIndex offsetElements = std::accumulate( + numberOfElementsPerRank.begin(), + numberOfElementsPerRank.begin() + thisRank, 0); + + globalIndex totalNumberOfElements = std::accumulate( + numberOfElementsPerRank.begin() + thisRank, + numberOfElementsPerRank.end(), offsetElements); + + topology["cells_shape"].set_uint64(totalNumberOfElements); + topology["cells_start"].set_uint64(offsetElements); + topology["cells_count"].set_uint64( + numberOfElementsPerRank[thisRank]); + } + }); + } } auto capsule = GenericConduitCapsule(&executeRoot); - if ( !CatalystExecute(&capsule) ) - { + if (!CatalystExecute(&capsule)) { GEOS_ERROR("CatalystOutput: execute: catalyst failed to execute."); return true; } @@ -270,29 +570,27 @@ bool CatalystOutput::execute( real64 const time_n, } /////////////////////////////////////////////////////////////////////////////////////////////////// -void CatalystOutput::cleanup( real64 const time_n, - integer const cycleNumber, - integer const eventCounter, - real64 const eventProgress, - DomainPartition & domain ) -{ +void CatalystOutput::cleanup(real64 const time_n, integer const cycleNumber, + integer const eventCounter, + real64 const eventProgress, + DomainPartition& domain) { GEOS_MARK_FUNCTION; - if ( this->execute( time_n, 0, cycleNumber, eventCounter, eventProgress, domain ) ) - { + if (this->execute(time_n, 0, cycleNumber, eventCounter, eventProgress, + domain)) { GEOS_ERROR("CatalystOutput: cleanup: last execute failed."); } conduit::Node emptyNode; auto capsule = GenericConduitCapsule(&emptyNode); - if ( !CatalystFinalize(&capsule) ) - { + if (!CatalystFinalize(&capsule)) { GEOS_ERROR("CatalystOutput: cleanup: finalize catalyst failed"); } this->internal->initialized = false; } -REGISTER_CATALOG_ENTRY( OutputBase, CatalystOutput, string const &, dataRepository::Group * const ) +REGISTER_CATALOG_ENTRY(OutputBase, CatalystOutput, string const&, + dataRepository::Group* const) -} +} // namespace geos From e8354fddca7799060d694ee6033bd5c56a3c166e Mon Sep 17 00:00:00 2001 From: Julien Fausty Date: Thu, 31 Aug 2023 10:59:26 +0200 Subject: [PATCH 07/17] CatalystOutput: remove now unrelevant shape information for adios --- .../fileIO/Outputs/CatalystOutput.cpp | 78 ------------------- src/docs/doxygen/GeosxConfig.hpp | 2 +- 2 files changed, 1 insertion(+), 79 deletions(-) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index cd1972c92de..8f153b41427 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -482,84 +482,6 @@ bool CatalystOutput::execute(real64 const time_n, real64 const /*dt*/, ScopedDataContainer dataScoping; ::SanitizeNode(channel, isInTransit, dataScoping); - // Add additional info if the implementation is adios based - if (isInTransit) { - int const thisRank = MpiWrapper::commRank(MPI_COMM_GEOSX); - int const commSize = MpiWrapper::commSize(MPI_COMM_GEOSX); - - MeshLevel const& meshLevel = domain.getMeshBody(0).getBaseDiscretization(); - - bool isMultiMesh = - (channel["type"].as_char8_str() == std::string("multimesh")); - conduit::index_t nMeshes = - (isMultiMesh ? channel["data"].number_of_children() : 1); - - for (conduit::index_t iMesh = 0; iMesh < nMeshes; ++iMesh) { - auto& mesh = - (isMultiMesh ? channel["data"].child(iMesh) : channel["data"]); - localIndex_array numberOfNodesPerRank(commSize); - - MpiWrapper::allGather(static_cast(mesh["coordsets"] - .child(0)["values"] - .child(0) - .dtype() - .number_of_elements()), - numberOfNodesPerRank); - - globalIndex offsetPoints = - std::accumulate(numberOfNodesPerRank.begin(), - numberOfNodesPerRank.begin() + thisRank, 0); - - globalIndex totalNumberOfNodes = - std::accumulate(numberOfNodesPerRank.begin() + thisRank, - numberOfNodesPerRank.end(), offsetPoints); - - mesh["coordsets/coords/points_shape"].set_uint64(totalNumberOfNodes); - mesh["coordsets/coords/points_start"].set_uint64(offsetPoints); - mesh["coordsets/coords/points_count"].set_uint64( - numberOfNodesPerRank[thisRank]); - - // WARNING - // what follows on the elements only works if all the MPI partitions have - // the same number of corresponding subRegions. I do not know if this is - // the case. If it is not, we expose ourselves to hanging MPI - // communication here. - auto& topologies = mesh["topologies"]; - if (topologies.number_of_children() != 1) { - GEOS_ERROR( - "CatalystOutput: execute: can only accept one topology per mesh"); - } - meshLevel.getElemManager() - .forElementSubRegionsComplete( - [&](localIndex, localIndex, ElementRegionBase const& region, - CellElementSubRegion const& subRegion) { - std::string const topologyName = - region.getName() + "-" + subRegion.getName(); - if (topologyName == topologies.child(0).name()) { - auto& topology = topologies[topologyName]; - - localIndex_array numberOfElementsPerRank(commSize); - - MpiWrapper::allGather(subRegion.size(), - numberOfElementsPerRank); - - globalIndex offsetElements = std::accumulate( - numberOfElementsPerRank.begin(), - numberOfElementsPerRank.begin() + thisRank, 0); - - globalIndex totalNumberOfElements = std::accumulate( - numberOfElementsPerRank.begin() + thisRank, - numberOfElementsPerRank.end(), offsetElements); - - topology["cells_shape"].set_uint64(totalNumberOfElements); - topology["cells_start"].set_uint64(offsetElements); - topology["cells_count"].set_uint64( - numberOfElementsPerRank[thisRank]); - } - }); - } - } - auto capsule = GenericConduitCapsule(&executeRoot); if (!CatalystExecute(&capsule)) { GEOS_ERROR("CatalystOutput: execute: catalyst failed to execute."); diff --git a/src/docs/doxygen/GeosxConfig.hpp b/src/docs/doxygen/GeosxConfig.hpp index 198305cfceb..7e222be7668 100644 --- a/src/docs/doxygen/GeosxConfig.hpp +++ b/src/docs/doxygen/GeosxConfig.hpp @@ -168,7 +168,7 @@ #define fmt_VERSION 9.1.0 /// Version information for python -#define Python3_VERSION 3.11.3 +#define Python3_VERSION 3.11.5 /// Version information for CUDAToolkit /* #undef CUDAToolkit_VERSION */ From 3482d3643b2ebc99b16ed19c46f5197835a28110 Mon Sep 17 00:00:00 2001 From: Charles Gueunet Date: Wed, 14 Feb 2024 17:10:02 +0100 Subject: [PATCH 08/17] Add python wrapping for Catalyst (attempt) --- src/coreComponents/fileIO/CMakeLists.txt | 2 + .../fileIO/python/PyCatalystOutput.cpp | 143 ++++++++++++++++++ .../fileIO/python/PyCatalystOutputType.hpp | 17 +++ 3 files changed, 162 insertions(+) create mode 100644 src/coreComponents/fileIO/python/PyCatalystOutput.cpp create mode 100644 src/coreComponents/fileIO/python/PyCatalystOutputType.hpp diff --git a/src/coreComponents/fileIO/CMakeLists.txt b/src/coreComponents/fileIO/CMakeLists.txt index d5abc37bc21..a466e54746a 100644 --- a/src/coreComponents/fileIO/CMakeLists.txt +++ b/src/coreComponents/fileIO/CMakeLists.txt @@ -31,10 +31,12 @@ set( fileIO_sources set( dependencyList ${parallelDeps} mesh constitutive hdf5 ) if( ENABLE_PYGEOSX ) list( APPEND fileIO_headers + python/PyCatalystOutputType.hpp python/PyHistoryCollectionType.hpp python/PyHistoryOutputType.hpp python/PyVTKOutputType.hpp ) list( APPEND fileIO_sources + python/PyCatalystOutput.cpp python/PyHistoryCollection.cpp python/PyHistoryOutput.cpp python/PyVTKOutput.cpp ) diff --git a/src/coreComponents/fileIO/python/PyCatalystOutput.cpp b/src/coreComponents/fileIO/python/PyCatalystOutput.cpp new file mode 100644 index 00000000000..ac666ca9316 --- /dev/null +++ b/src/coreComponents/fileIO/python/PyCatalystOutput.cpp @@ -0,0 +1,143 @@ +#define PY_SSIZE_T_CLEAN +#include + +// Source includes +#include "fileIO/Outputs/CatalystOutput.hpp" +#include "PyCatalystOutputType.hpp" +#include "dataRepository/python/PyGroupType.hpp" + + +#define VERIFY_NON_NULL_SELF( self ) \ + PYTHON_ERROR_IF( self == nullptr, PyExc_RuntimeError, "Passed a nullptr as self.", nullptr ) + +#define VERIFY_INITIALIZED( self ) \ + PYTHON_ERROR_IF( self->group == nullptr, PyExc_RuntimeError, "The PyCatalystOutput is not initialized.", nullptr ) + +namespace geos +{ +namespace python +{ + +struct PyCatalystOutput +{ + PyObject_HEAD + + static constexpr char const * docString = + "A Python interface to CatalystOutput."; + + geos::CatalystOutput * group; +}; + + +static PyObject * PyCatalystOutput_new( PyTypeObject *type, PyObject *args, PyObject *kwds ) +{ + GEOS_UNUSED_VAR( args, kwds ); + PyCatalystOutput *self; + + self = (PyCatalystOutput *)type->tp_alloc( type, 0 ); + if( self != nullptr ) + { + self->group = nullptr; + } + + return (PyObject *)self; +} + + +static PyObject * PyCatalystOutput_repr( PyObject * const obj ) noexcept +{ + PyCatalystOutput const * const pyCatalystOutput = LvArray::python::convert< PyCatalystOutput >( obj, getPyCatalystOutputType() ); + if( pyCatalystOutput == nullptr ) + { + return nullptr; + } + + VERIFY_INITIALIZED( pyCatalystOutput ); + + string repr; + string const path = pyCatalystOutput->group->getPath(); + string const type = LvArray::system::demangle( typeid( *(pyCatalystOutput->group) ).name() ); + repr = path + " ( " + type + " )"; + + return PyUnicode_FromString( repr.c_str() ); + +} + + +static PyObject * output( PyCatalystOutput * self, PyObject * args ) +{ + VERIFY_NON_NULL_SELF( self ); + VERIFY_INITIALIZED( self ); + + double time; + double dt; + + if( !PyArg_ParseTuple( args, "dd", &time, &dt ) ) + { + return nullptr; + } + + geos::DomainPartition & domain = self->group->getGroupByPath< DomainPartition >( "/Problem/domain" ); + + int cycleNumber = int(round( time/dt )); + try + { + self->group->execute( time, dt, cycleNumber, 0, 0, domain ); + } + catch( std::out_of_range const & e ) + { + std::cout << "Target not found. Impossible output."<< std::endl; + } + Py_RETURN_NONE; +} + +static PyObject * reinit( PyCatalystOutput * self, PyObject *args ) +{ + VERIFY_NON_NULL_SELF( self ); + VERIFY_INITIALIZED( self ); + GEOS_UNUSED_VAR( args ); + + self->group->reinit(); + + + Py_RETURN_NONE; +} + + +static PyMethodDef PyCatalystOutput_methods[] = { + { "output", (PyCFunction) output, METH_VARARGS, "wrapper to routine CatalystOutput::execute"}, + { "setOutputDir", (PyCFunction) setOutputDir, METH_VARARGS, "wrapper to change directory output"}, + { "setOutputFileRootName", (PyCFunction) setOutputFileRootName, METH_VARARGS, "wrapper to change the root name of the output files and subfolders"}, + { "reinit", (PyCFunction) reinit, METH_VARARGS, "reinitialization function"}, + { nullptr, nullptr, 0, nullptr } /* Sentinel */ +}; + + +/** + * Initialize the module object for Python with the exported functions + */ + +BEGIN_ALLOW_DESIGNATED_INITIALIZERS + +static PyTypeObject PyCatalystOutputType = { + PyVarObject_HEAD_INIT( nullptr, 0 ) + .tp_name = "pygeosx.CatalystOutput", + .tp_basicsize = sizeof( PyCatalystOutput ), + .tp_itemsize = 0, + .tp_repr = PyCatalystOutput_repr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = PyCatalystOutput::docString, + .tp_methods = PyCatalystOutput_methods, + .tp_base = getPyGroupType(), + .tp_new = PyCatalystOutput_new, +}; + +END_ALLOW_DESIGNATED_INITIALIZERS + +PyTypeObject * getPyCatalystOutputType() +{ + return &PyCatalystOutputType; +} + +} +} diff --git a/src/coreComponents/fileIO/python/PyCatalystOutputType.hpp b/src/coreComponents/fileIO/python/PyCatalystOutputType.hpp new file mode 100644 index 00000000000..b84f8e8b840 --- /dev/null +++ b/src/coreComponents/fileIO/python/PyCatalystOutputType.hpp @@ -0,0 +1,17 @@ +#ifndef GEOS_PYTHON_PYCATALYSTOUTPUTTYPE_HPP_ +#define GEOS_PYTHON_PYCATALYSTOUTPUTTYPE_HPP_ + +#include "LvArray/src/python/pythonForwardDeclarations.hpp" +#include "mesh/DomainPartition.hpp" + +namespace geos +{ +namespace python +{ + +PyTypeObject * getPyCatalystOutputType(); + +} // namespace python +} // namespace geos + +#endif From cea54ad6af3c5ad5fbe1811e815695601dc2f3fb Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 15 Feb 2024 12:14:40 +0100 Subject: [PATCH 09/17] removed to python function declaration that are not really wrapped --- src/coreComponents/fileIO/python/PyCatalystOutput.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreComponents/fileIO/python/PyCatalystOutput.cpp b/src/coreComponents/fileIO/python/PyCatalystOutput.cpp index ac666ca9316..82a5ef0b2a4 100644 --- a/src/coreComponents/fileIO/python/PyCatalystOutput.cpp +++ b/src/coreComponents/fileIO/python/PyCatalystOutput.cpp @@ -106,8 +106,6 @@ static PyObject * reinit( PyCatalystOutput * self, PyObject *args ) static PyMethodDef PyCatalystOutput_methods[] = { { "output", (PyCFunction) output, METH_VARARGS, "wrapper to routine CatalystOutput::execute"}, - { "setOutputDir", (PyCFunction) setOutputDir, METH_VARARGS, "wrapper to change directory output"}, - { "setOutputFileRootName", (PyCFunction) setOutputFileRootName, METH_VARARGS, "wrapper to change the root name of the output files and subfolders"}, { "reinit", (PyCFunction) reinit, METH_VARARGS, "reinitialization function"}, { nullptr, nullptr, 0, nullptr } /* Sentinel */ }; From 7826b1cf48cdf39dbaf371ea2045c035edeb12d4 Mon Sep 17 00:00:00 2001 From: Charles Gueunet Date: Thu, 15 Feb 2024 15:08:33 +0100 Subject: [PATCH 10/17] Add multi-writer sst filename --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 8f153b41427..0b059b41c91 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -362,6 +362,7 @@ struct CatalystOutput::CatalystInternals { std::string implementationPath = ""; std::string adiosConfig = ""; std::string channelName = "fullfield"; + std::string sstFileName = "gs.bp"; void initializeCatalyst() { std::string scriptsCopy = this->scripts; @@ -404,6 +405,11 @@ struct CatalystOutput::CatalystInternals { initializer["adios/config"] = this->adiosConfig; } + std::string envSSTfilename = std::getenv("CATALYST_SST_FILTENAME"); + if( envSSTfilename != "")) { + this->sstFileName = envSSTfilename; + } + auto capsule = GenericConduitCapsule(&initializer); if (!CatalystInitialize(&capsule)) { GEOS_ERROR("CatalystOutput: Constructor: catalyst failed to initialize."); @@ -466,6 +472,7 @@ bool CatalystOutput::execute(real64 const time_n, real64 const /*dt*/, auto& catalystState = executeRoot["catalyst/state"]; catalystState["timestep"].set(cycleNumber); catalystState["time"].set(time_n); + catalystState["sstFileName"].set(this->internal->sstFileName); auto& channel = executeRoot["catalyst/channels/" + this->internal->channelName]; From 14b7f11fd6e111ac11e74acea36d4bea6f5b1c52 Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 15 Feb 2024 16:14:39 +0100 Subject: [PATCH 11/17] added Catalyst output to pygeosx.cpp --- src/pygeosx/pygeosx.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pygeosx/pygeosx.cpp b/src/pygeosx/pygeosx.cpp index 3210a2365ea..926ab049a6a 100644 --- a/src/pygeosx/pygeosx.cpp +++ b/src/pygeosx/pygeosx.cpp @@ -25,6 +25,7 @@ #include "fileIO/python/PyHistoryCollectionType.hpp" #include "fileIO/python/PyHistoryOutputType.hpp" #include "fileIO/python/PyVTKOutputType.hpp" +#include "fileIO/python/PyCatalystOutputType.hpp" #include "mainInterface/initialization.hpp" #include "LvArray/src/python/PyArray.hpp" @@ -430,6 +431,11 @@ PyInit_pygeosx() return nullptr; } + if( !LvArray::python::addTypeToModule( module, geos::python::getPyCatalystOutputType(), "CatalystOutput" ) ) + { + return nullptr; + } + // Add the LvArray submodule. if( !LvArray::python::addPyLvArrayModule( module ) ) { From ab86833cede224c7f37a0cfd4bceae6f3223bcc1 Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 15 Feb 2024 16:42:01 +0100 Subject: [PATCH 12/17] added python type --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 7 +++++++ src/coreComponents/fileIO/Outputs/CatalystOutput.hpp | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 0b059b41c91..77f08077796 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -519,6 +519,13 @@ void CatalystOutput::cleanup(real64 const time_n, integer const cycleNumber, this->internal->initialized = false; } +#if defined(GEOSX_USE_PYGEOSX) +PyTypeObject * VTKOutput::getPythonType() const +{ + return python::getPyVTKOutputType(); +} +#endif + REGISTER_CATALOG_ENTRY(OutputBase, CatalystOutput, string const&, dataRepository::Group* const) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp index 2f52cd97d6f..05e60d7f016 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp @@ -73,6 +73,13 @@ class CatalystOutput : public BlueprintOutput integer const eventCounter, real64 const eventProgress, DomainPartition & domain ) override; + /** + * @brief Return PyVTKOutput type. + * @return Return PyVTKOutput type. + */ +#if defined(GEOSX_USE_PYGEOSX) + virtual PyTypeObject * getPythonType() const override; +#endif private: ///@{ From 34c6b256e7b79e0b82edb047a9075486b58dab3d Mon Sep 17 00:00:00 2001 From: Stefano Frambati Date: Thu, 15 Feb 2024 17:19:37 +0100 Subject: [PATCH 13/17] Removed spurious parenthesis --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 77f08077796..51ad7eae9cf 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -406,7 +406,7 @@ struct CatalystOutput::CatalystInternals { } std::string envSSTfilename = std::getenv("CATALYST_SST_FILTENAME"); - if( envSSTfilename != "")) { + if( envSSTfilename != "") { this->sstFileName = envSSTfilename; } From a4222039c8c737304e7b671eac6c44f9150fa916 Mon Sep 17 00:00:00 2001 From: j0405284 Date: Thu, 15 Feb 2024 17:26:09 +0100 Subject: [PATCH 14/17] VTK --> Catalyst leftover --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 6 +++--- src/coreComponents/fileIO/Outputs/CatalystOutput.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 77f08077796..62a7fa9a5f9 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -406,7 +406,7 @@ struct CatalystOutput::CatalystInternals { } std::string envSSTfilename = std::getenv("CATALYST_SST_FILTENAME"); - if( envSSTfilename != "")) { + if( envSSTfilename != "") { this->sstFileName = envSSTfilename; } @@ -520,9 +520,9 @@ void CatalystOutput::cleanup(real64 const time_n, integer const cycleNumber, } #if defined(GEOSX_USE_PYGEOSX) -PyTypeObject * VTKOutput::getPythonType() const +PyTypeObject * CatalystOutput::getPythonType() const { - return python::getPyVTKOutputType(); + return python::getPyCatalystOutputType(); } #endif diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp index 05e60d7f016..df0855df58f 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp @@ -74,8 +74,8 @@ class CatalystOutput : public BlueprintOutput real64 const eventProgress, DomainPartition & domain ) override; /** - * @brief Return PyVTKOutput type. - * @return Return PyVTKOutput type. + * @brief Return PyCatalystOutput type. + * @return Return PyCatalystOutput type. */ #if defined(GEOSX_USE_PYGEOSX) virtual PyTypeObject * getPythonType() const override; From 6e1ec041954bfaa02503ae2e96def480ea2fb6c7 Mon Sep 17 00:00:00 2001 From: Stefano Frambati Date: Thu, 15 Feb 2024 17:52:45 +0100 Subject: [PATCH 15/17] Added leftover python include --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 62a7fa9a5f9..0e9f3354ecf 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -19,6 +19,10 @@ // GEOS #include "CatalystOutput.hpp" +#if defined(GEOSX_USE_PYGEOSX) +#include "fileIO/python/PyCatalystOutputType.hpp" +#endif + #include "fileIO/Catalyst/CatalystActions.hpp" #include "fileIO/Catalyst/GenericConduitCapsule.tpp" #include "mesh/DomainPartition.hpp" From 4f21bca701b6b2ca47be50de3dccd095f7997fa9 Mon Sep 17 00:00:00 2001 From: j0405284 Date: Tue, 20 Feb 2024 12:11:13 +0100 Subject: [PATCH 16/17] corrected bug in stub implementation --- src/coreComponents/fileIO/Outputs/CatalystOutput.cpp | 12 +++++++++--- src/coreComponents/fileIO/Outputs/CatalystOutput.hpp | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp index 0e9f3354ecf..7826d4ebc84 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.cpp @@ -400,7 +400,7 @@ struct CatalystOutput::CatalystInternals { if (this->implementation == "adios" || (this->implementation.empty() && - std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios")) { + getEnv("CATALYST_IMPLEMENTATION_NAME") == "adios")) { if (this->adiosConfig == "") { GEOS_ERROR( "CatalystOutput: Constructor: Cannot use the catalyst/adios2 " @@ -409,7 +409,7 @@ struct CatalystOutput::CatalystInternals { initializer["adios/config"] = this->adiosConfig; } - std::string envSSTfilename = std::getenv("CATALYST_SST_FILTENAME"); + std::string envSSTfilename = getEnv("CATALYST_SST_FILTENAME"); if( envSSTfilename != "") { this->sstFileName = envSSTfilename; } @@ -488,7 +488,7 @@ bool CatalystOutput::execute(real64 const time_n, real64 const /*dt*/, bool isInTransit = this->internal->implementation == "adios" || (this->internal->implementation.empty() && - std::string(std::getenv("CATALYST_IMPLEMENTATION_NAME")) == "adios"); + getEnv("CATALYST_IMPLEMENTATION_NAME") == "adios"); ScopedDataContainer dataScoping; ::SanitizeNode(channel, isInTransit, dataScoping); @@ -502,6 +502,12 @@ bool CatalystOutput::execute(real64 const time_n, real64 const /*dt*/, return false; } +const std::string CatalystOutput::getEnv( const char* varname ) +{ + char* varptr = std::getenv( varname ); + return varptr == nullptr ? "" : std::string( varptr ); +} + /////////////////////////////////////////////////////////////////////////////////////////////////// void CatalystOutput::cleanup(real64 const time_n, integer const cycleNumber, integer const eventCounter, diff --git a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp index df0855df58f..7f252a4c75c 100644 --- a/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp +++ b/src/coreComponents/fileIO/Outputs/CatalystOutput.hpp @@ -82,6 +82,7 @@ class CatalystOutput : public BlueprintOutput #endif private: + static const std::string getEnv( const char* ); ///@{ /** */ From f21bfbdab48e1e8dfa0e97d80f1c966be8a03ea1 Mon Sep 17 00:00:00 2001 From: j0405284 Date: Tue, 20 Feb 2024 12:58:01 +0100 Subject: [PATCH 17/17] added correction from PR 2990 to test elastic workflow --- .../elementFormulations/Qk_Hexahedron_Lagrange_GaussLobatto.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreComponents/finiteElement/elementFormulations/Qk_Hexahedron_Lagrange_GaussLobatto.hpp b/src/coreComponents/finiteElement/elementFormulations/Qk_Hexahedron_Lagrange_GaussLobatto.hpp index e8cead63acf..8b1c73c1b85 100644 --- a/src/coreComponents/finiteElement/elementFormulations/Qk_Hexahedron_Lagrange_GaussLobatto.hpp +++ b/src/coreComponents/finiteElement/elementFormulations/Qk_Hexahedron_Lagrange_GaussLobatto.hpp @@ -1320,7 +1320,7 @@ computeFirstOrderStiffnessTerm( localIndex const q, const real64 w02 = w * gia * gjc; func( ibc, abj, w02 * detJ, J, 0, 2 ); func( abj, ibc, w02 * detJ, J, 2, 0 ); - const real64 w01 = w * gia * gjc; + const real64 w01 = w * gia * gjb; func( ibc, ajc, w01 * detJ, J, 0, 1 ); func( ajc, ibc, w01 * detJ, J, 1, 0 ); }