diff --git a/demos/alpine/PenningTrapManager.h b/demos/alpine/PenningTrapManager.h index 1c879ed27..86ba9e4c6 100644 --- a/demos/alpine/PenningTrapManager.h +++ b/demos/alpine/PenningTrapManager.h @@ -1,8 +1,8 @@ #ifndef IPPL_PENNING_TRAP_MANAGER_H #define IPPL_PENNING_TRAP_MANAGER_H -#include #include +#include #include "AlpineManager.h" #include "FieldContainer.hpp" @@ -49,12 +49,10 @@ class PenningTrapManager : public AlpineManager { double alpha_m; double DrInv_m; - - - #ifdef IPPL_ENABLE_CATALYST - public: +#ifdef IPPL_ENABLE_CATALYST +public: ippl::CatalystAdaptor cat_viz{std::string{TestName}}; - #endif +#endif public: void pre_run() override { @@ -135,23 +133,26 @@ class PenningTrapManager : public AlpineManager { this->grid2par(); - #ifdef IPPL_ENABLE_CATALYST - m << "Catalyst is enabled" << endl; - - std::shared_ptr runtime_steer_registry = ippl::MakeVisRegistryRuntimePtr(); - std::shared_ptr runtime_vis_registry = ippl::MakeVisRegistryRuntimePtr( - "density", this->fcontainer_m->getRho(), - "ions", this->pcontainer_m - ); - // runtime_vis_registry->add("potential", this->fcontainer_m->getRho() ); - runtime_vis_registry->add("electrostatic", this->fcontainer_m->getE() ); +#ifdef IPPL_ENABLE_CATALYST + m << "Catalyst is enabled" << endl; + + // New channel-based API: one mesh channel for fields, one + // particle channel for ions. Both use persistent staging + // buffers (no per-step allocation). + auto meshCh = cat_viz.addMeshChannel<3>("penning_mesh", this->fcontainer_m->getMesh(), + this->fcontainer_m->getFL()); + meshCh.addArray("density", this->fcontainer_m->getRho()); + meshCh.addArray("electrostatic", this->fcontainer_m->getE()); - static IpplTimings::TimerRef CAinit = IpplTimings::getTimer("CAinit"); - IpplTimings::startTimer(CAinit); - cat_viz.Initialize(runtime_vis_registry, runtime_steer_registry); - IpplTimings::stopTimer(CAinit); + auto steerReg = ippl::MakeVisRegistryRuntimePtr(); + cat_viz.addParticleChannel("ions", *this->pcontainer_m); - #endif + static IpplTimings::TimerRef CAinit = IpplTimings::getTimer("CAinit"); + IpplTimings::startTimer(CAinit); + cat_viz.Initialize(steerReg); + IpplTimings::stopTimer(CAinit); + +#endif this->dump(); @@ -268,10 +269,10 @@ class PenningTrapManager : public AlpineManager { static IpplTimings::TimerRef domainDecomposition = IpplTimings::getTimer("loadBalance"); static IpplTimings::TimerRef SolveTimer = IpplTimings::getTimer("solve"); - #ifdef IPPL_ENABLE_CATALYST - static IpplTimings::TimerRef TMR_CAremember = IpplTimings::getTimer("CAremember"); - static IpplTimings::TimerRef TMR_CAexecute = IpplTimings::getTimer("CAexecute"); - #endif +#ifdef IPPL_ENABLE_CATALYST + static IpplTimings::TimerRef TMR_CAremember = IpplTimings::getTimer("CAremember"); + static IpplTimings::TimerRef TMR_CAexecute = IpplTimings::getTimer("CAexecute"); +#endif double alpha = this->alpha_m; double Bext = this->Bext_m; @@ -332,12 +333,12 @@ class PenningTrapManager : public AlpineManager { // scatter the charge onto the underlying grid this->par2grid(); - // Save deep copy of the density field (for later). - #ifdef IPPL_ENABLE_CATALYST - IpplTimings::startTimer(TMR_CAremember); - cat_viz.rememberNow("density"); - IpplTimings::stopTimer(TMR_CAremember); - #endif +// Save deep copy of the density field (for later). +#ifdef IPPL_ENABLE_CATALYST + IpplTimings::startTimer(TMR_CAremember); + cat_viz.rememberNow("density"); + IpplTimings::stopTimer(TMR_CAremember); +#endif // Field solve IpplTimings::startTimer(SolveTimer); @@ -347,12 +348,12 @@ class PenningTrapManager : public AlpineManager { // gather E field this->grid2par(); - //trigger In Situ pipeline - #ifdef IPPL_ENABLE_CATALYST - IpplTimings::startTimer(TMR_CAexecute); - cat_viz.Execute(it, this->time_m); - IpplTimings::stopTimer(TMR_CAexecute); - #endif +// trigger In Situ pipeline +#ifdef IPPL_ENABLE_CATALYST + IpplTimings::startTimer(TMR_CAexecute); + cat_viz.Execute(it, this->time_m); + IpplTimings::stopTimer(TMR_CAexecute); +#endif IpplTimings::startTimer(PTimer); auto R2view = pc->R.getView(); diff --git a/demos/fel/FreeElectronLaserManager.h b/demos/fel/FreeElectronLaserManager.h index 7dbf296ef..1ca3528ec 100644 --- a/demos/fel/FreeElectronLaserManager.h +++ b/demos/fel/FreeElectronLaserManager.h @@ -7,17 +7,16 @@ #include #include -#include "Manager/BaseManager.h" - -#include "Interpolation/CurrentDeposition.hpp" +#include "datatypes.h" #include "Config.h" #include "FELFieldContainer.hpp" #include "FELParticleContainer.hpp" +#include "Interpolation/CurrentDeposition.hpp" #include "LorentzTransform.h" +#include "Manager/BaseManager.h" #include "MithraBunch.h" #include "Undulator.h" -#include "datatypes.h" #include "units.h" #ifdef IPPL_ENABLE_CATALYST @@ -51,8 +50,7 @@ class FreeElectronLaserManager : public ippl::BaseManager { , dt_m(0.0) , it_m(0) , frame_gamma_m(std::max( - T(1), cfg.bunch_gamma - / std::sqrt(1 + cfg.undulator_K * cfg.undulator_K * T(0.5)))) + T(1), cfg.bunch_gamma / std::sqrt(1 + cfg.undulator_K * cfg.undulator_K * T(0.5)))) , uparams_m(cfg.undulator_K, cfg.undulator_period, cfg.undulator_length) , frame_m(ippl::UniaxialLorentzframe::from_gamma(frame_gamma_m)) , undulator_m(uparams_m, 2.0 * cfg.sigma_position[2] * frame_gamma_m * frame_gamma_m) {} @@ -84,10 +82,10 @@ class FreeElectronLaserManager : public ippl::BaseManager { int nsubsteps_m = 3; ///< Boris sub-steps per FDTD step (reference behaviour). - T frame_gamma_m; ///< Lorentz factor of the co-moving frame. - ippl::undulator_parameters uparams_m; ///< Undulator parameters. - ippl::UniaxialLorentzframe frame_m; ///< Boost into the co-moving frame (z-axis). - ippl::Undulator undulator_m; ///< Static undulator field model. + T frame_gamma_m; ///< Lorentz factor of the co-moving frame. + ippl::undulator_parameters uparams_m; ///< Undulator parameters. + ippl::UniaxialLorentzframe frame_m; ///< Boost into the co-moving frame (z-axis). + ippl::Undulator undulator_m; ///< Static undulator field model. // --- narrow-band (resonant) radiation power diagnostic state --- // MITHRA reports the FEL output power as a sliding-window single-frequency @@ -152,7 +150,7 @@ class FreeElectronLaserManager : public ippl::BaseManager { void grid2par() { gatherFields(); } void depositCurrent() { - using value_type = typename SourceField_t::value_type; + using value_type = typename SourceField_t::value_type; this->fcontainer_m->getJ() = value_type(0); auto policy = Kokkos::RangePolicy<>(0, this->pcontainer_m->getLocalNum()); @@ -234,8 +232,9 @@ class FreeElectronLaserManager : public ippl::BaseManager { const ippl::Vector t2 = t1 + alpha * ippl::cross(t1, EB[1]); const ippl::Vector t3 = t1 - + ippl::cross(t2, T(2) * alpha - * (EB[1] / (1.0 + alpha * alpha * (EB[1].dot(EB[1]))))); + + ippl::cross( + t2, + T(2) * alpha * (EB[1] / (1.0 + alpha * alpha * (EB[1].dot(EB[1]))))); const ippl::Vector ngammabeta = t3 + charge * bunch_dt * EB[0] / (T(2) * mass); @@ -301,12 +300,11 @@ class FreeElectronLaserManager : public ippl::BaseManager { } m << "Discretization:" << endl - << "nt " << "(derived)" << " Np= " << this->totalP_m << " grid = " << this->nr_m - << endl; + << "nt " << "(derived)" << " Np= " << this->totalP_m << " grid = " << this->nr_m << endl; this->setFieldContainer(std::make_shared( - this->hr_m, this->rmin_m, this->rmax_m, this->decomp_m, this->domain_m, - this->origin_m, this->isAllPeriodic_m)); + this->hr_m, this->rmin_m, this->rmax_m, this->decomp_m, this->domain_m, this->origin_m, + this->isAllPeriodic_m)); this->fcontainer_m->initializeFields(); @@ -315,9 +313,8 @@ class FreeElectronLaserManager : public ippl::BaseManager { // The FDTD solver derives its own time step from the mesh (CFL); we read // it back to drive the particle push and diagnostics. - this->setFieldSolver(std::make_shared(this->fcontainer_m->getJ(), - this->fcontainer_m->getE(), - this->fcontainer_m->getB())); + this->setFieldSolver(std::make_shared( + this->fcontainer_m->getJ(), this->fcontainer_m->getE(), this->fcontainer_m->getB())); this->dt_m = this->solver_m->getDt(); this->nt_m = (int)std::ceil(this->m_config.total_time / this->dt_m); this->it_m = 0; @@ -328,13 +325,15 @@ class FreeElectronLaserManager : public ippl::BaseManager { initializeParticles(); #ifdef IPPL_ENABLE_CATALYST - auto runtime_vis_registry = ippl::MakeVisRegistryRuntimePtr( - "Particles", this->pcontainer_m, - "EField", this->fcontainer_m->getE(), - "Bfield", this->fcontainer_m->getB() - ); - auto runtime_steer_registry = ippl::MakeVisRegistryRuntimePtr(); - cat_viz.Initialize(runtime_vis_registry, runtime_steer_registry); + // New channel-based API + auto meshCh = cat_viz.addMeshChannel<3>("fel_mesh", this->fcontainer_m->getMesh(), + this->fcontainer_m->getFL()); + meshCh.addArray("EField", this->fcontainer_m->getE()); + meshCh.addArray("Bfield", this->fcontainer_m->getB()); + + auto steerReg = ippl::MakeVisRegistryRuntimePtr(); + cat_viz.addParticleChannel("Particles", *this->pcontainer_m); + cat_viz.Initialize(steerReg); #endif this->dump(); @@ -423,20 +422,19 @@ class FreeElectronLaserManager : public ippl::BaseManager { ippl::getRangePolicy(eview, 1), KOKKOS_LAMBDA(const size_t i, const size_t j, const size_t k, double& ref) { Kokkos::pair, ippl::Vector> buncheb{eview(i, j, k), - bview(i, j, k)}; - auto eblab = lb.inverse_transform_EB(buncheb); - uint32_t kg = (uint32_t)(k + ldom.first()[2]); + bview(i, j, k)}; + auto eblab = lb.inverse_transform_EB(buncheb); + uint32_t kg = (uint32_t)(k + ldom.first()[2]); if (kg == nz - 3) { ref += ippl::cross(eblab.first, eblab.second)[2]; } }, radiation); - double power_local = - radiation - * double(unit_powerdensity_in_watt_per_square_meter * unit_length_in_meters - * unit_length_in_meters) - * this->hr_m[0] * this->hr_m[1]; + double power_local = radiation + * double(unit_powerdensity_in_watt_per_square_meter + * unit_length_in_meters * unit_length_in_meters) + * this->hr_m[0] * this->hr_m[1]; double power_global = 0.0; MPI_Reduce(&power_local, &power_global, 1, MPI_DOUBLE, MPI_SUM, 0, ippl::Comm->getCommunicator()); @@ -475,10 +473,10 @@ class FreeElectronLaserManager : public ippl::BaseManager { if (!rp_init_m) { const double lambda_rad = this->m_config.undulator_period / frame_gamma_m; - rp_omega_m = 2.0 * M_PI / lambda_rad; // [1/unit_time], c = 1 - rp_Nf_m = std::max(1, (int)std::lround(3.0 * lambda_rad / this->dt_m)); - rp_fdt_m = Kokkos::View("FEL banded field buffer", rp_Nf_m, extx, exty, 4); - rp_init_m = true; + rp_omega_m = 2.0 * M_PI / lambda_rad; // [1/unit_time], c = 1 + rp_Nf_m = std::max(1, (int)std::lround(3.0 * lambda_rad / this->dt_m)); + rp_fdt_m = Kokkos::View("FEL banded field buffer", rp_Nf_m, extx, exty, 4); + rp_init_m = true; } const int Nf = rp_Nf_m; @@ -497,7 +495,7 @@ class FreeElectronLaserManager : public ippl::BaseManager { KOKKOS_LAMBDA(const int i, const int j) { Kokkos::pair, ippl::Vector> buncheb{ eview(i, j, kview), bview(i, j, kview)}; - auto eblab = lb.inverse_transform_EB(buncheb); + auto eblab = lb.inverse_transform_EB(buncheb); fdt(m, i, j, 0) = eblab.first[0]; // Ex_lab fdt(m, i, j, 1) = eblab.first[1]; // Ey_lab fdt(m, i, j, 2) = eblab.second[0]; // Bx_lab @@ -524,10 +522,14 @@ class FreeElectronLaserManager : public ippl::BaseManager { const double ey = fdt(mm, i, j, 1); const double bx = fdt(mm, i, j, 2); const double by = fdt(mm, i, j, 3); - ex_r += ex * cp; ex_i += ex * sp; - ey_r += ey * cp; ey_i += ey * sp; - bx_r += bx * cp; bx_i -= bx * sp; - by_r += by * cp; by_i -= by * sp; + ex_r += ex * cp; + ex_i += ex * sp; + ey_r += ey * cp; + ey_i += ey * sp; + bx_r += bx * cp; + bx_i -= bx * sp; + by_r += by * cp; + by_i -= by * sp; } ref += (ex_r * by_r - ex_i * by_i) - (ey_r * bx_r - ey_i * bx_i); }, @@ -599,9 +601,9 @@ class FreeElectronLaserManager : public ippl::BaseManager { : 0.0; // --- peak |E| and total EM field energy over the domain --- - const int nghost = fc->getE().getNghost(); - auto Eview = fc->getE().getView(); - auto Bview = fc->getB().getView(); + const int nghost = fc->getE().getNghost(); + auto Eview = fc->getE().getView(); + auto Bview = fc->getB().getView(); using index_array_type = typename ippl::RangePolicy::index_array_type; double localE2 = 0.0, localEmax = 0.0; ippl::parallel_reduce( @@ -681,27 +683,27 @@ class FreeElectronLaserManager : public ippl::BaseManager { ippl::Comm->barrier(); } - public: // NOTE: depositChargeDensity and destroyOutOfBounds must be public, not // protected/private, because they contain KOKKOS_LAMBDA expressions that // expand to extended __host__ __device__ lambdas under CUDA (nvcc). NVCC // forbids such lambdas inside protected or private member functions. - // See: https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#extended-lambda-restrictions + // See: + // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#extended-lambda-restrictions // Deposit charge density (CIC) into component [0] of the four-current field, // needed only when space charge is enabled. Component [1..Dim] is filled by // assemble_current_collocated. void depositChargeDensity() { - auto pc = this->pcontainer_m; - auto fc = this->fcontainer_m; - auto view = fc->getJ().getView(); - auto qview = pc->Q.getView(); - auto rview = pc->R.getView(); - const auto origin = fc->getMesh().getOrigin(); - const auto h = fc->getMesh().getMeshSpacing(); - const auto ldom = fc->getFL().getLocalNDIndex(); - const int nghost = fc->getJ().getNghost(); - T volume = T(1); + auto pc = this->pcontainer_m; + auto fc = this->fcontainer_m; + auto view = fc->getJ().getView(); + auto qview = pc->Q.getView(); + auto rview = pc->R.getView(); + const auto origin = fc->getMesh().getOrigin(); + const auto h = fc->getMesh().getMeshSpacing(); + const auto ldom = fc->getFL().getLocalNDIndex(); + const int nghost = fc->getJ().getNghost(); + T volume = T(1); for (unsigned d = 0; d < Dim; ++d) volume *= h[d]; @@ -747,7 +749,7 @@ class FreeElectronLaserManager : public ippl::BaseManager { Kokkos::parallel_reduce( pc->getLocalNum(), KOKKOS_LAMBDA(const size_t i, size_type& ref) { - bool out_of_bounds = false; + bool out_of_bounds = false; const ippl::Vector ppos = rview(i); for (unsigned d = 0; d < Dim; ++d) { out_of_bounds |= (ppos[d] <= origin[d]); diff --git a/src/Particle/ParticleAttrib.h b/src/Particle/ParticleAttrib.h index 6dcbe0b71..a22cd833d 100644 --- a/src/Particle/ParticleAttrib.h +++ b/src/Particle/ParticleAttrib.h @@ -271,20 +271,27 @@ namespace ippl { */ void internalCopy(const hash_type& indices) override; - #ifdef IPPL_ENABLE_CATALYST - void signConduitBlueprintNode( - const size_type Np_local - , conduit_cpp::Node& node_fields - , ViewRegistry& viewRegistry - , Inform& ca_m - , Inform& ca_warn - , const bool forceHostCopy - ) const override ; - #endif +#ifdef IPPL_ENABLE_CATALYST + void signConduitBlueprintNode(const size_type Np_local, conduit_cpp::Node& node_fields, + ViewRegistry& viewRegistry, Inform& ca_m, Inform& ca_warn, + const bool forceHostCopy) const override; + + void prepareConduitNode(const size_type Np_local, conduit_cpp::Node& node_fields, + Inform& ca_m) const override; + + void updateConduitData() const override; +#endif private: view_type dview_m{"ParticleAttrib::dview", 0}; view_type buf_m{"ParticleAttrib::buf", 0}; + +#ifdef IPPL_ENABLE_CATALYST + /// Persistent host mirror for in-situ visualization (allocated once, + /// reused on every Execute). Mutable because prepareConduitNode and + /// updateConduitData are const (like signConduitBlueprintNode). + mutable host_mirror_type hostMirrorForViz_m{"hostViz", 0}; +#endif }; namespace detail { diff --git a/src/Particle/ParticleAttrib.hpp b/src/Particle/ParticleAttrib.hpp index 31cc732a7..efdf53867 100644 --- a/src/Particle/ParticleAttrib.hpp +++ b/src/Particle/ParticleAttrib.hpp @@ -576,7 +576,13 @@ namespace ippl { /// matching bitwidth type is safe because the sizes and alignments match. template auto conduitCompatiblePtr(T* ptr) { - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + static_assert(sizeof(T) == sizeof(conduit_uint8), + "bool size must match conduit_uint8"); + static_assert(alignof(T) == alignof(conduit_uint8), + "bool alignment must match conduit_uint8"); + return reinterpret_cast(ptr); + } else if constexpr (std::is_same_v) { static_assert(sizeof(T) == sizeof(conduit_int64), "long long size must match conduit_int64"); static_assert(alignof(T) == alignof(conduit_int64), @@ -679,6 +685,76 @@ namespace ippl { } viewRegistry.set(hostMirror); } + + // ------------------------------------------------------------------ + // prepareConduitNode: allocate persistent mirror, set Conduit once. + // ------------------------------------------------------------------ + template + void ParticleAttrib::prepareConduitNode(const size_type Np_local, + conduit_cpp::Node& node_fields, + Inform& ca_m) const { + // Allocate (or resize) persistent host mirror + if (hostMirrorForViz_m.extent(0) != Np_local) { + Kokkos::resize(hostMirrorForViz_m, Np_local); + } + Kokkos::deep_copy(hostMirrorForViz_m, this->getView()); + + auto field = node_fields[this->name_m]; + field["association"].set_string("vertex"); + field["topology"].set_string("p_unstructured_topo"); + field["volume_dependent"].set_string("false"); + + if constexpr (std::is_scalar_v) { + ca_m << level4 << " prepareConduitNode: '" << this->name_m << "' (scalar, " << Np_local + << " particles)" << endl; + field["values"].set_external(detail::conduitCompatiblePtr(hostMirrorForViz_m.data()), + Np_local); + } else if constexpr (is_vector_v) { + ca_m << level4 << " prepareConduitNode: '" << this->name_m << "' (vector, " << Np_local + << " particles)" << endl; + using elem_t = std::remove_pointer_t; + const size_t stride = sizeof(elem_t); + if (Np_local > 0) { + field["values/x"].set_external( + detail::conduitCompatiblePtr(&hostMirrorForViz_m.data()[0][0]), Np_local, 0, + stride); + if constexpr (T::dim >= 2) + field["values/y"].set_external( + detail::conduitCompatiblePtr(&hostMirrorForViz_m.data()[0][1]), Np_local, 0, + stride); + if constexpr (T::dim >= 3) + field["values/z"].set_external( + detail::conduitCompatiblePtr(&hostMirrorForViz_m.data()[0][2]), Np_local, 0, + stride); + } else { + using component_type = typename T::value_type; + field["values/x"].set_external( + detail::conduitCompatiblePtr(static_cast(nullptr)), 0); + if constexpr (T::dim >= 2) + field["values/y"].set_external( + detail::conduitCompatiblePtr(static_cast(nullptr)), 0); + if constexpr (T::dim >= 3) + field["values/z"].set_external( + detail::conduitCompatiblePtr(static_cast(nullptr)), 0); + } + } + } + + // ------------------------------------------------------------------ + // updateConduitData: deep-copy device into existing persistent mirror. + // If Np_local changed, resize and re-set external pointers. + // ------------------------------------------------------------------ + template + void ParticleAttrib::updateConduitData() const { + const size_type Np = this->getParticleCount(); + if (hostMirrorForViz_m.extent(0) != Np) { + // Particle count changed — resize and let the next init re-set + // external pointers. For now just resize; the caller should + // detect the change and re-init. + Kokkos::resize(hostMirrorForViz_m, Np); + } + Kokkos::deep_copy(hostMirrorForViz_m, this->getView()); + } #endif } // namespace ippl diff --git a/src/Particle/ParticleAttribBase.h b/src/Particle/ParticleAttribBase.h index 761fe69a2..ccf30abb6 100644 --- a/src/Particle/ParticleAttribBase.h +++ b/src/Particle/ParticleAttribBase.h @@ -21,11 +21,11 @@ #include "Communicate/Archive.h" +#ifdef IPPL_ENABLE_CATALYST +#include - #ifdef IPPL_ENABLE_CATALYST - #include - #include "Stream/Registry/ViewRegistry.h" - #endif +#include "Stream/Registry/ViewRegistry.h" +#endif namespace ippl { namespace detail { @@ -100,16 +100,22 @@ namespace ippl { virtual void applyPermutation(const hash_type&) = 0; virtual void internalCopy(const hash_type&) = 0; - #ifdef IPPL_ENABLE_CATALYST - virtual void signConduitBlueprintNode( - const size_type Np_local - , conduit_cpp::Node& node_fields - , ViewRegistry& viewRegistry - , Inform& ca_m - , Inform& ca_warn - , const bool forceHostCopy - ) const = 0; - #endif +#ifdef IPPL_ENABLE_CATALYST + virtual void signConduitBlueprintNode(const size_type Np_local, + conduit_cpp::Node& node_fields, + ViewRegistry& viewRegistry, Inform& ca_m, + Inform& ca_warn, + const bool forceHostCopy) const = 0; + + /// Allocate a persistent host mirror and set Conduit external + /// pointers. Called once during channel init. + virtual void prepareConduitNode(const size_type Np_local, + conduit_cpp::Node& node_fields, Inform& ca_m) const = 0; + + /// Deep-copy device data into the persistent host mirror. + /// Called every Execute for dynamic particle channels. + virtual void updateConduitData() const = 0; +#endif protected: const size_type* localNum_mp; char name_m[ATTRIB_NAME_MAX_LEN]; diff --git a/src/Stream/CMakeLists.txt b/src/Stream/CMakeLists.txt index 7d5cc1c07..f88fd883d 100644 --- a/src/Stream/CMakeLists.txt +++ b/src/Stream/CMakeLists.txt @@ -2,4 +2,5 @@ # src/Stream/CMakeLists.txt # ----------------------------------------------------------------------------- add_subdirectory(InSitu) -add_subdirectory(Registry) \ No newline at end of file +add_subdirectory(Registry) +add_subdirectory(Steering) diff --git a/src/Stream/InSitu/CMakeLists.txt b/src/Stream/InSitu/CMakeLists.txt index 8e62bae37..c119fe7dc 100644 --- a/src/Stream/InSitu/CMakeLists.txt +++ b/src/Stream/InSitu/CMakeLists.txt @@ -8,12 +8,6 @@ target_include_directories(ippl ) if(IPPL_ENABLE_CATALYST) - # ProxyWriter uses Catalyst's Conduit API and is only needed by the Catalyst adaptor. - target_sources(ippl - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/ProxyWriter.cpp - ) - set(_ippl_catalyst_resource_source_dir "${CMAKE_CURRENT_SOURCE_DIR}/catalyst_scripts") set(_ippl_catalyst_resource_build_dir @@ -77,8 +71,6 @@ endif() install(FILES - ProxyWriter.h - ProxyWriter.hpp CatalystAdaptor.h CatalystAdaptorSteering.hpp CatalystAdaptor.hpp diff --git a/src/Stream/InSitu/CatalystAdaptor.h b/src/Stream/InSitu/CatalystAdaptor.h index ee9e93bc0..eb2fa80fb 100644 --- a/src/Stream/InSitu/CatalystAdaptor.h +++ b/src/Stream/InSitu/CatalystAdaptor.h @@ -3,7 +3,7 @@ * @brief Declarations and lightweight types for ParaView Catalyst in-situ integration. * * This header provides helper types, forward declarations, and includes used by the - * Catalyst adaptor. Heavy implementation details live in `CatalystAdaptor.hpp` and + * Catalyst adaptor. Heavy implementation details live in `CatalystAdaptor.hpp` and * `CatalystAdaptorSteering.hpp`. */ #ifndef CatalystAdaptor_h @@ -11,20 +11,22 @@ #include "Ippl.h" +#include #include #include - #include +#include #include +#include +#include +#include #include #include -#include -#include -#include #include -#include -#include #include +#include +#include +#include #if defined(MPI_VERSION) #include @@ -32,696 +34,510 @@ #include "Utility/IpplException.h" -#include "Stream/Registry/ViewRegistry.h" - -#include "Stream/InSitu/ProxyWriter.h" +#include "Stream/InSitu/Channel.h" +#include "Stream/ParticleAttributeSelection.h" #include "Stream/Registry/RegistryHelper.h" +#include "Stream/Registry/ViewRegistry.h" +#include "Stream/Steering/Button.h" +#include "Stream/Steering/ProxyWriter.h" +namespace ippl { -namespace ippl{ - -/* FORWARD DECLARATION */ -class VisRegistryRuntime; + /* FORWARD DECLARATION */ + class VisRegistryRuntime; + /** + * @class CatalystAdaptor + * @brief High-level orchestrator for Catalyst initialization, execution, and steering. + * + * The adaptor builds Conduit channels for registered meshes, mesh blocks and particle + * containers (addMeshChannel/addMeshBlock/addParticleChannel), invokes Catalyst scripts, + * and forwards/fetches steering values between the simulation and the GUI. + */ + class CatalystAdaptor { + public: + struct SteerInitVisitor; + struct SteerForwardVisitor; + struct SteerFetchVisitor; + using SteerVisitorVariant_t = + std::variant; -/** - * @struct Button - * @brief Momentary-action control for steering. - * - * A Button acts like an edge-trigger: it reports a single transition when - * pressed and then resets. Useful for one-shot actions triggered from the - * Catalyst GUI without latching state. - */ -struct Button { - Button() = default; - - // explicit not explicit allows: if(my_btn) to work - operator bool() const { return value_m; } - - // explicit Button(bool v) : value_m(v) {} - explicit Button(bool v){ - - if(v){ // Button is initialized pressd - value_m = true; - priorState_m=false; - } - else /* if(!v) */{ // Button is initialized unpressd - value_m=false; - priorState_m = false; - } - + private: + static std::string environmentValue(const char* name, std::string fallback) { + const char* value = std::getenv(name); + return value && *value ? std::string(value) : std::move(fallback); } - - // Assignment operator: Button = bool - Button& operator=(bool v) { - if(v) { // Button is being pressed - if(!value_m && !priorState_m) { // True unpressed state - value_m = true; - priorState_m = false; // Fixed typo: was "priot_state" - } - else if(value_m && !priorState_m) { // Button was pressed last iteration and is still "being pushed down" - // Internal button needs to snap back! And mark it as "freshly snapped" - value_m = false; - priorState_m = true; - } - else if(!value_m && priorState_m) { // Button was pressed at some previous iteration and is still "being pushed down" - // Button needs to stay in freshly snapped back state - value_m = false; - priorState_m = true; - } - else if(value_m && priorState_m) { // Impossible state - throw IpplException("CatalystAdaptor::Button Assignment", "Impossible State: Button is malfunctioning!!!"); - } - } - else { // Button is unpressed - value_m = false; - priorState_m = false; - } - return *this; // CRITICAL: Return reference to this object for chaining + static std::filesystem::path catalystOutputDirectory() { + const char* value = std::getenv("IPPL_CATALYST_OUTPUT_DIR"); + return value && *value ? std::filesystem::path(value) + : std::filesystem::current_path() / "catalyst"; } - // Friend function to overload << operator for output streams - friend std::ostream& operator<<(std::ostream& os, const Button& btn) { - os << (btn.value_m ? "PUSHED" : "not PUSHED"); - return os; + static std::optional normalizeExperimentName( + std::optional experimentName) { + if (experimentName && experimentName->empty()) { + experimentName.reset(); + } + return experimentName; } - private: - bool value_m = false; - bool priorState_m = false; -};// Button - - -/** - * @brief Host-side 1D mask view (0/1) used for ghost/halo flags. - */ -using HostMaskView1D_t = Kokkos::View; - + std::shared_ptr steerRegistry_m; -/** - * @brief Cache key for ghosted data. - * - * Tuple components: - * - pointer identifying the mesh/topology - * - pointer identifying the owning view/container - * - size/extent of the ghost region - */ -using GhostKey_t = std::tuple; + // Channel-based viz path. Steering uses steerRegistry_m. + std::vector> channels_m; + // True after the channel Conduit tree has been built (first Execute) + bool channelsBuilt_m = false; + bool initialized_m = false; + std::map particleAttributeSelections_m; + std::map()>> particleAttributeSources_m; + void validateParticleAttributeSelections() const; -/** - * @brief Hash functor for GhostKey_t used in unordered caches. - */ -struct GhostKeyHash { - std::size_t operator()(const GhostKey_t& k) const { - // Get the hash for each element in the tuple - auto h1 = std::hash{}(std::get<0>(k)); - auto h2 = std::hash{}(std::get<1>(k)); - auto h3 = std::hash{}(std::get<2>(k)); - - // Combine the hashes. This is a common pattern (based on boost::hash_combine) - // It xors and bit-shifts to mix the bits well. - h1 ^= h2 + 0x9e3779b9 + (h1 << 6) + (h1 >> 2); - h1 ^= h3 + 0x9e3779b9 + (h1 << 6) + (h1 >> 2); - return h1; - } -}; + ViewRegistry viewRegistry_m; + conduit_cpp::Node node_m; + conduit_cpp::Node results_m; - -/** - * @class CatalystAdaptor - * @brief High-level orchestrator for Catalyst initialization, execution, and steering. - * - * The adaptor wires up registered fields/particles and steerables to Conduit `node_m` instances, - * invokes Catalyst scripts, and forwards/fetches steering values between the - * simulation and the GUI. Use the runtime registry API (Initialize/Execute) - * for flexible, non-templated integration. - */ -class CatalystAdaptor { -public: - struct InitVisitor; - struct ExecVisitor; - struct SteerInitVisitor; - struct SteerForwardVisitor; - struct SteerFetchVisitor; - - using VisVisitorVariant_t = std::variant; - using SteerVisitorVariant_t = std::variant; - -private: - static std::string environmentValue(const char* name, std::string fallback) { - const char* value = std::getenv(name); - return value && *value ? std::string(value) : std::move(fallback); - } - - static std::filesystem::path catalystOutputDirectory() { - const char* value = std::getenv("IPPL_CATALYST_OUTPUT_DIR"); - return value && *value ? std::filesystem::path(value) - : std::filesystem::current_path() / "catalyst"; - } - - static std::optional normalizeExperimentName( - std::optional experimentName) { - if (experimentName && experimentName->empty()) { - experimentName.reset(); - } - return experimentName; - } - - std::shared_ptr visRegistry_m; - std::shared_ptr steerRegistry_m; - - ViewRegistry viewRegistry_m; - conduit_cpp::Node node_m; - conduit_cpp::Node results_m; public: - Inform catalystInfo_m; - Inform catalystWarn_m; - private: - - ProxyWriter proxyWriter_m; - std::unordered_map>> enumChoicesByType_m; - std::unordered_map>> enumChoices_m; - - - const std::optional experimentName_m; - const std::string catalystVis_m; - const std::string catalystLive_m; - const std::string catalystSteer_m; - const std::string catalystPng_m; - const std::string catalystVtk_m; - const std::string catalystVerbosity_m; - const std::string catalystGhostMask_m; - const std::string proxyOption_m; - - const bool visEnabled_m; - const bool liveEnabled_m; - const bool steerEnabled_m; - const bool pngExtracts_m ; - const bool vtkExtracts_m ; - const int outputLevel_m ; - const bool useGhostMasks_m; - - const std::filesystem::path resourceDir_m; - const std::filesystem::path outputDir_m; - - std::unordered_map forceHostCopy_m; - - std::unordered_map ghostMaskCache_m; - -public: - - - - - /** - * @brief Construct a Catalyst adaptor. - * - * @param experimentName Optional label forwarded to Catalyst extractor scripts. - * Runtime output is written below IPPL_CATALYST_OUTPUT_DIR, or ./catalyst when unset. - */ - explicit CatalystAdaptor(std::optional experimentName = std::nullopt) : - catalystInfo_m("CatalystAdaptor::", 0), // Only print on rank 0 - catalystWarn_m("CatalystAdaptor_WARNING", std::cerr, INFORM_ALL_NODES), - experimentName_m(normalizeExperimentName(std::move(experimentName))), - catalystVis_m(environmentValue("IPPL_CATALYST_VIS", "ON")), - catalystLive_m(environmentValue("IPPL_CATALYST_LIVE", "OFF")), - catalystSteer_m(environmentValue("IPPL_CATALYST_STEER", "OFF")), - catalystPng_m(environmentValue("IPPL_CATALYST_PNG", "OFF")), - catalystVtk_m(environmentValue("IPPL_CATALYST_VTK", "OFF")), - catalystVerbosity_m(environmentValue( - "IPPL_CATALYST_VERBOSITY", std::to_string(ippl::Info->getOutputLevel()))), - catalystGhostMask_m(environmentValue("IPPL_CATALYST_GHOST_MASKS", "OFF")), - proxyOption_m(environmentValue("IPPL_CATALYST_PROXY_OPTION", "ON")), - visEnabled_m(catalystVis_m != "OFF"), - liveEnabled_m(catalystLive_m == "ON"), - steerEnabled_m(catalystSteer_m == "ON"), - pngExtracts_m(catalystPng_m == "ON"), - vtkExtracts_m(catalystVtk_m == "ON"), - outputLevel_m(std::stoi(catalystVerbosity_m)), - useGhostMasks_m(catalystGhostMask_m == "ON"), - resourceDir_m(IPPL_CATALYST_SCRIPTS_DIR), - outputDir_m(catalystOutputDirectory()) - { - catalystInfo_m.setOutputLevel(outputLevel_m); - - // #if defined(MPI_VERSION) - // MPI_Barrier(MPI_COMM_WORLD); - // if(ippl::Comm->rank()==0) catalystWarn_m << "[rank = 0 size=" << ippl::Comm->size() << "]" << endl; - // MPI_Barrier(MPI_COMM_WORLD); - // if(ippl::Comm->rank()==1) catalystWarn_m << "[rank= 1 size=" << ippl::Comm->size() << "]" << endl; - // MPI_Barrier(MPI_COMM_WORLD); - // #endif - - catalystInfo_m << level4 << "::CatalystAdaptor() Global Output Level setting: " << ippl::Info->getOutputLevel() << endl; - catalystInfo_m << level4 << "::CatalystAdaptor() Catalyst Info Output Level setting: " << catalystInfo_m.getOutputLevel() << endl; - catalystInfo_m << level4 << "::CatalystAdaptor() Catalyst Warn Output Level setting: " << catalystWarn_m.getOutputLevel() << endl; - catalystInfo_m << level4 << "::CatalystAdaptor() using resourceDir_m = " << resourceDir_m.string() << endl; - catalystInfo_m << level4 << "::CatalystAdaptor() using outputDir_m = " << outputDir_m.string() << endl; - if (pngExtracts_m) - { catalystInfo_m << level4 << "::CatalystAdaptor() PNG extraction ACTIVATED" << endl;} - else{ catalystInfo_m << level4 << "::CatalystAdaptor() PNG extraction DEACTIVATED" << endl;} - if (vtkExtracts_m) - { catalystInfo_m << level4 << "::CatalystAdaptor() VTK extraction ACTIVATED" << endl;} - else{ catalystInfo_m << level4 << "::CatalystAdaptor() VTK extraction DEACTIVATED" << endl;} - if (steerEnabled_m) - { catalystInfo_m << level4 << "::CatalystAdaptor() Steering ACTIVATED" << endl;} - else{ catalystInfo_m << level4 << "::CatalystAdaptor() Steering DEACTIVATED" << endl;} - - } + Inform catalystInfo_m; + Inform catalystWarn_m; private: + ProxyWriter proxyWriter_m; + std::unordered_map>> + enumChoicesByType_m; + std::unordered_map>> enumChoices_m; + + const std::optional experimentName_m; + const std::string catalystVis_m; + const std::string catalystLive_m; + const std::string catalystSteer_m; + const std::string catalystPng_m; + const std::string catalystVtk_m; + const std::string catalystVerbosity_m; + const std::string catalystGhostMask_m; + const std::string proxyOption_m; + + const bool visEnabled_m; + const bool liveEnabled_m; + const bool steerEnabled_m; + const int outputLevel_m; + const bool useGhostMasks_m; + + const std::filesystem::path resourceDir_m; + const std::filesystem::path outputDir_m; - // ============================================================================================== - // HELPERS ===================================================================================== - // ============================================================================================== - /** - * @brief Sets a file path to a node, using an environment variable if available, otherwise a default path. - * - * @param nodePath The Conduit `node_m` instance to set the file path in. - * @param envVar The name of the environment variable to check. - * @param defaultFilePath The default file path to use if the environment variable is not set or invalid. - */ - void setNodeScript( - conduit_cpp::Node nodePath, - // const char* envVar, - const std::string envVar, - const std::filesystem::path defaultFilePath - ); - - - // ========================================================== - // VISUALIZATION CHANNEL INITIALIZERS ======================= - // ========================================================== - - - - /* SCALAR FIELDS - handles both reference and shared_ptr */ - // == ippl::Field, Cell>* - /** - * @brief Initializes a Conduit `node_m` instance entry for a scalar field. - * - * @tparam T Field value type. - * @tparam Dim Field dimension. - * @tparam ViewArgs Additional template arguments for the field. - * @param entry The scalar field to initialize. - * @param label The label for the field/channel. - */ - template - void InitVizChannel( - [[maybe_unused]] - const Field& entry - , const std::string label - ); - - - /* VECTOR FIELDS - handles both reference and shared_ptr */ - // == ippl::Field, 3, ippl::UniformCartesian, Cell>* - /** - * @brief Initializes a Conduit `node_m` instance entry for a vector field. - * - * @tparam T Vector value type. - * @tparam Dim Field dimension. - * @tparam Dim_v Vector dimension. - * @tparam ViewArgs Additional template arguments for the field. - * @param entry The vector field to initialize. - * @param label The label for the field/channel. - */ - template - void InitVizChannel( - [[maybe_unused]] - const Field, Dim, ViewArgs...>& entry - , const std::string label - ); - - - - // PARTICLECONTAINERS DERIVED FROM PARTICLEBASE: - // == ippl::ParticleBaseBase -> ParticleBase, ... , ... > - /** - * @brief Initializes a Conduit `node_m` instance entry for a particle container derived from ParticleBaseBase. - * - * @tparam T Particle container type (must derive from ippl::ParticleBaseBase). - * @param entry The particle container to initialize. - * @param label The label for the container/channel. - */ - template - requires std::derived_from, ParticleBaseBase> - void InitVizChannel( - [[maybe_unused]] - const T& entry - , const std::string label - ); - - - /* SHARED_PTR DISPATCHER - automatically unwraps and dispatches to appropriate overload */ - /** - * @brief Dispatcher for InitVizChannel: unwraps shared_ptr and dispatches to the appropriate overload. - * - * @tparam T Entry type. - * @param entry Shared pointer to the entry. - * @param label The label for the entry/channel. - */ - template - void InitVizChannel( - const std::shared_ptr& entry - , const std::string label - ); - - - - // BASE CASE: - // only enabled if EntryT is NOT derived from ippl::ParticleBaseBase - /** - * @brief Fallback for InitVizChannel: handles types not derived from ParticleBaseBase - * and not having specific overloads. Should never be called since these types are already - * filtered out inside a Visitor struct with AllowedVisType_v. - * - * @tparam T Entry type. - * @param entry The entry to initialize (not a particle container). - * @param label The label for the entry/channel. - */ - template - requires (!std::derived_from, ParticleBaseBase>) - void InitVizChannel( - [[maybe_unused]] - // T&& entry - const T& entry - , const std::string label - ); - - - - // ========================================================== - // VISUALIZATION CHANNEL EXECUTIONERS ======================= - // ========================================================== - - - /* SCALAR FIELDS - handles both reference and shared_ptr */ - /* VECTOR FIELDS - handles both reference and shared_ptr */ - // == ippl::Field, Cell> - // == ippl::Field, 3, ippl::UniformCartesian, Cell> - /** - * @brief Executes a scalar/vector field entry, populating the Conduit `node_m` instance and updating the view registry. - * - * @tparam T Field value type. - * @tparam Dim Field dimension. - * @tparam ViewArgs Additional template arguments for the field. - * @param entry The scalar/vector field to execute. - * @param label The label for the field/channel. - * - */ - template - void ExecVizChannel( - const Field& entry - , const std::string label - ); - - - // PARTICLECONTAINERS DERIVED FROM PARTICLEBASE: - /** - * @brief Executes a particle container entry (derived from ParticleBaseBase), populating the Conduit `node_m` instance and updating the view registry. - * - * @tparam T Particle container type (must derive from ippl::ParticleBaseBase). - * @param entry The particle container to execute. - * @param label The label for the container/channel. - */ - template - requires (std::derived_from, ParticleBaseBase>) - void ExecVizChannel( - const T& entry - , const std::string label - ); - - - // BASE CASE: only enabled if EntryT is NOT derived from ippl::ParticleBaseBase - /** - * @brief Fallback for ExecVizChannel: handles types not derived from ParticleBaseBase. - * - * @tparam T Entry type. - * @param label The label for the entry/channel. - * @param entry The entry to execute (not a particle container). - */ - template - requires (!std::derived_from, ParticleBaseBase>) - void ExecVizChannel( - [[maybe_unused]] T&& entry - , const std::string label - ); - - - /* SHARED_PTR DISPATCHER - automatically unwraps and dispatches to appropriate overload */ - /** - * @brief Dispatcher for ExecVizChannel: unwraps shared_ptr and dispatches to the appropriate overload. - * - * @tparam T Entry type. - * @param entry Shared pointer to the entry. - * @param label The label for the entry/channel. - */ - template - void ExecVizChannel( - const std::shared_ptr& entry - , const std::string label - ); - - - - // ========================================================== - // STEERING CHANNEL INITIALIZERS===== ======================= - // ========================================================== - - /** - * @brief Initializes a steerable channel in the Conduit `node_m` instance for runtime parameter adjustment. - * - * @tparam T Type of the steerable parameter. - * @param steerableScalarForwardpass The initial value to set. - * @param label The label for the steerable channel. - */ - template - requires (!std::is_enum_v>) - void InitSteerChannel( const T& steerableScalarForwardpass, const std::string& label ); - - // Enum overloads (arbitrary enum types) - template - requires (std::is_enum_v>) - void InitSteerChannel( const E& e, const std::string& label ); - - // Bool-like switch overload - void InitSteerChannel( const bool& sw, const std::string& label ); - - // Vector overloads for steerable channels - template - void InitSteerChannel( const ippl::Vector& steerableVecForwardpass, const std::string& label ); - - - void InitSteerChannel( const ippl::Button& btn, const std::string& label ); - - // Generic std::vector elements (arithmetic/bool/Button) for array steerables - template - requires (std::is_arithmetic_v> || std::is_enum_v> || std::is_same_v, bool> || std::is_same_v, ippl::Button>) - void InitSteerChannel( const std::vector& arr, const std::string& label ); - - // std::vector of ippl::Vector steerables - template - void InitSteerChannel( const std::vector>& arr, const std::string& label ); - - - // ========================================================== - // STEERING CHANNEL EXECUTIONERS============================= - // ========================================================== - - /** - * @brief Adds a steerable channel to the Conduit `node_m` instance for runtime parameter adjustment. - * - * @tparam T Type of the steerable parameter. - * @param steerableScalarForwardpass The value to pass forward. - * @param steerableSuffix Suffix for the steerable channel name. - */ - template - requires (!std::is_enum_v>) - void ForwardSteerChannel(const T& steerableScalarForwardpass, const std::string& steerableSuffix); - - // Enum overloads (arbitrary enum types) - template - requires (std::is_enum_v>) - void ForwardSteerChannel(const E& e, const std::string& steerableSuffix); - - // Bool-like switch overload - void ForwardSteerChannel(const bool& sw, const std::string& steerableSuffix); - - // Button-like push overloads - void ForwardSteerChannel(const ippl::Button& btn, const std::string& steerableSuffix); - - // Vector overloads for steerable channels - template - void ForwardSteerChannel(const ippl::Vector& steerableVecForwardpass, const std::string& steerableSuffix); - - // Generic std::vector elements (arithmetic/bool/Button) for array steerables - template - requires (std::is_arithmetic_v> || std::is_enum_v> || std::is_same_v, bool> || std::is_same_v, ippl::Button>) - void ForwardSteerChannel(const std::vector& arr, const std::string& label); - - // std::vector of ippl::Vector arrays forward - template - void ForwardSteerChannel(const std::vector>& arr, const std::string& label); - - - - // ========================================================== - // STEERING CHANNEL FETCHERS ============================= - // ========================================================== - - /** - * @brief Fetches the value of a steerable channel from Catalyst results. - * - * @tparam T Type of the steerable parameter. - * @param steerableScalarBackwardpass Reference to store the fetched value. - * @param steerableSuffix Suffix for the steerable channel name. - */ - template - requires (!std::is_enum_v>) - void FetchSteerChannel( T& steerableScalarBackwardpass, const std::string& steerableSuffix); - - // Enum overloads (arbitrary enum types) - template - requires (std::is_enum_v>) - void FetchSteerChannel( E& e, const std::string& steerableSuffix); - - // ippl Vector overload - template - void FetchSteerChannel( ippl::Vector& steerableVecBackwardpass, const std::string& steerableSuffix); - - // standard vector overload for basic types - template - requires (std::is_arithmetic_v> || std::is_enum_v> || std::is_same_v, bool> || std::is_same_v, ippl::Button>) - void FetchSteerChannel( std::vector& out, const std::string& label); - - // std::vector of ippl::Vector arrays backward - template - void FetchSteerChannel( std::vector>& out, const std::string& label); - - - /** - * @brief Retrieves results from Catalyst and populates the given Conduit `node_m` instance. - * - */ - void fetchResults(); - - - // ===================================================================================== - // CatalystAdaptor Public Methods - // ===================================================================================== public: + /** + * @brief Construct a Catalyst adaptor. + * + * @param experimentName Optional label forwarded to Catalyst extractor scripts. + * Runtime output is written below IPPL_CATALYST_OUTPUT_DIR, or ./catalyst when unset. + */ + explicit CatalystAdaptor(std::optional experimentName = std::nullopt) + : catalystInfo_m("CatalystAdaptor::", 0) + , // Only print on rank 0 + catalystWarn_m("CatalystAdaptor_WARNING", std::cerr, INFORM_ALL_NODES) + , experimentName_m(normalizeExperimentName(std::move(experimentName))) + , catalystVis_m(environmentValue("IPPL_CATALYST_VIS", "ON")) + , catalystLive_m(environmentValue("IPPL_CATALYST_LIVE", "OFF")) + , catalystSteer_m(environmentValue("IPPL_CATALYST_STEER", "OFF")) + , catalystPng_m(environmentValue("IPPL_CATALYST_PNG", "OFF")) + , catalystVtk_m(environmentValue("IPPL_CATALYST_VTK", "OFF")) + , catalystVerbosity_m(environmentValue("IPPL_CATALYST_VERBOSITY", + std::to_string(ippl::Info->getOutputLevel()))) + , catalystGhostMask_m(environmentValue("IPPL_CATALYST_GHOST_MASKS", "OFF")) + , proxyOption_m(environmentValue("IPPL_CATALYST_PROXY_OPTION", "ON")) + , visEnabled_m(catalystVis_m != "OFF") + , liveEnabled_m(catalystLive_m == "ON") + , steerEnabled_m(catalystSteer_m == "ON" && catalystLive_m == "ON") + , outputLevel_m(std::stoi(catalystVerbosity_m)) + , useGhostMasks_m(catalystGhostMask_m == "ON") + , resourceDir_m(IPPL_CATALYST_SCRIPTS_DIR) + , outputDir_m(catalystOutputDirectory()) { + catalystInfo_m.setOutputLevel(outputLevel_m); + + // #if defined(MPI_VERSION) + // MPI_Barrier(MPI_COMM_WORLD); + // if(ippl::Comm->rank()==0) catalystWarn_m << "[rank = 0 size=" << ippl::Comm->size() + // << "]" << endl; MPI_Barrier(MPI_COMM_WORLD); if(ippl::Comm->rank()==1) catalystWarn_m + // << "[rank= 1 size=" << ippl::Comm->size() << "]" << endl; + // MPI_Barrier(MPI_COMM_WORLD); + // #endif + + catalystInfo_m << level4 + << "::CatalystAdaptor() Global Output Level setting: " + << ippl::Info->getOutputLevel() << endl; + catalystInfo_m << level4 + << "::CatalystAdaptor() Catalyst Info Output Level setting: " + << catalystInfo_m.getOutputLevel() << endl; + catalystInfo_m << level4 + << "::CatalystAdaptor() Catalyst Warn Output Level setting: " + << catalystWarn_m.getOutputLevel() << endl; + catalystInfo_m << level4 << "::CatalystAdaptor() using resourceDir_m = " + << resourceDir_m.string() << endl; + catalystInfo_m << level4 + << "::CatalystAdaptor() using outputDir_m = " << outputDir_m.string() + << endl; + if (catalystPng_m == "ON") { + catalystInfo_m << level4 << "::CatalystAdaptor() PNG extraction ACTIVATED" + << endl; + } else { + catalystInfo_m << level4 << "::CatalystAdaptor() PNG extraction DEACTIVATED" + << endl; + } + if (catalystVtk_m == "ON") { + catalystInfo_m << level4 << "::CatalystAdaptor() VTK extraction ACTIVATED" + << endl; + } else { + catalystInfo_m << level4 << "::CatalystAdaptor() VTK extraction DEACTIVATED" + << endl; + } + if (steerEnabled_m) { + catalystInfo_m << level4 << "::CatalystAdaptor() Steering ACTIVATED" + << endl; + } else { + catalystInfo_m << level4 << "::CatalystAdaptor() Steering DEACTIVATED" + << endl; + } + if (catalystSteer_m == "ON" && !liveEnabled_m) { + catalystWarn_m << "::CatalystAdaptor() IPPL_CATALYST_STEER=ON requires " + "IPPL_CATALYST_LIVE=ON; steering is disabled." + << endl; + } + } - /** - * @brief Struct steering registration. - * - * Expose any user struct composed of already supported steerable member - * types (arithmetic, bool, ippl::Button, ippl::Vector<>, enums). Nested structs are not - * supported yet. Must be called before adding the struct instance to a - * runtime registry. Args must be an even-sized pack of: name(string-like), pointer-to-member. - * Validates each member type at registration (throws IpplException if invalid). - * Stores three lambdas which expand the pack and delegate to visitor overloads. - * - * @tparam T The struct type to register. - * @tparam Args Types of the member pointers. - * @param args Pointers to the struct members. - */ - template - static void RegisterStructMembers(Args&&... args); - - - - /* DEPRECATED atm ... */ - // Optional enum metadata: label -> list of (text,value) choices - // std::unordered_map>> enumChoices_m; - // /** - // * @brief Provides enum choices metadata so the GUI shows a dropdown. - // * - // * Use this before InitializeRuntime. - // * Example: RegisterEnumChoices("mode", {{"Off",0},{"Basic",1},{"Advanced",2}}); - // * - // * @param label The label for the enum choice. - // * @param entries A vector of pairs containing the display name and integer value. - // */ - // void RegisterEnumChoices(const std::string& label, - // const std::vector>& entries) { - // enumChoices_m[label] = entries; - // } - - // /** - // * @brief Provides typed enum choices metadata mapped to a specific label for the GUI dropdown. - // * - // * @tparam E The enumeration type. - // * @param label The specific label of the steerable channel to associate the choices with. - // * @param entries A vector of pairs containing the display name and the enumeration value. - // */ - // template - // requires (std::is_enum_v>) - // void RegisterEnumChoicesTyped(const std::string& label, const std::vector>& entries); - - /** - * @brief Provides typed enum choices metadata registered by the enum type, to be reused across labels. - * - * @tparam E The enumeration type. - * @param entries A vector of pairs containing the display name and the enumeration value. - */ - template - requires (std::is_enum_v>) - void RegisterEnumChoicesTyped(const std::vector>& entries); - - - - - /** - * @brief Initializes Catalyst using runtime registries (visualization and steering). - * - * @param visReg Shared pointer to the visualization runtime registry. - * @param steerReg Shared pointer to the steering runtime registry. - */ - void Initialize( - const std::shared_ptr& visReg, - const std::shared_ptr& steerReg - ); - - /** - * @brief Explicitly forces a host copy for a specifically labelled channel right now. - * - * @param label The label specifying which field or particle to remember currently. - */ - void rememberNow(const std::string label); - - /** - * @brief Executes Catalyst for a given timestep using the runtime registry. - * - * Populates forward steerable values and fetches back updated ones. - * - * @param cycle The current simulation cycle or timestep index. - * @param time The current simulation time. - * @param rank The MPI rank of the executing process (defaults to ippl::Comm->rank()). - */ - void Execute( - int cycle, double time, - int rank = ippl::Comm->rank() - ); - - - /** - * @brief Finalizes Catalyst and releases resources. - */ - void Finalize(); - + private: + // ============================================================================================== + // HELPERS + // ===================================================================================== + // ============================================================================================== + /** + * @brief Sets a file path to a node, using an environment variable if available, otherwise + * a default path. + * + * @param nodePath The Conduit `node_m` instance to set the file path in. + * @param envVar The name of the environment variable to check. + * @param defaultFilePath The default file path to use if the environment variable is not + * set or invalid. + */ + void setNodeScript(conduit_cpp::Node nodePath, + // const char* envVar, + const std::string envVar, const std::filesystem::path defaultFilePath); + private: + // ========================================================== + // STEERING CHANNEL INITIALIZERS===== ======================= + // ========================================================== + + /** + * @brief Initializes a steerable channel in the Conduit `node_m` instance for runtime + * parameter adjustment. + * + * @tparam T Type of the steerable parameter. + * @param steerableScalarForwardpass The initial value to set. + * @param label The label for the steerable channel. + */ + template + requires(!std::is_enum_v>) + void InitSteerChannel(const T& steerableScalarForwardpass, const std::string& label); + + // Enum overloads (arbitrary enum types) + template + requires(std::is_enum_v>) + void InitSteerChannel(const E& e, const std::string& label); + + // Bool-like switch overload + void InitSteerChannel(const bool& sw, const std::string& label); + + // Vector overloads for steerable channels + template + void InitSteerChannel(const ippl::Vector& steerableVecForwardpass, + const std::string& label); + + void InitSteerChannel(const ippl::Button& btn, const std::string& label); + + // Generic std::vector elements (arithmetic/bool/Button) for array steerables + template + requires(std::is_arithmetic_v> || std::is_enum_v> + || std::is_same_v, bool> + || std::is_same_v, ippl::Button>) + void InitSteerChannel(const std::vector& arr, const std::string& label); + + // std::vector of ippl::Vector steerables + template + void InitSteerChannel(const std::vector>& arr, + const std::string& label); + + // ========================================================== + // STEERING CHANNEL EXECUTIONERS============================= + // ========================================================== + + /** + * @brief Adds a steerable channel to the Conduit `node_m` instance for runtime parameter + * adjustment. + * + * @tparam T Type of the steerable parameter. + * @param steerableScalarForwardpass The value to pass forward. + * @param steerableSuffix Suffix for the steerable channel name. + */ + template + requires(!std::is_enum_v>) + void ForwardSteerChannel(const T& steerableScalarForwardpass, + const std::string& steerableSuffix); + + // Enum overloads (arbitrary enum types) + template + requires(std::is_enum_v>) + void ForwardSteerChannel(const E& e, const std::string& steerableSuffix); + + // Bool-like switch overload + void ForwardSteerChannel(const bool& sw, const std::string& steerableSuffix); + + // Button-like push overloads + void ForwardSteerChannel(const ippl::Button& btn, const std::string& steerableSuffix); + + // Vector overloads for steerable channels + template + void ForwardSteerChannel(const ippl::Vector& steerableVecForwardpass, + const std::string& steerableSuffix); + + // Generic std::vector elements (arithmetic/bool/Button) for array steerables + template + requires(std::is_arithmetic_v> || std::is_enum_v> + || std::is_same_v, bool> + || std::is_same_v, ippl::Button>) + void ForwardSteerChannel(const std::vector& arr, const std::string& label); + + // std::vector of ippl::Vector arrays forward + template + void ForwardSteerChannel(const std::vector>& arr, + const std::string& label); + + // ========================================================== + // STEERING CHANNEL FETCHERS ============================= + // ========================================================== + + /** + * @brief Fetches the value of a steerable channel from Catalyst results. + * + * @tparam T Type of the steerable parameter. + * @param steerableScalarBackwardpass Reference to store the fetched value. + * @param steerableSuffix Suffix for the steerable channel name. + */ + template + requires(!std::is_enum_v>) + void FetchSteerChannel(T& steerableScalarBackwardpass, const std::string& steerableSuffix); + + // Enum overloads (arbitrary enum types) + template + requires(std::is_enum_v>) + void FetchSteerChannel(E& e, const std::string& steerableSuffix); + + // ippl Vector overload + template + void FetchSteerChannel(ippl::Vector& steerableVecBackwardpass, + const std::string& steerableSuffix); + + // standard vector overload for basic types + template + requires(std::is_arithmetic_v> || std::is_enum_v> + || std::is_same_v, bool> + || std::is_same_v, ippl::Button>) + void FetchSteerChannel(std::vector& out, const std::string& label); + + // std::vector of ippl::Vector arrays backward + template + void FetchSteerChannel(std::vector>& out, const std::string& label); + + /** + * @brief Retrieves results from Catalyst and populates the given Conduit `node_m` instance. + * + */ + void fetchResults(); + + // ===================================================================================== + // CatalystAdaptor Public Methods + // ===================================================================================== + public: + /** + * @brief Struct steering registration. + * + * Expose any user struct composed of already supported steerable member + * types (arithmetic, bool, ippl::Button, ippl::Vector<>, enums). Nested structs are not + * supported yet. Must be called before adding the struct instance to a + * runtime registry. Args must be an even-sized pack of: name(string-like), + * pointer-to-member. Validates each member type at registration (throws IpplException if + * invalid). Stores three lambdas which expand the pack and delegate to visitor overloads. + * + * @tparam T The struct type to register. + * @tparam Args Types of the member pointers. + * @param args Pointers to the struct members. + */ + template + static void RegisterStructMembers(Args&&... args); + + /* DEPRECATED atm ... */ + // Optional enum metadata: label -> list of (text,value) choices + // std::unordered_map>> enumChoices_m; + // /** + // * @brief Provides enum choices metadata so the GUI shows a dropdown. + // * + // * Use this before InitializeRuntime. + // * Example: RegisterEnumChoices("mode", {{"Off",0},{"Basic",1},{"Advanced",2}}); + // * + // * @param label The label for the enum choice. + // * @param entries A vector of pairs containing the display name and integer value. + // */ + // void RegisterEnumChoices(const std::string& label, + // const std::vector>& entries) { + // enumChoices_m[label] = entries; + // } + + // /** + // * @brief Provides typed enum choices metadata mapped to a specific label for the GUI + // dropdown. + // * + // * @tparam E The enumeration type. + // * @param label The specific label of the steerable channel to associate the choices + // with. + // * @param entries A vector of pairs containing the display name and the enumeration + // value. + // */ + // template + // requires (std::is_enum_v>) + // void RegisterEnumChoicesTyped(const std::string& label, const + // std::vector>& entries); + + /** + * @brief Provides typed enum choices metadata registered by the enum type, to be reused + * across labels. + * + * @tparam E The enumeration type. + * @param entries A vector of pairs containing the display name and the enumeration value. + */ + template + requires(std::is_enum_v>) + void RegisterEnumChoicesTyped(const std::vector>& entries); + + /** + * @brief Initializes Catalyst using the new channel-based API. + * + * Visualization channels (registered via addMeshChannel/addParticleChannel) + * are initialized here. The steering registry is optional. + * + * @param steerReg Shared pointer to the steering runtime registry (may be null). + */ + void Initialize(const std::shared_ptr& steerReg = nullptr); + + /** + * @brief Registers a mesh channel with multiple field arrays. + * + * One Conduit channel is created per mesh; call addArray() on the + * returned handle to attach fields. + * + * @tparam Dim Mesh dimension (1, 2, or 3). + * @param name Channel name (e.g. "bunch_mesh"). + * @param mesh The IPPL mesh (must outlive the adaptor). + * @param layout The field layout (must outlive the adaptor). + * @param nghost Number of ghost layers (from the first field). + * @param useGhostMasks Whether to publish ghost cells with vtkGhostType. + * @return Handle for chaining addArray() calls. + */ + template + MeshChannelHandle addMeshChannel(const std::string& name, + const UniformCartesian& mesh, + const FieldLayout& layout, int nghost = 0, + bool useGhostMasks = false); + + /** + * @brief Registers a mesh block inside an existing particle multimesh. + * + * The mesh is written into + * catalyst/channels//data/ + * instead of a top-level channel. The caller is responsible for + * adding the corresponding assembly entry + * catalyst/channels//assembly/ = "" + * (typically done in the particle channel's init function). + * + * @tparam Dim Mesh dimension (1, 2, or 3). + * @param parentName Name of the parent particle channel. + * @param blockName Block name within the parent multimesh. + * @param mesh The IPPL mesh (must outlive the adaptor). + * @param layout The field layout (must outlive the adaptor). + * @param nghost Number of ghost layers. + * @param useGhostMasks Whether to publish ghost cells. + * @param geometryPolicy Whether to re-read mesh origin/spacing per step. + * @return Handle for chaining addArray() calls. + */ + template + MeshChannelHandle addMeshBlock(const std::string& parentName, const std::string& blockName, + const UniformCartesian& mesh, + const FieldLayout& layout, int nghost = 0, + bool useGhostMasks = false, + GeometryPolicy geometryPolicy = GeometryPolicy::Dynamic); + + /** + * @brief Registers a particle channel (one per particle container). + * + * Supports multi-bunch: call once per container. Each channel is + * independent with its own multimesh and optional transform. + * + * @tparam T Particle container type (deriving from ParticleBaseBase). + * @param name Channel name (e.g. "BEAM1", "BEAM2"). + * @param pc The particle container (must outlive the adaptor). + * @return Handle for chaining setTransform() calls. + */ + template + requires std::derived_from, ParticleBaseBase> + ParticleChannelHandle addParticleChannel(const std::string& name, const T& pc); + + /// Configure before Initialize(). Unconfigured particle channels use all(). + void setParticleAttributeSelection( + const std::string& label, + ParticleAttributeSelection selection = ParticleAttributeSelection::all()); + + const ParticleAttributeSelection& getParticleAttributeSelection(const std::string& label) const; + + /** + * @brief Forces a refresh of a specific array (replaces rememberNow). + * + * For Manual-policy arrays, copies device->host immediately. + * For Always/Static arrays, this is a no-op (Always copies on next + * Execute anyway; Static never copies). + * + * @param label The array label to refresh. + */ + void refresh(const std::string& label); + + /** + * @brief Deprecated. Use refresh(label) instead. + */ + void rememberNow(const std::string label) { refresh(label); } + + /** + * @brief Executes Catalyst for a given timestep. + * + * Iterates all registered channels, copies dynamic data into staging + * buffers, and calls catalyst_execute(). + * + * @param cycle The current simulation cycle or timestep index. + * @param time The current simulation time. + * @param rank The MPI rank of the executing process. + */ + void Execute(int cycle, double time, int rank = ippl::Comm->rank()); + + /** + * @brief Finalizes Catalyst and releases resources. + */ + void Finalize(); + + }; // class CatalystAdaptor +} // namespace ippl -};//class CatalystAdaptor -} //namespace ippl - -#include "Stream/InSitu/CatalystVisitors.h" -#include "Stream/Registry/VisRegistryRuntime.h" // visitor structs #include "CatalystAdaptor.hpp" - - +#include "Stream/InSitu/CatalystVisitors.h" +#include "Stream/Registry/VisRegistryRuntime.h" // visitor structs #endif diff --git a/src/Stream/InSitu/CatalystAdaptor.hpp b/src/Stream/InSitu/CatalystAdaptor.hpp index b195628d6..2d6cc4cac 100644 --- a/src/Stream/InSitu/CatalystAdaptor.hpp +++ b/src/Stream/InSitu/CatalystAdaptor.hpp @@ -1,1227 +1,823 @@ #pragma once -#include "Stream/InSitu/CatalystAdaptor.h" - #include "Ippl.h" -#include + #include -#include +#include #include -namespace ippl{ - -// ============================================================================================== -// HELPERS ===================================================================================== -// ============================================================================================== -void CatalystAdaptor::setNodeScript( - conduit_cpp::Node nodePath, - const std::string envVar, - const std::filesystem::path defaultFilePath -){ +#include + +#include "Stream/InSitu/CatalystAdaptor.h" +#include "Stream/InSitu/CatalystVisitors.h" +#include "Stream/Registry/VisRegistryRuntime.h" + +namespace { + /** + * @brief Device functor for zeroing the interior of a ghost-mask field. + * + * Using a named functor instead of a lambda avoids the NVCC restriction that + * extended __host__ __device__ lambdas cannot appear inside private/protected + * member functions. + */ + template + struct ZeroGhostMaskInterior { + View view; + + KOKKOS_INLINE_FUNCTION void operator()(const int i) const { view(i) = 0; } + + KOKKOS_INLINE_FUNCTION void operator()(const int i, const int j) const { view(i, j) = 0; } + + KOKKOS_INLINE_FUNCTION void operator()(const int i, const int j, const int k) const { + view(i, j, k) = 0; + } + }; +} // namespace + +namespace ippl { + + // ============================================================================================== + // HELPERS + // ===================================================================================== + // ============================================================================================== + void CatalystAdaptor::setNodeScript(conduit_cpp::Node nodePath, const std::string envVar, + const std::filesystem::path defaultFilePath) { const char* filePathEnv = std::getenv(envVar.c_str()); std::filesystem::path filePath; if (filePathEnv && std::filesystem::exists(filePathEnv)) { - catalystInfo_m << level4 <<"::Initialize()::setNodeScripts(...):\n" - << " Using " << envVar << " from environment:\n" - << " "<< filePathEnv << endl; - filePath = filePathEnv; + catalystInfo_m << level4 << "::Initialize()::setNodeScripts(...):\n" + << " Using " << envVar << " from environment:\n" + << " " << filePathEnv << endl; + filePath = filePathEnv; } else { - catalystInfo_m << level4 <<"::Initialize()::setNodeScripts(...): No valid " << envVar <<" set.\n" - << " Using default:\n" - << " " << defaultFilePath << endl; - filePath = defaultFilePath; + catalystInfo_m << level4 << "::Initialize()::setNodeScripts(...): No valid " << envVar + << " set.\n" + << " Using default:\n" + << " " << defaultFilePath << endl; + filePath = defaultFilePath; } nodePath.set(filePath.string()); -} - - -// ============================================================================================== -// VIS CHANNEL INITIALIZER:====================================================================== -// ============================================================================================== - -// == ippl::Field, Cell> -template -void CatalystAdaptor::InitVizChannel( [[maybe_unused]] const Field& entry , const std::string label) -{ - catalystInfo_m << level4 <<"::Initialize()::InitVizChannel(ippl::Field<" << typeid(T).name() << "," << Dim << ">) called" << endl; - forceHostCopy_m[label] = false; - - const std::string channelName = "ippl_sField_" + label; - if(pngExtracts_m){ - const std::string script = "catalyst/scripts/" + label; - setNodeScript( node_m[script + "/filename"], - "CATALYST_EXTRACTOR_SCRIPT_" +label, - resourceDir_m / "catalyst_extractors" / "png_ext_sfield.py" - ); - conduit_cpp::Node args = node_m[script + "/args"]; - args.append().set_string("--channel_name"); - args.append().set_string(channelName); - args.append().set_string("--label"); - args.append().set_string(label); - if(experimentName_m){ - args.append().set_string("--experiment_name"); - args.append().set_string(*experimentName_m); - } - - args.append().set_string("--verbosity"); - args.append().set_string(std::to_string(catalystInfo_m.getOutputLevel())); - } - - conduit_cpp::Node scriptArgs = node_m["catalyst/scripts/script/args"]; - scriptArgs.append().set_string(channelName); -} - - -// == ippl::Field, 3, ippl::UniformCartesian, Cell> -template -void CatalystAdaptor::InitVizChannel( [[maybe_unused]] const Field, Dim, ViewArgs...>& entry , const std::string label) -{ - catalystInfo_m << level4 << "::Initialize()::InitVizChannel(ippl::Field," << Dim - << ">) called" << endl; - - forceHostCopy_m[label] = false; - - - const std::string channelName = "ippl_vField_" + label; - if(pngExtracts_m){ - const std::string script = "catalyst/scripts/" + label; - - setNodeScript( node_m[script + "/filename"], - "CATALYST_EXTRACTOR_SCRIPT_" + label, - resourceDir_m / "catalyst_extractors" / "png_ext_vfield.py" - - ); - conduit_cpp::Node args = node_m[script + "/args"]; - args.append().set_string("--channel_name"); - args.append().set_string(channelName); - args.append().set_string("--label"); - args.append().set_string(label); - if(experimentName_m){ - args.append().set_string("--experiment_name"); - args.append().set_string(*experimentName_m); - } - args.append().set_string("--verbosity"); - args.append().set_string(std::to_string(catalystInfo_m.getOutputLevel())); - } - + } - conduit_cpp::Node scriptArgs = node_m["catalyst/scripts/script/args"]; - scriptArgs.append().set_string(channelName); - -} - -// PARTICLECONTAINERS derived from ParticleBaseBase: -// == ippl::ParticleBase,...>,...> -template -requires (std::derived_from, ParticleBaseBase>) -void CatalystAdaptor::InitVizChannel( [[maybe_unused]] const T& entry, const std::string label) -{ - catalystInfo_m << level4 << "::Initialize()::InitVizChannel(ParticleBase).name() << ","<< particle_dim_v - << ",...>...> [or subclass]) called" << endl; - - forceHostCopy_m[label] = false; - - const std::string channelName = "ippl_particles_" + label; - if(pngExtracts_m){ - const std::string script = "catalyst/scripts/"+ label; - - setNodeScript( - node_m[script + "/filename"], - "CATALYST_EXTRACTOR_SCRIPT_" +label, - resourceDir_m / "catalyst_extractors" / "png_ext_particle.py" - ); - - conduit_cpp::Node args = node_m[script + "/args"]; - args.append().set_string("--channel_name"); - args.append().set_string(channelName); - args.append().set_string("--label"); - args.append().set_string(label); - - if(experimentName_m){ - args.append().set_string("--experiment_name"); - args.append().set_string(*experimentName_m); - } +} // namespace ippl - args.append().set_string("--verbosity"); - args.append().set_string(std::to_string(catalystInfo_m.getOutputLevel())); - } +// ===================================================================================== +// STEERING: +// ===================================================================================== +#include "Stream/InSitu/CatalystAdaptorSteering.hpp" +// ===================================================================================== - conduit_cpp::Node scriptArgs = node_m["catalyst/scripts/script/args"]; - scriptArgs.append().set_string(channelName); -} - -/* SHARED_PTR DISPATCHER - automatically unwraps and dispatches to appropriate overload */ -template -void CatalystAdaptor::InitVizChannel( const std::shared_ptr& entry, const std::string label) -{ - if (entry) { - InitVizChannel( *entry - , label - ); - } - else { - catalystWarn_m << "::Initialize()InitVizChannel(nullptr): nullptr passed as entry." << endl - << " ID: "<< label << endl - << " ==> Channel will not be registered in Conduit Node." << endl; - } -} - - -// BASE CASE: -template -requires (!std::derived_from, ParticleBaseBase>) -void CatalystAdaptor::InitVizChannel([[maybe_unused]] const T& entry, const std::string label) -{ - catalystWarn_m << "::Initialize()InitVizChannel(nullptr): Entry type can't be processed." << endl - << " ID: "<< label << endl - << " Type: "<< typeid(std::decay_t).name() << endl - << " ==>Channel will not be registered in Conduit Node." << endl - << " If you see this something is wrong with: CatalystAdaptor::InitVisitor!!" << endl; -} - - - - -// == ippl::Field, Cell>* -// == ippl::Field, 3, ippl::UniformCartesian, Cell>* -template -void CatalystAdaptor::ExecVizChannel(const Field& entry, const std::string label) -{ - const bool refreshOnly = viewRegistry_m.contains(label); - if (refreshOnly && !forceHostCopy_m[label]) { - return; +namespace ippl { + /////////////////////////////////////////////////////////////// + // Note: + // this does not really need to be a separate function call, + // May we should just inline this into the execute functions ... + /////////////////////////////////////////////////////////////// + void CatalystAdaptor::fetchResults() { + catalyst_status err = catalyst_results(conduit_cpp::c_node(&results_m)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to execute Catalyst-results: " << err << std::endl; + } } - using Field_type = Field; - const Field_type* field = &entry; - - std::string channelName; - if constexpr (std::is_scalar_v) { - channelName = "ippl_sField_" + label; - catalystInfo_m << level4 <<"::Execute()::ExecVizChannel(" << label << ") | Type:ippl::Field<" << typeid(T).name() << "," << Dim << ">) called" << endl; - - } else if constexpr (is_vector_v) { - channelName = "ippl_vField_" + label; - catalystInfo_m << level4 <<"::Execute()::ExecVizChannel(" << label << ") | Type: ippl::Field::dim << ">," << Dim << ">)" << endl; - }else{ - channelName = "ippl_errorField_" + label; - - catalystInfo_m << level4 << "::Execute()::ExecVizChannel(Field<"<)\n" - << " For this type of Field the Conduit Blueprint description wasnt \n" - << " implemented in ippl. Therefore this type of field is not \n" - << " supported for visualisation." << endl; - } + // ===================================================================================== + // Channel-based Initialize / Execute + // ===================================================================================== - conduit_cpp::Node field_node; - typename Field_type::Layout_t& Layout_ = field->getLayout(); - typename Field_type::Mesh_t& Mesh_ = field->get_mesh(); - - const auto LocalNDIndex_ = Layout_.getLocalNDIndex(); - const auto Origin_ = Mesh_.getOrigin(); - const auto Spacing_ = Mesh_.getMeshSpacing(); - - const size_t nGhost = field->getNghost(); // returns int - - const size_t extra = (useGhostMasks_m) ? size_t(2*nGhost) : 0 ; - const size_t index_offset = (useGhostMasks_m) ? size_t(nGhost) : 0 ; - - int dims_n=1; - double extra_origin=0; - - static_assert(Dim >= 1 && Dim <= 3, - "Catalyst visualization supports fields with one to three dimensions"); - - const auto Ox = Origin_[0] - + (double(int(LocalNDIndex_[0].first()) - int(index_offset)) - + extra_origin) - * Spacing_[0]; - double Oy = 0.0; - double Oz = 0.0; - if constexpr (Dim >= 2) { - Oy = Origin_[1] - + (double(int(LocalNDIndex_[1].first()) - int(index_offset)) + extra_origin) - * Spacing_[1]; + inline void CatalystAdaptor::setParticleAttributeSelection( + const std::string& label, ParticleAttributeSelection selection) { + if (initialized_m) { + throw IpplException("CatalystAdaptor::setParticleAttributeSelection", + "Configure particle attributes before Initialize(), or after Finalize()."); } - if constexpr (Dim >= 3) { - Oz = Origin_[2] - + (double(int(LocalNDIndex_[2].first()) - int(index_offset)) + extra_origin) - * Spacing_[2]; + if (label.empty()) { + throw IpplException("CatalystAdaptor::setParticleAttributeSelection", "Empty channel label."); } + particleAttributeSelections_m.insert_or_assign(label, std::move(selection)); + } - const size_t nx = LocalNDIndex_[0].length() + extra; - size_t ny = 1; - size_t nz = 1; - if constexpr (Dim >= 2) { - ny = LocalNDIndex_[1].length() + extra; - } - if constexpr (Dim >= 3) { - nz = LocalNDIndex_[2].length() + extra; - } + inline const ParticleAttributeSelection& CatalystAdaptor::getParticleAttributeSelection( + const std::string& label) const { + static const auto DEFAULT_SELECTION = ParticleAttributeSelection::all(); + const auto it = particleAttributeSelections_m.find(label); + return it == particleAttributeSelections_m.end() ? DEFAULT_SELECTION : it->second; + } - const auto& fullDeviceView = field->getView(); // original view - using DeviceView_t = typename Field::view_type; - using HostView_t = Kokkos::View< - typename DeviceView_t::data_type, - Kokkos::LayoutLeft, - Kokkos::HostSpace - >; - - auto makeHostView = [&](const char* viewLabel) -> HostView_t { - if constexpr (Dim == 1) { - return HostView_t(viewLabel, nx); - } else if constexpr (Dim == 2) { - return HostView_t(viewLabel, nx, ny); - } else { - return HostView_t(viewLabel, nx, ny, nz); + inline void CatalystAdaptor::validateParticleAttributeSelections() const { + std::map> particleAttributeNames; + for (const auto& [label, source] : particleAttributeSources_m) { + particleAttributeNames.emplace(label, source()); + } + std::string error; + for (const auto& [label, selection] : particleAttributeSelections_m) { + if (!particleAttributeNames.contains(label)) { + error = "Selection refers to an unknown particle channel: '" + label + "'"; + break; } - }; + } - auto copyFieldToHost = [&](HostView_t& hostView, bool includeGhosts) { - if (includeGhosts) { - Kokkos::deep_copy(hostView, fullDeviceView); - } else if constexpr (Dim == 1) { - const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); - Kokkos::deep_copy(hostView, Kokkos::subview(fullDeviceView, r0)); - } else if constexpr (Dim == 2) { - const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); - const auto r1 = Kokkos::make_pair(nGhost, nGhost + ny); - Kokkos::deep_copy(hostView, Kokkos::subview(fullDeviceView, r0, r1)); - } else { - const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); - const auto r1 = Kokkos::make_pair(nGhost, nGhost + ny); - const auto r2 = Kokkos::make_pair(nGhost, nGhost + nz); - Kokkos::deep_copy(hostView, Kokkos::subview(fullDeviceView, r0, r1, r2)); - } + // Compare effective policies and schemas, irrespective of registration or request order. + std::string signature; + auto append = [&](const std::string& value) { + signature += std::to_string(value.size()) + ":" + value; }; - - if (refreshOnly) { - HostView_t* hostMirrorFinal = viewRegistry_m.find(label); - if (!hostMirrorFinal) { - throw IpplException("Stream::InSitu::CatalystAdaptor::ExecVizChannel", - "Missing host mirror for refresh: " + label); + for (const auto& [label, names] : particleAttributeNames) { + const auto& selection = getParticleAttributeSelection(label); + const auto reason = selection.validationError(names); + if (!reason.empty() && error.empty()) { + error = "Particle channel '" + label + "': " + reason; } - copyFieldToHost(*hostMirrorFinal, useGhostMasks_m); - return; + append(label); + append(std::to_string(static_cast(selection.mode()))); + append(selection.includesParticleIds() ? "ids" : "no ids"); + append(selection.includesRankIds() ? "ranks" : "no ranks"); + append(std::to_string(selection.names().size())); + for (const auto& name : selection.names()) append(name); + append(std::to_string(names.size())); + for (const auto& name : names) append(name); } - - // channel for this field of type mesh adheres to conduits mesh blueprint - auto channel = node_m["catalyst/channels/"+ channelName]; - auto channel_state = channel["state"]; - - channel["type"].set_string("mesh"); - auto data = channel["data"]; - - - auto fields = data["fields"]; - field_node = fields[label]; - data["topologies/fmesh_topo/type"].set_string("uniform"); - data["topologies/fmesh_topo/coordset"].set_string("cart_uniform_coords"); - data["coordsets/cart_uniform_coords/type"].set_string("uniform"); - - const void* meshKey = static_cast(&Mesh_); - const void* layoutKey = static_cast(&Layout_); - auto ghostKey = GhostKey_t{meshKey, layoutKey, nGhost}; - - const size_t localNumCells = nx * ny * nz; - const int rank = ippl::Comm->rank(); - - // using RankViewCells_t = Kokkos::View; - // RankViewCells_t rank_id_view_cells("rank_id_view_cells", localNumCells); - // auto host_policy = getRangePolicy(rank_id_view_cells); - - using RankViewCells_t = Kokkos::View; - using HostExecSpace = Kokkos::DefaultHostExecutionSpace; - RankViewCells_t rank_id_view_cells("rank_id_view_cells_3D", nx, ny, nz); - - if (localNumCells > 0) { - Kokkos::MDRangePolicy> host_policy( - {0, 0, 0}, // Start indices {i, j, k} - {nx, ny, nz} // End indices {i, j, k} - ); - Kokkos::parallel_for("fill_rank_ids_3D", host_policy, - KOKKOS_LAMBDA(const int i, const int j, const int k) { - rank_id_view_cells(i, j, k) = rank; - }); + if (signature.size() > static_cast(std::numeric_limits::max())) { + error = "Particle attribute configuration is too large for MPI validation."; } - auto rank_field = fields["RankID"]; - rank_field["association"].set_string("element"); // associate_m); - rank_field["topology"].set_string("fmesh_topo"); - rank_field["volume_dependent"].set_string("false"); - if (localNumCells > 0) { - rank_field["values"].set_external(rank_id_view_cells.data(), localNumCells); - } else { - rank_field["values"].set_external(static_cast(nullptr), 0); - } - data["metadata/vtk_fields/RankID/attribute_type"].set_string("ProcessIds"); - viewRegistry_m.set(label + "_rank_id_cells", rank_id_view_cells); - - - // auto print_ranked_mesh_info = [&](){ - // catalystWarn_m << "[ rank=" << ippl::Comm->rank() << "]" - // << " | dims(points)=" << nx << "x" << ny << "x" << nz - // << " | ghost: " << nGhost - // << " | origin=(" << Ox << "," << Oy << "," << Oz << ")" - // << " | spacing=(" << Spacing_[0] << "," << (Dim>=2?Spacing_[1]:0)<< "," << (Dim>=3?Spacing_[2]:0) << ")" << endl; - // }; - - // #if defined(MPI_VERSION) - // MPI_Barrier(MPI_COMM_WORLD); - // if(ippl::Comm->rank()==0) print_ranked_mesh_info(); - // MPI_Barrier(MPI_COMM_WORLD); - // if(ippl::Comm->rank()==1) print_ranked_mesh_info(); - // MPI_Barrier(MPI_COMM_WORLD); - // #endif - - { - data["coordsets/cart_uniform_coords/dims/i"].set(nx+ dims_n ); - data["coordsets/cart_uniform_coords/spacing/dx"].set(Spacing_[0]); - data["coordsets/cart_uniform_coords/origin/x"].set( Ox ); - data["topologies/fmesh_topo/origin/x"].set( Ox ); + // Every rank must reject invalid configuration before Catalyst enters collectives. + int invalid = !error.empty(); + MPI_Allreduce(MPI_IN_PLACE, &invalid, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (invalid) { + throw IpplException("CatalystAdaptor::Initialize", + error.empty() ? "Invalid particle attribute selection on another MPI rank." + : error); } - if constexpr(Dim >= 2){ - data["coordsets/cart_uniform_coords/dims/j"].set(ny+ dims_n); - data["coordsets/cart_uniform_coords/spacing/dy"].set(Spacing_[1]); - data["coordsets/cart_uniform_coords/origin/y"].set( Oy ); - data["topologies/fmesh_topo/origin/y"].set( Oy ); + int length = static_cast(signature.size()); + MPI_Bcast(&length, 1, MPI_INT, 0, MPI_COMM_WORLD); + std::string reference = signature; + reference.resize(length); + MPI_Bcast(reference.data(), length, MPI_CHAR, 0, MPI_COMM_WORLD); + invalid = reference != signature; + MPI_Allreduce(MPI_IN_PLACE, &invalid, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + if (invalid) { + throw IpplException("CatalystAdaptor::Initialize", + "Particle attribute names or selections differ between MPI ranks."); } - if constexpr(Dim >= 3){ - data["coordsets/cart_uniform_coords/dims/k"].set(nz+ dims_n); - data["coordsets/cart_uniform_coords/spacing/dz"].set(Spacing_[2]); - data["coordsets/cart_uniform_coords/origin/z"].set( Oz ); - data["topologies/fmesh_topo/origin/z"].set( Oz ); - } - - // ================================== - // Prepare Field Data in a HostMirror - // ================================== - - // Version 1: Cut out Ghost Cells from data during a deep copy into a new Kokkos View. - auto getHostMirrorView_noGhosts = [&]() -> HostView_t { - HostView_t hostMirrorFinal = makeHostView("hostMirrorNoGhosts_LayoutLeft"); - // This single deep_copy now performs: - // - Device-to-Host transfer - // - LayoutRight-to-LayoutLeft transpose - // - Cutting Ghost Cells from data. - copyFieldToHost(hostMirrorFinal, false); - return hostMirrorFinal ; - - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - // NOTE: - // for both lambdas: - // Technically, we need to use deep copy: 1. when explicitly forced 2. In case of Different memory spaces. - // 3.(?) when layouts don't match up - // Even if we can't use a subview directly since data of subview isn't meaningfull accessible in raw format - // with data() (so the subview can't be used by Conduit), but Conduit has it's own methods to access a - // substructures of arrays. See: Conduit Strided Structured Field descriptions. - // more efficient versions without default deep copies should be possible partially relying on shallow copies. - // - // if (!forceHostCopy_m[label] && std::is_same::value) - // HostView_t hostMirrorFinal = HostView_t("hostMirrorNoGhosts",nx,ny,nz); - // -> use data directy if possible eg - // HostView_t hostMirrorFinal = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), fullDeviceView); - // return hostMirrorFinal; - // - // - // (?) create_mirror_and_copy, adapts to spaces! but can convert from LayoutRight to LayoutLeft? - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - }; - - - // Version 2: Mark Ghost Cells in data with VTK meta data. - auto getHostMirrorView_withGhosts = [&]() -> HostView_t - { - using m_t = unsigned char; // Element Type for Field Mask - - /* Define the N-D Host View types we needed for the upcoming section/copying. */ - using DeviceMaskView_t = typename Field::view_type; //Fetch original Field View Type - using HostMaskView1D_t = Kokkos::View; // Host STORAGE - using HostMaskView_t = Kokkos::View< - typename DeviceMaskView_t::data_type, // e.g., unsigned char*** - Kokkos::LayoutLeft, - Kokkos::HostSpace - >; // Host WRAPPER - HostMaskView1D_t hostMaskView1D; // Declare storage view - - - // --- START OF CACHING LOGIC --- - auto it = ghostMaskCache_m.find(ghostKey); - if (it != ghostMaskCache_m.end()) - { - // --- CACHE HIT ---// Re-use the existing ghost view from the cache - catalystInfo_m << level4 <<"::Execute()::ExecVizChannel(" << label << ") | GhostCache HIT" << endl; - hostMaskView1D = it->second; - } - else - { - // --- CACHE MISS --- - catalystInfo_m << level4 <<"::Execute()::ExecVizChannel(" << label << ") | GhostCache MISS" << endl; - - /* Allocate a mask field matching the source field (same mesh/layout/nghost) */ - Field ghostMaskField(Mesh_, Layout_, static_cast(nGhost)); - - /* Fill entire allocation (owned + ghosts) with 1 */ - ghostMaskField = static_cast(1); - DeviceMaskView_t deviceMaskView = ghostMaskField.getView(); - auto interior = ghostMaskField.template getFieldRangePolicy<>(); - - /* Fill inner cells with 0 */ - // TODO(?): use ippl dimension independent iterators - if constexpr (Dim == 1) { - Kokkos::parallel_for("ZeroOwnedMask1D", interior, KOKKOS_LAMBDA(const int i) { - deviceMaskView(i) = static_cast(0); - }); - } else if constexpr (Dim == 2) { - Kokkos::parallel_for("ZeroOwnedMask2D", interior, KOKKOS_LAMBDA(const int i, const int j) { - deviceMaskView(i,j) = static_cast(0); - }); - } else if constexpr (Dim == 3) { - Kokkos::parallel_for("ZeroOwnedMask3D", interior, KOKKOS_LAMBDA(const int i, const int j, const int k) { - deviceMaskView(i,j,k) = static_cast(0); - }); - } - Kokkos::fence(); - - /* Allocate the 1D host view that will own the memory */ - hostMaskView1D = HostMaskView1D_t("hostGhostMask_1D", deviceMaskView.size()); - - // Wrap the flat host allocation in a view with the field's actual rank. This - // gives deep_copy matching source and destination ranks while retaining a single - // flat allocation for Conduit's external array. - HostMaskView_t hostMaskView_N_Rank; - if constexpr (Dim == 1) { - hostMaskView_N_Rank = - HostMaskView_t(hostMaskView1D.data(), deviceMaskView.extent(0)); - } else if constexpr (Dim == 2) { - hostMaskView_N_Rank = HostMaskView_t(hostMaskView1D.data(), - deviceMaskView.extent(0), - deviceMaskView.extent(1)); - } else { - hostMaskView_N_Rank = HostMaskView_t(hostMaskView1D.data(), - deviceMaskView.extent(0), - deviceMaskView.extent(1), - deviceMaskView.extent(2)); - } - - - // The deep_copy now performs: - // - Device-to-Host transfer - // - LayoutRight-to-LayoutLeft - // - Since ND wraps 1D; Copied data is in the hostMaskView1D owned data - Kokkos::deep_copy(hostMaskView_N_Rank, deviceMaskView); - - // Store the 1D view (which owns the memory) in the cache, enough to keep in memory - ghostMaskCache_m[ghostKey] = hostMaskView1D; - // --- END OF CACHING LOGIC --- - } - - + } + void CatalystAdaptor::Initialize(const std::shared_ptr& steerReg) { + if (!visEnabled_m) + return; - // auto ghostMask_field_meta = data["metadata/vtk_fields/GhostMask_field"]; // can't chooses arbitrary name!!!! - auto ghostMask_field_meta = data["metadata/vtk_fields/vtkGhostType"]; - ghostMask_field_meta["attribute_type"] = "Ghosts"; // same as set string??... - // auto ghostMask_field_node = fields["ghostMask_field"]; // can't chooses arbitrary name!!!! must be vtkGhostType - auto ghostMask_field_node = fields["vtkGhostType"]; - ghostMask_field_node["association"].set_string("element"); //associate_m); // vs vertex ... - ghostMask_field_node["topology"].set_string("fmesh_topo"); - ghostMask_field_node["volume_dependent"].set_string("false"); - // ghostMask_field_node["values"].set_external(hostMaskView.data(), hostMaskView.size()); - ghostMask_field_node["values"].set_external(hostMaskView1D.data(), hostMaskView1D.size()); - + if (initialized_m) { + throw IpplException("CatalystAdaptor::Initialize", "Finalize before reinitializing Catalyst."); + } + validateParticleAttributeSelections(); + node_m.reset(); - //////////////////////////////////////////////////////////////////////////////////////////// - // Note: - // Field name in the conduit nodes for data/fields and metadata/vtk_fields has to coincide - // so the meta data can be properly associated with the data. - //////////////////////////////////////////////////////////////////////////////////////////// + catalystInfo_m + << level4 + << "::Initialize() START============================================================= 0" + << endl; - HostView_t hostMirrorFinal = makeHostView("hostMirrorWithGhosts_LayoutLeft"); - copyFieldToHost(hostMirrorFinal, true); + int all_ready = 1; +#if defined(MPI_VERSION) + MPI_Allreduce(MPI_IN_PLACE, &all_ready, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); +#endif + catalystInfo_m << level4 << "::Initialize() Ranks ready for catalyst int: " << all_ready + << " ranks" << endl; + + steerRegistry_m = steerReg; + + const int fcomm = MPI_Comm_c2f(MPI_COMM_WORLD); + const int64_t fcomm64 = static_cast(fcomm); + node_m["catalyst/mpi_comm"].set(fcomm64); + + setNodeScript(node_m["catalyst/scripts/script/filename"], "CATALYST_PIPELINE_PATH", + resourceDir_m / "pipeline_default.py"); + conduit_cpp::Node args = node_m["catalyst/scripts/script/args"]; + + // Pass channel names from registered channels. Mesh blocks nested + // inside a parent multimesh are reported as "/" so + // scripts can locate them within the parent's producer. + args.append().set_string("--channel_names"); + for (const auto& ch : channels_m) { + const std::string parent = ch->basePath(); + args.append().set_string(parent.empty() ? ch->name() : parent + "/" + ch->name()); + } - return hostMirrorFinal; - // --- END FIX FOR MAIN FIELD --- - }; + // Pass channel types so scripts can dispatch without name-pattern matching + args.append().set_string("--channel_types"); + for (const auto& ch : channels_m) { + args.append().set_string(ch->conduitType()); + } - - HostView_t hostMirrorFinal = (useGhostMasks_m) ? getHostMirrorView_withGhosts() : getHostMirrorView_noGhosts(); - /* FOR BOTH CASES FINAL NODE SETTINGS ARE DONE AND WE HAVE THE DATA INSIDE hostMirrorFinal */ - using elem_t = std::remove_pointer_t; - // will return size of vector and amounts of vectors (not size of multiple doubles) - const auto n_elems = hostMirrorFinal.size(); - // Use true element size as stride (handles padding) - static constexpr size_t stride_bytes = sizeof(elem_t); - // offset is zero?? guaranteed? - const size_t offset = 0; - - - field_node["association"].set_string("element"); //associate_m); - field_node["topology"].set_string("fmesh_topo"); - field_node["volume_dependent"].set_string("false"); - if constexpr (std::is_scalar_v) { - // --- SCALAR FIELD CASE --- - field_node["values"].set_external(hostMirrorFinal.data(), n_elems); - } else if constexpr (is_vector_v) { - // --- VECTOR FIELD CASE --- - if (n_elems > 0) { - field_node["values/x"].set_external(&hostMirrorFinal.data()[0][0], n_elems, - offset, stride_bytes); - if constexpr (T::dim >= 2) { - field_node["values/y"].set_external(&hostMirrorFinal.data()[0][1], n_elems, - offset, stride_bytes); - } - if constexpr (T::dim >= 3) { - field_node["values/z"].set_external(&hostMirrorFinal.data()[0][2], n_elems, - offset, stride_bytes); - } - } else { - using component_type = typename T::value_type; - field_node["values/x"].set_external(static_cast(nullptr), 0); - if constexpr (T::dim >= 2) { - field_node["values/y"].set_external(static_cast(nullptr), 0); - } - if constexpr (T::dim >= 3) { - field_node["values/z"].set_external(static_cast(nullptr), 0); - } - } - } - // else { - // --- INVALID CASE --- - // } + // Ghost layers included in each channel's export (0 when stripped) + args.append().set_string("--channel_ghosts"); + for (const auto& ch : channels_m) { + args.append().set_string(std::to_string(ch->ghostLayers())); + } - /* save view so data isn't discarded */ - viewRegistry_m.set(label, hostMirrorFinal); -} + args.append().set_string("--verbosity"); + args.append().set_string(std::to_string(catalystInfo_m.getOutputLevel())); + args.append().set_string("--VTKextract"); + args.append().set_string(catalystVtk_m); + args.append().set_string("--png"); + args.append().set_string(catalystPng_m); -// == PARTICLECONTAINERS derived from ippl::ParticleBase,...>,...> -template -requires (std::derived_from, ParticleBaseBase>) -void CatalystAdaptor::ExecVizChannel(const T& entry, const std::string label) -{ - const bool refreshOnly = viewRegistry_m.contains(label); - if (refreshOnly && !forceHostCopy_m[label]) { - return; - } + // Directory for extracts written by the pipeline script + args.append().set_string("--extracts_output_dir"); + args.append().set_string((outputDir_m / "vtk_extracts").string()); - catalystInfo_m << level4 << "::Execute()::ExecVizChannel(" << label << ") | Type : ParticleBase).name() - << "," - << particle_dim_v - << ",...>...> [or subclass])" << endl; - const std::string channelName = "ippl_particles_" + label; - - auto particleContainer = &entry; - const size_t localNum = particleContainer->getLocalNum(); - assert((localNum == 0 || particleContainer->R.getView().data() != nullptr) - && "A non-empty R view should not be nullptr"); - - const std::string blockName = "block_allRanks"; - // const std::string blockName = "block_rank" + std::to_string(ippl::Comm->rank()); - - // channel for this particleContainer - // channel of type mesh adheres to conduits mesh blueprint - - - auto channel = node_m["catalyst/channels/"+ channelName]; - - - - channel["type"].set_string("multimesh"); - - - auto data = channel["data/block_main"]; - auto data_help = channel["data/block_help"]; - channel["assembly/main"] = "block_main"; - channel["assembly/help"] = "block_help"; - - //////////////////////////////////////////////////////// - // Note: - // Multimesh currently seems to have caused more headaches compared to what - // we have seemed to have gained from using it. So if the bugs or inconveniences stay - // in upcoming updates for ParaView catalyst we might want to use two normal meshes - // instead. - // channel["type"].set_string("mesh"); - //////////////////////////////////////////////////////// - - auto fields = data["fields"]; - data["type"].set_string("mesh"); - - const int rank = ippl::Comm->rank(); - - using IotaView_t = Kokkos::View; - using RankView_t = Kokkos::View; - IotaView_t iota_view("iota", localNum); - RankView_t rank_id_view("rank_id_view", localNum); - if (localNum > 0) { - using HostExecSpace = Kokkos::DefaultHostExecutionSpace; - Kokkos::RangePolicy host_policy(0, localNum); - Kokkos::parallel_for("fill_iota_host", host_policy, KOKKOS_LAMBDA(const int64_t i) { - iota_view(i) = i; - }); - Kokkos::parallel_for("fill_rank_ids", host_policy, KOKKOS_LAMBDA(const int64_t i) { - rank_id_view(i) = rank; - }); + if (experimentName_m) { + args.append().set_string("--experiment_name"); + args.append().set_string(*experimentName_m); } - viewRegistry_m.set(label + "_iota", iota_view); - viewRegistry_m.set(label + "_rank_id", rank_id_view); - - // Creates a host-accessible mirror view and copies the data from the device view to the host. - using RAttrib_t = std::remove_reference_tR)>; - using hostMirror_R_t = typename RAttrib_t::host_mirror_type; - hostMirror_R_t R_hostMirror; - - if (forceHostCopy_m[label]) { - R_hostMirror = particleContainer->R.getHostMirror(); - Kokkos::deep_copy(R_hostMirror, particleContainer->R.getView()); - viewRegistry_m.set(R_hostMirror); - } else { - R_hostMirror = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), particleContainer->R.getView()); - viewRegistry_m.set(R_hostMirror); + args.append().set_string("--live"); + args.append().set_string(catalystLive_m); + + args.append().set_string("--steer"); + args.append().set_string(catalystSteer_m); + + args.append().set_string("--steer_channel_names"); + + // Proxy writer (unchanged from old path) + std::filesystem::path proxyPath = outputDir_m / "catalyst_proxy.xml"; + bool useExistingProxy = false; + if (const char* proxyPathEnv = std::getenv("IPPL_CATALYST_PROXY_PATH"); + proxyPathEnv && *proxyPathEnv) { + proxyPath = proxyPathEnv; + useExistingProxy = true; + } else if (const char* legacyProxyPathEnv = std::getenv("CATALYST_PROXYS_PATH"); + legacyProxyPathEnv && *legacyProxyPathEnv) { + proxyPath = legacyProxyPathEnv; + useExistingProxy = true; + catalystWarn_m << "CATALYST_PROXYS_PATH is deprecated; use " + "IPPL_CATALYST_PROXY_PATH instead." + << endl; } - using hostMirror_ID_t = typename std::remove_reference_tID)>::host_mirror_type; - hostMirror_ID_t ID_hostMirror; - if constexpr (T::EnableIDs) { - if (forceHostCopy_m[label]) { - ID_hostMirror = particleContainer->ID.getHostMirror(); - Kokkos::deep_copy(ID_hostMirror, particleContainer->ID.getView()); - viewRegistry_m.set(label, ID_hostMirror); + std::string cfgYaml; + if (const char* cfg_env = std::getenv("IPPL_PROXY_CONFIG_YAML")) { + if (std::filesystem::exists(cfg_env)) { + cfgYaml = std::string(cfg_env); } else { - ID_hostMirror = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), particleContainer->ID.getView()); - viewRegistry_m.set(label, ID_hostMirror); + catalystInfo_m << level4 + << "::Initialize() IPPL_PROXY_CONFIG_YAML set but file not found: '" + << cfg_env << "', using default." << endl; } } - - using PLayout_t = T::Layout_t; - // using vector_t = T::Layout_t::vector_type; - // using value_t = T::Layout_t::value_type; - using R_elem_t = std::remove_pointer_t; //avoids padding etc (?) Rattrib_t - static constexpr size_t R_stride_bytes = sizeof(R_elem_t); - - /* checks if playput it spatial or pure layout */ - if constexpr (has_getRegionLayout_v){ - - using RLayout_t = PLayout_t::RegionLayout_t; - using NDRegion_t = RLayout_t::NDRegion_t; - constexpr unsigned dim_ = PLayout_t::dim; - const NDRegion_t ndr = particleContainer->getLayout().getRegionLayout().getDomain(); - - /* HELPER COORDINATES TO PASS THE BOUNDING BOX in vtk format*/ - /* HELPER TOPOLOGY TO PASS THE BOUNDING BOX (??even needed??) in vtk format */ - data_help["coordsets/bound_helper_coords/type"].set_string("uniform"); - data_help["topologies/bound_helper_topo/coordset"].set_string("bound_helper_coords"); - data_help["topologies/bound_helper_topo/type"].set_string("uniform"); - /* create unfirom coordinate mesh only consisting of the corner points of the domain */ - { - data_help["coordsets/bound_helper_coords/dims/i"].set(2); - data_help["coordsets/bound_helper_coords/spacing/dx"].set( ndr[0].max() - ndr[0].min() ); - data_help["coordsets/bound_helper_coords/origin/x"].set( ndr[0].min() ); - // data_help["topologies/bound_helper_topo/origin/x"].set( ndr[0].min() ); - } - if constexpr(dim_ >= 2){ - data_help["coordsets/bound_helper_coords/dims/j"].set(2); - data_help["coordsets/bound_helper_coords/spacing/dy"].set( ndr[1].max()- ndr[1].min() ); - data_help["coordsets/bound_helper_coords/origin/y"].set( ndr[1].min() ); - // data_help["topologies/bound_helper_topo/origin/y"].set( ndr[1].min() ); - } - if constexpr(dim_ >= 3){ - data_help["coordsets/bound_helper_coords/dims/k"].set(2); - data_help["coordsets/bound_helper_coords/spacing/dz"].set( ndr[2].max()- ndr[2].min() ); - data_help["coordsets/bound_helper_coords/origin/z"].set( ndr[2].min() ); - // data_help["topologies/bound_helper_topo/origin/z"].set( ndr[2].min() ); - } - } - // else { - /* will use raw particle data instead .. */ - // } - - - /* ATTRIBUTES HARDCODED IN PARTICELBASE are identity ID and position R */ - /* EXPLICIT COORDINATES -> EACH PARTICLE'S POSITION */ - - ///////////////////////////////////////////////////////////// - // Note: - // For debuggng purposes checking distribution of particles on ranks... - // #if defined(MPI_VERSION) - // // MPI_Barrier(MPI_COMM_WORLD); - // // if(ippl::Comm->rank()==0) catalystInfo_m << level4 <<"[Rank 0] Local Particles: " << localNum << endl; - // MPI_Barrier(MPI_COMM_WORLD); - // catalystWarn_m << "Local Particles: " << localNum << endl; - // MPI_Barrier(MPI_COMM_WORLD); - // #endif - ///////////////////////////////////////////////////////////// - - data["coordsets/p_explicit_coords/type"].set_string("explicit"); - /* unstructured topology relying on per rank unique particle ID */ - data["topologies/p_unstructured_topo/coordset"].set_string("p_explicit_coords"); - data["topologies/p_unstructured_topo/type"].set_string("unstructured"); - data["topologies/p_unstructured_topo/elements/shape"].set_string("point"); - data["topologies/p_unstructured_topo/elements/connectivity"].set_external(iota_view.data(),particleContainer->getLocalNum()); - - // left hardcodeed we already have the hostViews (instead of integrating the into the loop) - - /* Process ID ATTRIBUTE */ - auto rank_field = fields["RankID"]; - rank_field["association"].set_string("vertex"); - rank_field["topology"].set_string("p_unstructured_topo"); - rank_field["volume_dependent"].set_string("false"); - rank_field["values"].set_external(rank_id_view.data(), localNum); - data["metadata/vtk_fields/RankID/attribute_type"].set_string("ProcessIds"); - - /* Global ID ATTRIBUTE (only when particle IDs are enabled in ParticleBase) */ - if constexpr (T::EnableIDs) { - auto id_field = fields["ParticleIDs"]; - id_field["association"].set_string("vertex"); - id_field["topology"].set_string("p_unstructured_topo"); - id_field["volume_dependent"].set_string("false"); - if (localNum > 0) { - id_field["values"].set_external(ID_hostMirror.data(), localNum); - } else { - using id_value_t = typename hostMirror_ID_t::value_type; - id_field["values"].set_external(static_cast(nullptr), 0); - } - data["metadata/vtk_fields/ParticleIDs/attribute_type"].set_string("GlobalIds"); - } - - /* POSITION ATTRIBUTE */ - auto R_field = fields["position"]; - R_field["association"].set_string("vertex"); - R_field["topology"].set_string("p_unstructured_topo"); - R_field["volume_dependent"].set_string("false"); - - - constexpr unsigned ParticleDim = particle_dim_v; - static_assert(ParticleDim >= 1 && ParticleDim <= 3, - "Catalyst visualization supports particles with one to three dimensions"); - - if (localNum > 0) - { - /* COORDINATE DEFINITION... */ - data["coordsets/p_explicit_coords/values/x"].set_external(&R_hostMirror.data()[0][0], particleContainer->getLocalNum(), 0, R_stride_bytes); - if constexpr (ParticleDim >= 2) { - data["coordsets/p_explicit_coords/values/y"].set_external(&R_hostMirror.data()[0][1], particleContainer->getLocalNum(), 0, R_stride_bytes); - } - if constexpr (ParticleDim >= 3) { - data["coordsets/p_explicit_coords/values/z"].set_external(&R_hostMirror.data()[0][2], particleContainer->getLocalNum(), 0, R_stride_bytes); - } - - /* POSITION ATTRIBUTE */ - R_field["values/x"].set_external(&R_hostMirror.data()[0][0], particleContainer->getLocalNum(), 0, R_stride_bytes); - if constexpr (ParticleDim >= 2) { - R_field["values/y"].set_external(&R_hostMirror.data()[0][1], particleContainer->getLocalNum(), 0, R_stride_bytes); - } - if constexpr (ParticleDim >= 3) { - R_field["values/z"].set_external(&R_hostMirror.data()[0][2], particleContainer->getLocalNum(), 0, R_stride_bytes); - } - - /* concept for no copy in situ vis would be */ - //mesh["topologies/p_unstructured_topo/elements/connectivity"].set_external(particleContainer->ID.getView().data(),particleContainer->getLocalNum()); - }else - { - // In case a rank has no particles-> data()[0] is nulllptr dereferencing !!!!! - using component_type = typename R_elem_t::value_type; - data["coordsets/p_explicit_coords/values/x"].set_external(static_cast(nullptr), 0); - R_field["values/x"].set_external(static_cast(nullptr), 0); - if constexpr (ParticleDim >= 2) { - data["coordsets/p_explicit_coords/values/y"].set_external(static_cast(nullptr), 0); - R_field["values/y"].set_external(static_cast(nullptr), 0); - } - if constexpr (ParticleDim >= 3) { - data["coordsets/p_explicit_coords/values/z"].set_external(static_cast(nullptr), 0); - R_field["values/z"].set_external(static_cast(nullptr), 0); + if (cfgYaml.empty()) { + auto default_cfgYaml = (resourceDir_m / "proxy_default_config.yaml").string(); + if (std::filesystem::exists(default_cfgYaml)) { + cfgYaml = std::move(default_cfgYaml); } } - // Attribute containers are grouped by memory space, so their global ordinal is not - // stable. Identify ParticleBase's built-ins by object identity instead. - entry.forAllAttributes([&](const Attributes& atts) { - for (auto* attribute : atts) { - if (!entry.isBuiltinAttribute(attribute)) { - attribute->signConduitBlueprintNode(localNum, fields, viewRegistry_m, - catalystInfo_m, catalystWarn_m, - forceHostCopy_m[label]); - } + proxyWriter_m.initialize(proxyPath, cfgYaml); + if (steerEnabled_m) { + SteerInitVisitor steerInitV{*this}; + steerRegistry_m->forEach(steerInitV); + } + const bool generateProxy = !useExistingProxy && proxyOption_m != "OFF"; + int proxyReady = 1; + + if (ippl::Comm->rank() == 0) { + if (useExistingProxy) { + proxyReady = std::filesystem::is_regular_file(proxyPath) ? 1 : 0; + } else if (generateProxy) { + proxyReady = proxyWriter_m.produceUnified("SteerableParameters_SCALARS", + "SteerableParameters") + ? 1 + : 0; + } else { + proxyReady = std::filesystem::is_regular_file(proxyPath) ? 1 : 0; } - }); - //////////////////////////////////////////////////////////////////////////////////////////// - // Note: - // All ways on how to iterate over particle attributes rely on base class pointers. - // In whih case dimensions and types particle attributes is not retrievable from the - // a pointer instance. Therefore conduit maniupulation are done as a membermethod - // for paricleAttrib (overriding a virtual method in the base class). - // - // entry.template forAllAttributes( - // [&](const Attributes& atts) { - // for (auto* attribute : atts) { - //////////////////////////////////////////////////////////////////////////////////////////// - -} - - -// BASE CASE: only enabled if EntryT is NOT derived from ippl::ParticleBaseBase -template -requires (!std::derived_from, ParticleBaseBase>) - void CatalystAdaptor::ExecVizChannel( [[maybe_unused]] T&& entry, const std::string label) -{ - catalystInfo_m << level4 <<" Entry type can't be processed: ID "<< label <<" "<< typeid(std::decay_t).name() << endl; -} - - -/* SHARED_PTR DISPATCHER - automatically unwraps and dispatches to appropriate overload */ -template - void CatalystAdaptor::ExecVizChannel( const std::shared_ptr& entry,const std::string label ) -{ - if (entry) { - catalystInfo_m << level4 <<" dereferencing shared pointer and reattempting execute..." << endl; - ExecVizChannel(*entry, label ); - } else { - catalystInfo_m << level4 <<" Null shared_ptr encountered" << endl; } -} +#if defined(MPI_VERSION) + MPI_Bcast(&proxyReady, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); +#endif -} - - -// ===================================================================================== -// STEERING: -// ===================================================================================== -#include "Stream/InSitu/CatalystAdaptorSteering.hpp" -// ===================================================================================== - -namespace ippl{ -/////////////////////////////////////////////////////////////// -// Note: -// this does not really need to be a separate function call, -// May we should just inline this into the execute functions ... -/////////////////////////////////////////////////////////////// -void CatalystAdaptor::fetchResults() { - - catalyst_status err = catalyst_results(conduit_cpp::c_node(&results_m)); - if (err != catalyst_status_ok) - { - std::cerr << "Failed to execute Catalyst-results: " << err << std::endl; + if ((useExistingProxy || generateProxy) && !proxyReady) { + throw IpplException( + "Stream::InSitu::CatalystAdaptor::Initialize()", + "Could not create or read Catalyst proxy XML: " + proxyPath.string()); } - } - - -// ===================================================================================== -// Runtime registry based Initialize / Execute (non-templated registry) -// ===================================================================================== - -void CatalystAdaptor::Initialize( - const std::shared_ptr& visReg, - const std::shared_ptr& steerReg - ) { -if ( !visEnabled_m) return; + if (proxyReady) { + node_m["catalyst/proxies/proxy_/filename"].set(proxyPath.string()); + } else if (steerEnabled_m) { + catalystWarn_m << "Steering is enabled without a Catalyst proxy XML because " + "IPPL_CATALYST_PROXY_OPTION=OFF and no existing proxy was found at " + << proxyPath.string() << endl; + } - catalystInfo_m << level4 <<"::Initialize() START============================================================= 0" << endl; + if (proxyOption_m == "PRODUCE_ONLY") { + throw IpplException("Stream::InSitu::CatalystAdaptor", + "write_proxy_only_run: proxy available at " + proxyPath.string()); + } - int all_ready = 1; - #if defined(MPI_VERSION) - MPI_Allreduce(MPI_IN_PLACE, &all_ready, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - #endif - catalystInfo_m << level4 <<"::InitializeRuntime() Ranks ready for catalyst int: " << all_ready << " ranks" << endl; - - visRegistry_m = visReg; - steerRegistry_m = steerReg; - - - const int fcomm = MPI_Comm_c2f(MPI_COMM_WORLD); - const int64_t fcomm64 = static_cast(fcomm); - node_m["catalyst/mpi_comm"].set(fcomm64); - - - setNodeScript( node_m["catalyst/scripts/script/filename"], //where in node_m - "CATALYST_PIPELINE_PATH", // environment override - resourceDir_m / "pipeline_default.py") //default - ; - conduit_cpp::Node args = node_m["catalyst/scripts/script/args"]; - - - args.append().set_string("--channel_names"); - InitVisitor initV{*this}; - visRegistry_m->forEach(initV); - // Visitor will (also) append channel names here into the node_m (sequence of overall arguments is important!!) - - args.append().set_string("--verbosity"); - args.append().set_string(std::to_string(catalystInfo_m.getOutputLevel())); - - - args.append().set_string("--VTKextract"); - args.append().set_string(catalystVtk_m); - - args.append().set_string("--live"); - args.append().set_string(catalystLive_m); - - args.append().set_string("--steer"); - args.append().set_string(catalystSteer_m); - - - args.append().set_string("--steer_channel_names"); - - std::filesystem::path proxyPath = outputDir_m / "catalyst_proxy.xml"; - bool useExistingProxy = false; - if (const char* proxyPathEnv = std::getenv("IPPL_CATALYST_PROXY_PATH"); - proxyPathEnv && *proxyPathEnv) { - proxyPath = proxyPathEnv; - useExistingProxy = true; - } else if (const char* legacyProxyPathEnv = std::getenv("CATALYST_PROXYS_PATH"); - legacyProxyPathEnv && *legacyProxyPathEnv) { - proxyPath = legacyProxyPathEnv; - useExistingProxy = true; - catalystWarn_m << "CATALYST_PROXYS_PATH is deprecated; use " - "IPPL_CATALYST_PROXY_PATH instead." + catalystInfo_m << level4 + << "::Initialize() Printing Conduit `node_m` instance passed to " + "catalyst_initialize() =>" << endl; - } + catalystInfo_m << level4 << node_m.to_yaml() << endl; - std::string cfgYaml; - if (const char* cfg_env = std::getenv("IPPL_PROXY_CONFIG_YAML")) { - if (std::filesystem::exists(cfg_env)) { - cfgYaml = std::string(cfg_env); + catalyst_status err = catalyst_initialize(conduit_cpp::c_node(&node_m)); + if (err != catalyst_status_ok) { + catalystInfo_m << level4 << "::Initialize() Catalyst initialization failed." << endl; + throw IpplException("Stream::InSitu::CatalystAdaptor::Initialize()", + "Failed to initialize Catalyst!!!"); } else { - catalystInfo_m << level4 <<"::Initialize() IPPL_PROXY_CONFIG_YAML set but file not found: '" << cfg_env << "', using default." << endl; + catalystInfo_m << level4 << "::Initialize() Catalyst initialized successfully." + << endl; } - } - if (cfgYaml.empty()) { - auto default_cfgYaml = (resourceDir_m / "proxy_default_config.yaml").string(); - if (std::filesystem::exists(default_cfgYaml)) { - cfgYaml = std::move(default_cfgYaml); - } // else leave empty -> ProxyWriter can proceed without config - } + // Reset node_m (scripts are consumed by catalyst_initialize). + // Channel tree will be built on first Execute() and reused thereafter. + node_m.reset(); + channelsBuilt_m = false; + initialized_m = true; - proxyWriter_m.initialize(proxyPath, cfgYaml); - if (steerEnabled_m ) { - SteerInitVisitor steerInitV{*this}; - steerRegistry_m->forEach(steerInitV); - } - const bool generateProxy = !useExistingProxy && proxyOption_m != "OFF"; - int proxyReady = 1; - - if (ippl::Comm->rank() == 0) { - if (useExistingProxy) { - proxyReady = std::filesystem::is_regular_file(proxyPath) ? 1 : 0; - } else if (generateProxy) { - proxyReady = proxyWriter_m.produceUnified( - "SteerableParameters_SCALARS", "SteerableParameters") - ? 1 - : 0; - } else { - proxyReady = std::filesystem::is_regular_file(proxyPath) ? 1 : 0; - } + catalystInfo_m + << level4 + << "::Initialize() DONE============================================================= 1" + << endl; } + void CatalystAdaptor::Execute(int cycle, double time, + int rank /* default = ippl::Comm->rank() */) { + if (!visEnabled_m) + return; + + catalystInfo_m + << level4 + << "::Execute() START =============================================================== 0" + << endl; + + static IpplTimings::TimerRef TMRcatalyst_execute = + IpplTimings::getTimer("catalyst_execute"); + static IpplTimings::TimerRef TMRchannelExecute = IpplTimings::getTimer("channelExecute"); + static IpplTimings::TimerRef TMRexecSteerVisitor = + IpplTimings::getTimer("execSteerVisitor"); + + // Build channel tree on first Execute (after catalyst_initialize + // consumed the scripts node). + if (!channelsBuilt_m) { + for (auto& ch : channels_m) { + ch->init(node_m, rank); + } + channelsBuilt_m = true; + if (cycle == 0) { #if defined(MPI_VERSION) - MPI_Bcast(&proxyReady, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(MPI_COMM_WORLD); + catalystInfo_m << level4 + << "::Execute() [rank = 0] Printing first Conduit Node ==>" << endl; + if (catalystInfo_m.getOutputLevel() >= 4 && ippl::Comm->rank() == 0) + node_m.print(); + MPI_Barrier(MPI_COMM_WORLD); #endif + } + } - if ((useExistingProxy || generateProxy) && !proxyReady) { - throw IpplException( - "Stream::InSitu::CatalystAdaptor::Initialize()", - "Could not create or read Catalyst proxy XML: " + proxyPath.string()); - } + // Update state + auto state = node_m["catalyst/state"]; + state["cycle"].set(cycle); + state["time"].set(time); + state["domain_id"].set(rank); - if (proxyReady) { - node_m["catalyst/proxies/proxy_/filename"].set(proxyPath.string()); - } else if (steerEnabled_m) { - catalystWarn_m << "Steering is enabled without a Catalyst proxy XML because " - "IPPL_CATALYST_PROXY_OPTION=OFF and no existing proxy was found at " - << proxyPath.string() << endl; - } - - if (proxyOption_m == "PRODUCE_ONLY") { - throw IpplException( - "Stream::InSitu::CatalystAdaptor", - "write_proxy_only_run: proxy available at " + proxyPath.string()); - } + // Execute channels (deep-copy dynamic data into persistent mirrors) + IpplTimings::startTimer(TMRchannelExecute); + for (auto& ch : channels_m) { + ch->execute(node_m, cycle, time, rank); + } + IpplTimings::stopTimer(TMRchannelExecute); - catalystInfo_m << level4 <<"::Initialize() Printing Conduit `node_m` instance passed to catalyst_initialize() =>" << endl; - catalystInfo_m << level4 <forEach(steerV); + } + IpplTimings::stopTimer(TMRexecSteerVisitor); + // catalyst_execute + catalystInfo_m << level4 << "::Execute()::catalyst_execute() ==>" << endl; + IpplTimings::startTimer(TMRcatalyst_execute); + catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node_m)); + IpplTimings::stopTimer(TMRcatalyst_execute); + if (err != catalyst_status_ok) { + std::cerr << "::Execute() Failed to execute Catalyst (runtime path): " << err + << std::endl; + } + // Steering fetch + if (steerEnabled_m) { + static IpplTimings::TimerRef TMRfetchResult = + IpplTimings::getTimer("fetchSteerParameters"); + IpplTimings::startTimer(TMRfetchResult); -void CatalystAdaptor::rememberNow(const std::string label){ -if ( !visEnabled_m) return; + fetchResults(); + SteerFetchVisitor fetchV{*this}; + steerRegistry_m->forEach(fetchV); - // Validate inputs and state - auto it = forceHostCopy_m.find(label); - if (it == forceHostCopy_m.end()){ - throw IpplException("Stream::InSitu::CatalystAdaptor::rememberNow", "Label not present in Visualisation Registry: " + label); - } - if (!visRegistry_m) { - throw IpplException("Stream::InSitu::CatalystAdaptor::rememberNow", "Visualization registry is not initialized (nullptr)"); - } + IpplTimings::stopTimer(TMRfetchResult); + } - // Temporarily force a host copy for this label during an execute - bool tmp = it->second; - forceHostCopy_m[label] = true; - ExecVisitor execV{*this}; - const bool ok = visRegistry_m->forOne(label, execV); + // NO viewRegistry_m.clear(), NO node_m.reset(). + // The Conduit tree and all staging buffers persist across steps. - // Restore prior state - forceHostCopy_m[label] = tmp; - if (!ok) { - throw IpplException("Stream::InSitu::CatalystAdaptor::rememberNow", "Label not found in executable entries or has no execute callback: " + label); + catalystInfo_m + << level4 + << "::Execute() DONE =============================================================== 1" + << endl; } -} - -void CatalystAdaptor::Execute( int cycle, double time, int rank /* default = ippl::Comm->rank() */) { - if ( !visEnabled_m) return; - - catalystInfo_m << level4 <<"::Execute() START =============================================================== 0" << endl; - - static IpplTimings::TimerRef TMRcatalyst_execute = IpplTimings::getTimer("catalyst_execute"); - static IpplTimings::TimerRef TMRexecVizVisitor = IpplTimings::getTimer("execVizVisitor"); - static IpplTimings::TimerRef TMRexecSteerVisitor = IpplTimings::getTimer("execSteerVisitor"); - - - auto state = node_m["catalyst/state"]; - state["cycle"].set(cycle); - state["time"].set(time); - state["domain_id"].set(rank); + void CatalystAdaptor::Finalize() { + if (!visEnabled_m) + return; - IpplTimings::startTimer(TMRexecVizVisitor); - if ( !!visEnabled_m){ - // edit forward Node: add visualisation channels - ExecVisitor execV{*this}; - visRegistry_m->forEach(execV); + conduit_cpp::Node node; + catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node_m)); + if (err != catalyst_status_ok) { + std::cerr << "Failed to finalize Catalyst: " << err << std::endl; + } + initialized_m = false; + channelsBuilt_m = false; + node_m.reset(); + viewRegistry_m.clear(); } - IpplTimings::stopTimer(TMRexecVizVisitor); - - IpplTimings::startTimer(TMRexecSteerVisitor); - if (steerEnabled_m) { - // edit forward Node: add steering channels - SteerForwardVisitor steerV{*this}; - steerRegistry_m->forEach(steerV); + // ===================================================================================== + // New channel-based API + // ===================================================================================== + + template + MeshChannelHandle CatalystAdaptor::addMeshChannel(const std::string& name, + const UniformCartesian& mesh, + const FieldLayout& layout, int nghost, + bool useGhostMasks) { + auto ch = std::make_shared>(name, mesh, layout, nghost, useGhostMasks); + channels_m.push_back(ch); + catalystInfo_m << level4 << "::addMeshChannel('" << name << "', Dim=" << Dim + << ", nghost=" << nghost << ")" << endl; + return MeshChannelHandle(std::static_pointer_cast(ch), Dim); } - IpplTimings::stopTimer(TMRexecSteerVisitor); + template + MeshChannelHandle CatalystAdaptor::addMeshBlock(const std::string& parentName, + const std::string& blockName, + const UniformCartesian& mesh, + const FieldLayout& layout, int nghost, + bool useGhostMasks, + GeometryPolicy geometryPolicy) { + auto ch = std::make_shared>(blockName, mesh, layout, nghost, + useGhostMasks, geometryPolicy, parentName); + channels_m.push_back(ch); + catalystInfo_m << level4 << "::addMeshBlock(parent='" << parentName << "', block='" + << blockName << "', Dim=" << Dim << ", nghost=" << nghost << ", geometry=" + << (geometryPolicy == GeometryPolicy::Static ? "Static" : "Dynamic") << ")" + << endl; + return MeshChannelHandle(std::static_pointer_cast(ch), Dim); + } - if(cycle == 0){ - - #if defined(MPI_VERSION) - MPI_Barrier(MPI_COMM_WORLD); - catalystInfo_m << level4 <<"::Execute() [rank = 0] Printing first Conduit Node passed from to catalyst_execute() ==>" << endl; - if(catalystInfo_m.getOutputLevel() >= 4 && ippl::Comm->rank()==0) node_m.print(); - catalystInfo_m << level4 <<"::Execute() [rank = 1] Printing first Conduit Node passed from to catalyst_execute() ==>" << endl; - MPI_Barrier(MPI_COMM_WORLD); - if(catalystInfo_m.getOutputLevel() >= 4 && ippl::Comm->rank()==1) node_m.print(); - MPI_Barrier(MPI_COMM_WORLD); - #endif - // if(level >= 5 && ippl::Comm->rank()==0) node_m.print(); + template + requires std::derived_from, ParticleBaseBase> + ParticleChannelHandle CatalystAdaptor::addParticleChannel(const std::string& name, + const T& pc) { + if (initialized_m) { + throw IpplException("CatalystAdaptor::addParticleChannel", + "Register channels before Initialize()."); + } + if (name.empty() || particleAttributeSources_m.contains(name)) { + throw IpplException("CatalystAdaptor::addParticleChannel", + "Empty or duplicate particle channel name: " + name); + } + particleAttributeSources_m.emplace(name, [&pc] { + std::vector names; + pc.forAllAttributes([&](const Attributes& attributes) { + for (const auto* attribute : attributes) { + if (!pc.isBuiltinAttribute(attribute)) { + names.push_back(attribute->get_name()); + } + } + }); + std::sort(names.begin(), names.end()); + return names; + }); + auto ch = std::make_shared( + name, const_cast(static_cast(&pc))); + auto* raw = ch.get(); + + // Type-specific init function: builds multimesh, allocates mirrors, + // sets Conduit external pointers on first Execute and after count changes. + raw->setInitFn([this, name, &pc, raw](conduit_cpp::Node& channel, int rank, Inform& info) { + const size_t localNum = pc.getLocalNum(); + const auto& selection = getParticleAttributeSelection(name); + channel["type"].set_string("multimesh"); + channel["assembly/particles"] = "particles"; + channel["assembly/boundingbox"] = "boundingbox"; + channel["assembly/transform"] = "transform"; + + auto data = channel["data/particles"]; + auto data_help = channel["data/boundingbox"]; + data["type"].set_string("mesh"); + + // Persistent iota + rank_id + auto& iota = const_cast&>(raw->iota_m); + auto& rankId = const_cast&>(raw->rankId_m); + if (iota.extent(0) != localNum) { + Kokkos::resize(iota, localNum); + } + if (selection.includesRankIds() && rankId.extent(0) != localNum) { + Kokkos::resize(rankId, localNum); + } + for (size_t i = 0; i < localNum; ++i) { + iota(i) = static_cast(i); + if (selection.includesRankIds()) { + rankId(i) = rank; + } + } - catalystInfo_m << level4 << "::Execute() During first catalyst_execute() catalyst will\n" - << " for each passed script - in order how they were \n" - << " passed to the conduit node - run the globa scope,\n" - << " the initialize() and the execute()." << endl; - } + // Coordset + topology + data["coordsets/p_explicit_coords/type"].set_string("explicit"); + data["topologies/p_unstructured_topo/coordset"].set_string("p_explicit_coords"); + data["topologies/p_unstructured_topo/type"].set_string("unstructured"); + data["topologies/p_unstructured_topo/elements/shape"].set_string("point"); + data["topologies/p_unstructured_topo/elements/connectivity"].set_external(iota.data(), + localNum); + + // R host mirror (persistent) + using RAttrib_t = std::remove_reference_t; + using RHost_t = typename RAttrib_t::host_mirror_type; + RHost_t RHost; + if (auto* previous = viewRegistry_m.find(name + "_R")) { + RHost = *previous; + } + if (RHost.extent(0) != localNum) { + Kokkos::resize(RHost, localNum); + } + Kokkos::deep_copy(RHost, pc.R.getView()); + viewRegistry_m.set(name + "_R", RHost); + + constexpr unsigned ParticleDim = particle_dim_v; + static constexpr size_t R_stride = + sizeof(std::remove_pointer_t); + + if (localNum > 0) { + data["coordsets/p_explicit_coords/values/x"].set_external(&RHost.data()[0][0], + localNum, 0, R_stride); + if constexpr (ParticleDim >= 2) { + data["coordsets/p_explicit_coords/values/y"].set_external( + &RHost.data()[0][1], localNum, 0, R_stride); + } + if constexpr (ParticleDim >= 3) { + data["coordsets/p_explicit_coords/values/z"].set_external( + &RHost.data()[0][2], localNum, 0, R_stride); + } + } + if (localNum == 0) { + using Component = typename RAttrib_t::value_type::value_type; + const char* axes[] = {"x", "y", "z"}; + for (unsigned d = 0; d < ParticleDim && d < 3; ++d) { + data[std::string("coordsets/p_explicit_coords/values/") + axes[d]] + .set_external(static_cast(nullptr), 0); + } + } - //////////////////////////////////////////////////////////////// - // Note: - // Possibly helpful for further debugging. - // - // Kokkos::fence(); - // #if defined(MPI_VERSION) - // MPI_Barrier(MPI_COMM_WORLD); - // #endif - // - // int all_ready = 1; - // #if defined(MPI_VERSION) - // MPI_Allreduce(MPI_IN_PLACE, &all_ready, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - // #endif - // catalystInfo_m << level4 <<"::Execute() All ranks ready for catalyst_execute: " << all_ready << " ranks" << endl; - //////////////////////////////////////////////////////////////// - - catalystInfo_m << level4 <<"::Execute()::catalyst_execute() ==>" << endl; - IpplTimings::startTimer(TMRcatalyst_execute); - catalyst_status err = catalyst_execute(conduit_cpp::c_node(&node_m)); - IpplTimings::stopTimer(TMRcatalyst_execute); + // RankID field + auto fields = data["fields"]; + if (selection.includesRankIds()) { + auto rankField = fields["RankID"]; + rankField["association"].set_string("vertex"); + rankField["topology"].set_string("p_unstructured_topo"); + rankField["volume_dependent"].set_string("false"); + if (localNum > 0) { + rankField["values"].set_external(rankId.data(), localNum); + } else { + rankField["values"].set_external(static_cast(nullptr), 0); + } + data["metadata/vtk_fields/RankID/attribute_type"].set_string("ProcessIds"); + } - //////////////////////////////////////////////////////////////////////////////// - // Note: - // catalyst execute seems to be the current bottleneck of a medium sized simulation... - //////////////////////////////////////////////////////////////////////////////// + // ID field (if enabled) + if constexpr (T::EnableIDs) { + if (selection.includesParticleIds()) { + using IDAttrib_t = std::remove_reference_t; + using IDHost_t = typename IDAttrib_t::host_mirror_type; + IDHost_t IDHost; + if (auto* previous = viewRegistry_m.find(name + "_ID")) { + IDHost = *previous; + } + if (IDHost.extent(0) != localNum) { + Kokkos::resize(IDHost, localNum); + } + Kokkos::deep_copy(IDHost, pc.ID.getView()); + viewRegistry_m.set(name + "_ID", IDHost); + + auto idField = fields["ParticleIDs"]; + idField["association"].set_string("vertex"); + idField["topology"].set_string("p_unstructured_topo"); + idField["volume_dependent"].set_string("false"); + if (localNum > 0) { + idField["values"].set_external(IDHost.data(), localNum); + } else { + using id_value_t = typename IDHost_t::value_type; + idField["values"].set_external(static_cast(nullptr), 0); + } + data["metadata/vtk_fields/ParticleIDs/attribute_type"].set_string("GlobalIds"); + } + } + // Position field (references RHost) + auto R_field = fields["position"]; + R_field["association"].set_string("vertex"); + R_field["topology"].set_string("p_unstructured_topo"); + R_field["volume_dependent"].set_string("false"); + if (localNum > 0) { + R_field["values/x"].set_external(&RHost.data()[0][0], localNum, 0, R_stride); + if constexpr (ParticleDim >= 2) { + R_field["values/y"].set_external(&RHost.data()[0][1], localNum, 0, R_stride); + } + if constexpr (ParticleDim >= 3) { + R_field["values/z"].set_external(&RHost.data()[0][2], localNum, 0, R_stride); + } + } - if (err != catalyst_status_ok) { - std::cerr << "::Execute() Failed to execute Catalyst (runtime path): " << err << std::endl; - } + if (localNum == 0) { + using Component = typename RAttrib_t::value_type::value_type; + const char* axes[] = {"x", "y", "z"}; + for (unsigned d = 0; d < ParticleDim && d < 3; ++d) { + R_field[std::string("values/") + axes[d]] + .set_external(static_cast(nullptr), 0); + } + } - if (steerEnabled_m) { - - static IpplTimings::TimerRef TMRfetchResult = IpplTimings::getTimer("fetchSteerParameters"); - IpplTimings::startTimer(TMRfetchResult); + // User-defined attributes (persistent mirrors via prepareConduitNode) + pc.forAllAttributes([&](const Attributes& atts) { + for (auto* attribute : atts) { + if (!pc.isBuiltinAttribute(attribute) && selection.selects(attribute->get_name())) { + attribute->prepareConduitNode(localNum, fields, info); + } + } + }); - fetchResults(); - // backward Node: fetch updated steering values - SteerFetchVisitor fetchV{*this}; - steerRegistry_m->forEach(fetchV); + // Helper block (bounding box) + if constexpr (has_getRegionLayout_v) { + using RLayout_t = typename T::Layout_t::RegionLayout_t; + using NDRegion_t = typename RLayout_t::NDRegion_t; + constexpr unsigned dim_ = T::Layout_t::dim; + const NDRegion_t ndr = pc.getLayout().getRegionLayout().getDomain(); + + data_help["coordsets/bound_helper_coords/type"].set_string("uniform"); + data_help["topologies/bound_helper_topo/coordset"].set_string( + "bound_helper_coords"); + data_help["topologies/bound_helper_topo/type"].set_string("uniform"); + { + data_help["coordsets/bound_helper_coords/dims/i"].set(2); + data_help["coordsets/bound_helper_coords/spacing/dx"].set(ndr[0].max() + - ndr[0].min()); + data_help["coordsets/bound_helper_coords/origin/x"].set(ndr[0].min()); + } + if constexpr (dim_ >= 2) { + data_help["coordsets/bound_helper_coords/dims/j"].set(2); + data_help["coordsets/bound_helper_coords/spacing/dy"].set(ndr[1].max() + - ndr[1].min()); + data_help["coordsets/bound_helper_coords/origin/y"].set(ndr[1].min()); + } + if constexpr (dim_ >= 3) { + data_help["coordsets/bound_helper_coords/dims/k"].set(2); + data_help["coordsets/bound_helper_coords/spacing/dz"].set(ndr[2].max() + - ndr[2].min()); + data_help["coordsets/bound_helper_coords/origin/z"].set(ndr[2].min()); + } + } - IpplTimings::stopTimer(TMRfetchResult); + // Transform block (if registered via setTransform) + if (raw->hasTransform()) { + const auto& td = *raw->transform(); + auto data_tf = channel["data/transform"]; + data_tf["type"].set_string("mesh"); + data_tf["coordsets/tf_coords/type"].set_string("explicit"); + data_tf["coordsets/tf_coords/values/x"].set(0.0); + data_tf["coordsets/tf_coords/values/y"].set(0.0); + data_tf["coordsets/tf_coords/values/z"].set(0.0); + data_tf["topologies/tf_topo/coordset"].set_string("tf_coords"); + data_tf["topologies/tf_topo/type"].set_string("unstructured"); + data_tf["topologies/tf_topo/elements/shape"].set_string("point"); + data_tf["topologies/tf_topo/elements/connectivity"].set(0); + + auto setVec3Field = [&](const std::string& fieldName, + const ippl::Vector& v) { + auto f = data_tf[std::string("fields/") + fieldName]; + f["association"].set_string("vertex"); + f["topology"].set_string("tf_topo"); + f["volume_dependent"].set_string("false"); + f["values/x"].set(v[0]); + f["values/y"].set(v[1]); + f["values/z"].set(v[2]); + }; + setVec3Field("BunchOrigin", td.origin); + setVec3Field("BunchRotRow0", td.rows[0]); + setVec3Field("BunchRotRow1", td.rows[1]); + setVec3Field("BunchRotRow2", td.rows[2]); + setVec3Field("BunchInvOrigin", td.invOrigin); + setVec3Field("BunchInvRotRow0", td.invRows[0]); + setVec3Field("BunchInvRotRow1", td.invRows[1]); + setVec3Field("BunchInvRotRow2", td.invRows[2]); + } + raw->lastLocalNum_m = localNum; + info << level4 << " ParticleChannel '" << name << "' initialized (" << localNum + << " particles)" << endl; + }); - if(true){ - // if(cycle == 0){ - catalystInfo_m << level4 <<"::Execute() Printing Conduit Node received from catalyst_execute() ==>" << endl; - catalystInfo_m << level4 << results_m.to_yaml() << endl; - } + // Type-specific execute function: deep-copy into persistent mirrors. + // Called every Execute(). + raw->setExecuteFn( + [this, name, &pc, raw](conduit_cpp::Node& channel, int rank, Inform& info) { + const size_t localNum = pc.getLocalNum(); + const auto& selection = getParticleAttributeSelection(name); + + // If particle count changed, re-init the whole channel + // (re-allocate mirrors, re-set external pointers). + if (localNum != raw->lastLocalNum_m) { + catalystInfo_m << level4 << " ParticleChannel '" << name + << "' count changed: " << raw->lastLocalNum_m << " -> " + << localNum << ", re-initializing" << endl; + channel.reset(); + raw->initFn_m(channel, rank, info); + return; + } - } + // Deep-copy R into persistent mirror + using RAttrib_t = std::remove_reference_t; + using RHost_t = typename RAttrib_t::host_mirror_type; + auto* RHost = viewRegistry_m.find(name + "_R"); + if (RHost) { + Kokkos::deep_copy(*RHost, pc.R.getView()); + } + // Deep-copy ID (if enabled) + if constexpr (T::EnableIDs) { + using IDHost_t = + typename std::remove_reference_t::host_mirror_type; + auto* IDHost = viewRegistry_m.find(name + "_ID"); + if (selection.includesParticleIds() && IDHost) { + Kokkos::deep_copy(*IDHost, pc.ID.getView()); + } + } - viewRegistry_m.clear(); - ghostMaskCache_m.clear(); - node_m.reset(); + // Deep-copy selected user-defined attributes + pc.forAllAttributes([&](const Attributes& atts) { + for (auto* attribute : atts) { + if (!pc.isBuiltinAttribute(attribute) && selection.selects(attribute->get_name())) { + attribute->updateConduitData(); + } + } + }); + + // Update transform block values (if present) + if (raw->hasTransform()) { + const auto& td = *raw->transform(); + auto data_tf = channel["data/transform"]; + auto setVec3Field = [&](const std::string& fieldName, + const ippl::Vector& v) { + data_tf[std::string("fields/") + fieldName + "/values/x"].set(v[0]); + data_tf[std::string("fields/") + fieldName + "/values/y"].set(v[1]); + data_tf[std::string("fields/") + fieldName + "/values/z"].set(v[2]); + }; + setVec3Field("BunchOrigin", td.origin); + setVec3Field("BunchRotRow0", td.rows[0]); + setVec3Field("BunchRotRow1", td.rows[1]); + setVec3Field("BunchRotRow2", td.rows[2]); + setVec3Field("BunchInvOrigin", td.invOrigin); + setVec3Field("BunchInvRotRow0", td.invRows[0]); + setVec3Field("BunchInvRotRow1", td.invRows[1]); + setVec3Field("BunchInvRotRow2", td.invRows[2]); + } + }); - /////////////////////////////////////////////////// - // Note: - // We deliberately don't reset results since - // 1. they will be properly overwritten by Catalyst. - // 2. If the part of the Catalyst backend crashes and the - // results are not sent back, the old results will be used, possibly - // avoiding a problems during result retrieval. - // - // results.reset(); - /////////////////////////////////////////////////// - - - catalystInfo_m << level4 <<"::Execute() DONE =============================================================== 1" << endl; - -} - - -void CatalystAdaptor::Finalize() { - if ( !visEnabled_m) return; - - conduit_cpp::Node node; - catalyst_status err = catalyst_finalize(conduit_cpp::c_node(&node_m)); - if (err != catalyst_status_ok) { - std::cerr << "Failed to finalize Catalyst: " << err << std::endl; + channels_m.push_back(ch); + catalystInfo_m << level4 << "::addParticleChannel('" << name + << "', type=" << typeid(T).name() << ")" << endl; + return ParticleChannelHandle(raw); } -} - + void CatalystAdaptor::refresh(const std::string& label) { + if (!visEnabled_m) + return; + for (auto& ch : channels_m) { + auto* arr = ch->findArray(label); + if (arr) { + arr->markDirty(); + Inform info("CatalystAdaptor"); + arr->refresh(ippl::Comm->rank(), info); + return; + } + } + catalystWarn_m << "::refresh() label not found: '" << label << "'" << endl; + } -}//ippl +} // namespace ippl diff --git a/src/Stream/InSitu/CatalystVisitors.h b/src/Stream/InSitu/CatalystVisitors.h index 3bd080548..9155c1ec0 100644 --- a/src/Stream/InSitu/CatalystVisitors.h +++ b/src/Stream/InSitu/CatalystVisitors.h @@ -3,245 +3,173 @@ * @brief Visitor functors used by CatalystAdaptor for initialization, execution, and steering. */ #pragma once +#include +#include #include +#include #include #include #include -#include - -#include -#include - - - - -namespace ippl{ - -// ========================================= -// METAPROGRAMMING HELPER TRAITS -// ========================================= - -template -inline constexpr bool is_particle_v = std::is_base_of::type>::value; - -template -struct is_field : std::false_type {}; - -template -struct is_field> : std::true_type {}; - -template -inline constexpr bool is_field_v = is_field>::value; - - - -// Detect ippl::Vector -template -struct is_ippl_vector : std::false_type {}; -template -struct is_ippl_vector> : std::true_type {}; -template -inline constexpr bool is_ippl_vector_v = is_ippl_vector>::value; - -// detect std::vector of supported scalar/button types -template -struct is_std_vector_of_allowed_scalar : std::false_type {}; -template -struct is_std_vector_of_allowed_scalar> : std::bool_constant< - std::is_arithmetic_v> - || std::is_enum_v> - || std::is_same_v, bool> - || std::is_same_v, Button>> {}; - -// detect std::vector of ippl::Vector<...> -template -struct is_std_vector_of_ippl_vector : std::false_type {}; -template -struct is_std_vector_of_ippl_vector> : std::bool_constant> {}; - - - - -// detect any std::vector<...> (used to route struct arrays via StructMeta) -template -struct is_std_vector_any : std::false_type {}; -template -struct is_std_vector_any> : std::true_type {}; - -template -inline constexpr bool AllowedSteerType_v = - std::is_arithmetic_v> - || std::is_enum_v> - || std::is_same_v, Button> - || is_ippl_vector_v // ippl::Vector<> - || is_std_vector_of_allowed_scalar>::value - || is_std_vector_of_ippl_vector>::value - ; - - -template -inline constexpr bool AllowedVisType_v = is_particle_v || is_field_v; - -template -inline constexpr bool AllowedRegistryType_v = AllowedVisType_v - || AllowedSteerType_v - || is_std_vector_any>::value - ; - - -// Accept shared_ptr when U is allowed -template -struct is_allowed_shared_ptr : std::false_type {}; -template -struct is_allowed_shared_ptr> : std::bool_constant> {}; - - -template -inline constexpr bool AllowedRegistryTypeOrShared_v = - AllowedRegistryType_v || is_allowed_shared_ptr>::value; - - -// Checks for region Layout indicating Particle Spatial Layout vs Pure Layout -template -struct has_getRegionLayout : std::false_type {}; - -template -struct has_getRegionLayout().getRegionLayout())>> - : std::true_type {}; - -template -constexpr bool has_getRegionLayout_v = has_getRegionLayout::value; - - - - - -// ========================================= -// VISITORS -// ========================================= -/////////////////////////////////////////////////////////////////////////////////// -// Note: -// Cant overload identical function with requires conditions -// since requires does not affect signature so they end up with identical singature -// but we can switch reference or constness of arguments like the label, changiing -// function signature allowing for early throwing of exception (in this file) -// instead of the additional function overloading in the hpp files. -/////////////////////////////////////////////////////////////////////////////////// - - -/** - * @struct CatalystAdaptor::InitVisitor - * @brief Builds Conduit channel metadata for registered fields/particles. - */ -struct CatalystAdaptor::InitVisitor { - CatalystAdaptor& ca; - - template - requires(AllowedVisType_v) - void operator()(const std::string& label, const T& entry) const { - ca.InitVizChannel(entry, label); - } - - template - requires(!AllowedVisType_v) - void operator()(const std::string label, const T& entry) const { - throw IpplException("CatalystAdaptor::ExecVisitor", "Unsupported VIS type for channel: " + label); - } - -}; - - - - -/** - * @struct CatalystAdaptor::ExecVisitor - * @brief Publishes data views each step for registered fields/particles. - */ -struct CatalystAdaptor::ExecVisitor { - CatalystAdaptor& ca; - - template - requires(AllowedVisType_v) - void operator()(const std::string& label, const T& entry) const { - ca.ExecVizChannel(entry, label ); - } - - template - requires(!AllowedVisType_v) - void operator()(const std::string label, const T& entry) const { - throw IpplException("CatalystAdaptor::ExecVisitor", "Unsupported VIS type for channel: " + label); - } - -}; - - - - - - - -/** - * @struct CatalystAdaptor::SteerInitVisitor - * @brief Declares steerable properties (scalars/vectors/bools/buttons/enums) for the GUI. - */ -struct CatalystAdaptor::SteerInitVisitor { - CatalystAdaptor& ca; - - template - requires(AllowedSteerType_v) - void operator()(const std::string& label, const S& entry) const { - ca.InitSteerChannel(entry, label); - } - - template - requires(!AllowedSteerType_v) - void operator()(const std::string label , const S&) const { - throw IpplException("CatalystAdaptor::ForwardSteerChannel", "Unsupported steerable type for channel: " + label); - } -}; - - - -/** - * @struct CatalystAdaptor::SteerForwardVisitor - * @brief Sends current simulation values forward into Catalyst for display. - */ -struct CatalystAdaptor::SteerForwardVisitor { - CatalystAdaptor& ca; - - template - requires(AllowedSteerType_v) - void operator()(const std::string& label, const S& entry) const { - ca.ForwardSteerChannel(entry, label); - } - - template - requires(!AllowedSteerType_v) - void operator()(const std::string label , const S&) const { - throw IpplException("CatalystAdaptor::ForwardSteerChannel", "Unsupported steerable type for channel: " + label); - } -}; - -/** - * @struct CatalystAdaptor::SteerFetchVisitor - * @brief Retrieves updated values from Catalyst and writes back into the simulation. - */ -struct CatalystAdaptor::SteerFetchVisitor { - CatalystAdaptor& ca; - - template - requires( AllowedSteerType_v ) - void operator()(const std::string& label, S& value) const { - ca.FetchSteerChannel(value, label); - } - - template - requires(!AllowedSteerType_v) - void operator()(const std::string label, S&) const { - throw IpplException("CatalystAdaptor::FetchSteerChannel", "Unsupported steerable type for channel: " + label); - } -}; - - -} \ No newline at end of file +namespace ippl { + + // ========================================= + // METAPROGRAMMING HELPER TRAITS + // ========================================= + + template + inline constexpr bool is_particle_v = + std::is_base_of::type>::value; + + template + struct is_field : std::false_type {}; + + template + struct is_field> : std::true_type {}; + + template + inline constexpr bool is_field_v = is_field>::value; + + // Detect ippl::Vector + template + struct is_ippl_vector : std::false_type {}; + template + struct is_ippl_vector> : std::true_type {}; + template + inline constexpr bool is_ippl_vector_v = is_ippl_vector>::value; + + // detect std::vector of supported scalar/button types + template + struct is_std_vector_of_allowed_scalar : std::false_type {}; + template + struct is_std_vector_of_allowed_scalar> + : std::bool_constant< + std::is_arithmetic_v> || std::is_enum_v> + || std::is_same_v, bool> || std::is_same_v, Button>> { + }; + + // detect std::vector of ippl::Vector<...> + template + struct is_std_vector_of_ippl_vector : std::false_type {}; + template + struct is_std_vector_of_ippl_vector> : std::bool_constant> { + }; + + // detect any std::vector<...> (used to route struct arrays via StructMeta) + template + struct is_std_vector_any : std::false_type {}; + template + struct is_std_vector_any> : std::true_type {}; + + template + inline constexpr bool AllowedSteerType_v = + std::is_arithmetic_v> || std::is_enum_v> + || std::is_same_v, Button> || is_ippl_vector_v // ippl::Vector<> + || is_std_vector_of_allowed_scalar>::value + || is_std_vector_of_ippl_vector>::value; + + template + inline constexpr bool AllowedVisType_v = is_particle_v || is_field_v; + + template + inline constexpr bool AllowedRegistryType_v = + AllowedVisType_v || AllowedSteerType_v || is_std_vector_any>::value; + + // Accept shared_ptr when U is allowed + template + struct is_allowed_shared_ptr : std::false_type {}; + template + struct is_allowed_shared_ptr> + : std::bool_constant> {}; + + template + inline constexpr bool AllowedRegistryTypeOrShared_v = + AllowedRegistryType_v || is_allowed_shared_ptr>::value; + + // Checks for region Layout indicating Particle Spatial Layout vs Pure Layout + template + struct has_getRegionLayout : std::false_type {}; + + template + struct has_getRegionLayout().getRegionLayout())>> + : std::true_type {}; + + template + constexpr bool has_getRegionLayout_v = has_getRegionLayout::value; + + // ========================================= + // VISITORS + // ========================================= + /////////////////////////////////////////////////////////////////////////////////// + // Note: + // Cant overload identical function with requires conditions + // since requires does not affect signature so they end up with identical singature + // but we can switch reference or constness of arguments like the label, changiing + // function signature allowing for early throwing of exception (in this file) + // instead of the additional function overloading in the hpp files. + /////////////////////////////////////////////////////////////////////////////////// + + /** + * @struct CatalystAdaptor::SteerInitVisitor + * @brief Declares steerable properties (scalars/vectors/bools/buttons/enums) for the GUI. + */ + struct CatalystAdaptor::SteerInitVisitor { + CatalystAdaptor& ca; + + template + requires(AllowedSteerType_v) + void operator()(const std::string& label, const S& entry) const { + ca.InitSteerChannel(entry, label); + } + + template + requires(!AllowedSteerType_v) + void operator()(const std::string label, const S&) const { + throw IpplException("CatalystAdaptor::ForwardSteerChannel", + "Unsupported steerable type for channel: " + label); + } + }; + + /** + * @struct CatalystAdaptor::SteerForwardVisitor + * @brief Sends current simulation values forward into Catalyst for display. + */ + struct CatalystAdaptor::SteerForwardVisitor { + CatalystAdaptor& ca; + + template + requires(AllowedSteerType_v) + void operator()(const std::string& label, const S& entry) const { + ca.ForwardSteerChannel(entry, label); + } + + template + requires(!AllowedSteerType_v) + void operator()(const std::string label, const S&) const { + throw IpplException("CatalystAdaptor::ForwardSteerChannel", + "Unsupported steerable type for channel: " + label); + } + }; + + /** + * @struct CatalystAdaptor::SteerFetchVisitor + * @brief Retrieves updated values from Catalyst and writes back into the simulation. + */ + struct CatalystAdaptor::SteerFetchVisitor { + CatalystAdaptor& ca; + + template + requires(AllowedSteerType_v) + void operator()(const std::string& label, S& value) const { + ca.FetchSteerChannel(value, label); + } + + template + requires(!AllowedSteerType_v) + void operator()(const std::string label, S&) const { + throw IpplException("CatalystAdaptor::FetchSteerChannel", + "Unsupported steerable type for channel: " + label); + } + }; + +} // namespace ippl \ No newline at end of file diff --git a/src/Stream/InSitu/Channel.h b/src/Stream/InSitu/Channel.h new file mode 100644 index 000000000..f9f788b89 --- /dev/null +++ b/src/Stream/InSitu/Channel.h @@ -0,0 +1,323 @@ +/** + * @file Channel.h + * @brief Mesh-centric visualization channel types for Catalyst in-situ export. + * + * A Channel is the unit of data export: one Conduit channel per mesh (with + * multiple field arrays) or per particle container. Each channel owns + * persistent host staging buffers allocated once at registration and reused + * on every Execute, eliminating the per-step allocation/free churn of the + * previous flat-registry design. + */ +#ifndef IPPL_CHANNEL_H +#define IPPL_CHANNEL_H + +#include "Ippl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(MPI_VERSION) +#include +#endif + +namespace ippl { + + // ================================================================== + // Update policy + // ================================================================== + + enum class UpdatePolicy { + Always, ///< Copy device->host every Execute (default for dynamic data) + Static, ///< Copy once at registration; never refreshed + Manual, ///< Copy only when refresh(label) is called + }; + + // ================================================================== + // Geometry policy (viz-only) + // ================================================================== + + enum class GeometryPolicy { + Dynamic, ///< Re-read mesh origin/spacing every Execute (default; + ///< bunch may have moved) + Static, ///< Mesh origin/spacing set once at init, never re-read. + ///< Zero per-step geometry cost. + }; + + // ================================================================== + // Transform data (shared with CatalystAdaptor) + // ================================================================== + + struct TransformData { + ippl::Vector origin{}; + std::array, 3> rows{}; + ippl::Vector invOrigin{}; + std::array, 3> invRows{}; + }; + + // Forward declarations + class Channel; + class MeshFieldArrayBase; + template + class MeshChannelT; + class ParticleChannel; + class MeshChannelHandle; + class ParticleChannelHandle; + + // ================================================================== + // Channel base + // ================================================================== + + /** + * @brief Abstract base for a visualization export channel. + * + * Each concrete channel builds its Conduit node tree once in init() and + * copies data into persistent staging buffers every execute(). + */ + class Channel { + public: + virtual ~Channel() = default; + + virtual void init(conduit_cpp::Node& root, int rank) = 0; + virtual void execute(conduit_cpp::Node& root, int cycle, double time, int rank) = 0; + + /// Find a field array by name (mesh channels only). + /// Returns nullptr for particle channels or unknown names. + virtual MeshFieldArrayBase* findArray(const std::string& /*name*/) const { return nullptr; } + + /// Returns "mesh" for mesh channels, "multimesh" for particle channels. + /// Used by CatalystAdaptor to pass channel type metadata to scripts. + virtual const char* conduitType() const = 0; + + /// Parent channel name for blocks nested inside another channel + /// (empty for top-level channels). + virtual std::string basePath() const { return {}; } + + /// Ghost layers included in this channel's export (0 when ghost + /// layers are stripped or the channel has no mesh). + virtual int ghostLayers() const { return 0; } + + const std::string& name() const noexcept { return name_m; } + + protected: + explicit Channel(std::string name) + : name_m(std::move(name)) {} + std::string name_m; + }; + + // ================================================================== + // Type-erased field array on a mesh channel + // ================================================================== + + /** + * @brief Base class for a field array attached to a MeshChannel. + * + * Concrete instances are templated on the field type and own a persistent + * host mirror allocated once at init(). + */ + class MeshFieldArrayBase { + public: + virtual ~MeshFieldArrayBase() = default; + + virtual const std::string& arrayName() const = 0; + virtual UpdatePolicy policy() const = 0; + virtual void setPolicy(UpdatePolicy p) = 0; + + /// Allocate host mirror, copy device->host, set Conduit external pointers. + /// Called once during MeshChannel::init(). + virtual void initConduit(conduit_cpp::Node& data, const std::string& topoName, int rank, + Inform& info) = 0; + + /// Deep-copy device->host into the existing mirror. + /// Called every Execute for dynamic (Always/Manual-dirty) arrays. + virtual void refresh(int rank, Inform& info) = 0; + + bool needsRefresh() const { return policy_m == UpdatePolicy::Always || dirty_m; } + + void markDirty() { dirty_m = (policy_m == UpdatePolicy::Manual); } + void clearDirty() { dirty_m = false; } + + protected: + UpdatePolicy policy_m = UpdatePolicy::Always; + bool dirty_m = false; + }; + + // ================================================================== + // MeshChannel — one Conduit channel per mesh, multiple field arrays + // ================================================================== + + /** + * @brief A visualization channel that publishes one mesh with multiple + * field arrays. + * + * Shared per-mesh data (coordset, topology, RankID, ghost masks) is + * allocated once. Each field array owns its own persistent host mirror. + * + * @tparam Dim Mesh dimension (1, 2, or 3). + */ + template + class MeshChannelT : public Channel { + public: + using Mesh_t = UniformCartesian; + using Layout_t = FieldLayout; + + MeshChannelT(const std::string& name, const Mesh_t& mesh, const Layout_t& layout, + int nghost, bool useGhostMasks, + GeometryPolicy geometryPolicy = GeometryPolicy::Dynamic, + const std::string& basePath = ""); + + ~MeshChannelT() override = default; + + template + MeshChannelT& addArray(const std::string& arrayName, + const Field& field, + UpdatePolicy policy = UpdatePolicy::Always); + + /// Internal: add a pre-constructed type-erased array. + void addArray(std::unique_ptr array) { + arrays_m.push_back(std::move(array)); + } + + MeshFieldArrayBase* findArray(const std::string& arrayName) const override; + const char* conduitType() const override { return "mesh"; } + std::string basePath() const override { return basePath_m; } + int ghostLayers() const override { return useGhostMasks_m ? nghost_m : 0; } + + void init(conduit_cpp::Node& root, int rank) override; + void execute(conduit_cpp::Node& root, int cycle, double time, int rank) override; + + private: + void buildCoordset(conduit_cpp::Node& data, int rank); + void buildRankID(conduit_cpp::Node& data, int rank); + bool dimsChanged() const; + + const Mesh_t& mesh_m; + const Layout_t& layout_m; + int nghost_m; + bool useGhostMasks_m; + GeometryPolicy geometryPolicy_m = GeometryPolicy::Dynamic; + std::string basePath_m; + + std::vector> arrays_m; + + // Shared per-mesh arrays (allocated once) + using RankViewCells_t = Kokkos::View; + RankViewCells_t rankIdCells_m; + size_t localNumCells_m = 0; + + // Ghost mask cache (keyed by mesh+layout+nghost, reused across steps) + using HostMaskView1D_t = + Kokkos::View; + HostMaskView1D_t ghostMask_m; + bool ghostMaskReady_m = false; + + // Cached dimensions for change detection + long cachedNX_m = -1, cachedNY_m = -1, cachedNZ_m = -1; + }; + + // ================================================================== + // ParticleChannel — one Conduit channel per particle container + // ================================================================== + + /** + * @brief A visualization channel that publishes a particle container as a + * Conduit multimesh with optional bunch-frame transform. + * + * All host mirrors (R, ID, attributes, iota, rank_id) are allocated once + * and reused on every execute. The transform is published as a + * transform block (compatible with existing scripts). + * + * The type-specific work (attribute iteration, Conduit multimesh setup, + * D2H copies) is delegated to function objects set by + * CatalystAdaptor::addParticleChannel. This keeps ParticleChannel + * non-templated while preserving full type safety at registration time. + */ + class ParticleChannel : public Channel { + friend class CatalystAdaptor; + + public: + using InitFn = std::function; + using ExecuteFn = std::function; + + ParticleChannel(const std::string& name, void* pc); + ~ParticleChannel() override = default; + + void setInitFn(InitFn fn) { initFn_m = std::move(fn); } + void setExecuteFn(ExecuteFn fn) { execFn_m = std::move(fn); } + + void setTransform(const TransformData& td); + const char* conduitType() const override { return "multimesh"; } + bool hasTransform() const { return transform_m.has_value(); } + const std::optional& transform() const { return transform_m; } + + void init(conduit_cpp::Node& root, int rank) override; + void execute(conduit_cpp::Node& root, int cycle, double time, int rank) override; + + private: + void* pc_m; ///< Type-erased ParticleBaseBase* + + std::optional transform_m; + InitFn initFn_m; + ExecuteFn execFn_m; + + // Persistent staging (allocated once at init, reallocated on grow) + Kokkos::View iota_m{"iota", 0}; + Kokkos::View rankId_m{"rank_id", 0}; + size_t lastLocalNum_m = static_cast(-1); + }; + + // ================================================================== + // Handle classes + // ================================================================== + + /** + * @brief Lightweight handle returned by addMeshChannel(). + * Allows chaining addArray() calls. + */ + class MeshChannelHandle { + public: + explicit MeshChannelHandle(std::shared_ptr ch, int dim) + : channel_m(std::move(ch)) + , dim_m(dim) {} + explicit operator bool() const { return channel_m != nullptr; } + int dim() const { return dim_m; } + + template + MeshChannelHandle& addArray(const std::string& arrayName, + const Field& field, + UpdatePolicy policy = UpdatePolicy::Always); + + private: + std::shared_ptr channel_m; + int dim_m; + }; + + /** + * @brief Lightweight handle returned by addParticleChannel(). + */ + class ParticleChannelHandle { + public: + explicit ParticleChannelHandle(ParticleChannel* ch) + : channel_m(ch) {} + explicit operator bool() const { return channel_m != nullptr; } + + ParticleChannelHandle& setTransform(const TransformData& td) { + if (channel_m) + channel_m->setTransform(td); + return *this; + } + + private: + ParticleChannel* channel_m; + }; + +} // namespace ippl + +#include "Stream/InSitu/Channel.hpp" + +#endif // IPPL_CHANNEL_H diff --git a/src/Stream/InSitu/Channel.hpp b/src/Stream/InSitu/Channel.hpp new file mode 100644 index 000000000..7b81ea1a2 --- /dev/null +++ b/src/Stream/InSitu/Channel.hpp @@ -0,0 +1,475 @@ +#pragma once + +#include + +#include +#include + +#include "Utility/IpplException.h" + +#include "Stream/InSitu/Channel.h" + +namespace ippl { + + // ================================================================== + // Type-erased field array (templated implementation) + // ================================================================== + + namespace detail { + + template + class MeshFieldArrayImpl : public MeshFieldArrayBase { + public: + using Field_type = Field; + using DeviceView_t = typename Field_type::view_type; + using HostView_t = Kokkos::View; + + explicit MeshFieldArrayImpl(const std::string& arrayName, const Field_type& field) + : name_m(arrayName) + , field_m(field) {} + + const std::string& arrayName() const override { return name_m; } + UpdatePolicy policy() const override { return policy_m; } + void setPolicy(UpdatePolicy p) override { policy_m = p; } + + void initConduit(conduit_cpp::Node& data, const std::string& topoName, int rank, + Inform& info) override { + const auto& Layout_ = field_m.getLayout(); + const size_t nGhost = field_m.getNghost(); + const auto LocalNDIndex_ = Layout_.getLocalNDIndex(); + + const size_t nx = LocalNDIndex_[0].length(); + size_t ny = 1; + size_t nz = 1; + if constexpr (Dim >= 2) + ny = LocalNDIndex_[1].length(); + if constexpr (Dim >= 3) + nz = LocalNDIndex_[2].length(); + + // Allocate persistent host mirror (interior only, no ghosts) + if constexpr (Dim == 1) { + hostMirror_m = HostView_t("host_" + name_m, nx); + } else if constexpr (Dim == 2) { + hostMirror_m = HostView_t("host_" + name_m, nx, ny); + } else { + hostMirror_m = HostView_t("host_" + name_m, nx, ny, nz); + } + + nghostStored_m = nGhost; + dimsStored_m = {nx, ny, nz}; + + // First device->host copy + copyInterior(nGhost, nx, ny, nz); + + // Set Conduit field node (external pointers into persistent mirror) + auto fields = data["fields"]; + auto field_node = fields[name_m]; + + field_node["association"].set_string("element"); + field_node["topology"].set_string(topoName); + field_node["volume_dependent"].set_string("false"); + + const auto n_elems = hostMirror_m.size(); + if constexpr (std::is_scalar_v) { + field_node["values"].set_external(hostMirror_m.data(), n_elems); + } else if constexpr (is_vector_v) { + using elem_t = std::remove_pointer_t; + const size_t stride = sizeof(elem_t); + if (n_elems > 0) { + field_node["values/x"].set_external(&hostMirror_m.data()[0][0], n_elems, 0, + stride); + if constexpr (T::dim >= 2) + field_node["values/y"].set_external(&hostMirror_m.data()[0][1], n_elems, + 0, stride); + if constexpr (T::dim >= 3) + field_node["values/z"].set_external(&hostMirror_m.data()[0][2], n_elems, + 0, stride); + } else { + using component_type = typename T::value_type; + field_node["values/x"].set_external(static_cast(nullptr), + 0); + if constexpr (T::dim >= 2) + field_node["values/y"].set_external( + static_cast(nullptr), 0); + if constexpr (T::dim >= 3) + field_node["values/z"].set_external( + static_cast(nullptr), 0); + } + } + + info << level4 << " MeshChannel: initialized array '" << name_m << "' (" << nx + << "x" << ny << "x" << nz << ")" << endl; + } + + void refresh(int /*rank*/, Inform& /*info*/) override { + if (policy_m == UpdatePolicy::Static) + return; + copyInterior(nghostStored_m, dimsStored_m[0], dimsStored_m[1], dimsStored_m[2]); + clearDirty(); + } + + private: + std::string name_m; + const Field_type& field_m; + HostView_t hostMirror_m; + size_t nghostStored_m = 0; + std::array dimsStored_m{0, 0, 0}; + + void copyInterior(size_t nGhost, size_t nx, size_t ny, size_t nz) { + const auto& fullDeviceView = field_m.getView(); + // Copy full device view to host, then extract interior into + // the persistent mirror. The full mirror is temporary; only + // the interior buffer persists across steps. + auto hostMirrorFull = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), fullDeviceView); + if constexpr (Dim == 1) { + const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); + Kokkos::deep_copy(hostMirror_m, Kokkos::subview(hostMirrorFull, r0)); + } else if constexpr (Dim == 2) { + const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); + const auto r1 = Kokkos::make_pair(nGhost, nGhost + ny); + Kokkos::deep_copy(hostMirror_m, Kokkos::subview(hostMirrorFull, r0, r1)); + } else { + const auto r0 = Kokkos::make_pair(nGhost, nGhost + nx); + const auto r1 = Kokkos::make_pair(nGhost, nGhost + ny); + const auto r2 = Kokkos::make_pair(nGhost, nGhost + nz); + Kokkos::deep_copy(hostMirror_m, Kokkos::subview(hostMirrorFull, r0, r1, r2)); + } + } + }; + + } // namespace detail + + // ================================================================== + // MeshChannelHandle::addArray + // ================================================================== + + template + MeshChannelHandle& MeshChannelHandle::addArray(const std::string& arrayName, + const Field& field, + UpdatePolicy policy) { + if (dim_m != int(Dim)) { + throw IpplException("MeshChannelHandle::addArray", + "Field dimension does not match mesh channel dimension"); + } + auto* ch = static_cast*>(channel_m.get()); + auto array = + std::make_unique>(arrayName, field); + array->setPolicy(policy); + ch->addArray(std::move(array)); + return *this; + } + + // ================================================================== + // MeshChannelT implementation + // ================================================================== + + template + MeshChannelT::MeshChannelT(const std::string& name, const Mesh_t& mesh, + const Layout_t& layout, int nghost, bool useGhostMasks, + GeometryPolicy geometryPolicy, const std::string& basePath) + : Channel(name) + , mesh_m(mesh) + , layout_m(layout) + , nghost_m(nghost) + , useGhostMasks_m(useGhostMasks) + , geometryPolicy_m(geometryPolicy) + , basePath_m(basePath) {} + + template + template + MeshChannelT& MeshChannelT::addArray(const std::string& arrayName, + const Field& field, + UpdatePolicy policy) { + auto array = + std::make_unique>(arrayName, field); + array->setPolicy(policy); + arrays_m.push_back(std::move(array)); + return *this; + } + + template + MeshFieldArrayBase* MeshChannelT::findArray(const std::string& arrayName) const { + for (auto& a : arrays_m) { + if (a->arrayName() == arrayName) + return a.get(); + } + return nullptr; + } + + template + bool MeshChannelT::dimsChanged() const { + const auto LocalNDIndex_ = layout_m.getLocalNDIndex(); + const long nx = long(LocalNDIndex_[0].length()); + long ny = 1, nz = 1; + if constexpr (Dim >= 2) + ny = long(LocalNDIndex_[1].length()); + if constexpr (Dim >= 3) + nz = long(LocalNDIndex_[2].length()); + return nx != cachedNX_m || ny != cachedNY_m || nz != cachedNZ_m; + } + + template + void MeshChannelT::buildCoordset(conduit_cpp::Node& data, int rank) { + const auto LocalNDIndex_ = layout_m.getLocalNDIndex(); + const auto Origin_ = mesh_m.getOrigin(); + const auto Spacing_ = mesh_m.getMeshSpacing(); + + const size_t extra = useGhostMasks_m ? size_t(2 * nghost_m) : 0; + const size_t index_offset = useGhostMasks_m ? size_t(nghost_m) : 0; + const int dims_n = 1; // points = cells + 1 + + const size_t nx = LocalNDIndex_[0].length() + extra; + size_t ny = 1; + size_t nz = 1; + if constexpr (Dim >= 2) + ny = LocalNDIndex_[1].length() + extra; + if constexpr (Dim >= 3) + nz = LocalNDIndex_[2].length() + extra; + + const double Ox = + Origin_[0] + (double(int(LocalNDIndex_[0].first()) - int(index_offset))) * Spacing_[0]; + double Oy = 0.0, Oz = 0.0; + if constexpr (Dim >= 2) { + Oy = Origin_[1] + + (double(int(LocalNDIndex_[1].first()) - int(index_offset))) * Spacing_[1]; + } + if constexpr (Dim >= 3) { + Oz = Origin_[2] + + (double(int(LocalNDIndex_[2].first()) - int(index_offset))) * Spacing_[2]; + } + + data["coordsets/cart_uniform_coords/type"].set_string("uniform"); + data["topologies/fmesh_topo/type"].set_string("uniform"); + data["topologies/fmesh_topo/coordset"].set_string("cart_uniform_coords"); + + { + data["coordsets/cart_uniform_coords/dims/i"].set(nx + dims_n); + data["coordsets/cart_uniform_coords/spacing/dx"].set(Spacing_[0]); + data["coordsets/cart_uniform_coords/origin/x"].set(Ox); + data["topologies/fmesh_topo/origin/x"].set(Ox); + } + if constexpr (Dim >= 2) { + data["coordsets/cart_uniform_coords/dims/j"].set(ny + dims_n); + data["coordsets/cart_uniform_coords/spacing/dy"].set(Spacing_[1]); + data["coordsets/cart_uniform_coords/origin/y"].set(Oy); + data["topologies/fmesh_topo/origin/y"].set(Oy); + } + if constexpr (Dim >= 3) { + data["coordsets/cart_uniform_coords/dims/k"].set(nz + dims_n); + data["coordsets/cart_uniform_coords/spacing/dz"].set(Spacing_[2]); + data["coordsets/cart_uniform_coords/origin/z"].set(Oz); + data["topologies/fmesh_topo/origin/z"].set(Oz); + } + + // Cache dims + cachedNX_m = long(LocalNDIndex_[0].length()); + cachedNY_m = Dim >= 2 ? long(LocalNDIndex_[1].length()) : 1; + cachedNZ_m = Dim >= 3 ? long(LocalNDIndex_[2].length()) : 1; + localNumCells_m = nx * ny * nz; + } + + template + void MeshChannelT::buildRankID(conduit_cpp::Node& data, int rank) { + const auto LocalNDIndex_ = layout_m.getLocalNDIndex(); + const size_t extra = useGhostMasks_m ? size_t(2 * nghost_m) : 0; + + const size_t nx = LocalNDIndex_[0].length() + extra; + size_t ny = 1; + size_t nz = 1; + if constexpr (Dim >= 2) + ny = LocalNDIndex_[1].length() + extra; + if constexpr (Dim >= 3) + nz = LocalNDIndex_[2].length() + extra; + + const size_t localNumCells = nx * ny * nz; + + // Allocate (or reuse) RankID view + if (rankIdCells_m.extent(0) != nx || rankIdCells_m.extent(1) != ny + || rankIdCells_m.extent(2) != nz) { + rankIdCells_m = RankViewCells_t("rank_id_cells", nx, ny, nz); + } + + if (localNumCells > 0) { + using HostExecSpace = Kokkos::DefaultHostExecutionSpace; + Kokkos::MDRangePolicy> host_policy({0, 0, 0}, + {nx, ny, nz}); + // Host-only execution space: use a plain C++ lambda to avoid the + // NVCC restriction on extended __host__ __device__ lambdas inside + // private member functions. + Kokkos::parallel_for("fill_rank_ids_channel", host_policy, + [&](const int i, const int j, const int k) { + rankIdCells_m(i, j, k) = rank; + }); + } + + auto fields = data["fields"]; + auto rankField = fields["RankID"]; + rankField["association"].set_string("element"); + rankField["topology"].set_string("fmesh_topo"); + rankField["volume_dependent"].set_string("false"); + if (localNumCells > 0) { + rankField["values"].set_external(rankIdCells_m.data(), localNumCells); + } else { + rankField["values"].set_external(static_cast(nullptr), 0); + } + data["metadata/vtk_fields/RankID/attribute_type"].set_string("ProcessIds"); + } + + template + void MeshChannelT::init(conduit_cpp::Node& root, int rank) { + // When basePath_m is set, this mesh is a block inside a parent + // multimesh channel (catalyst/channels//data/). + // Otherwise it is a top-level channel (catalyst/channels/). + auto data = basePath_m.empty() + ? root["catalyst/channels/" + name_m]["data"] + : root["catalyst/channels/" + basePath_m]["data/" + name_m]; + if (basePath_m.empty()) { + root["catalyst/channels/" + name_m]["type"].set_string("mesh"); + } else { + // Each block in a multimesh needs its own type field. + // Also add the assembly entry so the parent knows about this block. + data["type"].set_string("mesh"); + root["catalyst/channels/" + basePath_m]["assembly/" + name_m] = name_m; + } + + // Build shared coordset + topology + buildCoordset(data, rank); + + // Build shared RankID + buildRankID(data, rank); + + // Initialize each field array (allocates persistent mirror, sets external) + for (auto& array : arrays_m) { + array->initConduit(data, "fmesh_topo", rank, + *std::make_unique("MeshChannel").get()); + } + } + + template + void MeshChannelT::execute(conduit_cpp::Node& root, int cycle, double time, int rank) { + // Update Catalyst state + auto state = root["catalyst/state"]; + state["cycle"].set(cycle); + state["time"].set(time); + state["domain_id"].set(rank); + + auto data = basePath_m.empty() + ? root["catalyst/channels/" + name_m]["data"] + : root["catalyst/channels/" + basePath_m]["data/" + name_m]; + + // When basePath_m is set, the parent particle channel may have + // reset this node (on particle count change). If the node is + // empty (no type field), re-initialize from scratch, including + // the assembly entry on the parent. + if (!basePath_m.empty() && !data.has_path("type")) { + data["type"].set_string("mesh"); + root["catalyst/channels/" + basePath_m]["assembly/" + name_m] = name_m; + buildCoordset(data, rank); + buildRankID(data, rank); + for (auto& array : arrays_m) { + array->initConduit(data, "fmesh_topo", rank, + *std::make_unique("MeshChannel").get()); + } + } + // Check for mesh repartition + else if (dimsChanged()) { + // Rebuild coordset + RankID, re-init all arrays. + // When basePath_m is set, the parent particle channel may have + // reset this node (on particle count change), so re-set type. + if (!basePath_m.empty()) { + data["type"].set_string("mesh"); + } + buildCoordset(data, rank); + buildRankID(data, rank); + for (auto& array : arrays_m) { + array->initConduit(data, "fmesh_topo", rank, + *std::make_unique("MeshChannel").get()); + } + } else if (geometryPolicy_m == GeometryPolicy::Dynamic) { + // Update coordset spacing/origin (bunch may have moved) + const auto Origin_ = mesh_m.getOrigin(); + const auto Spacing_ = mesh_m.getMeshSpacing(); + const auto LocalNDIndex_ = layout_m.getLocalNDIndex(); + const size_t index_offset = useGhostMasks_m ? size_t(nghost_m) : 0; + + const double Ox = + Origin_[0] + + (double(int(LocalNDIndex_[0].first()) - int(index_offset))) * Spacing_[0]; + data["coordsets/cart_uniform_coords/spacing/dx"].set(Spacing_[0]); + data["coordsets/cart_uniform_coords/origin/x"].set(Ox); + data["topologies/fmesh_topo/origin/x"].set(Ox); + if constexpr (Dim >= 2) { + const double Oy = + Origin_[1] + + (double(int(LocalNDIndex_[1].first()) - int(index_offset))) * Spacing_[1]; + data["coordsets/cart_uniform_coords/spacing/dy"].set(Spacing_[1]); + data["coordsets/cart_uniform_coords/origin/y"].set(Oy); + data["topologies/fmesh_topo/origin/y"].set(Oy); + } + if constexpr (Dim >= 3) { + const double Oz = + Origin_[2] + + (double(int(LocalNDIndex_[2].first()) - int(index_offset))) * Spacing_[2]; + data["coordsets/cart_uniform_coords/spacing/dz"].set(Spacing_[2]); + data["coordsets/cart_uniform_coords/origin/z"].set(Oz); + data["topologies/fmesh_topo/origin/z"].set(Oz); + } + } + // When GeometryPolicy::Static, skip geometry update entirely. + + // Refresh dynamic field arrays + { + auto info = std::make_unique("MeshChannel"); + for (auto& array : arrays_m) { + if (array->needsRefresh()) { + array->refresh(rank, *info); + } + } + } + } + + // ================================================================== + // ParticleChannel implementation + // ================================================================== + + inline ParticleChannel::ParticleChannel(const std::string& name, void* pc) + : Channel(name) + , pc_m(pc) {} + + inline void ParticleChannel::setTransform(const TransformData& td) { + transform_m = td; + } + + inline void ParticleChannel::init(conduit_cpp::Node& root, int rank) { + auto channel = root["catalyst/channels/" + name_m]; + channel["type"].set_string("multimesh"); + + // The init function (set by CatalystAdaptor::addParticleChannel) + // handles block assembly, coordset, topology, R/ID/attribute mirrors, + // and the optional transform block. + if (initFn_m) { + Inform info("ParticleChannel"); + initFn_m(channel, rank, info); + } + } + + inline void ParticleChannel::execute(conduit_cpp::Node& root, int cycle, double time, + int rank) { + auto state = root["catalyst/state"]; + state["cycle"].set(cycle); + state["time"].set(time); + state["domain_id"].set(rank); + + // The execute function (set by CatalystAdaptor::addParticleChannel) + // deep-copies R/ID/attributes into persistent mirrors and updates the + // transform block if present. + auto channel = root["catalyst/channels/" + name_m]; + if (execFn_m) { + Inform info("ParticleChannel"); + execFn_m(channel, rank, info); + } + } + +} // namespace ippl diff --git a/src/Stream/InSitu/catalyst_scripts/catalyst_extractors/png_ext_particle.py b/src/Stream/InSitu/catalyst_scripts/catalyst_extractors/png_ext_particle.py index 514426176..6702637cd 100644 --- a/src/Stream/InSitu/catalyst_scripts/catalyst_extractors/png_ext_particle.py +++ b/src/Stream/InSitu/catalyst_scripts/catalyst_extractors/png_ext_particle.py @@ -200,7 +200,7 @@ def _configure_particle_coloring(point_data_info): # subset_extractor # subset_extractor = ExtractBlock(registrationName='Selected_Parts', Input=ippl_producer) -# subset_extractor.Selectors = ['//block_main'] +# subset_extractor.Selectors = ['//particles'] # ippl_particle = MergeBlocks(registrationName='Merged_Subset', Input=subset_extractor) # ippl_particle.MergePartitionsOnly = 0 @@ -229,9 +229,9 @@ def _configure_particle_coloring(point_data_info): # registrationName=f"{cname[15:]}_bunch_png_ext", # Input=ippl_particle_p, # Assembly = 'Hierarchy', -# # Selectors=['//block_main'] +# # Selectors=['//particles'] # Selectors=['//main'] -# # Selectors=['/Root/block_main'] +# # Selectors=['/Root/particles'] # # Selectors=['/Root/main'] # ) # ippl_particle_e.UpdatePipeline() @@ -251,9 +251,9 @@ def _configure_particle_coloring(point_data_info): registrationName=f"{cname[15:]}_bunch_png_ext", Input=ippl_particle_p, Assembly = 'Hierarchy', - Selectors=['//block_main'] + Selectors=['//particles'] # Selectors=['//main'] - # Selectors=['/Root/block_main'] + # Selectors=['/Root/particles'] # Selectors=['/Root/main'] ) hide_source_from_gui(ippl_particle_bunch) @@ -262,9 +262,9 @@ def _configure_particle_coloring(point_data_info): registrationName=f"{cname[15:]}_box_png_ext", Input=ippl_particle_p, Assembly = 'Hierarchy', - Selectors=['//block_help'] + Selectors=['//boundingbox'] # Selectors=['//main'] - # Selectors=['/Root/block_main'] + # Selectors=['/Root/particles'] # Selectors=['/Root/main'] ) hide_source_from_gui(ippl_particle_box) diff --git a/src/Stream/InSitu/catalyst_scripts/pipeline_default.py b/src/Stream/InSitu/catalyst_scripts/pipeline_default.py index 21feaae95..da1a75f4e 100644 --- a/src/Stream/InSitu/catalyst_scripts/pipeline_default.py +++ b/src/Stream/InSitu/catalyst_scripts/pipeline_default.py @@ -1,7 +1,13 @@ r"""! \file pipeline_default.py -\brief Main ParaView Catalyst pipeline: live visualization, VTK extracts, and steering. +\brief Main ParaView Catalyst pipeline: live visualization, VTK extracts, PNG extracts, and steering. \details Discovers channel proxies, wires optional extractors, updates live views, -and forwards/fetches steerable parameters between the simulation an +and forwards/fetches steerable parameters between the simulation and ParaView. + +PNG extracts replicate the rendering of the former per-channel extractor scripts +(catalyst_extractors/png_ext_particle.py, png_ext_sfield.py and +png_ext_vfield.py): every channel - or every array of a mesh channel - gets its +own dedicated render view, styled and extracted exactly like those scripts did, +written to 'data_png_extracts_'. """ # script-version: 2.0 @@ -26,12 +32,20 @@ CreateSteerableParameters, PVTrivialProducer, Show, - MergeBlocks, + MergeBlocks, ExtractBlock, CellDatatoPointData, Glyph, GetColorTransferFunction, - ResampleToImage + GetOpacityTransferFunction, + GetTransferFunction2D, + GetScalarBar, + ColorBy, + CreateView, + SetActiveView, + GetMaterialLibrary, + ResampleToImage, + Threshold, # LoadPlugin, # ExtractSubset, # AdaptiveResampleToImage, @@ -52,9 +66,12 @@ from catalystSubroutines import ( print_proxy_overview, - get_global_spatial_bounds - # , - # create_VTPD_extractor + get_global_spatial_bounds, + get_global_range, + nice_bounds, + nice_bounds_sym, + auto_camera_from_bounds, + compute_bounding_box_scale ) #### disable automatic camera rest on 'Show' paraview.simple._DisableFirstRenderCameraReset() @@ -73,68 +90,30 @@ def create_VTPD_extractor(name, object, fr = 10): """Create a VTPD extractor for simple mesh data (uniform/structured grids). - + Use for: scalar fields, vector fields (Conduit type: "mesh") Output: .vtpd files (XML Partitioned Dataset) """ - vTPD = CreateExtractor('VTPD', object, registrationName='VTPD_'+ name) + safe_name = name.replace("/", "_") # nested channels are named parent/block + vTPD = CreateExtractor('VTPD', object, registrationName='VTPD_'+ safe_name) vTPD.Trigger.Frequency = fr - vTPD.Writer.FileName = 'ippl_'+name+'_{timestep:06d}.vtpd' + vTPD.Writer.FileName = 'ippl_'+safe_name+'_{timestep:06d}.vtpd' return vTPD def create_VTM_extractor(name, object, fr = 10): """Create a VTPC extractor for multimesh particle data (Partitioned Dataset Collection). - + Use for: particle data with multiple block types (Conduit type: "multimesh") Output: .vtpc files (XML Partitioned Dataset Collection) Note: Conduit multimesh → vtkPartitionedDataSetCollection (not vtkMultiBlockDataSet) """ # Use VTPC (Partitioned Dataset Collection) not VTM (MultiBlock) - vTPC = CreateExtractor('VTPC', object, registrationName='VTPC_'+ name) + safe_name = name.replace("/", "_") # nested channels are named parent/block + vTPC = CreateExtractor('VTPC', object, registrationName='VTPC_'+ safe_name) vTPC.Trigger.Frequency = fr - vTPC.Writer.FileName = 'ippl_'+name+'_{timestep:06d}.vtpc' + vTPC.Writer.FileName = 'ippl_'+safe_name+'_{timestep:06d}.vtpc' return vTPC -# Option 2: Extract only specific blocks (e.g., just particles without helper) - # Uncomment the following to extract only the main particle block: - # if parsed.VTKextract == "ON": - # particles_block = ExtractBlock( - # registrationName=f"{cname}_main_extract", - # Input=proxy, - # Selectors=['//main'] - # ) - # particles_block.UpdatePipeline() - # _log(f"Attaching VTPD extractor to MAIN particle block (via MergeBlocks) '{cname}'") - # _extractors[cname+"_main_only"] = create_extractor_from_single_block(cname+"_main", particles_block, 1)UI. -# Designed to orchestrate the per-channel extractor scripts in catalyst_extractors/. - -# def create_extractor_from_single_block(name, extract_block_filter, fr = 10): -# """Create an extractor for a single block from multimesh. - -# Use for: extracting just one block (e.g., only particles, no helper) -# Strategy: ExtractBlock → MergeBlocks to flatten → appropriate writer - -# Args: -# name: base name for output files -# extract_block_filter: an ExtractBlock filter with Selectors set -# fr: extraction frequency - -# Output: .vtpd or .vtu files depending on block content -# """ -# # Merge blocks to flatten the extracted subset into a single partitioned dataset -# merged = MergeBlocks( -# registrationName=f"{name}_merged", -# Input=extract_block_filter -# ) -# merged.MergePartitionsOnly = 1 # Keep partitions, just flatten hierarchy -# merged.UpdatePipeline() - -# # Now use VTPD which should work on the flattened structure -# vTPD = CreateExtractor('VTPD', merged, registrationName='VTPD_'+ name) -# vTPD.Trigger.Frequency = fr -# vTPD.Writer.FileName = 'ippl_'+name+'_{timestep:06d}.vtpd' -# return vTPD - _CATALYST_INFO_LEVEL = 4 @@ -147,7 +126,7 @@ def _log(msg): if controller: rank = controller.GetLocalProcessId() msg = "[rank " + str(rank) + "]: " + msg - + if verbosity >= _CATALYST_INFO_LEVEL: print(msg) @@ -160,7 +139,6 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): - #### disable automatic camera rest on 'Show' # paraview.simple._DisableFirstRenderCameraReset() colorPalette = GetSettingsProxy('ColorPalette') @@ -182,13 +160,6 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): colorPalette.Surface = [1.0, 0.0, 0.0] -# # 1. Set default representation to "Points" -# renderViewSettings.DefaultRepresentation = 'Points' -# colorPalette.Surface = [1.0, 0.0, 0.0] # [R, G, B] for Red -# renderViewSettings.DefaultPointSize = 10 -# renderViewSettings.DefaultRenderPointsAsSpheres = 1 # 1 means True - - # ---------------------------------------------------------------- # Parse arguments received via conduit node # ---------------------------------------------------------------- @@ -197,11 +168,18 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): parser = argparse.ArgumentParser() parser.add_argument("--channel_names", nargs="*", help="Pass All Channel Names for which we need to update the privial producer each round") +parser.add_argument("--channel_types", nargs="*", + help="Conduit type for each channel (mesh or multimesh)") parser.add_argument("--steer_channel_names", nargs="*", help="Pass All Channel Names for Steering scalar parameters") +parser.add_argument("--channel_ghosts", nargs="*", type=int, default=[], + help="Ghost layers included per channel export (informational; ghost handling in the pipelines detects vtkGhostType itself)") parser.add_argument("--verbosity", type=int, default=0, help="Communicate the Catalyst output level from the simulation") parser.add_argument("--VTKextract", default="OFF", help="Enable the VTK extracts of all incoming channels") +parser.add_argument("--png", default="OFF", help="Enable PNG screenshot extracts of the render view") +parser.add_argument("--extracts_output_dir", default="", + help="Extract output directory passed from the simulation (overrides the experiment-name default)") parser.add_argument("--live", default="OFF", help="Enable options.CatalystLive") parser.add_argument("--steer", default="OFF", help="Enable steering from catalyst python side") parser.add_argument("--show_forward_channels", default="OFF", help="Show forward steerable channels in the GUI (PVTrivialProducer)") @@ -211,6 +189,7 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): exp_string = parsed.experiment_name verbosity = parsed.verbosity +png_enabled = parsed.png == "ON" # ------------------------------------------------------------------------------ @@ -223,8 +202,12 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): print_info_(f"Parsed steer_channel_names: {parsed.steer_channel_names}") print_info_(f"Parsed channel_names: {parsed.channel_names}") +print_info_(f"Parsed channel_types: {parsed.channel_types}") +print_info_(f"Parsed channel_ghosts: {parsed.channel_ghosts}") print_info_(f"Parsed verbosity level: {parsed.verbosity}") print_info_(f"Parsed VTK extract options: {parsed.VTKextract}") +print_info_(f"Parsed PNG extract option: {parsed.png}") +print_info_(f"Parsed extracts output dir: {parsed.extracts_output_dir}") print_info_(f"Parsed steering option: {parsed.steer}") @@ -241,15 +224,26 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): options.EnableCatalystLive = 0 - options.GlobalTrigger = 'Time Step' options.CatalystLiveTrigger = 'Time Step' options.CatalystLiveURL = 'localhost:22222' #is also default -options.ExtractsOutputDirectory = 'data_vtk_extracts_' + exp_string +if png_enabled: + # The former png_ext_*.py scripts wrote their PNG extracts here; keep the + # same location so output remains comparable across the redesign. + options.ExtractsOutputDirectory = 'data_png_extracts_' + exp_string +elif parsed.extracts_output_dir: + # Directory passed from the simulation (below IPPL_CATALYST_OUTPUT_DIR) + options.ExtractsOutputDirectory = parsed.extracts_output_dir +else: + options.ExtractsOutputDirectory = 'data_vtk_extracts_' + exp_string +# Steering requires a Live connection: the backward-channel senders only +# make sense when a client can drive the parameters. steer_enabled = parsed.live == "ON" and parsed.steer == "ON" +if parsed.steer == "ON" and parsed.live != "ON": + print_info_("Steering requested (--steer ON) but Live is OFF: steering requires Live and stays disabled.") # ------------------------------------------------------------------------------ # Proactively remove the forward 0D mesh source from the GUI if it was auto-created @@ -296,179 +290,602 @@ def print_info_(s, level=_CATALYST_INFO_LEVEL): _filters = {} # _shows={} -# # ---------------------------------------------------------------- -# # create a new 'XML Partitioned Dataset Reader' -# # Dynamically create PVTrivialProducer objects for each channel name -# # ---------------------------------------------------------------- -# print_info_("=== SETTING TRIVIAL PRODUCERS (LIVE) ======="[0:40]+"|0") -# if parsed.channel_names: -# for cname in parsed.channel_names: -# _sources[cname] = PVTrivialProducer(registrationName=cname) +# Build a (name, type) list. If --channel_types was not passed (e.g. old +# C++ side), fall back to name-based heuristics for backward compatibility. +_channel_types = parsed.channel_types or [] +_channels = list(zip(parsed.channel_names or [], _channel_types)) +if not _channel_types: + _channels = [] + for cname in (parsed.channel_names or []): + if "particles" in cname: + _channels.append((cname, "multimesh")) + else: + _channels.append((cname, "mesh")) -# else: -# print_info_("No channel names provided in parsed.channel_names.") -# print_info_("=== SETTING TRIVIAL PRODUCERS (LIVE) ======="[0:40]+"|1") +# Live view for Catalyst Live connections. Plain default styling, exactly like +# the live branch of the former pipeline_default.py: the fancy per-channel +# rendering lives in the dedicated PNG views below. +live_view = None +if options.EnableCatalystLive: + live_view = CreateView('RenderView') + SetActiveView(live_view) -# # ------------------------------------------------------------------------------ -# # Optionally create VTPD extractors for each channel -# # ------------------------------------------------------------------------------ -# extractors = {} -# if parsed.VTKextract == "ON" and parsed.channel_names: -# print_info_("=== SETTING VTK DATA EXTRAXCTION================"[0:40]+"|0") -# for cname, reader in _sources.items(): -# extractors[cname] = create_VTPD_extractor(cname, reader, 1) -# print_info_("=== SETTING VTK DATA EXTRACTION==================="[0:40]+"|1") +# ------------------------------------------------------------------------------ +# Particle channel PNG machinery (recipe: catalyst_extractors/png_ext_particle.py) +# ------------------------------------------------------------------------------ +_BUILTIN_PARTICLE_ATTRIBUTES = { + "position", + "particleid", + "particleids", + "rankid", + "rankids", +} -# NEW / ALT APPROAH: -# Find proxies and set up pipelines in the global scope -pm = servermanager.ProxyManager() -for cname in parsed.channel_names: - # proxy = pm.GetProxy("sources", cname) - # if not proxy: - # _log(f"WARNING: Could not find auto-generated proxy for channel '{cname}'") - # continue - # _log(f"Found auto-generated source proxy: {cname} ({proxy.GetXMLName()})") - proxy = PVTrivialProducer(registrationName=cname) - proxy.UpdatePipeline() - _sources[cname] = proxy - - - if "particles" in cname: - # For Live visualization: create ExtractBlock filters to show specific blocks - if options.EnableCatalystLive: - _log(f"Creating ExtractBlock filter(s) for particle channel '{cname}' (Live view)") - particles = ExtractBlock( - # registrationName=f"{cname[15:]}.bunch", - registrationName=f"{cname}.bunch", - Input=proxy, - # // fails to enable visualisation by attributes exclusive to this block - Selectors=['//main', '//block_main'] - # Selectors=[] - ) - helper = ExtractBlock( - # registrationName=f"{cname[15:]}_box", - registrationName=f"{cname}.box", - Input=proxy, - Selectors=['//help', '//block_help'] - ) - particles.UpdatePipeline() - helper.UpdatePipeline() +def _normalized_attribute_name(name): + """Normalize spelling variants such as ParticleIDs and particle_ids.""" + return "".join(character for character in name.lower() if character.isalnum()) - _filters[cname+"_main"] = particles - _filters[cname+"_help"] = helper - - Show(particles) - Show(helper) +def _first_custom_particle_attribute(point_data_info): + """Return (name, component count) for the first non-built-in point array.""" + if point_data_info is None: + return None, 0 - # Particles come as multimesh with block_main and block_help - # Conduit multimesh → vtkPartitionedDataSetCollection → use VTPC writer - # Option 1: Extract entire multimesh (all blocks together) - if parsed.VTKextract == "ON": - _log(f"Attaching VTPC extractor to complete multimesh particle proxy '{cname}'") - _extractors[cname] = create_VTM_extractor(cname, proxy, 1) - + for index in range(point_data_info.GetNumberOfArrays()): + array_info = point_data_info.GetArrayInformation(index) + if array_info is None: + continue + name = array_info.GetName() + if not name or _normalized_attribute_name(name) in _BUILTIN_PARTICLE_ATTRIBUTES: + continue + components = array_info.GetNumberOfComponents() + if components > 0: + return name, components + return None, 0 - if "sField" in cname: - if options.EnableCatalystLive: - _log(" -> Using MergeBlocks for structured scalar field data.") - # merged = MergeBlocks(registrationName=cname[12:]+'_MergedBlocks', - merged = MergeBlocks(registrationName=cname +'.MergedBlocks', - Input=proxy) - merged.MergePartitionsOnly = 1 - Show(merged) - - _log(" -> Using CellDataToPointtData for structured scalar field data.") - # cell2point = CellDatatoPointData(registrationName=cname[12:]+'.Cell2Point', - cell2point = CellDatatoPointData(registrationName=cname+'.Cell2Point', - Input=merged) - Show(cell2point) - - _log(" -> Using ResampleToImage for structured scalar field data.") - - # Calculate dimensions (logic from png_ext_sfield.py) - info = proxy.GetDataInformation() - local_bounds = info.GetBounds() - local_extent = info.GetExtent() - global_bounds = get_global_spatial_bounds(local_bounds) - - nx = (local_extent[1] - local_extent[0] + 1) - ny = (local_extent[3] - local_extent[2] + 1) - nz = (local_extent[5] - local_extent[4] + 1) - - lx = (local_bounds[1] - local_bounds[0]) - ly = (local_bounds[3] - local_bounds[2]) - lz = (local_bounds[5] - local_bounds[4]) - - spacing_x = lx / max(nx - 1, 1) - spacing_y = ly / max(ny - 1, 1) - spacing_z = lz / max(nz - 1, 1) - - dx = max(spacing_x, 1e-12) - dy = max(spacing_y, 1e-12) - dz = max(spacing_z, 1e-12) - - - ghost_x = 1 - ghost_y = 1 - ghost_z = 1 - - dim_x = int(round((global_bounds[1] - global_bounds[0]) / dx)) - 2*ghost_x - dim_y = int(round((global_bounds[3] - global_bounds[2]) / dy)) - 2*ghost_y - dim_z = int(round((global_bounds[5] - global_bounds[4]) / dz)) - 2*ghost_z - global_extent = [dim_x, dim_y, dim_z] - print_info_(global_extent) - - - - resample = ResampleToImage(registrationName=cname+'.ResampleToImage', Input=merged) - resample.UseInputBounds = 1 - # resample.SamplingBounds = global_bounds - resample.SamplingDimensions = global_extent - Show(resample) - - # cell2point.CellDataArraytoprocess = ['RankID', 'density'] - _filters[cname[12:]+"_merge"] = merged - _filters[cname[12:]+"_c2p"] = merged - _filters[cname[12:]+"_resample"] = resample +def _configure_particle_coloring(state, point_data_info): + """Select the first custom attribute, or configure solid red coloring. - if parsed.VTKextract == "ON": - # _log(f"Attaching VTPD extractor to proxy '{cname}'") - _extractors[cname] = create_VTPD_extractor(cname, proxy, 1) + Ported from png_ext_particle.py; coloring state is kept per channel. + """ + array_name, components = _first_custom_particle_attribute(point_data_info) + if (state['color_initialized'] + and array_name == state['color_array_name'] + and components == state['color_components']): + return + + if state['color_lut'] is not None: + try: + GetScalarBar(state['color_lut'], state['view']).Visibility = 0 + except Exception: + pass + + state['color_array_name'] = array_name + state['color_components'] = components + state['color_lut'] = None + state['color_initialized'] = True + + display = state['display'] + if array_name is None: + display.ColorArrayName = ['POINTS', ''] + display.AmbientColor = [1.0, 0.0, 0.0] + display.DiffuseColor = [1.0, 0.0, 0.0] + print_info_("No custom particle attribute found; using solid red coloring.") + return + + color_spec = ("POINTS", array_name) + component_title = "" + if components > 1: + color_spec = ("POINTS", array_name, "Magnitude") + component_title = "Magnitude" + + ColorBy(display, color_spec) + lut = GetColorTransferFunction(array_name) + color_bar = GetScalarBar(lut, state['view']) + color_bar.Title = array_name + color_bar.ComponentTitle = component_title + color_bar.Visibility = 1 + display.SetScalarBarVisibility(state['view'], True) + state['color_lut'] = lut + + mode = "magnitude" if components > 1 else "scalar values" + print_info_(f"Coloring particles by {mode} of custom attribute '{array_name}'.") + + +def _setup_particle_png_channel(cname, producer, particles, helper): + """Dedicated PNG view for one particle channel (png_ext_particle.py recipe). + + Point-gaussian bunch with a data axes grid, solid red until the first + execute discovers custom attributes, yellow outline helper box, adaptive + camera and a PNG extractor writing '