From 6cc07bb8e9e9c7e9ae6c077fcf87af1242d30fb0 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Wed, 5 Nov 2025 21:01:06 +0000 Subject: [PATCH 01/10] doc(net): these comments should be rustdoc Signed-off-by: Daniel Noland --- net/src/packet/meta.rs | 74 +++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/net/src/packet/meta.rs b/net/src/packet/meta.rs index e0aa9b5c4..2ee1d644f 100644 --- a/net/src/packet/meta.rs +++ b/net/src/packet/meta.rs @@ -90,36 +90,58 @@ impl Display for VpcDiscriminant { } } } - #[allow(unused)] #[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] pub enum DoneReason { - InternalFailure, /* catch-all for internal issues */ - NotEthernet, /* could not get eth header */ - NotIp, /* could not get IP header - maybe it's not ip */ - UnsupportedTransport, /* unsupported transport layer */ - MacNotForUs, /* frame is not broadcast nor for us */ - InterfaceDetached, /* interface has not been attached to any VRF */ - InterfaceAdmDown, /* interface is admin down */ - InterfaceOperDown, /* interface is oper down : no link */ - InterfaceUnknown, /* the interface cannot be found */ - InterfaceUnsupported, /* the operation is not supported on the interface */ - NatOutOfResources, /* can't do NAT due to lack of resources */ - RouteFailure, /* missing routing information */ - RouteDrop, /* routing explicitly requests pkts to be dropped */ - HopLimitExceeded, /* TTL / Hop count was exceeded */ - Filtered, /* The packet was administratively filtered */ - Unhandled, /* there exists no support to handle this type of packet */ - MissL2resolution, /* adjacency failure: we don't know mac of some ip next-hop */ - InvalidDstMac, /* dropped the packet since it had to have an invalid destination mac */ - Malformed, /* the packet does not conform / is malformed */ - MissingEtherType, /* can't determine ethertype to use */ - Unroutable, /* we don't have state to forward the packet */ - NatFailure, /* It was not possible to NAT the packet */ - InternalDrop, /* the packet is dropped by the dataplane (e.g. due to lack of resources like queue space) */ - Delivered, /* the packet buffer was delivered by the NF - e.g. for xmit */ + /// Catch-all for internal issues + InternalFailure, + /// Could not get eth header + NotEthernet, + /// Could not get IP header - maybe it's not ip + NotIp, + /// Unsupported transport layer + UnsupportedTransport, + /// Frame is not broadcast nor for us + MacNotForUs, + /// Interface has not been attached to any VRF + InterfaceDetached, + /// Interface is admin down + InterfaceAdmDown, + /// Interface is oper down : no link + InterfaceOperDown, + /// The interface cannot be found + InterfaceUnknown, + /// The operation is not supported on the interface + InterfaceUnsupported, + /// Can't do NAT due to lack of resources + NatOutOfResources, + /// Missing routing information + RouteFailure, + /// Routing explicitly requests pkts to be dropped + RouteDrop, + /// TTL / Hop count was exceeded + HopLimitExceeded, + /// The packet was administratively filtered + Filtered, + /// There exists no support to handle this type of packet + Unhandled, + /// Adjacency failure: we don't know mac of some ip next-hop + MissL2resolution, + /// Dropped the packet since it had to have an invalid destination mac + InvalidDstMac, + /// The packet does not conform / is malformed + Malformed, + /// Can't determine ethertype to use + MissingEtherType, + /// We don't have state to forward the packet + Unroutable, + /// It was not possible to NAT the packet + NatFailure, + /// The packet is dropped by the dataplane (e.g. due to lack of resources like queue space) + InternalDrop, + /// The packet buffer was delivered by the NF - e.g. for xmit + Delivered, } - bitflags! { #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] struct MetaFlags: u16 { From 1966529ea1f7d6f21ed2262529557a106810d3dc Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Wed, 5 Nov 2025 21:03:35 +0000 Subject: [PATCH 02/10] feat(net): display meta flags fixes - remove inline - mark fmt as cold Signed-off-by: Daniel Noland --- net/src/packet/display.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/src/packet/display.rs b/net/src/packet/display.rs index c24e05334..23d30725d 100644 --- a/net/src/packet/display.rs +++ b/net/src/packet/display.rs @@ -245,7 +245,6 @@ impl Display for BridgeDomain { } } -#[inline] fn fmt_metadata_flags(meta: &PacketMeta, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, " Flags:")?; if meta.local() { @@ -272,6 +271,7 @@ fn fmt_metadata_flags(meta: &PacketMeta, f: &mut Formatter<'_>) -> std::fmt::Res writeln!(f) } impl Display for PacketMeta { + #[cold] fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { writeln!(f, " metadata:")?; write!(f, " ")?; From 752e4da6cce1ac11c8240820959a82561692a0af Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 31 Oct 2025 07:28:09 +0000 Subject: [PATCH 03/10] feat(init): rework init crate for memfd handoff This commit takes care of the actual passing of a memfd from dataplane init down into dataplane after a hardware scan and argument parsing. It basically takes care of the TODO from last time. Signed-off-by: Daniel Noland --- Cargo.lock | 8 + args/src/lib.rs | 644 ++++++++++++++++++++++++++++++++++++++--------- init/Cargo.toml | 23 +- init/src/main.rs | 465 ++++++++++++++++++++++++++++++++-- 4 files changed, 1003 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 881bada9f..3a6d116d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1208,16 +1208,24 @@ dependencies = [ name = "dataplane-init" version = "0.1.0" dependencies = [ + "color-eyre", + "command-fds", + "dataplane-args", "dataplane-dpdk-sysroot-helper", "dataplane-hardware", "dataplane-id", "dataplane-sysfs", + "miette", "nix 0.30.1", "procfs", + "rkyv", + "serde", + "serde_yaml_ng", "strum", "strum_macros", "thiserror 2.0.17", "tracing", + "tracing-error", "tracing-subscriber", ] diff --git a/args/src/lib.rs b/args/src/lib.rs index f1f2fcece..0a5ced045 100644 --- a/args/src/lib.rs +++ b/args/src/lib.rs @@ -1,6 +1,90 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Open Network Fabric Authors +//! Argument parsing and configuration management for the Hedgehog dataplane. +//! +//! This crate provides the infrastructure for safely passing configuration from the +//! `dataplane-init` process to the `dataplane` worker process using Linux memory file +//! descriptors (memfd). This approach enables zero-copy deserialization while maintaining +//! strong security guarantees through file sealing mechanisms. +//! +//! # Architecture +//! +//! The configuration flow follows this pattern: +//! +//! 1. **Parent Process (dataplane-init)**: +//! - Parses command-line arguments using [`CmdArgs`] +//! - Converts arguments into a [`LaunchConfiguration`] +//! - Serializes the configuration using `rkyv` for zero-copy deserialization +//! - Writes serialized data to a [`MemFile`] and finalizes it into a [`FinalizedMemFile`] +//! - Computes an [`IntegrityCheck`] (SHA-384 hash) of the configuration +//! - Passes both file descriptors to the child process at known FD numbers +//! +//! 2. **Child Process (dataplane)**: +//! - Inherits the configuration via [`LaunchConfiguration::inherit()`] +//! - Validates the integrity check matches the configuration +//! - Memory-maps the sealed memfd for zero-copy access +//! - Accesses the configuration through the rkyv archive format +//! +//! # Key Types +//! +//! - [`CmdArgs`]: Command-line argument parser using clap +//! - [`LaunchConfiguration`]: Complete dataplane configuration (driver, routing, metrics, etc.) +//! - [`MemFile`]: Mutable memfd wrapper for building configuration +//! - [`FinalizedMemFile`]: Immutable, sealed memfd for safe inter-process sharing +//! - [`IntegrityCheck`]: SHA-384-based validation for configuration authenticity +//! +//! # `FinalizedMemFile` Integrity +//! +//! [`FinalizedMemFile`] provides multiple layers of protection: +//! +//! - **Read-only mode**: File permissions are set to 0o400 (owner read-only) +//! - **Sealed against modification**: `F_SEAL_WRITE`, `F_SEAL_GROW`, `F_SEAL_SHRINK` prevent changes +//! - **Sealed seals**: `F_SEAL_SEAL` prevents removing the seals +//! - **Integrity checking**: SHA-384 hash validates the configuration hasn't been tampered with or corrupted. +//! - (optional) **Close-on-exec**: we have the ability to mark `MemFd` as close-on-exec to prevent accidental leaking +//! to subprocesses. +//! This can't be done in the parent process, but should be done by the child process as soon as the file descriptor +//! is identified. +//! +//! # Example Usage +//! +//! ## Parent Process (init) +//! +//! ```no_run +//! use dataplane_args::{CmdArgs, LaunchConfiguration, Parser, AsFinalizedMemFile}; +//! +//! // Parse command-line arguments +//! let args = CmdArgs::parse(); +//! +//! // Convert to launch configuration +//! let config: LaunchConfiguration = args.try_into() +//! .expect("invalid configuration"); +//! +//! // Serialize and finalize +//! let config_memfd = config.finalize(); +//! let mut config_memfd_mut = config_memfd; +//! let integrity_check = config_memfd_mut.integrity_check(); +//! let check_memfd = integrity_check.finalize(); +//! +//! // ... pass to child process ... +//! ``` +//! +//! ## Child Process (dataplane) +//! +//! ```no_run +//! use dataplane_args::LaunchConfiguration; +//! +//! // Inherit configuration from parent +//! let config = LaunchConfiguration::inherit(); +//! +//! // Configuration is now available with zero-copy access +//! println!("Using driver: {:?}", config.driver); +//! ``` + +#![deny(unsafe_code, clippy::pedantic)] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + pub use clap::Parser; use hardware::pci::address::InvalidPciAddress; use hardware::pci::address::PciAddress; @@ -12,7 +96,6 @@ use std::borrow::Borrow; use std::fmt::Display; use std::io::{Read, Seek, SeekFrom, Write}; use std::net::SocketAddr; -use std::num::NonZero; use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::path::PathBuf; use std::str::FromStr; @@ -57,26 +140,22 @@ impl FromStr for PortArg { impl FromStr for InterfaceArg { type Err = String; fn from_str(input: &str) -> Result { - match input.split_once('=') { - Some((first, second)) => { - let interface = InterfaceName::try_from(first) - .map_err(|e| format!("Bad interface name: {e}"))?; - - let port = PortArg::from_str(second)?; - Ok(InterfaceArg { - interface, - port: Some(port), - }) - } - // this branch will go away - None => { - let interface = InterfaceName::try_from(input) - .map_err(|e| format!("Bad interface name: {e}"))?; - Ok(InterfaceArg { - interface, - port: None, - }) - } + if let Some((first, second)) = input.split_once('=') { + let interface = InterfaceName::try_from(first) + .map_err(|e| format!("Bad interface name: {e}"))?; + + let port = PortArg::from_str(second)?; + Ok(InterfaceArg { + interface, + port: Some(port), + }) + } else { + let interface = InterfaceName::try_from(input) + .map_err(|e| format!("Bad interface name: {e}"))?; + Ok(InterfaceArg { + interface, + port: None, + }) } } } @@ -87,40 +166,50 @@ use bytecheck::CheckBytes; use nix::fcntl::{FcntlArg, FdFlag}; use nix::{fcntl::SealFlag, sys::memfd::MFdFlags}; +/// Default path to the dataplane's control plane unix socket. +/// +/// This socket is used by FRR to send route update messages to the dataplane process. pub const DEFAULT_DP_UX_PATH: &str = "/var/run/frr/hh/dataplane.sock"; + +/// Default path to the dataplane's CLI socket. +/// +/// This socket is used to accept connections from the dataplane CLI tool for +/// runtime inspection and control. pub const DEFAULT_DP_UX_PATH_CLI: &str = "/var/run/dataplane/cli.sock"; + +/// Default path to the FRR agent socket. +/// +/// This socket is used to connect to the FRR agent that controls FRR +/// configuration reloads. pub const DEFAULT_FRR_AGENT_PATH: &str = "/var/run/frr/frr-agent.sock"; /// A type wrapper around [`std::fs::File`] which is reserved to describe linux [memfd] files. /// -/// Our main use case for these files is passing ephemeral, launch-time configuration data to -/// the dataplane process from the dataplane-init process. -/// -/// # Note +/// Memory file descriptors are anonymous, file-like objects that exist only in memory +/// and are not backed by any filesystem. They are particularly useful for passing +/// ephemeral configuration data between processes. /// -/// [`MemFile`] is intended for mutation. Use the [`MemFile::finalize`] method to create a [`FinalizedMemFile`] to pass -/// to child processes. +/// # Mutability /// -/// [memfd]: https://man7.org/linux/man-pages/man2/memfd_create.2.html +/// [`MemFile`] is intended for mutation during construction. Once you've written your +/// data, use [`MemFile::finalize`] to create a [`FinalizedMemFile`] which provides +/// strong immutability guarantees suitable for inter-process sharing. #[derive(Debug)] pub struct MemFile(std::fs::File); -/// A type wrapper around [`MemFile`] for memfd files which are emphatically NOT intended for any kind of data mutation -/// ever again. +/// An immutable, sealed memory file descriptor that cannot be modified. /// /// Multiple protections are in place to deny all attempts to mutate the memory contents of these files. /// These protections make this type of file suitable for as-safe-as-practical zero-copy deserialization of data /// structure serialized by one process and given to a different process. /// -/// Protections include both basic linux DAC read only mode, as well as write, truncation, and extension sealing, as well -/// as sealing the seals in place to prevent their removal. +/// # Integrity Properties /// -/// # Note +/// Multiple protections are enforced to prevent any data mutation: /// /// If these files contain secrets (or even if they don't), it is usually best to mark the file as close-on-exec to /// further mitigate opportunities for the data to be corrupted / mutated. /// This task, by its nature, can not be done by the parent process (or the child would not get the file descriptor). -/// /// As a consequence, this marking step should be taken as soon as the file is received by the child process. /// The (unsafe) method [`FinalizedMemFile::from_fd`] takes this action automatically, and is the recommended way to /// receive and read the file from child processes. @@ -128,6 +217,11 @@ pub struct FinalizedMemFile(MemFile); impl MemFile { /// Create a new, blank [`MemFile`]. + /// + /// # Panics + /// + /// Panics if the operating system is unable to allocate an in-memory file descriptor. + #[must_use] pub fn new() -> MemFile { let id: id::Id = id::Id::new(); let descriptor = @@ -151,13 +245,14 @@ impl MemFile { /// /// # Panics /// - /// This method is intended for use only during early process initialization and make no attempt to recover from + /// This method is intended for use only during early process initialization and makes no attempt to recover from /// errors. /// /// This method will panic if /// /// 1. The file can not be modified to exclude write operations (basically chmod 400) /// 2. if the file can not be sealed against extension, truncation, mutation, and any attempt to remove the seals. + #[must_use] pub fn finalize(self) -> FinalizedMemFile { let mut this = self; // mark the file as read only @@ -259,11 +354,41 @@ impl From for FinalizedMemFile { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub enum GrpcAddress { + /// TCP socket address (IP address and port) Tcp(SocketAddr), + /// Unix domain socket path UnixSocket(String), } -/// Configuration for the driver used by the dataplane to process packets. +/// Configuration for the packet processing driver used by the dataplane. +/// +/// The dataplane supports two packet processing backends: +/// +/// - **DPDK (Data Plane Development Kit)**: High-performance userspace driver for +/// specialized network hardware. Provides kernel-bypass networking with direct access +/// to NIC hardware via PCI. +/// +/// - **Kernel**: Standard Linux kernel networking stack. Uses traditional network +/// interfaces and kernel packet processing. +/// +/// # Choosing a Driver +/// +/// - Use **DPDK** for maximum performance on supported hardware, typically in production +/// environments with dedicated NICs. +/// - Use **Kernel** for development, testing, or environments without DPDK-compatible +/// hardware. +/// +/// # Example +/// +/// ``` +/// use dataplane_args::{DriverConfigSection, KernelDriverConfigSection}; +/// use net::interface::InterfaceName; +/// +/// // Kernel driver configuration +/// let kernel_config = DriverConfigSection::Kernel(KernelDriverConfigSection { +/// interfaces: vec![], +/// }); +/// ``` #[derive( Debug, PartialEq, @@ -278,10 +403,28 @@ pub enum GrpcAddress { #[serde(rename_all = "snake_case")] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub enum DriverConfigSection { + /// DPDK userspace driver configuration Dpdk(DpdkDriverConfigSection), + /// Linux kernel driver configuration Kernel(KernelDriverConfigSection), } +/// Description of a network device by its bus address. +/// +/// Currently supports PCI-addressed devices, which is the standard addressing +/// scheme for NICs in modern systems. +/// +/// # Example +/// +/// ``` +/// use dataplane_args::NetworkDeviceDescription; +/// use hardware::pci::address::PciAddress; +/// +/// // PCI device at bus 0000:01:00.0 +/// let device = NetworkDeviceDescription::Pci( +/// PciAddress::try_from("0000:01:00.0").unwrap() +/// ); +/// ``` #[derive( Debug, Ord, @@ -298,6 +441,7 @@ pub enum DriverConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub enum NetworkDeviceDescription { + /// PCI-addressed network device Pci(hardware::pci::address::PciAddress), Kernel(InterfaceName), } @@ -315,23 +459,36 @@ impl Display for NetworkDeviceDescription { } } -pub type KiB = NonZero; - -#[derive( - Debug, - Default, - serde::Serialize, - serde::Deserialize, - rkyv::Serialize, - rkyv::Deserialize, - rkyv::Archive, -)] -pub enum WorkerStackSize { - #[default] - Default, - Size(KiB), -} - +/// Configuration for the DPDK (Data Plane Development Kit) driver. +/// +/// DPDK provides kernel-bypass networking for high-performance packet processing. +/// This configuration specifies which NICs to use and how to initialize the DPDK +/// Environment Abstraction Layer (EAL). +/// +/// # Fields +/// +/// - `use_nics`: List of network devices (by PCI address) to bind to DPDK +/// - `eal_args`: Arguments passed to DPDK's EAL initialization (e.g., core masks, +/// memory configuration, device allowlists) +/// +/// # Example +/// +/// ``` +/// use dataplane_args::{DpdkDriverConfigSection, NetworkDeviceDescription}; +/// use hardware::pci::address::PciAddress; +/// +/// let config = DpdkDriverConfigSection { +/// use_nics: vec![ +/// NetworkDeviceDescription::Pci( +/// PciAddress::try_from("0000:01:00.0").unwrap() +/// ) +/// ], +/// eal_args: vec![ +/// "--main-lcore".to_string(), +/// "2".to_string(), +/// ], +/// }; +/// ``` #[derive( Debug, PartialEq, @@ -345,10 +502,35 @@ pub enum WorkerStackSize { )] #[rkyv(attr(derive(Debug, PartialEq, Eq)))] pub struct DpdkDriverConfigSection { + /// Network devices to use with DPDK (identified by PCI address) pub use_nics: Vec, + /// DPDK EAL (Environment Abstraction Layer) initialization arguments pub eal_args: Vec, } +/// Configuration for the Linux kernel networking driver. +/// +/// Uses the standard Linux kernel network stack for packet processing. +/// This is suitable for development, testing, or environments without +/// DPDK-compatible hardware. +/// +/// # Fields +/// +/// - `interfaces`: List of kernel network interface names to manage +/// (e.g., `eth0`, `ens3`) +/// +/// # Example +/// +/// ``` +/// use dataplane_args::KernelDriverConfigSection; +/// use net::interface::InterfaceName; +/// +/// let config = KernelDriverConfigSection { +/// interfaces: vec![ +/// InterfaceName::try_from("eth0").unwrap(), +/// ], +/// }; +/// ``` #[derive( Debug, PartialEq, @@ -362,10 +544,14 @@ pub struct DpdkDriverConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct KernelDriverConfigSection { + /// Kernel network interfaces to manage pub interfaces: Vec, } -/// Configuration for the command line interface of the dataplane +/// Configuration for the dataplane's command-line interface (CLI). +/// +/// Specifies where the CLI server listens for connections from CLI clients +/// that want to inspect or control the running dataplane. #[derive( Debug, PartialEq, @@ -379,10 +565,15 @@ pub struct KernelDriverConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct CliConfigSection { + /// Unix socket path for CLI connections pub cli_sock_path: String, } -/// Configuration which defines how metrics are collected from the dataplane. +/// Configuration for metrics collection and export. +/// +/// Defines the HTTP endpoint where Prometheus-compatible metrics are exposed. +/// Metrics include packet counters, latency statistics, and other operational +/// telemetry. #[derive( Debug, PartialEq, @@ -396,6 +587,7 @@ pub struct CliConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct MetricsConfigSection { + /// Socket address (IP and port) where metrics HTTP endpoint listens pub address: SocketAddr, } @@ -413,10 +605,15 @@ pub struct MetricsConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct TracingConfigSection { + /// Display options for trace output pub show: TracingShowSection, + /// Tracing configuration string (e.g., "default=info,nat=debug") pub config: Option, // TODO: stronger typing on this config? } +/// Display option for trace metadata elements. +/// +/// Controls whether specific metadata is shown in trace output. #[derive( Debug, Default, @@ -433,11 +630,16 @@ pub struct TracingConfigSection { #[serde(rename_all = "snake_case")] #[repr(u8)] pub enum TracingDisplayOption { + /// Hide this metadata element #[default] Hide, + /// Show this metadata element Show, } +/// Display configuration for trace metadata. +/// +/// Controls which metadata elements are included in trace output. #[derive( Debug, Default, @@ -452,11 +654,16 @@ pub enum TracingDisplayOption { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct TracingShowSection { + /// Whether to display span/event tags pub tags: TracingDisplayOption, + /// Whether to display target module paths pub targets: TracingDisplayOption, } -/// Configuration which defines the interaction between the dataplane and the routing control plane. +/// Configuration for routing control plane integration. +/// +/// Defines how the dataplane communicates with FRR (Free Range Routing) and +/// related routing components. #[derive( Debug, PartialEq, @@ -469,12 +676,16 @@ pub struct TracingShowSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct RoutingConfigSection { + /// Unix socket path for receiving route updates from FRR pub control_plane_socket: String, + /// Unix socket path for FRR agent communication pub frr_agent_socket: String, } -/// Configuration section for the parameters of the dynamic configuration server which supplies -/// updated configuration to the dataplane at runtime. +/// Configuration for the dynamic configuration server. +/// +/// The configuration server provides runtime configuration updates to the dataplane +/// via gRPC. This allows modifying dataplane behavior without restarting the process. #[derive( Debug, PartialEq, @@ -488,19 +699,32 @@ pub struct RoutingConfigSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct ConfigServerSection { + /// gRPC server address (TCP or Unix socket) pub address: GrpcAddress, } -/// The configuration of the dataplane. +/// Complete dataplane launch configuration. +/// +/// This structure contains all configuration parameters needed to initialize and run +/// the dataplane process. It is typically constructed from command-line arguments in +/// the `dataplane-init` process, then serialized and passed to the `dataplane` worker +/// process via sealed memory file descriptors. +/// +/// # Architecture +/// +/// The configuration flow: +/// +/// 1. **Init Process**: Parses [`CmdArgs`] and converts to [`LaunchConfiguration`] +/// 2. **Serialization**: Configuration is serialized using `rkyv` for zero-copy access +/// 3. **Transfer**: Passed via sealed memfd to the worker process +/// 4. **Worker Process**: Calls [`LaunchConfiguration::inherit()`] to access the config /// -/// This structure should be computed from the command line arguments supplied to the dataplane-init. // TODO: implement bytecheck::Validate in addition to CheckBytes on all components of the launch config. #[derive( Debug, PartialEq, Eq, serde::Serialize, - serde::Deserialize, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, @@ -508,26 +732,61 @@ pub struct ConfigServerSection { )] #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct LaunchConfiguration { + /// Dynamic configuration server settings pub config_server: ConfigServerSection, + /// Packet processing driver configuration pub driver: DriverConfigSection, + /// CLI server configuration pub cli: CliConfigSection, + /// Routing control plane integration pub routing: RoutingConfigSection, + /// Logging and tracing configuration pub tracing: TracingConfigSection, + /// Metrics collection configuration pub metrics: MetricsConfigSection, } impl LaunchConfiguration { + /// Standard file descriptor number for the integrity check memfd. + /// + /// The parent process must pass the integrity check (SHA-384 hash) file at this + /// file descriptor number. pub const STANDARD_INTEGRITY_CHECK_FD: RawFd = 30; + + /// Standard file descriptor number for the configuration memfd. + /// + /// The parent process must pass the serialized configuration file at this + /// file descriptor number. pub const STANDARD_CONFIG_FD: RawFd = 40; - /// Inherit a launch configuration from your parent process (assuming it correctly specified one). + /// Inherit the launch configuration from the parent process. + /// + /// This method is called by the dataplane worker process to receive its configuration + /// from the init process. It expects two sealed memory file descriptors at the standard + /// FD numbers ([`STANDARD_INTEGRITY_CHECK_FD`](Self::STANDARD_INTEGRITY_CHECK_FD) and + /// [`STANDARD_CONFIG_FD`](Self::STANDARD_CONFIG_FD)). /// - /// This method assumes that agreed upon file descriptor numbers are assigned by the parent. + /// # Process + /// + /// 1. Receives integrity check and configuration file descriptors + /// 2. Validates the SHA-384 hash matches the configuration + /// 3. Memory-maps the configuration for zero-copy access + /// 4. Validates the archived data structure (alignment, bounds, enum variants) + /// 5. Deserializes the configuration /// /// # Panics /// - /// This method is intended for use at system startup and makes little attempt to recover from errors. - /// This method will panic if the configuration is missing, invalid, or otherwise impossible to manipulate. + /// This method is designed for early process initialization and will panic if: + /// + /// - File descriptors are missing or invalid + /// - Integrity check validation fails (hash mismatch) + /// - Memory mapping fails + /// - Archived data is misaligned or has invalid size + /// - Deserialization fails (corrupt or invalid data) + /// + /// These panics are intentional as the dataplane cannot start without valid configuration. + #[must_use] + #[allow(unsafe_code)] // no-escape from unsafety in this function as it involves constraints the compiler can't see pub fn inherit() -> LaunchConfiguration { let integrity_check_fd = unsafe { OwnedFd::from_raw_fd(Self::STANDARD_INTEGRITY_CHECK_FD) }; let launch_configuration_fd = unsafe { OwnedFd::from_raw_fd(Self::STANDARD_CONFIG_FD) }; @@ -546,28 +805,23 @@ impl LaunchConfiguration { .wrap_err("failed to memory map launch configuration") .unwrap(); - { - // VERY IMPORTANT: we must check for unaligned pointer here or risk undefined behavior. - // There is absolutely no reason to keep this pointer alive past this scope. - // Don't let it escape the scope (even if it is aligned). - const EXPECTED_ALIGNMENT: usize = std::mem::align_of::(); - const { - if !EXPECTED_ALIGNMENT.is_power_of_two() { - panic!("nonsense alignment computed for ArchivedLaunchConfiguration"); - } - } - let potentially_invalid_pointer = - launch_config_memmap.as_ptr() as *const ArchivedLaunchConfiguration; - if !potentially_invalid_pointer.is_aligned() { - panic!( - "invalid alignment for ArchivedLaunchConfiguration found in inherited memfd" - ); - } - } + // VERY IMPORTANT: we must check for unaligned pointer here or risk undefined behavior. + + // deactivate the lint because checking for alignment is _exactly_ what we are doing here + #[allow(clippy::cast_ptr_alignment)] + let is_aligned = launch_config_memmap + .as_ptr() + .cast::() + .is_aligned(); + assert!( + is_aligned, + "invalid alignment for ArchivedLaunchConfiguration found in inherited memfd" + ); - if launch_config_memmap.as_ref().len() < size_of::() { - panic!("invalid size for inherited memfd"); - } + assert!( + launch_config_memmap.as_ref().len() >= size_of::(), + "invalid size for inherited memfd" + ); // we slightly abuse the access method here just to get byte level validation. // The actual objective here is to ensure all enums are valid and that all pointers point within the @@ -602,14 +856,20 @@ impl AsFinalizedMemFile for LaunchConfiguration { .into_diagnostic() .wrap_err("failed to write dataplane configuration to memfd") .unwrap(); - // seal the memfd memfd.finalize() } } -/// Trait for data which may be "frozen" into a [`FinalizedMemFile`] +/// Trait for data that can be serialized into a sealed memory file descriptor. +/// +/// Types implementing this trait can be converted into a [`FinalizedMemFile`], +/// which provides strong immutability guarantees suitable for inter-process +/// communication. pub trait AsFinalizedMemFile { - /// Consume self and convert it into a [`FinalizedMemFile`] + /// Serialize and seal this data into a [`FinalizedMemFile`]. + /// + /// The returned file is immutable and suitable for zero-copy deserialization + /// in other processes. fn finalize(&self) -> FinalizedMemFile; } @@ -640,6 +900,7 @@ impl FinalizedMemFile { /// /// You should generally only call this method as when you are about to hand the file to a child process which is /// expecting such a file descriptor. + #[must_use] pub fn to_owned_fd(self) -> OwnedFd { OwnedFd::from(self.0.0) } @@ -666,6 +927,7 @@ impl FinalizedMemFile { /// 7. panics if the provided memfd can not be `seek`ed to the start of the file (very unlikely) /// 8. panics if the provided memfd can not be marked as close-on-exec (very unlikely) #[instrument(level = "debug", skip(fd))] + #[allow(unsafe_code)] // external contract documented and checked as well as I can for now pub unsafe fn from_fd(fd: OwnedFd) -> FinalizedMemFile { // TODO: is procfs actually mounted at /proc? Are we reading the correct file. Annoying to fix this properly. let os_str = @@ -678,19 +940,21 @@ impl FinalizedMemFile { .map_err(|_| std::io::Error::other("file descriptor readlink returned invalid unicode")) .into_diagnostic() .unwrap(); - if !readlink_result.starts_with("/memfd:") { - panic!("supplied file descriptor is not a memfd: {readlink_result}"); - } + assert!( + readlink_result.starts_with("/memfd:"), + "supplied file descriptor is not a memfd: {readlink_result}" + ); let stat = nix::sys::stat::fstat(fd.as_fd()) .into_diagnostic() .wrap_err("failed to stat memfd") .unwrap(); - if stat.st_mode != 0o100400 { - panic!( - "finalized memfd not in read only mode: given mode is {:o}", - stat.st_mode - ); - } + const EXPECTED_PERMISSIONS: u32 = 0o10_400; // expect read only + sticky bit + assert!( + stat.st_mode == EXPECTED_PERMISSIONS, + "finalized memfd not in read only mode: given mode is {:o}, expected {EXPECTED_PERMISSIONS:o}", + stat.st_mode + ); + let Some(seals) = SealFlag::from_bits( nix::fcntl::fcntl(fd.as_fd(), FcntlArg::F_GET_SEALS) .into_diagnostic() @@ -703,11 +967,10 @@ impl FinalizedMemFile { | SealFlag::F_SEAL_SHRINK | SealFlag::F_SEAL_WRITE | SealFlag::F_SEAL_SEAL; - if !seals.contains(expected_bits) { - panic!( - "missing seal bits on finalized memfd: bits set {seals:?}, bits expected: {expected_bits:?}" - ); - } + assert!( + seals.contains(expected_bits), + "missing seal bits on finalized memfd: bits set {seals:?}, bits expected: {expected_bits:?}" + ); let mut file = std::fs::File::from(fd); file.seek(SeekFrom::Start(0)) .into_diagnostic() @@ -721,7 +984,14 @@ impl FinalizedMemFile { FinalizedMemFile(MemFile(file)) } - /// Validate this file using an [`IntegrityCheck`] serialized into the provided check_file + /// Validate this file using an [`IntegrityCheck`] serialized into the provided `check_file` + /// + /// # Errors + /// + /// Returns an error if + /// + /// 1. unable to read the integrity check file + /// 2. invalid file (checksum mismatch) pub fn validate(&mut self, check_file: FinalizedMemFile) -> Result<(), miette::Report> { let mut check_file = check_file; check_file @@ -750,31 +1020,59 @@ impl FinalizedMemFile { } } +/// Errors that can occur during integrity check validation. #[derive(Debug, thiserror::Error, miette::Diagnostic)] pub enum IntegrityCheckError { + /// The integrity check file has an incorrect size. + /// + /// This typically indicates file corruption or an attempt to use an incompatible + /// hash algorithm. The expected size is [`SHA384_BYTE_LEN`] (48 bytes). #[error( "wrong check file length for hash type; received {0} bytes, expected {SHA384_BYTE_LEN} bytes" )] WrongCheckFileLength(u64), } +/// Size of SHA-384 hash in bytes (384 bits / 8 = 48 bytes). const SHA384_BYTE_LEN: usize = 384 / 8; + +/// Current size of integrity check in bytes (currently SHA-384). const INTEGRITY_CHECK_BYTE_LEN: usize = SHA384_BYTE_LEN; +/// Internal representation of SHA-384 hash bytes. #[repr(transparent)] #[derive(Debug, PartialEq, Eq)] struct Sha384Bytes([u8; SHA384_BYTE_LEN]); -/// An integrity check for a file. +/// Cryptographic integrity check for validating file contents. +/// +/// Currently implemented using SHA-384, providing a cryptographically secure hash +/// that can detect any tampering or corruption of the file contents. The hash +/// implementation may change in future versions without API changes. +/// +/// # Use Cases +/// +/// - Validating configuration files passed between processes +/// - Detecting corruption in sealed memory file descriptors +/// - Ensuring data integrity during process handoff +/// +/// # Security Properties +/// +/// SHA-384 is a member of the SHA-2 family and provides: /// -/// Currently implemented as SHA384, but without any contractual requirement to continue using that hash in the future. +/// - 384-bit (48-byte) hash output +/// - Cryptographic collision resistance +/// - Pre-image resistance (cannot reverse the hash to find the original data) #[must_use] #[derive(Debug, PartialEq, Eq)] pub struct IntegrityCheck { sha384: Sha384Bytes, } -/// A byte array which may hold an [`IntegrityCheck`] +/// Byte array representation of an [`IntegrityCheck`]. +/// +/// This type can hold the serialized form of an integrity check (currently 48 bytes +/// for SHA-384). pub type IntegrityCheckBytes = [u8; INTEGRITY_CHECK_BYTE_LEN]; impl IntegrityCheck { @@ -841,12 +1139,31 @@ impl AsFinalizedMemFile for IntegrityCheck { } } +/// Errors that can occur when parsing or validating command-line arguments. +/// +/// These errors occur during the conversion from [`CmdArgs`] to [`LaunchConfiguration`] +/// when argument values are invalid or inconsistent. #[derive(Debug, thiserror::Error, miette::Diagnostic)] pub enum InvalidCmdArguments { + /// Invalid gRPC address specification. + /// + /// This occurs when: + /// - TCP address cannot be parsed as `IP:PORT` + /// - Unix socket path is not absolute when `--grpc-unix-socket` is set #[error("Illegal grpc address: {0}")] InvalidGrpcAddress(String), // TODO: this should have a stronger error type + + /// Invalid PCI device address format. + /// + /// PCI addresses must follow the format: `domain:bus:device.function` + /// (e.g., `0000:01:00.0`) #[error(transparent)] InvalidPciAddress(#[from] InvalidPciAddress), + + /// Invalid network interface name. + /// + /// Interface names must be valid Linux network interface names + /// (e.g., `eth0`, `ens3`) #[error(transparent)] InvalidInterfaceName(#[from] IllegalInterfaceName), #[error("\"{0}\" is not a valid driver. Must be dpdk or kernel")] @@ -938,13 +1255,15 @@ impl TryFrom for LaunchConfiguration { }, tracing: TracingConfigSection { show: TracingShowSection { - tags: match value.show_tracing_tags() { - true => TracingDisplayOption::Show, - false => TracingDisplayOption::Hide, + tags: if value.show_tracing_tags() { + TracingDisplayOption::Show + } else { + TracingDisplayOption::Hide }, - targets: match value.show_tracing_targets() { - true => TracingDisplayOption::Show, - false => TracingDisplayOption::Hide, + targets: if value.show_tracing_targets() { + TracingDisplayOption::Show + } else { + TracingDisplayOption::Hide }, }, config: value.tracing.clone(), @@ -1071,6 +1390,11 @@ E.g. default=error,all=info,nat=debug will set the default target to error, and } impl CmdArgs { + /// Get the configured driver name. + /// + /// Returns `"dpdk"` if no driver was explicitly specified (the default), + /// otherwise returns the specified driver name (`"dpdk"` or `"kernel"`). + #[must_use] pub fn driver_name(&self) -> &str { match &self.driver { None => "dpdk", @@ -1078,23 +1402,86 @@ impl CmdArgs { } } + /// Check if the `--show-tracing-tags` flag was set. + /// + /// When true, the application should display available tracing tags and exit. + /// + /// # Returns + /// + /// `true` if `--show-tracing-tags` was passed, `false` otherwise. + #[must_use] pub fn show_tracing_tags(&self) -> bool { self.show_tracing_tags } + + /// Check if the `--show-tracing-targets` flag was set. + /// + /// When true, the application should display configurable tracing targets and exit. + /// + /// # Returns + /// + /// `true` if `--show-tracing-targets` was passed, `false` otherwise. + #[must_use] pub fn show_tracing_targets(&self) -> bool { self.show_tracing_targets } + + /// Check if the `--tracing-config-generate` flag was set. + /// + /// When true, the application should generate a tracing configuration string + /// as output and exit. + /// + /// # Returns + /// + /// `true` if `--tracing-config-generate` was passed, `false` otherwise. + #[must_use] pub fn tracing_config_generate(&self) -> bool { self.tracing_config_generate } + + /// Get the tracing configuration string, if provided. + /// + /// Returns the value of the `--tracing` argument, which specifies log levels + /// for different components in the format `target1=level1,target2=level2`. + /// + /// # Returns + /// + /// `Some(&String)` if a tracing configuration was provided, `None` otherwise. + #[must_use] pub fn tracing(&self) -> Option<&String> { self.tracing.as_ref() } + /// Get the number of worker threads for the kernel driver. + /// + /// This value comes from the `--num-workers` argument (default: 1, range: 1-64). + /// + /// # Returns + /// + /// The number of worker threads as a `usize`. + /// + /// # Note + /// + /// This value is only relevant when using the kernel driver. The DPDK driver + /// uses its own threading model configured via EAL arguments. + #[must_use] pub fn kernel_num_workers(&self) -> usize { self.num_workers.into() } - // backwards-compatible, to deprecate + + /// Get the list of kernel network interfaces to use. + /// + /// Returns the interfaces specified via `--interface` arguments. + /// + /// # Returns + /// + /// A vector of interface name strings (e.g., `vec!["eth0", "eth1"]`). + /// + /// # Note + /// + /// This is only used with the kernel driver. For DPDK, use [`Self::allow`] to + /// specify PCI devices instead. + #[must_use] pub fn kernel_interfaces(&self) -> Vec { self.interface .iter() @@ -1107,7 +1494,16 @@ impl CmdArgs { self.interface.iter().cloned() } - /// Get the gRPC server address configuration + /// Parse and validate the gRPC server address configuration. + /// + /// This method interprets the `--grpc-address` and `--grpc-unix-socket` arguments + /// to determine the appropriate gRPC listening address. + /// + /// # Errors + /// + /// Returns an error if: + /// - Unix socket path is not absolute when `--grpc-unix-socket` is set + /// - TCP address cannot be parsed as a valid `IP:PORT` combination pub fn grpc_address(&self) -> Result { // If UNIX socket flag is set, treat the address as a UNIX socket path if self.grpc_unix_socket { @@ -1132,19 +1528,35 @@ impl CmdArgs { } } + /// Get the control plane interface socket path. + /// + /// Returns the path where FRR (Free Range Routing) sends route updates to the dataplane. + #[must_use] pub fn cpi_sock_path(&self) -> String { self.cpi_sock_path.clone() } + /// Get the CLI socket path. + /// + /// Returns the path where the dataplane CLI server listens for client connections. + #[must_use] pub fn cli_sock_path(&self) -> String { self.cli_sock_path.clone() } + /// Get the FRR agent socket path. + /// + /// Returns the path to connect to the FRR agent that controls FRR configuration reloads. + #[must_use] pub fn frr_agent_path(&self) -> String { self.frr_agent_path.clone() } - /// Get the metrics bind address, returns None if metrics are disabled + /// Get the Prometheus metrics HTTP endpoint address. + /// + /// Returns the socket address (IP and port) where the dataplane exposes + /// Prometheus-compatible metrics for scraping. + #[must_use] pub fn metrics_address(&self) -> SocketAddr { self.metrics_address } diff --git a/init/Cargo.toml b/init/Cargo.toml index 6ac33af3c..175903cb7 100644 --- a/init/Cargo.toml +++ b/init/Cargo.toml @@ -2,23 +2,32 @@ name = "dataplane-init" version = "0.1.0" edition = "2024" -publish = false license = "Apache-2.0" +publish = false [dependencies] + # internal -hardware = { workspace = true, features = ["serde", "scan"] } -id = { workspace = true } -sysfs = { workspace = true } +args = { workspace = true, features = [] } +hardware = { workspace = true, features = ["scan", "serde"] } +id = { workspace = true, features = [] } # external -nix = { workspace = true, features = ["mount", "fs"] } +color-eyre = { workspace = true, features = ["capture-spantrace"] } +command-fds = { workspace = true, features = [] } +miette = { workspace = true, features = ["derive", "fancy"] } +nix = { workspace = true, features = ["fs", "mount"] } procfs = { workspace = true, features = [] } +rkyv = { workspace = true, features = ["alloc", "bytecheck", "std"] } +serde = { workspace = true, features = ["derive", "std"] } +serde_yaml_ng = { workspace = true, default-features = false, features = [] } strum = { workspace = true, features = ["derive"] } strum_macros = { workspace = true, features = [] } -thiserror = { workspace = true } +sysfs = { workspace = true, features = [] } +thiserror = { workspace = true, features = ["std"] } tracing = { workspace = true, features = ["attributes"] } -tracing-subscriber = { workspace = true, features = ["fmt"] } +tracing-error = { workspace = true, features = ["traced-error"]} +tracing-subscriber = { workspace = true, features = ["ansi", "fmt"] } [dev-dependencies] # internal diff --git a/init/src/main.rs b/init/src/main.rs index e32833636..55f0ab854 100644 --- a/init/src/main.rs +++ b/init/src/main.rs @@ -2,23 +2,460 @@ // Copyright Open Network Fabric Authors #![doc = include_str!("../README.md")] -#![deny(clippy::pedantic, missing_docs)] +// #![deny(clippy::pedantic, missing_docs)] // TEMP: don't merge till uncommented -use hardware::nic::{BindToVfioPci, PciNic}; +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Display, + os::unix::process::CommandExt, +}; -fn main() { - tracing_subscriber::fmt() - .with_ansi(false) +use args::{AsFinalizedMemFile, LaunchConfiguration, NetworkDeviceDescription, Parser}; +use command_fds::{CommandFdExt, FdMapping}; +use hardware::{ + NodeAttributes, + nic::{BindToVfioPci, PciNic}, + support::SupportedDevice, +}; +use miette::{Context, IntoDiagnostic}; +use tracing::{debug, error, info, trace, warn}; +use tracing_subscriber::layer::SubscriberExt; + +fn early_init() { + let subscriber = tracing_subscriber::fmt() + .with_ansi(true) + .with_thread_ids(true) .with_file(true) .with_level(true) + .with_max_level(tracing::Level::DEBUG) .with_line_number(true) - .init(); - // TODO: proper argument parsing - // -- hack add a real command line parser - let mut args = std::env::args().skip(1); - // -- end hack - // TODO: fix unwraps in the next PR. These can't be properly addressed before the arg parser is done. - let address = hardware::pci::address::PciAddress::try_from(args.next().unwrap()).unwrap(); - let mut device = PciNic::new(address).unwrap(); - device.bind_to_vfio_pci().unwrap(); + .with_test_writer() + .finish() + .with(tracing_error::ErrorLayer::default()); + tracing::subscriber::set_global_default(subscriber) + .into_diagnostic() + .wrap_err("failed to set tracing subscriber") + .unwrap(); + info!("tracing initialized"); + color_eyre::install().unwrap(); + debug!("color-eyre enabled"); +} + +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +#[diagnostic(code(dataplane::initialization::error))] +pub enum InitializationError { + #[error("no network devices specified for use in the dataplane")] + NoDevicesSpecified, + #[error("no network devices available for use in the dataplane")] + NoDevicesAvailable, + #[error(transparent)] + DevicesNotFound(#[from] DevicesNotFound), + #[error(transparent)] + DevicesNotSupported(#[from] DevicesNotSupported), +} + +#[derive(Debug, thiserror::Error, miette::Diagnostic)] +pub struct DevicesNotFound { + missing: BTreeSet, +} + +impl std::fmt::Display for DevicesNotFound { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let devices_str = self + .missing + .iter() + .map(|elem| elem.to_string()) + .collect::>() + .join(", "); + write!(f, "missing devices: {devices_str}") + } +} + +impl DevicesNotFound { + pub(crate) fn new<'a>( + missing: impl Iterator, + ) -> DevicesNotFound { + Self { + missing: missing.cloned().collect(), + } + } +} + +#[derive(Debug, thiserror::Error, miette::Diagnostic, serde::Serialize)] +#[error("unsupported devices\n---\n{0}")] +pub struct DevicesNotSupported(String); + +impl DevicesNotSupported { + pub(crate) fn new(not_supported: &BTreeMap) -> DevicesNotSupported { + let yaml = serde_yaml_ng::to_string(not_supported) + .into_diagnostic() + .wrap_err("failed to serialize unsupported devices list") + .unwrap(); + DevicesNotSupported(yaml) + } +} + +pub struct DeviceSearch { + /// Devices which we were instructed to use. + /// These are user supplied _requests_ and may include unsupported devices or devices which we can't find in a + /// hardware scan. + requested: BTreeSet, + /// Total hardware scan + scan: hardware::Node, +} + +impl DeviceSearch { + #[tracing::instrument(level = "info", skip(requested))] + pub fn new<'a>(requested: impl Iterator) -> DeviceSearch { + let requested: BTreeSet<_> = requested.cloned().collect(); + info!("scanning hardware for devices {requested:?}"); + let scan = hardware::Node::scan_all(); + DeviceSearch { requested, scan } + } + + /// Walk the full system hardware scan and collect the list of devices which match the requested devices + /// + /// # Note + /// + /// This method can and will return devices which we can find in the hardware but which we do not support. + #[tracing::instrument(level = "info", skip(self))] + pub fn matching(&self) -> BTreeMap { + self.scan + .iter() + .filter_map(|node| { + if let Some(attributes) = node.attributes() { + match attributes { + hardware::NodeAttributes::Pci(attributes) => { + let target = NetworkDeviceDescription::Pci(attributes.address()); + if self.requested.contains(&target) { + Some((target, node)) + } else { + None + } + } + _ => None, + } + } else { + None + } + }) + .collect() + } + + /// Walk the full system hardware scan and collect the list of devices which are requested but which can not be + /// found in the scan at all. + pub fn missing(&self) -> BTreeSet { + let matching: BTreeSet<_> = self.matching().keys().cloned().collect(); + self.requested.difference(&matching).cloned().collect() + } + + /// Walk a full system hardware scan and collect the list of supportable network devices. + /// + /// # Note + /// + /// The result can and may include devices which were not requested by the user. + /// + /// # Panics + /// + /// Panics if unable to serialize the device list as yaml. + #[tracing::instrument(level = "info", skip(self))] + pub fn supportable(&self) -> BTreeMap { + // hardware which we see and which we could use for packet processing if requested to do so. + let supportable_hardware: BTreeMap = self.scan.iter().filter_map(|node| { + let attributes = node.attributes()?; + match attributes { + hardware::NodeAttributes::Pci(attributes) => { + match SupportedDevice::try_from((attributes.vendor_id(), attributes.device_id())) { + Ok(supported) => { + let yaml_description_of_node = serde_yaml_ng::to_string(node) + .into_diagnostic() + .wrap_err("failed to construct yaml description of hardware node") + .unwrap(); + info!( + "found supported device {supported}:\n---\n{yaml_description_of_node}" + ); + let nic_desc = NetworkDeviceDescription::Pci(attributes.address()); + Some((nic_desc, node)) + } + Err(unsupported) => { + trace!("found unsupported pci device device {unsupported}"); + None + } + } + } + _ => None, + } + }).collect(); + + let supportable_hardware_list_yaml = serde_yaml_ng::to_string(&supportable_hardware) + .into_diagnostic() + .wrap_err("failed to serialize supportable_hardware list to yaml") + .unwrap(); + + info!( + "supportable hardware found:\n---\n{}", + supportable_hardware_list_yaml + ); + supportable_hardware + } + + /// Walk the hardware scan and find the devices which are both supported and requested. + pub fn scheduled_for_use(&self) -> BTreeMap { + let matching: BTreeSet<_> = self.matching().keys().cloned().collect(); + let supportable = self.supportable(); + let supportable_keys: BTreeSet<_> = supportable.keys().cloned().collect(); + let to_use: BTreeSet<_> = matching.intersection(&supportable_keys).collect(); + supportable + .iter() + .filter_map(|(desc, node)| { + if to_use.contains(desc) { + Some((desc.clone(), *node)) + } else { + None + } + }) + .collect() + } + + pub fn requested_but_not_supported(&self) -> BTreeMap { + let matching = self.matching(); + let matching_keys: BTreeSet<_> = matching.keys().cloned().collect(); + let supportable = self.supportable(); + let supportable_keys: BTreeSet<_> = supportable.keys().cloned().collect(); + let unsupported: BTreeSet<_> = matching_keys + .difference(&supportable_keys) + .cloned() + .collect(); + matching + .iter() + .filter_map(|(desc, &node)| { + if unsupported.contains(desc) { + // TODO: conversion to string is required to prevent serialization failure downstream. + // I don't understand why this is. It may be a bug in serde. + Some((desc.to_string(), node)) + } else { + None + } + }) + .collect() + } + + pub fn report(&self) -> NetworkDeviceSearchReport<'_> { + NetworkDeviceSearchReport::new(self) + } +} + +#[derive(Debug, serde::Serialize)] +pub struct NetworkDeviceSearchReport<'search> { + requested: BTreeSet, + matching: BTreeMap, + missing: BTreeSet, + supportable: BTreeMap, + scheduled_for_use: BTreeMap, + // TODO: I don't understand why this needs to be a string. The requested_but_not_supported method crashes otherwise + requested_but_not_supported: BTreeMap, +} + +pub enum StartupViability { + Clean, + Warn(Vec), + Fail(Vec), +} + +impl<'search> NetworkDeviceSearchReport<'search> { + fn new(search: &'search DeviceSearch) -> NetworkDeviceSearchReport<'search> { + NetworkDeviceSearchReport { + requested: search.requested.clone(), + matching: search + .matching() + .iter() + .map(|(k, &v)| (k.to_string(), v)) + .collect(), + missing: search.missing(), + supportable: search + .supportable() + .iter() + .map(|(k, &v)| (k.to_string(), v)) + .collect(), + requested_but_not_supported: search.requested_but_not_supported(), + scheduled_for_use: search + .scheduled_for_use() + .iter() + .map(|(k, &v)| (k.to_string(), v)) + .collect(), + } + } + + #[tracing::instrument(level = "info", skip(self))] + fn viability(&self) -> StartupViability { + if self.requested.is_empty() { + return StartupViability::Fail(vec![InitializationError::NoDevicesSpecified]); + } + if self.scheduled_for_use.is_empty() { + return StartupViability::Fail(vec![InitializationError::NoDevicesAvailable]); + } + let missing = DevicesNotFound::new(self.missing.iter()); + let not_supported = DevicesNotSupported::new(&self.requested_but_not_supported); + if missing.missing.is_empty() && not_supported.0.is_empty() { + info!("all requested network devices supported and detected"); + StartupViability::Clean + } else { + let mut ret = vec![]; + if !missing.missing.is_empty() { + ret.push(missing.into()); + } + if !not_supported.0.is_empty() { + ret.push(not_supported.into()); + } + StartupViability::Warn(ret) + } + } +} + +impl Display for NetworkDeviceSearchReport<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let as_yaml = serde_yaml_ng::to_string(self) + .into_diagnostic() + .wrap_err("failed to serialize NetworkDeviceSearchReport") + .unwrap(); + write!(f, "{as_yaml}") + } +} + +pub enum NetworkDeviceDetectionError { + NotFound(BTreeSet), +} + +#[allow(clippy::too_many_lines)] // I don't think breaking up this function helps anything +fn main() { + early_init(); + let main = tracing::span!(tracing::Level::INFO, "main"); + let _main = main.enter(); + debug!( + "received command line arguments: \n {cli_args}\n", + cli_args = std::env::args().collect::>().join("\n ") + ); + let args = args::CmdArgs::parse(); + + // todo: format launch config as yaml + let launch_args_yaml = serde_yaml_ng::to_string(&args) + .into_diagnostic() + .wrap_err("failed to serialize launch cli arguments as yaml") + .unwrap(); + info!("parsed command line arguments as:\n---\n{launch_args_yaml}"); + let launch_config = args::LaunchConfiguration::try_from(args) + .into_diagnostic() + .wrap_err("invalid command line arguments given") + .unwrap(); + let launch_config_yaml = serde_yaml_ng::to_string(&launch_config) + .into_diagnostic() + .wrap_err("failed to serialize launch configuration as yaml") + .unwrap(); + info!("interpreted requested lanunch configuration as\n---\n{launch_config_yaml}"); + + match &launch_config.driver { + args::DriverConfigSection::Dpdk(dpdk_section) => { + let search = DeviceSearch::new(dpdk_section.use_nics.iter()); + let report = search.report(); + let report_yml = serde_yaml_ng::to_string(&report) + .into_diagnostic() + .wrap_err("failed to serialize hardware scan report") + .unwrap(); + info!("hardware scan report:\n---\n{report_yml}"); + match report.viability() { + StartupViability::Clean => {} + StartupViability::Warn(initialization_warnings) => { + for wrn in initialization_warnings { + let diagnostic = Result::<(), _>::Err(wrn).into_diagnostic().unwrap_err(); + warn!("{diagnostic}"); + } + } + StartupViability::Fail(initialization_errors) => { + for err in initialization_errors { + let diagnostic = Result::<(), _>::Err(err) + .into_diagnostic() + .wrap_err("fatal error in dataplane startup") + .unwrap_err(); + error!("{diagnostic}"); + } + error!("dataplane failed to initialize"); + panic!("dataplane failed to initialize"); + } + }; + search + .scheduled_for_use() + .iter() + .for_each(|(desc, &node)| { + let (pci_address, vendor_id, device_id) = match node.attributes() { + Some(NodeAttributes::Pci(pci_attributes)) => { + (pci_attributes.address(), pci_attributes.vendor_id(), pci_attributes.device_id()) + } + Some(_) | None => todo!(), + }; + match SupportedDevice::try_from((vendor_id, device_id)) { + Ok(supported) => match supported { + SupportedDevice::IntelE1000 + | SupportedDevice::IntelX710 + | SupportedDevice::IntelX710VirtualFunction + | SupportedDevice::VirtioNet => match desc { + NetworkDeviceDescription::Pci(pci_address) => { + let mut nic = PciNic::new(*pci_address) + .into_diagnostic() + .wrap_err("failed to find expected network device") + .unwrap(); + nic.bind_to_vfio_pci() + .into_diagnostic() + .wrap_err( + "failed to ensure network device is bound to vfio", + ) + .unwrap(); + } + NetworkDeviceDescription::Kernel(_) => { + // nothing to do here + }, + }, + SupportedDevice::MellanoxConnectX6DX + | SupportedDevice::MellanoxConnectX7 + | SupportedDevice::MellanoxConnectX8 + | SupportedDevice::MellanoxBlueField2 + | SupportedDevice::MellanoxBlueField3 => { + info!("device {supported} ({pci_address}) uses bifurcated driver: not attempting to bind it to vfio"); + }, + }, + Err(_) => unreachable!(), // TODO: restructure to remove this branch + }; + }); + } + args::DriverConfigSection::Kernel(_kernel_section) => todo!(), + }; + + let mut launch_config = launch_config.finalize(); + let integrity_check = launch_config.integrity_check().finalize().to_owned_fd(); + + let launch_config = launch_config.to_owned_fd(); + + let io_err = std::process::Command::new( + "/home/dnoland/code/githedgehog/dataplane/target/x86_64-unknown-linux-gnu/debug/dataplane", + ) + .fd_mappings(vec![ + FdMapping { + parent_fd: integrity_check, + child_fd: LaunchConfiguration::STANDARD_INTEGRITY_CHECK_FD, + }, + FdMapping { + parent_fd: launch_config, + child_fd: LaunchConfiguration::STANDARD_CONFIG_FD, + }, + ]) + .into_diagnostic() + .wrap_err("failed to set file descriptor mapping for child process") + .unwrap() + .env_clear() + .exec(); + + // if we got here then we failed to exec somehow + Result::<(), _>::Err(io_err) + .into_diagnostic() + .wrap_err("failed to exec child process") + .unwrap(); } From 23099a1b5648d6c8319815b2a65daa1a2ba3cad6 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Fri, 31 Oct 2025 07:27:13 +0000 Subject: [PATCH 04/10] feat(dataplane): make datplane consume memfd With this commit the dataplane now consumes and acts on the memfd created by dataplane init from the previous commit. Co-authored-by: Fredi Raspal Signed-off-by: Daniel Noland --- Cargo.lock | 1 + Dockerfile | 3 +- args/src/lib.rs | 134 ++++++++++++++------------ dataplane/Cargo.toml | 1 + dataplane/src/drivers/dpdk.rs | 66 +++++++------ dataplane/src/drivers/kernel.rs | 11 +-- dataplane/src/main.rs | 165 +++++++++++++++----------------- init/src/main.rs | 2 +- 8 files changed, 194 insertions(+), 189 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a6d116d7..e6727ef18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1037,6 +1037,7 @@ dependencies = [ "parking_lot", "pyroscope", "pyroscope_pprofrs", + "rkyv", "serde", "tokio", "tracing", diff --git a/Dockerfile b/Dockerfile index 6a1cd5433..bf82b6730 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM $BASE AS dataplane ARG ARTIFACT ARG ARTIFACT_CLI COPY --link --chown=0:0 "${ARTIFACT}" /dataplane +COPY --link --chown=0:0 "${ARTIFACT}" /dataplane-init COPY --link --chown=0:0 "${ARTIFACT_CLI}" /dataplane-cli WORKDIR / -ENTRYPOINT ["/dataplane"] +ENTRYPOINT ["/dataplane-init"] diff --git a/args/src/lib.rs b/args/src/lib.rs index 0a5ced045..0ffda2376 100644 --- a/args/src/lib.rs +++ b/args/src/lib.rs @@ -100,20 +100,26 @@ use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd, RawFd}; use std::path::PathBuf; use std::str::FromStr; -#[derive(Debug, Clone, PartialEq, serde::Serialize)] -pub enum PortArg { - PCI(PciAddress), // DPDK driver - KERNEL(InterfaceName), // kernel driver -} - -#[derive(Debug, Clone, serde::Serialize)] +#[derive( + CheckBytes, + Clone, + Debug, + Eq, + PartialEq, + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, + serde::Deserialize, + serde::Serialize, +)] +#[rkyv(attr(derive(PartialEq, Eq, Debug)))] #[allow(unused)] pub struct InterfaceArg { pub interface: InterfaceName, - pub port: Option, + pub port: NetworkDeviceDescription, } -impl FromStr for PortArg { +impl FromStr for NetworkDeviceDescription { type Err = String; fn from_str(input: &str) -> Result { let (disc, value) = input @@ -123,12 +129,12 @@ impl FromStr for PortArg { match disc { "pci" => { let pciaddr = PciAddress::try_from(value).map_err(|e| e.to_string())?; - Ok(PortArg::PCI(pciaddr)) + Ok(NetworkDeviceDescription::Pci(pciaddr)) } "kernel" => { let kernelif = InterfaceName::try_from(value) .map_err(|e| format!("Bad kernel interface name: {e}"))?; - Ok(PortArg::KERNEL(kernelif)) + Ok(NetworkDeviceDescription::Kernel(kernelif)) } _ => Err(format!( "Unknown discriminant '{disc}': allowed values are pci|kernel" @@ -141,21 +147,12 @@ impl FromStr for InterfaceArg { type Err = String; fn from_str(input: &str) -> Result { if let Some((first, second)) = input.split_once('=') { - let interface = InterfaceName::try_from(first) - .map_err(|e| format!("Bad interface name: {e}"))?; - - let port = PortArg::from_str(second)?; - Ok(InterfaceArg { - interface, - port: Some(port), - }) + let interface = + InterfaceName::try_from(first).map_err(|e| format!("Bad interface name: {e}"))?; + let port = NetworkDeviceDescription::from_str(second)?; + Ok(InterfaceArg { interface, port }) } else { - let interface = InterfaceName::try_from(input) - .map_err(|e| format!("Bad interface name: {e}"))?; - Ok(InterfaceArg { - interface, - port: None, - }) + Err(format!("invalid interface argument: {input}")) } } } @@ -503,7 +500,7 @@ impl Display for NetworkDeviceDescription { #[rkyv(attr(derive(Debug, PartialEq, Eq)))] pub struct DpdkDriverConfigSection { /// Network devices to use with DPDK (identified by PCI address) - pub use_nics: Vec, + pub interfaces: Vec, /// DPDK EAL (Environment Abstraction Layer) initialization arguments pub eal_args: Vec, } @@ -545,7 +542,7 @@ pub struct DpdkDriverConfigSection { #[rkyv(attr(derive(PartialEq, Eq, Debug)))] pub struct KernelDriverConfigSection { /// Kernel network interfaces to manage - pub interfaces: Vec, + pub interfaces: Vec, } /// Configuration for the dataplane's command-line interface (CLI). @@ -734,6 +731,8 @@ pub struct ConfigServerSection { pub struct LaunchConfiguration { /// Dynamic configuration server settings pub config_server: ConfigServerSection, + /// Number of dataplane worker threads / cores + pub dataplane_workers: usize, /// Packet processing driver configuration pub driver: DriverConfigSection, /// CLI server configuration @@ -744,6 +743,32 @@ pub struct LaunchConfiguration { pub tracing: TracingConfigSection, /// Metrics collection configuration pub metrics: MetricsConfigSection, + /// Profileing collection configuration + pub profiling: ProfilingConfigSection, +} + +#[derive( + Debug, Clone, PartialEq, Eq, serde::Serialize, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, +)] +#[rkyv(attr(derive(PartialEq, Eq, Debug)))] +pub struct ProfilingConfigSection { + /// The URL of the pryroscope url + pub pyroscope_url: Option, + /// Frequency with which we collect stack traces + pub frequency: u32, +} + +impl ProfilingConfigSection { + pub const DEFAULT_FREQUENCY: u32 = 100; +} + +impl Default for ProfilingConfigSection { + fn default() -> Self { + Self { + pyroscope_url: None, + frequency: Self::DEFAULT_FREQUENCY, + } + } } impl LaunchConfiguration { @@ -1191,26 +1216,19 @@ impl TryFrom for LaunchConfiguration { type Error = InvalidCmdArguments; fn try_from(value: CmdArgs) -> Result { - let use_nics: Vec<_> = value - .interfaces() - .map(|x| match x.port { - Some(PortArg::KERNEL(name)) => NetworkDeviceDescription::Kernel(name), - Some(PortArg::PCI(address)) => NetworkDeviceDescription::Pci(address), - None => todo!(), // I am not clear what this case means - }) - .collect(); Ok(LaunchConfiguration { config_server: ConfigServerSection { address: value .grpc_address() .map_err(InvalidCmdArguments::InvalidGrpcAddress)?, }, + dataplane_workers: value.num_workers.into(), driver: match &value.driver { Some(driver) if driver == "dpdk" => { // TODO: adjust command line to specify lcore usage more flexibly in next PR - let eal_args = use_nics - .iter() - .map(|nic| match nic { + let eal_args = value + .interfaces() + .map(|nic| match nic.port { NetworkDeviceDescription::Pci(pci_address) => { Ok(["--allow".to_string(), format!("{pci_address}")]) } @@ -1224,23 +1242,15 @@ impl TryFrom for LaunchConfiguration { .into_iter() .flatten() .collect(); - DriverConfigSection::Dpdk(DpdkDriverConfigSection { use_nics, eal_args }) + + DriverConfigSection::Dpdk(DpdkDriverConfigSection { + interfaces: value.interfaces().collect(), + eal_args, + }) } Some(driver) if driver == "kernel" => { DriverConfigSection::Kernel(KernelDriverConfigSection { - interfaces: use_nics - .iter() - .map(|nic| match nic { - NetworkDeviceDescription::Pci(address) => { - Err(InvalidCmdArguments::UnsupportedByDriver( - UnsupportedByDriver::Kernel(*address), - )) - } - NetworkDeviceDescription::Kernel(interface) => { - Ok(interface.clone()) - } - }) - .collect::>()?, + interfaces: value.interfaces().collect(), }) } Some(other) => Err(InvalidCmdArguments::InvalidDriver(other.clone()))?, @@ -1271,6 +1281,10 @@ impl TryFrom for LaunchConfiguration { metrics: MetricsConfigSection { address: value.metrics_address(), }, + profiling: ProfilingConfigSection { + pyroscope_url: value.pyroscope_url().map(std::string::ToString::to_string), + frequency: ProfilingConfigSection::DEFAULT_FREQUENCY, + }, }) } } @@ -1561,6 +1575,7 @@ impl CmdArgs { self.metrics_address } + #[must_use] pub fn pyroscope_url(&self) -> Option<&url::Url> { self.pyroscope_url.as_ref() } @@ -1575,7 +1590,7 @@ mod tests { use hardware::pci::function::Function; use net::interface::InterfaceName; - use crate::{InterfaceArg, PortArg}; + use crate::{InterfaceArg, NetworkDeviceDescription}; use std::str::FromStr; #[test] @@ -1585,12 +1600,12 @@ mod tests { assert_eq!(spec.interface.as_ref(), "GbEth1.9000"); assert_eq!( spec.port, - Some(PortArg::PCI(PciAddress::new( + NetworkDeviceDescription::Pci(PciAddress::new( Domain::from(0), Bus::new(2), Device::try_from(1).unwrap(), Function::try_from(7).unwrap() - ))) + )) ); // interface + port as kernel interface @@ -1598,16 +1613,9 @@ mod tests { assert_eq!(spec.interface.as_ref(), "GbEth1.9000"); assert_eq!( spec.port, - Some(PortArg::KERNEL( - InterfaceName::try_from("enp2s1.100").unwrap() - )) + NetworkDeviceDescription::Kernel(InterfaceName::try_from("enp2s1.100").unwrap()) ); - // interface only (backwards compatibility) - let spec = InterfaceArg::from_str("GbEth1.9000").unwrap(); - assert_eq!(spec.interface.as_ref(), "GbEth1.9000"); - assert!(spec.port.is_none()); - // bad pci address assert!(InterfaceArg::from_str("GbEth1.9000=pci@0000:02:01").is_err()); diff --git a/dataplane/Cargo.toml b/dataplane/Cargo.toml index a39ab5ec2..32013c942 100644 --- a/dataplane/Cargo.toml +++ b/dataplane/Cargo.toml @@ -34,6 +34,7 @@ pkt-io = { workspace = true } pkt-meta = { workspace = true } pyroscope = { workspace = true } pyroscope_pprofrs = { workspace = true } +rkyv = { workspace = true, features = [] } routing = { workspace = true } serde = { workspace = true, features = ["derive"] } stats = { workspace = true } diff --git a/dataplane/src/drivers/dpdk.rs b/dataplane/src/drivers/dpdk.rs index f5f9b2a57..28157fa77 100644 --- a/dataplane/src/drivers/dpdk.rs +++ b/dataplane/src/drivers/dpdk.rs @@ -14,7 +14,6 @@ use dpdk::queue::tx::{TxQueueConfig, TxQueueIndex}; use dpdk::{dev, eal, socket}; use tracing::{debug, error, info, trace, warn}; -use crate::CmdArgs; use net::buffer::PacketBufferMut; use net::packet::Packet; use pipeline::sample_nfs::Passthrough; @@ -96,34 +95,42 @@ fn start_rte_workers(devices: &[Dev], setup_pipeline: &(impl Sync + Fn() -> DynP info!("Starting RTE Worker on {lcore_id:?}"); WorkerThread::launch(lcore_id, move || { let mut pipeline = setup_pipeline(); - let rx_queue = devices[0] - .rx_queue(RxQueueIndex(u16::try_from(i).unwrap())) - .unwrap(); - let tx_queue = devices[0] - .tx_queue(TxQueueIndex(u16::try_from(i).unwrap())) - .unwrap(); + let queues: Vec<_> = devices + .iter() + .map(|device| { + let rx_queue = device + .rx_queue(RxQueueIndex(u16::try_from(i).unwrap())) + .unwrap(); + let tx_queue = device + .tx_queue(TxQueueIndex(u16::try_from(i).unwrap())) + .unwrap(); + (rx_queue, tx_queue) + }) + .collect(); loop { - let mbufs = rx_queue.receive(); - let pkts = mbufs.filter_map(|mbuf| match Packet::new(mbuf) { - Ok(pkt) => { - debug!("packet: {pkt:?}"); - Some(pkt) - } - Err(e) => { - trace!("Failed to parse packet: {e:?}"); - None - } - }); + for (rx_queue, tx_queue) in &queues { + let mbufs = rx_queue.receive(); + let pkts = mbufs.filter_map(|mbuf| match Packet::new(mbuf) { + Ok(pkt) => { + debug!("packet: {pkt:?}"); + Some(pkt) + } + Err(e) => { + trace!("Failed to parse packet: {e:?}"); + None + } + }); - let pkts_out = pipeline.process(pkts); - let buffers = pkts_out.filter_map(|pkt| match pkt.serialize() { - Ok(buf) => Some(buf), - Err(e) => { - error!("{e:?}"); - None - } - }); - tx_queue.transmit(buffers); + let pkts_out = pipeline.process(pkts); + let buffers = pkts_out.filter_map(|pkt| match pkt.serialize() { + Ok(buf) => Some(buf), + Err(e) => { + error!("{e:?}"); + None + } + }); + tx_queue.transmit(buffers); + } } }); }); @@ -135,9 +142,10 @@ impl DriverDpdk { pub fn start( args: impl IntoIterator>, setup_pipeline: &(impl Sync + Fn() -> DynPipeline), - ) { - let eal = init_eal(args); + ) -> (Eal, Vec) { + let eal = eal::init(args); let devices = init_devices(&eal); start_rte_workers(&devices, setup_pipeline); + (eal, devices) } } diff --git a/dataplane/src/drivers/kernel.rs b/dataplane/src/drivers/kernel.rs index 55d81d4fe..016927dfe 100644 --- a/dataplane/src/drivers/kernel.rs +++ b/dataplane/src/drivers/kernel.rs @@ -13,7 +13,7 @@ )] use afpacket::sync::RawPacketStream; -use args::{InterfaceArg, PortArg}; +use args::{InterfaceArg, NetworkDeviceDescription}; use concurrency::sync::Arc; use concurrency::thread; @@ -312,11 +312,11 @@ impl DriverKernel { // populate the kernel interface table with the desired interfaces for ifarg in args { match ifarg.port { - Some(PortArg::PCI(_)) => { + NetworkDeviceDescription::Pci(_) => { error!("kernel driver does not support PCI ports"); return Err("kernel driver does not support PCI ports".to_string()); } - Some(PortArg::KERNEL(name)) => { + NetworkDeviceDescription::Kernel(name) => { let Some(ifindex) = get_interface_ifindex(&inventory_kern_ifs, name.as_ref()) else { return Err(format!("Could not find kernel interface {name}")); @@ -326,11 +326,6 @@ impl DriverKernel { return Err(e); } } - _ => { - // TODO: remove Option<> from PortArg as it will need to be mandatory - // after the integration - return Err("Port specification is mandatory".to_string()); - } } } // we allow starting the dataplane without any kernel interface. diff --git a/dataplane/src/main.rs b/dataplane/src/main.rs index 5c13d9fa7..59a4b4979 100644 --- a/dataplane/src/main.rs +++ b/dataplane/src/main.rs @@ -11,7 +11,7 @@ mod statistics; use crate::packet_processor::start_router; use crate::statistics::MetricsServer; -use args::{CmdArgs, Parser}; +use args::{LaunchConfiguration, TracingConfigSection}; use drivers::kernel::DriverKernel; use mgmt::{ConfigProcessorParams, MgmtParams, start_mgmt}; @@ -37,137 +37,128 @@ fn init_logging() { .expect("Setting default loglevel failed"); } -fn process_tracing_cmds(args: &CmdArgs) { - if let Some(tracing) = args.tracing() +fn process_tracing_cmds(cfg: &TracingConfigSection) { + if let Some(tracing) = &cfg.config && let Err(e) = get_trace_ctl().setup_from_string(tracing) { error!("Invalid tracing configuration: {e}"); panic!("Invalid tracing configuration: {e}"); } - if args.show_tracing_tags() { - let out = get_trace_ctl() - .as_string_by_tag() - .unwrap_or_else(|e| e.to_string()); - println!("{out}"); - std::process::exit(0); + match cfg.show.tags { + args::TracingDisplayOption::Hide => {} + args::TracingDisplayOption::Show => { + let out = get_trace_ctl() + .as_string_by_tag() + .unwrap_or_else(|e| e.to_string()); + println!("{out}"); + std::process::exit(0); + } } - if args.show_tracing_targets() { + if cfg.show.targets == args::TracingDisplayOption::Show { let out = get_trace_ctl() .as_string() .unwrap_or_else(|e| e.to_string()); println!("{out}"); std::process::exit(0); } - if args.tracing_config_generate() { - let out = get_trace_ctl() - .as_config_string() - .unwrap_or_else(|e| e.to_string()); - println!("{out}"); - std::process::exit(0); - } + // if args.tracing_config_generate() { + // let out = get_trace_ctl() + // .as_config_string() + // .unwrap_or_else(|e| e.to_string()); + // println!("{out}"); + // std::process::exit(0); + // } } fn main() { + let launch_config = LaunchConfiguration::inherit(); init_logging(); - let args = CmdArgs::parse(); - let agent_running = args.pyroscope_url().and_then(|url| { - match PyroscopeAgent::builder(url.as_str(), "hedgehog-dataplane") - .backend(pprof_backend( - PprofConfig::new() - .sample_rate(100) // Hz - .report_thread_name(), - )) - .build() - { - Ok(agent) => match agent.start() { - Ok(running) => Some(running), + let agent_running = launch_config + .profiling + .pyroscope_url + .as_ref() + .and_then(|url| { + match PyroscopeAgent::builder(url.as_str(), "hedgehog-dataplane") + .backend(pprof_backend( + PprofConfig::new() + .sample_rate(launch_config.profiling.frequency) // Hz + .report_thread_name(), + )) + .build() + { + Ok(agent) => match agent.start() { + Ok(running) => Some(running), + Err(e) => { + error!("Pyroscope start failed: {e}"); + None + } + }, Err(e) => { - error!("Pyroscope start failed: {e}"); + error!("Pyroscope build failed: {e}"); None } - }, - Err(e) => { - error!("Pyroscope build failed: {e}"); - None } - } - }); - process_tracing_cmds(&args); + }); + info!("launch config: {launch_config:?}"); + process_tracing_cmds(&launch_config.tracing); + info!("Starting gateway process..."); let (stop_tx, stop_rx) = std::sync::mpsc::channel(); ctrlc::set_handler(move || stop_tx.send(()).expect("Error sending SIGINT signal")) .expect("failed to set SIGINT handler"); - let grpc_addr = match args.grpc_address() { - Ok(addr) => addr, - Err(e) => { - error!("Invalid gRPC address configuration: {e}"); - panic!("Management service configuration error. Aborting..."); - } - }; + let grpc_addr = launch_config.config_server.address; /* router parameters */ - let Ok(config) = RouterParamsBuilder::default() - .metrics_addr(args.metrics_address()) - .cli_sock_path(args.cli_sock_path()) - .cpi_sock_path(args.cpi_sock_path()) - .frr_agent_path(args.frr_agent_path()) + let config = match RouterParamsBuilder::default() + .metrics_addr(launch_config.metrics.address) + .cli_sock_path(launch_config.cli.cli_sock_path) + .cpi_sock_path(launch_config.routing.control_plane_socket) + .frr_agent_path(launch_config.routing.frr_agent_socket) .build() - else { - error!("Bad router configuration"); - panic!("Bad router configuration"); + { + Ok(config) => config, + Err(e) => { + error!("error building router parameters: {e}"); + panic!("error building router parameters: {e}"); + } }; // start the router; returns control-plane handles and a pipeline factory (Arc<... Fn() -> DynPipeline<_> >) let setup = start_router(config).expect("failed to start router"); - MetricsServer::new(args.metrics_address(), setup.stats); + let _metrics_server = MetricsServer::new(launch_config.metrics.address, setup.stats); // pipeline builder let pipeline_factory = setup.pipeline; // Start driver with the provided pipeline builder. Driver should create a portmap table, // populate it with [`PortSpec`]s and return the writer - let pmapw = match args.driver_name() { - "dpdk" => { - info!("Using driver DPDK..."); - todo!(); - } - "kernel" => { - info!("Using driver kernel..."); - DriverKernel::start( - args.interfaces(), - args.kernel_num_workers(), - &pipeline_factory, - ) - } - other => { - error!("Unknown driver '{other}'. Aborting..."); - panic!("Packet processing pipeline failed to start. Aborting..."); - } + let (pmapw, (_handle, iom_ctl)) = { + let pmapw = match &launch_config.driver { + args::DriverConfigSection::Dpdk(_section) => { + info!("Using driver DPDK..."); + todo!(); + } + args::DriverConfigSection::Kernel(section) => { + info!("Using driver kernel..."); + DriverKernel::start( + section.interfaces.clone().into_iter(), + launch_config.dataplane_workers, + &pipeline_factory, + ) + } + }; + let (_handle, iom_ctl) = + start_io::(setup.puntq, setup.injectq, TestBufferPool) + .unwrap(); + (pmapw, (_handle, iom_ctl)) }; // always log port mappings pmapw.log_pmap_table(); - // start IO service - let (_handle, iom_ctl) = match args.driver_name() { - /* - "dpdk" => { - let pool_cfg = - PoolConfig::new("fixme", PoolParams::default()).expect("Bad pool config"); - let pool = Pool::new_pkt_pool(pool_cfg).expect("Failed to create DPDK buffer pool"); - start_io::(setup.puntq, setup.injectq, pool) - } - */ - "kernel" => { - start_io::(setup.puntq, setup.injectq, TestBufferPool) - } - &_ => todo!(), - } - .expect("Failed to start IO manager"); - // prepare parameters for mgmt let mgmt_params = MgmtParams { grpc_addr, diff --git a/init/src/main.rs b/init/src/main.rs index 55f0ab854..1f9c30bdf 100644 --- a/init/src/main.rs +++ b/init/src/main.rs @@ -355,7 +355,7 @@ fn main() { match &launch_config.driver { args::DriverConfigSection::Dpdk(dpdk_section) => { - let search = DeviceSearch::new(dpdk_section.use_nics.iter()); + let search = DeviceSearch::new(dpdk_section.interfaces.iter().map(|it| &it.port)); let report = search.report(); let report_yml = serde_yaml_ng::to_string(&report) .into_diagnostic() From 0e1795d0ffbddeadea175bf4b4ae7f0ab7d12e0a Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Tue, 4 Nov 2025 00:16:18 +0000 Subject: [PATCH 05/10] fixup: init rework --- init/src/main.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/init/src/main.rs b/init/src/main.rs index 1f9c30bdf..e612365c4 100644 --- a/init/src/main.rs +++ b/init/src/main.rs @@ -2,7 +2,7 @@ // Copyright Open Network Fabric Authors #![doc = include_str!("../README.md")] -// #![deny(clippy::pedantic, missing_docs)] // TEMP: don't merge till uncommented +#![deny(clippy::pedantic)] use std::{ collections::{BTreeMap, BTreeSet}, @@ -64,7 +64,7 @@ impl std::fmt::Display for DevicesNotFound { let devices_str = self .missing .iter() - .map(|elem| elem.to_string()) + .map(std::string::ToString::to_string) .collect::>() .join(", "); write!(f, "missing devices: {devices_str}") @@ -144,6 +144,7 @@ impl DeviceSearch { /// Walk the full system hardware scan and collect the list of devices which are requested but which can not be /// found in the scan at all. + #[must_use] pub fn missing(&self) -> BTreeSet { let matching: BTreeSet<_> = self.matching().keys().cloned().collect(); self.requested.difference(&matching).cloned().collect() @@ -200,6 +201,7 @@ impl DeviceSearch { } /// Walk the hardware scan and find the devices which are both supported and requested. + #[must_use] pub fn scheduled_for_use(&self) -> BTreeMap { let matching: BTreeSet<_> = self.matching().keys().cloned().collect(); let supportable = self.supportable(); @@ -217,6 +219,7 @@ impl DeviceSearch { .collect() } + #[must_use] pub fn requested_but_not_supported(&self) -> BTreeMap { let matching = self.matching(); let matching_keys: BTreeSet<_> = matching.keys().cloned().collect(); @@ -240,6 +243,7 @@ impl DeviceSearch { .collect() } + #[must_use] pub fn report(&self) -> NetworkDeviceSearchReport<'_> { NetworkDeviceSearchReport::new(self) } @@ -252,7 +256,8 @@ pub struct NetworkDeviceSearchReport<'search> { missing: BTreeSet, supportable: BTreeMap, scheduled_for_use: BTreeMap, - // TODO: I don't understand why this needs to be a string. The requested_but_not_supported method crashes otherwise + // TODO: I don't understand why this key needs to be a string. + // The requested_but_not_supported method crashes otherwise when serializing. requested_but_not_supported: BTreeMap, } @@ -381,7 +386,7 @@ fn main() { error!("dataplane failed to initialize"); panic!("dataplane failed to initialize"); } - }; + } search .scheduled_for_use() .iter() @@ -423,11 +428,11 @@ fn main() { }, }, Err(_) => unreachable!(), // TODO: restructure to remove this branch - }; + } }); } args::DriverConfigSection::Kernel(_kernel_section) => todo!(), - }; + } let mut launch_config = launch_config.finalize(); let integrity_check = launch_config.integrity_check().finalize().to_owned_fd(); @@ -435,7 +440,7 @@ fn main() { let launch_config = launch_config.to_owned_fd(); let io_err = std::process::Command::new( - "/home/dnoland/code/githedgehog/dataplane/target/x86_64-unknown-linux-gnu/debug/dataplane", + "/bin/dataplane", ) .fd_mappings(vec![ FdMapping { From e2525e73574338169e67d5ad1232ed22f6748259 Mon Sep 17 00:00:00 2001 From: Daniel Noland Date: Tue, 4 Nov 2025 01:52:48 +0000 Subject: [PATCH 06/10] chore(init): fmt --- init/src/main.rs | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/init/src/main.rs b/init/src/main.rs index e612365c4..22c35e7a2 100644 --- a/init/src/main.rs +++ b/init/src/main.rs @@ -439,24 +439,22 @@ fn main() { let launch_config = launch_config.to_owned_fd(); - let io_err = std::process::Command::new( - "/bin/dataplane", - ) - .fd_mappings(vec![ - FdMapping { - parent_fd: integrity_check, - child_fd: LaunchConfiguration::STANDARD_INTEGRITY_CHECK_FD, - }, - FdMapping { - parent_fd: launch_config, - child_fd: LaunchConfiguration::STANDARD_CONFIG_FD, - }, - ]) - .into_diagnostic() - .wrap_err("failed to set file descriptor mapping for child process") - .unwrap() - .env_clear() - .exec(); + let io_err = std::process::Command::new("/bin/dataplane") + .fd_mappings(vec![ + FdMapping { + parent_fd: integrity_check, + child_fd: LaunchConfiguration::STANDARD_INTEGRITY_CHECK_FD, + }, + FdMapping { + parent_fd: launch_config, + child_fd: LaunchConfiguration::STANDARD_CONFIG_FD, + }, + ]) + .into_diagnostic() + .wrap_err("failed to set file descriptor mapping for child process") + .unwrap() + .env_clear() + .exec(); // if we got here then we failed to exec somehow Result::<(), _>::Err(io_err) From 780d1de8e2d54d04d7f345ba2a44caeb7f5c3b59 Mon Sep 17 00:00:00 2001 From: Fredi Raspall Date: Thu, 6 Nov 2025 15:22:15 +0100 Subject: [PATCH 07/10] feat(pkt-io,dataplane): remove portmap & mapper We could keep them, but at this point this functionality is not needed and forces us to maintain code that may not be needed. Note: this breakes the kernel driver. Signed-off-by: Fredi Raspall --- dataplane/src/drivers/kernel.rs | 36 +-- dataplane/src/main.rs | 14 +- mgmt/src/processor/proc.rs | 18 +- mgmt/src/tests/mgmt.rs | 4 - pkt-io/src/lib.rs | 4 - pkt-io/src/portmap.rs | 469 -------------------------------- pkt-io/src/portmapper.rs | 55 ---- 7 files changed, 8 insertions(+), 592 deletions(-) delete mode 100644 pkt-io/src/portmap.rs delete mode 100644 pkt-io/src/portmapper.rs diff --git a/dataplane/src/drivers/kernel.rs b/dataplane/src/drivers/kernel.rs index 016927dfe..da8f8ca5b 100644 --- a/dataplane/src/drivers/kernel.rs +++ b/dataplane/src/drivers/kernel.rs @@ -40,8 +40,6 @@ use pipeline::{DynPipeline, NetworkFunction}; #[allow(unused)] use tracing::{debug, error, info, trace, warn}; -use pkt_io::{PortMapWriter, PortSpec, build_portmap}; - // Flow-key based symmetric hashing use pkt_meta::flow_table::flow_key::{Bidi, FlowKey}; @@ -336,32 +334,6 @@ impl DriverKernel { Ok(kiftable) } - /// Register devices in the port map and return back the writer and factory - fn register_devices(kiftable: &mut KifTable) -> PortMapWriter { - // build port specs from the kifs to populate portmap - let pspecs: Vec<_> = kiftable - .by_token - .values() - .map(|kif| PortSpec::new(kif.name.to_string(), kif.pindex, kif.tapname.clone())) - .collect(); - - // populate port-map - let mapw = build_portmap(pspecs.into_iter()); - - // burn the tap ifindex in the kif so that we need not look it up - let rh = mapw.factory().handle(); - kiftable.by_token.values_mut().for_each(|kif| { - kif.tapifindex = Some( - rh.get_by_pdesc(&kif.name.to_string()) - .unwrap_or_else(|| unreachable!()) - .ifindex, - ); - }); - - // give ownership of portmap writer - mapw - } - /// Start the kernel IO thread for rx/tx fn start_kernel_io_thread( to_workers: Vec, @@ -466,7 +438,7 @@ impl DriverKernel { interfaces: impl Iterator, num_workers: usize, setup_pipeline: &Arc DynPipeline>, - ) -> PortMapWriter { + ) { // init port devices let mut kiftable = match Self::init_devices(interfaces) { Ok(kiftable) => kiftable, @@ -476,9 +448,6 @@ impl DriverKernel { } }; - // register port devices - let mapt_w = Self::register_devices(&mut kiftable); - // Spawn pipeline workers let (to_workers, from_workers) = Self::spawn_workers(num_workers, setup_pipeline); if to_workers.len() != num_workers { @@ -495,9 +464,6 @@ impl DriverKernel { // Spawn io thread Self::start_kernel_io_thread(to_workers, from_workers, kiftable); - - // return maptable writer - mapt_w } fn recv_packets( diff --git a/dataplane/src/main.rs b/dataplane/src/main.rs index 59a4b4979..08903fd4c 100644 --- a/dataplane/src/main.rs +++ b/dataplane/src/main.rs @@ -135,8 +135,8 @@ fn main() { // Start driver with the provided pipeline builder. Driver should create a portmap table, // populate it with [`PortSpec`]s and return the writer - let (pmapw, (_handle, iom_ctl)) = { - let pmapw = match &launch_config.driver { + let (_handle, iom_ctl) = { + match &launch_config.driver { args::DriverConfigSection::Dpdk(_section) => { info!("Using driver DPDK..."); todo!(); @@ -150,15 +150,10 @@ fn main() { ) } }; - let (_handle, iom_ctl) = - start_io::(setup.puntq, setup.injectq, TestBufferPool) - .unwrap(); - (pmapw, (_handle, iom_ctl)) + start_io::(setup.puntq, setup.injectq, TestBufferPool) + .expect("Failed to start IO manager") }; - // always log port mappings - pmapw.log_pmap_table(); - // prepare parameters for mgmt let mgmt_params = MgmtParams { grpc_addr, @@ -170,7 +165,6 @@ fn main() { vpcmapw: setup.vpcmapw, vpc_stats_store: setup.vpc_stats_store, iom_ctl, - pmapw, }, }; // start mgmt diff --git a/mgmt/src/processor/proc.rs b/mgmt/src/processor/proc.rs index ae498db73..9e8f621be 100644 --- a/mgmt/src/processor/proc.rs +++ b/mgmt/src/processor/proc.rs @@ -23,7 +23,7 @@ use crate::processor::confbuild::router::generate_router_config; use nat::stateful::NatAllocatorWriter; use nat::stateless::NatTablesWriter; use nat::stateless::setup::{build_nat_configuration, validate_nat_configuration}; -use pkt_io::{IoManagerCtl, PortMapReader, PortMapWriter}; +use pkt_io::IoManagerCtl; use pkt_meta::dst_vpcd_lookup::VpcDiscTablesWriter; use pkt_meta::dst_vpcd_lookup::setup::build_dst_vni_lookup_configuration; use routing::frr::FrrAppliedConfig; @@ -123,9 +123,6 @@ pub struct ConfigProcessorParams { // IO manager control pub iom_ctl: IoManagerCtl, - - // writer for portmap table - pub pmapw: PortMapWriter, } impl ConfigProcessor { @@ -449,11 +446,7 @@ impl VpcManager { } } -async fn config_io_manager( - internal: &InternalConfig, - iom_ctl: &mut IoManagerCtl, - _pmapr: PortMapReader, -) { +async fn config_io_manager(internal: &InternalConfig, iom_ctl: &mut IoManagerCtl) { iom_ctl.clear(); for vrfconfig in internal.vrfs.all_vrfs() { for iface in vrfconfig.interfaces.values().filter(|ifcfg| ifcfg.is_eth()) { @@ -643,12 +636,7 @@ async fn apply_gw_config( apply_router_config(&kernel_vrfs, config, &mut params.router_ctl).await?; /* reconfigure IO manager */ - config_io_manager( - internal, - &mut params.iom_ctl, - params.pmapw.factory().handle(), - ) - .await; + config_io_manager(internal, &mut params.iom_ctl).await; info!("Successfully applied config for genid {genid}"); Ok(()) diff --git a/mgmt/src/tests/mgmt.rs b/mgmt/src/tests/mgmt.rs index 69cb991b0..824eb5b81 100644 --- a/mgmt/src/tests/mgmt.rs +++ b/mgmt/src/tests/mgmt.rs @@ -402,9 +402,6 @@ pub mod test { let injectq = PktQueue::new(1); let (iom_handle, mut iom_ctl) = start_io(puntq, injectq, TestBufferPool).unwrap(); - /* create portmap */ - let pmapw = PortMapWriter::new(); - /* build configuration of mgmt config processor */ let processor_config = ConfigProcessorParams { router_ctl, @@ -414,7 +411,6 @@ pub mod test { vpcdtablesw, vpc_stats_store, iom_ctl: iom_ctl.clone(), // we pass a clone to keep one to stop IOM in test - pmapw, }; /* start config processor to test the processing of a config. The processor embeds the config database diff --git a/pkt-io/src/lib.rs b/pkt-io/src/lib.rs index b96bd438d..04fe63b2f 100644 --- a/pkt-io/src/lib.rs +++ b/pkt-io/src/lib.rs @@ -10,8 +10,6 @@ mod ctl; mod io; mod nf; -mod portmap; -mod portmapper; mod tests; // re-exports @@ -19,5 +17,3 @@ pub use ctl::IoManagerCtl; pub use io::{IoManagerError, start_io}; pub use nf::PktIo; pub use nf::PktQueue; -pub use portmap::{PortMap, PortMapReader, PortMapReaderFactory, PortMapWriter}; -pub use portmapper::{PortSpec, build_portmap, build_portmap_async}; diff --git a/pkt-io/src/portmap.rs b/pkt-io/src/portmap.rs deleted file mode 100644 index 01e5beec4..000000000 --- a/pkt-io/src/portmap.rs +++ /dev/null @@ -1,469 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright Open Network Fabric Authors - -//! Portid-to-interface mappings. This module implements a table to map port identifiers -//! and interface identifiers. Port identifiers are represented by [`PortIndex`] type, -//! while interface identifiers by type [`InterfaceIndex`]. - -#![allow(unused)] -#![deny( - unsafe_code, - clippy::all, - clippy::pedantic, - clippy::unwrap_used, - clippy::expect_used, - clippy::panic -)] - -use ahash::RandomState; -use left_right::ReadHandleFactory; -use left_right::{Absorb, ReadGuard, ReadHandle, WriteHandle}; -use net::interface::{InterfaceIndex, InterfaceName}; -use net::packet::PortIndex; -use net::vlan::Vid; -use std::collections::HashMap; -use std::sync::Arc; -use tracing::info; - -/// Type to describe a port. This definition is temporary -pub type NetworkDeviceDescription = String; - -/// A key to look up port mappings [`PortMap`] in a [`PortMapTable`]. -/// This type is internal to this module and needs not be exposed. -#[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)] -enum PortMapKey { - Port(PortIndex), - Iface(InterfaceIndex), -} -impl PortMapKey { - const fn from_port(pindex: PortIndex) -> Self { - PortMapKey::Port(pindex) - } - const fn from_iface(ifindex: InterfaceIndex) -> Self { - PortMapKey::Iface(ifindex) - } -} - -/// A (port+vlan)-to-interface mapping entry. -#[derive(Clone, Debug, PartialEq)] -pub struct PortMap { - pub pdesc: NetworkDeviceDescription, - pindex: PortIndex, - pub ifname: InterfaceName, - pub ifindex: InterfaceIndex, -} -impl PortMap { - const fn new( - pdesc: NetworkDeviceDescription, - pindex: PortIndex, - ifname: InterfaceName, - ifindex: InterfaceIndex, - ) -> Self { - Self { - pdesc, - pindex, - ifname, - ifindex, - } - } - const fn build_keys(&self) -> (PortMapKey, PortMapKey) { - let one = PortMapKey::from_port(self.pindex); - let two = PortMapKey::from_iface(self.ifindex); - (one, two) - } -} - -/// Table to look up [`PortMap`]'s. Every [`PortMap`] is doubly indexed by two keys -/// so that it can be queried from interface or port & vlan. -/// This table is wrapped in left-right and needs not be exposed. -#[derive(Clone, Debug)] -struct PortMapTable(HashMap, RandomState>); -impl PortMapTable { - #[must_use] - fn new() -> Self { - Self(HashMap::with_hasher(RandomState::with_seed(0))) - } - fn add_replace(&mut self, pmap: PortMap) { - let pmap = Arc::new(pmap); - let (pkey, ifkey) = pmap.build_keys(); - - self.del(pkey); - self.del(ifkey); - self.0.insert(pkey, pmap.clone()); - self.0.insert(ifkey, pmap.clone()); - - debug_assert!(self.0.len().is_multiple_of(2)); - debug_assert!(self.get(pkey) == Some(&pmap)); - debug_assert!(self.get(ifkey) == Some(&pmap)); - } - fn del_by_port(&mut self, portid: PortIndex) { - if let Some(pmap) = self.0.remove(&PortMapKey::from_port(portid)) { - self.0.remove(&PortMapKey::from_iface(pmap.ifindex)); - } - debug_assert!(self.0.len().is_multiple_of(2)); - } - fn del_by_interface(&mut self, ifindex: InterfaceIndex) { - if let Some(pmap) = self.0.remove(&PortMapKey::from_iface(ifindex)) { - self.0.remove(&PortMapKey::from_port(pmap.pindex)); - } - debug_assert!(self.0.len().is_multiple_of(2)); - } - fn del(&mut self, key: PortMapKey) { - match key { - PortMapKey::Port(pindex) => self.del_by_port(pindex), - PortMapKey::Iface(ifid) => self.del_by_interface(ifid), - } - debug_assert!(self.0.len().is_multiple_of(2)); - } - - fn get(&self, key: PortMapKey) -> Option<&PortMap> { - self.0.get(&key).map(std::convert::AsRef::as_ref) - } - pub(crate) fn get_by_pindex(&self, pindex: PortIndex) -> Option<&PortMap> { - self.get(PortMapKey::from_port(pindex)) - } - pub(crate) fn get_by_ifindex(&self, ifindex: InterfaceIndex) -> Option<&PortMap> { - self.get(PortMapKey::from_iface(ifindex)) - } - pub(crate) fn get_by_pdesc(&self, pdesc: &NetworkDeviceDescription) -> Option<&PortMap> { - self.0 - .values() - .find(|pmap| &pmap.pdesc == pdesc) - .map(|v| &**v) - } -} - -enum PortMapChange { - AddReplace(PortMap), - Del(PortMapKey), -} -impl Absorb for PortMapTable { - fn absorb_first(&mut self, change: &mut PortMapChange, _: &Self) { - match change { - PortMapChange::AddReplace(pmap) => self.add_replace(pmap.clone()), - PortMapChange::Del(pmapkey) => self.del(*pmapkey), - } - } - fn sync_with(&mut self, first: &Self) { - *self = first.clone(); - } -} - -pub struct PortMapWriter(WriteHandle); -impl PortMapWriter { - #[must_use] - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - let (writer, _) = - left_right::new_from_empty::(PortMapTable::new()); - PortMapWriter(writer) - } - pub fn add_replace( - &mut self, - pdesc: NetworkDeviceDescription, - ifname: InterfaceName, - pindex: PortIndex, - ifindex: InterfaceIndex, - ) { - let pmap = PortMap::new(pdesc, pindex, ifname, ifindex); - self.0.append(PortMapChange::AddReplace(pmap)); - self.0.publish(); - } - pub fn del_by_interface(&mut self, ifindex: InterfaceIndex) { - let pmapkey = PortMapKey::from_iface(ifindex); - self.0.append(PortMapChange::Del(pmapkey)); - self.0.publish(); - } - pub fn del_by_port(&mut self, pindex: PortIndex) { - let pmapkey = PortMapKey::from_port(pindex); - self.0.append(PortMapChange::Del(pmapkey)); - self.0.publish(); - } - pub fn factory(&self) -> PortMapReaderFactory { - PortMapReaderFactory(self.0.clone().factory()) - } - pub fn log_pmap_table(&self) { - let table = &*self.0.enter().unwrap_or_else(|| unreachable!()); - info!("{table}",); - } -} - -pub struct PortMapReader(ReadHandle); -impl PortMapReader { - #[cfg(test)] - fn get_by_pindex(&self, pindex: PortIndex) -> Option> { - let g = self.0.enter()?; - g.get_by_pindex(pindex)?; - Some(ReadGuard::map(g, |table| { - table - .get_by_pindex(pindex) - .unwrap_or_else(|| unreachable!()) - })) - } - #[cfg(test)] - fn get_by_ifindex(&self, ifindex: InterfaceIndex) -> Option> { - let g = self.0.enter()?; - g.get_by_ifindex(ifindex)?; - Some(ReadGuard::map(g, |table| { - table - .get_by_ifindex(ifindex) - .unwrap_or_else(|| unreachable!()) - })) - } - - pub fn get_by_pdesc(&self, pdesc: &NetworkDeviceDescription) -> Option> { - let g = self.0.enter()?; - g.get_by_pdesc(pdesc)?; // This is ugly - Some(ReadGuard::map(g, |table| { - table.get_by_pdesc(pdesc).unwrap_or_else(|| unreachable!()) - })) - } - pub fn lookup_iface_by_pindex(&self, pindex: PortIndex) -> Option { - self.0 - .enter()? - .get_by_pindex(pindex) - .map(|pmap| pmap.ifindex) - } - pub fn lookup_port_by_ifindex(&self, ifindex: InterfaceIndex) -> Option { - self.0 - .enter()? - .get_by_ifindex(ifindex) - .map(|pmap| pmap.pindex) - } -} - -pub struct PortMapReaderFactory(ReadHandleFactory); -impl PortMapReaderFactory { - #[must_use] - pub fn handle(&self) -> PortMapReader { - PortMapReader(self.0.handle()) - } -} - -use std::fmt::Display; - -macro_rules! PORTMAP_FMT { - () => { - " {:<8} {:<20} {:<8} {:<16} {:<8}" - }; -} -fn fmt_portmap_heading(f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "{}", - format_args!( - PORTMAP_FMT!(), - "key", "Device", "portid", "interface", "ifindex" - ) - ) -} - -impl Display for PortMapKey { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PortMapKey::Iface(ifindex) => write!(f, "{ifindex}"), - PortMapKey::Port(pindex) => write!(f, "{pindex}"), - } - } -} - -fn fmt_pmap_with_key( - f: &mut std::fmt::Formatter<'_>, - pmap: &PortMap, - key: PortMapKey, -) -> std::fmt::Result { - writeln!( - f, - "{}", - format_args!( - PORTMAP_FMT!(), - key.to_string(), - pmap.pdesc.clone(), - pmap.pindex.to_string(), - pmap.ifname.to_string(), - pmap.ifindex - ) - ) -} - -impl Display for PortMapTable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!( - f, - "\n━━━━━━━━━━━━━━━━━━━━━━━━━━━ PortMap table ━━━━━━━━━━━━━━━━━━━━━━━━━━" - )?; - fmt_portmap_heading(f)?; - for (key, pmap) in &self.0 { - fmt_pmap_with_key(f, pmap, *key)?; - } - writeln!( - f, - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - )?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::portmap::{NetworkDeviceDescription, PortMap, PortMapTable, PortMapWriter}; - use net::interface::{InterfaceIndex, InterfaceName}; - use net::packet::PortIndex; - use tracing_test::traced_test; - - fn build_portmap(pdesc: &str, ifname: &str, pindex: u16, ifindex: u32) -> PortMap { - let pdesc = pdesc.to_string(); - let ifname = InterfaceName::try_from(ifname).unwrap(); - let pindex = PortIndex::new(pindex); - let ifindex = InterfaceIndex::try_new(ifindex).unwrap(); - PortMap::new(pdesc, pindex, ifname, ifindex) - } - - #[test] - fn test_portmap_table_internal() { - let mut pmap_t = PortMapTable::new(); - - { - println!("test: insertion"); - let pmap = build_portmap("0000:03:02.1", "eth1", 1, 800); - pmap_t.add_replace(pmap.clone()); - - let lookup1 = pmap_t.get_by_ifindex(pmap.ifindex).unwrap(); - let lookup2 = pmap_t.get_by_pindex(pmap.pindex).unwrap(); - assert_eq!(lookup1, &pmap); - assert_eq!(lookup2, &pmap); - assert_eq!(pmap_t.0.len(), 2); - } - - { - println!("test: idempotence"); - let pmap = build_portmap("0000:03:02.1", "eth1", 1, 800); - pmap_t.add_replace(pmap.clone()); - - let lookup1 = pmap_t.get_by_ifindex(pmap.ifindex).unwrap(); - let lookup2 = pmap_t.get_by_pindex(pmap.pindex).unwrap(); - assert_eq!(lookup1, &pmap); - assert_eq!(lookup2, &pmap); - assert_eq!(pmap_t.0.len(), 2); - } - - { - println!("test: update non key fields"); - let pmap = build_portmap("0000:03:02.7", "ethFoo", 1, 800); - pmap_t.add_replace(pmap.clone()); - - let lookup1 = pmap_t.get_by_ifindex(pmap.ifindex).unwrap(); - let lookup2 = pmap_t.get_by_pindex(pmap.pindex).unwrap(); - assert_eq!(lookup1, &pmap); - assert_eq!(lookup2, &pmap); - assert_eq!(pmap_t.0.len(), 2); - } - - { - println!("test: replacement: change port index"); - let pmap = build_portmap("0000:03:02.1", "eth1", 2, 800); - pmap_t.add_replace(pmap.clone()); - - let lookup1 = pmap_t.get_by_ifindex(pmap.ifindex).unwrap(); - let lookup2 = pmap_t.get_by_pindex(pmap.pindex).unwrap(); - assert_eq!(lookup1, &pmap); - assert_eq!(lookup2, &pmap); - assert_eq!(pmap_t.0.len(), 2); - } - - { - println!("test: replacement: change ifindex"); - let pmap = build_portmap("0000:03:02.1", "eth1", 2, 900); - pmap_t.add_replace(pmap.clone()); - - let lookup1 = pmap_t.get_by_ifindex(pmap.ifindex).unwrap(); - let lookup2 = pmap_t.get_by_pindex(pmap.pindex).unwrap(); - assert_eq!(lookup1, &pmap); - assert_eq!(lookup2, &pmap); - assert_eq!(pmap_t.0.len(), 2); - } - - { - println!("test: deletion by portid"); - let pindex = PortIndex::new(2); - let ifindex = InterfaceIndex::try_new(900).unwrap(); - pmap_t.del_by_port(pindex); - - assert!(pmap_t.get_by_ifindex(ifindex).is_none()); - assert!(pmap_t.get_by_pindex(pindex).is_none()); - assert!(pmap_t.0.is_empty()); - } - - { - println!("test: restore and deletion by ifindex"); - let pmap = build_portmap("0000:03:02.1", "eth1", 2, 900); - pmap_t.add_replace(pmap.clone()); - assert_eq!(pmap_t.0.len(), 2); - - pmap_t.del_by_interface(pmap.ifindex); - assert!(pmap_t.get_by_ifindex(pmap.ifindex).is_none()); - assert!(pmap_t.get_by_pindex(pmap.pindex).is_none()); - assert!(pmap_t.0.is_empty()); - } - } - - #[traced_test] - #[test] - fn test_portmap_table() { - let mut writer = PortMapWriter::new(); - let reader = writer.factory().handle(); - - // insert some port map - let pmap = build_portmap("0000:03:02.1", "eth1", 1, 101); - writer.add_replace( - pmap.pdesc.clone(), - pmap.ifname.clone(), - pmap.pindex, - pmap.ifindex, - ); - writer.log_pmap_table(); - - // check reader sees it - let found = reader.get_by_ifindex(pmap.ifindex).unwrap(); - assert_eq!(&pmap, found.as_ref()); - drop(found); - - // lookups - assert_eq!( - reader.lookup_iface_by_pindex(pmap.pindex).unwrap(), - pmap.ifindex - ); - let pindex = reader.lookup_port_by_ifindex(pmap.ifindex).unwrap(); - assert_eq!(pindex, pmap.pindex); - - // update a port map: same port, distinct interface and vlan - let pmap = build_portmap("0000:03:02.1", "eth2", 1, 102); - writer.add_replace( - pmap.pdesc.clone(), - pmap.ifname.clone(), - pmap.pindex, - pmap.ifindex, - ); - let found = reader.get_by_ifindex(pmap.ifindex).unwrap(); - assert_eq!(&pmap, found.as_ref()); - drop(found); - - // lookups - assert_eq!( - reader.lookup_iface_by_pindex(pmap.pindex).unwrap(), - pmap.ifindex - ); - let pindex = reader.lookup_port_by_ifindex(pmap.ifindex).unwrap(); - assert_eq!(pindex, pmap.pindex); - - // Remove port map - let pmap = build_portmap("0000:03:02.1", "eth2", 1, 102); - writer.del_by_port(pmap.pindex); - assert!(reader.get_by_ifindex(pmap.ifindex).is_none()); - assert!(reader.get_by_pindex(pmap.pindex).is_none()); - - // lookups - assert!(reader.lookup_iface_by_pindex(pmap.pindex).is_none()); - assert!(reader.lookup_port_by_ifindex(pmap.ifindex).is_none()); - } -} diff --git a/pkt-io/src/portmapper.rs b/pkt-io/src/portmapper.rs deleted file mode 100644 index abae0f04f..000000000 --- a/pkt-io/src/portmapper.rs +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright Open Network Fabric Authors - -//! Portmapper. The port mapper is responsible for creating tap devices for the -//! available ports reported by drivers and populate the [`PortMapTable`]. - -#![deny(unsafe_code, clippy::all, clippy::pedantic, clippy::unwrap_used)] -#![allow(unused)] -#![allow(clippy::panic, clippy::missing_panics_doc)] - -use crate::portmap::{NetworkDeviceDescription, PortMapReaderFactory, PortMapWriter}; -use interface_manager::interface::TapDevice; -use net::interface::{Interface, InterfaceName}; -use net::packet::PortIndex; -use tokio::runtime::Runtime; - -pub struct PortSpec { - pdesc: NetworkDeviceDescription, // port description - pindex: PortIndex, // driver must guarantee uniqueness - ifname: InterfaceName, -} -impl PortSpec { - #[must_use] - pub fn new(pdesc: NetworkDeviceDescription, pindex: PortIndex, ifname: InterfaceName) -> Self { - Self { - pdesc, - pindex, - ifname, - } - } -} - -#[must_use] -pub async fn build_portmap_async(port_specs: impl Iterator) -> PortMapWriter { - let mut mapt_w = PortMapWriter::new(); - for spec in port_specs { - let Ok(tap) = TapDevice::open(&spec.ifname).await else { - // clearly, we should not proceed further if this fails. - panic!("Failed to build tap '{}'", spec.ifname); - }; - - // add mapping entry - mapt_w.add_replace(spec.pdesc, spec.ifname.clone(), spec.pindex, tap.ifindex()); - // N.B. we drop the tap device here. This is fine and desired. The tap interface is persisted - // and we don't need nor want to hold any file descriptor for it here. - } - mapt_w -} - -#[must_use] -pub fn build_portmap(port_specs: impl Iterator) -> PortMapWriter { - Runtime::new() - .expect("Tokio runtime creation failed!") - .block_on(build_portmap_async(port_specs)) -} From 81061a1e4e84d0885bfe87e2dec89686d5126236 Mon Sep 17 00:00:00 2001 From: Fredi Raspall Date: Thu, 6 Nov 2025 15:41:37 +0100 Subject: [PATCH 08/10] feat(kernel-driver): let kernel driver learn taps Signed-off-by: Fredi Raspall --- Cargo.lock | 1 + dataplane/Cargo.toml | 1 + dataplane/src/drivers/kernel.rs | 30 ++++++++++++++++-------------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6727ef18..3c10fc4f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,7 @@ dependencies = [ "mio", "n-vm", "netdev", + "nix 0.30.1", "once_cell", "ordermap", "parking_lot", diff --git a/dataplane/Cargo.toml b/dataplane/Cargo.toml index 32013c942..2cb8863c5 100644 --- a/dataplane/Cargo.toml +++ b/dataplane/Cargo.toml @@ -25,6 +25,7 @@ mgmt = { workspace = true } mio = { workspace = true, features = ["os-ext", "net"] } nat = { workspace = true } net = { workspace = true, features = ["test_buffer"] } +nix = { workspace = true, default-features = false, features = ["ioctl", "net"] } netdev = { workspace = true } once_cell = { workspace = true } ordermap = { workspace = true, features = ["std"] } diff --git a/dataplane/src/drivers/kernel.rs b/dataplane/src/drivers/kernel.rs index da8f8ca5b..bace462ea 100644 --- a/dataplane/src/drivers/kernel.rs +++ b/dataplane/src/drivers/kernel.rs @@ -36,6 +36,8 @@ use net::buffer::test_buffer::TestBuffer; use net::interface::{InterfaceIndex, InterfaceName}; use net::packet::{DoneReason, Packet, PortIndex}; use netdev::Interface; +use nix::net::if_::if_nametoindex; + use pipeline::{DynPipeline, NetworkFunction}; #[allow(unused)] use tracing::{debug, error, info, trace, warn}; @@ -61,8 +63,8 @@ pub struct Kif { raw_fd: RawFd, // raw desc of packet socket pindex: PortIndex, // port index. This is how the kernel interface is externally identified - tapname: InterfaceName, // name of tap interface - tapifindex: Option, // tap ifindex + tapname: InterfaceName, // name of tap interface + tapifindex: InterfaceIndex, // tap ifindex } impl Kif { @@ -74,6 +76,7 @@ impl Kif { name: &InterfaceName, token: Token, tapname: &InterfaceName, + tapifindex: InterfaceIndex, ) -> Result { let mut sock = RawPacketStream::new() .map_err(|e| format!("Failed to open raw sock for interface {name}: {e}"))?; @@ -92,7 +95,7 @@ impl Kif { raw_fd, pindex, tapname: tapname.clone(), - tapifindex: None, + tapifindex, }; debug!("Successfully created interface '{name}'"); Ok(iface) @@ -126,13 +129,17 @@ impl KifTable { ) -> Result<(), String> { debug!("Adding interface '{name}'..."); let token = Token(self.next_token); - let interface = Kif::new(ifindex, name, token, tapname)?; - let mut source = SourceFd(&interface.raw_fd); + let tapifindex = if_nametoindex(tapname.as_ref()) + .map_err(|e| format!("Could not find ifindex for tap {tapname}: {e}"))?; + let tapifindex = InterfaceIndex::try_from(tapifindex).map_err(|e| e.to_string())?; + + let kif = Kif::new(ifindex, name, token, tapname, tapifindex)?; + let mut source = SourceFd(&kif.raw_fd); self.poll .registry() .register(&mut source, token, Interest::READABLE) .map_err(|e| format!("Failed to register interface '{name}': {e}"))?; - self.by_token.insert(token, interface); + self.by_token.insert(token, kif); self.next_token += 1; debug!("Successfully registered interface '{name}' with token {token:?}"); Ok(()) @@ -146,7 +153,7 @@ impl KifTable { pub fn get_mut_by_tap_index(&mut self, tapifindex: InterfaceIndex) -> Option<&mut Kif> { self.by_token .values_mut() - .find(|kif| kif.tapifindex == Some(tapifindex)) + .find(|kif| kif.tapifindex == tapifindex) } } @@ -165,11 +172,6 @@ fn fmt_heading(f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { impl Display for Kif { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let tapifindex = if let Some(i) = self.tapifindex { - i.to_string() - } else { - "--".to_string() - }; writeln!( f, KIF_FMT!(), @@ -177,7 +179,7 @@ impl Display for Kif { self.ifindex.to_string(), self.pindex.to_string(), self.tapname.to_string(), - tapifindex + self.tapifindex ) } } @@ -494,7 +496,7 @@ impl DriverKernel { Ok(mut incoming) => { // we'll probably ditch iport, but for the time being.... incoming.get_meta_mut().iport = Some(kif.pindex); - incoming.get_meta_mut().iif = kif.tapifindex; + incoming.get_meta_mut().iif = Some(kif.tapifindex); pkts.push(Box::new(incoming)); } Err(e) => { From 60d597afd47862e7b901ca877c7c07d606d8672e Mon Sep 17 00:00:00 2001 From: Fredi Raspall Date: Thu, 6 Nov 2025 15:58:18 +0100 Subject: [PATCH 09/10] feat(pkt-io): add initial tap creation routines Signed-off-by: Fredi Raspall --- Cargo.lock | 1 + pkt-io/Cargo.toml | 1 + pkt-io/src/lib.rs | 3 ++- pkt-io/src/tapinit.rs | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 pkt-io/src/tapinit.rs diff --git a/Cargo.lock b/Cargo.lock index 3c10fc4f3..01932fb7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,6 +1396,7 @@ dependencies = [ "ahash", "caps", "crossbeam", + "dataplane-args", "dataplane-interface-manager", "dataplane-net", "dataplane-pipeline", diff --git a/pkt-io/Cargo.toml b/pkt-io/Cargo.toml index ef7b3da6a..d5cb2516d 100644 --- a/pkt-io/Cargo.toml +++ b/pkt-io/Cargo.toml @@ -7,6 +7,7 @@ license = "Apache-2.0" [dependencies] # internal +args = { workspace = true } interface-manager = { workspace = true } net = { workspace = true } pipeline = { workspace = true } diff --git a/pkt-io/src/lib.rs b/pkt-io/src/lib.rs index 04fe63b2f..49e7a2ea9 100644 --- a/pkt-io/src/lib.rs +++ b/pkt-io/src/lib.rs @@ -8,8 +8,8 @@ mod ctl; mod io; - mod nf; +mod tapinit; mod tests; // re-exports @@ -17,3 +17,4 @@ pub use ctl::IoManagerCtl; pub use io::{IoManagerError, start_io}; pub use nf::PktIo; pub use nf::PktQueue; +pub use tapinit::{tap_init, tap_init_async}; diff --git a/pkt-io/src/tapinit.rs b/pkt-io/src/tapinit.rs new file mode 100644 index 000000000..28816ee9c --- /dev/null +++ b/pkt-io/src/tapinit.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Open Network Fabric Authors + +//! Tap initialization + +use args::InterfaceArg; +use interface_manager::interface::TapDevice; +use tokio::runtime::Runtime; +use tracing::{error, info}; + +/// Creates a tap device for each of the [`InterfaceArg`]s provided. +/// +/// # Errors +/// +/// This function fails if any of the taps cannot be created. +pub async fn tap_init_async(ifargs: &[InterfaceArg]) -> std::io::Result<()> { + info!("Creating tap devices"); + for ifarg in ifargs.iter() { + if let Err(e) = TapDevice::open(&ifarg.interface).await { + error!("Failed to create tap '{}':{e}", ifarg.interface); + return Err(e); + } else { + info!("Created tap device '{}'", ifarg.interface); + } + } + Ok(()) +} + +/// Creates a tap device for each of the [`InterfaceArg`]s provided. +/// This is a sync wrapper to `tap_init_async`. +/// +/// # Errors +/// +/// This function fails if any of the taps cannot be created. +pub fn tap_init(port_specs: &[InterfaceArg]) -> std::io::Result<()> { + Runtime::new() + .expect("Tokio runtime creation failed!") + .block_on(tap_init_async(port_specs)) +} From a70bdf61b01ca459d849bc89ef0bb025e9227185 Mon Sep 17 00:00:00 2001 From: Fredi Raspall Date: Thu, 6 Nov 2025 16:35:29 +0100 Subject: [PATCH 10/10] feat(dataplane): create taps at initialization Create taps for the interfaces specified via cmd line args. Taps should be created before drivers start so that the latter can learn their during their initialization. Signed-off-by: Fredi Raspall --- dataplane/src/main.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dataplane/src/main.rs b/dataplane/src/main.rs index 08903fd4c..9301580ea 100644 --- a/dataplane/src/main.rs +++ b/dataplane/src/main.rs @@ -20,7 +20,7 @@ use pyroscope::PyroscopeAgent; use pyroscope_pprofrs::{PprofConfig, pprof_backend}; use net::buffer::{TestBuffer, TestBufferPool}; -use pkt_io::start_io; +use pkt_io::{start_io, tap_init}; use routing::RouterParamsBuilder; use tracectl::{custom_target, get_trace_ctl, trace_target}; @@ -133,15 +133,18 @@ fn main() { // pipeline builder let pipeline_factory = setup.pipeline; - // Start driver with the provided pipeline builder. Driver should create a portmap table, - // populate it with [`PortSpec`]s and return the writer + // Start driver with the provided pipeline builder. Taps must have been created before this + // happens so that their ifindex is available when drivers initialize. let (_handle, iom_ctl) = { match &launch_config.driver { - args::DriverConfigSection::Dpdk(_section) => { + args::DriverConfigSection::Dpdk(section) => { + tap_init(§ion.interfaces).expect("Tap initialization failed"); + info!("Using driver DPDK..."); todo!(); } args::DriverConfigSection::Kernel(section) => { + tap_init(§ion.interfaces).expect("Tap initialization failed"); info!("Using driver kernel..."); DriverKernel::start( section.interfaces.clone().into_iter(),