From f81fe7f396dc5083fe14d38ddf066a3034ac1fa5 Mon Sep 17 00:00:00 2001 From: drcpu Date: Fri, 3 Oct 2025 19:25:56 +0200 Subject: [PATCH 1/5] fix(data_structures): remove unknown lint --- data_structures/src/proto/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs index e0508af27..b5af1ea6f 100644 --- a/data_structures/src/proto/mod.rs +++ b/data_structures/src/proto/mod.rs @@ -9,7 +9,6 @@ use protobuf::Message; use std::{convert::TryFrom, fmt::Debug}; #[allow(elided_lifetimes_in_paths)] -#[allow(mismatched_lifetime_syntaxes)] #[allow(renamed_and_removed_lints)] pub mod schema; pub mod versioning; From d84a6c7cba7578cd7cfdff9e009c6459ae244f1a Mon Sep 17 00:00:00 2001 From: drcpu Date: Fri, 3 Oct 2025 22:52:41 +0200 Subject: [PATCH 2/5] feat(data_structures): implement resolving of data requests with API key placeholder in the URL or headers --- Cargo.lock | 9 +- config/src/config.rs | 39 +++- config/src/defaults.rs | 19 ++ data_structures/Cargo.toml | 1 + data_structures/src/capabilities.rs | 12 +- data_structures/src/chain/mod.rs | 77 ++++++- data_structures/src/data_request.rs | 271 +++++++++++++++++++++++- data_structures/src/error.rs | 4 + data_structures/src/proto/mod.rs | 8 + data_structures/src/staking/stakes.rs | 18 +- node/src/actors/chain_manager/actor.rs | 20 +- node/src/actors/chain_manager/mining.rs | 67 +++++- node/src/actors/chain_manager/mod.rs | 7 + rad/src/error.rs | 3 + rad/src/lib.rs | 52 +++-- schemas/witnet/witnet.proto | 2 + validations/src/eligibility/current.rs | 24 ++- validations/src/validations.rs | 9 +- witnet.toml | 26 ++- 19 files changed, 616 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15d2c381c..9347c5e89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4064,9 +4064,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -4076,9 +4076,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -6652,6 +6652,7 @@ dependencies = [ "protobuf-convert", "rand 0.8.5", "rand_distr", + "regex", "serde", "serde_cbor", "serde_json", diff --git a/config/src/config.rs b/config/src/config.rs index 1cb378096..56f79fe06 100644 --- a/config/src/config.rs +++ b/config/src/config.rs @@ -268,6 +268,15 @@ pub struct Connections { pub requested_blocks_batch_limit: u32, } +/// Struct to store all values of an API key +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, PartialStruct, Serialize)] +pub struct ApiKey { + pub id: String, + pub key: String, + pub reward: Option, + pub rate: Option, +} + /// Witnessing-specific configuration. #[derive(Clone, Debug, Eq, PartialEq, PartialStruct)] #[partial_struct(derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize))] @@ -289,6 +298,19 @@ pub struct Witnessing { /// and we are taking as small of a risk as possible when committing to specially crafted data /// requests that may be potentially ill-intended. pub proxies: Vec, + + /// Binary flag telling whether to enable default data request witnessing + #[partial_struct(serde(default))] + pub enabled: bool, + + /// Binary flag telling whether to enable data request witnessing using API keys + #[partial_struct(serde(default))] + pub enabled_with_keys: bool, + + /// Collection of the API keys defined in the config file + /// The + #[partial_struct(serde(default))] + pub api_keys: Vec, } /// Available storage backends @@ -346,7 +368,7 @@ pub struct JsonRPC { #[derive(PartialStruct, Debug, Clone, PartialEq, Eq)] #[partial_struct(derive(Deserialize, Serialize, Default, Debug, Clone, PartialEq, Eq))] pub struct Mining { - /// Binary flag telling whether to enable the MiningManager or not + /// Binary flag telling whether to enable the mining of blocks or not pub enabled: bool, /// Limits the number of retrievals to perform during a single epoch. /// This tries to prevent nodes from forking out or being left in a @@ -1147,6 +1169,18 @@ impl Witnessing { .proxies .clone() .unwrap_or_else(|| defaults.witnessing_proxies()), + enabled: config + .enabled + .to_owned() + .unwrap_or_else(|| defaults.witnessing_enabled()), + enabled_with_keys: config + .enabled_with_keys + .to_owned() + .unwrap_or_else(|| defaults.witnessing_with_keys_enabled()), + api_keys: config + .api_keys + .to_owned() + .unwrap_or_else(|| defaults.api_keys()), } } @@ -1155,6 +1189,9 @@ impl Witnessing { allow_unproxied: Some(self.allow_unproxied), paranoid_percentage: Some(self.paranoid_percentage), proxies: Some(self.proxies.clone()), + enabled: Some(self.enabled), + enabled_with_keys: Some(self.enabled_with_keys), + api_keys: Some(self.api_keys.clone()), } } diff --git a/config/src/defaults.rs b/config/src/defaults.rs index dee900207..c0903e224 100644 --- a/config/src/defaults.rs +++ b/config/src/defaults.rs @@ -2,6 +2,8 @@ //! //! This module contains per-environment default values for the Witnet //! protocol params. +use crate::config::ApiKey; + use std::{ collections::{HashMap, HashSet}, net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -146,6 +148,23 @@ pub trait Defaults { vec![] } + /// Node should have witnessing enabled by default + fn witnessing_enabled(&self) -> bool { + true + } + + /// Witnessing using API keys is disabled by default since it requires setting up (paid) API + /// keys by the operator. + fn witnessing_with_keys_enabled(&self) -> bool { + false + } + + /// Adds a collation of API keys to resolve data requests which contain an API key placeholder + /// in the URL or headers + fn api_keys(&self) -> Vec { + vec![] + } + /// Timestamp at the start of epoch 0 fn consensus_constants_checkpoint_zero_timestamp(&self) -> i64; diff --git a/data_structures/Cargo.toml b/data_structures/Cargo.toml index 325568505..833a89f8f 100644 --- a/data_structures/Cargo.toml +++ b/data_structures/Cargo.toml @@ -31,6 +31,7 @@ protobuf = { version = "2.28.0", features = ["with-serde"] } protobuf-convert = "0.4.0" rand = "0.8.5" rand_distr = "0.4.3" +regex = "1.11.3" serde = { version = "1.0.104", features = ["derive"] } serde_cbor = "0.11.1" serde_json = "1.0.48" diff --git a/data_structures/src/capabilities.rs b/data_structures/src/capabilities.rs index 1429728c5..2e2139888 100644 --- a/data_structures/src/capabilities.rs +++ b/data_structures/src/capabilities.rs @@ -24,9 +24,15 @@ pub enum Capability { Mining = 0, /// The universal HTTP GET / HTTP POST / WIP-0019 RNG capability Witnessing = 1, + /// The HTTP GET / HTTP POST capability which requires API keys + WitnessingWithKey = 2, } -pub const ALL_CAPABILITIES: [Capability; 2] = [Capability::Mining, Capability::Witnessing]; +pub const ALL_CAPABILITIES: [Capability; 3] = [ + Capability::Mining, + Capability::Witnessing, + Capability::WitnessingWithKey, +]; #[derive(Copy, Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct CapabilityMap @@ -35,6 +41,7 @@ where { pub mining: T, pub witnessing: T, + pub witnessing_with_key: T, } impl CapabilityMap @@ -46,6 +53,7 @@ where match capability { Capability::Mining => self.mining, Capability::Witnessing => self.witnessing, + Capability::WitnessingWithKey => self.witnessing_with_key, } } @@ -54,6 +62,7 @@ where match capability { Capability::Mining => self.mining = value, Capability::Witnessing => self.witnessing = value, + Capability::WitnessingWithKey => self.witnessing_with_key = value, } } @@ -61,5 +70,6 @@ where pub fn update_all(&mut self, value: T) { self.mining = value; self.witnessing = value; + self.witnessing_with_key = value; } } diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 4d5bd9203..3025ebf4a 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -4,6 +4,7 @@ use ethereum_types::U256; use futures::future::BoxFuture; use ordered_float::OrderedFloat; use partial_struct::PartialStruct; +use regex::Regex; use serde::{Deserialize, Serialize}; use std::{ cell::{Cell, RefCell}, @@ -2077,6 +2078,12 @@ pub enum RADType { /// HTTP HEAD request #[serde(rename = "HTTP-HEAD")] HttpHead, + /// HTTP GET request requiring an API key + #[serde(rename = "HTTP-GET-KEY")] + HttpGetKey, + /// HTTP POST request requiring an API key + #[serde(rename = "HTTP-POST-KEY")] + HttpPostKey, } impl RADType { @@ -2086,6 +2093,10 @@ impl RADType { RADType::HttpGet | RADType::HttpPost | RADType::HttpHead ) } + + pub fn is_http_key(&self) -> bool { + matches!(self, RADType::HttpGetKey | RADType::HttpPostKey) + } } /// RAD request data structure @@ -2233,9 +2244,11 @@ impl RADRetrieve { // Anything is fine Ok(()) } - RADType::HttpGet => check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]), + RADType::HttpGet | RADType::HttpGetKey => { + check(&[Field::Kind, Field::Url, Field::Script], &[Field::Headers]) + } RADType::Rng => check(&[Field::Kind, Field::Script], &[]), - RADType::HttpPost => { + RADType::HttpPost | RADType::HttpPostKey => { // In HttpPost the body is optional because empty body should also be allowed check( &[Field::Kind, Field::Url, Field::Script], @@ -2297,6 +2310,66 @@ impl RADRetrieve { .saturating_add(body_weight) .saturating_add(headers_weight) } + + /// Function to check if an URL contains an API key placeholder + pub fn api_key_in_url(&self) -> bool { + if self.kind != RADType::HttpGetKey && self.kind != RADType::HttpPostKey { + return false; + } + + let rex = Regex::new(r".+<[a-zA-Z0-9_-]+_API_KEY>.*").unwrap(); + + rex.is_match(&self.url) + } + + /// Function to get the placeholder of API key + pub fn url_extract_api_key(&self) -> Option<&str> { + if self.kind != RADType::HttpGetKey && self.kind != RADType::HttpPostKey { + return None; + } + + let rex = Regex::new(r".+(<[a-zA-Z0-9_-]+_API_KEY>).*").unwrap(); + + match rex.captures(&self.url) { + Some(captures) => Some(captures.get(1).unwrap().as_str()), + None => None, + } + } + + /// Function to check if the value of a header is an API key placeholder + pub fn api_key_in_headers(&self) -> bool { + if self.kind != RADType::HttpGetKey && self.kind != RADType::HttpPostKey { + return false; + } + + let rex = Regex::new(r".*<[a-zA-Z0-9_-]+_API_KEY>.*").unwrap(); + + self.headers.iter().any(|(_, value)| rex.is_match(&value)) + } + + /// Function to get the placeholder of API key + pub fn headers_extract_api_key(&self) -> Option<(usize, &str)> { + if self.kind != RADType::HttpGetKey && self.kind != RADType::HttpPostKey { + return None; + } + + let rex = Regex::new(r".*(<[a-zA-Z0-9_-]+_API_KEY>).*").unwrap(); + + for (idx, (_, header_value)) in self.headers.iter().enumerate() { + if rex.is_match(&header_value) { + return Some(( + idx, + rex.captures(&header_value) + .unwrap() + .get(1) + .unwrap() + .as_str(), + )); + } + } + + None + } } /// Filter stage diff --git a/data_structures/src/data_request.rs b/data_structures/src/data_request.rs index f2cf5bb1f..a45f2e667 100644 --- a/data_structures/src/data_request.rs +++ b/data_structures/src/data_request.rs @@ -11,7 +11,7 @@ use witnet_crypto::hash::calculate_sha256; use crate::{ chain::{ DataRequestInfo, DataRequestOutput, DataRequestStage, DataRequestState, Environment, Epoch, - Hash, Hashable, PublicKeyHash, ValueTransferOutput, tapi::ActiveWips, + Hash, Hashable, PublicKeyHash, RADRequest, RADType, ValueTransferOutput, tapi::ActiveWips, }, error::{DataRequestError, TransactionError}, get_environment, get_protocol_version_activation_epoch, @@ -588,6 +588,72 @@ pub fn data_request_has_too_many_witnesses( } } +/// Function to check if any of the URL's in a data request requires an API key +pub fn data_request_requires_api_key(request: &RADRequest) -> bool { + request.retrieve.iter().any(|retrieval| { + if retrieval.kind == RADType::HttpGetKey || retrieval.kind == RADType::HttpPostKey { + let url_with_api_key = retrieval.api_key_in_url(); + let headers_with_api_key = retrieval.api_key_in_headers(); + + url_with_api_key || headers_with_api_key + } else { + false + } + }) +} + +/// Function to replace API key placeholders with the actual API key (assuming it is found) +pub fn data_request_replace_api_keys( + rad_request: &mut RADRequest, + api_keys: &HashMap, +) -> Result { + let mut key = ""; + + for retrieval in rad_request.retrieve.iter_mut() { + // Replace API keys inside an URL + match retrieval.url_extract_api_key() { + Some(api_key) => { + match api_keys.get_key_value(api_key) { + Some((k, v)) => { + retrieval.url = retrieval.url.replace(api_key, &v.0); + key = k; + } + // API key not found, we cannot solve this data request + None => { + return Err(DataRequestError::NoApiKeyFound { + api_key: api_key.to_string(), + } + .into()); + } + } + } + None => (), + }; + + // Replace API keys in a header + match retrieval.headers_extract_api_key() { + Some((idx, api_key)) => { + match api_keys.get_key_value(api_key) { + Some((k, v)) => { + retrieval.headers[idx].1 = retrieval.headers[idx].1.replace(api_key, &v.0); + key = k; + } + // API key not found, we cannot solve this data request + None => { + return Err(DataRequestError::NoApiKeyFound { + api_key: retrieval.headers[idx].1.to_string(), + } + .into()); + } + } + } + None => (), + }; + } + + Ok(key.to_string()) +} + /// Saturating version of `u64::div_ceil`. /// /// Calculates the quotient of `lhs` and `rhs`, rounding the result towards positive infinity. @@ -1518,4 +1584,207 @@ mod tests { DataRequestStage::REVEAL ); } + + #[test] + fn test_api_key_required() { + let rad_retrieve_1 = RADRetrieve { + url: "https://api.coingecko.com/api/v3/ping".to_string(), + kind: RADType::HttpGet, + ..RADRetrieve::default() + }; + let rad_retrieve_2 = RADRetrieve { + url: "https://api.coingecko.com/api/v3/ping?x_cg_demo_api_key=" + .to_string(), + kind: RADType::HttpGetKey, + ..RADRetrieve::default() + }; + let rad_retrieve_3 = RADRetrieve { + url: "https://api.coingecko.com/api/v3/ping?x_cg_demo_api_key=".to_string(), + )], + ..RADRetrieve::default() + }; + let rad_retrieve_5 = RADRetrieve { + url: "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest".to_string(), + kind: RADType::HttpPostKey, + headers: vec![( + "X-CMC_PRO_API_KEY".to_string(), + "This is my API key: ".to_string(), + )], + ..RADRetrieve::default() + }; + + let rad_request_1 = RADRequest { + retrieve: vec![rad_retrieve_1.clone()], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_1) == false); + + let rad_request_2 = RADRequest { + retrieve: vec![rad_retrieve_2.clone()], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_2)); + + let rad_request_3 = RADRequest { + retrieve: vec![rad_retrieve_3.clone()], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_3) == false); + + let rad_request_4 = RADRequest { + retrieve: vec![rad_retrieve_1, rad_retrieve_2], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_4)); + + let rad_request_5 = RADRequest { + retrieve: vec![rad_retrieve_4], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_5)); + + let rad_request_6 = RADRequest { + retrieve: vec![rad_retrieve_5], + ..RADRequest::default() + }; + assert!(data_request_requires_api_key(&rad_request_6)); + } + + #[test] + fn test_api_key_replacement() { + let api_keys = HashMap::from([ + ( + "".to_string(), + ("I_promise_this_is_a_real_api_key".to_string(), 0, 0, 0), + ), + ( + "".to_string(), + ("it_is_definitely_real".to_string(), 0, 0, 0), + ), + ]); + + let rad_retrieve_1 = RADRetrieve { + url: "https://api.coingecko.com/api/v3/ping".to_string(), + kind: RADType::HttpGet, + ..RADRetrieve::default() + }; + let rad_retrieve_2 = RADRetrieve { + url: "https://api.coingecko.com/api/v3/ping?x_cg_demo_api_key=" + .to_string(), + kind: RADType::HttpGetKey, + ..RADRetrieve::default() + }; + let rad_retrieve_3 = RADRetrieve { + url: "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest".to_string(), + kind: RADType::HttpPostKey, + headers: vec![( + "X-CMC_PRO_API_KEY".to_string(), + "".to_string(), + )], + ..RADRetrieve::default() + }; + let rad_retrieve_4 = RADRetrieve { + url: "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest".to_string(), + kind: RADType::HttpPostKey, + headers: vec![( + "X-CMC_PRO_API_KEY".to_string(), + "This is my API key: ".to_string(), + )], + ..RADRetrieve::default() + }; + + // No API key to replace + let mut rad_request_1 = RADRequest { + retrieve: vec![rad_retrieve_1.clone()], + ..RADRequest::default() + }; + data_request_replace_api_keys(&mut rad_request_1, &api_keys).unwrap(); + assert_eq!( + rad_request_1.retrieve[0].url, + "https://api.coingecko.com/api/v3/ping".to_string() + ); + + // Replace an API key in an url + let mut rad_request_2 = RADRequest { + retrieve: vec![rad_retrieve_2.clone()], + ..RADRequest::default() + }; + data_request_replace_api_keys(&mut rad_request_2, &api_keys).unwrap(); + assert_eq!(rad_request_2.retrieve[0].url, "https://api.coingecko.com/api/v3/ping?x_cg_demo_api_key=I_promise_this_is_a_real_api_key".to_string()); + + // Replace an API key in a header + let mut rad_request_3 = RADRequest { + retrieve: vec![rad_retrieve_3.clone()], + ..RADRequest::default() + }; + data_request_replace_api_keys(&mut rad_request_3, &api_keys).unwrap(); + assert_eq!( + rad_request_3.retrieve[0].url, + "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest".to_string() + ); + assert_eq!( + rad_request_3.retrieve[0].headers, + vec![( + "X-CMC_PRO_API_KEY".to_string(), + "it_is_definitely_real".to_string() + )] + ); + + let mut rad_request_4 = RADRequest { + retrieve: vec![rad_retrieve_4.clone()], + ..RADRequest::default() + }; + data_request_replace_api_keys(&mut rad_request_4, &api_keys).unwrap(); + assert_eq!( + rad_request_4.retrieve[0].url, + "https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest".to_string() + ); + assert_eq!( + rad_request_4.retrieve[0].headers, + vec![( + "X-CMC_PRO_API_KEY".to_string(), + "This is my API key: it_is_definitely_real".to_string() + )] + ); + + // API key in URL not found + let mut rad_request_5 = RADRequest { + retrieve: vec![rad_retrieve_2.clone()], + ..RADRequest::default() + }; + assert_eq!( + data_request_replace_api_keys(&mut rad_request_5, &HashMap::new()) + .unwrap_err() + .downcast::() + .unwrap(), + DataRequestError::NoApiKeyFound { + api_key: "".to_string() + } + ); + + // API key in header not found + let mut rad_request_6 = RADRequest { + retrieve: vec![rad_retrieve_3.clone()], + ..RADRequest::default() + }; + assert_eq!( + data_request_replace_api_keys(&mut rad_request_6, &HashMap::new()) + .unwrap_err() + .downcast::() + .unwrap(), + DataRequestError::NoApiKeyFound { + api_key: "".to_string() + } + ); + } } diff --git a/data_structures/src/error.rs b/data_structures/src/error.rs index 3ee072c18..7711f4ea3 100644 --- a/data_structures/src/error.rs +++ b/data_structures/src/error.rs @@ -493,12 +493,16 @@ pub enum DataRequestError { tx_hash: Hash, dr_pointer: Hash, }, + #[error("API key {api_key} required to solve this data request was not found")] + NoApiKeyFound { api_key: String }, #[error("Received a commitment and Data Request is not in Commit stage")] NotCommitStage, #[error("Received a reveal and Data Request is not in Reveal stage")] NotRevealStage, #[error("Received a tally and Data Request is not in Tally stage")] NotTallyStage, + #[error("No API key found in the URL or headers")] + RequestWithoutApiKey, #[error("Cannot persist unfinished data request (with no Tally)")] UnfinishedDataRequest, #[error("The data request is not valid since it has no retrieval sources")] diff --git a/data_structures/src/proto/mod.rs b/data_structures/src/proto/mod.rs index b5af1ea6f..5c7c33c9b 100644 --- a/data_structures/src/proto/mod.rs +++ b/data_structures/src/proto/mod.rs @@ -56,6 +56,10 @@ impl ProtobufConvert for chain::RADType { chain::RADType::Rng => witnet::DataRequestOutput_RADRequest_RADType::Rng, chain::RADType::HttpPost => witnet::DataRequestOutput_RADRequest_RADType::HttpPost, chain::RADType::HttpHead => witnet::DataRequestOutput_RADRequest_RADType::HttpHead, + chain::RADType::HttpGetKey => witnet::DataRequestOutput_RADRequest_RADType::HttpGetKey, + chain::RADType::HttpPostKey => { + witnet::DataRequestOutput_RADRequest_RADType::HttpPostKey + } } } @@ -66,6 +70,10 @@ impl ProtobufConvert for chain::RADType { witnet::DataRequestOutput_RADRequest_RADType::Rng => chain::RADType::Rng, witnet::DataRequestOutput_RADRequest_RADType::HttpPost => chain::RADType::HttpPost, witnet::DataRequestOutput_RADRequest_RADType::HttpHead => chain::RADType::HttpHead, + witnet::DataRequestOutput_RADRequest_RADType::HttpGetKey => chain::RADType::HttpGetKey, + witnet::DataRequestOutput_RADRequest_RADType::HttpPostKey => { + chain::RADType::HttpPostKey + } }) } } diff --git a/data_structures/src/staking/stakes.rs b/data_structures/src/staking/stakes.rs index cf7a301fa..353d85b76 100644 --- a/data_structures/src/staking/stakes.rs +++ b/data_structures/src/staking/stakes.rs @@ -1038,7 +1038,8 @@ mod tests { 100, CapabilityMap { mining: 100, - witnessing: 100 + witnessing: 100, + witnessing_with_key: 100 }, 100, ) @@ -1062,7 +1063,8 @@ mod tests { 150, CapabilityMap { mining: 166, - witnessing: 166 + witnessing: 166, + witnessing_with_key: 166 }, 300, ) @@ -1093,7 +1095,8 @@ mod tests { 500, CapabilityMap { mining: 1_000, - witnessing: 1_000 + witnessing: 1_000, + witnessing_with_key: 1_000 }, 1_000, ) @@ -1377,7 +1380,8 @@ mod tests { 20, CapabilityMap { mining: 30, - witnessing: 30 + witnessing: 30, + witnessing_with_key: 30 }, 30, ) @@ -1393,7 +1397,8 @@ mod tests { 20, CapabilityMap { mining: 30, - witnessing: 30 + witnessing: 30, + witnessing_with_key: 30 }, 30, ) @@ -1417,7 +1422,8 @@ mod tests { 20, CapabilityMap { mining: 30, - witnessing: 30 + witnessing: 30, + witnessing_with_key: 30 }, 30, ) diff --git a/node/src/actors/chain_manager/actor.rs b/node/src/actors/chain_manager/actor.rs index 8c669a43e..2234d678a 100644 --- a/node/src/actors/chain_manager/actor.rs +++ b/node/src/actors/chain_manager/actor.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, pin::Pin, str::FromStr, time::Duration}; +use std::{collections::HashMap, path::PathBuf, pin::Pin, str::FromStr, time::Duration}; use actix::prelude::*; use actix::{ActorTryFutureExt, ContextFutureSpawner, WrapFuture}; @@ -172,6 +172,24 @@ impl ChainManager { log::debug!("Initial WIT supply: {}", act.initial_supply); } + // Enable witnessing + act.witnessing_enabled = config.witnessing.enabled; + + // Enable witnessing with API keys + act.witnessing_with_keys_enabled = config.witnessing.enabled_with_keys; + + // Store the API keys defined in the configuration + if !act.witnessing_with_keys_enabled { + log::info!("Using following API keys with the node: {:?}", config.witnessing.api_keys); + } + act.api_keys = HashMap::new(); + for key_data in config.witnessing.api_keys.iter() { + act.api_keys.insert( + "<".to_owned() + &key_data.id + ">", + (key_data.key.clone(), key_data.reward.unwrap_or(0), key_data.rate.unwrap_or(0), 0) + ); + } + storage_mngr::get_chain_state(storage_keys::chain_state_key(magic)) .into_actor(act) .then(|chain_state_from_storage, _, _| { diff --git a/node/src/actors/chain_manager/mining.rs b/node/src/actors/chain_manager/mining.rs index 48d0b3ff7..8a149e959 100644 --- a/node/src/actors/chain_manager/mining.rs +++ b/node/src/actors/chain_manager/mining.rs @@ -18,6 +18,7 @@ use futures::future::{FutureExt, try_join_all}; use witnet_data_structures::{ DEFAULT_VALIDATOR_COUNT_FOR_TESTS, + capabilities::Capability, chain::{ Block, BlockHeader, BlockMerkleRoots, BlockTransactions, Bn256PublicKey, CheckpointBeacon, CheckpointVRF, ConsensusConstantsWit2, DataRequestOutput, EpochConstants, Hash, Hashable, @@ -27,7 +28,8 @@ use witnet_data_structures::{ data_request::{ DataRequestPool, calculate_witness_reward, calculate_witness_reward_before_second_hard_fork, create_tally, - data_request_has_too_many_witnesses, + data_request_has_too_many_witnesses, data_request_replace_api_keys, + data_request_requires_api_key, }, error::TransactionError, get_environment, get_protocol_version, @@ -292,7 +294,7 @@ impl ChainManager { &mut self, ctx: &mut Context, ) -> Result<(), ChainManagerError> { - if !self.mining_enabled { + if !self.witnessing_enabled { return Err(ChainManagerError::MiningIsDisabled); } @@ -347,7 +349,7 @@ impl ChainManager { let current_retrieval_count = Arc::new(AtomicU16::new(0u16)); let maximum_retrieval_count = self.data_request_max_retrievals_per_epoch; - for (dr_pointer, dr_state) in dr_pointers.into_iter().filter_map(|dr_pointer| { + for (dr_pointer, mut dr_state) in dr_pointers.into_iter().filter_map(|dr_pointer| { // Filter data requests that are not in data_request_pool self.chain_state .data_request_pool @@ -370,6 +372,53 @@ impl ChainManager { continue; } + // Replace the API key place holders in the URL's + // If we do not have the API key required, just continue to the next data request + let mut rad_request = dr_state.data_request.data_request.clone(); + let used_api_key = if data_request_requires_api_key(&rad_request) { + if !self.witnessing_with_keys_enabled { + continue; + } + + match data_request_replace_api_keys(&mut rad_request, &self.api_keys) { + // Check that the reward and rate needed to solve a data request with this API key + // are adhered to. + Ok(key) => { + let (_, reward, rate, last_epoch) = self.api_keys[&key]; + if dr_state.data_request.witness_reward < reward { + log::info!( + "Reward for the data request requiring key {} was too low: {} < {}", + key, + dr_state.data_request.witness_reward, + reward, + ); + continue; + } + if last_epoch != 0 && current_epoch < last_epoch + rate { + log::info!( + "Resolved a data request requiring key {} too recently: {} < {} + {}", + key, + current_epoch, + last_epoch, + rate, + ); + continue; + } + + Some(key) + } + // API key not found, we cannot solve this data request + // No need to even start eligibility calculations etc + Err(e) => { + log::warn!("{e}"); + continue; + } + } + } else { + None + }; + dr_state.data_request.data_request = rad_request; + let (checkpoint_period, max_rounds) = match &self.chain_state.chain_info { Some(x) => ( // Unwraps should be safe if we have a chain_info object @@ -401,6 +450,11 @@ impl ChainManager { .consensus_constants_wit2 .get_collateral_age(&active_wips); let (target_hash, probability) = if protocol_version >= V2_0 { + let capability = + match data_request_requires_api_key(&dr_state.data_request.data_request) { + true => Capability::WitnessingWithKey, + false => Capability::Witnessing, + }; let (eligibility, target_hash, probability) = self .chain_state .stakes @@ -410,6 +464,7 @@ impl ChainManager { num_witnesses, round, max_rounds, + capability, ) .map_err(ChainManagerError::Staking)?; @@ -434,6 +489,12 @@ impl ChainManager { ) }; + // We are eligible to resolve a data request with an API key, update the last epoch + match used_api_key { + Some(key) => self.api_keys.get_mut(&key).unwrap().3 = current_epoch, + None => (), + } + // Grab a reference to `current_retrieval_count` let cloned_retrieval_count = Arc::clone(¤t_retrieval_count); let cloned_retrieval_count2 = Arc::clone(¤t_retrieval_count); diff --git a/node/src/actors/chain_manager/mod.rs b/node/src/actors/chain_manager/mod.rs index 4cab1e353..634d3c317 100644 --- a/node/src/actors/chain_manager/mod.rs +++ b/node/src/actors/chain_manager/mod.rs @@ -260,6 +260,13 @@ pub struct ChainManager { consensus_constants_wit2: ConsensusConstantsWit2, /// Initial WIT supply initial_supply: u64, + /// Witnessing of default data requests is enabled + witnessing_enabled: bool, + /// Witnessing of data requests requiring an API key is enabled + witnessing_with_keys_enabled: bool, + /// A collection of API keys indexed by the name and containing a key value, a minimum reward + /// and a block rate + api_keys: HashMap, } impl ChainManager { diff --git a/rad/src/error.rs b/rad/src/error.rs index 6c80aed14..0d1b5f84c 100644 --- a/rad/src/error.rs +++ b/rad/src/error.rs @@ -176,6 +176,9 @@ pub enum RadError { /// Error while parsing retrieval URL #[error("URL parse error: {inner}: url={url:?}")] UrlParseError { inner: url::ParseError, url: String }, + /// Error while parsing retrieval URL + #[error("No API key found in url {url:?}")] + UrlApiKeyError { url: String }, /// Timeout during retrieval phase #[error("Timeout during retrieval phase")] RetrieveTimeout, diff --git a/rad/src/lib.rs b/rad/src/lib.rs index 238810ade..929359a24 100644 --- a/rad/src/lib.rs +++ b/rad/src/lib.rs @@ -246,7 +246,7 @@ pub fn run_retrieval_with_data_report( settings: RadonScriptExecutionSettings, ) -> Result> { match retrieve.kind { - RADType::HttpGet | RADType::HttpPost => { + RADType::HttpGet | RADType::HttpPost | RADType::HttpGetKey | RADType::HttpPostKey => { string_response_with_data_report(retrieve, response, context, settings) } RADType::Rng => rng_response_with_data_report(response, context), @@ -308,22 +308,26 @@ async fn http_response( // Populate the builder and generate the body for different types of retrievals let mut request_builder = match retrieve.kind { - RADType::HttpGet => client - .clone() - .get( - witnet_net::Uri::parse(&retrieve.url).map_err(|e| RadError::UrlParseError { - url: retrieve.url.clone(), - inner: e, - })?, - ), - RADType::HttpPost => client - .clone() - .post( - witnet_net::Uri::parse(&retrieve.url).map_err(|e| RadError::UrlParseError { - url: retrieve.url.clone(), - inner: e, - })?, - ), + RADType::HttpGet | RADType::HttpGetKey => { + client + .clone() + .get(witnet_net::Uri::parse(&retrieve.url).map_err(|e| { + RadError::UrlParseError { + url: retrieve.url.clone(), + inner: e, + } + })?) + } + RADType::HttpPost | RADType::HttpPostKey => { + client + .clone() + .post(witnet_net::Uri::parse(&retrieve.url).map_err(|e| { + RadError::UrlParseError { + url: retrieve.url.clone(), + inner: e, + } + })?) + } RADType::HttpHead => client .clone() .head( @@ -340,7 +344,9 @@ async fn http_response( // Using `Vec` as the body sets the content type header to `application/octet-stream` { request_builder = match retrieve.kind { - RADType::HttpPost => request_builder.body(WitnetHttpBody::from(retrieve.body.clone())), + RADType::HttpPost | RADType::HttpPostKey => { + request_builder.body(WitnetHttpBody::from(retrieve.body.clone())) + } _ => request_builder, }; @@ -491,10 +497,12 @@ pub async fn run_retrieval_report( context.set_protocol_version(protocol_version); match retrieve.kind { - RADType::HttpGet => http_response(retrieve, context, settings, client).await, + RADType::HttpGet + | RADType::HttpPost + | RADType::HttpHead + | RADType::HttpGetKey + | RADType::HttpPostKey => http_response(retrieve, context, settings, client).await, RADType::Rng => rng_response(context, settings).await, - RADType::HttpPost => http_response(retrieve, context, settings, client).await, - RADType::HttpHead => http_response(retrieve, context, settings, client).await, _ => Err(RadError::UnknownRetrieval), } } @@ -532,7 +540,7 @@ pub async fn run_paranoid_retrieval( witnessing: WitnessingConfig, ) -> Result> { // We can skip paranoid checks for retrieval types that don't use networking (e.g. RNG) - if !retrieve.kind.is_http() { + if !retrieve.kind.is_http() && !retrieve.kind.is_http_key() { return run_retrieval_report(retrieve, settings, active_wips, protocol_version, None).await; } diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto index fc0d7c044..432b2bae7 100644 --- a/schemas/witnet/witnet.proto +++ b/schemas/witnet/witnet.proto @@ -180,6 +180,8 @@ message DataRequestOutput { Rng = 2; HttpPost = 3; HttpHead = 4; + HttpGetKey = 5; + HttpPostKey = 6; } message RADFilter { uint32 op = 1; diff --git a/validations/src/eligibility/current.rs b/validations/src/eligibility/current.rs index e8a067360..665d4b079 100644 --- a/validations/src/eligibility/current.rs +++ b/validations/src/eligibility/current.rs @@ -77,6 +77,7 @@ where witnesses: u16, round: u16, max_rounds: u16, + capability: Capability, ) -> StakesResult<(Eligible, Hash, f64), Address, Coins, Epoch> where ISK: Into
; @@ -90,11 +91,14 @@ where witnesses: u16, round: u16, max_rounds: u16, + capability: Capability, ) -> bool where ISK: Into
, { - match self.witnessing_eligibility(validator, epoch, witnesses, round, max_rounds) { + match self + .witnessing_eligibility(validator, epoch, witnesses, round, max_rounds, capability) + { Ok((eligible, _, _)) => matches!(eligible, Eligible::Yes), Err(_) => false, } @@ -199,11 +203,12 @@ where witnesses: u16, round: u16, max_rounds: u16, + capability: Capability, ) -> StakesResult<(Eligible, Hash, f64), Address, Coins, Epoch> where ISK: Into
, { - let power = match self.query_power(key, Capability::Witnessing, epoch) { + let power = match self.query_power(key, capability, epoch) { Ok(p) => p, Err(e) => { // Early exit if the stake key does not exist @@ -225,7 +230,7 @@ where )); } - let mut rank = self.by_rank(Capability::Witnessing, epoch); + let mut rank = self.by_rank(capability, epoch); let (_, max_power) = rank.next().unwrap_or_default(); // Requirement no. 2 from the WIP: @@ -324,14 +329,14 @@ mod tests { let stakes = StakesTester::default(); let isk = "validator"; - let eligibility = stakes.witnessing_eligibility(isk, 0, 10, 0, 4); + let eligibility = stakes.witnessing_eligibility(isk, 0, 10, 0, 4, Capability::Witnessing); assert!(matches!( eligibility, Ok((Eligible::No(IneligibilityReason::NotStaking), _, _)) )); assert!(!stakes.witnessing_eligibility_bool(isk, 0, 10, 0, 4)); - let eligibility = stakes.witnessing_eligibility(isk, 100, 10, 0, 4); + let eligibility = stakes.witnessing_eligibility(isk, 100, 10, 0, 4, Capability::Witnessing); assert!(matches!( eligibility, Ok((Eligible::No(IneligibilityReason::NotStaking), _, _)) @@ -348,14 +353,14 @@ mod tests { .add_stake(isk, 10_000_000_000, 0, true, MIN_STAKE_NANOWITS) .unwrap(); - let eligibility = stakes.witnessing_eligibility(isk, 0, 10, 0, 4); + let eligibility = stakes.witnessing_eligibility(isk, 0, 10, 0, 4, Capability::Witnessing); assert!(matches!( eligibility, Ok((Eligible::No(IneligibilityReason::InsufficientPower), _, _)) )); assert!(!stakes.witnessing_eligibility_bool(isk, 0, 10, 0, 4)); - let eligibility = stakes.witnessing_eligibility(isk, 100, 10, 0, 4); + let eligibility = stakes.witnessing_eligibility(isk, 100, 10, 0, 4, Capability::Witnessing); assert!(matches!(eligibility, Ok((Eligible::Yes, _, _)))); assert!(stakes.witnessing_eligibility_bool(isk, 100, 10, 0, 4)); } @@ -381,7 +386,7 @@ mod tests { .add_stake(isk_4, 40_000_000_000, 0, true, MIN_STAKE_NANOWITS) .unwrap(); - let eligibility = stakes.witnessing_eligibility(isk_1, 0, 2, 0, 4); + let eligibility = stakes.witnessing_eligibility(isk_1, 0, 2, 0, 4, Capability::Witnessing); // TODO: verify target hash assert!(matches!( eligibility, @@ -389,7 +394,8 @@ mod tests { )); assert!(!stakes.witnessing_eligibility_bool(isk_1, 0, 10, 0, 4)); - let eligibility = stakes.witnessing_eligibility(isk_1, 100, 2, 0, 4); + let eligibility = + stakes.witnessing_eligibility(isk_1, 100, 2, 0, 4, Capability::Witnessing); // TODO: verify target hash assert!(matches!( eligibility, diff --git a/validations/src/validations.rs b/validations/src/validations.rs index d52a01a21..222d3de75 100644 --- a/validations/src/validations.rs +++ b/validations/src/validations.rs @@ -14,6 +14,7 @@ use witnet_crypto::{ signature::{PublicKey, Signature, verify}, }; use witnet_data_structures::{ + capabilities::Capability, chain::{ Block, BlockMerkleRoots, CheckpointBeacon, CheckpointVRF, ConsensusConstants, ConsensusConstantsWit2, DataRequestOutput, DataRequestStage, DataRequestState, Epoch, @@ -377,7 +378,12 @@ pub fn validate_rad_request( } for path in retrieval_paths { - if active_wips.wip0020() { + if (path.kind == RADType::HttpGetKey || path.kind == RADType::HttpPostKey) + && !path.api_key_in_url() + && !path.api_key_in_headers() + { + return Err(DataRequestError::RequestWithoutApiKey.into()); + } else if active_wips.wip0020() { path.check_fields()?; unpack_radon_script(path.script.as_slice())?; @@ -703,6 +709,7 @@ pub fn validate_commit_transaction( dr_state.data_request.witnesses, dr_state.info.current_commit_round, max_rounds, + Capability::Witnessing, ) { Ok((eligibility, target_hash, _)) => { if matches!(eligibility, Eligible::No(_)) { diff --git a/witnet.toml b/witnet.toml index fb31da670..ac988e82e 100644 --- a/witnet.toml +++ b/witnet.toml @@ -86,7 +86,7 @@ ws_address = "127.0.0.1:21340" update_period_seconds = 1024 [mining] -# Enable or disable mining and participation in resolving data requests. +# Enable or disable mining of blocks. enabled = true # Limit the number of retrievals that the node will perform during a single epoch. Due to the locking, highly # side-effected nature of performing HTTP GET requests, a limit needs to be enforced on the number of retrievals that @@ -123,6 +123,30 @@ paranoid_percentage = 51 # The currently supported proxy protocols are HTTP, HTTPS, SOCKS4 (with and without authentication) and SOCKS5 (with and # without authentication) proxies = [] +# Enable witnessing of data requests by default. +enabled = true +# Disable witnessing of data requests which requires an API key by default. Note that uncommenting below section to +# add API keys will not enable this boolean, it needs to be turned on separately. This option serves to disable +# witnessing of data requests using an API key without having to clear below section. +enabled_with_keys = false + +# Define API keys by uncommenting below section and fields +# [[witnessing.api_keys]] +# This is the name of the API key as it is used as a wildcard in the URL or headers of a data request. +# id = "MY_API_KEY_1" +# The value of the API key your node should subsitute in the wildcard. +# key = "A_real_API_key_1" +# The minimum reward in nanowit you want as a node operator to solve a data request with the specified API key. +# reward = 1_000_000_000 +# The number of blocks which have to elapse between the resolving of two subsequent data requests with this key. +# This can serve as a rate limiter to prevent your API key being blacklisted or your credits running out. +# rate = 10 +# +# [[witnessing.api_keys]] +# id = "MY_API_KEY_2" +# key = "A_real_API_key_2" +# reward = 1_000_000_000 +# rate = 10 [log] # Logging level, i.e. from more verbose to quieter: "trace" > "debug" > "info" > "warn" > "error" > "none" From 5e49b14111cb334dcdeb2a533f16366848a19f85 Mon Sep 17 00:00:00 2001 From: drcpu Date: Sat, 4 Oct 2025 21:13:52 +0200 Subject: [PATCH 3/5] feat(jsonrpc): add RPC endpoints to manipulate the registered API keys (add, get, remove) --- node/src/actors/chain_manager/handlers.rs | 101 ++++++++++++++++-- node/src/actors/json_rpc/api.rs | 119 ++++++++++++++++++++-- node/src/actors/messages.rs | 53 ++++++++++ src/cli/node/json_rpc_client.rs | 84 ++++++++++++++- src/cli/node/with_node.rs | 61 +++++++++++ 5 files changed, 398 insertions(+), 20 deletions(-) diff --git a/node/src/actors/chain_manager/handlers.rs b/node/src/actors/chain_manager/handlers.rs index a43051712..3830dc96c 100644 --- a/node/src/actors/chain_manager/handlers.rs +++ b/node/src/actors/chain_manager/handlers.rs @@ -42,16 +42,17 @@ use crate::{ actors::{ chain_manager::{BlockCandidate, handlers::BlockBatches::*}, messages::{ - AddBlocks, AddCandidates, AddCommitReveal, AddSuperBlock, AddSuperBlockVote, - AddTransaction, Broadcast, BuildDrt, BuildStake, BuildUnstake, BuildVtt, - EpochNotification, EstimatePriority, GetBalance, GetBalance2, GetBalanceTarget, - GetBlocksEpochRange, GetDataRequestInfo, GetHighestCheckpointBeacon, + AddApiKey, AddBlocks, AddCandidates, AddCommitReveal, AddSuperBlock, AddSuperBlockVote, + AddTransaction, ApiKeyData, Broadcast, BuildDrt, BuildStake, BuildUnstake, BuildVtt, + EpochNotification, EstimatePriority, GetApiKeys, GetBalance, GetBalance2, + GetBalanceTarget, GetBlocksEpochRange, GetDataRequestInfo, GetHighestCheckpointBeacon, GetMemoryTransaction, GetMempool, GetMempoolResult, GetNodeStats, GetProtocolInfo, GetReputation, GetReputationResult, GetSignalingInfo, GetState, GetSuperBlockVotes, GetSupplyInfo, GetSupplyInfo2, GetUtxoInfo, IsConfirmedBlock, PeersBeacons, - QueryStakes, QueryStakesOrderByOptions, QueryStakingPowers, ReputationStats, Rewind, - SendLastBeacon, SessionUnitResult, SetLastBeacon, SetPeersLimits, SignalingInfo, - SnapshotExport, SnapshotImport, TryMineBlock, try_do_magic_into_pkh, + QueryStakes, QueryStakesOrderByOptions, QueryStakingPowers, RemoveApiKey, + ReputationStats, Rewind, SendLastBeacon, SessionUnitResult, SetLastBeacon, + SetPeersLimits, SignalingInfo, SnapshotExport, SnapshotImport, TryMineBlock, + try_do_magic_into_pkh, }, sessions_manager::SessionsManager, }, @@ -2471,6 +2472,92 @@ impl Handler for ChainManager { } } +impl Handler for ChainManager { + type Result = ::Result; + + fn handle(&mut self, _msg: GetApiKeys, _ctx: &mut Self::Context) -> Self::Result { + let api_keys = self + .api_keys + .iter() + .map(|(id, (key, reward, rate, last_epoch))| ApiKeyData { + id: id.to_string(), + key: key.to_string(), + reward: *reward, + rate: *rate, + last_epoch: *last_epoch, + }) + .collect(); + + Ok(api_keys) + } +} + +impl Handler for ChainManager { + type Result = ::Result; + + fn handle(&mut self, msg: AddApiKey, _ctx: &mut Self::Context) -> Self::Result { + // Prepend and append the placeholder delimiter symbols if necessary + let id = if msg.id.chars().nth(0).unwrap() == '<' { + if msg.id.chars().nth(msg.id.len() - 1).unwrap() == '>' { + msg.id + } else { + msg.id + ">" + } + } else { + "<".to_owned() + &msg.id + ">" + }; + + let success = if self.api_keys.contains_key(&id) { + if msg.replace { + self.api_keys.insert( + id, + (msg.key, msg.reward.unwrap_or(0), msg.rate.unwrap_or(0), 0), + ); + + true + } else { + false + } + } else { + self.api_keys.insert( + id, + (msg.key, msg.reward.unwrap_or(0), msg.rate.unwrap_or(0), 0), + ); + + true + }; + + Ok(success) + } +} + +impl Handler for ChainManager { + type Result = ::Result; + + fn handle(&mut self, msg: RemoveApiKey, _ctx: &mut Self::Context) -> Self::Result { + // Prepend and append the placeholder delimiter symbols if necessary + let id = if msg.id.chars().nth(0).unwrap() == '<' { + if msg.id.chars().nth(msg.id.len() - 1).unwrap() == '>' { + msg.id + } else { + msg.id + ">" + } + } else { + "<".to_owned() + &msg.id + ">" + }; + + let success = if self.api_keys.contains_key(&id) { + self.api_keys.remove(&id); + + true + } else { + false + }; + + Ok(success) + } +} + #[derive(Debug, Eq, PartialEq)] pub enum BlockBatches { TargetNotReached(Vec), diff --git a/node/src/actors/json_rpc/api.rs b/node/src/actors/json_rpc/api.rs index c613f11d1..8bd85558d 100644 --- a/node/src/actors/json_rpc/api.rs +++ b/node/src/actors/json_rpc/api.rs @@ -28,16 +28,16 @@ use crate::{ inventory_manager::{InventoryManager, InventoryManagerError}, json_rpc::Subscriptions, messages::{ - AddCandidates, AddPeers, AddTransaction, AuthorizeStake, BuildDrt, BuildStake, - BuildStakeParams, BuildStakeResponse, BuildUnstake, BuildUnstakeParams, BuildVtt, - ClearPeers, DropAllPeers, EstimatePriority, GetBalance, GetBalance2, GetBalance2Limits, - GetBalanceTarget, GetBlocksEpochRange, GetConsolidatedPeers, GetDataRequestInfo, - GetEpoch, GetEpochConstants, GetHighestCheckpointBeacon, GetItemBlock, - GetItemSuperblock, GetItemTransaction, GetKnownPeers, GetMemoryTransaction, GetMempool, - GetNodeStats, GetProtocolInfo, GetReputation, GetSignalingInfo, GetState, - GetSupplyInfo, GetSupplyInfo2, GetUtxoInfo, InitializePeers, IsConfirmedBlock, - MagicEither, QueryStakes, QueryStakingPowers, Rewind, SnapshotExport, SnapshotImport, - StakeAuthorization, + AddApiKey, AddCandidates, AddPeers, AddTransaction, AuthorizeStake, BuildDrt, + BuildStake, BuildStakeParams, BuildStakeResponse, BuildUnstake, BuildUnstakeParams, + BuildVtt, ClearPeers, DropAllPeers, EstimatePriority, GetApiKeys, GetBalance, + GetBalance2, GetBalance2Limits, GetBalanceTarget, GetBlocksEpochRange, + GetConsolidatedPeers, GetDataRequestInfo, GetEpoch, GetEpochConstants, + GetHighestCheckpointBeacon, GetItemBlock, GetItemSuperblock, GetItemTransaction, + GetKnownPeers, GetMemoryTransaction, GetMempool, GetNodeStats, GetProtocolInfo, + GetReputation, GetSignalingInfo, GetState, GetSupplyInfo, GetSupplyInfo2, GetUtxoInfo, + InitializePeers, IsConfirmedBlock, MagicEither, QueryStakes, QueryStakingPowers, + RemoveApiKey, Rewind, SnapshotExport, SnapshotImport, StakeAuthorization, }, peers_manager::PeersManager, sessions_manager::SessionsManager, @@ -315,6 +315,33 @@ pub fn attach_sensitive_methods( |params| unstake(params.parse()), )) }); + + server.add_actix_method(system, "getApiKeys", move |params| { + Box::pin(if_authorized( + enable_sensitive_methods, + "getApiKeys", + params, + |_params| get_api_keys(), + )) + }); + + server.add_actix_method(system, "addApiKey", move |params| { + Box::pin(if_authorized( + enable_sensitive_methods, + "addApiKey", + params, + |params| add_api_key(params.parse()), + )) + }); + + server.add_actix_method(system, "removeApiKey", move |params| { + Box::pin(if_authorized( + enable_sensitive_methods, + "removeApiKey", + params, + |params| remove_api_key(params.parse()), + )) + }); } fn extract_topic_and_params(params: Params) -> Result<(String, Value), Error> { @@ -3078,6 +3105,78 @@ async fn get_block_epoch(block_hash: Hash) -> Result<(u32, bool), Error> { } } +/// Get all registered API keys +pub async fn get_api_keys() -> JsonRpcResult { + let chain_manager_addr = ChainManager::from_registry(); + + chain_manager_addr + .send(GetApiKeys {}) + .map(|res| { + res.map_err(internal_error) + .and_then(|api_keys| match api_keys { + Ok(x) => match serde_json::to_value(x) { + Ok(x) => Ok(x), + Err(e) => { + let err = internal_error_s(e); + Err(err) + } + }, + Err(e) => Err(internal_error_s(e)), + }) + }) + .await +} + +/// Add a (new) API keys +pub async fn add_api_key(params: Result) -> JsonRpcResult { + // Short-circuit if parameters are wrong + let msg = params?; + + let chain_manager_addr = ChainManager::from_registry(); + + chain_manager_addr + .send(msg) + .map(|res| { + res.map_err(internal_error) + .and_then(|success| match success { + Ok(x) => match serde_json::to_value(x) { + Ok(x) => Ok(x), + Err(e) => { + let err = internal_error_s(e); + Err(err) + } + }, + Err(e) => Err(internal_error_s(e)), + }) + }) + .await +} + +/// Remove an API keys +pub async fn remove_api_key(params: Result) -> JsonRpcResult { + // Short-circuit if parameters are wrong + let msg = params?; + + let chain_manager_addr = ChainManager::from_registry(); + + chain_manager_addr + .send(msg) + .map(|res| { + res.map_err(internal_error) + .and_then(|success| match success { + Ok(x) => match serde_json::to_value(x) { + Ok(x) => Ok(x), + Err(e) => { + let err = internal_error_s(e); + Err(err) + } + }, + Err(e) => Err(internal_error_s(e)), + }) + }) + .await +} + #[cfg(test)] mod mock_actix { use actix::{MailboxError, Message}; diff --git a/node/src/actors/messages.rs b/node/src/actors/messages.rs index 43e4e307e..ca36d23ae 100644 --- a/node/src/actors/messages.rs +++ b/node/src/actors/messages.rs @@ -835,6 +835,59 @@ impl Message for SetEpochConstants { type Result = (); } +/// Get all registered API keys +#[derive(Clone, Debug)] +pub struct GetApiKeys; + +/// Return value for an API key +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct ApiKeyData { + /// The API key id + pub id: String, + /// The API key value + pub key: String, + /// Reward: the minimum reward required before a data request requiring this API key will be solved + pub reward: u64, + /// Rate: blocks between solving two consecutive data requests requiring the same API key + pub rate: u32, + /// Last epoch: the last epoch when this API key was used to attempt to solve a data request + pub last_epoch: u32, +} + +impl Message for GetApiKeys { + type Result = Result, anyhow::Error>; +} + +/// Add a new API key +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct AddApiKey { + /// The API key id + pub id: String, + /// The API key value + pub key: String, + /// Reward: the minimum reward required before a data request requiring this API key will be solved + pub reward: Option, + /// Rate: blocks between solving two consecutive data requests requiring the same API key + pub rate: Option, + /// Replace the API key if it already exists + pub replace: bool, +} + +impl Message for AddApiKey { + type Result = Result; +} + +/// Add an API key +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct RemoveApiKey { + /// The API key id + pub id: String, +} + +impl Message for RemoveApiKey { + type Result = Result; +} + //////////////////////////////////////////////////////////////////////////////////////// // MESSAGES FROM CONNECTIONS MANAGER //////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/cli/node/json_rpc_client.rs b/src/cli/node/json_rpc_client.rs index 79eac0541..08984872f 100644 --- a/src/cli/node/json_rpc_client.rs +++ b/src/cli/node/json_rpc_client.rs @@ -54,9 +54,10 @@ use witnet_node::actors::{ AddrType, GetBlockChainParams, GetTransactionOutput, PeersResult, QueryStakingPowersRecord, }, messages::{ - AuthorizeStake, BuildDrt, BuildStakeParams, BuildStakeResponse, BuildUnstakeParams, - BuildVtt, GetBalanceTarget, GetReputationResult, MagicEither, QueryStakes, - QueryStakesFilter, QueryStakingPowers, SignalingInfo, StakeAuthorization, + AddApiKey, ApiKeyData, AuthorizeStake, BuildDrt, BuildStakeParams, BuildStakeResponse, + BuildUnstakeParams, BuildVtt, GetBalanceTarget, GetReputationResult, MagicEither, + QueryStakes, QueryStakesFilter, QueryStakingPowers, RemoveApiKey, SignalingInfo, + StakeAuthorization, }, }; use witnet_rad::types::RadonTypes; @@ -2035,6 +2036,83 @@ pub fn query_powers( Ok(()) } +pub fn get_api_keys(addr: SocketAddr) -> Result<(), anyhow::Error> { + let mut stream = start_client(addr)?; + let request = r#"{"jsonrpc": "2.0","method": "getApiKeys", "id": "1"}"#; + let response = send_request(&mut stream, request)?; + let api_keys: Vec = parse_response(&response)?; + + for api_key in api_keys.iter() { + println!(""); + println!("[{}]", api_key.id[1..api_key.id.len() - 1].to_string()); + println!("key: {}", api_key.key); + println!( + "reward: {} WIT", + Wit::from_nanowits(api_key.reward).to_string() + ); + println!("rate: 1 per {} blocks", api_key.rate); + println!( + "API key was used last in data request at epoch {}\n", + api_key.last_epoch + ); + } + + Ok(()) +} + +pub fn add_api_key( + addr: SocketAddr, + id: String, + key: String, + reward: Option, + rate: Option, + replace: bool, +) -> Result<(), anyhow::Error> { + let mut stream = start_client(addr)?; + + let params = AddApiKey { + id, + key, + reward, + rate, + replace, + }; + + let request = format!( + r#"{{"jsonrpc": "2.0","method": "addApiKey", "params": {}, "id": "1"}}"#, + serde_json::to_string(¶ms).unwrap() + ); + let response = send_request(&mut stream, &request)?; + let response: bool = parse_response(&response)?; + if response { + println!("Successfully added API key"); + } else { + bail!("Failed to add API key"); + } + + Ok(()) +} + +pub fn remove_api_key(addr: SocketAddr, id: String) -> Result<(), anyhow::Error> { + let mut stream = start_client(addr)?; + + let params = RemoveApiKey { id }; + + let request = format!( + r#"{{"jsonrpc": "2.0","method": "removeApiKey", "params": {}, "id": "1"}}"#, + serde_json::to_string(¶ms).unwrap() + ); + let response = send_request(&mut stream, &request)?; + let response: bool = parse_response(&response)?; + if response { + println!("Successfully removed API key"); + } else { + bail!("Failed to removed API key"); + } + + Ok(()) +} + #[derive(Serialize, Deserialize)] struct SignatureWithData { address: String, diff --git a/src/cli/node/with_node.rs b/src/cli/node/with_node.rs index d06791a15..da06fe1e5 100644 --- a/src/cli/node/with_node.rs +++ b/src/cli/node/with_node.rs @@ -315,6 +315,25 @@ pub fn exec_cmd( fee, dry_run, ), + Command::GetApiKeys { node } => rpc::get_api_keys(node.unwrap_or(default_jsonrpc)), + Command::AddApiKey { + node, + id, + key, + reward, + rate, + replace, + } => rpc::add_api_key( + node.unwrap_or(default_jsonrpc), + id, + key, + reward, + rate, + replace, + ), + Command::RemoveApiKey { node, id } => { + rpc::remove_api_key(node.unwrap_or(default_jsonrpc), id) + } } } @@ -869,6 +888,48 @@ pub enum Command { #[structopt(long = "dry-run")] dry_run: bool, }, + #[structopt(name = "getApiKeys", about = "Get the registered API keys")] + GetApiKeys { + /// Socket address of the Witnet node to query + #[structopt(short = "n", long = "node")] + node: Option, + }, + #[structopt( + name = "addApiKey", + about = "Add a new API key while the node is running" + )] + AddApiKey { + /// Socket address of the Witnet node to query + #[structopt(short = "n", long = "node")] + node: Option, + /// The id of the key used as a placeholder in a data request + #[structopt(long = "id")] + id: String, + /// The actual API key + #[structopt(long = "key")] + key: String, + /// The minimum reward expected by this node to solve a data request with this API key + #[structopt(long = "reward")] + reward: Option, + /// The minimum number of blocks between two subsequent data requests + #[structopt(long = "rate")] + rate: Option, + /// Whether to replace an already existing key with this one + #[structopt(long = "replace")] + replace: bool, + }, + #[structopt( + name = "removeApiKey", + about = "Remove an API key while the node is running" + )] + RemoveApiKey { + /// Socket address of the Witnet node to query + #[structopt(short = "n", long = "node")] + node: Option, + /// The id of the key used as a placeholder in a data request + #[structopt(long = "id")] + id: String, + }, } #[derive(Debug, StructOpt)] From 514c2a8a1cba6cb139fe4a3687ba4eb8f8c98520 Mon Sep 17 00:00:00 2001 From: drcpu Date: Tue, 7 Oct 2025 23:05:13 +0200 Subject: [PATCH 4/5] feat(node): share registered API keys in a custom message --- data_structures/src/builders.rs | 14 ++- data_structures/src/chain/mod.rs | 5 + data_structures/src/proto/versioning.rs | 6 ++ data_structures/src/types.rs | 42 +++++++- node/src/actors/chain_manager/handlers.rs | 116 +++++++++++++++++++--- node/src/actors/chain_manager/mod.rs | 2 + node/src/actors/messages.rs | 33 +++++- node/src/actors/session/handlers.rs | 111 ++++++++++++++++++--- schemas/witnet/witnet.proto | 14 +++ validations/src/validations.rs | 5 + 10 files changed, 316 insertions(+), 32 deletions(-) diff --git a/data_structures/src/builders.rs b/data_structures/src/builders.rs index f1bf1cbfe..490543c34 100644 --- a/data_structures/src/builders.rs +++ b/data_structures/src/builders.rs @@ -13,7 +13,7 @@ use crate::{ transaction::Transaction, types::{ Address, Command, GetPeers, InventoryAnnouncement, InventoryRequest, IpAddress, LastBeacon, - Message, Peers, Verack, Version, + Message, Peers, RegisteredApiKeys, SignedRegisteredApiKeys, Verack, Version, }, }; @@ -150,6 +150,18 @@ impl Message { Message::build_message(magic, Command::SuperBlockVote(superblock_vote)) } + /// Function to build SignedRegisteredApiKeys messages + pub fn build_api_keys_message( + magic: u16, + keys: RegisteredApiKeys, + signature: KeyedSignature, + ) -> Message { + Message::build_message( + magic, + Command::SignedRegisteredApiKeys(SignedRegisteredApiKeys { keys, signature }), + ) + } + /// Function to build a message from a command fn build_message(magic: u16, command: Command) -> Message { Message { diff --git a/data_structures/src/chain/mod.rs b/data_structures/src/chain/mod.rs index 3025ebf4a..fc9284cb2 100644 --- a/data_structures/src/chain/mod.rs +++ b/data_structures/src/chain/mod.rs @@ -55,6 +55,7 @@ use crate::{ RevealTransaction, StakeTransaction, TallyTransaction, Transaction, TxInclusionProof, UnstakeTransaction, VTTransaction, }, + types::RegisteredApiKeys, utxo_pool::{OldUnspentOutputsPool, OwnUnspentOutputsPool, UnspentOutputsPool}, vrf::{BlockEligibilityClaim, DataRequestEligibilityClaim}, wit::{WIT_DECIMAL_PLACES, Wit}, @@ -4914,6 +4915,10 @@ pub enum SignaturesToVerify { SuperBlockVote { superblock_vote: SuperBlockVote, }, + SignedRegisteredApiKeys { + keys: RegisteredApiKeys, + signature: KeyedSignature, + }, } // Auxiliar functions for test diff --git a/data_structures/src/proto/versioning.rs b/data_structures/src/proto/versioning.rs index 78642fadb..6818f2517 100644 --- a/data_structures/src/proto/versioning.rs +++ b/data_structures/src/proto/versioning.rs @@ -761,6 +761,9 @@ impl From for LegacyMessage_LegacyCommand_oneof_kind Message_Command_oneof_kind::SuperBlock(x) => { LegacyMessage_LegacyCommand_oneof_kind::SuperBlock(x) } + Message_Command_oneof_kind::SignedRegisteredApiKeys(x) => { + LegacyMessage_LegacyCommand_oneof_kind::SignedRegisteredApiKeys(x) + } } } } @@ -801,6 +804,9 @@ impl From for Message_Command_oneof_kind LegacyMessage_LegacyCommand_oneof_kind::SuperBlock(x) => { Message_Command_oneof_kind::SuperBlock(x) } + LegacyMessage_LegacyCommand_oneof_kind::SignedRegisteredApiKeys(x) => { + Message_Command_oneof_kind::SignedRegisteredApiKeys(x) + } } } } diff --git a/data_structures/src/types.rs b/data_structures/src/types.rs index 78d47eb1d..671170621 100644 --- a/data_structures/src/types.rs +++ b/data_structures/src/types.rs @@ -4,8 +4,13 @@ use std::fmt; pub use num_traits::ops::wrapping::WrappingAdd; use serde::{Deserialize, Serialize}; +use witnet_crypto::hash::calculate_sha256; + use crate::{ - chain::{Block, CheckpointBeacon, Hashable, InventoryEntry, SuperBlock, SuperBlockVote}, + chain::{ + Block, CheckpointBeacon, Epoch, Hash, Hashable, InventoryEntry, KeyedSignature, SuperBlock, + SuperBlockVote, + }, proto::{ProtobufConvert, schema::witnet}, transaction::Transaction, }; @@ -42,6 +47,9 @@ pub enum Command { // Superblock SuperBlockVote(SuperBlockVote), + + // API keys + SignedRegisteredApiKeys(SignedRegisteredApiKeys), } impl fmt::Display for Command { @@ -88,6 +96,12 @@ impl fmt::Display for Command { sbv.superblock_hash ), Command::SuperBlock(sb) => write!(f, "SUPERBLOCK #{}: {}", sb.index, sb.hash()), + Command::SignedRegisteredApiKeys(srak) => write!( + f, + "API keys at {} for {}", + srak.keys.epoch, + srak.signature.public_key.pkh(), + ), } } } @@ -147,6 +161,32 @@ pub struct LastBeacon { pub highest_superblock_checkpoint: CheckpointBeacon, } +/////////////////////////////////////////////////////////// +// API KEY MESSAGE +/////////////////////////////////////////////////////////// + +#[derive(Debug, Eq, PartialEq, Clone, ProtobufConvert)] +#[protobuf_convert(source = "witnet::RegisteredApiKeys")] +pub struct RegisteredApiKeys { + pub ids: Vec, + pub rewards: Vec, + pub rates: Vec, + pub epoch: Epoch, +} + +impl Hashable for RegisteredApiKeys { + fn hash(&self) -> Hash { + calculate_sha256(&self.to_pb_bytes().unwrap()).into() + } +} + +#[derive(Debug, Eq, PartialEq, Clone, ProtobufConvert)] +#[protobuf_convert(source = "witnet::SignedRegisteredApiKeys")] +pub struct SignedRegisteredApiKeys { + pub keys: RegisteredApiKeys, + pub signature: KeyedSignature, +} + /////////////////////////////////////////////////////////// // AUX TYPES /////////////////////////////////////////////////////////// diff --git a/node/src/actors/chain_manager/handlers.rs b/node/src/actors/chain_manager/handlers.rs index 3830dc96c..6a3667e9b 100644 --- a/node/src/actors/chain_manager/handlers.rs +++ b/node/src/actors/chain_manager/handlers.rs @@ -7,9 +7,10 @@ use std::{ }; use actix::{ActorFutureExt, WrapFuture, prelude::*}; -use futures::future::Either; - +use futures::future::{Either, FutureExt}; use itertools::Itertools; +use rand::Rng; + use witnet_data_structures::{ chain::{ Block, ChainState, CheckpointBeacon, ConsensusConstantsWit2, DataRequestInfo, Epoch, Hash, @@ -28,7 +29,7 @@ use witnet_data_structures::{ DRTransaction, StakeTransaction, Transaction, UnstakeTransaction, VTTransaction, }, transaction_factory::{self, NodeBalance, NodeBalance2}, - types::LastBeacon, + types::{LastBeacon, RegisteredApiKeys}, utxo_pool::{UtxoDiff, UtxoInfo, get_utxo_info}, wit::Wit, }; @@ -42,17 +43,17 @@ use crate::{ actors::{ chain_manager::{BlockCandidate, handlers::BlockBatches::*}, messages::{ - AddApiKey, AddBlocks, AddCandidates, AddCommitReveal, AddSuperBlock, AddSuperBlockVote, - AddTransaction, ApiKeyData, Broadcast, BuildDrt, BuildStake, BuildUnstake, BuildVtt, - EpochNotification, EstimatePriority, GetApiKeys, GetBalance, GetBalance2, - GetBalanceTarget, GetBlocksEpochRange, GetDataRequestInfo, GetHighestCheckpointBeacon, - GetMemoryTransaction, GetMempool, GetMempoolResult, GetNodeStats, GetProtocolInfo, - GetReputation, GetReputationResult, GetSignalingInfo, GetState, GetSuperBlockVotes, - GetSupplyInfo, GetSupplyInfo2, GetUtxoInfo, IsConfirmedBlock, PeersBeacons, - QueryStakes, QueryStakesOrderByOptions, QueryStakingPowers, RemoveApiKey, - ReputationStats, Rewind, SendLastBeacon, SessionUnitResult, SetLastBeacon, - SetPeersLimits, SignalingInfo, SnapshotExport, SnapshotImport, TryMineBlock, - try_do_magic_into_pkh, + AddApiKey, AddBlocks, AddCandidates, AddCommitReveal, AddReceivedApiKeys, + AddSuperBlock, AddSuperBlockVote, AddTransaction, ApiKeyData, Broadcast, BuildDrt, + BuildStake, BuildUnstake, BuildVtt, EpochNotification, EstimatePriority, GetApiKeys, + GetBalance, GetBalance2, GetBalanceTarget, GetBlocksEpochRange, GetDataRequestInfo, + GetHighestCheckpointBeacon, GetMemoryTransaction, GetMempool, GetMempoolResult, + GetNodeStats, GetProtocolInfo, GetReputation, GetReputationResult, GetSignalingInfo, + GetState, GetSuperBlockVotes, GetSupplyInfo, GetSupplyInfo2, GetUtxoInfo, + IsConfirmedBlock, PeersBeacons, QueryStakes, QueryStakesOrderByOptions, + QueryStakingPowers, RemoveApiKey, ReputationStats, Rewind, SendLastBeacon, + SendRegisteredApiKeys, SessionUnitResult, SetLastBeacon, SetPeersLimits, SignalingInfo, + SnapshotExport, SnapshotImport, TryMineBlock, try_do_magic_into_pkh, }, sessions_manager::SessionsManager, }, @@ -150,6 +151,42 @@ impl Handler> for ChainManager { refresh_protocol_version(current_epoch); } + // Periodically, but randomly send an API keys message + let mut rng = rand::thread_rng(); + if rng.gen_range(0, 10) <= 1 { + let (mut ids, mut rewards, mut rates) = + (Vec::::new(), Vec::::new(), Vec::::new()); + for (id, (_, reward, rate, _)) in self.api_keys.iter() { + ids.push(id.to_string()); + rewards.push(*reward); + rates.push(*rate); + } + + let keys = RegisteredApiKeys { + ids, + rewards, + rates, + epoch: msg.checkpoint, + }; + + // Create a hash of the api_keys struct and sign it + signature_mngr::sign_data(keys.hash().data()) + .map(move |res| { + res.map_err(|e| log::error!("Failed to sign API keys message: {e}")) + .map(|signature| { + log::debug!("Broadcasting API keys message at epoch {}", keys.epoch); + + SessionsManager::from_registry().do_send(Broadcast { + command: SendRegisteredApiKeys { keys, signature }, + only_inbound: false, + }); + }) + }) + .into_actor(self) + .map(|_res: Result<(), ()>, _act, _ctx| ()) + .wait(ctx); + } + match self.sm_state { StateMachine::WaitingConsensus => { if let Some(chain_info) = &self.chain_state.chain_info { @@ -797,6 +834,57 @@ impl Handler for ChainManager { } } +/// Handler for AddReceivedApiKeys message +impl Handler for ChainManager { + type Result = Result<(), anyhow::Error>; + + fn handle( + &mut self, + AddReceivedApiKeys { keys, signature }: AddReceivedApiKeys, + _ctx: &mut Context, + ) -> Self::Result { + let sender = signature.public_key.pkh(); + // Already have a set of API keys from this node + if self.received_api_keys.contains_key(&sender) { + // Based on the epoch, received an updated set of keys, track and forward them + if self.received_api_keys.get(&sender).unwrap().0 != keys.epoch { + let key_data = keys + .ids + .iter() + .zip(keys.rewards.iter()) + .zip(keys.rates.iter()) + .map(|((id, reward), rate)| (id.to_string(), *reward, *rate)) + .collect(); + self.received_api_keys + .insert(sender, (keys.epoch, key_data)); + + SessionsManager::from_registry().do_send(Broadcast { + command: SendRegisteredApiKeys { keys, signature }, + only_inbound: false, + }); + } + } else { + // Never received keys from this node, track and forward them + let key_data = keys + .ids + .iter() + .zip(keys.rewards.iter()) + .zip(keys.rates.iter()) + .map(|((id, reward), rate)| (id.to_string(), *reward, *rate)) + .collect(); + self.received_api_keys + .insert(sender, (keys.epoch, key_data)); + + SessionsManager::from_registry().do_send(Broadcast { + command: SendRegisteredApiKeys { keys, signature }, + only_inbound: false, + }); + } + + Ok(()) + } +} + /// Handler for GetBlocksEpochRange impl Handler for ChainManager { type Result = Result, ChainManagerError>; diff --git a/node/src/actors/chain_manager/mod.rs b/node/src/actors/chain_manager/mod.rs index 634d3c317..ef4316122 100644 --- a/node/src/actors/chain_manager/mod.rs +++ b/node/src/actors/chain_manager/mod.rs @@ -267,6 +267,8 @@ pub struct ChainManager { /// A collection of API keys indexed by the name and containing a key value, a minimum reward /// and a block rate api_keys: HashMap, + /// Track the received API key messages + received_api_keys: HashMap)>, } impl ChainManager { diff --git a/node/src/actors/messages.rs b/node/src/actors/messages.rs index ca36d23ae..6625929ba 100644 --- a/node/src/actors/messages.rs +++ b/node/src/actors/messages.rs @@ -35,7 +35,7 @@ use witnet_data_structures::{ UnstakeTransaction, VTTransaction, }, transaction_factory::{NodeBalance, NodeBalance2}, - types::LastBeacon, + types::{LastBeacon, RegisteredApiKeys}, utxo_pool::{UtxoInfo, UtxoSelectionStrategy}, wit::{WIT_DECIMAL_PLACES, Wit}, }; @@ -1391,6 +1391,37 @@ impl fmt::Display for SendSuperBlockVote { } } +/// Messages to send and receive registered API keys through the network +#[derive(Clone, Debug)] +pub struct SendRegisteredApiKeys { + /// The API key data + pub keys: RegisteredApiKeys, + /// The signature associated with this message + pub signature: KeyedSignature, +} + +impl Message for SendRegisteredApiKeys { + type Result = (); +} + +impl fmt::Display for SendRegisteredApiKeys { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "SendRegisteredApiKeys") + } +} + +/// Add a superblock vote +pub struct AddReceivedApiKeys { + /// The API key data + pub keys: RegisteredApiKeys, + /// The signature associated with this message + pub signature: KeyedSignature, +} + +impl Message for AddReceivedApiKeys { + type Result = Result<(), anyhow::Error>; +} + /// Message to close an open session #[derive(Clone, Debug)] pub struct CloseSession; diff --git a/node/src/actors/session/handlers.rs b/node/src/actors/session/handlers.rs index db90f9f5e..4d9c30436 100644 --- a/node/src/actors/session/handlers.rs +++ b/node/src/actors/session/handlers.rs @@ -5,38 +5,42 @@ use actix::{ StreamHandler, SystemService, WrapFuture, io::WriteHandler, }; use bytes::BytesMut; -use futures::future::Either; +use futures::future::{Either, FutureExt}; use thiserror::Error; use witnet_data_structures::{ builders::from_address, chain::{ - Block, CheckpointBeacon, Epoch, InventoryEntry, InventoryItem, SuperBlock, SuperBlockVote, + Block, CheckpointBeacon, Epoch, InventoryEntry, InventoryItem, KeyedSignature, + SignaturesToVerify, SuperBlock, SuperBlockVote, }, get_protocol_version, proto::versioning::{Versioned, VersionedHashable}, transaction::Transaction, types::{ Address, Command, InventoryAnnouncement, InventoryRequest, LastBeacon, - Message as WitnetMessage, Peers, Version, + Message as WitnetMessage, Peers, RegisteredApiKeys, SignedRegisteredApiKeys, Version, }, }; use witnet_p2p::sessions::{SessionStatus, SessionType}; use witnet_util::timestamp::get_timestamp; -use crate::actors::{ - chain_manager::ChainManager, - inventory_manager::InventoryManager, - messages::{ - AddBlocks, AddCandidates, AddConsolidatedPeer, AddPeers, AddSuperBlock, AddSuperBlockVote, - AddTransaction, CloseSession, Consolidate, EpochNotification, GetBlocksEpochRange, - GetHighestCheckpointBeacon, GetItem, GetSuperBlockVotes, PeerBeacon, - RemoveAddressesFromTried, RequestPeers, SendGetPeers, SendInventoryAnnouncement, - SendInventoryItem, SendInventoryRequest, SendLastBeacon, SendSuperBlockVote, - SessionUnitResult, +use crate::{ + actors::{ + chain_manager::ChainManager, + inventory_manager::InventoryManager, + messages::{ + AddBlocks, AddCandidates, AddConsolidatedPeer, AddPeers, AddReceivedApiKeys, + AddSuperBlock, AddSuperBlockVote, AddTransaction, CloseSession, Consolidate, + EpochNotification, GetBlocksEpochRange, GetHighestCheckpointBeacon, GetItem, + GetSuperBlockVotes, PeerBeacon, RemoveAddressesFromTried, RequestPeers, SendGetPeers, + SendInventoryAnnouncement, SendInventoryItem, SendInventoryRequest, SendLastBeacon, + SendRegisteredApiKeys, SendSuperBlockVote, SessionUnitResult, + }, + peers_manager::PeersManager, + sessions_manager::SessionsManager, }, - peers_manager::PeersManager, - sessions_manager::SessionsManager, + signature_mngr, }; use super::Session; @@ -390,6 +394,16 @@ impl StreamHandler> for Session { process_superblock_vote(self, sbv) } + ///////////////////////////////// + // REGISTERED API KEYS MESSAGE // + ///////////////////////////////// + (_, SessionStatus::Consolidated, Command::SignedRegisteredApiKeys(rap)) => { + match validate_api_key_message(self, ctx, rap.clone()) { + Ok(_) => process_registered_api_keys(self, rap), + Err(_) => (), + }; + } + ///////////////////// // NOT SUPPORTED // ///////////////////// @@ -494,6 +508,22 @@ impl Handler for Session { } } +impl Handler for Session { + type Result = SessionUnitResult; + + fn handle( + &mut self, + SendRegisteredApiKeys { keys, signature }: SendRegisteredApiKeys, + _ctx: &mut Context, + ) { + log::trace!( + "Sending SignedRegisteredApiKeys to peer at {:?}", + self.remote_addr + ); + send_registered_api_keys(self, keys, signature); + } +} + impl Handler for Session { type Result = SessionUnitResult; @@ -1128,6 +1158,57 @@ fn process_superblock_vote(_session: &mut Session, superblock_vote: SuperBlockVo chain_manager_addr.do_send(AddSuperBlockVote { superblock_vote }); } +/// Function called when the `Session` actor recieves a `SendRegisteredApiKeys` message +/// Send a `SignedRegisteredApiKeys` message to this peer +fn send_registered_api_keys( + session: &mut Session, + keys: RegisteredApiKeys, + signature: KeyedSignature, +) { + let registered_api_keys_msg = + WitnetMessage::build_api_keys_message(session.magic_number, keys, signature); + // Send SignedRegisteredApiKeys msg + session.send_message(registered_api_keys_msg); +} + +fn validate_api_key_message( + session: &mut Session, + ctx: &mut Context, + signed_keys: SignedRegisteredApiKeys, +) -> Result<(), anyhow::Error> { + signature_mngr::verify_signatures(vec![SignaturesToVerify::SignedRegisteredApiKeys { + keys: signed_keys.keys, + signature: signed_keys.signature.clone(), + }]) + .map(move |res| { + res.map_err(|e| { + let sender = signed_keys.signature.public_key.pkh(); + log::error!( + "Verification of API key message failed for {}: {}", + sender, + e + ); + }) + }) + .into_actor(session) + .map(|_res: Result<(), ()>, _act, _ctx| ()) + .wait(ctx); + + Ok(()) +} + +/// Function called when SignedRegisteredApiKeys message is received from another peer +fn process_registered_api_keys( + _session: &mut Session, + SignedRegisteredApiKeys { keys, signature }: SignedRegisteredApiKeys, +) { + // Get ChainManager address + let chain_manager_addr = ChainManager::from_registry(); + + // Send a message to the ChainManager to try to validate this superblock vote + chain_manager_addr.do_send(AddReceivedApiKeys { keys, signature }); +} + #[cfg(test)] mod tests { use witnet_data_structures::chain::Hash; diff --git a/schemas/witnet/witnet.proto b/schemas/witnet/witnet.proto index 432b2bae7..69837d739 100644 --- a/schemas/witnet/witnet.proto +++ b/schemas/witnet/witnet.proto @@ -16,6 +16,7 @@ message LegacyMessage { Transaction Transaction = 9; SuperBlockVote SuperBlockVote = 10; SuperBlock SuperBlock = 11; + SignedRegisteredApiKeys SignedRegisteredApiKeys = 12; } } @@ -38,6 +39,7 @@ message Message { Transaction Transaction = 9; SuperBlockVote SuperBlockVote = 10; SuperBlock SuperBlock = 11; + SignedRegisteredApiKeys SignedRegisteredApiKeys = 12; } } @@ -456,3 +458,15 @@ message SuperBlockVote { Hash superblock_hash = 3; fixed32 superblock_index = 4; } + +message RegisteredApiKeys { + repeated string ids = 1; + repeated uint64 rewards = 2; + repeated uint32 rates = 3; + uint32 epoch = 4; +} + +message SignedRegisteredApiKeys { + RegisteredApiKeys keys = 1; + KeyedSignature signature = 2; +} diff --git a/validations/src/validations.rs b/validations/src/validations.rs index 222d3de75..0190135d3 100644 --- a/validations/src/validations.rs +++ b/validations/src/validations.rs @@ -2972,6 +2972,11 @@ pub fn verify_signatures( .unwrap(), )?; } + SignaturesToVerify::SignedRegisteredApiKeys { keys, signature } => verify( + &signature.public_key.try_into().unwrap(), + &keys.hash().data(), + &signature.signature.try_into().unwrap(), + )?, } } From e6562b928ed97c0144d567c46285805ba4e07e5c Mon Sep 17 00:00:00 2001 From: drcpu Date: Wed, 8 Oct 2025 20:19:35 +0200 Subject: [PATCH 5/5] feat(jsonrpc): extend the getApiKeys RPC to list all known API keys from all validators --- node/src/actors/chain_manager/handlers.rs | 44 +++++++++++---- node/src/actors/json_rpc/api.rs | 9 ++- node/src/actors/messages.rs | 9 ++- src/cli/node/json_rpc_client.rs | 69 ++++++++++++++++------- src/cli/node/with_node.rs | 7 ++- 5 files changed, 101 insertions(+), 37 deletions(-) diff --git a/node/src/actors/chain_manager/handlers.rs b/node/src/actors/chain_manager/handlers.rs index 6a3667e9b..c36a41c98 100644 --- a/node/src/actors/chain_manager/handlers.rs +++ b/node/src/actors/chain_manager/handlers.rs @@ -2563,18 +2563,38 @@ impl Handler for ChainManager { impl Handler for ChainManager { type Result = ::Result; - fn handle(&mut self, _msg: GetApiKeys, _ctx: &mut Self::Context) -> Self::Result { - let api_keys = self - .api_keys - .iter() - .map(|(id, (key, reward, rate, last_epoch))| ApiKeyData { - id: id.to_string(), - key: key.to_string(), - reward: *reward, - rate: *rate, - last_epoch: *last_epoch, - }) - .collect(); + fn handle(&mut self, msg: GetApiKeys, _ctx: &mut Self::Context) -> Self::Result { + let all = msg.all; + + let api_keys = if all { + let mut api_key_data = vec![]; + for (address, (epoch, api_keys)) in self.received_api_keys.iter() { + for (id, reward, rate) in api_keys.iter() { + api_key_data.push(ApiKeyData { + address: address.to_string(), + id: id.to_string(), + key: "".to_string(), + reward: *reward, + rate: *rate, + last_epoch: *epoch, + }) + } + } + + api_key_data + } else { + self.api_keys + .iter() + .map(|(id, (key, reward, rate, last_epoch))| ApiKeyData { + address: self.own_pkh.unwrap().to_string(), + id: id.to_string(), + key: key.to_string(), + reward: *reward, + rate: *rate, + last_epoch: *last_epoch, + }) + .collect() + }; Ok(api_keys) } diff --git a/node/src/actors/json_rpc/api.rs b/node/src/actors/json_rpc/api.rs index 8bd85558d..738cb3922 100644 --- a/node/src/actors/json_rpc/api.rs +++ b/node/src/actors/json_rpc/api.rs @@ -321,7 +321,7 @@ pub fn attach_sensitive_methods( enable_sensitive_methods, "getApiKeys", params, - |_params| get_api_keys(), + |params| get_api_keys(params.parse()), )) }); @@ -3106,11 +3106,14 @@ async fn get_block_epoch(block_hash: Hash) -> Result<(u32, bool), Error> { } /// Get all registered API keys -pub async fn get_api_keys() -> JsonRpcResult { +pub async fn get_api_keys(params: Result) -> JsonRpcResult { + // Short-circuit if parameters are wrong + let msg = params?; + let chain_manager_addr = ChainManager::from_registry(); chain_manager_addr - .send(GetApiKeys {}) + .send(msg) .map(|res| { res.map_err(internal_error) .and_then(|api_keys| match api_keys { diff --git a/node/src/actors/messages.rs b/node/src/actors/messages.rs index 6625929ba..ea426f2fe 100644 --- a/node/src/actors/messages.rs +++ b/node/src/actors/messages.rs @@ -836,12 +836,17 @@ impl Message for SetEpochConstants { } /// Get all registered API keys -#[derive(Clone, Debug)] -pub struct GetApiKeys; +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct GetApiKeys { + /// If enabled, return the available API key data of all validators + pub all: bool, +} /// Return value for an API key #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct ApiKeyData { + /// The address owning this API key + pub address: String, /// The API key id pub id: String, /// The API key value diff --git a/src/cli/node/json_rpc_client.rs b/src/cli/node/json_rpc_client.rs index 08984872f..2655503a2 100644 --- a/src/cli/node/json_rpc_client.rs +++ b/src/cli/node/json_rpc_client.rs @@ -55,9 +55,9 @@ use witnet_node::actors::{ }, messages::{ AddApiKey, ApiKeyData, AuthorizeStake, BuildDrt, BuildStakeParams, BuildStakeResponse, - BuildUnstakeParams, BuildVtt, GetBalanceTarget, GetReputationResult, MagicEither, - QueryStakes, QueryStakesFilter, QueryStakingPowers, RemoveApiKey, SignalingInfo, - StakeAuthorization, + BuildUnstakeParams, BuildVtt, GetApiKeys, GetBalanceTarget, GetReputationResult, + MagicEither, QueryStakes, QueryStakesFilter, QueryStakingPowers, RemoveApiKey, + SignalingInfo, StakeAuthorization, }, }; use witnet_rad::types::RadonTypes; @@ -2036,25 +2036,56 @@ pub fn query_powers( Ok(()) } -pub fn get_api_keys(addr: SocketAddr) -> Result<(), anyhow::Error> { +pub fn get_api_keys(addr: SocketAddr, all: bool) -> Result<(), anyhow::Error> { + let params = GetApiKeys { all }; let mut stream = start_client(addr)?; - let request = r#"{"jsonrpc": "2.0","method": "getApiKeys", "id": "1"}"#; - let response = send_request(&mut stream, request)?; + let request = format!( + r#"{{"jsonrpc": "2.0","method": "getApiKeys", "params": {}, "id": "1"}}"#, + serde_json::to_string(¶ms).unwrap() + ); + let response = send_request(&mut stream, &request)?; let api_keys: Vec = parse_response(&response)?; - for api_key in api_keys.iter() { - println!(""); - println!("[{}]", api_key.id[1..api_key.id.len() - 1].to_string()); - println!("key: {}", api_key.key); - println!( - "reward: {} WIT", - Wit::from_nanowits(api_key.reward).to_string() - ); - println!("rate: 1 per {} blocks", api_key.rate); - println!( - "API key was used last in data request at epoch {}\n", - api_key.last_epoch - ); + if all { + let mut last_address = ""; + for api_key in api_keys.iter() { + if last_address != api_key.address { + println!(""); + println!( + "API keys for {} since epoch {}:", + api_key.address, api_key.last_epoch + ); + println!( + "\tID: {}, reward: {}, rate: {}", + api_key.id, + Wit::from_nanowits(api_key.reward).to_string(), + api_key.rate + ); + last_address = &api_key.address; + } else { + println!( + "\tID: {}, reward: {}, rate: {}", + api_key.id, + Wit::from_nanowits(api_key.reward).to_string(), + api_key.rate + ); + } + } + } else { + for api_key in api_keys.iter() { + println!(""); + println!("[{}]", api_key.id[1..api_key.id.len() - 1].to_string()); + println!("key: {}", api_key.key); + println!( + "reward: {} WIT", + Wit::from_nanowits(api_key.reward).to_string() + ); + println!("rate: 1 per {} blocks", api_key.rate); + println!( + "API key was used last in data request at epoch {}\n", + api_key.last_epoch + ); + } } Ok(()) diff --git a/src/cli/node/with_node.rs b/src/cli/node/with_node.rs index da06fe1e5..95b198b91 100644 --- a/src/cli/node/with_node.rs +++ b/src/cli/node/with_node.rs @@ -315,7 +315,9 @@ pub fn exec_cmd( fee, dry_run, ), - Command::GetApiKeys { node } => rpc::get_api_keys(node.unwrap_or(default_jsonrpc)), + Command::GetApiKeys { node, all } => { + rpc::get_api_keys(node.unwrap_or(default_jsonrpc), all) + } Command::AddApiKey { node, id, @@ -893,6 +895,9 @@ pub enum Command { /// Socket address of the Witnet node to query #[structopt(short = "n", long = "node")] node: Option, + /// Whether to get a list of the keys from all validators in the network + #[structopt(long = "all")] + all: bool, }, #[structopt( name = "addApiKey",