From 7333525306aa5d7e5e782b1ea86457a9baf579ee Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 14 Aug 2022 15:04:15 +0300 Subject: [PATCH 01/76] Remove ffi with new `logging` and some improvements in arch and safe --- doublets-ffi/Cargo.toml | 13 +- doublets-ffi/env-decorators/Cargo.toml | 12 - doublets-ffi/env-decorators/src/lib.rs | 78 ---- doublets-ffi/ffi-attributes/src/lib.rs | 19 + doublets-ffi/src/constants.rs | 78 ++++ doublets-ffi/src/export.rs | 41 +++ doublets-ffi/src/lib.rs | 476 +------------------------ doublets-ffi/src/logging.rs | 105 ++++++ doublets-ffi/src/store.rs | 287 +++++++++++++++ 9 files changed, 553 insertions(+), 556 deletions(-) delete mode 100644 doublets-ffi/env-decorators/Cargo.toml delete mode 100644 doublets-ffi/env-decorators/src/lib.rs create mode 100644 doublets-ffi/src/constants.rs create mode 100644 doublets-ffi/src/export.rs create mode 100644 doublets-ffi/src/logging.rs create mode 100644 doublets-ffi/src/store.rs diff --git a/doublets-ffi/Cargo.toml b/doublets-ffi/Cargo.toml index f1672de..d921162 100644 --- a/doublets-ffi/Cargo.toml +++ b/doublets-ffi/Cargo.toml @@ -7,14 +7,15 @@ edition = "2021" crate-type = ["cdylib", "staticlib"] [dependencies] -log = "0.4.14" -libc = "0.2.100" -tracing-subscriber = "0.3.3" -tracing-log = "0.1.2" -tracing = "0.1.29" doublets = { path = "../doublets" } ffi-attributes = { path = "ffi-attributes" } -env-decorators = { path = "env-decorators" } + +tap = { version = "1.0.1" } +log-panics = { version = "2.1.0" } + +crossbeam-channel = { version = "0.5.6" } +tracing = { version = "0.1.36" } +tracing-subscriber = { version = "0.3.15", features = ["env-filter", "json"] } [package.log] features = ["release_max_level_error"] diff --git a/doublets-ffi/env-decorators/Cargo.toml b/doublets-ffi/env-decorators/Cargo.toml deleted file mode 100644 index 7475483..0000000 --- a/doublets-ffi/env-decorators/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "env-decorators" -version = "0.1.0" -edition = "2021" - -[lib] -proc-macro = true - -[dependencies] -syn = "1.0.86" -proc-macro2 = "1.0.36" -quote = "1.0.15" \ No newline at end of file diff --git a/doublets-ffi/env-decorators/src/lib.rs b/doublets-ffi/env-decorators/src/lib.rs deleted file mode 100644 index b9bfca9..0000000 --- a/doublets-ffi/env-decorators/src/lib.rs +++ /dev/null @@ -1,78 +0,0 @@ -use proc_macro::TokenStream; -use quote::quote; -use syn::{ - parse::{Parse, ParseStream}, - Expr, LitStr, Token, Type, -}; - -struct EnvInput { - env: String, - expr: Expr, -} - -impl Parse for EnvInput { - fn parse(input: ParseStream) -> syn::Result { - let env = input.parse::()?.value(); - input.parse::()?; - let expr = input.parse::()?; - Ok(EnvInput { env, expr }) - } -} - -#[proc_macro] -pub fn env_value(item: TokenStream) -> TokenStream { - let EnvInput { env, expr } = syn::parse_macro_input!(item as EnvInput); - let mut expr = quote! { #expr }; - std::env::var(env) - .unwrap_or_default() - .split_whitespace() - .rev() - .for_each(|name| { - let ty_name: proc_macro2::TokenStream = name.parse().unwrap(); - expr = quote! { - #ty_name :: new(#expr) - } - }); - TokenStream::from(expr) -} - -struct EnvType { - env: String, - pat: String, - default: Type, -} - -impl Parse for EnvType { - fn parse(input: ParseStream) -> syn::Result { - let env = input.parse::()?.value(); - input.parse::()?; - let pat = input.parse::()?.value(); - input.parse::()?; - let ty = input.parse::()?; - Ok(EnvType { - env, - pat, - default: ty, - }) - } -} - -#[proc_macro] -pub fn env_type(item: TokenStream) -> TokenStream { - let EnvType { env, pat, default } = syn::parse_macro_input!(item as EnvType); - let mut ty = quote! { #default }; - std::env::var(env) - .unwrap_or_default() - .split_whitespace() - .rev() - .for_each(|name| { - let new_pat = pat.replace('*', &ty.to_string()); - println!("{}", new_pat); - let pat_ty: proc_macro2::TokenStream = new_pat.parse().unwrap(); - let ty_name: proc_macro2::TokenStream = name.parse().unwrap(); - ty = quote! { - #ty_name #pat_ty - } - }); - TokenStream::from(ty) -} diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 82379cd..b0d2b27 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -29,10 +29,28 @@ fn csharp_convention(s: String) -> String { .to_string() } +fn rust_convention(s: String) -> String { + match s.as_str() { + "i8" => "i8", + "u8" => "u8", + "i16" => "i16", + "u16" => "u16", + "i32" => "i32", + "u32" => "u32", + "i64" => "i64", + "u64" => "u64", + s => { + panic!("{} is incompatible with doublets-ffi type", s) + } + } + .to_string() +} + #[derive(FromMeta, PartialEq, Eq, Debug)] #[allow(non_camel_case_types)] enum Conventions { csharp, + rust, } #[derive(FromMeta)] @@ -136,6 +154,7 @@ pub fn specialize_for(args: TokenStream, input: TokenStream) -> TokenStream { '*', match &args.convention { Conventions::csharp => csharp_convention(ty.clone()), + Conventions::rust => rust_convention(ty.clone()), _ => { panic!("unknown convention") } diff --git a/doublets-ffi/src/constants.rs b/doublets-ffi/src/constants.rs new file mode 100644 index 0000000..a8ba4c3 --- /dev/null +++ b/doublets-ffi/src/constants.rs @@ -0,0 +1,78 @@ +use doublets::data::{LinkType, LinksConstants}; +use std::{mem::MaybeUninit, ops::RangeInclusive}; + +#[derive(Eq, PartialEq)] +#[repr(C)] +pub struct Range(pub T, pub T); + +#[repr(C)] +pub struct Constants { + pub index_part: T, + pub source_part: T, + pub target_part: T, + pub null: T, + pub r#continue: T, + pub r#break: T, + pub skip: T, + pub any: T, + pub itself: T, + pub error: T, + pub internal_range: Range, + // `MaybeUninit` is transparent - `Range` is repr(C) + pub external_range: MaybeUninit>, + pub external_is_some: bool, +} + +impl From> for Constants { + fn from(c: LinksConstants) -> Self { + let mut new = Self { + index_part: c.index_part, + source_part: c.source_part, + target_part: c.target_part, + null: c.null, + r#continue: c.r#continue, + r#break: c.r#break, + skip: c.skip, + any: c.any, + itself: c.itself, + error: c.error, + internal_range: Range(*c.internal_range.start(), *c.internal_range.end()), + // external_range: c.external_range.map(|r| Range(*r.start(), *r.end())), + external_range: MaybeUninit::uninit(), + external_is_some: false, + }; + if let Some(r) = c.external_range { + new.external_is_some = true; + new.external_range.write(Range(*r.start(), *r.end())); + } + new + } +} + +#[allow(clippy::from_over_into)] +impl Into> for Constants { + fn into(self) -> LinksConstants { + LinksConstants { + index_part: self.index_part, + source_part: self.source_part, + target_part: self.target_part, + r#break: self.r#break, + null: self.null, + r#continue: self.r#continue, + skip: self.skip, + any: self.any, + itself: self.itself, + error: self.error, + internal_range: RangeInclusive::new(self.internal_range.0, self.internal_range.1), + external_range: if self.external_is_some { + // SAFETY: `self.external_range` is init + unsafe { + let range = self.external_range.assume_init(); + Some(RangeInclusive::new(range.0, range.1)) + } + } else { + None + }, + } + } +} diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs new file mode 100644 index 0000000..6699863 --- /dev/null +++ b/doublets-ffi/src/export.rs @@ -0,0 +1,41 @@ +use crate::{ + c_char, + logging::{DoubletsFFILogHandle, LogFFICallback}, + FFICallbackContext, +}; +use std::ffi::CStr; +use tracing::error; + +#[no_mangle] +pub extern "C" fn doublets_activate_env_logger() { + if tracing_subscriber::fmt::try_init().is_err() { + error!("Cannot re-init env logger, this should only be called once"); + } +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_create_log_handle( + callback: LogFFICallback, + ctx: FFICallbackContext, + max_level: *const c_char, + use_ansi: bool, + use_json: bool, +) -> *mut DoubletsFFILogHandle { + assert!(!max_level.is_null()); + // if str isn't utf-8 just panic + let max_level_str = CStr::from_ptr(max_level).to_str().unwrap(); + Box::into_raw(Box::new(DoubletsFFILogHandle::new( + callback, + ctx, + max_level_str, + use_ansi, + use_json, + ))) +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_free_log_handle(ptr: *mut DoubletsFFILogHandle) { + if !ptr.is_null() { + let _ = Box::from_raw(ptr); + } +} diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index c9c571a..78405a5 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -2,467 +2,23 @@ #![feature(box_syntax)] #![feature(try_trait_v2)] -use std::{ - error::Error, - ffi::CStr, - fmt::Display, - fs::File, - mem, - ops::{RangeInclusive, Try}, - ptr::{drop_in_place, null_mut}, -}; - -use doublets::{ - data::{ - query, - Flow::{Break, Continue}, - LinkType, LinksConstants, Query, ToQuery, - }, - mem::FileMapped, - Link, Links, -}; -use libc::c_char; -use log::{error, warn}; - +mod constants; +mod export; +mod logging; +mod store; + +// It is not useless: CLion highlight +// `c_char` as alias - italic +// `c_void` as type or alias - non-italic #[allow(non_camel_case_types)] -type c_void = core::ffi::c_void; - -use doublets::{parts, unit, Doublets}; -use ffi_attributes as ffi; - -fn result_into_log(result: Result, default: R) -> R { - result.unwrap_or_else(|e| { - error!("{e}"); - default - }) -} - -unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: usize) -> Query<'a, T> { - // it not require `#[cfg(debug_assertions)]`, - // because it is used in debug log mode only (llvm optimization:)) - if query.is_null() && len != 0 { - warn!("if `query` is null then `len` must be 0"); - } - - if query.is_null() { - query![] - } else { - std::slice::from_raw_parts(query, len).to_query() - } -} - -unsafe fn unnull_or_error<'a, P, R>(ptr: *mut P) -> &'a mut R { - if ptr.is_null() { - // todo: use std::Backtrace or crates/tracing - error!("Null pointer"); - panic!("Null pointer"); - } else { - &mut *(ptr as *mut _) - } -} - -// TODO: remove ::mem:: in doublets crate -type UnitedLinks = unit::Store>>; - -type WrappedLinks = Box>; - -type EachCallback = extern "C" fn(Link) -> T; - -type CUDCallback = extern "C" fn(Link, Link) -> T; - -#[derive(Eq, PartialEq)] -#[repr(C)] -pub struct Range(pub T, pub T); - -#[repr(C)] -pub struct Constants { - pub index_part: T, - pub source_part: T, - pub target_part: T, - pub null: T, - pub r#continue: T, - pub r#break: T, - pub skip: T, - pub any: T, - pub itself: T, - pub error: T, - pub internal_range: Range, - pub external_range: Range, - pub _opt_marker: bool, -} - -impl From> for Constants { - fn from(c: LinksConstants) -> Self { - Self { - index_part: c.index_part, - source_part: c.source_part, - target_part: c.target_part, - null: c.null, - r#continue: c.r#continue, - r#break: c.r#break, - skip: c.skip, - any: c.any, - itself: c.itself, - error: c.error, - internal_range: Range(*c.internal_range.start(), *c.internal_range.end()), - // external_range: c.external_range.map(|r| Range(*r.start(), *r.end())), - external_range: c - .clone() - .external_range - .map_or(Range(T::funty(0), T::funty(0)), |r| { - Range(*r.start(), *r.end()) - }), - _opt_marker: c.external_range.is_some(), - } - } -} - -#[allow(clippy::from_over_into)] -impl Into> for Constants { - fn into(self) -> LinksConstants { - LinksConstants { - index_part: self.index_part, - source_part: self.source_part, - target_part: self.target_part, - r#break: self.r#break, - null: self.null, - r#continue: self.r#continue, - skip: self.skip, - any: self.any, - itself: self.itself, - error: self.error, - internal_range: RangeInclusive::new(self.internal_range.0, self.internal_range.1), - external_range: if self._opt_marker { - Some(RangeInclusive::new( - self.external_range.0, - self.external_range.1, - )) - } else { - None - }, - } - } -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_New" -)] -unsafe fn new_united_links(path: *const c_char) -> *mut c_void { - new_with_constants_united_links::(path, LinksConstants::external().into()) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_NewWithConstants" -)] -unsafe fn new_with_constants_united_links( - path: *const c_char, - constants: Constants, -) -> *mut c_void { - let result: Result<_, Box> = try { - let path = CStr::from_ptr(path).to_str()?; - let file = File::options() - .create(true) - .read(true) - .write(true) - .open(path)?; - let mem = FileMapped::new(file)?; - let mut links: Box> = - box UnitedLinks::::with_constants(mem, constants.into())?; - let ptr = links.as_mut() as *mut _ as *mut c_void; - mem::forget(links); - ptr - }; - result_into_log(result, null_mut()) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Drop" -)] -unsafe fn drop_united_links(this: *mut c_void) { - let links: &mut WrappedLinks = unnull_or_error(this); - drop_in_place(links); -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_GetConstants" -)] -unsafe fn get_constants_united_links(this: *mut c_void) -> Constants { - let links: &mut WrappedLinks = unnull_or_error(this); - links.constants().clone().into() -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Create" -)] -unsafe fn create_united( - this: *mut c_void, - query: *const T, - len: usize, - callback: CUDCallback, -) -> T { - let links: &mut WrappedLinks = unnull_or_error(this); - let continue_ = links.constants().r#continue; - let break_ = links.constants().r#break; - let result = { - let query = query_from_raw(query, len); - let handler = |before: Link<_>, after: Link<_>| { - if callback(before, after) == continue_ { - Break - } else { - Continue - } - }; - links.create_by_with(query, handler) - }; - result_into_log( - result.map(|flow| { - if flow.branch().is_continue() { - continue_ - } else { - break_ - } - }), - links.constants().error, - ) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_SmartCreate" -)] -unsafe fn smart_create_united(this: *mut c_void) -> T { - let links: &mut WrappedLinks = unnull_or_error(this); - let result = links.create(); - result_into_log(result, links.constants().error) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_SmartUpdate" -)] -unsafe fn smart_update_united(this: *mut c_void, index: T, source: T, target: T) -> T { - let links: &mut WrappedLinks = unnull_or_error(this); - let result = links.update(index, source, target); - result_into_log(result, links.constants().error) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Each" -)] -unsafe fn each_united( - this: *mut c_void, - query: *const T, - len: usize, - callback: EachCallback, -) -> T { - let links: &mut WrappedLinks = unnull_or_error(this); - let query = query_from_raw(query, len); - let r#continue = links.constants().r#continue; - let r#break = links.constants().r#break; - let result = links.each_by(query, move |link| { - if callback(link) == r#continue { - Continue - } else { - Break - } - }); - match result { - Continue => r#continue, - Break => r#break, - } -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Count" -)] -unsafe fn count_united(this: *mut c_void, query: *const T, len: usize) -> T { - let links: &mut WrappedLinks = unnull_or_error(this); - let query = query_from_raw(query, len); - links.count_by(query) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Update" -)] -unsafe fn update_united( - this: *mut c_void, - restrictions: *const T, - len_r: usize, - substitutuion: *const T, - len_s: usize, - callback: CUDCallback, -) -> T { - let restrictions = query_from_raw(restrictions, len_r); - let substitutuion = query_from_raw(substitutuion, len_s); - let links: &mut WrappedLinks = unnull_or_error(this); - let continue_ = links.constants().r#continue; - let break_ = links.constants().r#break; - let result = { - let handler = move |before: Link, after: Link| { - if callback(before, after) == continue_ { - Continue - } else { - Break - } - }; - links.update_by_with(restrictions, substitutuion, handler) - }; - result_into_log( - result.map(|flow| { - if flow.branch().is_continue() { - continue_ - } else { - break_ - } - }), - links.constants().error, - ) -} - -#[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "csharp", - name = "*Links_Delete" -)] -unsafe fn delete_united( - this: *mut c_void, - query: *const T, - len: usize, - callback: CUDCallback, -) -> T { - let query = query_from_raw(query, len); - let links: &mut WrappedLinks = unnull_or_error(this); - let continue_ = links.constants().r#continue; - let break_ = links.constants().r#break; - let result = { - let handler = move |before: Link<_>, after: Link<_>| { - if callback(before, after) == break_ { - Break - } else { - Continue - } - }; - links.delete_by_with(query, handler) - }; - result_into_log( - result.map(|flow| { - if flow.branch().is_continue() { - continue_ - } else { - break_ - } - }), - links.constants().error, - ) -} - -#[repr(C)] -pub struct SharedLogger { - formatter: for<'a> extern "C" fn(&'a log::Record<'_>), -} - -impl log::Log for SharedLogger { - fn enabled(&self, _: &log::Metadata) -> bool { - true - } - fn log(&self, record: &log::Record) { - (self.formatter)(record) - } - fn flush(&self) {} -} - -pub fn build_shared_logger() -> SharedLogger { - extern "C" fn formatter(r: &log::Record<'_>) { - tracing_log::format_trace(r).unwrap() - } - SharedLogger { formatter } -} - -#[no_mangle] -pub extern "C" fn setup_shared_logger(logger: SharedLogger) { - log::set_max_level(log::STATIC_MAX_LEVEL); - - let subscriber = tracing_subscriber::fmt::fmt() - .with_max_level(tracing::Level::TRACE) - .finish(); - if let Err(err) = tracing::subscriber::set_global_default(subscriber) { - warn!("subscriber error: {}", err) - } - - if let Err(err) = log::set_boxed_logger(Box::new(logger)) { - warn!("{}", err) - } -} +type c_void = std::ffi::c_void; +#[allow(non_camel_case_types)] +type c_char = std::ffi::c_char; -#[no_mangle] -pub extern "C" fn init_fmt_logger() { - let logger = build_shared_logger(); - setup_shared_logger(logger); -} +type FFICallbackContext = *mut c_void; -mod tests { - #[test] - fn error_log() { - use crate::{build_shared_logger, setup_shared_logger}; - use log::{debug, error, info, trace, warn}; +#[derive(Clone, Copy)] +pub struct FFICallbackContextWrapper(FFICallbackContext); - let logger = build_shared_logger(); - setup_shared_logger(logger); - trace!("trace"); - debug!("debug"); - info!("info"); - warn!("warn"); - error!("error"); - } -} +unsafe impl Send for FFICallbackContextWrapper {} +unsafe impl Sync for FFICallbackContextWrapper {} diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs new file mode 100644 index 0000000..2398ea1 --- /dev/null +++ b/doublets-ffi/src/logging.rs @@ -0,0 +1,105 @@ +use super::{c_char, FFICallbackContext}; +use crate::FFICallbackContextWrapper; +use crossbeam_channel::{self as mpsc, Sender}; +use std::{ffi::CString, io, str::FromStr, thread}; +use tracing::{error, info_span, warn}; +use tracing_subscriber::{ + filter::{EnvFilter, LevelFilter}, + fmt::MakeWriter, +}; + +struct ChannelWriter { + sender: Sender>, +} + +impl ChannelWriter { + pub fn new(sender: Sender>) -> Self { + Self { sender } + } +} + +impl io::Write for ChannelWriter { + fn write(&mut self, buf: &[u8]) -> Result { + let len = buf.len(); + let _ = self.sender.send(buf.to_vec()); + Ok(len) + } + + fn flush(&mut self) -> Result<(), io::Error> { + Ok(()) + } +} + +impl MakeWriter<'_> for ChannelWriter { + type Writer = ChannelWriter; + + fn make_writer(&self) -> Self::Writer { + ChannelWriter { + sender: self.sender.clone(), + } + } +} + +pub type LogFFICallback = extern "C" fn(FFICallbackContext, *const c_char); + +pub struct DoubletsFFILogHandle {} + +impl DoubletsFFILogHandle { + pub fn new( + callback: LogFFICallback, + ctx: FFICallbackContext, + max_level: &str, + use_ansi: bool, + use_json: bool, + ) -> Self { + log_panics::init(); + let wrapper = FFICallbackContextWrapper(ctx); + let (sender, receiver) = mpsc::bounded(256); + + let callback = move |ctx: FFICallbackContextWrapper, ptr| { + callback(ctx.0, ptr); + }; + + thread::spawn(move || { + // We can't use `while let Ok(msg) = receiver.recv()` + // here because the receiver will be blocked + + loop { + // info_span!("Logging loop").in_scope(|| { + if let Ok(msg) = receiver.recv() { + let str = CString::new(msg) + .expect("Only UTF-8 format strings are allowed in logging"); + callback(wrapper, str.as_ptr()); + } + // }); + } + }); + + let filter = EnvFilter::from_default_env() + .add_directive(LevelFilter::from_str(max_level).unwrap().into()); + if use_json { + if tracing_subscriber::fmt() + .json() + .with_ansi(use_ansi) + .with_writer(ChannelWriter::new(sender)) + .with_env_filter(filter) + .with_filter_reloading() + .try_init() + .is_err() + { + error!("Log handler already set, cannot currently change log levels."); + } + } else if tracing_subscriber::fmt() + .with_ansi(use_ansi) + .with_writer(ChannelWriter::new(sender)) + .with_env_filter(filter) + .with_filter_reloading() + .try_init() + .is_err() + { + error!("Log handler already set, cannot currently change log levels."); + }; + + Self {} + } +} diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs new file mode 100644 index 0000000..7bd5b0a --- /dev/null +++ b/doublets-ffi/src/store.rs @@ -0,0 +1,287 @@ +use crate::{c_char, c_void, constants::Constants}; +use doublets::{ + data::{query, Flow, LinkType, Query, ToQuery}, + mem::FileMapped, + parts, unit, Doublets, Link, Links, +}; +use ffi_attributes as ffi; +use std::{error, ffi::CStr, marker::PhantomData, ptr, slice}; +use tap::Pipe; +use tracing::{debug, error, warn}; + +// TODO: remove ::mem:: in doublets crate +type UnitedLinks = unit::Store>>; + +type EachCallback = extern "C" fn(Link) -> T; + +type CUDCallback = extern "C" fn(Link, Link) -> T; + +#[repr(transparent)] +pub struct StoreHandle { + pub(crate) ptr: *mut c_void, // dyn Doublets + marker: PhantomData, +} + +impl StoreHandle { + pub fn new(store: Box>) -> Self { + let raw = Box::into_raw(Box::new(store)); + // SAFETY: box contains valid ptr to store + unsafe { Self::from_raw(raw.cast()) } + } + + pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { + Self { + ptr: raw, + marker: PhantomData, + } + } + + pub unsafe fn assume(&mut self) -> &mut Box> { + &mut *self.ptr.cast() + } + + pub fn invalid(err: Box) -> Self { + error!(err); + // we not have access to self inner + Self { + ptr: ptr::null_mut(), + marker: PhantomData, + } + } + + pub fn drop(mut handle: Self) { + // SAFETY: `self.store` is valid `Store` ptr + unsafe { + if !handle.ptr.is_null() { + let _ = Box::from_raw(handle.assume()); + } + } + } +} + +unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { + // it not require `#[cfg(debug_assertions)]`, + // because it is used in debug log mode only (llvm optimization:)) + if query.is_null() && len != 0 { + warn!("if `query` is null then `len` must be 0"); + } + + // fixme: may be use `assert!(!query.is_null())` + if query.is_null() { + query![] + } else { + slice::from_raw_parts(query, len as usize).to_query() + } +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_create_united_store_*" +)] +pub unsafe fn create_united_store( + path: *const c_char, + constants: Constants, +) -> StoreHandle { + let result: Result<_, Box> = try { + let path = CStr::from_ptr(path).to_str().unwrap(); + let mem = FileMapped::from_path(path)?; + StoreHandle::new(Box::new(UnitedLinks::::with_constants( + mem, + constants.into(), + )?)) + }; + result.unwrap_or_else(StoreHandle::invalid) +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_free_store_*" +)] +pub unsafe fn free_store(this: *mut c_void) { + StoreHandle::drop(StoreHandle::::from_raw(this)) +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_constants_*" +)] +pub unsafe fn constants_for_store(this: *mut c_void) -> Constants { + StoreHandle::from_raw(this) + .assume() + .constants() + .clone() // fixme: useless .clone + .into() +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_create_unit_*" +)] +pub unsafe fn create_united( + this: *mut c_void, + query: *const T, + len: u32, + callback: CUDCallback, +) -> T { + let mut handle = StoreHandle::::from_raw(this); + let store = handle.assume(); + let constants = store.constants().clone(); + let (cnt, brk) = (constants.r#continue, constants.r#break); + + let query = query_from_raw(query, len); + let handler = move |before, after| { + if callback(before, after) == cnt { + Flow::Continue + } else { + Flow::Break + } + }; + store + .create_by_with(query, handler) + // fixme: add `.is_break` for `Flow` + .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) + .unwrap_or_else(|err| { + debug!(operation_error = %err); + constants.error + }) +} + +#[tracing::instrument(level = "debug", name = "united::each")] +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_each_unit_*" +)] +pub unsafe fn each_united( + this: *mut c_void, + query: *const T, + len: u32, + callback: EachCallback, +) -> T { + let mut handle = StoreHandle::::from_raw(this); + let store = handle.assume(); + let constants = store.constants(); + let (cnt, brk) = (constants.r#continue, constants.r#break); + + let query = query_from_raw(query, len); + let handler = move |link| { + if callback(link) == cnt { + Flow::Continue + } else { + Flow::Break + } + }; + store + .each_by(query, handler) + // fixme: add `.is_break` for `Flow` + .pipe(|flow| if let Flow::Continue = flow { cnt } else { brk }) +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_count_unit_*" +)] +pub unsafe fn count_united(this: *mut c_void, query: *const T, len: u32) -> T { + let mut handle = StoreHandle::::from_raw(this); + let query = query_from_raw(query, len); + handle.assume().count_by(query) +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_update_unit_*" +)] +pub unsafe fn update_united( + this: *mut c_void, + query: *const T, + len_r: u32, + change: *const T, + len_s: u32, + callback: CUDCallback, +) -> T { + let query = query_from_raw(query, len_r); + let change = query_from_raw(change, len_s); + let mut handle = StoreHandle::::from_raw(this); + let store = handle.assume(); + let constants = store.constants().clone(); + let (cnt, brk) = (constants.r#continue, constants.r#break); + let handler = move |before, after| { + if callback(before, after) == cnt { + Flow::Continue + } else { + Flow::Break + } + }; + store + .update_by_with(query, change, handler) + // fixme: add `.is_break` for `Flow` + .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) + .unwrap_or_else(|err| { + debug!(operation_error = %err); + constants.error + }) +} + +#[ffi::specialize_for( + types = "u8", + types = "u16", + types = "u32", + types = "u64", + convention = "rust", + name = "doublets_delete_unit_*" +)] +pub unsafe fn delete_united( + this: *mut c_void, + query: *const T, + len: u32, + callback: CUDCallback, +) -> T { + let mut handle = StoreHandle::::from_raw(this); + let store = handle.assume(); + let constants = store.constants().clone(); + let (cnt, brk) = (constants.r#continue, constants.r#break); + + let query = query_from_raw(query, len); + let handler = move |before, after| { + if callback(before, after) == cnt { + Flow::Continue + } else { + Flow::Break + } + }; + store + .delete_by_with(query, handler) + // fixme: add `.is_break` for `Flow` + .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) + .unwrap_or_else(|err| { + debug!(operation_error = %err); + constants.error + }) +} From 4d59bae354b2a6f8c2879d361c59ab214443ca69 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 14 Aug 2022 17:46:34 +0300 Subject: [PATCH 02/76] Improve logging (via `instrument`s) and naming --- doublets-ffi/src/store.rs | 92 +++++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 7bd5b0a..3e8869f 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -7,7 +7,7 @@ use doublets::{ use ffi_attributes as ffi; use std::{error, ffi::CStr, marker::PhantomData, ptr, slice}; use tap::Pipe; -use tracing::{debug, error, warn}; +use tracing::{debug, error, trace, warn}; // TODO: remove ::mem:: in doublets crate type UnitedLinks = unit::Store>>; @@ -59,6 +59,15 @@ impl StoreHandle { } } +unsafe fn thin_query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { + // fixme: may be use `assert!(!query.is_null())` + if query.is_null() { + query![] + } else { + slice::from_raw_parts(query, len as usize).to_query() + } +} + unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { // it not require `#[cfg(debug_assertions)]`, // because it is used in debug log mode only (llvm optimization:)) @@ -66,14 +75,16 @@ unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a warn!("if `query` is null then `len` must be 0"); } - // fixme: may be use `assert!(!query.is_null())` - if query.is_null() { - query![] - } else { - slice::from_raw_parts(query, len as usize).to_query() - } + thin_query_from_raw(query, len) } +#[tracing::instrument( + skip_all, + fields( + path = ?CStr::from_ptr(path).to_str(), + path.ptr = ?path, + ), +)] #[ffi::specialize_for( types = "u8", types = "u16", @@ -82,7 +93,7 @@ unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a convention = "rust", name = "doublets_create_united_store_*" )] -pub unsafe fn create_united_store( +pub unsafe fn create_unit_store( path: *const c_char, constants: Constants, ) -> StoreHandle { @@ -125,15 +136,23 @@ pub unsafe fn constants_for_store(this: *mut c_void) -> Constants( +pub unsafe fn create( this: *mut c_void, query: *const T, len: u32, @@ -162,16 +181,23 @@ pub unsafe fn create_united( }) } -#[tracing::instrument(level = "debug", name = "united::each")] +#[tracing::instrument( + skip_all, + fields( + query = ?&thin_query_from_raw(query, len)[..], + query.ptr = ?query, + query.len = len, + ), +)] #[ffi::specialize_for( types = "u8", types = "u16", types = "u32", types = "u64", convention = "rust", - name = "doublets_each_unit_*" + name = "doublets_each_*" )] -pub unsafe fn each_united( +pub unsafe fn each( this: *mut c_void, query: *const T, len: u32, @@ -196,38 +222,58 @@ pub unsafe fn each_united( .pipe(|flow| if let Flow::Continue = flow { cnt } else { brk }) } +#[tracing::instrument( + skip_all, + fields( + query = ?&thin_query_from_raw(query, len)[..], + query.ptr = ?query, + query.len = len, + ), +)] #[ffi::specialize_for( types = "u8", types = "u16", types = "u32", types = "u64", convention = "rust", - name = "doublets_count_unit_*" + name = "doublets_count_*" )] -pub unsafe fn count_united(this: *mut c_void, query: *const T, len: u32) -> T { +pub unsafe fn count(this: *mut c_void, query: *const T, len: u32) -> T { let mut handle = StoreHandle::::from_raw(this); let query = query_from_raw(query, len); handle.assume().count_by(query) } +#[tracing::instrument( + skip_all, + fields( + query = ?&thin_query_from_raw(query, len_q)[..], + query.ptr = ?query, + query.len = len_q, + + change = ?&thin_query_from_raw(query, len_q)[..], + change.ptr = ?change, + change.len = len_c, + ), +)] #[ffi::specialize_for( types = "u8", types = "u16", types = "u32", types = "u64", convention = "rust", - name = "doublets_update_unit_*" + name = "doublets_update_*" )] -pub unsafe fn update_united( +pub unsafe fn update( this: *mut c_void, query: *const T, - len_r: u32, + len_q: u32, change: *const T, - len_s: u32, + len_c: u32, callback: CUDCallback, ) -> T { - let query = query_from_raw(query, len_r); - let change = query_from_raw(change, len_s); + let query = query_from_raw(query, len_q); + let change = query_from_raw(change, len_c); let mut handle = StoreHandle::::from_raw(this); let store = handle.assume(); let constants = store.constants().clone(); @@ -255,9 +301,9 @@ pub unsafe fn update_united( types = "u32", types = "u64", convention = "rust", - name = "doublets_delete_unit_*" + name = "doublets_delete_*" )] -pub unsafe fn delete_united( +pub unsafe fn delete( this: *mut c_void, query: *const T, len: u32, From e0edd835c331d41306f284033e67ac68515ce044 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 14 Aug 2022 22:24:29 +0300 Subject: [PATCH 03/76] Add ffi context to callbacks --- doublets-ffi/src/store.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 3e8869f..2242802 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,4 +1,4 @@ -use crate::{c_char, c_void, constants::Constants}; +use crate::{c_char, c_void, constants::Constants, FFICallbackContext}; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, mem::FileMapped, @@ -12,9 +12,9 @@ use tracing::{debug, error, trace, warn}; // TODO: remove ::mem:: in doublets crate type UnitedLinks = unit::Store>>; -type EachCallback = extern "C" fn(Link) -> T; +type EachCallback = extern "C" fn(FFICallbackContext, Link) -> T; -type CUDCallback = extern "C" fn(Link, Link) -> T; +type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> T; #[repr(transparent)] pub struct StoreHandle { @@ -156,6 +156,7 @@ pub unsafe fn create( this: *mut c_void, query: *const T, len: u32, + ctx: FFICallbackContext, callback: CUDCallback, ) -> T { let mut handle = StoreHandle::::from_raw(this); @@ -165,7 +166,7 @@ pub unsafe fn create( let query = query_from_raw(query, len); let handler = move |before, after| { - if callback(before, after) == cnt { + if callback(ctx, before, after) == cnt { Flow::Continue } else { Flow::Break @@ -201,6 +202,7 @@ pub unsafe fn each( this: *mut c_void, query: *const T, len: u32, + ctx: FFICallbackContext, callback: EachCallback, ) -> T { let mut handle = StoreHandle::::from_raw(this); @@ -210,7 +212,7 @@ pub unsafe fn each( let query = query_from_raw(query, len); let handler = move |link| { - if callback(link) == cnt { + if callback(ctx, link) == cnt { Flow::Continue } else { Flow::Break @@ -270,6 +272,7 @@ pub unsafe fn update( len_q: u32, change: *const T, len_c: u32, + ctx: FFICallbackContext, callback: CUDCallback, ) -> T { let query = query_from_raw(query, len_q); @@ -279,7 +282,7 @@ pub unsafe fn update( let constants = store.constants().clone(); let (cnt, brk) = (constants.r#continue, constants.r#break); let handler = move |before, after| { - if callback(before, after) == cnt { + if callback(ctx, before, after) == cnt { Flow::Continue } else { Flow::Break @@ -307,6 +310,7 @@ pub unsafe fn delete( this: *mut c_void, query: *const T, len: u32, + ctx: FFICallbackContext, callback: CUDCallback, ) -> T { let mut handle = StoreHandle::::from_raw(this); @@ -316,7 +320,7 @@ pub unsafe fn delete( let query = query_from_raw(query, len); let handler = move |before, after| { - if callback(before, after) == cnt { + if callback(ctx, before, after) == cnt { Flow::Continue } else { Flow::Break From e8691c2ad5586aaece0c7e6fd7c66dfc7b05f6ad Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Mon, 15 Aug 2022 17:18:38 +0300 Subject: [PATCH 04/76] Add ffi `examples` crate --- Cargo.toml | 3 +- doublets-ffi/Cargo.toml | 2 +- doublets-ffi/examples/rust/Cargo.toml | 10 +++ .../examples/rust/examples/all-logs.rs | 31 ++++++++++ .../rust/examples/doublets-context.rs | 62 +++++++++++++++++++ .../examples/rust/examples/log-context.rs | 33 ++++++++++ doublets-ffi/src/export.rs | 4 +- doublets-ffi/src/lib.rs | 10 +-- doublets-ffi/src/logging.rs | 11 +++- 9 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 doublets-ffi/examples/rust/Cargo.toml create mode 100644 doublets-ffi/examples/rust/examples/all-logs.rs create mode 100644 doublets-ffi/examples/rust/examples/doublets-context.rs create mode 100644 doublets-ffi/examples/rust/examples/log-context.rs diff --git a/Cargo.toml b/Cargo.toml index 8176e2d..5f81d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,9 @@ [workspace] members = [ - "doublets-ffi", "doublets", + "doublets-ffi", + "doublets-ffi/examples/rust", # dev "dev-deps/mem-rs", diff --git a/doublets-ffi/Cargo.toml b/doublets-ffi/Cargo.toml index d921162..4beb88e 100644 --- a/doublets-ffi/Cargo.toml +++ b/doublets-ffi/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [lib] -crate-type = ["cdylib", "staticlib"] +crate-type = ["cdylib", "staticlib", "lib"] [dependencies] doublets = { path = "../doublets" } diff --git a/doublets-ffi/examples/rust/Cargo.toml b/doublets-ffi/examples/rust/Cargo.toml new file mode 100644 index 0000000..40f1adc --- /dev/null +++ b/doublets-ffi/examples/rust/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "doublets-ffi-examples" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +doublets-ffi = { path = "../../../doublets-ffi" } +doublets = { path = "../../../doublets" } +tracing = { version = "0.1.36" } \ No newline at end of file diff --git a/doublets-ffi/examples/rust/examples/all-logs.rs b/doublets-ffi/examples/rust/examples/all-logs.rs new file mode 100644 index 0000000..19e0c42 --- /dev/null +++ b/doublets-ffi/examples/rust/examples/all-logs.rs @@ -0,0 +1,31 @@ +use doublets_ffi::{ + export::{doublets_create_log_handle, doublets_free_log_handle}, + FFICallbackContext, +}; +use std::{ + ffi::{c_char, CStr, CString}, + ptr, +}; + +unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { + let cstr = CStr::from_ptr(ptr); + print!("{}", cstr.to_str().unwrap()); +} + +fn main() { + let ctx = ptr::null_mut(); + let level = CString::new("trace").unwrap(); + let use_ansi = true; + let use_json = false; + unsafe { + let handle = doublets_create_log_handle(ctx, callback, level.as_ptr(), use_ansi, use_json); + + tracing::error!("SOMETHING IS SERIOUSLY WRONG!!!"); + tracing::warn!("important informational messages; might indicate an error"); + tracing::info!("general informational messages relevant to users"); + tracing::debug!("diagnostics used for internal debugging of a library or application"); + tracing::trace!("very verbose diagnostic events"); + + doublets_free_log_handle(handle); + } +} diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs new file mode 100644 index 0000000..2493278 --- /dev/null +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -0,0 +1,62 @@ +use doublets::{data::LinksConstants, Link}; +use doublets_ffi::{ + constants::Constants, + export::{doublets_create_log_handle, doublets_free_log_handle}, + store::{create, doublets_constants_u64, doublets_create_united_store_u64}, + FFICallbackContext, +}; +use std::{ + ffi::{c_char, c_void, CStr, CString}, + fs, + ptr::{null, null_mut}, +}; + +unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { + print!("{}", CStr::from_ptr(ptr).to_str().unwrap()); +} + +extern "C" fn create_cb(ctx: FFICallbackContext, before: Link, after: Link) -> u64 +where + F: FnMut(Link, Link), +{ + unsafe { + let &mut (store, ref mut handler) = &mut *(ctx as *mut (*mut c_void, F)); + (*handler)(before, after); + let constants = doublets_constants_u64(store); + constants.r#continue + } +} + +unsafe fn magic_create(ptr: *mut c_void, handler: F) +where + F: FnMut(Link, Link), +{ + let mut ctx = (ptr, handler); + let _ = create( + ptr, + null(), + 0, + (&mut ctx as *mut (*mut c_void, F)).cast(), + create_cb::, + ); +} + +fn main() { + let level = CString::new("trace").unwrap(); + unsafe { + let handle = doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); + + let path = CString::new("doublets.links").unwrap(); + let mut store = doublets_create_united_store_u64( + path.as_ptr(), + Constants::from(LinksConstants::external()), + ); + + magic_create(store.assume() as *mut _ as *mut _, |before, after| { + print!("{before:?}\n{after:?}\n"); + }); + + doublets_free_log_handle(handle); + } + let _ = fs::remove_file("doublets.links"); +} diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs new file mode 100644 index 0000000..a9915a3 --- /dev/null +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -0,0 +1,33 @@ +use doublets_ffi::{ + export::{doublets_create_log_handle, doublets_free_log_handle}, + FFICallbackContext, +}; +use std::ffi::{c_char, CStr, CString}; + +unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { + let str = CStr::from_ptr(ptr).to_str().unwrap(); + let ctx = &mut *(ctx as *mut usize); + *ctx += 1; + + if *ctx % 2 == 0 { + print!("{str}"); + } else { + eprint!("{str}"); + } +} + +fn main() { + let ctx = &mut 0usize as *mut usize; + let level = CString::new("trace").unwrap(); + unsafe { + let handle = doublets_create_log_handle(ctx.cast(), callback, level.as_ptr(), true, false); + + tracing::error!("SOMETHING IS SERIOUSLY WRONG!!!"); + tracing::warn!("important informational messages; might indicate an error"); + tracing::info!("general informational messages relevant to users"); + tracing::debug!("diagnostics used for internal debugging of a library or application"); + tracing::trace!("very verbose diagnostic events"); + + doublets_free_log_handle(handle); + } +} diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index 6699863..c458f07 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -15,8 +15,8 @@ pub extern "C" fn doublets_activate_env_logger() { #[no_mangle] pub unsafe extern "C" fn doublets_create_log_handle( - callback: LogFFICallback, ctx: FFICallbackContext, + callback: LogFFICallback, max_level: *const c_char, use_ansi: bool, use_json: bool, @@ -25,8 +25,8 @@ pub unsafe extern "C" fn doublets_create_log_handle( // if str isn't utf-8 just panic let max_level_str = CStr::from_ptr(max_level).to_str().unwrap(); Box::into_raw(Box::new(DoubletsFFILogHandle::new( - callback, ctx, + callback, max_level_str, use_ansi, use_json, diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 78405a5..dba0ca2 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -2,10 +2,10 @@ #![feature(box_syntax)] #![feature(try_trait_v2)] -mod constants; -mod export; -mod logging; -mod store; +pub mod constants; +pub mod export; +pub mod logging; +pub mod store; // It is not useless: CLion highlight // `c_char` as alias - italic @@ -15,7 +15,7 @@ type c_void = std::ffi::c_void; #[allow(non_camel_case_types)] type c_char = std::ffi::c_char; -type FFICallbackContext = *mut c_void; +pub type FFICallbackContext = *mut c_void; #[derive(Clone, Copy)] pub struct FFICallbackContextWrapper(FFICallbackContext); diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 2398ea1..e05d256 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -40,14 +40,16 @@ impl MakeWriter<'_> for ChannelWriter { } } -pub type LogFFICallback = extern "C" fn(FFICallbackContext, *const c_char); +/// # Safety +/// This callback is safe if all the rules of Rust are followed +pub type LogFFICallback = unsafe extern "C" fn(FFICallbackContext, *const c_char); pub struct DoubletsFFILogHandle {} impl DoubletsFFILogHandle { pub fn new( - callback: LogFFICallback, ctx: FFICallbackContext, + callback: LogFFICallback, max_level: &str, use_ansi: bool, use_json: bool, @@ -57,7 +59,10 @@ impl DoubletsFFILogHandle { let (sender, receiver) = mpsc::bounded(256); let callback = move |ctx: FFICallbackContextWrapper, ptr| { - callback(ctx.0, ptr); + // SAFETY: caller must guarantee - we only delegate callback + unsafe { + callback(ctx.0, ptr); + } }; thread::spawn(move || { From 63be554033de962b72417eeec0dbf2ba08a8a943 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 17 Aug 2022 17:46:50 +0300 Subject: [PATCH 05/76] Fix too long ptr cast --- .../examples/rust/examples/doublets-context.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index 2493278..443eccd 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -31,14 +31,8 @@ unsafe fn magic_create(ptr: *mut c_void, handler: F) where F: FnMut(Link, Link), { - let mut ctx = (ptr, handler); - let _ = create( - ptr, - null(), - 0, - (&mut ctx as *mut (*mut c_void, F)).cast(), - create_cb::, - ); + let ctx = &mut (ptr, handler); + let _ = create(ptr, null(), 0, ctx as *mut _ as *mut _, create_cb::); } fn main() { From b1a3d779a378d2f2ee746ce95e8a64f02f39fb48 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 17 Aug 2022 23:18:20 +0300 Subject: [PATCH 06/76] Improve safety docs --- doublets-ffi/src/store.rs | 46 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 2242802..6771479 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -29,6 +29,43 @@ impl StoreHandle { unsafe { Self::from_raw(raw.cast()) } } + /// # Examples + /// + /// Safe usage: + /// + /// ``` + /// # use std::ffi::c_void; + /// # use doublets_ffi::store::StoreHandle; + /// extern "C" fn create_u64_store() -> *mut c_void { + /// todo!("todo: simple but full example") + /// } + /// + /// // SAFETY: caller must guarantee `from_raw` invariants + /// unsafe extern "C" fn free_u64_store(ptr: *mut c_void) { + /// StoreHandle::drop(StoreHandle::::from_raw(ptr)) + /// } + /// ``` + /// + /// Undefined Behaviour usage: + /// ```no_run + /// # use std::ffi::c_void; + /// # use doublets_ffi::store::StoreHandle; + /// + /// unsafe extern "C" fn should_crush(ptr: *mut c_void) { + /// // two handle for one store is safe + /// let (mut a, mut b) = ( + /// StoreHandle::::from_raw(ptr), + /// StoreHandle::::from_raw(ptr), + /// ); + /// // but it is ub + /// let (a, b) = (a.assume(), b.assume()); + /// } + /// ``` + /// + /// # Safety + /// `raw` must be valid ptr to `Box>` + /// allocated in `Box` + /// without owner pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { Self { ptr: raw, @@ -36,8 +73,11 @@ impl StoreHandle { } } - pub unsafe fn assume(&mut self) -> &mut Box> { - &mut *self.ptr.cast() + pub fn assume(&mut self) -> &mut Box> { + // SAFETY: `StoreHandle` must be create from safe `new()` + // or unsafe `Self::from_raw` + // then it guarantee by `Self::from_raw()` caller + unsafe { &mut *self.ptr.cast() } } pub fn invalid(err: Box) -> Self { @@ -52,6 +92,8 @@ impl StoreHandle { pub fn drop(mut handle: Self) { // SAFETY: `self.store` is valid `Store` ptr unsafe { + // assume ptr is not null + // guarantee by `from_raw` and `new` if !handle.ptr.is_null() { let _ = Box::from_raw(handle.assume()); } From f4c7cb4f6b602a5e75d67b928334d3fa6b9f3c7c Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Thu, 18 Aug 2022 14:26:19 +0300 Subject: [PATCH 07/76] Improve error handling: - use more debug logs - use more high-level types - fix useless code in ffi bindings - add function to work with error data (incomplete) - improve `doublets`, `platform-data` to make it compatible with new changes --- .../rust/examples/doublets-context.rs | 13 +- .../examples/rust/examples/error-handling.rs | 72 ++++++++++ doublets-ffi/src/errors.rs | 107 ++++++++++++++ doublets-ffi/src/lib.rs | 4 + doublets-ffi/src/store.rs | 136 +++++++++--------- doublets/src/data/doublet.rs | 1 + 6 files changed, 256 insertions(+), 77 deletions(-) create mode 100644 doublets-ffi/examples/rust/examples/error-handling.rs create mode 100644 doublets-ffi/src/errors.rs diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index 443eccd..165a74d 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -1,6 +1,10 @@ -use doublets::{data::LinksConstants, Link}; +use doublets::{ + data::{Flow, LinksConstants}, + Link, +}; use doublets_ffi::{ constants::Constants, + errors::DoubletsErrorKind, export::{doublets_create_log_handle, doublets_free_log_handle}, store::{create, doublets_constants_u64, doublets_create_united_store_u64}, FFICallbackContext, @@ -15,15 +19,14 @@ unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { print!("{}", CStr::from_ptr(ptr).to_str().unwrap()); } -extern "C" fn create_cb(ctx: FFICallbackContext, before: Link, after: Link) -> u64 +extern "C" fn create_cb(ctx: FFICallbackContext, before: Link, after: Link) -> Flow where F: FnMut(Link, Link), { unsafe { - let &mut (store, ref mut handler) = &mut *(ctx as *mut (*mut c_void, F)); + let handler = &mut *(ctx as *mut F); (*handler)(before, after); - let constants = doublets_constants_u64(store); - constants.r#continue + Flow::Continue } } diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs new file mode 100644 index 0000000..caeb6a2 --- /dev/null +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -0,0 +1,72 @@ +use doublets::{ + data::{Flow, LinksConstants}, + Link, +}; +use doublets_ffi::{ + constants::Constants, + errors::{ + doublets_read_error, doublets_read_error_as_not_found, doublets_read_error_message, + DoubletsErrorKind, + }, + export::{doublets_create_log_handle, doublets_free_log_handle}, + store::{constants_for_store, create_unit_store, delete, free_store}, + FFICallbackContext, +}; +use std::{ + ffi::{c_char, CStr, CString}, + fs, + ptr::{null, null_mut}, +}; + +unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { + print!("{}", CStr::from_ptr(ptr).to_str().unwrap()); +} + +extern "C" fn create_cb(_: FFICallbackContext, _: Link, _: Link) -> Flow { + Flow::Continue +} + +fn main() { + let level = CString::new("trace").unwrap(); + unsafe { + let handle = doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); + + let path = CString::new("doublets.links").unwrap(); + let mut store = + create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); + + let ptr = store.assume() as *mut _ as *mut _; + + let any = constants_for_store::(ptr).any; + + let query = [1 /* not exists index */, any, any]; + let result = delete::(ptr, query.as_ptr(), 3, null_mut(), create_cb); + + if result as u8 != DoubletsErrorKind::None as u8 { + let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); + + // last error - DON'T USE PTR AFTER NEW DOUBLETS OPERATION + let err = doublets_read_error(); + let mut msg_buf = vec![0u8; 256]; + + doublets_read_error_message(msg_buf.as_mut_ptr().cast(), 256, err); + + msg_buf.drain(memchr(&msg_buf) + 1..); + + let cstring = CString::from_vec_with_nul(msg_buf).unwrap(); + let str = cstring.to_str().unwrap(); + tracing::error!("{}", str); + + // forget `err` ptr - we not in manage it deallocation + let not_exists = doublets_read_error_as_not_found(err); + tracing::error!("duplication: link {} does not exists", not_exists); + + // forget `err` ptr - we not in manage it deallocation + } + + free_store::(ptr); + + doublets_free_log_handle(handle); + } + let _ = fs::remove_file("doublets.links"); +} diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs new file mode 100644 index 0000000..fc084e0 --- /dev/null +++ b/doublets-ffi/src/errors.rs @@ -0,0 +1,107 @@ +use crate::c_char; +use core::ffi::c_size_t; +use doublets::{data::LinkType, Doublet, Error, Link}; +use std::{cell::RefCell, cmp, error, ffi::c_short, io, mem::MaybeUninit, ptr}; + +#[repr(u8)] +pub enum DoubletsErrorKind { + None, + NotExists, + HasUsages, + AlreadyExists, + LimitReached, + AllocFailed, + Other, +} + +thread_local! { + static ERROR_BUF: RefCell>> = + RefCell::new(MaybeUninit::uninit()); +} + +fn link_cast( + Link { + index, + source, + target, + }: Link, +) -> Link { + Link::new(index.as_usize(), source.as_usize(), target.as_usize()) +} + +pub(crate) fn place_error(error: Error) { + use doublets::Error::*; + ERROR_BUF.with(|buf| { + buf.borrow_mut().write(match error { + NotExists(link) => NotExists(link.as_usize()), + HasUsages(usages) => HasUsages(usages.into_iter().map(link_cast).collect()), + AlreadyExists(Doublet { source, target }) => { + AlreadyExists(Doublet::new(source.as_usize(), target.as_usize())) + } + LimitReached(limit) => LimitReached(limit.as_usize()), + AllocFailed(alloc) => AllocFailed(alloc), + Other(other) => Other(other), + }); + }); +} + +unsafe fn write_raw_msg(buf: *mut c_char, size: c_short, msg: &str) { + let cap = cmp::min(size as usize, msg.len()) - 1; + ptr::copy_nonoverlapping(msg.as_ptr(), buf.cast(), cap); + ptr::write(buf.add(cap), 0); +} + +#[no_mangle] +pub extern "C" fn doublets_read_error() -> *const Error { + ERROR_BUF.with(|buf| buf.borrow().as_ptr()) +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_read_error_message( + buf: *mut c_char, + size: c_short, + err: *const Error, +) { + let error_msg = format!("{}", &*err); + write_raw_msg(buf, size, &error_msg); +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_read_error_backtrace( + buf: *mut c_char, + size: c_short, + err: *const Error, +) { + let error_msg = format!("{:?}", error::Error::source(&*err)); + write_raw_msg(buf, size, &error_msg); +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_read_error_as_not_found(err: *const Error) -> c_size_t { + match &*err { + Error::NotExists(link) => *link as c_size_t, + _ => panic!("error type is not `NotExists`"), + } +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_read_error_as_already_exists( + err: *const Error, +) -> Doublet { + match &*err { + Error::AlreadyExists(Doublet { source, target }) => { + Doublet::new(*source as c_size_t, *target as c_size_t) + } + _ => panic!("error type is not `AlreadyExists`"), + } +} + +#[no_mangle] +pub unsafe extern "C" fn doublets_read_error_as_limit_reached( + err: *const Error, +) -> c_size_t { + match &*err { + Error::LimitReached(limit) => *limit as c_size_t, + _ => panic!("error type is not `LimitReached`"), + } +} diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index dba0ca2..bde6e4c 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -1,8 +1,12 @@ #![feature(try_blocks)] #![feature(box_syntax)] #![feature(try_trait_v2)] +#![feature(c_size_t)] + +extern crate core; pub mod constants; +pub mod errors; pub mod export; pub mod logging; pub mod store; diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 6771479..4e859d9 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,20 +1,20 @@ -use crate::{c_char, c_void, constants::Constants, FFICallbackContext}; +use crate::{c_char, c_void, constants::Constants, errors::DoubletsErrorKind, FFICallbackContext}; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, mem::FileMapped, - parts, unit, Doublets, Link, Links, + parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; use std::{error, ffi::CStr, marker::PhantomData, ptr, slice}; use tap::Pipe; -use tracing::{debug, error, trace, warn}; +use tracing::{debug, error, warn}; // TODO: remove ::mem:: in doublets crate type UnitedLinks = unit::Store>>; -type EachCallback = extern "C" fn(FFICallbackContext, Link) -> T; +type EachCallback = extern "C" fn(FFICallbackContext, Link) -> Flow; -type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> T; +type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> Flow; #[repr(transparent)] pub struct StoreHandle { @@ -73,6 +73,14 @@ impl StoreHandle { } } + /// # Safety + /// should not live more than what is allowed + pub unsafe fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { + let leak = Self::from_raw(raw); + // SAFETY: Guarantee by caller + &mut *leak.ptr.cast() + } + pub fn assume(&mut self) -> &mut Box> { // SAFETY: `StoreHandle` must be create from safe `new()` // or unsafe `Self::from_raw` @@ -120,6 +128,36 @@ unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a thin_query_from_raw(query, len) } +fn place_error(err: Error) { + // It can be very expensive to handle each error + debug!(op_error = % err); + super::errors::place_error(err); +} + +impl DoubletsErrorKind { + pub fn leak(err: &Error) -> Self { + match err { + Error::NotExists(_) => DoubletsErrorKind::NotExists, + Error::HasUsages(_) => DoubletsErrorKind::HasUsages, + Error::AlreadyExists(_) => DoubletsErrorKind::AlreadyExists, + Error::LimitReached(_) => DoubletsErrorKind::LimitReached, + Error::AllocFailed(_) => DoubletsErrorKind::AllocFailed, + Error::Other(_) => DoubletsErrorKind::Other, + } + } +} + +fn acquire_result(result: Result>) -> DoubletsErrorKind { + match result { + Ok(_) => DoubletsErrorKind::None, + Err(err) => { + let ret = DoubletsErrorKind::leak(&err); + place_error(err); + ret + } + } +} + #[tracing::instrument( skip_all, fields( @@ -200,28 +238,11 @@ pub unsafe fn create( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> T { - let mut handle = StoreHandle::::from_raw(this); - let store = handle.assume(); - let constants = store.constants().clone(); - let (cnt, brk) = (constants.r#continue, constants.r#break); - +) -> DoubletsErrorKind { let query = query_from_raw(query, len); - let handler = move |before, after| { - if callback(ctx, before, after) == cnt { - Flow::Continue - } else { - Flow::Break - } - }; - store - .create_by_with(query, handler) - // fixme: add `.is_break` for `Flow` - .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) - .unwrap_or_else(|err| { - debug!(operation_error = %err); - constants.error - }) + let store = StoreHandle::::from_raw_assume(this); + let handler = move |before, after| callback(ctx, before, after); + store.create_by_with(query, handler).pipe(acquire_result) } #[tracing::instrument( @@ -253,13 +274,7 @@ pub unsafe fn each( let (cnt, brk) = (constants.r#continue, constants.r#break); let query = query_from_raw(query, len); - let handler = move |link| { - if callback(ctx, link) == cnt { - Flow::Continue - } else { - Flow::Break - } - }; + let handler = move |link| callback(ctx, link); store .each_by(query, handler) // fixme: add `.is_break` for `Flow` @@ -316,30 +331,24 @@ pub unsafe fn update( len_c: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> T { +) -> DoubletsErrorKind { let query = query_from_raw(query, len_q); let change = query_from_raw(change, len_c); - let mut handle = StoreHandle::::from_raw(this); - let store = handle.assume(); - let constants = store.constants().clone(); - let (cnt, brk) = (constants.r#continue, constants.r#break); - let handler = move |before, after| { - if callback(ctx, before, after) == cnt { - Flow::Continue - } else { - Flow::Break - } - }; + let store = StoreHandle::::from_raw_assume(this); + let handler = move |before, after| callback(ctx, before, after); store .update_by_with(query, change, handler) - // fixme: add `.is_break` for `Flow` - .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) - .unwrap_or_else(|err| { - debug!(operation_error = %err); - constants.error - }) + .pipe(acquire_result) } +#[tracing::instrument( + skip_all, + fields( + query = ?&thin_query_from_raw(query, len)[..], + query.ptr = ?query, + query.len = len, + ) +)] #[ffi::specialize_for( types = "u8", types = "u16", @@ -354,26 +363,9 @@ pub unsafe fn delete( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> T { - let mut handle = StoreHandle::::from_raw(this); - let store = handle.assume(); - let constants = store.constants().clone(); - let (cnt, brk) = (constants.r#continue, constants.r#break); - +) -> DoubletsErrorKind { let query = query_from_raw(query, len); - let handler = move |before, after| { - if callback(ctx, before, after) == cnt { - Flow::Continue - } else { - Flow::Break - } - }; - store - .delete_by_with(query, handler) - // fixme: add `.is_break` for `Flow` - .map(|flow| if let Flow::Continue = flow { cnt } else { brk }) - .unwrap_or_else(|err| { - debug!(operation_error = %err); - constants.error - }) + let store = StoreHandle::::from_raw_assume(this); + let handler = move |before, after| callback(ctx, before, after); + store.delete_by_with(query, handler).pipe(acquire_result) } diff --git a/doublets/src/data/doublet.rs b/doublets/src/data/doublet.rs index 2a5d39b..7286842 100644 --- a/doublets/src/data/doublet.rs +++ b/doublets/src/data/doublet.rs @@ -3,6 +3,7 @@ use std::fmt::{Debug, Display, Formatter}; use data::LinkType; #[derive(Debug, Eq, PartialEq, Hash, Clone)] +#[repr(C)] pub struct Doublet { pub source: T, pub target: T, From a40e21ed002aba708c8506b85b30eeba1cf51b29 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 19 Aug 2022 15:48:07 +0300 Subject: [PATCH 08/76] Use `thread_local` attribute for `ERROR_PLACE` --- doublets-ffi/src/errors.rs | 29 +++++++++++++---------------- doublets-ffi/src/lib.rs | 1 + 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index fc084e0..39076c5 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -14,10 +14,8 @@ pub enum DoubletsErrorKind { Other, } -thread_local! { - static ERROR_BUF: RefCell>> = - RefCell::new(MaybeUninit::uninit()); -} +#[thread_local] +static ERROR_PLACE: RefCell>> = RefCell::new(MaybeUninit::uninit()); fn link_cast( Link { @@ -31,17 +29,16 @@ fn link_cast( pub(crate) fn place_error(error: Error) { use doublets::Error::*; - ERROR_BUF.with(|buf| { - buf.borrow_mut().write(match error { - NotExists(link) => NotExists(link.as_usize()), - HasUsages(usages) => HasUsages(usages.into_iter().map(link_cast).collect()), - AlreadyExists(Doublet { source, target }) => { - AlreadyExists(Doublet::new(source.as_usize(), target.as_usize())) - } - LimitReached(limit) => LimitReached(limit.as_usize()), - AllocFailed(alloc) => AllocFailed(alloc), - Other(other) => Other(other), - }); + + ERROR_PLACE.borrow_mut().write(match error { + NotExists(link) => NotExists(link.as_usize()), + HasUsages(usages) => HasUsages(usages.into_iter().map(link_cast).collect()), + AlreadyExists(Doublet { source, target }) => { + AlreadyExists(Doublet::new(source.as_usize(), target.as_usize())) + } + LimitReached(limit) => LimitReached(limit.as_usize()), + AllocFailed(alloc) => AllocFailed(alloc), + Other(other) => Other(other), }); } @@ -53,7 +50,7 @@ unsafe fn write_raw_msg(buf: *mut c_char, size: c_short, msg: &str) { #[no_mangle] pub extern "C" fn doublets_read_error() -> *const Error { - ERROR_BUF.with(|buf| buf.borrow().as_ptr()) + ERROR_PLACE.borrow().as_ptr() } #[no_mangle] diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index bde6e4c..5e89f46 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -2,6 +2,7 @@ #![feature(box_syntax)] #![feature(try_trait_v2)] #![feature(c_size_t)] +#![feature(thread_local)] extern crate core; From 7dd3cd9a33968f6f4d7c529401ea867b9300875d Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 19 Aug 2022 16:13:16 +0300 Subject: [PATCH 09/76] Improve `warn` message in `query_from_raw` --- doublets-ffi/src/store.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 4e859d9..960b271 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -119,10 +119,8 @@ unsafe fn thin_query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Que } unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { - // it not require `#[cfg(debug_assertions)]`, - // because it is used in debug log mode only (llvm optimization:)) if query.is_null() && len != 0 { - warn!("if `query` is null then `len` must be 0"); + warn!("query ptr is null, but len is not null: this could be a potential mistake."); } thin_query_from_raw(query, len) From 46000619bbe74367d25fe45921b6f90d989fcd93 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 19 Aug 2022 16:32:00 +0300 Subject: [PATCH 10/76] Improvements list: - Fix forgotten `each` in moder style - Rename `DoubletsErrorKind` to `DoubletsResultKind` (it must contain `Continue`/`Break`) - improve error handling for create store (and fix example) --- doublets-ffi/examples/rust/examples/doublets-context.rs | 2 +- doublets-ffi/examples/rust/examples/error-handling.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index 165a74d..b92781c 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -4,7 +4,7 @@ use doublets::{ }; use doublets_ffi::{ constants::Constants, - errors::DoubletsErrorKind, + errors::DoubletsResultKind, export::{doublets_create_log_handle, doublets_free_log_handle}, store::{create, doublets_constants_u64, doublets_create_united_store_u64}, FFICallbackContext, diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index caeb6a2..0d42de1 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -6,7 +6,7 @@ use doublets_ffi::{ constants::Constants, errors::{ doublets_read_error, doublets_read_error_as_not_found, doublets_read_error_message, - DoubletsErrorKind, + DoubletsResultKind, }, export::{doublets_create_log_handle, doublets_free_log_handle}, store::{constants_for_store, create_unit_store, delete, free_store}, @@ -35,6 +35,11 @@ fn main() { let mut store = create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); + // `StoreHandle` is transparent - in really FFI we must use raw ptr + if store.as_ptr().is_null() { + unreachable!("it would be better for errors not to occur in the examples") + } + let ptr = store.assume() as *mut _ as *mut _; let any = constants_for_store::(ptr).any; @@ -42,7 +47,7 @@ fn main() { let query = [1 /* not exists index */, any, any]; let result = delete::(ptr, query.as_ptr(), 3, null_mut(), create_cb); - if result as u8 != DoubletsErrorKind::None as u8 { + if result as u8 >= DoubletsResultKind::NotExists as u8 { let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); // last error - DON'T USE PTR AFTER NEW DOUBLETS OPERATION From 4f6b95b3f18f58e4ec89dbfca596fb75953f3abd Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 19 Aug 2022 16:32:31 +0300 Subject: [PATCH 11/76] Improvements list: - Fix forgotten `each` in moder style - Rename `DoubletsErrorKind` to `DoubletsResultKind` (it must contain `Continue`/`Break`) - improve error handling for create store (and fix example) --- doublets-ffi/src/errors.rs | 7 ++-- doublets-ffi/src/store.rs | 69 +++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 39076c5..855966d 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -4,8 +4,11 @@ use doublets::{data::LinkType, Doublet, Error, Link}; use std::{cell::RefCell, cmp, error, ffi::c_short, io, mem::MaybeUninit, ptr}; #[repr(u8)] -pub enum DoubletsErrorKind { - None, +pub enum DoubletsResultKind { + // oks + Break, + Continue, + // errors NotExists, HasUsages, AlreadyExists, diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 960b271..635044f 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,4 +1,4 @@ -use crate::{c_char, c_void, constants::Constants, errors::DoubletsErrorKind, FFICallbackContext}; +use crate::{c_char, c_void, constants::Constants, errors::DoubletsResultKind, FFICallbackContext}; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, mem::FileMapped, @@ -88,8 +88,8 @@ impl StoreHandle { unsafe { &mut *self.ptr.cast() } } - pub fn invalid(err: Box) -> Self { - error!(err); + pub fn invalid(err: Error) -> Self { + acquire_error(err); // we not have access to self inner Self { ptr: ptr::null_mut(), @@ -97,6 +97,10 @@ impl StoreHandle { } } + pub fn as_ptr(&self) -> *const c_void { + self.ptr + } + pub fn drop(mut handle: Self) { // SAFETY: `self.store` is valid `Store` ptr unsafe { @@ -132,27 +136,37 @@ fn place_error(err: Error) { super::errors::place_error(err); } -impl DoubletsErrorKind { +impl DoubletsResultKind { + pub fn branch(flow: Flow) -> Self { + if let Flow::Continue = flow { + DoubletsResultKind::Continue + } else { + DoubletsResultKind::Break + } + } + pub fn leak(err: &Error) -> Self { match err { - Error::NotExists(_) => DoubletsErrorKind::NotExists, - Error::HasUsages(_) => DoubletsErrorKind::HasUsages, - Error::AlreadyExists(_) => DoubletsErrorKind::AlreadyExists, - Error::LimitReached(_) => DoubletsErrorKind::LimitReached, - Error::AllocFailed(_) => DoubletsErrorKind::AllocFailed, - Error::Other(_) => DoubletsErrorKind::Other, + Error::NotExists(_) => DoubletsResultKind::NotExists, + Error::HasUsages(_) => DoubletsResultKind::HasUsages, + Error::AlreadyExists(_) => DoubletsResultKind::AlreadyExists, + Error::LimitReached(_) => DoubletsResultKind::LimitReached, + Error::AllocFailed(_) => DoubletsResultKind::AllocFailed, + Error::Other(_) => DoubletsResultKind::Other, } } } -fn acquire_result(result: Result>) -> DoubletsErrorKind { +fn acquire_error(err: Error) -> DoubletsResultKind { + let ret = DoubletsResultKind::leak(&err); + place_error(err); + ret +} + +fn acquire_result(result: Result>) -> DoubletsResultKind { match result { - Ok(_) => DoubletsErrorKind::None, - Err(err) => { - let ret = DoubletsErrorKind::leak(&err); - place_error(err); - ret - } + Ok(flow) => DoubletsResultKind::branch(flow), + Err(err) => acquire_error(err), } } @@ -175,10 +189,10 @@ pub unsafe fn create_unit_store( path: *const c_char, constants: Constants, ) -> StoreHandle { - let result: Result<_, Box> = try { + let result: Result<_, Error> = try { let path = CStr::from_ptr(path).to_str().unwrap(); let mem = FileMapped::from_path(path)?; - StoreHandle::new(Box::new(UnitedLinks::::with_constants( + StoreHandle::new(Box::new(UnitedLinks::with_constants( mem, constants.into(), )?)) @@ -236,7 +250,7 @@ pub unsafe fn create( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsErrorKind { +) -> DoubletsResultKind { let query = query_from_raw(query, len); let store = StoreHandle::::from_raw_assume(this); let handler = move |before, after| callback(ctx, before, after); @@ -265,18 +279,13 @@ pub unsafe fn each( len: u32, ctx: FFICallbackContext, callback: EachCallback, -) -> T { - let mut handle = StoreHandle::::from_raw(this); - let store = handle.assume(); - let constants = store.constants(); - let (cnt, brk) = (constants.r#continue, constants.r#break); - +) -> DoubletsResultKind { let query = query_from_raw(query, len); + let store = StoreHandle::::from_raw_assume(this); let handler = move |link| callback(ctx, link); store .each_by(query, handler) - // fixme: add `.is_break` for `Flow` - .pipe(|flow| if let Flow::Continue = flow { cnt } else { brk }) + .pipe(DoubletsResultKind::branch) } #[tracing::instrument( @@ -329,7 +338,7 @@ pub unsafe fn update( len_c: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsErrorKind { +) -> DoubletsResultKind { let query = query_from_raw(query, len_q); let change = query_from_raw(change, len_c); let store = StoreHandle::::from_raw_assume(this); @@ -361,7 +370,7 @@ pub unsafe fn delete( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsErrorKind { +) -> DoubletsResultKind { let query = query_from_raw(query, len); let store = StoreHandle::::from_raw_assume(this); let handler = move |before, after| callback(ctx, before, after); From a11188f20097bb05842983c35fc8617df319dfea Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 19 Aug 2022 16:34:58 +0300 Subject: [PATCH 12/76] Update outdated data --- dev-deps/data-rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-deps/data-rs b/dev-deps/data-rs index 95b7d4d..cd3cd4f 160000 --- a/dev-deps/data-rs +++ b/dev-deps/data-rs @@ -1 +1 @@ -Subproject commit 95b7d4deb04e1ba34d2fa3187dca80cc7f62c78e +Subproject commit cd3cd4f1ece69b8342b099e39a7d3e8a014dc5c7 From edd0b87700f1a2a0593ec83480c14ceb21e0a7a5 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Mon, 22 Aug 2022 17:04:22 +0300 Subject: [PATCH 13/76] Use `NonNull` instead of raw pointer --- doublets-ffi/src/store.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 635044f..d4abf38 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,3 +1,5 @@ +#![allow(clippy::missing_safety_doc)] + use crate::{c_char, c_void, constants::Constants, errors::DoubletsResultKind, FFICallbackContext}; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, @@ -5,9 +7,9 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{error, ffi::CStr, marker::PhantomData, ptr, slice}; +use std::{ffi::CStr, marker::PhantomData, ptr, ptr::NonNull, slice}; use tap::Pipe; -use tracing::{debug, error, warn}; +use tracing::{debug, warn}; // TODO: remove ::mem:: in doublets crate type UnitedLinks = unit::Store>>; @@ -18,7 +20,7 @@ type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> Flo #[repr(transparent)] pub struct StoreHandle { - pub(crate) ptr: *mut c_void, // dyn Doublets + pub(crate) ptr: NonNull, // thin ptr to dyn Doublets marker: PhantomData, } @@ -67,8 +69,10 @@ impl StoreHandle { /// allocated in `Box` /// without owner pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { + debug_assert!(!raw.is_null()); + Self { - ptr: raw, + ptr: NonNull::new_unchecked(raw), marker: PhantomData, } } @@ -78,37 +82,33 @@ impl StoreHandle { pub unsafe fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { let leak = Self::from_raw(raw); // SAFETY: Guarantee by caller - &mut *leak.ptr.cast() + leak.ptr.cast().as_mut() } pub fn assume(&mut self) -> &mut Box> { // SAFETY: `StoreHandle` must be create from safe `new()` // or unsafe `Self::from_raw` // then it guarantee by `Self::from_raw()` caller - unsafe { &mut *self.ptr.cast() } + unsafe { self.ptr.cast().as_mut() } } pub fn invalid(err: Error) -> Self { acquire_error(err); // we not have access to self inner Self { - ptr: ptr::null_mut(), + ptr: NonNull::dangling(), marker: PhantomData, } } pub fn as_ptr(&self) -> *const c_void { - self.ptr + self.ptr.as_ptr() } pub fn drop(mut handle: Self) { // SAFETY: `self.store` is valid `Store` ptr unsafe { - // assume ptr is not null - // guarantee by `from_raw` and `new` - if !handle.ptr.is_null() { - let _ = Box::from_raw(handle.assume()); - } + let _ = Box::from_raw(handle.assume()); } } } From 646f1aec4e7a7882ea784dde6462ca1ca0a6b67d Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 23 Aug 2022 12:40:13 +0300 Subject: [PATCH 14/76] `cargo fix` unused imports --- doublets-ffi/src/errors.rs | 2 +- doublets-ffi/src/logging.rs | 2 +- doublets-ffi/src/store.rs | 104 +++++++++++++++++++----------------- 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 855966d..27db57e 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,7 +1,7 @@ use crate::c_char; use core::ffi::c_size_t; use doublets::{data::LinkType, Doublet, Error, Link}; -use std::{cell::RefCell, cmp, error, ffi::c_short, io, mem::MaybeUninit, ptr}; +use std::{cell::RefCell, cmp, error, ffi::c_short, mem::MaybeUninit, ptr}; #[repr(u8)] pub enum DoubletsResultKind { diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index e05d256..4625830 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -2,7 +2,7 @@ use super::{c_char, FFICallbackContext}; use crate::FFICallbackContextWrapper; use crossbeam_channel::{self as mpsc, Sender}; use std::{ffi::CString, io, str::FromStr, thread}; -use tracing::{error, info_span, warn}; +use tracing::{error}; use tracing_subscriber::{ filter::{EnvFilter, LevelFilter}, fmt::MakeWriter, diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index d4abf38..483a733 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -7,7 +7,7 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{ffi::CStr, marker::PhantomData, ptr, ptr::NonNull, slice}; +use std::{ffi::CStr, marker::PhantomData, ptr::NonNull, slice}; use tap::Pipe; use tracing::{debug, warn}; @@ -178,12 +178,13 @@ fn acquire_result(result: Result>) -> DoubletsResult ), )] #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_create_united_store_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_create_united_store_*", )] pub unsafe fn create_unit_store( path: *const c_char, @@ -201,23 +202,25 @@ pub unsafe fn create_unit_store( } #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_free_store_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_free_store_*", )] pub unsafe fn free_store(this: *mut c_void) { StoreHandle::drop(StoreHandle::::from_raw(this)) } #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ) name = "doublets_constants_*" )] pub unsafe fn constants_for_store(this: *mut c_void) -> Constants { @@ -237,12 +240,13 @@ pub unsafe fn constants_for_store(this: *mut c_void) -> Constants( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_create_*", )] pub unsafe fn create( this: *mut c_void, @@ -266,12 +270,13 @@ pub unsafe fn create( ), )] #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_each_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_each_*", )] pub unsafe fn each( this: *mut c_void, @@ -297,12 +302,13 @@ pub unsafe fn each( ), )] #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_count_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_count_*", )] pub unsafe fn count(this: *mut c_void, query: *const T, len: u32) -> T { let mut handle = StoreHandle::::from_raw(this); @@ -323,12 +329,13 @@ pub unsafe fn count(this: *mut c_void, query: *const T, len: u32) - ), )] #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_update_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_update_*", )] pub unsafe fn update( this: *mut c_void, @@ -357,12 +364,13 @@ pub unsafe fn update( ) )] #[ffi::specialize_for( - types = "u8", - types = "u16", - types = "u32", - types = "u64", - convention = "rust", - name = "doublets_delete_*" + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_delete_*", )] pub unsafe fn delete( this: *mut c_void, From 6d7c353294846a9ed1185d179ac193a6ce49e0ca Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 23 Aug 2022 12:40:27 +0300 Subject: [PATCH 15/76] `cargo fix` unused imports --- doublets-ffi/examples/rust/examples/doublets-context.rs | 3 +-- doublets-ffi/examples/rust/examples/error-handling.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index b92781c..c222bcf 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -4,9 +4,8 @@ use doublets::{ }; use doublets_ffi::{ constants::Constants, - errors::DoubletsResultKind, export::{doublets_create_log_handle, doublets_free_log_handle}, - store::{create, doublets_constants_u64, doublets_create_united_store_u64}, + store::{create, doublets_create_united_store_u64}, FFICallbackContext, }; use std::{ diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 0d42de1..bd9ae27 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -15,7 +15,7 @@ use doublets_ffi::{ use std::{ ffi::{c_char, CStr, CString}, fs, - ptr::{null, null_mut}, + ptr::{null_mut}, }; unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { From abee06b6d10081b5847cea82936317cc2afa22cc Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 23 Aug 2022 12:42:24 +0300 Subject: [PATCH 16/76] Improve `ffi-attributes`: - use modern syntax: ```rust #[ffi::specialize_for( types( u8 => "u8", u16 => "uint16", u32 => "uint", u64 => "ll", ), name = "...*", )] ``` - add warnings handler - use more beautiful code --- doublets-ffi/ffi-attributes/src/expand.rs | 193 ++++++++++ doublets-ffi/ffi-attributes/src/lib.rs | 413 ++++++++++++--------- doublets-ffi/ffi-attributes/src/prepare.rs | 67 ++++ 3 files changed, 496 insertions(+), 177 deletions(-) create mode 100644 doublets-ffi/ffi-attributes/src/expand.rs create mode 100644 doublets-ffi/ffi-attributes/src/prepare.rs diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs new file mode 100644 index 0000000..b889bb4 --- /dev/null +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -0,0 +1,193 @@ +use crate::{prepare, FromMeta, SpecializeArgs}; +use proc_macro::Diagnostic; +use proc_macro2::{Ident, TokenStream}; +use quote::{quote, ToTokens}; + +use syn::{ + punctuated::Punctuated, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, Token, TypeParam, +}; + +fn filter_generics<'gen>( + generics: impl Iterator + 'gen, + param: &'gen Ident, +) -> impl Iterator + 'gen { + generics + .filter(move |&par| { + if let GenericParam::Type(TypeParam { ident, .. }) = par { + ident != param + } else { + true + } + }) + .cloned() +} + +fn prepare_fn_args<'args>( + fn_args: impl Iterator, + param: &Ident, + ident: &Ident, +) { + for arg in fn_args { + match arg { + FnArg::Typed(PatType { box ty, .. }) => { + *ty = prepare::replace_ty_in_param(ty.clone(), param, ident); + } + FnArg::Receiver(_) => { + todo!() + } + } + } +} + +fn prepare_output_type(output: &mut ReturnType, param: &Ident, ident: &Ident) { + if let ReturnType::Type(_, box ty) = output { + *ty = prepare::replace_ty_in_param(ty.clone(), param, ident); + } +} + +fn args_idents<'args>( + args: impl Iterator, +) -> impl Iterator { + args.map(|arg| match arg { + FnArg::Typed(PatType { box pat, .. }) => pat as &dyn ToTokens, + _ => { + todo!() + } + }) +} + +fn build_fn(input: ItemFn, real_fn: &Ident, param: &Ident) -> TokenStream { + let ItemFn { + attrs, vis, sig, .. + } = input; + + let Signature { + output: return_type, + inputs: params, + unsafety, + asyncness, + constness, + abi, + ident, + generics: + syn::Generics { + params: gen_params, + where_clause, + .. + }, + .. + } = sig; + + let args = args_idents(params.iter()); + quote! { + #(#attrs) * + #vis #constness #unsafety #asyncness #abi fn #ident<#gen_params>(#params) #return_type + #where_clause + { + #real_fn::<#param>(#(#args),*) + } + } +} + +fn gen_new_def(mut fn_list: TokenStream, input: ItemFn, args: SpecializeArgs) -> TokenStream { + let real_name = input.sig.ident.to_string(); + let name_pat = args + .name + .map(|name| name.to_token_stream().to_string()) + .unwrap_or_else(|| real_name + "_*"); + let name_pat = name_pat.trim_matches('"'); + + for (ty, lit) in args.aliases { + let ItemFn { + attrs, + vis, + sig, + block, + } = input.clone(); + + let Signature { + output: mut return_type, + inputs: mut params, + generics: syn::Generics { + params: gen_params, .. + }, + .. + } = sig.clone(); + + let param = args.param.as_ref().unwrap(); + let generics: Punctuated<_, Token![,]> = + Punctuated::from_iter(filter_generics(gen_params.iter(), param)); + prepare_fn_args(params.iter_mut(), param, &ty); + prepare_output_type(&mut return_type, param, &ty); + + let new_ident: Ident = + Ident::from_string(&name_pat.replace('*', &lit.to_token_stream().to_string())).unwrap(); + + let real_fn = sig.ident.clone(); + let sig = Signature { + ident: new_ident, + output: return_type, + inputs: params, + generics: syn::Generics { + params: generics, + ..sig.generics + }, + ..sig + }; + let new_fn = build_fn( + ItemFn { + attrs, + vis, + sig, + block, + }, + &real_fn, + &lit, + ); + fn_list = quote! { + #fn_list + #new_fn + } + } + + fn_list +} + +pub(crate) fn gen_function(input: ItemFn, args: SpecializeArgs) -> TokenStream { + args.warnings().for_each(Diagnostic::emit); + + let ItemFn { + attrs, + vis, + sig, + block, + } = input.clone(); + + let Signature { + output: return_type, + inputs: params, + unsafety, + asyncness, + constness, + abi, + ident, + generics: + syn::Generics { + params: gen_params, + where_clause, + .. + }, + .. + } = sig.clone(); + + let this = quote! { + #(#attrs) * + #vis #constness #unsafety #asyncness #abi fn #ident<#gen_params>(#params) #return_type + #where_clause + { + #block + } + }; + + gen_new_def(this, input, args) +} diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index b0d2b27..505eb97 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -1,208 +1,267 @@ #![feature(box_syntax)] +#![feature(proc_macro_diagnostic)] +#![feature(box_patterns)] -use proc_macro::TokenStream; +mod expand; +mod prepare; + +use proc_macro::{Level, Span}; +use std::{collections::HashMap, marker::PhantomData}; use darling::FromMeta; -use quote::{quote, ToTokens}; -use syn::{parse::Parser, punctuated::Punctuated}; + +use syn::punctuated::Punctuated; + use syn::{ - parse_macro_input, AttributeArgs, FnArg, GenericArgument, GenericParam, Ident, ItemFn, - PathArguments, ReturnType, Type, + parse::{Parse, ParseStream}, + parse_macro_input, + spanned::Spanned, + token::Token, Ident, ItemFn, LitStr, Token, }; -fn csharp_convention(s: String) -> String { - match s.as_str() { - "i8" => "SByte", - "u8" => "Byte", - "i16" => "Int16", - "u16" => "UInt16", - "i32" => "Int32", - "u32" => "UInt32", - "i64" => "Int64", - "u64" => "UInt64", - s => { - panic!("{} is incompatible with doublets-ffi type", s) - } +mod kw { + syn::custom_keyword!(types); + syn::custom_keyword!(name); +} + +#[derive(Clone, Default, Debug)] +struct SpecializeArgs { + name: Option, + param: Option, + aliases: HashMap, + /// Errors describing any unrecognized parse inputs that we skipped. + parse_warnings: Vec, +} + +impl SpecializeArgs { + pub(crate) fn warnings(&self) -> impl Iterator + '_ { + self.parse_warnings.iter().map(|err| { + let msg = format!("found unrecognized input, {}", err); + proc_macro::Diagnostic::spanned::, _>( + vec![err.span().unwrap()], + Level::Warning, + msg, + ) + }) } - .to_string() } -fn rust_convention(s: String) -> String { - match s.as_str() { - "i8" => "i8", - "u8" => "u8", - "i16" => "i16", - "u16" => "u16", - "i32" => "i32", - "u32" => "u32", - "i64" => "i64", - "u64" => "u64", - s => { - panic!("{} is incompatible with doublets-ffi type", s) +impl Parse for SpecializeArgs { + fn parse(input: ParseStream<'_>) -> syn::Result { + let mut args = Self::default(); + + while !input.is_empty() { + let lookahead = input.lookahead1(); + + if lookahead.peek(kw::name) { + if args.name.is_some() { + return Err(input.error("expected only a single `name` argument")); + } + let name = input.parse::>()?.value; + args.name = Some(name); + } else if lookahead.peek(kw::types) { + if !args.aliases.is_empty() { + return Err(input.error("expected only a single `types` argument")); + } + let AliasArg { param, aliases } = input.parse::()?; + args.param = Some(param); + args.aliases = aliases; + } else if lookahead.peek(Token![,]) { + let _ = input.parse::()?; + } else { + // Parse the unrecognized tokens stream, + // and ignore it away so we can keep parsing. + args.parse_warnings.push(lookahead.error()); + let _ = input.parse::(); + } } + + Ok(args) } - .to_string() } -#[derive(FromMeta, PartialEq, Eq, Debug)] -#[allow(non_camel_case_types)] -enum Conventions { - csharp, - rust, +struct StrArg { + value: LitStr, + _marker: PhantomData, } -#[derive(FromMeta)] -struct MacroArgs { - convention: Conventions, - #[darling(multiple)] - types: Vec, - name: String, +impl Parse for StrArg { + fn parse(input: ParseStream<'_>) -> syn::Result { + let _ = input.parse::()?; + let _ = input.parse::()?; + let value = input.parse()?; + Ok(Self { + value, + _marker: PhantomData, + }) + } } +struct AliasLine(Ident, Ident); -fn ty_from_to(ty: Type, from: &str, to: &str) -> Type { - match ty { - Type::Array(arr) => ty_from_to(*arr.elem, from, to), - Type::Path(mut path) => { - path.path.segments.iter_mut().for_each(|seg| { - if seg.ident.to_string().as_str() == from { - seg.ident = Ident::from_string(to).unwrap(); - } - match seg.arguments { - PathArguments::AngleBracketed(ref mut angle) => { - for arg in angle.args.iter_mut() { - match arg { - GenericArgument::Type(gty) => { - *gty = ty_from_to(gty.clone(), from, to); - } - _ => { - panic!("not doublets-ffi compatible generic") - } - } - } - } - PathArguments::Parenthesized(_) => { - todo!() - } - _ => { /* ignore */ } - } - }); - Type::Path(path) - } - Type::Ptr(mut ptr) => { - *ptr.elem = ty_from_to(*ptr.elem, from, to); - Type::Ptr(ptr) - } - Type::Reference(mut refer) => { - *refer.elem = ty_from_to(*refer.elem, from, to); - Type::Reference(refer) - } - _ => { - panic!("unexpected doublets-ffi type"); - } +impl Parse for AliasLine { + fn parse(input: ParseStream<'_>) -> syn::Result { + let ty = input.parse::()?; + let _ = input.parse::]>()?; + let str = input.parse::()?; + Ok(Self(ty, str)) } } -#[proc_macro_attribute] -pub fn specialize_for(args: TokenStream, input: TokenStream) -> TokenStream { - let attr_args = parse_macro_input!(args as AttributeArgs); - let input = parse_macro_input!(input as ItemFn); - let input_clone: ItemFn = input.clone(); - let ident = input.sig.ident; - // TODO: use args - let args = match MacroArgs::from_list(&attr_args) { - Ok(v) => v, - Err(e) => { - return TokenStream::from(e.write_errors()); - } - }; - //println!("{:?}", args.types); - - let inputs = input.sig.inputs; - let generic_name = { - let mut generics_names: Vec<_> = input - .sig - .generics - .params - .iter() - .map(|param| match param { - GenericParam::Lifetime(_) => { - panic!("`lifetime` generic is not supported") - } - GenericParam::Const(_) => { - panic!("`const` generic is not supported") - } - GenericParam::Type(ty) => ty.ident.to_string(), - }) - .collect(); - assert_eq!(generics_names.len(), 1); - generics_names.remove(0) - }; - - let fn_pat = args.name; - let asterisk_count = fn_pat.chars().filter(|c| *c == '*').count(); - assert_eq!(asterisk_count, 1); - - let mut out = quote! { #input_clone }; - - for ty in args.types { - let ty_str = ty.as_str(); - let ty_tt: proc_macro2::TokenStream = ty.parse().unwrap(); - let fn_pat: proc_macro2::TokenStream = fn_pat - .replace( - '*', - match &args.convention { - Conventions::csharp => csharp_convention(ty.clone()), - Conventions::rust => rust_convention(ty.clone()), - _ => { - panic!("unknown convention") - } - } - .as_str(), - ) - .parse() - .unwrap(); +struct AliasArg { + param: Ident, + aliases: HashMap, +} - let mut inputs: Punctuated = inputs.clone(); +impl Parse for AliasArg { + fn parse(input: ParseStream<'_>) -> syn::Result { + let _ = input.parse::()?; + let _ = input.parse::()?; + let _ = input.parse::()?; + let param = input.parse::()?; + let _ = input.parse::]>()?; + let content; + let _ = syn::parenthesized!(content in input); + let aliases: Punctuated = + content.parse_terminated(AliasLine::parse)?; - let output_ty: proc_macro2::TokenStream = match &input.sig.output { - ReturnType::Default => "()".parse().unwrap(), - ReturnType::Type(_, ty) => { - ty_from_to((**ty).clone(), &generic_name, ty_str).to_token_stream() + let mut map = HashMap::new(); + for AliasLine(ty, lit) in aliases { + #[allow(clippy::map_entry)] + if map.contains_key(&ty) { + return Err(syn::Error::new( + ty.span().join(lit.span()).unwrap(), + "tried to skip the same field twice", + )); + } else { + map.insert(ty, lit); } - }; + } - inputs.iter_mut().for_each(|arg| match arg { - FnArg::Receiver(_) => { - panic!("function with `self` is not supported") - } - FnArg::Typed(pat_type) => { - pat_type.ty = box ty_from_to(*(pat_type.ty).clone(), &generic_name, &ty); - } - }); - - let _generic_name: proc_macro2::TokenStream = generic_name.parse().unwrap(); - let input_args: Vec<_> = inputs - .iter() - .map(|arg| match arg { - FnArg::Receiver(_) => { - unreachable!() - } - FnArg::Typed(ty) => ty.pat.to_token_stream(), - }) - .collect(); - - out = quote! { - #out - #[no_mangle] - pub unsafe extern "C" fn #fn_pat(#inputs) -> #output_ty { - #ident::<#ty_tt>(#(#input_args),*) - } - }; + Ok(Self { + param, + aliases: map, + }) } +} + +fn specialize_precise( + args: SpecializeArgs, + item: proc_macro::TokenStream, +) -> syn::Result { + let input = syn::parse::(item)?; + Ok(expand::gen_function(input, args).into()) +} + +#[proc_macro_attribute] +pub fn specialize_for( + args: proc_macro::TokenStream, + item: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + let args = parse_macro_input!(args as SpecializeArgs); + + specialize_precise(args, item.clone()).unwrap_or_else(|_err| todo!()) + + /* + let input = parse_macro_input!(input as ItemFn); + let input_clone: ItemFn = input.clone(); + let ident = input.sig.ident; + // TODO: use args + let args = match MacroArgs::from_list(&attr_args) { + Ok(v) => v, + Err(e) => { + return TokenStream::from(e.write_errors()); + } + }; + //println!("{:?}", args.types); + + let inputs = input.sig.inputs; + let generic_name = { + let mut generics_names: Vec<_> = input + .sig + .generics + .params + .iter() + .map(|param| match param { + GenericParam::Lifetime(_) => { + panic!("`lifetime` generic is not supported") + } + GenericParam::Const(_) => { + panic!("`const` generic is not supported") + } + GenericParam::Type(ty) => ty.ident.to_string(), + }) + .collect(); + assert_eq!(generics_names.len(), 1); + generics_names.remove(0) + }; + + let fn_pat = args.name; + let asterisk_count = fn_pat.chars().filter(|c| *c == '*').count(); + assert_eq!(asterisk_count, 1); + + let mut out = quote! { #input_clone }; + + for ty in args.types { + let ty_str = ty.as_str(); + let ty_tt: proc_macro2::TokenStream = ty.parse().unwrap(); + let fn_pat: proc_macro2::TokenStream = fn_pat + .replace( + '*', + match &args.convention { + Conventions::csharp => csharp_convention(ty.clone()), + Conventions::rust => rust_convention(ty.clone()), + _ => { + panic!("unknown convention") + } + } + .as_str(), + ) + .parse() + .unwrap(); + + let mut inputs: Punctuated = inputs.clone(); + + let output_ty: proc_macro2::TokenStream = match &input.sig.output { + ReturnType::Default => "()".parse().unwrap(), + ReturnType::Type(_, ty) => { + ty_from_to((**ty).clone(), &generic_name, ty_str).to_token_stream() + } + }; + + inputs.iter_mut().for_each(|arg| match arg { + FnArg::Receiver(_) => { + panic!("function with `self` is not supported") + } + FnArg::Typed(pat_type) => { + pat_type.ty = box ty_from_to(*(pat_type.ty).clone(), &generic_name, &ty); + } + }); + + let _generic_name: proc_macro2::TokenStream = generic_name.parse().unwrap(); + let input_args: Vec<_> = inputs + .iter() + .map(|arg| match arg { + FnArg::Receiver(_) => { + unreachable!() + } + FnArg::Typed(ty) => ty.pat.to_token_stream(), + }) + .collect(); + + out = quote! { + #out + #[no_mangle] + pub unsafe extern "C" fn #fn_pat(#inputs) -> #output_ty { + #ident::<#ty_tt>(#(#input_args),*) + } + }; + } + + //println!("{}", out); - //println!("{}", out); + out.into() - out.into() + */ } diff --git a/doublets-ffi/ffi-attributes/src/prepare.rs b/doublets-ffi/ffi-attributes/src/prepare.rs new file mode 100644 index 0000000..b469aed --- /dev/null +++ b/doublets-ffi/ffi-attributes/src/prepare.rs @@ -0,0 +1,67 @@ +use proc_macro2::Ident; +use syn::{ + GenericArgument, ParenthesizedGenericArguments, PathArguments, ReturnType, Type, TypePath, +}; + +pub(crate) fn prepare_path(mut path: TypePath, from: &Ident, to: &Ident) -> TypePath { + path.path.segments.iter_mut().for_each(|seg| { + if &seg.ident == from { + seg.ident = to.clone(); + } + match &mut seg.arguments { + PathArguments::AngleBracketed(angle) => { + for arg in &mut angle.args { + match arg { + GenericArgument::Type(gty) => { + *gty = replace_ty_in_param(gty.clone(), from, to); + } + _ => { /* ignore */ } + } + } + } + PathArguments::Parenthesized(ParenthesizedGenericArguments { + inputs, output, .. + }) => { + for input in inputs { + *input = replace_ty_in_param(input.clone(), from, to); + } + if let ReturnType::Type(_, box ty) = output { + *ty = replace_ty_in_param(ty.clone(), from, to); + } + } + _ => { /* ignore */ } + } + }); + path +} + +pub(crate) fn replace_ty_in_param(ty: Type, from: &Ident, to: &Ident) -> Type { + match ty { + Type::Path(path) => Type::Path(prepare_path(path, from, to)), + Type::Array(mut arr) => { + *arr.elem = replace_ty_in_param(*arr.elem, from, to); + Type::Array(arr) + } + Type::Ptr(mut ptr) => { + *ptr.elem = replace_ty_in_param(*ptr.elem, from, to); + Type::Ptr(ptr) + } + Type::Reference(mut refer) => { + *refer.elem = replace_ty_in_param(*refer.elem, from, to); + Type::Reference(refer) + } + Type::Slice(mut slice) => { + *slice.elem = replace_ty_in_param(*slice.elem, from, to); + Type::Slice(slice) + } + Type::Tuple(mut tuple) => { + for elem in &mut tuple.elems { + *elem = replace_ty_in_param(elem.clone(), from, to); + } + Type::Tuple(tuple) + } + _ => { + todo!() + } + } +} From 8420a12b008f8bb67fc3b11d4a2255c419ef1b5e Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 23 Aug 2022 12:45:48 +0300 Subject: [PATCH 17/76] Remove forgotten comment --- doublets-ffi/ffi-attributes/src/lib.rs | 108 +------------------------ 1 file changed, 2 insertions(+), 106 deletions(-) diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 505eb97..9bca096 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -10,15 +10,14 @@ use std::{collections::HashMap, marker::PhantomData}; use darling::FromMeta; - use syn::punctuated::Punctuated; - use syn::{ parse::{Parse, ParseStream}, parse_macro_input, spanned::Spanned, - token::Token, Ident, ItemFn, LitStr, Token, + token::Token, + Ident, ItemFn, LitStr, Token, }; mod kw { @@ -160,108 +159,5 @@ pub fn specialize_for( item: proc_macro::TokenStream, ) -> proc_macro::TokenStream { let args = parse_macro_input!(args as SpecializeArgs); - specialize_precise(args, item.clone()).unwrap_or_else(|_err| todo!()) - - /* - let input = parse_macro_input!(input as ItemFn); - let input_clone: ItemFn = input.clone(); - let ident = input.sig.ident; - // TODO: use args - let args = match MacroArgs::from_list(&attr_args) { - Ok(v) => v, - Err(e) => { - return TokenStream::from(e.write_errors()); - } - }; - //println!("{:?}", args.types); - - let inputs = input.sig.inputs; - let generic_name = { - let mut generics_names: Vec<_> = input - .sig - .generics - .params - .iter() - .map(|param| match param { - GenericParam::Lifetime(_) => { - panic!("`lifetime` generic is not supported") - } - GenericParam::Const(_) => { - panic!("`const` generic is not supported") - } - GenericParam::Type(ty) => ty.ident.to_string(), - }) - .collect(); - assert_eq!(generics_names.len(), 1); - generics_names.remove(0) - }; - - let fn_pat = args.name; - let asterisk_count = fn_pat.chars().filter(|c| *c == '*').count(); - assert_eq!(asterisk_count, 1); - - let mut out = quote! { #input_clone }; - - for ty in args.types { - let ty_str = ty.as_str(); - let ty_tt: proc_macro2::TokenStream = ty.parse().unwrap(); - let fn_pat: proc_macro2::TokenStream = fn_pat - .replace( - '*', - match &args.convention { - Conventions::csharp => csharp_convention(ty.clone()), - Conventions::rust => rust_convention(ty.clone()), - _ => { - panic!("unknown convention") - } - } - .as_str(), - ) - .parse() - .unwrap(); - - let mut inputs: Punctuated = inputs.clone(); - - let output_ty: proc_macro2::TokenStream = match &input.sig.output { - ReturnType::Default => "()".parse().unwrap(), - ReturnType::Type(_, ty) => { - ty_from_to((**ty).clone(), &generic_name, ty_str).to_token_stream() - } - }; - - inputs.iter_mut().for_each(|arg| match arg { - FnArg::Receiver(_) => { - panic!("function with `self` is not supported") - } - FnArg::Typed(pat_type) => { - pat_type.ty = box ty_from_to(*(pat_type.ty).clone(), &generic_name, &ty); - } - }); - - let _generic_name: proc_macro2::TokenStream = generic_name.parse().unwrap(); - let input_args: Vec<_> = inputs - .iter() - .map(|arg| match arg { - FnArg::Receiver(_) => { - unreachable!() - } - FnArg::Typed(ty) => ty.pat.to_token_stream(), - }) - .collect(); - - out = quote! { - #out - #[no_mangle] - pub unsafe extern "C" fn #fn_pat(#inputs) -> #output_ty { - #ident::<#ty_tt>(#(#input_args),*) - } - }; - } - - //println!("{}", out); - - out.into() - - */ } From 4ec685ac4e3aafbdc8a6147668dff699686dc518 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 23 Aug 2022 12:53:01 +0300 Subject: [PATCH 18/76] Fix type `lit` => `ty` --- doublets-ffi/ffi-attributes/src/expand.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs index b889bb4..c3c1812 100644 --- a/doublets-ffi/ffi-attributes/src/expand.rs +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -4,7 +4,8 @@ use proc_macro2::{Ident, TokenStream}; use quote::{quote, ToTokens}; use syn::{ - punctuated::Punctuated, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, Token, TypeParam, + punctuated::Punctuated, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, Token, + TypeParam, }; fn filter_generics<'gen>( @@ -142,7 +143,7 @@ fn gen_new_def(mut fn_list: TokenStream, input: ItemFn, args: SpecializeArgs) -> block, }, &real_fn, - &lit, + &ty, ); fn_list = quote! { #fn_list From 6c7176027c932c138d7ed495872155937b7a168d Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 10:28:51 +0300 Subject: [PATCH 19/76] Add `attributes` keyword to delegate some attrs to parent function implementation - fix ffi build --- doublets-ffi/ffi-attributes/src/expand.rs | 19 ++++++--- doublets-ffi/ffi-attributes/src/lib.rs | 13 +++++- doublets-ffi/src/store.rs | 50 +++++++++++++++++------ 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs index c3c1812..d2a0739 100644 --- a/doublets-ffi/ffi-attributes/src/expand.rs +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -1,11 +1,11 @@ -use crate::{prepare, FromMeta, SpecializeArgs}; +use crate::{attributes, prepare, FromMeta, SpecializeArgs}; use proc_macro::Diagnostic; use proc_macro2::{Ident, TokenStream}; use quote::{quote, ToTokens}; use syn::{ - punctuated::Punctuated, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, Token, - TypeParam, + punctuated::Punctuated, Attribute, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, + Token, TypeParam, }; fn filter_generics<'gen>( @@ -57,7 +57,12 @@ fn args_idents<'args>( }) } -fn build_fn(input: ItemFn, real_fn: &Ident, param: &Ident) -> TokenStream { +fn build_fn( + input: ItemFn, + real_fn: &Ident, + param: &Ident, + attributes: &[Attribute], +) -> TokenStream { let ItemFn { attrs, vis, sig, .. } = input; @@ -81,7 +86,10 @@ fn build_fn(input: ItemFn, real_fn: &Ident, param: &Ident) -> TokenStream { let args = args_idents(params.iter()); quote! { - #(#attrs) * + // delegate attributes below the self one + #(#attrs)* + // delegate provided attributes + #(#attributes)* #vis #constness #unsafety #asyncness #abi fn #ident<#gen_params>(#params) #return_type #where_clause { @@ -144,6 +152,7 @@ fn gen_new_def(mut fn_list: TokenStream, input: ItemFn, args: SpecializeArgs) -> }, &real_fn, &ty, + &args.attributes, ); fn_list = quote! { #fn_list diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 9bca096..239d9aa 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -12,17 +12,19 @@ use darling::FromMeta; use syn::punctuated::Punctuated; +use crate::kw::attributes; use syn::{ parse::{Parse, ParseStream}, parse_macro_input, spanned::Spanned, token::Token, - Ident, ItemFn, LitStr, Token, + Attribute, Ident, ItemFn, LitStr, Token, }; mod kw { syn::custom_keyword!(types); syn::custom_keyword!(name); + syn::custom_keyword!(attributes); } #[derive(Clone, Default, Debug)] @@ -30,6 +32,7 @@ struct SpecializeArgs { name: Option, param: Option, aliases: HashMap, + attributes: Vec, /// Errors describing any unrecognized parse inputs that we skipped. parse_warnings: Vec, } @@ -67,6 +70,14 @@ impl Parse for SpecializeArgs { let AliasArg { param, aliases } = input.parse::()?; args.param = Some(param); args.aliases = aliases; + } else if lookahead.peek(kw::attributes) { + if !args.attributes.is_empty() { + return Err(input.error("expected only a single `attributes` argument")); + } + let _ = input.parse::()?; + let content; + let _ = syn::parenthesized!(content in input); + args.attributes = content.call(Attribute::parse_outer)?; } else if lookahead.peek(Token![,]) { let _ = input.parse::()?; } else { diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 483a733..c66971b 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -68,7 +68,8 @@ impl StoreHandle { /// `raw` must be valid ptr to `Box>` /// allocated in `Box` /// without owner - pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { + /// #[no_mangle + pub unsafe extern "C" fn from_raw(raw: *mut c_void) -> StoreHandle { debug_assert!(!raw.is_null()); Self { @@ -78,8 +79,9 @@ impl StoreHandle { } /// # Safety - /// should not live more than what is allowed - pub unsafe fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { + /// should not live + /// #[no_manglemore than what is allowed + pub unsafe extern "C" fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { let leak = Self::from_raw(raw); // SAFETY: Guarantee by caller leak.ptr.cast().as_mut() @@ -185,8 +187,11 @@ fn acquire_result(result: Result>) -> DoubletsResult u64 => u64, ), name = "doublets_create_united_store_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn create_unit_store( +pub unsafe extern "C" fn create_unit_store( path: *const c_char, constants: Constants, ) -> StoreHandle { @@ -209,8 +214,11 @@ pub unsafe fn create_unit_store( u64 => u64, ), name = "doublets_free_store_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn free_store(this: *mut c_void) { +pub unsafe extern "C" fn free_store(this: *mut c_void) { StoreHandle::drop(StoreHandle::::from_raw(this)) } @@ -220,10 +228,13 @@ pub unsafe fn free_store(this: *mut c_void) { u16 => u16, u32 => u32, u64 => u64, + ), + name = "doublets_constants_*", + attributes( + #[no_mangle] ) - name = "doublets_constants_*" )] -pub unsafe fn constants_for_store(this: *mut c_void) -> Constants { +pub unsafe extern "C" fn constants_for_store(this: *mut c_void) -> Constants { StoreHandle::from_raw(this) .assume() .constants() @@ -247,8 +258,11 @@ pub unsafe fn constants_for_store(this: *mut c_void) -> Constants u64, ), name = "doublets_create_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn create( +pub unsafe extern "C" fn create( this: *mut c_void, query: *const T, len: u32, @@ -277,8 +291,11 @@ pub unsafe fn create( u64 => u64, ), name = "doublets_each_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn each( +pub unsafe extern "C" fn each( this: *mut c_void, query: *const T, len: u32, @@ -309,8 +326,11 @@ pub unsafe fn each( u64 => u64, ), name = "doublets_count_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn count(this: *mut c_void, query: *const T, len: u32) -> T { +pub unsafe extern "C" fn count(this: *mut c_void, query: *const T, len: u32) -> T { let mut handle = StoreHandle::::from_raw(this); let query = query_from_raw(query, len); handle.assume().count_by(query) @@ -336,8 +356,11 @@ pub unsafe fn count(this: *mut c_void, query: *const T, len: u32) - u64 => u64, ), name = "doublets_update_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn update( +pub unsafe extern "C" fn update( this: *mut c_void, query: *const T, len_q: u32, @@ -371,8 +394,11 @@ pub unsafe fn update( u64 => u64, ), name = "doublets_delete_*", + attributes( + #[no_mangle] + ) )] -pub unsafe fn delete( +pub unsafe extern "C" fn delete( this: *mut c_void, query: *const T, len: u32, From cc2eee12e5bf64f62b94f040335562e476cd45bf Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 10:30:44 +0300 Subject: [PATCH 20/76] Fix some externs --- doublets-ffi/src/store.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index c66971b..4acfbe7 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -66,10 +66,8 @@ impl StoreHandle { /// /// # Safety /// `raw` must be valid ptr to `Box>` - /// allocated in `Box` - /// without owner - /// #[no_mangle - pub unsafe extern "C" fn from_raw(raw: *mut c_void) -> StoreHandle { + /// allocated in `Box` without owner + pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { debug_assert!(!raw.is_null()); Self { @@ -80,8 +78,8 @@ impl StoreHandle { /// # Safety /// should not live - /// #[no_manglemore than what is allowed - pub unsafe extern "C" fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { + /// more than what is allowed + pub unsafe fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { let leak = Self::from_raw(raw); // SAFETY: Guarantee by caller leak.ptr.cast().as_mut() From 2c10ab3532085e7abb964e2acaa3fb489c9d5d42 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 10:34:59 +0300 Subject: [PATCH 21/76] Add some doc comments to `Range` --- doublets-ffi/src/constants.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doublets-ffi/src/constants.rs b/doublets-ffi/src/constants.rs index a8ba4c3..f624c06 100644 --- a/doublets-ffi/src/constants.rs +++ b/doublets-ffi/src/constants.rs @@ -1,6 +1,9 @@ use doublets::data::{LinkType, LinksConstants}; use std::{mem::MaybeUninit, ops::RangeInclusive}; +/// FFI repr to [`Inclusive Range`] +/// +/// [`Inclusive Range`]: https://doc.rust-lang.org/std/ops/struct.RangeInclusive.html #[derive(Eq, PartialEq)] #[repr(C)] pub struct Range(pub T, pub T); From fdc90c91269ff147c66bf6704ae50df3178a57b6 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 10:35:28 +0300 Subject: [PATCH 22/76] Fix brackets in `error-handling` example --- doublets-ffi/examples/rust/examples/error-handling.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index bd9ae27..373b477 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -15,7 +15,7 @@ use doublets_ffi::{ use std::{ ffi::{c_char, CStr, CString}, fs, - ptr::{null_mut}, + ptr::null_mut, }; unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { From f48486c267ca0ca9b57dbf0f0b64c5855436ff72 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 15:16:55 +0300 Subject: [PATCH 23/76] Use `usize` instead of `c_size_t` --- doublets-ffi/src/errors.rs | 17 +++++++---------- doublets-ffi/src/lib.rs | 1 - 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 27db57e..e658d17 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,5 +1,4 @@ use crate::c_char; -use core::ffi::c_size_t; use doublets::{data::LinkType, Doublet, Error, Link}; use std::{cell::RefCell, cmp, error, ffi::c_short, mem::MaybeUninit, ptr}; @@ -8,7 +7,7 @@ pub enum DoubletsResultKind { // oks Break, Continue, - // errors + // errors NotExists, HasUsages, AlreadyExists, @@ -77,9 +76,9 @@ pub unsafe extern "C" fn doublets_read_error_backtrace( } #[no_mangle] -pub unsafe extern "C" fn doublets_read_error_as_not_found(err: *const Error) -> c_size_t { +pub unsafe extern "C" fn doublets_read_error_as_not_found(err: *const Error) -> usize { match &*err { - Error::NotExists(link) => *link as c_size_t, + Error::NotExists(link) => *link as usize, _ => panic!("error type is not `NotExists`"), } } @@ -87,21 +86,19 @@ pub unsafe extern "C" fn doublets_read_error_as_not_found(err: *const Error, -) -> Doublet { +) -> Doublet { match &*err { Error::AlreadyExists(Doublet { source, target }) => { - Doublet::new(*source as c_size_t, *target as c_size_t) + Doublet::new(*source as usize, *target as usize) } _ => panic!("error type is not `AlreadyExists`"), } } #[no_mangle] -pub unsafe extern "C" fn doublets_read_error_as_limit_reached( - err: *const Error, -) -> c_size_t { +pub unsafe extern "C" fn doublets_read_error_as_limit_reached(err: *const Error) -> usize { match &*err { - Error::LimitReached(limit) => *limit as c_size_t, + Error::LimitReached(limit) => *limit as usize, _ => panic!("error type is not `LimitReached`"), } } diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 5e89f46..3e7ae0f 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -1,7 +1,6 @@ #![feature(try_blocks)] #![feature(box_syntax)] #![feature(try_trait_v2)] -#![feature(c_size_t)] #![feature(thread_local)] extern crate core; From b7e8a2586c37cd434cc29e3a6a92b79779f4b2be Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 16:42:58 +0300 Subject: [PATCH 24/76] Use modern `StoreHandle` without `Box::from_raw/into_raw` --- .../rust/examples/doublets-context.rs | 10 +- .../examples/rust/examples/error-handling.rs | 22 +-- doublets-ffi/src/lib.rs | 4 +- doublets-ffi/src/store.rs | 158 +++++++----------- 4 files changed, 75 insertions(+), 119 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index c222bcf..cc1703a 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -5,7 +5,7 @@ use doublets::{ use doublets_ffi::{ constants::Constants, export::{doublets_create_log_handle, doublets_free_log_handle}, - store::{create, doublets_create_united_store_u64}, + store::{create, doublets_create_united_store_u64, StoreHandle}, FFICallbackContext, }; use std::{ @@ -29,12 +29,12 @@ where } } -unsafe fn magic_create(ptr: *mut c_void, handler: F) +unsafe fn magic_create(handle: &mut StoreHandle, mut handler: F) where F: FnMut(Link, Link), { - let ctx = &mut (ptr, handler); - let _ = create(ptr, null(), 0, ctx as *mut _ as *mut _, create_cb::); + let ctx = &mut handler as *mut _; + let _ = create(handle, null(), 0, ctx as *mut _, create_cb::); } fn main() { @@ -48,7 +48,7 @@ fn main() { Constants::from(LinksConstants::external()), ); - magic_create(store.assume() as *mut _ as *mut _, |before, after| { + magic_create(&mut store, |before, after| { print!("{before:?}\n{after:?}\n"); }); diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 373b477..a52ba17 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -9,7 +9,7 @@ use doublets_ffi::{ DoubletsResultKind, }, export::{doublets_create_log_handle, doublets_free_log_handle}, - store::{constants_for_store, create_unit_store, delete, free_store}, + store::{constants_from_store, create_unit_store, delete, free_store}, FFICallbackContext, }; use std::{ @@ -29,23 +29,17 @@ extern "C" fn create_cb(_: FFICallbackContext, _: Link, _: Link) -> Fl fn main() { let level = CString::new("trace").unwrap(); unsafe { - let handle = doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); + let log_handle = + doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); let path = CString::new("doublets.links").unwrap(); - let mut store = + let mut handle = create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); - // `StoreHandle` is transparent - in really FFI we must use raw ptr - if store.as_ptr().is_null() { - unreachable!("it would be better for errors not to occur in the examples") - } - - let ptr = store.assume() as *mut _ as *mut _; - - let any = constants_for_store::(ptr).any; + let any = constants_from_store::(&handle).any; let query = [1 /* not exists index */, any, any]; - let result = delete::(ptr, query.as_ptr(), 3, null_mut(), create_cb); + let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); if result as u8 >= DoubletsResultKind::NotExists as u8 { let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); @@ -69,9 +63,9 @@ fn main() { // forget `err` ptr - we not in manage it deallocation } - free_store::(ptr); + free_store::(handle); - doublets_free_log_handle(handle); + doublets_free_log_handle(log_handle); } let _ = fs::remove_file("doublets.links"); } diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 3e7ae0f..38c403b 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -3,7 +3,7 @@ #![feature(try_trait_v2)] #![feature(thread_local)] -extern crate core; +use std::marker::{PhantomData, PhantomPinned}; pub mod constants; pub mod errors; @@ -26,3 +26,5 @@ pub struct FFICallbackContextWrapper(FFICallbackContext); unsafe impl Send for FFICallbackContextWrapper {} unsafe impl Sync for FFICallbackContextWrapper {} + +pub(crate) type Marker = PhantomData<(*mut u8, PhantomPinned)>; diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 4acfbe7..480ccd2 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -7,7 +7,7 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{ffi::CStr, marker::PhantomData, ptr::NonNull, slice}; +use std::{ffi::CStr, marker::PhantomData, mem::MaybeUninit, ptr::NonNull, slice}; use tap::Pipe; use tracing::{debug, warn}; @@ -18,98 +18,49 @@ type EachCallback = extern "C" fn(FFICallbackContext, Link) -> Flow; type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> Flow; -#[repr(transparent)] pub struct StoreHandle { - pub(crate) ptr: NonNull, // thin ptr to dyn Doublets - marker: PhantomData, + pointer: MaybeUninit>>, } impl StoreHandle { - pub fn new(store: Box>) -> Self { - let raw = Box::into_raw(Box::new(store)); - // SAFETY: box contains valid ptr to store - unsafe { Self::from_raw(raw.cast()) } + pub fn new(store: Box>) -> Box { + Box::new(Self { + pointer: MaybeUninit::new(store), + }) } - /// # Examples - /// - /// Safe usage: - /// - /// ``` - /// # use std::ffi::c_void; - /// # use doublets_ffi::store::StoreHandle; - /// extern "C" fn create_u64_store() -> *mut c_void { - /// todo!("todo: simple but full example") - /// } - /// - /// // SAFETY: caller must guarantee `from_raw` invariants - /// unsafe extern "C" fn free_u64_store(ptr: *mut c_void) { - /// StoreHandle::drop(StoreHandle::::from_raw(ptr)) - /// } - /// ``` - /// - /// Undefined Behaviour usage: - /// ```no_run - /// # use std::ffi::c_void; - /// # use doublets_ffi::store::StoreHandle; - /// - /// unsafe extern "C" fn should_crush(ptr: *mut c_void) { - /// // two handle for one store is safe - /// let (mut a, mut b) = ( - /// StoreHandle::::from_raw(ptr), - /// StoreHandle::::from_raw(ptr), - /// ); - /// // but it is ub - /// let (a, b) = (a.assume(), b.assume()); - /// } - /// ``` - /// - /// # Safety - /// `raw` must be valid ptr to `Box>` - /// allocated in `Box` without owner - pub unsafe fn from_raw(raw: *mut c_void) -> StoreHandle { - debug_assert!(!raw.is_null()); - - Self { - ptr: NonNull::new_unchecked(raw), - marker: PhantomData, - } - } - - /// # Safety - /// should not live - /// more than what is allowed - pub unsafe fn from_raw_assume<'a>(raw: *mut c_void) -> &'a mut Box> { - let leak = Self::from_raw(raw); - // SAFETY: Guarantee by caller - leak.ptr.cast().as_mut() + pub unsafe fn assume(&mut self) -> &mut Box> { + // SAFETY: `StoreHandle` must be create from safe `new()` + // or unsafe `Self::from_raw` + // then it guarantee by `Self::from_raw()` caller + self.pointer.assume_init_mut() } - pub fn assume(&mut self) -> &mut Box> { + pub unsafe fn assume_ref(&self) -> &Box> { // SAFETY: `StoreHandle` must be create from safe `new()` // or unsafe `Self::from_raw` // then it guarantee by `Self::from_raw()` caller - unsafe { self.ptr.cast().as_mut() } + self.pointer.assume_init_ref() } - pub fn invalid(err: Error) -> Self { + /// This function is actually unsafe + /// + /// # Safety + /// + /// Caller guarantee that will not drop handle + pub fn invalid(err: Error) -> Box { acquire_error(err); // we not have access to self inner - Self { - ptr: NonNull::dangling(), - marker: PhantomData, - } - } - - pub fn as_ptr(&self) -> *const c_void { - self.ptr.as_ptr() + Box::new(Self { + pointer: MaybeUninit::uninit(), + }) } +} - pub fn drop(mut handle: Self) { - // SAFETY: `self.store` is valid `Store` ptr - unsafe { - let _ = Box::from_raw(handle.assume()); - } +impl Drop for StoreHandle { + fn drop(&mut self) { + // Caller guarantee `StoreHandle` is valid + unsafe { self.pointer.assume_init_drop() }; } } @@ -124,7 +75,7 @@ unsafe fn thin_query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Que unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { if query.is_null() && len != 0 { - warn!("query ptr is null, but len is not null: this could be a potential mistake."); + warn!("query ptr is null, but len is not null: handle could be a potential mistake."); } thin_query_from_raw(query, len) @@ -192,7 +143,7 @@ fn acquire_result(result: Result>) -> DoubletsResult pub unsafe extern "C" fn create_unit_store( path: *const c_char, constants: Constants, -) -> StoreHandle { +) -> Box> { let result: Result<_, Error> = try { let path = CStr::from_ptr(path).to_str().unwrap(); let mem = FileMapped::from_path(path)?; @@ -216,8 +167,8 @@ pub unsafe extern "C" fn create_unit_store( #[no_mangle] ) )] -pub unsafe extern "C" fn free_store(this: *mut c_void) { - StoreHandle::drop(StoreHandle::::from_raw(this)) +pub unsafe extern "C" fn free_store(handle: Box>) { + let _ = handle; } #[ffi::specialize_for( @@ -232,9 +183,11 @@ pub unsafe extern "C" fn free_store(this: *mut c_void) { #[no_mangle] ) )] -pub unsafe extern "C" fn constants_for_store(this: *mut c_void) -> Constants { - StoreHandle::from_raw(this) - .assume() +pub unsafe extern "C" fn constants_from_store( + handle: &StoreHandle, +) -> Constants { + handle + .assume_ref() .constants() .clone() // fixme: useless .clone .into() @@ -261,16 +214,18 @@ pub unsafe extern "C" fn constants_for_store(this: *mut c_void) -> ) )] pub unsafe extern "C" fn create( - this: *mut c_void, + handle: &mut StoreHandle, query: *const T, len: u32, ctx: FFICallbackContext, callback: CUDCallback, ) -> DoubletsResultKind { let query = query_from_raw(query, len); - let store = StoreHandle::::from_raw_assume(this); let handler = move |before, after| callback(ctx, before, after); - store.create_by_with(query, handler).pipe(acquire_result) + handle + .assume() + .create_by_with(query, handler) + .pipe(acquire_result) } #[tracing::instrument( @@ -294,16 +249,16 @@ pub unsafe extern "C" fn create( ) )] pub unsafe extern "C" fn each( - this: *mut c_void, + handle: &StoreHandle, query: *const T, len: u32, ctx: FFICallbackContext, callback: EachCallback, ) -> DoubletsResultKind { let query = query_from_raw(query, len); - let store = StoreHandle::::from_raw_assume(this); let handler = move |link| callback(ctx, link); - store + handle + .assume_ref() .each_by(query, handler) .pipe(DoubletsResultKind::branch) } @@ -328,8 +283,11 @@ pub unsafe extern "C" fn each( #[no_mangle] ) )] -pub unsafe extern "C" fn count(this: *mut c_void, query: *const T, len: u32) -> T { - let mut handle = StoreHandle::::from_raw(this); +pub unsafe extern "C" fn count( + handle: &mut StoreHandle, + query: *const T, + len: u32, +) -> T { let query = query_from_raw(query, len); handle.assume().count_by(query) } @@ -359,7 +317,7 @@ pub unsafe extern "C" fn count(this: *mut c_void, query: *const T, ) )] pub unsafe extern "C" fn update( - this: *mut c_void, + handle: &mut StoreHandle, query: *const T, len_q: u32, change: *const T, @@ -367,11 +325,11 @@ pub unsafe extern "C" fn update( ctx: FFICallbackContext, callback: CUDCallback, ) -> DoubletsResultKind { + let handler = move |before, after| callback(ctx, before, after); let query = query_from_raw(query, len_q); let change = query_from_raw(change, len_c); - let store = StoreHandle::::from_raw_assume(this); - let handler = move |before, after| callback(ctx, before, after); - store + handle + .assume() .update_by_with(query, change, handler) .pipe(acquire_result) } @@ -397,14 +355,16 @@ pub unsafe extern "C" fn update( ) )] pub unsafe extern "C" fn delete( - this: *mut c_void, + handle: &mut StoreHandle, query: *const T, len: u32, ctx: FFICallbackContext, callback: CUDCallback, ) -> DoubletsResultKind { - let query = query_from_raw(query, len); - let store = StoreHandle::::from_raw_assume(this); let handler = move |before, after| callback(ctx, before, after); - store.delete_by_with(query, handler).pipe(acquire_result) + let query = query_from_raw(query, len); + handle + .assume() + .delete_by_with(query, handler) + .pipe(acquire_result) } From 7454ec0f9e22a0b70c02f74100cd012d5b818277 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 18:32:31 +0300 Subject: [PATCH 25/76] Add error handling without bottleneck (`ERROR_PLACE`) and `take_last_error` :) --- .../examples/rust/examples/error-handling.rs | 19 +- doublets-ffi/src/errors.rs | 206 +++++++++++------- doublets-ffi/src/lib.rs | 2 + doublets-ffi/src/store.rs | 57 ++--- doublets/src/data/error.rs | 2 +- 5 files changed, 164 insertions(+), 122 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index a52ba17..74166bb 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -4,10 +4,7 @@ use doublets::{ }; use doublets_ffi::{ constants::Constants, - errors::{ - doublets_read_error, doublets_read_error_as_not_found, doublets_read_error_message, - DoubletsResultKind, - }, + errors::{free_error, read_error, DoubletsResult}, export::{doublets_create_log_handle, doublets_free_log_handle}, store::{constants_from_store, create_unit_store, delete, free_store}, FFICallbackContext, @@ -41,14 +38,14 @@ fn main() { let query = [1 /* not exists index */, any, any]; let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); - if result as u8 >= DoubletsResultKind::NotExists as u8 { + if let DoubletsResult::Continue | DoubletsResult::Break = result { + unreachable!() + } else { let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); - // last error - DON'T USE PTR AFTER NEW DOUBLETS OPERATION - let err = doublets_read_error(); let mut msg_buf = vec![0u8; 256]; - doublets_read_error_message(msg_buf.as_mut_ptr().cast(), 256, err); + read_error::(msg_buf.as_mut_ptr().cast(), 256, &result); msg_buf.drain(memchr(&msg_buf) + 1..); @@ -56,11 +53,7 @@ fn main() { let str = cstring.to_str().unwrap(); tracing::error!("{}", str); - // forget `err` ptr - we not in manage it deallocation - let not_exists = doublets_read_error_as_not_found(err); - tracing::error!("duplication: link {} does not exists", not_exists); - - // forget `err` ptr - we not in manage it deallocation + free_error::(result); } free_store::(handle); diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index e658d17..aefe5d8 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,47 +1,115 @@ -use crate::c_char; -use doublets::{data::LinkType, Doublet, Error, Link}; -use std::{cell::RefCell, cmp, error, ffi::c_short, mem::MaybeUninit, ptr}; +use crate::{c_char, Marker}; +use doublets::{data::LinkType, mem, Doublet, Error, Link}; +use std::{ + cell::RefCell, + cmp, error, + ffi::c_short, + fmt, + fmt::{Debug, Display, Formatter}, + mem::MaybeUninit, + ptr, + ptr::NonNull, +}; +use tracing::warn; -#[repr(u8)] -pub enum DoubletsResultKind { +type OpaqueError = Box; + +/// `OpaqueSlice` is a FFI-Safe `Box<[T]>` +#[repr(C)] +pub struct OpaqueSlice { + pub ptr: NonNull, + pub len: usize, +} + +impl OpaqueSlice { + pub fn leak(place: Box<[T]>) -> Self { + let leak = NonNull::from(Box::leak(place)); + OpaqueSlice { + ptr: leak.as_non_null_ptr(), + len: leak.len(), + } + } + + pub fn as_slice(&self) -> &[T] { + let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); + // SAFETY: `Self` is opaque we create Box and we drop it + unsafe { slice.as_ref() } + } +} + +impl Debug for OpaqueSlice { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +impl Drop for OpaqueSlice { + fn drop(&mut self) { + let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); + let _ = unsafe { Box::from_raw(slice.as_ptr()) }; + } +} + +#[repr(C, u8)] +#[derive(Debug)] +pub enum DoubletsResult { // oks Break, Continue, // errors - NotExists, - HasUsages, - AlreadyExists, - LimitReached, - AllocFailed, - Other, + NotExists(T), + LimitReached(T), + HasUsages(OpaqueSlice>), + AlreadyExists(Doublet), + AllocFailed(Box), + Other(Box), } -#[thread_local] -static ERROR_PLACE: RefCell>> = RefCell::new(MaybeUninit::uninit()); - -fn link_cast( - Link { - index, - source, - target, - }: Link, -) -> Link { - Link::new(index.as_usize(), source.as_usize(), target.as_usize()) +impl Display for DoubletsResult { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + DoubletsResult::NotExists(exists) => { + write!(f, "link {exists} does not exist.") + } + DoubletsResult::LimitReached(limit) => { + write!( + f, + "limit for the number of links in the storage has been reached: {limit}" + ) + } + DoubletsResult::HasUsages(usages) => { + write!(f, "link {usages:?} has dependencies") + } + DoubletsResult::AlreadyExists(exists) => { + write!(f, "link {exists} already exists") + } + DoubletsResult::AllocFailed(alloc) => { + write!(f, "unable to allocate memory for links storage: `{alloc}`") + } + DoubletsResult::Other(other) => { + write!(f, "other internal error: `{other}`") + } + other @ _ => Debug::fmt(other, f), + } + } } -pub(crate) fn place_error(error: Error) { - use doublets::Error::*; +use ffi_attributes as ffi; - ERROR_PLACE.borrow_mut().write(match error { - NotExists(link) => NotExists(link.as_usize()), - HasUsages(usages) => HasUsages(usages.into_iter().map(link_cast).collect()), - AlreadyExists(Doublet { source, target }) => { - AlreadyExists(Doublet::new(source.as_usize(), target.as_usize())) - } - LimitReached(limit) => LimitReached(limit.as_usize()), - AllocFailed(alloc) => AllocFailed(alloc), - Other(other) => Other(other), - }); +#[ffi::specialize_for( + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_free_error_*", + attributes( + #[no_mangle] + ) +)] +pub extern "C" fn free_error(err: DoubletsResult) { + let _ = err; } unsafe fn write_raw_msg(buf: *mut c_char, size: c_short, msg: &str) { @@ -50,55 +118,33 @@ unsafe fn write_raw_msg(buf: *mut c_char, size: c_short, msg: &str) { ptr::write(buf.add(cap), 0); } -#[no_mangle] -pub extern "C" fn doublets_read_error() -> *const Error { - ERROR_PLACE.borrow().as_ptr() -} - -#[no_mangle] -pub unsafe extern "C" fn doublets_read_error_message( +#[ffi::specialize_for( + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_read_error_message_*", + attributes( + #[no_mangle] + ) +)] +pub unsafe extern "C" fn read_error( buf: *mut c_char, size: c_short, - err: *const Error, + error: &DoubletsResult, ) { - let error_msg = format!("{}", &*err); - write_raw_msg(buf, size, &error_msg); -} - -#[no_mangle] -pub unsafe extern "C" fn doublets_read_error_backtrace( - buf: *mut c_char, - size: c_short, - err: *const Error, -) { - let error_msg = format!("{:?}", error::Error::source(&*err)); - write_raw_msg(buf, size, &error_msg); -} - -#[no_mangle] -pub unsafe extern "C" fn doublets_read_error_as_not_found(err: *const Error) -> usize { - match &*err { - Error::NotExists(link) => *link as usize, - _ => panic!("error type is not `NotExists`"), - } -} - -#[no_mangle] -pub unsafe extern "C" fn doublets_read_error_as_already_exists( - err: *const Error, -) -> Doublet { - match &*err { - Error::AlreadyExists(Doublet { source, target }) => { - Doublet::new(*source as usize, *target as usize) + match error { + /* invalid @ */ + DoubletsResult::Break | DoubletsResult::Continue => { + warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); + } + valid => { + let msg = valid.to_string(); + let cap = cmp::min(size as usize, msg.len()) - 1; + ptr::copy_nonoverlapping(msg.as_ptr(), buf.cast(), cap); + ptr::write(buf.add(cap), 0); } - _ => panic!("error type is not `AlreadyExists`"), - } -} - -#[no_mangle] -pub unsafe extern "C" fn doublets_read_error_as_limit_reached(err: *const Error) -> usize { - match &*err { - Error::LimitReached(limit) => *limit as usize, - _ => panic!("error type is not `LimitReached`"), } } diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 38c403b..9b9a5ea 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -2,6 +2,8 @@ #![feature(box_syntax)] #![feature(try_trait_v2)] #![feature(thread_local)] +#![feature(nonnull_slice_from_raw_parts)] +#![feature(slice_ptr_get)] use std::marker::{PhantomData, PhantomPinned}; diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 480ccd2..f8d2c78 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,6 +1,11 @@ #![allow(clippy::missing_safety_doc)] -use crate::{c_char, c_void, constants::Constants, errors::DoubletsResultKind, FFICallbackContext}; +use crate::{ + c_char, c_void, + constants::Constants, + errors::{DoubletsResult, OpaqueSlice}, + FFICallbackContext, +}; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, mem::FileMapped, @@ -81,42 +86,38 @@ unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a thin_query_from_raw(query, len) } -fn place_error(err: Error) { - // It can be very expensive to handle each error - debug!(op_error = % err); - super::errors::place_error(err); -} - -impl DoubletsResultKind { +impl DoubletsResult { pub fn branch(flow: Flow) -> Self { if let Flow::Continue = flow { - DoubletsResultKind::Continue + DoubletsResult::Continue } else { - DoubletsResultKind::Break + DoubletsResult::Break } } - pub fn leak(err: &Error) -> Self { + pub fn from_err(err: Error) -> Self { match err { - Error::NotExists(_) => DoubletsResultKind::NotExists, - Error::HasUsages(_) => DoubletsResultKind::HasUsages, - Error::AlreadyExists(_) => DoubletsResultKind::AlreadyExists, - Error::LimitReached(_) => DoubletsResultKind::LimitReached, - Error::AllocFailed(_) => DoubletsResultKind::AllocFailed, - Error::Other(_) => DoubletsResultKind::Other, + Error::NotExists(link) => Self::NotExists(link), + Error::HasUsages(usages) => Self::HasUsages(OpaqueSlice::leak(usages)), + Error::AlreadyExists(exists) => Self::AlreadyExists(exists), + Error::LimitReached(limit) => Self::LimitReached(limit), + // these errors are difficult to handle as data + // I hope no one will be offended if we alloc them at the heap + Error::AllocFailed(alloc) => Self::AllocFailed(Box::new(alloc)), + Error::Other(other) => Self::Other(Box::new(other)), } } } -fn acquire_error(err: Error) -> DoubletsResultKind { - let ret = DoubletsResultKind::leak(&err); - place_error(err); - ret +fn acquire_error(err: Error) -> DoubletsResult { + // It can be very expensive to handle each error + debug!(op_error = % err); + DoubletsResult::from_err(err) } -fn acquire_result(result: Result>) -> DoubletsResultKind { +fn acquire_result(result: Result>) -> DoubletsResult { match result { - Ok(flow) => DoubletsResultKind::branch(flow), + Ok(flow) => DoubletsResult::branch(flow), Err(err) => acquire_error(err), } } @@ -219,7 +220,7 @@ pub unsafe extern "C" fn create( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResultKind { +) -> DoubletsResult { let query = query_from_raw(query, len); let handler = move |before, after| callback(ctx, before, after); handle @@ -254,13 +255,13 @@ pub unsafe extern "C" fn each( len: u32, ctx: FFICallbackContext, callback: EachCallback, -) -> DoubletsResultKind { +) -> DoubletsResult { let query = query_from_raw(query, len); let handler = move |link| callback(ctx, link); handle .assume_ref() .each_by(query, handler) - .pipe(DoubletsResultKind::branch) + .pipe(DoubletsResult::branch) } #[tracing::instrument( @@ -324,7 +325,7 @@ pub unsafe extern "C" fn update( len_c: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResultKind { +) -> DoubletsResult { let handler = move |before, after| callback(ctx, before, after); let query = query_from_raw(query, len_q); let change = query_from_raw(change, len_c); @@ -360,7 +361,7 @@ pub unsafe extern "C" fn delete( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResultKind { +) -> DoubletsResult { let handler = move |before, after| callback(ctx, before, after); let query = query_from_raw(query, len); handle diff --git a/doublets/src/data/error.rs b/doublets/src/data/error.rs index 5cb0659..37b0195 100644 --- a/doublets/src/data/error.rs +++ b/doublets/src/data/error.rs @@ -8,7 +8,7 @@ pub enum Error { NotExists(T), #[error("link {0:?} has dependencies")] - HasUsages(Vec>), + HasUsages(Box<[Link]>), #[error("link {0} already exists")] AlreadyExists(Doublet), From d54db5dc21263c2dc9d4e92627b2bd7a40d31ac4 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 24 Aug 2022 22:54:44 +0300 Subject: [PATCH 26/76] Improve `OpaqueSlice` in favor of `OwnedSlice` --- doublets-ffi/src/errors.rs | 34 +++++++++++++++++++++++----------- doublets-ffi/src/store.rs | 4 ++-- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index aefe5d8..e67df94 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -14,17 +14,17 @@ use tracing::warn; type OpaqueError = Box; -/// `OpaqueSlice` is a FFI-Safe `Box<[T]>` +/// `OwnedSlice` is a FFI-Safe `Box<[T]>` representation #[repr(C)] -pub struct OpaqueSlice { - pub ptr: NonNull, - pub len: usize, +pub struct OwnedSlice { + ptr: NonNull, + len: usize, } -impl OpaqueSlice { +impl OwnedSlice { pub fn leak(place: Box<[T]>) -> Self { let leak = NonNull::from(Box::leak(place)); - OpaqueSlice { + OwnedSlice { ptr: leak.as_non_null_ptr(), len: leak.len(), } @@ -35,18 +35,30 @@ impl OpaqueSlice { // SAFETY: `Self` is opaque we create Box and we drop it unsafe { slice.as_ref() } } + + /// # Safety + /// forget `self` after `.keep_own` + pub unsafe fn keep_own(&self) -> Box<[T]> { + let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); + unsafe { Box::from_raw(slice.as_ptr()) } + } + + pub fn into_owned(self) -> Box<[T]> { + // SAFETY: `self` drop after call `.into_owned` + unsafe { self.keep_own() } + } } -impl Debug for OpaqueSlice { +impl Debug for OwnedSlice { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self.as_slice()) } } -impl Drop for OpaqueSlice { +impl Drop for OwnedSlice { fn drop(&mut self) { - let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); - let _ = unsafe { Box::from_raw(slice.as_ptr()) }; + // SAFETY: `self` drop at end of this scope + let _ = unsafe { self.keep_own() }; } } @@ -59,7 +71,7 @@ pub enum DoubletsResult { // errors NotExists(T), LimitReached(T), - HasUsages(OpaqueSlice>), + HasUsages(OwnedSlice>), AlreadyExists(Doublet), AllocFailed(Box), Other(Box), diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index f8d2c78..7a52f77 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -3,7 +3,7 @@ use crate::{ c_char, c_void, constants::Constants, - errors::{DoubletsResult, OpaqueSlice}, + errors::{DoubletsResult, OwnedSlice}, FFICallbackContext, }; use doublets::{ @@ -98,7 +98,7 @@ impl DoubletsResult { pub fn from_err(err: Error) -> Self { match err { Error::NotExists(link) => Self::NotExists(link), - Error::HasUsages(usages) => Self::HasUsages(OpaqueSlice::leak(usages)), + Error::HasUsages(usages) => Self::HasUsages(OwnedSlice::leak(usages)), Error::AlreadyExists(exists) => Self::AlreadyExists(exists), Error::LimitReached(limit) => Self::LimitReached(limit), // these errors are difficult to handle as data From fba6e78c32c198b894de0b2a9a80b9d146e326dd Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Thu, 25 Aug 2022 22:05:24 +0300 Subject: [PATCH 27/76] Fix some typos in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b1582e..346cf52 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ later A basic operations in doublets: ```rust -use doublets::{data, mem, unit, Doublets, Links}; +use doublets::{data, mem, unit, Doublets, DoubletsExt, Links}; fn main() -> Result<(), doublets::Error> { // use file as memory for doublets @@ -19,7 +19,7 @@ fn main() -> Result<(), doublets::Error> { let mut store = unit::Store::::new(mem)?; // create 1: 1 1 - it's point: link where source and target it self - let mut point = store.create_link(1, 1)?; + let point = store.create_link(1, 1)?; // `any` constant denotes any link let any = store.constants().any; From d5025c01c27cb978cca45824433cd7a723512b71 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 16:25:31 +0300 Subject: [PATCH 28/76] Revert changes in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 346cf52..6b1582e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ later A basic operations in doublets: ```rust -use doublets::{data, mem, unit, Doublets, DoubletsExt, Links}; +use doublets::{data, mem, unit, Doublets, Links}; fn main() -> Result<(), doublets::Error> { // use file as memory for doublets @@ -19,7 +19,7 @@ fn main() -> Result<(), doublets::Error> { let mut store = unit::Store::::new(mem)?; // create 1: 1 1 - it's point: link where source and target it self - let point = store.create_link(1, 1)?; + let mut point = store.create_link(1, 1)?; // `any` constant denotes any link let any = store.constants().any; From 089029309120e59188be1da9e67160ec5ce07c75 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 16:26:38 +0300 Subject: [PATCH 29/76] Fix some typos in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b1582e..346cf52 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ later A basic operations in doublets: ```rust -use doublets::{data, mem, unit, Doublets, Links}; +use doublets::{data, mem, unit, Doublets, DoubletsExt, Links}; fn main() -> Result<(), doublets::Error> { // use file as memory for doublets @@ -19,7 +19,7 @@ fn main() -> Result<(), doublets::Error> { let mut store = unit::Store::::new(mem)?; // create 1: 1 1 - it's point: link where source and target it self - let mut point = store.create_link(1, 1)?; + let point = store.create_link(1, 1)?; // `any` constant denotes any link let any = store.constants().any; From 56ae6bcf03eda721f9f82c0fedd3bcad87d7e1b7 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 22:02:10 +0300 Subject: [PATCH 30/76] Use `Box` instead of raw ptrs --- doublets-ffi/src/export.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index c458f07..eeef69f 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -20,22 +20,22 @@ pub unsafe extern "C" fn doublets_create_log_handle( max_level: *const c_char, use_ansi: bool, use_json: bool, -) -> *mut DoubletsFFILogHandle { - assert!(!max_level.is_null()); +) -> Box { + assert!( + !max_level.is_null(), /* `CStr` immediately uses `strlen` */ + ); // if str isn't utf-8 just panic let max_level_str = CStr::from_ptr(max_level).to_str().unwrap(); - Box::into_raw(Box::new(DoubletsFFILogHandle::new( + Box::new(DoubletsFFILogHandle::new( ctx, callback, max_level_str, use_ansi, use_json, - ))) + )) } #[no_mangle] -pub unsafe extern "C" fn doublets_free_log_handle(ptr: *mut DoubletsFFILogHandle) { - if !ptr.is_null() { - let _ = Box::from_raw(ptr); - } +pub unsafe extern "C" fn doublets_free_log_handle(log_handle: Box) { + let _ = log_handle; } From 7061c37179bcd26bce7fc49b66caff96c2c8cdf6 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 22:15:24 +0300 Subject: [PATCH 31/76] Stop logging if channel is disconnected --- doublets-ffi/src/logging.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 4625830..706356f 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -2,7 +2,7 @@ use super::{c_char, FFICallbackContext}; use crate::FFICallbackContextWrapper; use crossbeam_channel::{self as mpsc, Sender}; use std::{ffi::CString, io, str::FromStr, thread}; -use tracing::{error}; +use tracing::error; use tracing_subscriber::{ filter::{EnvFilter, LevelFilter}, fmt::MakeWriter, @@ -75,6 +75,8 @@ impl DoubletsFFILogHandle { let str = CString::new(msg) .expect("Only UTF-8 format strings are allowed in logging"); callback(wrapper, str.as_ptr()); + } else { + break; } // }); } From 2de822acfe4e83e5700f3d61416c1c46a4b0d419 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 22:17:18 +0300 Subject: [PATCH 32/76] Remove useless `todo` message --- doublets-ffi/src/store.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 7a52f77..72248f9 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -16,7 +16,6 @@ use std::{ffi::CStr, marker::PhantomData, mem::MaybeUninit, ptr::NonNull, slice} use tap::Pipe; use tracing::{debug, warn}; -// TODO: remove ::mem:: in doublets crate type UnitedLinks = unit::Store>>; type EachCallback = extern "C" fn(FFICallbackContext, Link) -> Flow; From 0f4d4fa0a59d1874aefe1916312df8ea20ba2a30 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 26 Aug 2022 22:25:05 +0300 Subject: [PATCH 33/76] Improve repr of `StoreHandle` --- doublets-ffi/src/store.rs | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 72248f9..b17abf3 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -12,7 +12,14 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{ffi::CStr, marker::PhantomData, mem::MaybeUninit, ptr::NonNull, slice}; +use std::{ + ffi::CStr, + marker::PhantomData, + mem, + mem::MaybeUninit, + ptr::{null_mut, NonNull}, + slice, +}; use tap::Pipe; use tracing::{debug, warn}; @@ -23,28 +30,20 @@ type EachCallback = extern "C" fn(FFICallbackContext, Link) -> Flow; type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> Flow; pub struct StoreHandle { - pointer: MaybeUninit>>, + pointer: Box>, } impl StoreHandle { pub fn new(store: Box>) -> Box { - Box::new(Self { - pointer: MaybeUninit::new(store), - }) + Box::new(Self { pointer: store }) } pub unsafe fn assume(&mut self) -> &mut Box> { - // SAFETY: `StoreHandle` must be create from safe `new()` - // or unsafe `Self::from_raw` - // then it guarantee by `Self::from_raw()` caller - self.pointer.assume_init_mut() + &mut self.pointer } pub unsafe fn assume_ref(&self) -> &Box> { - // SAFETY: `StoreHandle` must be create from safe `new()` - // or unsafe `Self::from_raw` - // then it guarantee by `Self::from_raw()` caller - self.pointer.assume_init_ref() + &self.pointer } /// This function is actually unsafe @@ -52,19 +51,12 @@ impl StoreHandle { /// # Safety /// /// Caller guarantee that will not drop handle + // fixme: may be we can port `result::Result` to C pub fn invalid(err: Error) -> Box { acquire_error(err); - // we not have access to self inner - Box::new(Self { - pointer: MaybeUninit::uninit(), - }) - } -} -impl Drop for StoreHandle { - fn drop(&mut self) { - // Caller guarantee `StoreHandle` is valid - unsafe { self.pointer.assume_init_drop() }; + // SAFETY: Box is repr to `*mut T` and must forgot + unsafe { mem::transmute(null_mut::()) } } } From 9c58bb089a90e739840a481ad522a86213622f74 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 28 Aug 2022 11:50:59 +0300 Subject: [PATCH 34/76] Fix macro error message --- doublets-ffi/ffi-attributes/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 239d9aa..5d0939f 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -142,7 +142,7 @@ impl Parse for AliasArg { if map.contains_key(&ty) { return Err(syn::Error::new( ty.span().join(lit.span()).unwrap(), - "tried to skip the same field twice", + "tried to ad lias to same field twice", )); } else { map.insert(ty, lit); From f570601104356dfd678a121181d3041f665319b1 Mon Sep 17 00:00:00 2001 From: Mitron57 Date: Sun, 28 Aug 2022 19:03:44 +0400 Subject: [PATCH 35/76] Change pair from Ident => Ident to Type => Ident --- doublets-ffi/ffi-attributes/src/expand.rs | 13 ++++--------- doublets-ffi/ffi-attributes/src/lib.rs | 10 +++++----- doublets-ffi/ffi-attributes/src/prepare.rs | 14 +++++++------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs index d2a0739..f865890 100644 --- a/doublets-ffi/ffi-attributes/src/expand.rs +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -5,7 +5,7 @@ use quote::{quote, ToTokens}; use syn::{ punctuated::Punctuated, Attribute, FnArg, GenericParam, ItemFn, PatType, ReturnType, Signature, - Token, TypeParam, + Token, Type, TypeParam, }; fn filter_generics<'gen>( @@ -26,7 +26,7 @@ fn filter_generics<'gen>( fn prepare_fn_args<'args>( fn_args: impl Iterator, param: &Ident, - ident: &Ident, + ident: &Type, ) { for arg in fn_args { match arg { @@ -40,7 +40,7 @@ fn prepare_fn_args<'args>( } } -fn prepare_output_type(output: &mut ReturnType, param: &Ident, ident: &Ident) { +fn prepare_output_type(output: &mut ReturnType, param: &Ident, ident: &Type) { if let ReturnType::Type(_, box ty) = output { *ty = prepare::replace_ty_in_param(ty.clone(), param, ident); } @@ -57,12 +57,7 @@ fn args_idents<'args>( }) } -fn build_fn( - input: ItemFn, - real_fn: &Ident, - param: &Ident, - attributes: &[Attribute], -) -> TokenStream { +fn build_fn(input: ItemFn, real_fn: &Ident, param: &Type, attributes: &[Attribute]) -> TokenStream { let ItemFn { attrs, vis, sig, .. } = input; diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 5d0939f..38d605e 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -18,7 +18,7 @@ use syn::{ parse_macro_input, spanned::Spanned, token::Token, - Attribute, Ident, ItemFn, LitStr, Token, + Attribute, Ident, ItemFn, LitStr, Token, Type, }; mod kw { @@ -31,7 +31,7 @@ mod kw { struct SpecializeArgs { name: Option, param: Option, - aliases: HashMap, + aliases: HashMap, attributes: Vec, /// Errors describing any unrecognized parse inputs that we skipped. parse_warnings: Vec, @@ -108,11 +108,11 @@ impl Parse for StrArg { }) } } -struct AliasLine(Ident, Ident); +struct AliasLine(Type, Ident); impl Parse for AliasLine { fn parse(input: ParseStream<'_>) -> syn::Result { - let ty = input.parse::()?; + let ty = input.parse::()?; let _ = input.parse::]>()?; let str = input.parse::()?; Ok(Self(ty, str)) @@ -121,7 +121,7 @@ impl Parse for AliasLine { struct AliasArg { param: Ident, - aliases: HashMap, + aliases: HashMap, } impl Parse for AliasArg { diff --git a/doublets-ffi/ffi-attributes/src/prepare.rs b/doublets-ffi/ffi-attributes/src/prepare.rs index b469aed..74b5e62 100644 --- a/doublets-ffi/ffi-attributes/src/prepare.rs +++ b/doublets-ffi/ffi-attributes/src/prepare.rs @@ -3,11 +3,11 @@ use syn::{ GenericArgument, ParenthesizedGenericArguments, PathArguments, ReturnType, Type, TypePath, }; -pub(crate) fn prepare_path(mut path: TypePath, from: &Ident, to: &Ident) -> TypePath { +pub(crate) fn prepare_path(mut path: TypePath, from: &Ident, to: &Type) -> Type { + if path.path.is_ident(from) { + return to.clone(); + } path.path.segments.iter_mut().for_each(|seg| { - if &seg.ident == from { - seg.ident = to.clone(); - } match &mut seg.arguments { PathArguments::AngleBracketed(angle) => { for arg in &mut angle.args { @@ -32,12 +32,12 @@ pub(crate) fn prepare_path(mut path: TypePath, from: &Ident, to: &Ident) -> Type _ => { /* ignore */ } } }); - path + Type::Path(path) } -pub(crate) fn replace_ty_in_param(ty: Type, from: &Ident, to: &Ident) -> Type { +pub(crate) fn replace_ty_in_param(ty: Type, from: &Ident, to: &Type) -> Type { match ty { - Type::Path(path) => Type::Path(prepare_path(path, from, to)), + Type::Path(path) => prepare_path(path, from, to), Type::Array(mut arr) => { *arr.elem = replace_ty_in_param(*arr.elem, from, to); Type::Array(arr) From a2ba5063cfd2d490cf50e5e269ba7f6f4ebf7ce1 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 28 Aug 2022 20:29:09 +0300 Subject: [PATCH 36/76] Make code more cleaned and self-doc+friendly for parsing --- doublets-ffi/ffi-attributes/Cargo.toml | 8 +- doublets-ffi/ffi-attributes/src/expand.rs | 12 ++- doublets-ffi/ffi-attributes/src/lib.rs | 115 +++++++++++++--------- 3 files changed, 77 insertions(+), 58 deletions(-) diff --git a/doublets-ffi/ffi-attributes/Cargo.toml b/doublets-ffi/ffi-attributes/Cargo.toml index df5ede5..cf7ec7d 100644 --- a/doublets-ffi/ffi-attributes/Cargo.toml +++ b/doublets-ffi/ffi-attributes/Cargo.toml @@ -7,8 +7,6 @@ edition = "2021" proc-macro = true [dependencies] -syn = "1.0.80" -proc-macro2 = "1.0.30" -darling = "0.13.0" -quote = "1.0.10" -serde = { version = "1.0.130", features = ["derive"] } \ No newline at end of file +quote = { version = "1.0.21" } +proc-macro2 = { version = "1.0.43" } +syn = { version = "1.0.99", features = ["full", "extra-traits"] } \ No newline at end of file diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs index f865890..8039bfa 100644 --- a/doublets-ffi/ffi-attributes/src/expand.rs +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -1,6 +1,6 @@ -use crate::{attributes, prepare, FromMeta, SpecializeArgs}; +use crate::{attributes, prepare, AliasLine, SpecializeArgs}; use proc_macro::Diagnostic; -use proc_macro2::{Ident, TokenStream}; +use proc_macro2::{Ident, Span, TokenStream}; use quote::{quote, ToTokens}; use syn::{ @@ -101,7 +101,7 @@ fn gen_new_def(mut fn_list: TokenStream, input: ItemFn, args: SpecializeArgs) -> .unwrap_or_else(|| real_name + "_*"); let name_pat = name_pat.trim_matches('"'); - for (ty, lit) in args.aliases { + for AliasLine { ty, ident: lit, .. } in args.aliases { let ItemFn { attrs, vis, @@ -124,8 +124,10 @@ fn gen_new_def(mut fn_list: TokenStream, input: ItemFn, args: SpecializeArgs) -> prepare_fn_args(params.iter_mut(), param, &ty); prepare_output_type(&mut return_type, param, &ty); - let new_ident: Ident = - Ident::from_string(&name_pat.replace('*', &lit.to_token_stream().to_string())).unwrap(); + let new_ident: Ident = Ident::new( + &name_pat.replace('*', &lit.to_token_stream().to_string()), + lit.span(), + ); let real_fn = sig.ident.clone(); let sig = Signature { diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 38d605e..26ecadb 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -6,18 +6,15 @@ mod expand; mod prepare; use proc_macro::{Level, Span}; -use std::{collections::HashMap, marker::PhantomData}; - -use darling::FromMeta; - -use syn::punctuated::Punctuated; +use std::collections::HashMap; use crate::kw::attributes; use syn::{ parse::{Parse, ParseStream}, parse_macro_input, + punctuated::Punctuated, spanned::Spanned, - token::Token, + token::Paren, Attribute, Ident, ItemFn, LitStr, Token, Type, }; @@ -31,7 +28,7 @@ mod kw { struct SpecializeArgs { name: Option, param: Option, - aliases: HashMap, + aliases: Punctuated, attributes: Vec, /// Errors describing any unrecognized parse inputs that we skipped. parse_warnings: Vec, @@ -61,13 +58,13 @@ impl Parse for SpecializeArgs { if args.name.is_some() { return Err(input.error("expected only a single `name` argument")); } - let name = input.parse::>()?.value; + let name = input.parse::>()?.lit; args.name = Some(name); } else if lookahead.peek(kw::types) { if !args.aliases.is_empty() { return Err(input.error("expected only a single `types` argument")); } - let AliasArg { param, aliases } = input.parse::()?; + let AliasArg { param, aliases, .. } = input.parse::()?; args.param = Some(param); args.aliases = aliases; } else if lookahead.peek(kw::attributes) { @@ -92,67 +89,89 @@ impl Parse for SpecializeArgs { } } +// custom_kw = "literal" +#[derive(Debug, Clone)] +#[allow(dead_code)] struct StrArg { - value: LitStr, - _marker: PhantomData, + kw: T, + // track issue: https://github.com/dtolnay/syn/issues/1209 + eq: syn::token::Eq, + lit: LitStr, } impl Parse for StrArg { fn parse(input: ParseStream<'_>) -> syn::Result { - let _ = input.parse::()?; - let _ = input.parse::()?; - let value = input.parse()?; Ok(Self { - value, - _marker: PhantomData, + kw: input.parse()?, + eq: input.parse()?, + lit: input.parse()?, }) } } -struct AliasLine(Type, Ident); + +// MyType => mu_type_suffix +#[derive(Clone, Debug)] +#[allow(dead_code)] +struct AliasLine { + ty: Type, + to: Token![=>], + ident: Ident, +} impl Parse for AliasLine { fn parse(input: ParseStream<'_>) -> syn::Result { - let ty = input.parse::()?; - let _ = input.parse::]>()?; - let str = input.parse::()?; - Ok(Self(ty, str)) + Ok(Self { + ty: input.parse()?, + to: input.parse()?, + ident: input.parse()?, + }) } } +// types::( +// u32 => integral, +// f32 => floating, +// (u32, Option) => magic, +// ) +#[allow(dead_code)] struct AliasArg { + kw: kw::types, + colon: Token![::], + lt_token: Token![<], param: Ident, - aliases: HashMap, + gt_toke: Token![>], + paren_token: Paren, + aliases: Punctuated, +} + +fn alias_validation(aliases: &Punctuated) -> Result<(), syn::Error> { + let mut map = HashMap::new(); + aliases.iter().try_for_each(|AliasLine { ty, ident, .. }| { + if let Some(twice) = map.insert(ty.clone(), ident.clone()) { + Err(syn::Error::new( + ty.span().join(ident.span()).unwrap(), + format!("tried to add alias to `{twice}` twice"), + )) + } else { + Ok(()) + } + }) } impl Parse for AliasArg { fn parse(input: ParseStream<'_>) -> syn::Result { - let _ = input.parse::()?; - let _ = input.parse::()?; - let _ = input.parse::()?; - let param = input.parse::()?; - let _ = input.parse::]>()?; let content; - let _ = syn::parenthesized!(content in input); - let aliases: Punctuated = - content.parse_terminated(AliasLine::parse)?; - - let mut map = HashMap::new(); - for AliasLine(ty, lit) in aliases { - #[allow(clippy::map_entry)] - if map.contains_key(&ty) { - return Err(syn::Error::new( - ty.span().join(lit.span()).unwrap(), - "tried to ad lias to same field twice", - )); - } else { - map.insert(ty, lit); - } - } - - Ok(Self { - param, - aliases: map, - }) + let new = Self { + kw: input.parse()?, + colon: input.parse()?, + lt_token: input.parse()?, + param: input.parse()?, + gt_toke: input.parse()?, + paren_token: syn::parenthesized!(content in input), + aliases: content.parse_terminated(AliasLine::parse)?, + }; + + alias_validation(&new.aliases).map(|_| new) } } From 4752904cee5af70943d7b5f12a33e5bbbd2f3baa Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 30 Aug 2022 19:42:28 +0300 Subject: [PATCH 37/76] Some improvements: - `DoubletsFFILogHandle` use `dyn Subscriber` with decl macro instead of too `if`s - replace `repr(u8)` to `repr(usize)` in `enum`s - use `enum`s to pass `Format` and `Level` instead of `bool` and `str` --- .../examples/rust/examples/all-logs.rs | 20 ++++- .../rust/examples/doublets-context.rs | 5 +- .../examples/rust/examples/error-handling.rs | 4 +- .../examples/rust/examples/log-context.rs | 4 +- doublets-ffi/src/errors.rs | 2 +- doublets-ffi/src/export.rs | 20 ++--- doublets-ffi/src/logging.rs | 89 +++++++++++++------ 7 files changed, 93 insertions(+), 51 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/all-logs.rs b/doublets-ffi/examples/rust/examples/all-logs.rs index 19e0c42..e44111d 100644 --- a/doublets-ffi/examples/rust/examples/all-logs.rs +++ b/doublets-ffi/examples/rust/examples/all-logs.rs @@ -1,5 +1,8 @@ +#![feature(const_option_ext)] + use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, + logging::{Format, Level}, FFICallbackContext, }; use std::{ @@ -12,13 +15,24 @@ unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { print!("{}", cstr.to_str().unwrap()); } +const FORMAT: &str = option_env!("RUST_EXAMPLES_FORMAT").unwrap_or("virgin"); + fn main() { let ctx = ptr::null_mut(); - let level = CString::new("trace").unwrap(); + let level = Level::Trace; let use_ansi = true; - let use_json = false; + + let format = match &FORMAT.to_ascii_lowercase()[..] { + "virgin" => Format::Virgin, + "pretty" => Format::Pretty, + "json" => Format::Json, + _ => { + panic!("allow only: `virgin`, `pretty`, `json`") + } + }; + unsafe { - let handle = doublets_create_log_handle(ctx, callback, level.as_ptr(), use_ansi, use_json); + let handle = doublets_create_log_handle(ctx, callback, level, format, use_ansi); tracing::error!("SOMETHING IS SERIOUSLY WRONG!!!"); tracing::warn!("important informational messages; might indicate an error"); diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index cc1703a..07d6149 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -5,6 +5,7 @@ use doublets::{ use doublets_ffi::{ constants::Constants, export::{doublets_create_log_handle, doublets_free_log_handle}, + logging::{Format, Level}, store::{create, doublets_create_united_store_u64, StoreHandle}, FFICallbackContext, }; @@ -38,9 +39,9 @@ where } fn main() { - let level = CString::new("trace").unwrap(); unsafe { - let handle = doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); + let handle = + doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, false); let path = CString::new("doublets.links").unwrap(); let mut store = doublets_create_united_store_u64( diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 74166bb..f35e4ed 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -6,6 +6,7 @@ use doublets_ffi::{ constants::Constants, errors::{free_error, read_error, DoubletsResult}, export::{doublets_create_log_handle, doublets_free_log_handle}, + logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, FFICallbackContext, }; @@ -24,10 +25,9 @@ extern "C" fn create_cb(_: FFICallbackContext, _: Link, _: Link) -> Fl } fn main() { - let level = CString::new("trace").unwrap(); unsafe { let log_handle = - doublets_create_log_handle(null_mut(), callback, level.as_ptr(), true, false); + doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, false); let path = CString::new("doublets.links").unwrap(); let mut handle = diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index a9915a3..fa91181 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -1,5 +1,6 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, + logging::{Format, Level}, FFICallbackContext, }; use std::ffi::{c_char, CStr, CString}; @@ -20,7 +21,8 @@ fn main() { let ctx = &mut 0usize as *mut usize; let level = CString::new("trace").unwrap(); unsafe { - let handle = doublets_create_log_handle(ctx.cast(), callback, level.as_ptr(), true, false); + let handle = + doublets_create_log_handle(ctx.cast(), callback, Level::Trace, Format::Virgin, false); tracing::error!("SOMETHING IS SERIOUSLY WRONG!!!"); tracing::warn!("important informational messages; might indicate an error"); diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index e67df94..2a77376 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -62,7 +62,7 @@ impl Drop for OwnedSlice { } } -#[repr(C, u8)] +#[repr(C, usize)] #[derive(Debug)] pub enum DoubletsResult { // oks diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index eeef69f..81c70de 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -1,11 +1,12 @@ use crate::{ c_char, - logging::{DoubletsFFILogHandle, LogFFICallback}, + logging::{DoubletsFFILogHandle, Format, Level, LogFFICallback}, FFICallbackContext, }; use std::ffi::CStr; use tracing::error; +/// Basic logger. For advanced use [`doublets_create_log_handle`] #[no_mangle] pub extern "C" fn doublets_activate_env_logger() { if tracing_subscriber::fmt::try_init().is_err() { @@ -17,21 +18,12 @@ pub extern "C" fn doublets_activate_env_logger() { pub unsafe extern "C" fn doublets_create_log_handle( ctx: FFICallbackContext, callback: LogFFICallback, - max_level: *const c_char, - use_ansi: bool, - use_json: bool, + max_level: Level, + format: Format, + ansi: bool, ) -> Box { - assert!( - !max_level.is_null(), /* `CStr` immediately uses `strlen` */ - ); - // if str isn't utf-8 just panic - let max_level_str = CStr::from_ptr(max_level).to_str().unwrap(); Box::new(DoubletsFFILogHandle::new( - ctx, - callback, - max_level_str, - use_ansi, - use_json, + ctx, callback, max_level, format, ansi, )) } diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 706356f..44ac722 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -2,10 +2,12 @@ use super::{c_char, FFICallbackContext}; use crate::FFICallbackContextWrapper; use crossbeam_channel::{self as mpsc, Sender}; use std::{ffi::CString, io, str::FromStr, thread}; -use tracing::error; +use tap::Pipe; +use tracing::{dispatcher, error, subscriber, Dispatch, Subscriber}; use tracing_subscriber::{ filter::{EnvFilter, LevelFilter}, - fmt::MakeWriter, + fmt::{format, FormatFields, MakeWriter, SubscriberBuilder}, + util::SubscriberInitExt, }; struct ChannelWriter { @@ -44,15 +46,34 @@ impl MakeWriter<'_> for ChannelWriter { /// This callback is safe if all the rules of Rust are followed pub type LogFFICallback = unsafe extern "C" fn(FFICallbackContext, *const c_char); -pub struct DoubletsFFILogHandle {} +#[repr(usize)] +pub enum Level { + Trace = 0, + Debug = 1, + Info = 2, + Warn = 3, + Error = 4, + // FFI binding can contain + // Off = 5 + // But `tracing` must ignore it +} + +#[repr(usize)] +pub enum Format { + Virgin, + Pretty, + Json, +} + +pub struct DoubletsFFILogHandle {/* opaque */} impl DoubletsFFILogHandle { pub fn new( ctx: FFICallbackContext, callback: LogFFICallback, - max_level: &str, - use_ansi: bool, - use_json: bool, + max_level: Level, + format: Format, + ansi: bool, ) -> Self { log_panics::init(); let wrapper = FFICallbackContextWrapper(ctx); @@ -82,29 +103,41 @@ impl DoubletsFFILogHandle { } }); - let filter = EnvFilter::from_default_env() - .add_directive(LevelFilter::from_str(max_level).unwrap().into()); - if use_json { - if tracing_subscriber::fmt() - .json() - .with_ansi(use_ansi) - .with_writer(ChannelWriter::new(sender)) - .with_env_filter(filter) - .with_filter_reloading() - .try_init() - .is_err() - { - error!("Log handler already set, cannot currently change log levels."); - } - } else if tracing_subscriber::fmt() - .with_ansi(use_ansi) - .with_writer(ChannelWriter::new(sender)) - .with_env_filter(filter) - .with_filter_reloading() - .try_init() - .is_err() + let filter = EnvFilter::from_default_env().add_directive( + LevelFilter::from_level(match max_level { + Level::Trace => tracing::Level::TRACE, + Level::Debug => tracing::Level::DEBUG, + Level::Info => tracing::Level::INFO, + Level::Warn => tracing::Level::WARN, + Level::Error => tracing::Level::ERROR, + }) + .into(), + ); + + macro_rules! subscribe { + ($($methods:tt)*) => { + tracing_subscriber::fmt() + $($methods)* + .with_ansi(ansi) + .with_writer(ChannelWriter::new(sender)) + .with_env_filter(filter) + .with_filter_reloading() + .finish() + }; + } + + if match format { + Format::Virgin => Box::new(subscribe!()) as Box, + Format::Pretty => Box::new(subscribe! { .pretty() }), + Format::Json => Box::new(subscribe! { .json() }), + } + .pipe(subscriber::set_global_default) + .is_err() { - error!("Log handler already set, cannot currently change log levels."); + error!( + "Log handler already set, cannot currently change: track issue \ + `https://github.com/linksplatform/doublets-rs/issues/12`" + ); }; Self {} From 18ed088de58f80fde71953e9b90488fddfa6078b Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 17:37:46 +0300 Subject: [PATCH 38/76] Fix useless `.clone` in `ffi-attributes` --- doublets-ffi/ffi-attributes/src/expand.rs | 6 +++--- doublets-ffi/ffi-attributes/src/lib.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doublets-ffi/ffi-attributes/src/expand.rs b/doublets-ffi/ffi-attributes/src/expand.rs index 8039bfa..b6077d6 100644 --- a/doublets-ffi/ffi-attributes/src/expand.rs +++ b/doublets-ffi/ffi-attributes/src/expand.rs @@ -1,6 +1,6 @@ -use crate::{attributes, prepare, AliasLine, SpecializeArgs}; +use crate::{prepare, AliasLine, SpecializeArgs}; use proc_macro::Diagnostic; -use proc_macro2::{Ident, Span, TokenStream}; +use proc_macro2::{Ident, TokenStream}; use quote::{quote, ToTokens}; use syn::{ @@ -185,7 +185,7 @@ pub(crate) fn gen_function(input: ItemFn, args: SpecializeArgs) -> TokenStream { .. }, .. - } = sig.clone(); + } = sig; let this = quote! { #(#attrs) * diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 26ecadb..8e047d2 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -8,7 +8,7 @@ mod prepare; use proc_macro::{Level, Span}; use std::collections::HashMap; -use crate::kw::attributes; + use syn::{ parse::{Parse, ParseStream}, parse_macro_input, @@ -189,5 +189,5 @@ pub fn specialize_for( item: proc_macro::TokenStream, ) -> proc_macro::TokenStream { let args = parse_macro_input!(args as SpecializeArgs); - specialize_precise(args, item.clone()).unwrap_or_else(|_err| todo!()) + specialize_precise(args, item).unwrap_or_else(|_err| todo!()) } From 6a7a90fedfe2bc4f607c72c841a7346d08027cb2 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 17:38:34 +0300 Subject: [PATCH 39/76] Fix imports and useless in examples --- doublets-ffi/examples/rust/examples/all-logs.rs | 2 +- doublets-ffi/examples/rust/examples/doublets-context.rs | 2 +- doublets-ffi/examples/rust/examples/log-context.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/all-logs.rs b/doublets-ffi/examples/rust/examples/all-logs.rs index e44111d..213a020 100644 --- a/doublets-ffi/examples/rust/examples/all-logs.rs +++ b/doublets-ffi/examples/rust/examples/all-logs.rs @@ -6,7 +6,7 @@ use doublets_ffi::{ FFICallbackContext, }; use std::{ - ffi::{c_char, CStr, CString}, + ffi::{c_char, CStr}, ptr, }; diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index 07d6149..19dab53 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -10,7 +10,7 @@ use doublets_ffi::{ FFICallbackContext, }; use std::{ - ffi::{c_char, c_void, CStr, CString}, + ffi::{c_char, CStr, CString}, fs, ptr::{null, null_mut}, }; diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index fa91181..d2886ae 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -19,7 +19,6 @@ unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { fn main() { let ctx = &mut 0usize as *mut usize; - let level = CString::new("trace").unwrap(); unsafe { let handle = doublets_create_log_handle(ctx.cast(), callback, Level::Trace, Format::Virgin, false); From a57cec3da3062a94d8c97083a110f180f40316ee Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 17:41:50 +0300 Subject: [PATCH 40/76] Add `backtrace` feature and make `doublets-ffi` features-free: - use rustc feature `error_generic_member_access` - remove useless features - add `read_backtrace` ffi function - some improvements)) --- doublets-ffi/src/errors.rs | 82 +++++++++++++++++++++++++++++-------- doublets-ffi/src/export.rs | 3 +- doublets-ffi/src/lib.rs | 15 +++---- doublets-ffi/src/logging.rs | 15 ++++--- doublets-ffi/src/store.rs | 22 ++++------ doublets-ffi/src/utils.rs | 11 +++++ 6 files changed, 95 insertions(+), 53 deletions(-) create mode 100644 doublets-ffi/src/utils.rs diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 2a77376..c6e16b9 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,12 +1,12 @@ -use crate::{c_char, Marker}; -use doublets::{data::LinkType, mem, Doublet, Error, Link}; +use crate::c_char; +use doublets::{data::LinkType, mem, Doublet, Link}; +#[cfg(feature = "backtrace")] +use std::backtrace::Backtrace; use std::{ - cell::RefCell, cmp, error, ffi::c_short, fmt, fmt::{Debug, Display, Formatter}, - mem::MaybeUninit, ptr, ptr::NonNull, }; @@ -22,16 +22,23 @@ pub struct OwnedSlice { } impl OwnedSlice { + #[inline] + pub fn slice_from_raw_parts(data: NonNull, len: usize) -> NonNull<[T]> { + // SAFETY: `data` is a `NonNull` pointer which is necessarily non-null + unsafe { NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(data.as_ptr(), len)) } + } + pub fn leak(place: Box<[T]>) -> Self { let leak = NonNull::from(Box::leak(place)); OwnedSlice { - ptr: leak.as_non_null_ptr(), + // ptr: leak.as_non_null_ptr(), + ptr: unsafe { NonNull::new_unchecked(leak.as_ptr() as *mut _) }, len: leak.len(), } } pub fn as_slice(&self) -> &[T] { - let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); + let slice = Self::slice_from_raw_parts(self.ptr, self.len); // SAFETY: `Self` is opaque we create Box and we drop it unsafe { slice.as_ref() } } @@ -39,7 +46,7 @@ impl OwnedSlice { /// # Safety /// forget `self` after `.keep_own` pub unsafe fn keep_own(&self) -> Box<[T]> { - let slice = NonNull::slice_from_raw_parts(self.ptr, self.len); + let slice = Self::slice_from_raw_parts(self.ptr, self.len); unsafe { Box::from_raw(slice.as_ptr()) } } @@ -77,6 +84,25 @@ pub enum DoubletsResult { Other(Box), } +#[rustfmt::skip] +impl DoubletsResult { + #[cfg(feature = "backtrace")] + fn backtrace(&self) -> Option<&Backtrace> { + match self { + DoubletsResult::AllocFailed(err) => { + (&**err as &dyn error::Error).request_ref::() + } + DoubletsResult::Other(err) => { + (&***err as &dyn error::Error).request_ref::() + } + DoubletsResult::Break | DoubletsResult::Continue => { + panic!("`backtrace` not allowed for ok results") + } + _ => None, + } + } +} + impl Display for DoubletsResult { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { @@ -101,7 +127,7 @@ impl Display for DoubletsResult { DoubletsResult::Other(other) => { write!(f, "other internal error: `{other}`") } - other @ _ => Debug::fmt(other, f), + other => Debug::fmt(other, f), } } } @@ -147,16 +173,36 @@ pub unsafe extern "C" fn read_error( size: c_short, error: &DoubletsResult, ) { - match error { - /* invalid @ */ - DoubletsResult::Break | DoubletsResult::Continue => { - warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); - } - valid => { - let msg = valid.to_string(); - let cap = cmp::min(size as usize, msg.len()) - 1; - ptr::copy_nonoverlapping(msg.as_ptr(), buf.cast(), cap); - ptr::write(buf.add(cap), 0); + if let DoubletsResult::Break | DoubletsResult::Continue = error { + warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); + } else { + write_raw_msg(buf, size, &error.to_string()); + } +} + +#[cfg(feature = "backtrace")] +#[ffi::specialize_for( + types::( + u8 => u8, + u16 => u16, + u32 => u32, + u64 => u64, + ), + name = "doublets_read_backtrace_*", + attributes( + #[no_mangle] + ) +)] +pub unsafe extern "C" fn read_backtrace( + buf: *mut c_char, + size: c_short, + error: &DoubletsResult, +) { + if let DoubletsResult::Break | DoubletsResult::Continue = error { + warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); + } else { + if let Some(backtrace) = error.backtrace() { + write_raw_msg(buf, size, &backtrace.to_string()); } } } diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index 81c70de..f769989 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -1,9 +1,8 @@ use crate::{ - c_char, logging::{DoubletsFFILogHandle, Format, Level, LogFFICallback}, FFICallbackContext, }; -use std::ffi::CStr; + use tracing::error; /// Basic logger. For advanced use [`doublets_create_log_handle`] diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 9b9a5ea..334a57d 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -1,17 +1,13 @@ -#![feature(try_blocks)] -#![feature(box_syntax)] -#![feature(try_trait_v2)] -#![feature(thread_local)] -#![feature(nonnull_slice_from_raw_parts)] -#![feature(slice_ptr_get)] - -use std::marker::{PhantomData, PhantomPinned}; +#![cfg_attr(feature = "backtrace", feature(error_generic_member_access))] pub mod constants; pub mod errors; pub mod export; pub mod logging; pub mod store; +mod utils; + +pub(crate) use utils::stable_try; // It is not useless: CLion highlight // `c_char` as alias - italic @@ -26,7 +22,6 @@ pub type FFICallbackContext = *mut c_void; #[derive(Clone, Copy)] pub struct FFICallbackContextWrapper(FFICallbackContext); +/// Guarantee by caller side unsafe impl Send for FFICallbackContextWrapper {} unsafe impl Sync for FFICallbackContextWrapper {} - -pub(crate) type Marker = PhantomData<(*mut u8, PhantomPinned)>; diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 44ac722..a185ab0 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -1,13 +1,12 @@ use super::{c_char, FFICallbackContext}; use crate::FFICallbackContextWrapper; use crossbeam_channel::{self as mpsc, Sender}; -use std::{ffi::CString, io, str::FromStr, thread}; +use std::{ffi::CString, io, thread}; use tap::Pipe; -use tracing::{dispatcher, error, subscriber, Dispatch, Subscriber}; +use tracing::{error, subscriber, Subscriber}; use tracing_subscriber::{ filter::{EnvFilter, LevelFilter}, - fmt::{format, FormatFields, MakeWriter, SubscriberBuilder}, - util::SubscriberInitExt, + fmt::MakeWriter, }; struct ChannelWriter { @@ -21,13 +20,13 @@ impl ChannelWriter { } impl io::Write for ChannelWriter { - fn write(&mut self, buf: &[u8]) -> Result { + fn write(&mut self, buf: &[u8]) -> io::Result { let len = buf.len(); let _ = self.sender.send(buf.to_vec()); Ok(len) } - fn flush(&mut self) -> Result<(), io::Error> { + fn flush(&mut self) -> io::Result<()> { Ok(()) } } @@ -65,7 +64,7 @@ pub enum Format { Json, } -pub struct DoubletsFFILogHandle {/* opaque */} +pub struct DoubletsFFILogHandle; impl DoubletsFFILogHandle { pub fn new( @@ -140,6 +139,6 @@ impl DoubletsFFILogHandle { ); }; - Self {} + Self } } diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index b17abf3..85843f7 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -1,10 +1,10 @@ #![allow(clippy::missing_safety_doc)] use crate::{ - c_char, c_void, + c_char, constants::Constants, errors::{DoubletsResult, OwnedSlice}, - FFICallbackContext, + stable_try as tri, FFICallbackContext, }; use doublets::{ data::{query, Flow, LinkType, Query, ToQuery}, @@ -12,14 +12,7 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{ - ffi::CStr, - marker::PhantomData, - mem, - mem::MaybeUninit, - ptr::{null_mut, NonNull}, - slice, -}; +use std::{ffi::CStr, mem, ptr::null_mut, slice}; use tap::Pipe; use tracing::{debug, warn}; @@ -61,7 +54,6 @@ impl StoreHandle { } unsafe fn thin_query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { - // fixme: may be use `assert!(!query.is_null())` if query.is_null() { query![] } else { @@ -78,7 +70,7 @@ unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a } impl DoubletsResult { - pub fn branch(flow: Flow) -> Self { + pub fn from_branch(flow: Flow) -> Self { if let Flow::Continue = flow { DoubletsResult::Continue } else { @@ -108,7 +100,7 @@ fn acquire_error(err: Error) -> DoubletsResult { fn acquire_result(result: Result>) -> DoubletsResult { match result { - Ok(flow) => DoubletsResult::branch(flow), + Ok(flow) => DoubletsResult::from_branch(flow), Err(err) => acquire_error(err), } } @@ -136,7 +128,7 @@ pub unsafe extern "C" fn create_unit_store( path: *const c_char, constants: Constants, ) -> Box> { - let result: Result<_, Error> = try { + let result: Result<_, Error> = tri! { let path = CStr::from_ptr(path).to_str().unwrap(); let mem = FileMapped::from_path(path)?; StoreHandle::new(Box::new(UnitedLinks::with_constants( @@ -252,7 +244,7 @@ pub unsafe extern "C" fn each( handle .assume_ref() .each_by(query, handler) - .pipe(DoubletsResult::branch) + .pipe(DoubletsResult::from_branch) } #[tracing::instrument( diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs new file mode 100644 index 0000000..552de46 --- /dev/null +++ b/doublets-ffi/src/utils.rs @@ -0,0 +1,11 @@ +/// Stable `try` block only for `Result` +macro_rules! stable_try { + ($($block:tt)*) => { + (|| { + let ret = { $($block)* }; + core::result::Result::Ok(ret) + })() + }; +} + +pub(crate) use stable_try; From 67afd8997df19441fb9b9802d59923dab7a8ecc0 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 18:18:05 +0300 Subject: [PATCH 41/76] Update building guide --- doublets-ffi/Cargo.toml | 3 +++ doublets-ffi/readme.md | 42 ++++++++++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/doublets-ffi/Cargo.toml b/doublets-ffi/Cargo.toml index 4beb88e..d909b21 100644 --- a/doublets-ffi/Cargo.toml +++ b/doublets-ffi/Cargo.toml @@ -19,3 +19,6 @@ tracing-subscriber = { version = "0.3.15", features = ["env-filter", "json"] } [package.log] features = ["release_max_level_error"] + +[features] +backtrace = [] \ No newline at end of file diff --git a/doublets-ffi/readme.md b/doublets-ffi/readme.md index 796bdfe..6eaf4cc 100644 --- a/doublets-ffi/readme.md +++ b/doublets-ffi/readme.md @@ -1,52 +1,68 @@ - - ## Build dynamic or static library ### Basic build library -Install [rustup](https://rustup.rs/) and setup Rust language tools. +Before you can start writing an actix application, you’ll need a version of Rust installed. +We recommend you use [rustup](https://rustup.rs/) to install or configure such latest version. + +Please note that some platforms support multiple variants of toolchains -on linux: -```shell -rustup toolchain install nightly -``` -on windows: ```shell +# windows rustup toolchain install nightly-[gnu|msvc] ``` Run cargo build in this folder: + ```shell -cargo +nightly build --release +# build with `dev` profile +cargo build +# build with `release` profile +cargo build --release ``` Great! Your libray is located in the `target/release` folder. ### Advanced build library + You can configure your build in the __`Cargo.toml`__ file: Try write the following code: + ```toml [profile.release] debug = true overflow-checks = true ``` -And rerun build +And rerun build What is it?\ `debug` - controls the amount of debug information included in the compiled binary.\ -`overflow-checks` - controls the behavior of runtime [integer overflow](https://doc.rust-lang.org/reference/expressions/operator-expr.html#overflow). +`overflow-checks` - controls the behavior of +runtime [integer overflow](https://doc.rust-lang.org/reference/expressions/operator-expr.html#overflow). + +Also, you can add it flags to `RUSTFLAGS` env: `RUSTFLAGS="-C debuginfo=2 -C overflow-checks=yes"` + +codegen flags:\ +[debuginfo](https://doc.rust-lang.org/rustc/codegen-options/index.html#debuginfo)\ +[overflow-checks](https://doc.rust-lang.org/rustc/codegen-options/index.html#overflow-checks) [More options](https://doc.rust-lang.org/cargo/reference/profiles.html) -Also you can configure log level.\ +Also you can configure compiler builtin log level.\ Try replace + ```toml [package.log] features = ["release_max_level_error"] ``` + To + ```toml [package.log] features = ["release_max_level_info"] -``` \ No newline at end of file +``` + +### Features +You can build with `--features backtrace` to enable `backtrace` feature and provide appropriate methods \ No newline at end of file From 367b05afc8ce800c43d083aacec33c0d40196ff7 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 18:25:34 +0300 Subject: [PATCH 42/76] Fix copy typo in readme --- doublets-ffi/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/readme.md b/doublets-ffi/readme.md index 6eaf4cc..543d9e5 100644 --- a/doublets-ffi/readme.md +++ b/doublets-ffi/readme.md @@ -2,7 +2,7 @@ ### Basic build library -Before you can start writing an actix application, you’ll need a version of Rust installed. +Before you can start writing a binding over doublets library, you’ll need a version of Rust installed. We recommend you use [rustup](https://rustup.rs/) to install or configure such latest version. Please note that some platforms support multiple variants of toolchains From 74fbc82c67f5d41b58cc558c108f5cc2b1fa5b19 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 31 Aug 2022 18:26:39 +0300 Subject: [PATCH 43/76] Install stable rust in readme --- doublets-ffi/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/readme.md b/doublets-ffi/readme.md index 543d9e5..1086111 100644 --- a/doublets-ffi/readme.md +++ b/doublets-ffi/readme.md @@ -9,7 +9,7 @@ Please note that some platforms support multiple variants of toolchains ```shell # windows -rustup toolchain install nightly-[gnu|msvc] +rustup toolchain install stable-[gnu|msvc] ``` Run cargo build in this folder: From 804823b0ad3ffa3fc4d9d05aa2a2377aaf02bfb3 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Thu, 1 Sep 2022 19:20:44 +0300 Subject: [PATCH 44/76] Add `Off` for `Level` and simplify `EnvFilter` to `match` --- doublets-ffi/src/logging.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index a185ab0..2dedb66 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -52,9 +52,7 @@ pub enum Level { Info = 2, Warn = 3, Error = 4, - // FFI binding can contain - // Off = 5 - // But `tracing` must ignore it + Off = 5, } #[repr(usize)] @@ -103,13 +101,14 @@ impl DoubletsFFILogHandle { }); let filter = EnvFilter::from_default_env().add_directive( - LevelFilter::from_level(match max_level { - Level::Trace => tracing::Level::TRACE, - Level::Debug => tracing::Level::DEBUG, - Level::Info => tracing::Level::INFO, - Level::Warn => tracing::Level::WARN, - Level::Error => tracing::Level::ERROR, - }) + match max_level { + Level::Trace => LevelFilter::TRACE, + Level::Debug => LevelFilter::DEBUG, + Level::Info => LevelFilter::INFO, + Level::Warn => LevelFilter::WARN, + Level::Error => LevelFilter::ERROR, + Level::Off => LevelFilter::OFF, + } .into(), ); From 056c7e21a9ef7ae10edf392edc18fb1336c75ac0 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Thu, 1 Sep 2022 19:36:16 +0300 Subject: [PATCH 45/76] Add colored shows to `log-context` example --- doublets-ffi/examples/rust/Cargo.toml | 4 ++- .../examples/rust/examples/log-context.rs | 33 +++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/doublets-ffi/examples/rust/Cargo.toml b/doublets-ffi/examples/rust/Cargo.toml index 40f1adc..9df334d 100644 --- a/doublets-ffi/examples/rust/Cargo.toml +++ b/doublets-ffi/examples/rust/Cargo.toml @@ -7,4 +7,6 @@ publish = false [dependencies] doublets-ffi = { path = "../../../doublets-ffi" } doublets = { path = "../../../doublets" } -tracing = { version = "0.1.36" } \ No newline at end of file +tracing = { version = "0.1.36" } +# dev +termcolor = { version = "1.1.3" } diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index d2886ae..b523739 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -1,20 +1,41 @@ +#![feature(try_blocks)] + use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, FFICallbackContext, }; -use std::ffi::{c_char, CStr, CString}; +use std::{ + ffi::{c_char, CStr, CString}, + io::{self, Write}, +}; +use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { let str = CStr::from_ptr(ptr).to_str().unwrap(); let ctx = &mut *(ctx as *mut usize); *ctx += 1; - if *ctx % 2 == 0 { - print!("{str}"); - } else { - eprint!("{str}"); - } + let mut stdout = StandardStream::stdout(ColorChoice::Always); + + let _: io::Result<_> = try { + match *ctx - 1 % 5 { + 0..=2 => stdout.set_color( + ColorSpec::new() + .set_fg(Some(Color::Green)) + .set_bg(Some(Color::Red)), + )?, + + 3..=5 => stdout.set_color( + ColorSpec::new() + .set_fg(Some(Color::Red)) + .set_bg(Some(Color::Green)), + )?, + _ => unreachable!(), + } + + write!(&mut stdout, "{str}")?; + }; } fn main() { From 4adab7756ff0b28874d1dd359b54368c297f4595 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Thu, 1 Sep 2022 19:37:58 +0300 Subject: [PATCH 46/76] Fix typo in example --- doublets-ffi/examples/rust/examples/log-context.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index b523739..37e149d 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -14,12 +14,11 @@ use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { let str = CStr::from_ptr(ptr).to_str().unwrap(); let ctx = &mut *(ctx as *mut usize); - *ctx += 1; let mut stdout = StandardStream::stdout(ColorChoice::Always); let _: io::Result<_> = try { - match *ctx - 1 % 5 { + match *ctx % 5 { 0..=2 => stdout.set_color( ColorSpec::new() .set_fg(Some(Color::Green)) @@ -36,6 +35,8 @@ unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { write!(&mut stdout, "{str}")?; }; + + *ctx += 1; } fn main() { From 6b1345af40b47f2f1fdbc087ad8f5e5b44e2fac9 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sat, 3 Sep 2022 15:24:53 +0300 Subject: [PATCH 47/76] Improve colors and reset --- .../examples/rust/examples/log-context.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index 37e149d..657cac4 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -19,21 +19,27 @@ unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { let _: io::Result<_> = try { match *ctx % 5 { - 0..=2 => stdout.set_color( + 0..=1 => stdout.set_color( ColorSpec::new() - .set_fg(Some(Color::Green)) - .set_bg(Some(Color::Red)), + .set_fg(Some(Color::Rgb(0, 0, 255))) + .set_bg(Some(Color::Rgb(255, 165, 0))), + )?, + 2 => stdout.set_color( + ColorSpec::new() + .set_fg(Some(Color::Rgb(255, 165, 0))) + .set_bg(Some(Color::Rgb(0, 0, 255))), )?, - 3..=5 => stdout.set_color( ColorSpec::new() - .set_fg(Some(Color::Red)) - .set_bg(Some(Color::Green)), + .set_fg(Some(Color::Rgb(0, 0, 255))) + .set_bg(Some(Color::Rgb(255, 165, 0))), )?, _ => unreachable!(), } write!(&mut stdout, "{str}")?; + + stdout.reset()?; }; *ctx += 1; From 3338ba8705088d2be630e21ec77b473f438e8d4c Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 14:59:25 +0300 Subject: [PATCH 48/76] Start queries reworking --- doublets/src/mem/mod.rs | 3 + doublets/src/mem/split/store.rs | 176 ++++++++++++-------------------- doublets/src/mem/unit/store.rs | 125 ++++++----------------- doublets/src/mem/utils.rs | 5 + 4 files changed, 105 insertions(+), 204 deletions(-) create mode 100644 doublets/src/mem/utils.rs diff --git a/doublets/src/mem/mod.rs b/doublets/src/mem/mod.rs index 73d88da..083ff77 100644 --- a/doublets/src/mem/mod.rs +++ b/doublets/src/mem/mod.rs @@ -6,6 +6,9 @@ mod header; pub mod split; mod traits; pub mod unit; +mod utils; + +pub(crate) use utils::detach_query; #[cfg(feature = "mem")] pub use mem::*; diff --git a/doublets/src/mem/split/store.rs b/doublets/src/mem/split/store.rs index 0e214b4..d8f50d7 100644 --- a/doublets/src/mem/split/store.rs +++ b/doublets/src/mem/split/store.rs @@ -2,6 +2,7 @@ use std::{cmp::Ordering, default::default, error::Error, mem::transmute, ptr::No use crate::{ mem::{ + detach_query, split::{ DataPart, ExternalSourcesRecursionlessTree, ExternalTargetsRecursionlessTree, IndexPart, InternalSourcesLinkedList, InternalSourcesRecursionlessTree, @@ -336,9 +337,7 @@ impl< // TODO: use attributes expressions feature // TODO: use `Range::contains` - link >= *constants.internal_range.start() - && link <= header.allocated - && !self.is_unused(link) + link >= constants.internal.start && link <= header.allocated && !self.is_unused(link) } // SAFETY: must be link exists @@ -350,90 +349,39 @@ impl< } fn try_each_by_core(&self, handler: ReadHandler<'_, T>, query: &[T]) -> Flow { - let query = query.to_query(); + let [index, source, target] = detach_query(query); - if query.is_empty() { - for index in T::funty(1)..=self.get_header().allocated { - if let Some(link) = self.get_link(index) { - handler(link)?; - } - } - return Flow::Continue; - } + let is_virtual_source = self.is_virtual(source); + let is_virtual_target = self.is_virtual(target); - let constants = self.constants.clone(); - let any = constants.any; - let index = query[constants.index_part.as_usize()]; - if query.len() == 1 { - return if index == any { - self.try_each_by_core(handler, &[]) - } else if let Some(link) = self.get_link(index) { - handler(link) - } else { - Flow::Continue - }; - } - // - if query.len() == 2 { - let value = query[1]; - return if index == any { - if value == any { - self.try_each_by_core(handler, &[]) + let any = T::ANY; + + return if index == any { + if (source, target) == (any, any) { + self.try_each_by_core(handler, &[any, any, any]) + } else if source == any { + if is_virtual_target { + self.external_targets.each_usages(target, handler) } else { - self.try_each_by_core(handler, &[index, value, any])?; - self.try_each_by_core(handler, &[index, any, value]) + self.internal_targets.each_usages(target, handler) } - } else if let Some(link) = self.get_link(index) { - if value == any || link.source == value || link.target == value { - handler(link) + } else if target == any { + if is_virtual_source { + self.external_sources.each_usages(source, handler) + } else if Self::USE_LIST { + self.sources_list.each_usages(source, handler) } else { - Flow::Continue + self.internal_sources.each_usages(source, handler) } } else { - Flow::Continue - }; - } - // - if query.len() == 3 { - let source = query[constants.source_part.as_usize()]; - let target = query[constants.target_part.as_usize()]; - let is_virtual_source = self.is_virtual(source); - let is_virtual_target = self.is_virtual(target); - - return if index == any { - if (source, target) == (any, any) { - self.try_each_by_core(handler, &[]) - } else if source == any { - if is_virtual_target { - self.external_targets.each_usages(target, handler) - } else { - self.internal_targets.each_usages(target, handler) - } - } else if target == any { - if is_virtual_source { - self.external_sources.each_usages(source, handler) - } else if Self::USE_LIST { - self.sources_list.each_usages(source, handler) - } else { - self.internal_sources.each_usages(source, handler) - } - } else { - let link = if true { - if is_virtual_source && is_virtual_target { + let link = if true { + if is_virtual_source && is_virtual_target { + self.external_sources.search(source, target) + } else if is_virtual_source { + self.internal_targets.search(source, target) + } else if is_virtual_target { + if Self::USE_LIST { self.external_sources.search(source, target) - } else if is_virtual_source { - self.internal_targets.search(source, target) - } else if is_virtual_target { - if Self::USE_LIST { - self.external_sources.search(source, target) - } else { - self.internal_sources.search(source, target) - } - } else if Self::USE_LIST - || self.internal_sources.count_usages(source) - > self.internal_targets.count_usages(target) - { - self.internal_targets.search(source, target) } else { self.internal_sources.search(source, target) } @@ -444,44 +392,50 @@ impl< self.internal_targets.search(source, target) } else { self.internal_sources.search(source, target) - }; - return if link == constants.null { - Flow::Continue - } else { - // SAFETY: link 100% exists - let link = unsafe { self.get_link(link).unwrap_unchecked() }; - handler(link) - }; + } + } else if Self::USE_LIST + || self.internal_sources.count_usages(source) + > self.internal_targets.count_usages(target) + { + self.internal_targets.search(source, target) + } else { + self.internal_sources.search(source, target) + }; + return if link == constants.null { + Flow::Continue + } else { + // SAFETY: link 100% exists + let link = unsafe { self.get_link(link).unwrap_unchecked() }; + handler(link) + }; + } + } else if let Some(link) = self.get_link(index) { + if (source, target) == (any, any) { + handler(link) + } else if source != any && target != any { + if (link.source, link.target) == (source, target) { + handler(link) + } else { + Flow::Continue } - } else if let Some(link) = self.get_link(index) { - if (source, target) == (any, any) { + } else if source == any { + if link.target == target { + handler(link) + } else { + Flow::Continue + } + } else if target == any { + if link.source == source { handler(link) - } else if source != any && target != any { - if (link.source, link.target) == (source, target) { - handler(link) - } else { - Flow::Continue - } - } else if source == any { - if link.target == target { - handler(link) - } else { - Flow::Continue - } - } else if target == any { - if link.source == source { - handler(link) - } else { - Flow::Continue - } } else { Flow::Continue } } else { Flow::Continue - }; - } - todo!() + } + } else { + Flow::Continue + }; } fn resolve_danglind_internal(&mut self, index: T) { @@ -697,7 +651,7 @@ impl< let header = self.get_header(); let mut free = header.first_free; if free == constants.null { - let max_inner = *constants.internal_range.end(); + let max_inner = *constants.internal.end(); if header.allocated >= max_inner { return Err(LinksError::LimitReached(max_inner)); } diff --git a/doublets/src/mem/unit/store.rs b/doublets/src/mem/unit/store.rs index 5231732..bb3584e 100644 --- a/doublets/src/mem/unit/store.rs +++ b/doublets/src/mem/unit/store.rs @@ -209,9 +209,7 @@ impl>, TS: UnitTree, TT: UnitTree, TU: let constants = self.constants(); let header = self.get_header(); - link >= *constants.internal_range.start() - && link <= header.allocated - && !self.is_unused(link) + link >= *constants.internal.start() && link <= header.allocated && !self.is_unused(link) } fn update_mem(&mut self, mem: NonNull<[LinkPart]>) { @@ -328,102 +326,43 @@ impl>, TS: UnitTree, TT: UnitTree, TU: } fn count_links(&self, query: &[T]) -> T { - if query.is_empty() { - return self.get_total(); - }; - - let constants = self.constants(); - let any = constants.any; - let index = query[constants.index_part.as_usize()]; - - if query.len() == 1 { - return if index == any { - self.get_total() - } else if self.exists(index) { - T::funty(1) - } else { - T::funty(0) - }; - } - - if query.len() == 2 { - let value = query[1]; - return if index == any { - if value == any { - self.get_total() - } else { - self.targets.count_usages(value) + self.sources.count_usages(value) - } - } else { - if !self.exists(index) { - return T::funty(0); - } - if value == any { - return T::funty(1); - } - - return self.get_link(index).map_or_else( - || T::funty(0), - |stored| { - if stored.source == value || stored.target == value { - T::funty(1) - } else { - T::funty(0) - } - }, - ); - }; - } - - if query.len() == 3 { - let source = query[constants.source_part.as_usize()]; - let target = query[constants.target_part.as_usize()]; - - return if index == any { - if (target, source) == (any, any) { - self.get_total() - } else if source == any { - self.targets.count_usages(target) - } else if target == any { - self.sources.count_usages(source) - } else { - let link = self.sources.search(source, target); - if link == constants.null { + let [index, source, target] = query; + let any = self.constants().any; + + if index == any { + match (source, target) { + (T::ANY, T::ANY) => self.get_total(), + (T::ANY, _) => self.targets.count_usages(target), + (_, T::ANY) => self.targets.count_usages(source), + (_, _) => { + if self.sources.search(source, target) == T::funty(0) { T::funty(0) } else { T::funty(1) } } - } else if !self.exists(index) { - T::funty(0) - } else if (source, target) == (any, any) { - T::funty(1) - } else { - let link = unsafe { self.get_link_unchecked(index) }; - if source != any && target != any { - if (link.source, link.target) == (source, target) { - T::funty(1) - } else { - T::funty(0) - } - } else if source == any { - if link.target == target { - T::funty(1) - } else { - T::funty(0) - } - } else if target == any { - if link.source == source { - T::funty(1) - } else { - T::funty(0) - } - } else { - T::funty(0) + } + } else { + macro_rules! bool { + ($bool:expr) => { + if $bool { T::funty(1) } else { T::funty(0) } + }; + } + + if !self.exists(index) { + return T::funty(0); + } + + let link = unsafe { self.get_link_unchecked(index) }; + match (source, target) { + (T::ANY, T::ANY) => T::funty(1), + (T::ANY, _) => bool!(link.target == target), + (_, T::ANY) => bool!(link.source == target), + (_, _) => { + bool!((link.source, link.target) == (source, target)) } - }; + } } - todo!() } fn create_links( @@ -435,7 +374,7 @@ impl>, TS: UnitTree, TT: UnitTree, TU: let header = self.get_header(); let mut free = header.first_free; if free == constants.null { - let max_inner = *constants.internal_range.end(); + let max_inner = *constants.internal.end(); if header.allocated >= max_inner { return Err(LinksError::LimitReached(max_inner)); } diff --git a/doublets/src/mem/utils.rs b/doublets/src/mem/utils.rs new file mode 100644 index 0000000..e603b01 --- /dev/null +++ b/doublets/src/mem/utils.rs @@ -0,0 +1,5 @@ +pub fn detach_query(query: &[T]) -> [T; N] { + query[..] + .try_into() + .unwrap_or_else(|_| panic!("expected `query` with {N} len")) +} From 97c2640c55d2650710a50f8d56f591ae2f23fca0 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 15:12:50 +0300 Subject: [PATCH 49/76] Revert queries reworking --- doublets/src/mem/mod.rs | 3 - doublets/src/mem/split/store.rs | 176 ++++++++++++++++++++------------ doublets/src/mem/unit/store.rs | 125 +++++++++++++++++------ doublets/src/mem/utils.rs | 5 - 4 files changed, 204 insertions(+), 105 deletions(-) delete mode 100644 doublets/src/mem/utils.rs diff --git a/doublets/src/mem/mod.rs b/doublets/src/mem/mod.rs index 083ff77..73d88da 100644 --- a/doublets/src/mem/mod.rs +++ b/doublets/src/mem/mod.rs @@ -6,9 +6,6 @@ mod header; pub mod split; mod traits; pub mod unit; -mod utils; - -pub(crate) use utils::detach_query; #[cfg(feature = "mem")] pub use mem::*; diff --git a/doublets/src/mem/split/store.rs b/doublets/src/mem/split/store.rs index d8f50d7..0e214b4 100644 --- a/doublets/src/mem/split/store.rs +++ b/doublets/src/mem/split/store.rs @@ -2,7 +2,6 @@ use std::{cmp::Ordering, default::default, error::Error, mem::transmute, ptr::No use crate::{ mem::{ - detach_query, split::{ DataPart, ExternalSourcesRecursionlessTree, ExternalTargetsRecursionlessTree, IndexPart, InternalSourcesLinkedList, InternalSourcesRecursionlessTree, @@ -337,7 +336,9 @@ impl< // TODO: use attributes expressions feature // TODO: use `Range::contains` - link >= constants.internal.start && link <= header.allocated && !self.is_unused(link) + link >= *constants.internal_range.start() + && link <= header.allocated + && !self.is_unused(link) } // SAFETY: must be link exists @@ -349,39 +350,90 @@ impl< } fn try_each_by_core(&self, handler: ReadHandler<'_, T>, query: &[T]) -> Flow { - let [index, source, target] = detach_query(query); + let query = query.to_query(); - let is_virtual_source = self.is_virtual(source); - let is_virtual_target = self.is_virtual(target); - - let any = T::ANY; + if query.is_empty() { + for index in T::funty(1)..=self.get_header().allocated { + if let Some(link) = self.get_link(index) { + handler(link)?; + } + } + return Flow::Continue; + } - return if index == any { - if (source, target) == (any, any) { - self.try_each_by_core(handler, &[any, any, any]) - } else if source == any { - if is_virtual_target { - self.external_targets.each_usages(target, handler) + let constants = self.constants.clone(); + let any = constants.any; + let index = query[constants.index_part.as_usize()]; + if query.len() == 1 { + return if index == any { + self.try_each_by_core(handler, &[]) + } else if let Some(link) = self.get_link(index) { + handler(link) + } else { + Flow::Continue + }; + } + // + if query.len() == 2 { + let value = query[1]; + return if index == any { + if value == any { + self.try_each_by_core(handler, &[]) } else { - self.internal_targets.each_usages(target, handler) + self.try_each_by_core(handler, &[index, value, any])?; + self.try_each_by_core(handler, &[index, any, value]) } - } else if target == any { - if is_virtual_source { - self.external_sources.each_usages(source, handler) - } else if Self::USE_LIST { - self.sources_list.each_usages(source, handler) + } else if let Some(link) = self.get_link(index) { + if value == any || link.source == value || link.target == value { + handler(link) } else { - self.internal_sources.each_usages(source, handler) + Flow::Continue } } else { - let link = if true { - if is_virtual_source && is_virtual_target { - self.external_sources.search(source, target) - } else if is_virtual_source { - self.internal_targets.search(source, target) - } else if is_virtual_target { - if Self::USE_LIST { + Flow::Continue + }; + } + // + if query.len() == 3 { + let source = query[constants.source_part.as_usize()]; + let target = query[constants.target_part.as_usize()]; + let is_virtual_source = self.is_virtual(source); + let is_virtual_target = self.is_virtual(target); + + return if index == any { + if (source, target) == (any, any) { + self.try_each_by_core(handler, &[]) + } else if source == any { + if is_virtual_target { + self.external_targets.each_usages(target, handler) + } else { + self.internal_targets.each_usages(target, handler) + } + } else if target == any { + if is_virtual_source { + self.external_sources.each_usages(source, handler) + } else if Self::USE_LIST { + self.sources_list.each_usages(source, handler) + } else { + self.internal_sources.each_usages(source, handler) + } + } else { + let link = if true { + if is_virtual_source && is_virtual_target { self.external_sources.search(source, target) + } else if is_virtual_source { + self.internal_targets.search(source, target) + } else if is_virtual_target { + if Self::USE_LIST { + self.external_sources.search(source, target) + } else { + self.internal_sources.search(source, target) + } + } else if Self::USE_LIST + || self.internal_sources.count_usages(source) + > self.internal_targets.count_usages(target) + { + self.internal_targets.search(source, target) } else { self.internal_sources.search(source, target) } @@ -392,50 +444,44 @@ impl< self.internal_targets.search(source, target) } else { self.internal_sources.search(source, target) - } - } else if Self::USE_LIST - || self.internal_sources.count_usages(source) - > self.internal_targets.count_usages(target) - { - self.internal_targets.search(source, target) - } else { - self.internal_sources.search(source, target) - }; - return if link == constants.null { - Flow::Continue - } else { - // SAFETY: link 100% exists - let link = unsafe { self.get_link(link).unwrap_unchecked() }; - handler(link) - }; - } - } else if let Some(link) = self.get_link(index) { - if (source, target) == (any, any) { - handler(link) - } else if source != any && target != any { - if (link.source, link.target) == (source, target) { - handler(link) - } else { - Flow::Continue - } - } else if source == any { - if link.target == target { - handler(link) - } else { - Flow::Continue + }; + return if link == constants.null { + Flow::Continue + } else { + // SAFETY: link 100% exists + let link = unsafe { self.get_link(link).unwrap_unchecked() }; + handler(link) + }; } - } else if target == any { - if link.source == source { + } else if let Some(link) = self.get_link(index) { + if (source, target) == (any, any) { handler(link) + } else if source != any && target != any { + if (link.source, link.target) == (source, target) { + handler(link) + } else { + Flow::Continue + } + } else if source == any { + if link.target == target { + handler(link) + } else { + Flow::Continue + } + } else if target == any { + if link.source == source { + handler(link) + } else { + Flow::Continue + } } else { Flow::Continue } } else { Flow::Continue - } - } else { - Flow::Continue - }; + }; + } + todo!() } fn resolve_danglind_internal(&mut self, index: T) { @@ -651,7 +697,7 @@ impl< let header = self.get_header(); let mut free = header.first_free; if free == constants.null { - let max_inner = *constants.internal.end(); + let max_inner = *constants.internal_range.end(); if header.allocated >= max_inner { return Err(LinksError::LimitReached(max_inner)); } diff --git a/doublets/src/mem/unit/store.rs b/doublets/src/mem/unit/store.rs index bb3584e..5231732 100644 --- a/doublets/src/mem/unit/store.rs +++ b/doublets/src/mem/unit/store.rs @@ -209,7 +209,9 @@ impl>, TS: UnitTree, TT: UnitTree, TU: let constants = self.constants(); let header = self.get_header(); - link >= *constants.internal.start() && link <= header.allocated && !self.is_unused(link) + link >= *constants.internal_range.start() + && link <= header.allocated + && !self.is_unused(link) } fn update_mem(&mut self, mem: NonNull<[LinkPart]>) { @@ -326,43 +328,102 @@ impl>, TS: UnitTree, TT: UnitTree, TU: } fn count_links(&self, query: &[T]) -> T { - let [index, source, target] = query; - let any = self.constants().any; - - if index == any { - match (source, target) { - (T::ANY, T::ANY) => self.get_total(), - (T::ANY, _) => self.targets.count_usages(target), - (_, T::ANY) => self.targets.count_usages(source), - (_, _) => { - if self.sources.search(source, target) == T::funty(0) { + if query.is_empty() { + return self.get_total(); + }; + + let constants = self.constants(); + let any = constants.any; + let index = query[constants.index_part.as_usize()]; + + if query.len() == 1 { + return if index == any { + self.get_total() + } else if self.exists(index) { + T::funty(1) + } else { + T::funty(0) + }; + } + + if query.len() == 2 { + let value = query[1]; + return if index == any { + if value == any { + self.get_total() + } else { + self.targets.count_usages(value) + self.sources.count_usages(value) + } + } else { + if !self.exists(index) { + return T::funty(0); + } + if value == any { + return T::funty(1); + } + + return self.get_link(index).map_or_else( + || T::funty(0), + |stored| { + if stored.source == value || stored.target == value { + T::funty(1) + } else { + T::funty(0) + } + }, + ); + }; + } + + if query.len() == 3 { + let source = query[constants.source_part.as_usize()]; + let target = query[constants.target_part.as_usize()]; + + return if index == any { + if (target, source) == (any, any) { + self.get_total() + } else if source == any { + self.targets.count_usages(target) + } else if target == any { + self.sources.count_usages(source) + } else { + let link = self.sources.search(source, target); + if link == constants.null { T::funty(0) } else { T::funty(1) } } - } - } else { - macro_rules! bool { - ($bool:expr) => { - if $bool { T::funty(1) } else { T::funty(0) } - }; - } - - if !self.exists(index) { - return T::funty(0); - } - - let link = unsafe { self.get_link_unchecked(index) }; - match (source, target) { - (T::ANY, T::ANY) => T::funty(1), - (T::ANY, _) => bool!(link.target == target), - (_, T::ANY) => bool!(link.source == target), - (_, _) => { - bool!((link.source, link.target) == (source, target)) + } else if !self.exists(index) { + T::funty(0) + } else if (source, target) == (any, any) { + T::funty(1) + } else { + let link = unsafe { self.get_link_unchecked(index) }; + if source != any && target != any { + if (link.source, link.target) == (source, target) { + T::funty(1) + } else { + T::funty(0) + } + } else if source == any { + if link.target == target { + T::funty(1) + } else { + T::funty(0) + } + } else if target == any { + if link.source == source { + T::funty(1) + } else { + T::funty(0) + } + } else { + T::funty(0) } - } + }; } + todo!() } fn create_links( @@ -374,7 +435,7 @@ impl>, TS: UnitTree, TT: UnitTree, TU: let header = self.get_header(); let mut free = header.first_free; if free == constants.null { - let max_inner = *constants.internal.end(); + let max_inner = *constants.internal_range.end(); if header.allocated >= max_inner { return Err(LinksError::LimitReached(max_inner)); } diff --git a/doublets/src/mem/utils.rs b/doublets/src/mem/utils.rs deleted file mode 100644 index e603b01..0000000 --- a/doublets/src/mem/utils.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub fn detach_query(query: &[T]) -> [T; N] { - query[..] - .try_into() - .unwrap_or_else(|_| panic!("expected `query` with {N} len")) -} From 918d2552298d2125f66831c0a09b4f4a488f2d8a Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 15:17:42 +0300 Subject: [PATCH 50/76] Forbid dynamic lib to ffi --- doublets-ffi/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/Cargo.toml b/doublets-ffi/Cargo.toml index d909b21..4d71747 100644 --- a/doublets-ffi/Cargo.toml +++ b/doublets-ffi/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [lib] -crate-type = ["cdylib", "staticlib", "lib"] +crate-type = ["staticlib", "lib"] [dependencies] doublets = { path = "../doublets" } From 26927a7a95c381ad12d1529458c1f27c7709584b Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 17:15:53 +0300 Subject: [PATCH 51/76] Add docs to `FFICallbackContextWrapper` --- doublets-ffi/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 334a57d..fcb7157 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -19,6 +19,12 @@ type c_char = std::ffi::c_char; pub type FFICallbackContext = *mut c_void; +/// [`Send`] and [`Sync`] wrapper on [`FFICallbackContext`] +/// +/// WARNING: value of `FFICallbackContext` Context must be transferred across thread boundaries +/// and safe to share references between threads. +/// +/// Otherwise value not use in multithreading context #[derive(Clone, Copy)] pub struct FFICallbackContextWrapper(FFICallbackContext); From 8561b07badedc3d33ca1bdd86585074796c39c41 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 17:53:05 +0300 Subject: [PATCH 52/76] Improve `Range` and add `Maybe` to simplify constants conversion --- doublets-ffi/src/constants.rs | 81 +++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/doublets-ffi/src/constants.rs b/doublets-ffi/src/constants.rs index f624c06..3250347 100644 --- a/doublets-ffi/src/constants.rs +++ b/doublets-ffi/src/constants.rs @@ -6,7 +6,58 @@ use std::{mem::MaybeUninit, ops::RangeInclusive}; /// [`Inclusive Range`]: https://doc.rust-lang.org/std/ops/struct.RangeInclusive.html #[derive(Eq, PartialEq)] #[repr(C)] -pub struct Range(pub T, pub T); +pub struct Range { + start: T, + end: T, +} + +impl From> for Range { + fn from(range: RangeInclusive) -> Self { + Self { + start: *range.start(), + end: *range.end(), + } + } +} + +impl From> for RangeInclusive { + fn from(Range { start, end }: Range) -> Self { + RangeInclusive::new(start, end) + } +} + +#[repr(C)] +pub struct Maybe { + // `MaybeUninit` is transparent - `Range` is repr(C) + value: MaybeUninit, + some: bool, +} + +impl From> for Maybe { + fn from(opt: Option) -> Self { + match opt { + Some(val) => Self { + value: MaybeUninit::new(val), + some: false, + }, + None => Self { + value: MaybeUninit::uninit(), + some: true, + }, + } + } +} + +impl From> for Option { + fn from(maybe: Maybe) -> Self { + if maybe.some { + // SAFETY: value is some + unsafe { Some(maybe.value.assume_init()) } + } else { + None + } + } +} #[repr(C)] pub struct Constants { @@ -21,14 +72,12 @@ pub struct Constants { pub itself: T, pub error: T, pub internal_range: Range, - // `MaybeUninit` is transparent - `Range` is repr(C) - pub external_range: MaybeUninit>, - pub external_is_some: bool, + pub external_range: Maybe>, } impl From> for Constants { fn from(c: LinksConstants) -> Self { - let mut new = Self { + Self { index_part: c.index_part, source_part: c.source_part, target_part: c.target_part, @@ -39,16 +88,10 @@ impl From> for Constants { any: c.any, itself: c.itself, error: c.error, - internal_range: Range(*c.internal_range.start(), *c.internal_range.end()), + internal_range: Range::from(c.internal_range), // external_range: c.external_range.map(|r| Range(*r.start(), *r.end())), - external_range: MaybeUninit::uninit(), - external_is_some: false, - }; - if let Some(r) = c.external_range { - new.external_is_some = true; - new.external_range.write(Range(*r.start(), *r.end())); + external_range: c.external_range.map(Range::from).into(), } - new } } @@ -66,16 +109,8 @@ impl Into> for Constants { any: self.any, itself: self.itself, error: self.error, - internal_range: RangeInclusive::new(self.internal_range.0, self.internal_range.1), - external_range: if self.external_is_some { - // SAFETY: `self.external_range` is init - unsafe { - let range = self.external_range.assume_init(); - Some(RangeInclusive::new(range.0, range.1)) - } - } else { - None - }, + internal_range: RangeInclusive::from(self.internal_range), + external_range: Option::from(self.external_range).map(|range: Range<_>| range.into()), } } } From 903d07e44ea2748a92f26b6b7b80b8cfb506f101 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 17:55:21 +0300 Subject: [PATCH 53/76] Fix typo --- doublets-ffi/src/constants.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doublets-ffi/src/constants.rs b/doublets-ffi/src/constants.rs index 3250347..d96113e 100644 --- a/doublets-ffi/src/constants.rs +++ b/doublets-ffi/src/constants.rs @@ -38,11 +38,11 @@ impl From> for Maybe { match opt { Some(val) => Self { value: MaybeUninit::new(val), - some: false, + some: true, }, None => Self { value: MaybeUninit::uninit(), - some: true, + some: false, }, } } From ae182b5b47549b60da6b63db6f027c2441054f5c Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 21:54:25 +0300 Subject: [PATCH 54/76] Improve `backtrace` feature in favor of `unstable_backtrace` add `.gitignore` with `.cargo` --- .gitignore | 2 ++ doublets-ffi/Cargo.toml | 3 --- doublets-ffi/readme.md | 4 +++- doublets-ffi/src/errors.rs | 20 ++++++++++---------- doublets-ffi/src/lib.rs | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d52dd4a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.cargo +/target diff --git a/doublets-ffi/Cargo.toml b/doublets-ffi/Cargo.toml index 4d71747..86f9bd7 100644 --- a/doublets-ffi/Cargo.toml +++ b/doublets-ffi/Cargo.toml @@ -19,6 +19,3 @@ tracing-subscriber = { version = "0.3.15", features = ["env-filter", "json"] } [package.log] features = ["release_max_level_error"] - -[features] -backtrace = [] \ No newline at end of file diff --git a/doublets-ffi/readme.md b/doublets-ffi/readme.md index 1086111..5ba422a 100644 --- a/doublets-ffi/readme.md +++ b/doublets-ffi/readme.md @@ -65,4 +65,6 @@ features = ["release_max_level_info"] ``` ### Features -You can build with `--features backtrace` to enable `backtrace` feature and provide appropriate methods \ No newline at end of file + +You can build with `--cfg unstable_backtrace` in `RUSTFLAGS` +to enable `backtrace` feature and provide appropriate methods \ No newline at end of file diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index c6e16b9..452b58b 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,6 +1,6 @@ use crate::c_char; use doublets::{data::LinkType, mem, Doublet, Link}; -#[cfg(feature = "backtrace")] +#[cfg(unstable_backtrace)] use std::backtrace::Backtrace; use std::{ cmp, error, @@ -10,6 +10,7 @@ use std::{ ptr, ptr::NonNull, }; +use tap::Pipe; use tracing::warn; type OpaqueError = Box; @@ -84,17 +85,16 @@ pub enum DoubletsResult { Other(Box), } -#[rustfmt::skip] impl DoubletsResult { - #[cfg(feature = "backtrace")] + #[cfg(unstable_backtrace)] fn backtrace(&self) -> Option<&Backtrace> { + fn erasure(err: &mem::Error) -> &dyn error::Error { + err as _ + } + match self { - DoubletsResult::AllocFailed(err) => { - (&**err as &dyn error::Error).request_ref::() - } - DoubletsResult::Other(err) => { - (&***err as &dyn error::Error).request_ref::() - } + DoubletsResult::AllocFailed(err) => erasure(err).request_ref(), + DoubletsResult::Other(err) => err.request_ref(), DoubletsResult::Break | DoubletsResult::Continue => { panic!("`backtrace` not allowed for ok results") } @@ -180,7 +180,7 @@ pub unsafe extern "C" fn read_error( } } -#[cfg(feature = "backtrace")] +#[cfg(unstable_backtrace)] #[ffi::specialize_for( types::( u8 => u8, diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index fcb7157..4dc69f3 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(feature = "backtrace", feature(error_generic_member_access))] +#![cfg_attr(unstable_backtrace, feature(error_generic_member_access))] pub mod constants; pub mod errors; From 3eb54ef95903e579badd119bce1b33648e5a212d Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 4 Sep 2022 22:05:01 +0300 Subject: [PATCH 55/76] Fix `if-else-let` lint --- doublets-ffi/src/errors.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 452b58b..4372c09 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -200,9 +200,7 @@ pub unsafe extern "C" fn read_backtrace( ) { if let DoubletsResult::Break | DoubletsResult::Continue = error { warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); - } else { - if let Some(backtrace) = error.backtrace() { - write_raw_msg(buf, size, &backtrace.to_string()); - } + } else if let Some(backtrace) = error.backtrace() { + write_raw_msg(buf, size, &backtrace.to_string()); } } From d6de95af86e6b4b6d44516e7558cd97225e1dc68 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 6 Sep 2022 18:37:56 +0300 Subject: [PATCH 56/76] Add marker> to `OwnedSlice` --- doublets-ffi/src/errors.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 4372c09..13d55fb 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -7,10 +7,10 @@ use std::{ ffi::c_short, fmt, fmt::{Debug, Display, Formatter}, + marker::PhantomData, ptr, ptr::NonNull, }; -use tap::Pipe; use tracing::warn; type OpaqueError = Box; @@ -20,6 +20,8 @@ type OpaqueError = Box; pub struct OwnedSlice { ptr: NonNull, len: usize, + // actually it's still a `Box<[T]>` + _marker: PhantomData>, } impl OwnedSlice { @@ -35,6 +37,7 @@ impl OwnedSlice { // ptr: leak.as_non_null_ptr(), ptr: unsafe { NonNull::new_unchecked(leak.as_ptr() as *mut _) }, len: leak.len(), + _marker: PhantomData, } } From c2a4813be8ff2cd4febe8fc0276c9fe64e283355 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 6 Sep 2022 21:05:54 +0300 Subject: [PATCH 57/76] allow(clippy::borrowed_box) for `assume_ref` --- doublets-ffi/src/store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 85843f7..816cc80 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -35,6 +35,7 @@ impl StoreHandle { &mut self.pointer } + #[allow(clippy::borrowed_box)] // needs for `Self: Sized` also can use `&impl Doublets` pub unsafe fn assume_ref(&self) -> &Box> { &self.pointer } From 766df154338839b5ffdda2a21cc361de43cf00b5 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 6 Sep 2022 21:13:20 +0300 Subject: [PATCH 58/76] Use `while let` instead of `loop { if let }` in logging loop --- doublets-ffi/src/logging.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 2dedb66..d7f4900 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -84,20 +84,16 @@ impl DoubletsFFILogHandle { }; thread::spawn(move || { - // We can't use `while let Ok(msg) = receiver.recv()` - // here because the receiver will be blocked - - loop { - // info_span!("Logging loop").in_scope(|| { - if let Ok(msg) = receiver.recv() { - let str = CString::new(msg) - .expect("Only UTF-8 format strings are allowed in logging"); - callback(wrapper, str.as_ptr()); - } else { - break; - } - // }); + // We can use `while let Ok(msg) = receiver.recv()` + // `crossbeam::recv` is blocking it is base + + // info_span!("Logging loop").in_scope(|| { + while let Ok(msg) = receiver.recv() { + let str = + CString::new(msg).expect("Only UTF-8 format strings are allowed in logging"); + callback(wrapper, str.as_ptr()); } + // }); }); let filter = EnvFilter::from_default_env().add_directive( From 2e7e592fada69590bda1e837de90bd3328fa488b Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Tue, 6 Sep 2022 21:34:53 +0300 Subject: [PATCH 59/76] Use slice as queries --- doublets-ffi/src/store.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 816cc80..cad3d63 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -7,7 +7,7 @@ use crate::{ stable_try as tri, FFICallbackContext, }; use doublets::{ - data::{query, Flow, LinkType, Query, ToQuery}, + data::{Flow, LinkType}, mem::FileMapped, parts, unit, Doublets, Error, Link, Links, }; @@ -54,15 +54,15 @@ impl StoreHandle { } } -unsafe fn thin_query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { +unsafe fn thin_query_from_raw<'a, T>(query: *const T, len: u32) -> &'a [T] { if query.is_null() { - query![] + &[] } else { - slice::from_raw_parts(query, len as usize).to_query() + slice::from_raw_parts(query, len as usize) } } -unsafe fn query_from_raw<'a, T: LinkType>(query: *const T, len: u32) -> Query<'a, T> { +unsafe fn query_from_raw<'a, T>(query: *const T, len: u32) -> &'a [T] { if query.is_null() && len != 0 { warn!("query ptr is null, but len is not null: handle could be a potential mistake."); } @@ -181,7 +181,7 @@ pub unsafe extern "C" fn constants_from_store( #[tracing::instrument( skip_all, fields( - query = ?&thin_query_from_raw(query, len)[..], + query = ?thin_query_from_raw(query, len), query.ptr = ?query, query.len = len, ), @@ -216,7 +216,7 @@ pub unsafe extern "C" fn create( #[tracing::instrument( skip_all, fields( - query = ?&thin_query_from_raw(query, len)[..], + query = ?thin_query_from_raw(query, len), query.ptr = ?query, query.len = len, ), @@ -251,7 +251,7 @@ pub unsafe extern "C" fn each( #[tracing::instrument( skip_all, fields( - query = ?&thin_query_from_raw(query, len)[..], + query = ?thin_query_from_raw(query, len), query.ptr = ?query, query.len = len, ), @@ -280,11 +280,11 @@ pub unsafe extern "C" fn count( #[tracing::instrument( skip_all, fields( - query = ?&thin_query_from_raw(query, len_q)[..], + query = ?thin_query_from_raw(query, len_q), query.ptr = ?query, query.len = len_q, - change = ?&thin_query_from_raw(query, len_q)[..], + change = ?thin_query_from_raw(query, len_q), change.ptr = ?change, change.len = len_c, ), @@ -322,7 +322,7 @@ pub unsafe extern "C" fn update( #[tracing::instrument( skip_all, fields( - query = ?&thin_query_from_raw(query, len)[..], + query = ?thin_query_from_raw(query, len), query.ptr = ?query, query.len = len, ) From 155f475cad9b6baff987b215f95b5ad1a1c002f0 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 7 Sep 2022 18:44:27 +0300 Subject: [PATCH 60/76] Add error handling to store creation --- .../examples/rust/examples/error-handling.rs | 9 +++- doublets-ffi/src/constants.rs | 36 +------------ doublets-ffi/src/errors.rs | 6 ++- doublets-ffi/src/store.rs | 51 ++++++++++++++---- doublets-ffi/src/utils.rs | 53 +++++++++++++++++++ 5 files changed, 107 insertions(+), 48 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index f35e4ed..d93f803 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -30,8 +30,15 @@ fn main() { doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, false); let path = CString::new("doublets.links").unwrap(); - let mut handle = + let result = create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); + let mut handle = match result { + DoubletsResult::Handle(handle) => handle, + DoubletsResult::Continue | DoubletsResult::Break => unreachable!(), + error => { + panic!("unexpected error: {}", error) + } + }; let any = constants_from_store::(&handle).any; diff --git a/doublets-ffi/src/constants.rs b/doublets-ffi/src/constants.rs index d96113e..bb485ed 100644 --- a/doublets-ffi/src/constants.rs +++ b/doublets-ffi/src/constants.rs @@ -1,5 +1,6 @@ +use crate::utils::Maybe; use doublets::data::{LinkType, LinksConstants}; -use std::{mem::MaybeUninit, ops::RangeInclusive}; +use std::ops::RangeInclusive; /// FFI repr to [`Inclusive Range`] /// @@ -26,39 +27,6 @@ impl From> for RangeInclusive { } } -#[repr(C)] -pub struct Maybe { - // `MaybeUninit` is transparent - `Range` is repr(C) - value: MaybeUninit, - some: bool, -} - -impl From> for Maybe { - fn from(opt: Option) -> Self { - match opt { - Some(val) => Self { - value: MaybeUninit::new(val), - some: true, - }, - None => Self { - value: MaybeUninit::uninit(), - some: false, - }, - } - } -} - -impl From> for Option { - fn from(maybe: Maybe) -> Self { - if maybe.some { - // SAFETY: value is some - unsafe { Some(maybe.value.assume_init()) } - } else { - None - } - } -} - #[repr(C)] pub struct Constants { pub index_part: T, diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 13d55fb..bc2409c 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -79,6 +79,7 @@ pub enum DoubletsResult { // oks Break, Continue, + Handle(Box>), // errors NotExists(T), LimitReached(T), @@ -135,6 +136,7 @@ impl Display for DoubletsResult { } } +use crate::store::StoreHandle; use ffi_attributes as ffi; #[ffi::specialize_for( @@ -176,7 +178,7 @@ pub unsafe extern "C" fn read_error( size: c_short, error: &DoubletsResult, ) { - if let DoubletsResult::Break | DoubletsResult::Continue = error { + if let DoubletsResult::Break | DoubletsResult::Continue | DoubletsResult::Handle(_) = error { warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); } else { write_raw_msg(buf, size, &error.to_string()); @@ -201,7 +203,7 @@ pub unsafe extern "C" fn read_backtrace( size: c_short, error: &DoubletsResult, ) { - if let DoubletsResult::Break | DoubletsResult::Continue = error { + if let DoubletsResult::Break | DoubletsResult::Continue | DoubletsResult::Handle(_) = error { warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); } else if let Some(backtrace) = error.backtrace() { write_raw_msg(buf, size, &backtrace.to_string()); diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index cad3d63..ad964ff 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -4,7 +4,9 @@ use crate::{ c_char, constants::Constants, errors::{DoubletsResult, OwnedSlice}, - stable_try as tri, FFICallbackContext, + stable_try as tri, + utils::Maybe, + FFICallbackContext, }; use doublets::{ data::{Flow, LinkType}, @@ -12,7 +14,11 @@ use doublets::{ parts, unit, Doublets, Error, Link, Links, }; use ffi_attributes as ffi; -use std::{ffi::CStr, mem, ptr::null_mut, slice}; +use std::{ + ffi::CStr, + fmt::{self, Debug, Formatter}, + slice, +}; use tap::Pipe; use tracing::{debug, warn}; @@ -26,9 +32,17 @@ pub struct StoreHandle { pointer: Box>, } +impl Debug for StoreHandle { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreHandle") + .field("pointer", &(self.pointer.as_ref() as *const _)) + .finish() + } +} + impl StoreHandle { - pub fn new(store: Box>) -> Box { - Box::new(Self { pointer: store }) + pub fn new(store: Box>) -> Self { + Self { pointer: store } } pub unsafe fn assume(&mut self) -> &mut Box> { @@ -46,11 +60,10 @@ impl StoreHandle { /// /// Caller guarantee that will not drop handle // fixme: may be we can port `result::Result` to C - pub fn invalid(err: Error) -> Box { + pub fn invalid(err: Error) -> Maybe { acquire_error(err); - // SAFETY: Box is repr to `*mut T` and must forgot - unsafe { mem::transmute(null_mut::()) } + Maybe::none() } } @@ -93,15 +106,31 @@ impl DoubletsResult { } } +impl From for DoubletsResult { + fn from(flow: Flow) -> Self { + Self::from_branch(flow) + } +} + +impl From>> for DoubletsResult { + fn from(handle: Box>) -> Self { + Self::Handle(handle) + } +} + fn acquire_error(err: Error) -> DoubletsResult { // It can be very expensive to handle each error debug!(op_error = % err); DoubletsResult::from_err(err) } -fn acquire_result(result: Result>) -> DoubletsResult { +fn acquire_result(result: Result>) -> DoubletsResult +where + T: LinkType, + Ok: Into>, +{ match result { - Ok(flow) => DoubletsResult::from_branch(flow), + Ok(ok) => ok.into(), Err(err) => acquire_error(err), } } @@ -128,7 +157,7 @@ fn acquire_result(result: Result>) -> DoubletsResult pub unsafe extern "C" fn create_unit_store( path: *const c_char, constants: Constants, -) -> Box> { +) -> DoubletsResult { let result: Result<_, Error> = tri! { let path = CStr::from_ptr(path).to_str().unwrap(); let mem = FileMapped::from_path(path)?; @@ -137,7 +166,7 @@ pub unsafe extern "C" fn create_unit_store( constants.into(), )?)) }; - result.unwrap_or_else(StoreHandle::invalid) + result.map(Box::new).pipe(acquire_result) } #[ffi::specialize_for( diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs index 552de46..978f171 100644 --- a/doublets-ffi/src/utils.rs +++ b/doublets-ffi/src/utils.rs @@ -9,3 +9,56 @@ macro_rules! stable_try { } pub(crate) use stable_try; +use std::mem::MaybeUninit; + +#[repr(C)] +pub struct Maybe { + // `MaybeUninit` is transparent - `Range` is repr(C) + value: MaybeUninit, + some: bool, +} + +impl Maybe { + pub fn some(value: T) -> Self { + Self { + value: MaybeUninit::new(value), + some: true, + } + } + + pub fn none() -> Self { + Self { + value: MaybeUninit::uninit(), + some: false, + } + } + + pub fn unwrap(self) -> T { + if self.some { + // SAFETY: value is some + unsafe { self.value.assume_init() } + } else { + panic!("called `Maybe::unwrap()` on a `none` value") + } + } +} + +impl From> for Maybe { + fn from(opt: Option) -> Self { + match opt { + Some(val) => Maybe::some(val), + None => Maybe::none(), + } + } +} + +impl From> for Option { + fn from(maybe: Maybe) -> Self { + if maybe.some { + // SAFETY: value is some + unsafe { Some(maybe.value.assume_init()) } + } else { + None + } + } +} From d6195603561691f0b6bcc199b2e86f403ef767e1 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 7 Sep 2022 19:12:02 +0300 Subject: [PATCH 61/76] explain `if let` to `.is_ok` --- doublets-ffi/src/errors.rs | 6 +++--- doublets-ffi/src/store.rs | 8 ++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index bc2409c..b624dab 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -99,7 +99,7 @@ impl DoubletsResult { match self { DoubletsResult::AllocFailed(err) => erasure(err).request_ref(), DoubletsResult::Other(err) => err.request_ref(), - DoubletsResult::Break | DoubletsResult::Continue => { + _ if self.is_ok() => { panic!("`backtrace` not allowed for ok results") } _ => None, @@ -178,7 +178,7 @@ pub unsafe extern "C" fn read_error( size: c_short, error: &DoubletsResult, ) { - if let DoubletsResult::Break | DoubletsResult::Continue | DoubletsResult::Handle(_) = error { + if error.is_ok() { warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); } else { write_raw_msg(buf, size, &error.to_string()); @@ -203,7 +203,7 @@ pub unsafe extern "C" fn read_backtrace( size: c_short, error: &DoubletsResult, ) { - if let DoubletsResult::Break | DoubletsResult::Continue | DoubletsResult::Handle(_) = error { + if error.is_ok() { warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); } else if let Some(backtrace) = error.backtrace() { write_raw_msg(buf, size, &backtrace.to_string()); diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index ad964ff..81ceeb8 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -104,6 +104,14 @@ impl DoubletsResult { Error::Other(other) => Self::Other(Box::new(other)), } } + + pub fn is_ok(&self) -> bool { + matches!(self, Self::Break | Self::Continue | Self::Handle(_)) + } + + pub fn is_err(&self) -> bool { + !self.is_ok() + } } impl From for DoubletsResult { From af3fef01ac93f3b2e7dfe6ce4dff6a0f04e3da75 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 7 Sep 2022 21:07:47 +0300 Subject: [PATCH 62/76] Use shortest error messages --- doublets-ffi/src/errors.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index b624dab..4fa5d66 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -114,10 +114,7 @@ impl Display for DoubletsResult { write!(f, "link {exists} does not exist.") } DoubletsResult::LimitReached(limit) => { - write!( - f, - "limit for the number of links in the storage has been reached: {limit}" - ) + write!(f, "links limit in storage has been reached: {limit}") } DoubletsResult::HasUsages(usages) => { write!(f, "link {usages:?} has dependencies") @@ -126,7 +123,7 @@ impl Display for DoubletsResult { write!(f, "link {exists} already exists") } DoubletsResult::AllocFailed(alloc) => { - write!(f, "unable to allocate memory for links storage: `{alloc}`") + write!(f, "alloc memory error: `{alloc}`") } DoubletsResult::Other(other) => { write!(f, "other internal error: `{other}`") From 722094c6ccf0f165333adf9c0037938a9d04287b Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Wed, 7 Sep 2022 21:09:34 +0300 Subject: [PATCH 63/76] Shortest ptr cast --- doublets-ffi/src/errors.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index 4fa5d66..a065109 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -35,7 +35,7 @@ impl OwnedSlice { let leak = NonNull::from(Box::leak(place)); OwnedSlice { // ptr: leak.as_non_null_ptr(), - ptr: unsafe { NonNull::new_unchecked(leak.as_ptr() as *mut _) }, + ptr: leak.cast(), len: leak.len(), _marker: PhantomData, } From 8da33d52f7ddfad0afa8033e6670882d4baf0078 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 17:23:54 +0300 Subject: [PATCH 64/76] Improve error handling in favor of Rust looks like --- .../examples/rust/examples/error-handling.rs | 8 +- doublets-ffi/src/errors.rs | 112 +++------------ doublets-ffi/src/store.rs | 66 ++------- doublets-ffi/src/utils.rs | 131 ++++++++++++++---- 4 files changed, 143 insertions(+), 174 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index d93f803..f035bba 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -4,7 +4,7 @@ use doublets::{ }; use doublets_ffi::{ constants::Constants, - errors::{free_error, read_error, DoubletsResult}, + errors::{free_error, read_error, DoubletsError}, export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, @@ -33,8 +33,8 @@ fn main() { let result = create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); let mut handle = match result { - DoubletsResult::Handle(handle) => handle, - DoubletsResult::Continue | DoubletsResult::Break => unreachable!(), + DoubletsError::Handle(handle) => handle, + DoubletsError::Continue | DoubletsError::Break => unreachable!(), error => { panic!("unexpected error: {}", error) } @@ -45,7 +45,7 @@ fn main() { let query = [1 /* not exists index */, any, any]; let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); - if let DoubletsResult::Continue | DoubletsResult::Break = result { + if let DoubletsError::Continue | DoubletsError::Break = result { unreachable!() } else { let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index a065109..c108ef4 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -5,82 +5,16 @@ use std::backtrace::Backtrace; use std::{ cmp, error, ffi::c_short, - fmt, - fmt::{Debug, Display, Formatter}, - marker::PhantomData, + fmt::{self, Debug, Display, Formatter}, ptr, - ptr::NonNull, }; use tracing::warn; type OpaqueError = Box; -/// `OwnedSlice` is a FFI-Safe `Box<[T]>` representation -#[repr(C)] -pub struct OwnedSlice { - ptr: NonNull, - len: usize, - // actually it's still a `Box<[T]>` - _marker: PhantomData>, -} - -impl OwnedSlice { - #[inline] - pub fn slice_from_raw_parts(data: NonNull, len: usize) -> NonNull<[T]> { - // SAFETY: `data` is a `NonNull` pointer which is necessarily non-null - unsafe { NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(data.as_ptr(), len)) } - } - - pub fn leak(place: Box<[T]>) -> Self { - let leak = NonNull::from(Box::leak(place)); - OwnedSlice { - // ptr: leak.as_non_null_ptr(), - ptr: leak.cast(), - len: leak.len(), - _marker: PhantomData, - } - } - - pub fn as_slice(&self) -> &[T] { - let slice = Self::slice_from_raw_parts(self.ptr, self.len); - // SAFETY: `Self` is opaque we create Box and we drop it - unsafe { slice.as_ref() } - } - - /// # Safety - /// forget `self` after `.keep_own` - pub unsafe fn keep_own(&self) -> Box<[T]> { - let slice = Self::slice_from_raw_parts(self.ptr, self.len); - unsafe { Box::from_raw(slice.as_ptr()) } - } - - pub fn into_owned(self) -> Box<[T]> { - // SAFETY: `self` drop after call `.into_owned` - unsafe { self.keep_own() } - } -} - -impl Debug for OwnedSlice { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.as_slice()) - } -} - -impl Drop for OwnedSlice { - fn drop(&mut self) { - // SAFETY: `self` drop at end of this scope - let _ = unsafe { self.keep_own() }; - } -} - #[repr(C, usize)] #[derive(Debug)] -pub enum DoubletsResult { - // oks - Break, - Continue, - Handle(Box>), - // errors +pub enum DoubletsError { NotExists(T), LimitReached(T), HasUsages(OwnedSlice>), @@ -89,7 +23,7 @@ pub enum DoubletsResult { Other(Box), } -impl DoubletsResult { +impl DoubletsError { #[cfg(unstable_backtrace)] fn backtrace(&self) -> Option<&Backtrace> { fn erasure(err: &mem::Error) -> &dyn error::Error { @@ -97,43 +31,39 @@ impl DoubletsResult { } match self { - DoubletsResult::AllocFailed(err) => erasure(err).request_ref(), - DoubletsResult::Other(err) => err.request_ref(), - _ if self.is_ok() => { - panic!("`backtrace` not allowed for ok results") - } + DoubletsError::AllocFailed(err) => erasure(err).request_ref(), + DoubletsError::Other(err) => err.request_ref(), _ => None, } } } -impl Display for DoubletsResult { +impl Display for DoubletsError { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { - DoubletsResult::NotExists(exists) => { + DoubletsError::NotExists(exists) => { write!(f, "link {exists} does not exist.") } - DoubletsResult::LimitReached(limit) => { + DoubletsError::LimitReached(limit) => { write!(f, "links limit in storage has been reached: {limit}") } - DoubletsResult::HasUsages(usages) => { + DoubletsError::HasUsages(usages) => { write!(f, "link {usages:?} has dependencies") } - DoubletsResult::AlreadyExists(exists) => { + DoubletsError::AlreadyExists(exists) => { write!(f, "link {exists} already exists") } - DoubletsResult::AllocFailed(alloc) => { + DoubletsError::AllocFailed(alloc) => { write!(f, "alloc memory error: `{alloc}`") } - DoubletsResult::Other(other) => { + DoubletsError::Other(other) => { write!(f, "other internal error: `{other}`") } - other => Debug::fmt(other, f), } } } -use crate::store::StoreHandle; +use crate::utils::OwnedSlice; use ffi_attributes as ffi; #[ffi::specialize_for( @@ -148,7 +78,7 @@ use ffi_attributes as ffi; #[no_mangle] ) )] -pub extern "C" fn free_error(err: DoubletsResult) { +pub extern "C" fn free_error(err: DoubletsError) { let _ = err; } @@ -173,13 +103,9 @@ unsafe fn write_raw_msg(buf: *mut c_char, size: c_short, msg: &str) { pub unsafe extern "C" fn read_error( buf: *mut c_char, size: c_short, - error: &DoubletsResult, + error: &DoubletsError, ) { - if error.is_ok() { - warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); - } else { - write_raw_msg(buf, size, &error.to_string()); - } + write_raw_msg(buf, size, &error.to_string()); } #[cfg(unstable_backtrace)] @@ -198,11 +124,9 @@ pub unsafe extern "C" fn read_error( pub unsafe extern "C" fn read_backtrace( buf: *mut c_char, size: c_short, - error: &DoubletsResult, + error: &DoubletsError, ) { - if error.is_ok() { - warn!("`DoubletsResult` is expected to contain an error, got: `{error:?}`"); - } else if let Some(backtrace) = error.backtrace() { + if let Some(backtrace) = error.backtrace() { write_raw_msg(buf, size, &backtrace.to_string()); } } diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 81ceeb8..d38c8a5 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -3,9 +3,9 @@ use crate::{ c_char, constants::Constants, - errors::{DoubletsResult, OwnedSlice}, + errors::DoubletsError, stable_try as tri, - utils::Maybe, + utils::{Fallible, Maybe, OwnedSlice}, FFICallbackContext, }; use doublets::{ @@ -83,16 +83,8 @@ unsafe fn query_from_raw<'a, T>(query: *const T, len: u32) -> &'a [T] { thin_query_from_raw(query, len) } -impl DoubletsResult { - pub fn from_branch(flow: Flow) -> Self { - if let Flow::Continue = flow { - DoubletsResult::Continue - } else { - DoubletsResult::Break - } - } - - pub fn from_err(err: Error) -> Self { +impl From> for DoubletsError { + fn from(err: Error) -> Self { match err { Error::NotExists(link) => Self::NotExists(link), Error::HasUsages(usages) => Self::HasUsages(OwnedSlice::leak(usages)), @@ -104,43 +96,16 @@ impl DoubletsResult { Error::Other(other) => Self::Other(Box::new(other)), } } - - pub fn is_ok(&self) -> bool { - matches!(self, Self::Break | Self::Continue | Self::Handle(_)) - } - - pub fn is_err(&self) -> bool { - !self.is_ok() - } -} - -impl From for DoubletsResult { - fn from(flow: Flow) -> Self { - Self::from_branch(flow) - } -} - -impl From>> for DoubletsResult { - fn from(handle: Box>) -> Self { - Self::Handle(handle) - } } -fn acquire_error(err: Error) -> DoubletsResult { +fn acquire_error(err: Error) -> DoubletsError { // It can be very expensive to handle each error debug!(op_error = % err); - DoubletsResult::from_err(err) + err.into() } -fn acquire_result(result: Result>) -> DoubletsResult -where - T: LinkType, - Ok: Into>, -{ - match result { - Ok(ok) => ok.into(), - Err(err) => acquire_error(err), - } +fn acquire_result(result: Result>) -> Fallible> { + result.map_err(acquire_error).into() } #[tracing::instrument( @@ -165,7 +130,7 @@ where pub unsafe extern "C" fn create_unit_store( path: *const c_char, constants: Constants, -) -> DoubletsResult { +) -> Fallible>, DoubletsError> { let result: Result<_, Error> = tri! { let path = CStr::from_ptr(path).to_str().unwrap(); let mem = FileMapped::from_path(path)?; @@ -241,7 +206,7 @@ pub unsafe extern "C" fn create( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResult { +) -> Fallible> { let query = query_from_raw(query, len); let handler = move |before, after| callback(ctx, before, after); handle @@ -276,13 +241,10 @@ pub unsafe extern "C" fn each( len: u32, ctx: FFICallbackContext, callback: EachCallback, -) -> DoubletsResult { +) -> Flow { let query = query_from_raw(query, len); let handler = move |link| callback(ctx, link); - handle - .assume_ref() - .each_by(query, handler) - .pipe(DoubletsResult::from_branch) + handle.assume_ref().each_by(query, handler) } #[tracing::instrument( @@ -346,7 +308,7 @@ pub unsafe extern "C" fn update( len_c: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResult { +) -> Fallible> { let handler = move |before, after| callback(ctx, before, after); let query = query_from_raw(query, len_q); let change = query_from_raw(change, len_c); @@ -382,7 +344,7 @@ pub unsafe extern "C" fn delete( len: u32, ctx: FFICallbackContext, callback: CUDCallback, -) -> DoubletsResult { +) -> Fallible> { let handler = move |before, after| callback(ctx, before, after); let query = query_from_raw(query, len); handle diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs index 978f171..6cbc11a 100644 --- a/doublets-ffi/src/utils.rs +++ b/doublets-ffi/src/utils.rs @@ -9,36 +9,62 @@ macro_rules! stable_try { } pub(crate) use stable_try; -use std::mem::MaybeUninit; +use std::{ + fmt::{self, Debug, Formatter}, + marker::PhantomData, + ptr::{self, NonNull}, +}; -#[repr(C)] -pub struct Maybe { - // `MaybeUninit` is transparent - `Range` is repr(C) - value: MaybeUninit, - some: bool, +/// С/C++ not allow zero-sized types. `u8` it's a repr of (in C/C++) +/// with `sizeof == 1` and `alignof == 1`: +/// +/// +/// struct Void {}; +/// +#[repr(transparent)] +#[derive(Default)] +pub struct Void { + _nonzero: u8, } -impl Maybe { - pub fn some(value: T) -> Self { - Self { - value: MaybeUninit::new(value), - some: true, +#[repr(C, usize)] +pub enum Fallible { + Ok(T), + Err(E), +} + +impl Fallible { + pub fn unwrap(self) -> T { + if let Self::Ok(val) = self { + val + } else { + panic!("called `Fallible::unwrap()` on a `Err` value") } } +} +pub type Maybe = Fallible; + +impl Maybe { pub fn none() -> Self { - Self { - value: MaybeUninit::uninit(), - some: false, + Self::Err(Void::default()) + } +} + +impl From> for Fallible { + fn from(result: Result) -> Self { + match result { + Ok(ok) => Self::Ok(ok), + Err(err) => Self::Err(err), } } +} - pub fn unwrap(self) -> T { - if self.some { - // SAFETY: value is some - unsafe { self.value.assume_init() } - } else { - panic!("called `Maybe::unwrap()` on a `none` value") +impl From> for Result { + fn from(fallible: Fallible) -> Self { + match fallible { + Fallible::Ok(ok) => Ok(ok), + Fallible::Err(err) => Err(err), } } } @@ -46,7 +72,7 @@ impl Maybe { impl From> for Maybe { fn from(opt: Option) -> Self { match opt { - Some(val) => Maybe::some(val), + Some(val) => Maybe::Ok(val), None => Maybe::none(), } } @@ -54,11 +80,68 @@ impl From> for Maybe { impl From> for Option { fn from(maybe: Maybe) -> Self { - if maybe.some { - // SAFETY: value is some - unsafe { Some(maybe.value.assume_init()) } + if let Maybe::Ok(val) = maybe { + Some(val) } else { None } } } + +/// `OwnedSlice` is a FFI-Safe `Box<[T]>` representation +#[repr(C)] +pub struct OwnedSlice { + ptr: NonNull, + len: usize, + // actually it's still a `Box<[T]>` + _marker: PhantomData>, +} + +impl OwnedSlice { + #[inline] + pub fn slice_from_raw_parts(data: NonNull, len: usize) -> NonNull<[T]> { + // SAFETY: `data` is a `NonNull` pointer which is necessarily non-null + unsafe { NonNull::new_unchecked(ptr::slice_from_raw_parts_mut(data.as_ptr(), len)) } + } + + pub fn leak(place: Box<[T]>) -> Self { + let leak = NonNull::from(Box::leak(place)); + OwnedSlice { + // ptr: leak.as_non_null_ptr(), + ptr: leak.cast(), + len: leak.len(), + _marker: PhantomData, + } + } + + pub fn as_slice(&self) -> &[T] { + let slice = Self::slice_from_raw_parts(self.ptr, self.len); + // SAFETY: `Self` is opaque we create Box and we drop it + unsafe { slice.as_ref() } + } + + /// # Safety + /// forget `self` after `.keep_own` + pub unsafe fn keep_own(&self) -> Box<[T]> { + let slice = Self::slice_from_raw_parts(self.ptr, self.len); + unsafe { Box::from_raw(slice.as_ptr()) } + } + + pub fn into_owned(self) -> Box<[T]> { + // SAFETY: `self` drop after call `.into_owned` + unsafe { self.keep_own() } + } +} + +impl Debug for OwnedSlice { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_slice()) + } +} + +impl Drop for OwnedSlice { + fn drop(&mut self) { + // SAFETY: `self` drop at end of this scope + let _ = unsafe { self.keep_own() }; + } +} From 3bf0a7e6c1c5855a3ed7fa7907bc7acbbee62539 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 17:27:42 +0300 Subject: [PATCH 65/76] Fix priv `utils` mod --- .../examples/rust/examples/error-handling.rs | 25 ++++++++----------- doublets-ffi/src/lib.rs | 2 +- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index f035bba..0336919 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -8,6 +8,7 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, + utils::Fallible, FFICallbackContext, }; use std::{ @@ -27,32 +28,24 @@ extern "C" fn create_cb(_: FFICallbackContext, _: Link, _: Link) -> Fl fn main() { unsafe { let log_handle = - doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, false); + doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, true); let path = CString::new("doublets.links").unwrap(); - let result = - create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())); - let mut handle = match result { - DoubletsError::Handle(handle) => handle, - DoubletsError::Continue | DoubletsError::Break => unreachable!(), - error => { - panic!("unexpected error: {}", error) - } - }; + let mut handle = + create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())) + .unwrap(); let any = constants_from_store::(&handle).any; let query = [1 /* not exists index */, any, any]; let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); - if let DoubletsError::Continue | DoubletsError::Break = result { - unreachable!() - } else { + if let Fallible::Err(error) = result { let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); let mut msg_buf = vec![0u8; 256]; - read_error::(msg_buf.as_mut_ptr().cast(), 256, &result); + read_error::(msg_buf.as_mut_ptr().cast(), 256, &error); msg_buf.drain(memchr(&msg_buf) + 1..); @@ -60,7 +53,9 @@ fn main() { let str = cstring.to_str().unwrap(); tracing::error!("{}", str); - free_error::(result); + free_error::(error); + } else { + unreachable!() } free_store::(handle); diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index 4dc69f3..a2e7913 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -5,7 +5,7 @@ pub mod errors; pub mod export; pub mod logging; pub mod store; -mod utils; +pub mod utils; pub(crate) use utils::stable_try; From 0e71c0e6404820372b60e2710cf2a14f70dc21a1 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 17:31:44 +0300 Subject: [PATCH 66/76] Disable clippy with bug --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b586808..9f5228e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,8 @@ jobs: needs: - test - miri - - clippy + # temporary ignore bug in clippy + # - clippy steps: - run: exit 0 From 8433376a66688c55ef475ef18f82b6b78cf2172f Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 17:36:40 +0300 Subject: [PATCH 67/76] Fix fallible example --- doublets-ffi/examples/rust/examples/doublets-context.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index 19dab53..a747f45 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -47,7 +47,8 @@ fn main() { let mut store = doublets_create_united_store_u64( path.as_ptr(), Constants::from(LinksConstants::external()), - ); + ) + .unwrap(); magic_create(&mut store, |before, after| { print!("{before:?}\n{after:?}\n"); From a5156eda8ff2a4416a31ff7c7926b676ec832060 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 17:43:28 +0300 Subject: [PATCH 68/76] Really disable clippy --- .github/workflows/ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5228e..759ce96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,22 +52,22 @@ jobs: - name: Run tests run: cargo test ${{ matrix.features }} --release -- --nocapture - clippy: - name: clippy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install Rust clippy - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true - components: clippy - - uses: Swatinem/rust-cache@v1 - - name: "clippy --all" - run: cargo clippy --all --tests --all-features + # clippy: + # name: clippy + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # submodules: true + # - name: Install Rust clippy + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: nightly + # override: true + # components: clippy + # - uses: Swatinem/rust-cache@v1 + # - name: "clippy --all" + # run: cargo clippy --all --tests --all-features miri: runs-on: ubuntu-latest From a95d580b6d7eaa26f9fbf80a7999ffc354a459c9 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 20:22:48 +0300 Subject: [PATCH 69/76] Use shortest names --- doublets-ffi/examples/rust/examples/all-logs.rs | 4 ++-- .../examples/rust/examples/doublets-context.rs | 6 +++--- .../examples/rust/examples/error-handling.rs | 6 +++--- doublets-ffi/examples/rust/examples/log-context.rs | 4 ++-- doublets-ffi/src/export.rs | 4 ++-- doublets-ffi/src/lib.rs | 8 ++++---- doublets-ffi/src/logging.rs | 12 ++++++------ doublets-ffi/src/store.rs | 14 +++++++------- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/all-logs.rs b/doublets-ffi/examples/rust/examples/all-logs.rs index 213a020..93efdc0 100644 --- a/doublets-ffi/examples/rust/examples/all-logs.rs +++ b/doublets-ffi/examples/rust/examples/all-logs.rs @@ -3,14 +3,14 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, - FFICallbackContext, + FFIContext, }; use std::{ ffi::{c_char, CStr}, ptr, }; -unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { +unsafe extern "C" fn callback(_: FFIContext, ptr: *const c_char) { let cstr = CStr::from_ptr(ptr); print!("{}", cstr.to_str().unwrap()); } diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index a747f45..c6e42ab 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -7,7 +7,7 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{create, doublets_create_united_store_u64, StoreHandle}, - FFICallbackContext, + FFIContext, }; use std::{ ffi::{c_char, CStr, CString}, @@ -15,11 +15,11 @@ use std::{ ptr::{null, null_mut}, }; -unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { +unsafe extern "C" fn callback(_: FFIContext, ptr: *const c_char) { print!("{}", CStr::from_ptr(ptr).to_str().unwrap()); } -extern "C" fn create_cb(ctx: FFICallbackContext, before: Link, after: Link) -> Flow +extern "C" fn create_cb(ctx: FFIContext, before: Link, after: Link) -> Flow where F: FnMut(Link, Link), { diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 0336919..7d9b382 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -9,7 +9,7 @@ use doublets_ffi::{ logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, utils::Fallible, - FFICallbackContext, + FFIContext, }; use std::{ ffi::{c_char, CStr, CString}, @@ -17,11 +17,11 @@ use std::{ ptr::null_mut, }; -unsafe extern "C" fn callback(_: FFICallbackContext, ptr: *const c_char) { +unsafe extern "C" fn callback(_: FFIContext, ptr: *const c_char) { print!("{}", CStr::from_ptr(ptr).to_str().unwrap()); } -extern "C" fn create_cb(_: FFICallbackContext, _: Link, _: Link) -> Flow { +extern "C" fn create_cb(_: FFIContext, _: Link, _: Link) -> Flow { Flow::Continue } diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index 657cac4..8ba652d 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -3,7 +3,7 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, - FFICallbackContext, + FFIContext, }; use std::{ ffi::{c_char, CStr, CString}, @@ -11,7 +11,7 @@ use std::{ }; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; -unsafe extern "C" fn callback(ctx: FFICallbackContext, ptr: *const c_char) { +unsafe extern "C" fn callback(ctx: FFIContext, ptr: *const c_char) { let str = CStr::from_ptr(ptr).to_str().unwrap(); let ctx = &mut *(ctx as *mut usize); diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index f769989..eccc6a8 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -1,6 +1,6 @@ use crate::{ logging::{DoubletsFFILogHandle, Format, Level, LogFFICallback}, - FFICallbackContext, + FFIContext, }; use tracing::error; @@ -15,7 +15,7 @@ pub extern "C" fn doublets_activate_env_logger() { #[no_mangle] pub unsafe extern "C" fn doublets_create_log_handle( - ctx: FFICallbackContext, + ctx: FFIContext, callback: LogFFICallback, max_level: Level, format: Format, diff --git a/doublets-ffi/src/lib.rs b/doublets-ffi/src/lib.rs index a2e7913..dcd64b9 100644 --- a/doublets-ffi/src/lib.rs +++ b/doublets-ffi/src/lib.rs @@ -17,7 +17,7 @@ type c_void = std::ffi::c_void; #[allow(non_camel_case_types)] type c_char = std::ffi::c_char; -pub type FFICallbackContext = *mut c_void; +pub type FFIContext = *mut c_void; /// [`Send`] and [`Sync`] wrapper on [`FFICallbackContext`] /// @@ -26,8 +26,8 @@ pub type FFICallbackContext = *mut c_void; /// /// Otherwise value not use in multithreading context #[derive(Clone, Copy)] -pub struct FFICallbackContextWrapper(FFICallbackContext); +pub struct FFIContextWrapper(FFIContext); /// Guarantee by caller side -unsafe impl Send for FFICallbackContextWrapper {} -unsafe impl Sync for FFICallbackContextWrapper {} +unsafe impl Send for FFIContextWrapper {} +unsafe impl Sync for FFIContextWrapper {} diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index d7f4900..7d8ce3e 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -1,5 +1,5 @@ -use super::{c_char, FFICallbackContext}; -use crate::FFICallbackContextWrapper; +use super::{c_char, FFIContext}; +use crate::FFIContextWrapper; use crossbeam_channel::{self as mpsc, Sender}; use std::{ffi::CString, io, thread}; use tap::Pipe; @@ -43,7 +43,7 @@ impl MakeWriter<'_> for ChannelWriter { /// # Safety /// This callback is safe if all the rules of Rust are followed -pub type LogFFICallback = unsafe extern "C" fn(FFICallbackContext, *const c_char); +pub type LogFFICallback = unsafe extern "C" fn(FFIContext, *const c_char); #[repr(usize)] pub enum Level { @@ -66,17 +66,17 @@ pub struct DoubletsFFILogHandle; impl DoubletsFFILogHandle { pub fn new( - ctx: FFICallbackContext, + ctx: FFIContext, callback: LogFFICallback, max_level: Level, format: Format, ansi: bool, ) -> Self { log_panics::init(); - let wrapper = FFICallbackContextWrapper(ctx); + let wrapper = FFIContextWrapper(ctx); let (sender, receiver) = mpsc::bounded(256); - let callback = move |ctx: FFICallbackContextWrapper, ptr| { + let callback = move |ctx: FFIContextWrapper, ptr| { // SAFETY: caller must guarantee - we only delegate callback unsafe { callback(ctx.0, ptr); diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index d38c8a5..62586ea 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -6,7 +6,7 @@ use crate::{ errors::DoubletsError, stable_try as tri, utils::{Fallible, Maybe, OwnedSlice}, - FFICallbackContext, + FFIContext, }; use doublets::{ data::{Flow, LinkType}, @@ -24,9 +24,9 @@ use tracing::{debug, warn}; type UnitedLinks = unit::Store>>; -type EachCallback = extern "C" fn(FFICallbackContext, Link) -> Flow; +type EachCallback = extern "C" fn(FFIContext, Link) -> Flow; -type CUDCallback = extern "C" fn(FFICallbackContext, Link, Link) -> Flow; +type CUDCallback = extern "C" fn(FFIContext, Link, Link) -> Flow; pub struct StoreHandle { pointer: Box>, @@ -204,7 +204,7 @@ pub unsafe extern "C" fn create( handle: &mut StoreHandle, query: *const T, len: u32, - ctx: FFICallbackContext, + ctx: FFIContext, callback: CUDCallback, ) -> Fallible> { let query = query_from_raw(query, len); @@ -239,7 +239,7 @@ pub unsafe extern "C" fn each( handle: &StoreHandle, query: *const T, len: u32, - ctx: FFICallbackContext, + ctx: FFIContext, callback: EachCallback, ) -> Flow { let query = query_from_raw(query, len); @@ -306,7 +306,7 @@ pub unsafe extern "C" fn update( len_q: u32, change: *const T, len_c: u32, - ctx: FFICallbackContext, + ctx: FFIContext, callback: CUDCallback, ) -> Fallible> { let handler = move |before, after| callback(ctx, before, after); @@ -342,7 +342,7 @@ pub unsafe extern "C" fn delete( handle: &mut StoreHandle, query: *const T, len: u32, - ctx: FFICallbackContext, + ctx: FFIContext, callback: CUDCallback, ) -> Fallible> { let handler = move |before, after| callback(ctx, before, after); From fb70b9ca01d977ad3065fc246aee7a05b413887c Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 20:36:48 +0300 Subject: [PATCH 70/76] Improve code for future readers --- .../rust/examples/doublets-context.rs | 10 ++++----- .../examples/rust/examples/error-handling.rs | 21 ++++++++++++------- .../examples/rust/examples/log-context.rs | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/doublets-context.rs b/doublets-ffi/examples/rust/examples/doublets-context.rs index c6e42ab..44995aa 100644 --- a/doublets-ffi/examples/rust/examples/doublets-context.rs +++ b/doublets-ffi/examples/rust/examples/doublets-context.rs @@ -23,11 +23,9 @@ extern "C" fn create_cb(ctx: FFIContext, before: Link, after: Link) where F: FnMut(Link, Link), { - unsafe { - let handler = &mut *(ctx as *mut F); - (*handler)(before, after); - Flow::Continue - } + let handler = unsafe { &mut *(ctx as *mut F) }; + (*handler)(before, after); + Flow::Continue } unsafe fn magic_create(handle: &mut StoreHandle, mut handler: F) @@ -41,7 +39,7 @@ where fn main() { unsafe { let handle = - doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, false); + doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, true); let path = CString::new("doublets.links").unwrap(); let mut store = doublets_create_united_store_u64( diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 7d9b382..aace1c4 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -1,3 +1,5 @@ +#![feature(cstr_from_bytes_until_nul)] + use doublets::{ data::{Flow, LinksConstants}, Link, @@ -12,6 +14,7 @@ use doublets_ffi::{ FFIContext, }; use std::{ + error::Error, ffi::{c_char, CStr, CString}, fs, ptr::null_mut, @@ -25,12 +28,12 @@ extern "C" fn create_cb(_: FFIContext, _: Link, _: Link) -> Flow { Flow::Continue } -fn main() { +fn main() -> Result<(), Box> { unsafe { let log_handle = doublets_create_log_handle(null_mut(), callback, Level::Trace, Format::Virgin, true); - let path = CString::new("doublets.links").unwrap(); + let path = CStr::from_bytes_until_nul(b"doublets.links\0")?; let mut handle = create_unit_store::(path.as_ptr(), Constants::from(LinksConstants::external())) .unwrap(); @@ -41,16 +44,16 @@ fn main() { let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); if let Fallible::Err(error) = result { - let memchr = |buf: &[u8]| buf.iter().position(|x| *x == 0).unwrap(); + let memchr = |buf: &[u8]| { + buf.iter() + .position(|x| *x == 0) + .expect("it is unreachable: C strings every has \\'0' at the end") + }; let mut msg_buf = vec![0u8; 256]; - read_error::(msg_buf.as_mut_ptr().cast(), 256, &error); - msg_buf.drain(memchr(&msg_buf) + 1..); - - let cstring = CString::from_vec_with_nul(msg_buf).unwrap(); - let str = cstring.to_str().unwrap(); + let str = CStr::from_bytes_until_nul(&msg_buf[..=memchr(&msg_buf)])?.to_str()?; tracing::error!("{}", str); free_error::(error); @@ -63,4 +66,6 @@ fn main() { doublets_free_log_handle(log_handle); } let _ = fs::remove_file("doublets.links"); + + Ok(()) } diff --git a/doublets-ffi/examples/rust/examples/log-context.rs b/doublets-ffi/examples/rust/examples/log-context.rs index 8ba652d..96a9983 100644 --- a/doublets-ffi/examples/rust/examples/log-context.rs +++ b/doublets-ffi/examples/rust/examples/log-context.rs @@ -6,7 +6,7 @@ use doublets_ffi::{ FFIContext, }; use std::{ - ffi::{c_char, CStr, CString}, + ffi::{c_char, CStr}, io::{self, Write}, }; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; From 9042ce8cc7af727e5bac77b7aabc215995bd818f Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Fri, 9 Sep 2022 20:38:40 +0300 Subject: [PATCH 71/76] Improve code for future readers - two-step --- doublets-ffi/examples/rust/examples/error-handling.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index aace1c4..84cc2a6 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -44,16 +44,10 @@ fn main() -> Result<(), Box> { let result = delete::(&mut handle, query.as_ptr(), 3, null_mut(), create_cb); if let Fallible::Err(error) = result { - let memchr = |buf: &[u8]| { - buf.iter() - .position(|x| *x == 0) - .expect("it is unreachable: C strings every has \\'0' at the end") - }; - let mut msg_buf = vec![0u8; 256]; read_error::(msg_buf.as_mut_ptr().cast(), 256, &error); - let str = CStr::from_bytes_until_nul(&msg_buf[..=memchr(&msg_buf)])?.to_str()?; + let str = CStr::from_bytes_until_nul(&msg_buf)?.to_str()?; tracing::error!("{}", str); free_error::(error); From 3754990eb64f6857b055f69ce76fbf90430d0a40 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sat, 10 Sep 2022 22:55:58 +0300 Subject: [PATCH 72/76] Add `cbindgen` config for cxx (but with problems): - `Fallible` can't drop without generics (can impl manually dropping by C++) --- cbindgen-cxx.toml | 112 ++++++++++++++++++++++++++++++++++++++ doublets-ffi/src/utils.rs | 2 +- 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 cbindgen-cxx.toml diff --git a/cbindgen-cxx.toml b/cbindgen-cxx.toml new file mode 100644 index 0000000..4422c13 --- /dev/null +++ b/cbindgen-cxx.toml @@ -0,0 +1,112 @@ +language = "C++" + +############## Options for Wrapping the Contents of the Header ################# + +# header = "/* Text to put at the beginning of the generated file. Probably a license. */" +# trailer = "/* Text to put at the end of the generated file */" +# include_guard = "my_bindings_h" +pragma_once = true +# autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */" +include_version = false +# namespace = "my_namespace" +namespaces = [] +using_namespaces = [] +sys_includes = [] +includes = [] +no_includes = false +after_includes = """ + +template +using Box = T*; + +template +using ManuallyDrop = T; + +""" + +braces = "SameLine" +line_length = 100 +tab_width = 2 +documentation = true +documentation_style = "auto" +documentation_length = "full" +line_endings = "LF" # also "CR", "CRLF", "Native" + +style = "both" +sort_by = "Name" # default for `fn.sort_by` and `const.sort_by` +usize_is_size_t = true + +[defines] +unstable_backtrace = "UNSTABLE_BACKTRACE" +# "target_os = freebsd" = "DEFINE_FREEBSD" +# "feature = serde" = "DEFINE_SERDE" + +[export] +include = [] +exclude = [ + "Box", "ManuallyDrop" +] +# prefix = "CAPI_" +item_types = [] +renaming_overrides_prefixing = false + +[export.rename] + +[export.body] + +[export.mangle] + +[fn] +rename_args = "None" +# must_use = "MUST_USE_FUNC" +# no_return = "NO_RETURN" +# prefix = "START_FUNC" +# postfix = "END_FUNC" +args = "auto" +sort_by = "Name" + +[struct] +rename_fields = "None" +# must_use = "MUST_USE_STRUCT" +derive_constructor = false +derive_eq = false +derive_neq = false +derive_lt = false +derive_lte = false +derive_gt = false +derive_gte = false + +[enum] +rename_variants = "None" +# must_use = "MUST_USE_ENUM" +add_sentinel = false +prefix_with_name = false +derive_helper_methods = false +derive_const_casts = false +derive_mut_casts = false +# cast_assert_name = "ASSERT" +derive_tagged_enum_destructor = false +derive_tagged_enum_copy_constructor = false +enum_class = true +private_default_tagged_enum_constructor = false + +[const] +allow_static_const = true +allow_constexpr = true +sort_by = "Name" + +[macro_expansion] +# bitflags = false + +[parse] +parse_deps = true +include = ["doublets", "platform-data"] +exclude = [] +clean = false +extra_bindings = [] + +[parse.expand] +crates = ["doublets-ffi"] +all_features = true +default_features = true +features = [] \ No newline at end of file diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs index 6cbc11a..1c1bb2e 100644 --- a/doublets-ffi/src/utils.rs +++ b/doublets-ffi/src/utils.rs @@ -24,7 +24,7 @@ use std::{ #[repr(transparent)] #[derive(Default)] pub struct Void { - _nonzero: u8, + _nonzero: [u8; 0], } #[repr(C, usize)] From 3b272d9b8123d7f1ef8af4da1176290b5c53a1ab Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Mon, 12 Sep 2022 20:03:17 +0300 Subject: [PATCH 73/76] Cosmetic improvements --- doublets-ffi/src/errors.rs | 8 ++------ doublets-ffi/src/export.rs | 10 ---------- doublets-ffi/src/logging.rs | 8 ++++++-- doublets-ffi/src/store.rs | 2 +- doublets-ffi/src/utils.rs | 10 +++------- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index c108ef4..a5012a3 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -19,19 +19,15 @@ pub enum DoubletsError { LimitReached(T), HasUsages(OwnedSlice>), AlreadyExists(Doublet), - AllocFailed(Box), + AllocFailed(Box), Other(Box), } impl DoubletsError { #[cfg(unstable_backtrace)] fn backtrace(&self) -> Option<&Backtrace> { - fn erasure(err: &mem::Error) -> &dyn error::Error { - err as _ - } - match self { - DoubletsError::AllocFailed(err) => erasure(err).request_ref(), + DoubletsError::AllocFailed(err) => err.request_ref(), DoubletsError::Other(err) => err.request_ref(), _ => None, } diff --git a/doublets-ffi/src/export.rs b/doublets-ffi/src/export.rs index eccc6a8..bbe7970 100644 --- a/doublets-ffi/src/export.rs +++ b/doublets-ffi/src/export.rs @@ -3,16 +3,6 @@ use crate::{ FFIContext, }; -use tracing::error; - -/// Basic logger. For advanced use [`doublets_create_log_handle`] -#[no_mangle] -pub extern "C" fn doublets_activate_env_logger() { - if tracing_subscriber::fmt::try_init().is_err() { - error!("Cannot re-init env logger, this should only be called once"); - } -} - #[no_mangle] pub unsafe extern "C" fn doublets_create_log_handle( ctx: FFIContext, diff --git a/doublets-ffi/src/logging.rs b/doublets-ffi/src/logging.rs index 7d8ce3e..fe2453e 100644 --- a/doublets-ffi/src/logging.rs +++ b/doublets-ffi/src/logging.rs @@ -62,7 +62,9 @@ pub enum Format { Json, } -pub struct DoubletsFFILogHandle; +pub struct DoubletsFFILogHandle { + _non_exhaustive: (), +} impl DoubletsFFILogHandle { pub fn new( @@ -134,6 +136,8 @@ impl DoubletsFFILogHandle { ); }; - Self + Self { + _non_exhaustive: (), + } } } diff --git a/doublets-ffi/src/store.rs b/doublets-ffi/src/store.rs index 62586ea..2340dba 100644 --- a/doublets-ffi/src/store.rs +++ b/doublets-ffi/src/store.rs @@ -92,7 +92,7 @@ impl From> for DoubletsError { Error::LimitReached(limit) => Self::LimitReached(limit), // these errors are difficult to handle as data // I hope no one will be offended if we alloc them at the heap - Error::AllocFailed(alloc) => Self::AllocFailed(Box::new(alloc)), + Error::AllocFailed(alloc) => Self::AllocFailed(Box::new(Box::new(alloc))), Error::Other(other) => Self::Other(Box::new(other)), } } diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs index 1c1bb2e..470aba4 100644 --- a/doublets-ffi/src/utils.rs +++ b/doublets-ffi/src/utils.rs @@ -15,13 +15,9 @@ use std::{ ptr::{self, NonNull}, }; -/// С/C++ not allow zero-sized types. `u8` it's a repr of (in C/C++) -/// with `sizeof == 1` and `alignof == 1`: -/// -/// -/// struct Void {}; -/// -#[repr(transparent)] +/// repr C type unit type +/// with `sizeof == 0` and `alignof == 1`: +#[repr(C)] #[derive(Default)] pub struct Void { _nonzero: [u8; 0], From fb2aad6eec5b2ef2016f60717151943494de4b85 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 27 Nov 2022 17:24:01 +0300 Subject: [PATCH 74/76] Remove unused imports --- doublets-ffi/examples/rust/examples/error-handling.rs | 4 ++-- doublets-ffi/ffi-attributes/src/lib.rs | 1 - doublets-ffi/src/errors.rs | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 84cc2a6..dc1e392 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -6,7 +6,7 @@ use doublets::{ }; use doublets_ffi::{ constants::Constants, - errors::{free_error, read_error, DoubletsError}, + errors::{free_error, read_error}, export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, @@ -15,7 +15,7 @@ use doublets_ffi::{ }; use std::{ error::Error, - ffi::{c_char, CStr, CString}, + ffi::{c_char, CStr}, fs, ptr::null_mut, }; diff --git a/doublets-ffi/ffi-attributes/src/lib.rs b/doublets-ffi/ffi-attributes/src/lib.rs index 8e047d2..4547f3c 100644 --- a/doublets-ffi/ffi-attributes/src/lib.rs +++ b/doublets-ffi/ffi-attributes/src/lib.rs @@ -8,7 +8,6 @@ mod prepare; use proc_macro::{Level, Span}; use std::collections::HashMap; - use syn::{ parse::{Parse, ParseStream}, parse_macro_input, diff --git a/doublets-ffi/src/errors.rs b/doublets-ffi/src/errors.rs index a5012a3..120f7cc 100644 --- a/doublets-ffi/src/errors.rs +++ b/doublets-ffi/src/errors.rs @@ -1,5 +1,5 @@ use crate::c_char; -use doublets::{data::LinkType, mem, Doublet, Link}; +use doublets::{data::LinkType, Doublet, Link}; #[cfg(unstable_backtrace)] use std::backtrace::Backtrace; use std::{ @@ -8,7 +8,6 @@ use std::{ fmt::{self, Debug, Display, Formatter}, ptr, }; -use tracing::warn; type OpaqueError = Box; From ce9449a53f3ddd1e11d6dbbb61f9723e5771f4cb Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 27 Nov 2022 17:31:45 +0300 Subject: [PATCH 75/76] Improve `Fallible` messages and idioms --- .../examples/rust/examples/error-handling.rs | 2 +- doublets-ffi/src/utils.rs | 30 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index dc1e392..01fc463 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -10,7 +10,7 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, - utils::Fallible, + utils::{Fallible, Maybe}, FFIContext, }; use std::{ diff --git a/doublets-ffi/src/utils.rs b/doublets-ffi/src/utils.rs index 470aba4..cc2671d 100644 --- a/doublets-ffi/src/utils.rs +++ b/doublets-ffi/src/utils.rs @@ -18,32 +18,44 @@ use std::{ /// repr C type unit type /// with `sizeof == 0` and `alignof == 1`: #[repr(C)] -#[derive(Default)] -pub struct Void { +pub struct None { _nonzero: [u8; 0], } +impl None { + pub fn new() -> Self { + Self { _nonzero: [] } + } +} + +impl Debug for None { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "None value") + } +} + #[repr(C, usize)] pub enum Fallible { Ok(T), Err(E), } -impl Fallible { +impl Fallible { pub fn unwrap(self) -> T { - if let Self::Ok(val) = self { - val - } else { - panic!("called `Fallible::unwrap()` on a `Err` value") + match self { + Self::Ok(val) => val, + Self::Err(err) => { + panic!("called `Fallible::unwrap()` on a `Err` value: {err:?}") + } } } } -pub type Maybe = Fallible; +pub type Maybe = Fallible; impl Maybe { pub fn none() -> Self { - Self::Err(Void::default()) + Self::Err(None::new()) } } From 97e12b29be683b9b1948d2c19ca752a654cca702 Mon Sep 17 00:00:00 2001 From: uselessgoddess Date: Sun, 27 Nov 2022 17:32:14 +0300 Subject: [PATCH 76/76] Remove type - useless import --- doublets-ffi/examples/rust/examples/error-handling.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doublets-ffi/examples/rust/examples/error-handling.rs b/doublets-ffi/examples/rust/examples/error-handling.rs index 01fc463..dc1e392 100644 --- a/doublets-ffi/examples/rust/examples/error-handling.rs +++ b/doublets-ffi/examples/rust/examples/error-handling.rs @@ -10,7 +10,7 @@ use doublets_ffi::{ export::{doublets_create_log_handle, doublets_free_log_handle}, logging::{Format, Level}, store::{constants_from_store, create_unit_store, delete, free_store}, - utils::{Fallible, Maybe}, + utils::Fallible, FFIContext, }; use std::{