add some documentation

Author: etseidl

parquet/src/arrow/arrow_reader/mod.rs

@@ -519,6 +519,12 @@ impl ArrowReaderOptions {
     }
 
     /// Set whether to convert `encoding_stats` to a bitmask.
+    ///
+    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for an explanation of why this
+    /// might be desirable.
+    ///
+    /// [`ColumnChunkMetaData::page_encoding_stats_mask`]:
+    /// crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask
     pub fn with_encoding_stats_as_mask(mut self, val: bool) -> Self {
         self.metadata_options.set_encoding_stats_as_mask(val);
         self

parquet/src/file/metadata/mod.rs

@@ -1051,18 +1051,39 @@ impl ColumnChunkMetaData {
         self.geo_statistics.as_deref()
     }
 
-    /// Returns the page encoding stats,
-    /// or `None` if no page encoding stats are available.
+    /// Returns the page encoding statistics, or `None` if no page encoding statistics
+    /// are available.
     pub fn page_encoding_stats(&self) -> Option<&Vec<PageEncodingStats>> {
         self.encoding_stats.as_ref()
     }
 
-    /// Returns the page encoding stats reduced to a bitmask.
+    /// Returns the page encoding statistics reduced to a bitmask, or `None` if statistics are
+    /// not available.
     ///
-    /// Decoding the full page encoding statistics can be costly, and is not always necessary.
-    /// This field contains a mask of all encodings used for data pages. This can still support
-    /// some uses of the full statistics, such as determining if all data pages are dictionary
-    /// encoded.
+    /// The [`PageEncodingStats`] struct was added to the Parquet specification specifically to
+    /// enable fast determination of whether all pages in a column chunk are dictionary encoded
+    /// (see <https://github.com/apache/parquet-format/pull/16>).
+    /// Decoding the full page encoding statistics, however, can be very costly, and is not
+    /// necessary to support the aforementioned use case. As an alternative, this crate can
+    /// instead distill the list of `PageEncodingStats` down to a bitmask of just the encodings
+    /// used for data pages
+    /// (see [`ParquetMetaDataOptions::set_encoding_stats_as_mask`]).
+    /// To test for an all-dictionary-encoded chunk one could use this bitmask in the following way:
+    ///
+    /// ```rust
+    /// use parquet::basic::Encoding;
+    /// use parquet::file::metadata::ColumnChunkMetaData;
+    /// // test if all data pages in the column chunk are dictionary encoded
+    /// fn is_all_dictionary_encoded(col_meta: &ColumnChunkMetaData) -> bool {
+    ///     // check that dictionary encoding was used
+    ///     col_meta.dictionary_page_offset().is_some()
+    ///         && col_meta.page_encoding_stats_mask().is_some_and(|mask| {
+    ///             // mask should only have one bit set, either for PLAIN_DICTIONARY or
+    ///             // RLE_DICTIONARY
+    ///             mask.is_only(Encoding::PLAIN_DICTIONARY) || mask.is_only(Encoding::RLE_DICTIONARY)
+    ///         })
+    /// }
+    /// ```
     pub fn page_encoding_stats_mask(&self) -> Option<&EncodingMask> {
         self.encoding_stats_mask.as_ref()
     }

parquet/src/file/metadata/options.rs

@@ -75,14 +75,24 @@ impl ParquetMetaDataOptions {
 
     /// Returns whether to present the `encoding_stats` field of the `ColumnMetaData` as a
     /// bitmask.
+    ///
+    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for an explanation of why this
+    /// might be desirable.
+    ///
+    /// [`ColumnChunkMetaData::page_encoding_stats_mask`]:
+    /// crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask
     pub fn encoding_stats_as_mask(&self) -> bool {
         self.encoding_stats_as_mask
     }
 
     /// Convert `encoding_stats` from a vector of [`PageEncodingStats`] to a bitmask. This can
     /// speed up metadata decoding while still enabling some use cases served by the full stats.
     ///
+    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for more information.
+    ///
     /// [`PageEncodingStats`]: crate::file::metadata::PageEncodingStats
+    /// [`ColumnChunkMetaData::page_encoding_stats_mask`]:
+    /// crate::file::metadata::ColumnChunkMetaData::page_encoding_stats_mask
     pub fn set_encoding_stats_as_mask(&mut self, val: bool) {
         self.encoding_stats_as_mask = val;
     }
@@ -100,16 +110,20 @@ impl ParquetMetaDataOptions {
             .is_some_and(|oset| oset.as_ref().is_none_or(|keep| !keep.contains(&col_index)))
     }
 
-    /// Skip decoding of all `encoding_stats`. Takes precedence over `encoding_stats_as_mask`.
+    /// Skip decoding of all `encoding_stats`. Takes precedence over
+    /// [`Self::encoding_stats_as_mask`].
     pub fn set_skip_encoding_stats(&mut self, val: bool) {
         self.skip_encoding_stats = if val { Some(None) } else { None };
     }
 
     // with_skip_encoding_stats
     add_mutator!(skip_encoding_stats, bool);
 
-    /// Skip decoding of `encoding_stats`, but decode the stats for those column in
-    /// provided list of column indices.
+    /// Skip decoding of `encoding_stats`, but decode the stats for those columns in
+    /// the provided list of column indices.
+    ///
+    /// This allows for optimizations such as only decoding the page encoding statistics
+    /// for columns present in a predicate.
     pub fn set_keep_encoding_stats(&mut self, keep: &[usize]) {
         if keep.is_empty() {
             self.set_skip_encoding_stats(true);

parquet/src/file/serialized_reader.rs

@@ -161,6 +161,9 @@ impl ReadOptionsBuilder {
     }
 
     /// Set whether to convert `encoding_stats` to a bitmask.
+    ///
+    /// See [`ColumnChunkMetaData::page_encoding_stats_mask`] for an explanation of why this
+    /// might be desirable.
     pub fn with_encoding_stats_as_mask(mut self, val: bool) -> Self {
         self.metadata_options.set_encoding_stats_as_mask(val);
         self