Refactoring, docs, tests, v0.6 (#22)

* moved all rust code to `rust` mod
* moved cpp code to `cpp` mod
* both are now optional
* docs and an example for native
* use main C++ repo
* added myself to authors (hope it's ok)
* add `--all-features` to all tests

Author: nyurik

.github/workflows/ci.yml

@@ -17,21 +17,18 @@ jobs:
         with: { tool: just }
       - uses: actions/checkout@v4
         with: { submodules: recursive }
-#
-# Uncomment this and below once the crate has been published to crates.io
-#
-#      - name: Ensure this crate has not yet been published (on release)
-#        if: github.event_name == 'release' && matrix.type == 'latest'
-#        run: just check-if-published
+      - name: Ensure this crate has not yet been published (on release)
+        if: github.event_name == 'release' && matrix.type == 'latest'
+        run: just check-if-published
       - uses: Swatinem/rust-cache@v2
         if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
       - run: just -v ci-test
-#      - name: Check semver
-#        uses: obi1kenobi/cargo-semver-checks-action@v2
-#      - name: Test packaging for publish
-#        run: |
-#          rustup install nightly --profile minimal
-#          just test-publish
+      - name: Check semver
+        uses: obi1kenobi/cargo-semver-checks-action@v2
+      - name: Test packaging for publish
+        run: |
+          rustup install nightly --profile minimal
+          just test-publish
 
   msrv:
     name: Test MSRV

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "cpp"]
 	path = cpp
-	url = https://github.com/nyurik/FastPFor-cpp.git
+	url = https://github.com/fast-pack/FastPFOR.git

Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "fastpfor"
-version = "0.5.0"
+version = "0.6.0"
 description = "FastPFOR library written in Rust."
-authors = ["Francisco Jimenez <jjcfrank@gmail.com>"]
+authors = ["Francisco Jimenez <jjcfrank@gmail.com>", "Yuri Astrakhan <YuriAstrakhan@gmail.com>"]
 repository = "https://github.com/jjcfrancisco/fastpfor"
 edition = "2021"
 license = "MIT OR Apache-2.0"
@@ -11,25 +11,27 @@ categories = ["compression"]
 rust-version = "1.83.0"
 
 [features]
-# We probably want to build without the C++ bindings by default.
+# Eventually we may want to build without the C++ bindings by default.
 # Keeping it on for now to simplify development.
 default = ["cpp"]
 cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"]
+rust = ["dep:thiserror"]
 
 [dependencies]
-rand = "0.8.5"
-thiserror = "2.0.7"
 cxx = { version = "1.0.136", optional = true }
+thiserror = { version = "2.0.7" , optional = true}
+
+[dev-dependencies]
+rand = "0.9.0"
 
 [build-dependencies]
 cmake = { version = "0.1.52", optional = true }
 cxx-build = { version = "1.0.136", optional = true }
 
 [lints.rust]
-unused_assignments = "allow"
 dead_code = "allow"
+unused_assignments = "allow"
+unused_qualifications = "warn"
 
 [lints.clippy]
-unnecessary_cast = "allow"
-ptr_arg = "allow"
 needless_range_loop = "allow"

README.md

@@ -1,31 +1,101 @@
-# FastPFor
+# FastPFor for Rust
 
 [![GitHub](https://img.shields.io/badge/github-fastpfor-8da0cb?logo=github)](https://github.com/jjcfrancisco/fastpfor)
 [![crates.io version](https://img.shields.io/crates/v/fastpfor.svg)](https://crates.io/crates/fastpfor)
 [![docs.rs docs](https://docs.rs/fastpfor/badge.svg)](https://docs.rs/fastpfor)
-[![crates.io version](https://img.shields.io/crates/l/fastpfor.svg)](https://github.com/jjcfrancisco/fastpfor/blob/main/LICENSE-APACHE)
+[![license](https://img.shields.io/crates/l/fastpfor.svg)](https://github.com/jjcfrancisco/fastpfor/blob/main/LICENSE-APACHE)
 [![CI build](https://github.com/jjcfrancisco/fastpfor/actions/workflows/ci.yml/badge.svg)](https://github.com/jjcfrancisco/fastpfor/actions)
 
-FastPFOR library written in Rust.
+This is a Rust wrapper for the [C++ FastPFor library](https://github.com/fast-pack/FastPFor), as well as a pure Rust re-implementation (work in progress).  Supports 32-bit and 64-bit integers, and SIMD-optimized codecs for 128-bit and 256-bit vectors. Based on the [Decoding billions of integers per second through vectorization, 2012](https://arxiv.org/abs/1209.2137) paper.
 
-## Requirements
+### Supported algorithms
+Unless otherwise specified, all codecs support `&[u32]` only. 
+* BP32
+* Copy
+* FastBinaryPacking8
+* FastPFor128 (both `&[u32]` and `&[u64]`)
+* FastPFor256 (both `&[u32]` and `&[u64]`)
+* FastBinaryPacking16
+* FastBinaryPacking32
+* MaskedVByte
+* NewPFor
+* OptPFor
+* PFor2008
+* PFor
+* SimdBinaryPacking
+* SimdFastPFor128
+* SimdFastPFor256
+* SimdGroupSimple
+* SimdGroupSimpleRingBuf
+* SimdNewPFor
+* SimdOptPFor
+* SimdPFor
+* SimdSimplePFor
+* Simple16
+* Simple8b
+* Simple8bRle
+* Simple9
+* Simple9Rle
+* SimplePFor
+* StreamVByte
+* VByte
+* VarInt (both `&[u32]` and `&[u64]`)
+* VarIntGb
 
-# MacOS
-To build FastPFor on macOS, you'll need to install SIMDe. Since Homebrew installs packages in /opt/homebrew (for Apple Silicon), you'll also need to explicitly set the include paths.
+## Usage
+
+### Crate Features
+* `cpp` - C++ implementation (default)
+* `rust` - Rust implementation (work in progress, opt-in)
+
+### Using C++ Wrapper
+
+```rust
+use fastpfor::cpp::{Codec32 as _, SimdFastPFor128Codec};
+
+fn main() {
+  let mut codec = SimdFastPFor128Codec::new();
+
+  // Encode
+  let mut input = vec![1, 2, 3, 4, 5];
+  let mut output = vec![0; 10];  // must be large enough
+  let enc_slice = codec.encode32(&input, &mut output).unwrap();
+
+  // Decode
+  let mut decoded = vec![0; 10]; // must be large enough
+  let dec_slice = codec.decode32(&enc_slice, &mut decoded).unwrap();
+
+  assert_eq!(input, dec_slice);
+}
+```
+
+## Build Requirements
+
+When using the Rust implementation, no additional dependencies are required.
+When using the C++ implementation, you need to have a C++ compiler that supports C++14 and SIMD intrinsics. See [FastPFor C++ requirements](https://github.com/fast-pack/FastPFor?tab=readme-ov-file#software-requirements).
+
+### Linux
+The default GitHub action runner for Linux has all the needed dependencies. For local development, you may need to install the following packages.
+
+```bash
+# This list may be incomplete
+sudo apt-get install build-essential libsimde-dev
+```
+
+### macOS
+To build FastPFor on macOS, you'll need to install SIMDe. Since Homebrew installs packages in `/opt/homebrew` (for Apple Silicon), you'll also need to explicitly set the include paths.
 
-First, install SIMDe via Homebrew:
 ```bash
+# install SIMDe via Homebrew
 brew install simde
 ```
-Then, ensure the compiler can find the required headers by setting:
+
 ```bash
+# Ensure the compiler can find the required headers before building
 export CXXFLAGS="-I/opt/homebrew/include"
 export CFLAGS="-I/opt/homebrew/include"
 ```
 
-## To do:
-- [ x ] Rust wrapper for the [C++ library](https://github.com/fast-pack/FastPFor): try [`cxxbridge`](https://github.com/dtolnay/cxx) or [`bindgen`](https://crates.io/crates/bindgen)
-
 ## License
 
 Licensed under either of

build.rs

@@ -1,28 +1,28 @@
-use std::path::Path;
-
-use cmake::Config;
-
 fn main() {
-    if !Path::new("cpp/CMakeLists.txt").exists() {
-        panic!("FastPFOR submodule not initialized. Run `git submodule update --init`.");
-    }
+    #[cfg(feature = "cpp")]
+    {
+        use std::path::Path;
 
-    // Compile FastPFOR using CMake
-    let lib_path = Config::new("cpp").build().join("lib");
-    let lib_path = lib_path.to_str().unwrap();
-    println!("cargo:rerun-if-changed=cpp");
+        if !Path::new("cpp/CMakeLists.txt").exists() {
+            panic!("FastPFOR submodule not initialized. Run `git submodule update --init`.");
+        }
 
-    // Compile the bridge
-    cxx_build::bridge("src/bridge/mod.rs")
-        .include("cpp/headers")
-        .include("src/bridge")
-        .std("c++14")
-        .compile("fastpfor_bridge");
-    println!("cargo:rerun-if-changed=src/ffi.rs");
-    println!("cargo:rerun-if-changed=src/bridge/fastpfor_bridge.cc");
-    println!("cargo:rerun-if-changed=src/bridge/fastpfor_bridge.h");
+        // Compile FastPFOR using CMake
+        println!("cargo:rerun-if-changed=cpp");
+        let lib_path = cmake::Config::new("cpp").build().join("lib");
+        let lib_path = lib_path.to_str().unwrap();
 
-    // Link the FastPFOR library - must be done after the bridge is compiled
-    println!("cargo:rustc-link-search=native={lib_path}");
-    println!("cargo:rustc-link-lib=static=FastPFOR");
+        // Compile the bridge
+        println!("cargo:rerun-if-changed=src/cpp/fastpfor_bridge.h");
+        println!("cargo:rerun-if-changed=src/cpp/mod.rs");
+        cxx_build::bridge("src/cpp/mod.rs")
+            .include("cpp/headers")
+            .include("src/cpp")
+            .std("c++14")
+            .compile("fastpfor_bridge");
+
+        // Link the FastPFOR library - must be done after the bridge is compiled
+        println!("cargo:rustc-link-search=native={lib_path}");
+        println!("cargo:rustc-link-lib=static=FastPFOR");
+    }
 }

cpp

@@ -15,7 +15,7 @@ update:
 
 # Run cargo clippy
 clippy:
-    cargo clippy --all-targets --workspace -- -D warnings
+    cargo clippy --all-targets --workspace --all-features -- -D warnings
 
 # Test code formatting
 test-fmt:
@@ -27,27 +27,27 @@ fmt:
 
 # Build and open code documentation
 docs:
-    cargo doc --no-deps --open
+    cargo doc --no-deps --all-features --open
 
 # Quick compile
 check:
-    cargo check --all-targets --workspace
+    cargo check --all-targets --workspace --all-features
 
 # Default build
 build *ARGS:
-    cargo build --all-targets --workspace {{ARGS}}
+    cargo build --all-targets --workspace --all-features {{ARGS}}
 
 # Run all tests
 test *ARGS: build
-    cargo test --all-targets --workspace {{ARGS}}
+    cargo test --all-targets --workspace --all-features {{ARGS}}
 
 # Find the minimum supported Rust version. Install it with `cargo install cargo-msrv`
 msrv:
-    cargo msrv find --component rustfmt -- {{just_executable()}} ci-test-msrv
+    cargo msrv find --component rustfmt --all-features -- {{just_executable()}} ci-test-msrv
 
 # Find unused dependencies. Install it with `cargo install cargo-udeps`
 udeps:
-    cargo +nightly udeps --all-targets --workspace
+    cargo +nightly udeps --all-targets --workspace --all-features
 
 # Check semver compatibility with prior published version. Install it with `cargo install cargo-semver-checks`
 semver *ARGS:
@@ -72,22 +72,19 @@ test-publish:
 
 # Run tests, and accept their results. Requires insta to be installed.
 bless:
-    TRYBUILD=overwrite cargo insta test --accept
+    TRYBUILD=overwrite cargo insta test --accept --all-features
 
 # Test documentation
 test-doc:
-    cargo test --doc
-    cargo doc --no-deps
+    cargo test --doc --all-features
+    cargo doc --no-deps --all-features
 
 rust-info:
     rustc --version
     cargo --version
 
-# Run tests only relevant to the latest version
-ci-test-extras: test-doc
-
 # Run all tests as expected by CI
-ci-test: rust-info test-fmt clippy test build
+ci-test: rust-info test-fmt clippy test test-doc
 
 # Run minimal subset of tests to ensure compatibility with MSRV
 ci-test-msrv: rust-info test