feat: SIMD compilation mode - CPU portable vs native mode selection (#67)

This pull request introduces a new SIMD mode configuration system for
the FastPFOR crate, allowing users to select between portable, native,
and runtime SIMD instruction sets for the C++ backend. The changes
improve build flexibility, CI reliability, and documentation clarity,
while updating the crate version and standardizing feature usage in
development scripts.

### SIMD Mode Configuration and Build System

* Added support for selecting SIMD mode (`portable`, `native`, or
`runtime`) via Cargo features and the `FASTPFOR_SIMD_MODE` environment
variable; the build script now warns if multiple SIMD features are
enabled and passes the selected mode to CMake. (`build.rs`,
`Cargo.toml`)
[[1]](diffhunk://#diff-d0d98998092552a1d3259338c2c71e118a5b8343dd4703c0c7f552ada7f9cb42L16-R59)
[[2]](diffhunk://#diff-2e9d962a08321605940b5a657135052fbcef87b5e360662bb527c96d9a615542R28-R35)
* Updated GitHub Actions CI workflow to test all combinations of OS and
SIMD mode, and to cache builds separately for each SIMD mode.
(`.github/workflows/ci.yml`)

### Documentation Improvements

* Expanded `README.md` with a clear explanation of SIMD modes, their use
cases, and configuration recommendations for end users.

### Development Scripts and Testing

* Standardized the use of the `_all_compatible` feature group in the
`justfile` for building, testing, linting, and coverage, and added
recipes to test all SIMD modes.
[[1]](diffhunk://#diff-deb9bb56fb122db0b605aa5b63f95a4665c905b18dd670e1fa6c877576a94ff1L4-R30)
[[2]](diffhunk://#diff-deb9bb56fb122db0b605aa5b63f95a4665c905b18dd670e1fa6c877576a94ff1L51-R65)
[[3]](diffhunk://#diff-deb9bb56fb122db0b605aa5b63f95a4665c905b18dd670e1fa6c877576a94ff1L95-R130)
[[4]](diffhunk://#diff-deb9bb56fb122db0b605aa5b63f95a4665c905b18dd670e1fa6c877576a94ff1L130-R145)
* Improved installation logic in the `justfile` for development tools,
with clearer output and better CI compatibility.

### Version Update

* Bumped the crate version from `0.7.0` to `0.8.0` to reflect these
breaking and feature enhancements. (`Cargo.toml`)

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: nyurik

.github/workflows/ci.yml

@@ -13,19 +13,22 @@ defaults:
 
 jobs:
   test:
-    name: Test on ${{ matrix.os }}
+    name: Test ${{ matrix.os }} (${{ matrix.simd_mode }})
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-          - os: macos-latest
+        os: [ubuntu-latest, macos-latest]
+        simd_mode: [portable, native]
+    env:
+      FASTPFOR_SIMD_MODE: ${{ matrix.simd_mode }}
     steps:
       - uses: actions/checkout@v6
         with: { submodules: recursive }
       - if: github.event_name != 'release' && github.event_name != 'workflow_dispatch'
         uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v0-${{ matrix.simd_mode }}"
       - uses: taiki-e/install-action@v2
         with: { tool: 'just,cargo-binstall' }
       - run: just ci-test
@@ -60,42 +63,35 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: ${{ steps.msrv.outputs.value }}
-          components: rustfmt
       - run: just ci_mode=0 ci-test-msrv  # Ignore warnings in MSRV
+
   fuzz:
     name: Fuzz
     runs-on: ubuntu-latest
-
     env:
       # The number of seconds to run the fuzz target.
       FUZZ_TIME: 60
-
     strategy:
       matrix:
         include:
           - fuzz_target: cpp_roundtrip
           - fuzz_target: rust_compress_oracle
           - fuzz_target: rust_decompress_oracle
-
     steps:
       - uses: actions/checkout@v6
         with: {persist-credentials: false, submodules: recursive}
-
       # Install the nightly Rust channel.
       - run: rustup toolchain install nightly
       - run: rustup default nightly
-
       # Install and cache `cargo-fuzz`.
       - uses: taiki-e/install-action@v2
         with:
           tool: cargo-binstall
       - run: cargo binstall -y cargo-fuzz@0.13.1 # Pinned to avoid breakage.
-
       # Build and then run the fuzz target.
       #  --target x86_64-unknown-linux-gnu is necessary to not default to musl, which is not supported by cargo-fuzz.
       - run: cargo fuzz build --target x86_64-unknown-linux-gnu ${{ matrix.fuzz_target }}
       - run: cargo fuzz run --target x86_64-unknown-linux-gnu ${{ matrix.fuzz_target }} -- -max_total_time=${{ env.FUZZ_TIME }}
-
       # Upload fuzzing artifacts on failure for post-mortem debugging.
       - uses: actions/upload-artifact@v7
         if: failure()
@@ -130,7 +126,7 @@ jobs:
     steps:
       - name: Result of the needed steps
         run: echo "${{ toJSON(needs) }}"
-      - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
+      - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
         run: exit 1
 
   release:

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fastpfor"
-version = "0.7.0"
+version = "0.8.0"
 description = "FastPFOR lib with C++ Rust wrapper and pure Rust implementation"
 authors = [
     "Francisco Jimenez <jjcfrank@gmail.com>",
@@ -25,6 +25,14 @@ harness = false
 # Eventually we may want to build without the C++ bindings by default.
 # Keeping it on for now to simplify development.
 default = ["cpp"]
+# Used internally for testing and benchmarking. Not intended for public use.
+_all_compatible = ["cpp_portable", "rust"]
+# Use portable C++ code that will not rely on the latest CPU features. This is the default for the C++ bindings.
+cpp_portable = ["cpp"]
+# Optimize FastPFOR for the current CPU.
+cpp_native = ["cpp"]
+# Experimental mode for dynamic dispatch of FastPFOR implementations.
+cpp_runtime = ["cpp"]
 cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"]
 rust = ["dep:thiserror", "dep:bytes"]
 

README.md

@@ -48,9 +48,22 @@ Unless otherwise specified, all codecs support `&[u32]` only.
 ## Usage
 
 ### Crate Features
-* `cpp` - C++ implementation (default)
+* `cpp` - C++ implementation (default, uses portable SIMD mode)
 * `rust` - Rust implementation (work in progress, opt-in)
 
+#### SIMD Mode Configuration
+
+The C++ backend can be compiled with different SIMD instruction sets. Control this by enabling one of these features:
+| Mode | Description |
+|------|-------------|
+| `cpp_portable` | **Default.** Uses SSE4.2 baseline only. Binaries run on any x86-64 CPU from ~2008+. Best for distributable libraries. |
+| `cpp_native` | Uses `-march=native` to enable all SIMD instructions supported by the build machine (AVX, AVX2, etc.). Maximum performance but may crash on CPUs lacking those instructions. |
+| `cpp_runtime` | Experimental. Reserved for future runtime CPU dispatch. |
+
+Feature selection can be overridden with the `FASTPFOR_SIMD_MODE` environment variable set to "portable", "native", or "runtime".
+
+**Recommendation:** Use `portable` (default) for libraries and distributed binaries. Use `native` only when building for a specific machine where you need maximum performance.
+
 ### Using C++ Wrapper
 
 ```rust

build.rs

@@ -13,7 +13,57 @@ fn build_fastpfor() {
 
     // Compile FastPFOR using CMake
     println!("cargo:rerun-if-changed=cpp");
-    let cmake_out = cmake::Config::new("cpp").define("WITH_TEST", "OFF").build();
+
+    // Warn if more than one feature is enabled. The order is important, must match the if else block below.
+    let simd_features = [
+        (cfg!(feature = "cpp_portable"), "'cpp_portable'"),
+        (cfg!(feature = "cpp_runtime"), "'cpp_runtime'"),
+        (cfg!(feature = "cpp_native"), "'cpp_native'"),
+    ];
+    let enabled_simd_features: Vec<_> = simd_features
+        .into_iter()
+        .filter_map(|(enabled, name)| enabled.then_some(name))
+        .collect();
+
+    // SIMD mode configuration via environment variable:
+    // - native: Use -march=native for maximum performance (not portable across CPUs)
+    // - portable: Use baseline SSE4.2 only for maximum compatibility (default)
+    // - runtime: Use function multi-versioning for runtime CPU dispatch (experimental)
+    let simd_mode = env::var("FASTPFOR_SIMD_MODE");
+    if enabled_simd_features.len() > 1 {
+        let feats = enabled_simd_features.join(", ");
+        if let Ok(simd_mode) = &simd_mode {
+            println!(
+                "cargo::warning=Multiple SIMD mode features are enabled: {feats}, but FASTPFOR_SIMD_MODE overrides it with {simd_mode}."
+            );
+        } else {
+            println!(
+                "cargo::warning=Multiple SIMD mode features enabled: {feats}. Defaulting to {}.",
+                enabled_simd_features[0]
+            );
+        }
+    }
+
+    let simd_mode = simd_mode.as_deref().unwrap_or({
+        {
+            // The order is important, must match the list above.
+            if cfg!(feature = "cpp_portable") {
+                "portable"
+            } else if cfg!(feature = "cpp_runtime") {
+                "runtime"
+            } else if cfg!(feature = "cpp_native") {
+                "native"
+            } else {
+                "portable" // fallback
+            }
+        }
+    });
+    println!("cargo:rerun-if-env-changed=FASTPFOR_SIMD_MODE");
+
+    let cmake_out = cmake::Config::new("cpp")
+        .define("FASTPFOR_WITH_TEST", "OFF")
+        .define("FASTPFOR_SIMD_MODE", simd_mode)
+        .build();
     let lib_path = cmake_out.join("lib");
     let lib_path = lib_path.to_str().unwrap();
 

justfile

@@ -1,33 +1,33 @@
 #!/usr/bin/env just --justfile
 
 main_crate := 'fastpfor'
-features_flag := '--all-features'
+# How to call the current just executable. Note that just_executable() may have `\` in Windows paths, so we need to quote it.
+just := quote(just_executable())
+# cargo-binstall needs a workaround due to caching when used in CI
+binstall_args := if env('CI', '') != '' {'--no-confirm --no-track --disable-telemetry'} else {''}
 
 # if running in CI, treat warnings as errors by setting RUSTFLAGS and RUSTDOCFLAGS to '-D warnings' unless they are already set
 # Use `CI=true just ci-test` to run the same tests as in GitHub CI.
 # Use `just env-info` to see the current values of RUSTFLAGS and RUSTDOCFLAGS
 ci_mode := if env('CI', '') != '' {'1'} else {''}
-# cargo-binstall needs a workaround due to caching
-# ci_mode might be manually set by user, so re-check the env var
-binstall_args := if env('CI', '') != '' {'--no-track'} else {''}
 export RUSTFLAGS := env('RUSTFLAGS', if ci_mode == '1' {'-D warnings'} else {''})
 export RUSTDOCFLAGS := env('RUSTDOCFLAGS', if ci_mode == '1' {'-D warnings'} else {''})
-export RUST_BACKTRACE := env('RUST_BACKTRACE', if ci_mode == '1' {'1'} else {''})
+export RUST_BACKTRACE := env('RUST_BACKTRACE', if ci_mode == '1' {'1'} else {'0'})
 
 @_default:
-    {{just_executable()}} --list
+    {{just}} --list
 
 # Run integration tests and save its output as the new expected output
 bless *args:  (cargo-install 'cargo-insta')
-    cargo insta test --accept --unreferenced=delete {{features_flag}} {{args}}
+    cargo insta test --accept --unreferenced=delete --features _all_compatible {{args}}
 
 # Build the project
 build:
-    cargo build --workspace --all-targets {{features_flag}}
+    cargo build --workspace --all-targets --features _all_compatible
 
 # Quick compile without building a binary
 check:
-    cargo check --workspace --all-targets {{features_flag}}
+    cargo check --workspace --all-targets --features _all_compatible
 
 # Generate code coverage report to upload to codecov.io
 ci-coverage: env-info && \
@@ -48,21 +48,21 @@ clean:
 
 # Run cargo clippy to lint the code
 clippy *args:
-    cargo clippy --workspace --all-targets {{features_flag}} {{args}}
+    cargo clippy --workspace --all-targets --features _all_compatible {{args}}
 
 # Generate code coverage report. Will install `cargo llvm-cov` if missing.
 coverage *args='--no-clean --open':  (cargo-install 'cargo-llvm-cov')
-    cargo llvm-cov --workspace --all-targets {{features_flag}} --include-build-script {{args}}
+    cargo llvm-cov --workspace --all-targets --features _all_compatible --include-build-script {{args}}
 
 # Build and open code documentation
 docs *args='--open':
-    DOCS_RS=1 cargo doc --no-deps {{args}} --workspace {{features_flag}}
+    DOCS_RS=1 cargo doc --no-deps {{args}} --workspace --features _all_compatible
 
 # Print environment info
 env-info:
-    @echo "Running {{if ci_mode == '1' {'in CI mode'} else {'in dev mode'} }} on {{os()}} / {{arch()}}"
-    @echo "PWD $(pwd)"
-    {{just_executable()}} --version
+    @echo "Running for '{{main_crate}}' crate {{if ci_mode == '1' {'in CI mode'} else {'in dev mode'} }} on {{os()}} / {{arch()}}"
+    @echo "PWD {{justfile_directory()}}"
+    {{just}} --version
     rustc --version
     cargo --version
     rustup --version
@@ -92,29 +92,42 @@ fmt:
 fmt-toml *args:  (cargo-install 'cargo-sort')
     cargo sort --workspace --grouped {{args}}
 
-# Get any package's field from the metadata
+# Get a package field from the metadata
 get-crate-field field package=main_crate:  (assert-cmd 'jq')
-    cargo metadata --format-version 1 | jq -e -r '.packages | map(select(.name == "{{package}}")) | first | .{{field}} | select(. != null)'
+    cargo metadata --format-version 1 | jq -e -r '.packages | map(select(.name == "{{package}}")) | first | .{{field}} // error("Field \"{{field}}\" is missing in Cargo.toml for package {{package}}")'
 
 # Get the minimum supported Rust version (MSRV) for the crate
 get-msrv package=main_crate:  (get-crate-field 'rust_version' package)
 
 # Find the minimum supported Rust version (MSRV) using cargo-msrv extension, and update Cargo.toml
 msrv:  (cargo-install 'cargo-msrv')
-    cargo msrv find --write-msrv --component rustfmt {{features_flag}} --ignore-lockfile -- {{just_executable()}} ci-test-msrv
+    cargo msrv find --write-msrv --features _all_compatible --ignore-lockfile -- {{just}} ci-test-msrv
 
 # Run cargo-release
 release *args='':  (cargo-install 'release-plz')
     release-plz {{args}}
 
 # Check semver compatibility with prior published version. Install it with `cargo install cargo-semver-checks`
 semver *args:  (cargo-install 'cargo-semver-checks')
-    cargo semver-checks {{features_flag}} {{args}}
+    cargo semver-checks --features _all_compatible {{args}}
 
 # Run all unit and integration tests
 test:
-    cargo test --workspace --all-targets {{features_flag}}
-    cargo test --workspace --doc {{features_flag}}
+    cargo test --workspace --all-targets --features _all_compatible
+    cargo test --workspace --doc --features _all_compatible
+
+# Test with a specific SIMD mode (portable, native, or runtime)
+test-simd mode='portable':
+    cargo test --workspace --all-targets --features cpp_{{mode}}
+
+# Test all SIMD modes
+test-all-simd-modes:
+    cargo clean -p fastpfor
+    {{just}} test-simd portable
+    cargo clean -p fastpfor
+    {{just}} test-simd native
+    cargo clean -p fastpfor
+    {{just}} test-simd runtime
 
 # Test documentation generation
 test-doc:  (docs '')
@@ -127,9 +140,9 @@ test-fmt: && (fmt-toml '--check' '--check-format')
 test-publish:
     cargo +nightly -Z package-workspace publish --dry-run
 
-# Find unused dependencies. Install it with `cargo install cargo-udeps`
+# Find unused dependencies. Uses `cargo-udeps`
 udeps:  (cargo-install 'cargo-udeps')
-    cargo +nightly udeps --workspace --all-targets {{features_flag}}
+    cargo +nightly udeps --workspace --all-targets --features _all_compatible
 
 # Update all dependencies, including breaking changes. Requires nightly toolchain (install with `rustup install nightly`)
 update:
@@ -161,11 +174,14 @@ cargo-install $COMMAND $INSTALL_CMD='' *args='':
     #!/usr/bin/env bash
     set -euo pipefail
     if ! command -v $COMMAND > /dev/null; then
+        echo "$COMMAND could not be found. Installing..."
         if ! command -v cargo-binstall > /dev/null; then
-            echo "$COMMAND could not be found. Installing it with    cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}}"
+            set -x
             cargo install ${INSTALL_CMD:-$COMMAND} --locked {{args}}
+            { set +x; } 2>/dev/null
         else
-            echo "$COMMAND could not be found. Installing it with    cargo binstall ${INSTALL_CMD:-$COMMAND} {{binstall_args}} --locked {{args}}"
+            set -x
             cargo binstall ${INSTALL_CMD:-$COMMAND} {{binstall_args}} --locked {{args}}
+            { set +x; } 2>/dev/null
         fi
     fi