Skip to content

Commit 9b4c57a

Browse files
nyuriknyurik
authored
Move ffi to different location, use different submodule (#20)
* Get rid of the factory - it causes binary size bloat, and makes things more error prone (not typed) * Introduce one Rust type per codec, that implement `Codec32` and optionally `Codec64` if supported -- compiler time verification if supported
1 parent 8d75810 commit 9b4c57a

6 files changed

Lines changed: 280 additions & 364 deletions

File tree

‎build.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ fn main() {
1313
println!("cargo:rerun-if-changed=cpp");
1414

1515
// Compile the bridge
16-
cxx_build::bridge("src/ffi_wrapper.rs")
16+
cxx_build::bridge("src/bridge/mod.rs")
1717
.include("cpp/headers")
1818
.include("src/bridge")
1919
.std("c++14")

‎cpp‎

‎src/bridge/fastpfor_bridge.h‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,29 +8,9 @@
88
// Adding things to the same namespace as lib to keep things simpler
99
namespace FastPForLib {
1010

11-
// Instantiate Coder Factory
12-
inline std::unique_ptr<CODECFactory> new_codec_factory() {
13-
return std::make_unique<CODECFactory>();
14-
}
15-
16-
// Get a list of codec names
17-
inline std::unique_ptr<std::vector<std::string>> codec_factory_all_names(
18-
const CODECFactory& factory
19-
) {
20-
return std::make_unique<std::vector<std::string>>(factory.allNames());
21-
}
22-
23-
// Get codec by name from a factory
24-
inline std::shared_ptr<IntegerCODEC> codec_factory_get_from_name(
25-
const CODECFactory& factory,
26-
rust::Str name
27-
) {
28-
return factory.getFromName(std::string(name));
29-
}
30-
3111
// Encode helpers
3212
inline size_t codec_encode32(
33-
const std::shared_ptr<IntegerCODEC>& codec,
13+
const std::unique_ptr<IntegerCODEC>& codec,
3414
const rust::Slice<const uint32_t> in,
3515
rust::Slice<uint32_t> out
3616
) {
@@ -40,7 +20,7 @@ inline size_t codec_encode32(
4020
}
4121

4222
inline size_t codec_encode64(
43-
const std::shared_ptr<IntegerCODEC>& codec,
23+
const std::unique_ptr<IntegerCODEC>& codec,
4424
const rust::Slice<const uint64_t> in,
4525
rust::Slice<uint32_t> out
4626
) {
@@ -51,7 +31,7 @@ inline size_t codec_encode64(
5131

5232
// Decode helper: returns consumed input elements
5333
inline size_t codec_decode32(
54-
const std::shared_ptr<IntegerCODEC>& codec,
34+
const std::unique_ptr<IntegerCODEC>& codec,
5535
const rust::Slice<const uint32_t> in,
5636
rust::Slice<uint32_t> out
5737
) {
@@ -61,7 +41,7 @@ inline size_t codec_decode32(
6141
}
6242

6343
inline size_t codec_decode64(
64-
const std::shared_ptr<IntegerCODEC>& codec,
44+
const std::unique_ptr<IntegerCODEC>& codec,
6545
const rust::Slice<const uint32_t> in,
6646
rust::Slice<uint64_t> out
6747
) {

‎src/bridge/mod.rs‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
use cxx::UniquePtr;
2+
3+
#[cxx::bridge(namespace = "FastPForLib")]
4+
mod ffi {
5+
unsafe extern "C++" {
6+
include!("fastpfor_bridge.h");
7+
8+
type IntegerCODEC;
9+
10+
fn fastbinarypacking8_codec() -> UniquePtr<IntegerCODEC>;
11+
fn fastbinarypacking16_codec() -> UniquePtr<IntegerCODEC>;
12+
fn fastbinarypacking32_codec() -> UniquePtr<IntegerCODEC>;
13+
fn BP32_codec() -> UniquePtr<IntegerCODEC>;
14+
fn vsencoding_codec() -> UniquePtr<IntegerCODEC>;
15+
fn fastpfor128_codec() -> UniquePtr<IntegerCODEC>;
16+
fn fastpfor256_codec() -> UniquePtr<IntegerCODEC>;
17+
fn simdfastpfor128_codec() -> UniquePtr<IntegerCODEC>;
18+
fn simdfastpfor256_codec() -> UniquePtr<IntegerCODEC>;
19+
fn simplepfor_codec() -> UniquePtr<IntegerCODEC>;
20+
fn simdsimplepfor_codec() -> UniquePtr<IntegerCODEC>;
21+
fn pfor_codec() -> UniquePtr<IntegerCODEC>;
22+
fn simdpfor_codec() -> UniquePtr<IntegerCODEC>;
23+
fn pfor2008_codec() -> UniquePtr<IntegerCODEC>;
24+
fn simdnewpfor_codec() -> UniquePtr<IntegerCODEC>;
25+
fn newpfor_codec() -> UniquePtr<IntegerCODEC>;
26+
fn optpfor_codec() -> UniquePtr<IntegerCODEC>;
27+
fn simdoptpfor_codec() -> UniquePtr<IntegerCODEC>;
28+
fn varint_codec() -> UniquePtr<IntegerCODEC>;
29+
fn vbyte_codec() -> UniquePtr<IntegerCODEC>;
30+
fn maskedvbyte_codec() -> UniquePtr<IntegerCODEC>;
31+
fn streamvbyte_codec() -> UniquePtr<IntegerCODEC>;
32+
fn varintgb_codec() -> UniquePtr<IntegerCODEC>;
33+
fn simple16_codec() -> UniquePtr<IntegerCODEC>;
34+
fn simple9_codec() -> UniquePtr<IntegerCODEC>;
35+
fn simple9_rle_codec() -> UniquePtr<IntegerCODEC>;
36+
fn simple8b_codec() -> UniquePtr<IntegerCODEC>;
37+
fn simple8b_rle_codec() -> UniquePtr<IntegerCODEC>;
38+
// TODO: conditional with #ifdef, might support later
39+
// fn varintg8iu_codec() -> UniquePtr<IntegerCODEC>;
40+
// fn snappy_codec() -> UniquePtr<IntegerCODEC>;
41+
fn simdbinarypacking_codec() -> UniquePtr<IntegerCODEC>;
42+
fn simdgroupsimple_codec() -> UniquePtr<IntegerCODEC>;
43+
fn simdgroupsimple_ringbuf_codec() -> UniquePtr<IntegerCODEC>;
44+
fn copy_codec() -> UniquePtr<IntegerCODEC>;
45+
46+
// Encode methods: returns number of output values
47+
48+
fn codec_encode32(
49+
codec: &UniquePtr<IntegerCODEC>,
50+
input: &[u32],
51+
output: &mut [u32],
52+
) -> Result<usize>;
53+
54+
fn codec_encode64(
55+
codec: &UniquePtr<IntegerCODEC>,
56+
input: &[u64],
57+
output: &mut [u32],
58+
) -> Result<usize>;
59+
60+
// Decode methods: returns consumed input values
61+
62+
fn codec_decode32(
63+
codec: &UniquePtr<IntegerCODEC>,
64+
input: &[u32],
65+
output: &mut [u32],
66+
) -> Result<usize>;
67+
68+
fn codec_decode64(
69+
codec: &UniquePtr<IntegerCODEC>,
70+
input: &[u32],
71+
output: &mut [u64],
72+
) -> Result<usize>;
73+
}
74+
}
75+
76+
trait CodecWrapper {
77+
fn codec(&self) -> &UniquePtr<ffi::IntegerCODEC>;
78+
}
79+
80+
#[expect(private_bounds)]
81+
pub trait Codec32: CodecWrapper {
82+
/// Encode a slice of 32-bit integers.
83+
fn encode32<'out>(
84+
&self,
85+
input: &[u32],
86+
output: &'out mut [u32],
87+
) -> Result<&'out mut [u32], cxx::Exception> {
88+
let n = ffi::codec_encode32(self.codec(), input, output)?;
89+
Ok(&mut output[..n])
90+
}
91+
92+
/// Decode a slice of 32-bit integers.
93+
fn decode32<'out>(
94+
&self,
95+
input: &[u32],
96+
output: &'out mut [u32],
97+
) -> Result<&'out mut [u32], cxx::Exception> {
98+
let n = ffi::codec_decode32(self.codec(), input, output)?;
99+
Ok(&mut output[..n])
100+
}
101+
}
102+
103+
#[expect(private_bounds)]
104+
pub trait Codec64: CodecWrapper {
105+
/// Encode a slice of 64-bit integers.
106+
fn encode64<'out>(
107+
&self,
108+
input: &[u64],
109+
output: &'out mut [u32],
110+
) -> Result<&'out mut [u32], cxx::Exception> {
111+
let n = ffi::codec_encode64(self.codec(), input, output)?;
112+
Ok(&mut output[..n])
113+
}
114+
115+
/// Decode a slice of 64-bit integers.
116+
fn decode64<'out>(
117+
&self,
118+
input: &[u32],
119+
output: &'out mut [u64],
120+
) -> Result<&'out mut [u64], cxx::Exception> {
121+
let n = ffi::codec_decode64(self.codec(), input, output)?;
122+
Ok(&mut output[..n])
123+
}
124+
}
125+
126+
macro_rules! implement_codecs {
127+
($($name:ident $(+ $extra:ident)? => $ffi:ident , )*) => {
128+
$(
129+
pub struct $name(UniquePtr<ffi::IntegerCODEC>);
130+
131+
impl $name {
132+
pub fn new() -> Self {
133+
Self(ffi::$ffi())
134+
}
135+
}
136+
137+
impl Default for $name {
138+
fn default() -> Self {
139+
Self::new()
140+
}
141+
}
142+
143+
impl CodecWrapper for $name {
144+
fn codec(&self) -> &UniquePtr<ffi::IntegerCODEC> {
145+
&self.0
146+
}
147+
}
148+
149+
impl Codec32 for $name {}
150+
$(impl $extra for $name {})*
151+
)*
152+
153+
#[cfg(test)]
154+
mod codec_tests {
155+
use super::*;
156+
157+
$(
158+
#[test]
159+
#[allow(non_snake_case)]
160+
fn $name() {
161+
roundtrip_32($name::new());
162+
$(
163+
// if $extra is Codec64, call roundtrip_64, but ignore the name
164+
let _ = stringify!($extra);
165+
roundtrip_64($name::new());
166+
)*
167+
}
168+
)*
169+
170+
fn roundtrip_32(codec: impl Codec32) {
171+
let input = vec![1, 2, 3, 4, 5];
172+
let mut output = vec![0; 10];
173+
let encoded = codec.encode32(&input, &mut output).unwrap();
174+
let mut decoded = vec![0; 10];
175+
let decoded = codec.decode32(encoded, &mut decoded).unwrap();
176+
assert_eq!(decoded, input);
177+
}
178+
179+
fn roundtrip_64(codec: impl Codec64) {
180+
let input = vec![1, 2, 3, 4, 5];
181+
let mut output = vec![0; 10];
182+
let encoded = codec.encode64(&input, &mut output).unwrap();
183+
184+
let mut decoded = vec![0; 10];
185+
let decoded = codec.decode64(encoded, &mut decoded).unwrap();
186+
assert_eq!(decoded, input);
187+
}
188+
}
189+
};
190+
}
191+
192+
implement_codecs! {
193+
BP32Codec => BP32_codec,
194+
CopyCodec => copy_codec,
195+
FastBinaryPacking8Codec => fastbinarypacking8_codec,
196+
FastPFor128Codec + Codec64 => fastpfor128_codec,
197+
FastPFor256Codec + Codec64 => fastpfor256_codec,
198+
FastBinaryPacking16Codec => fastbinarypacking16_codec,
199+
FastBinaryPacking32Codec => fastbinarypacking32_codec,
200+
MaskedVByteCodec => maskedvbyte_codec,
201+
NewPForCodec => newpfor_codec,
202+
OptPForCodec => optpfor_codec,
203+
PFor2008Codec => pfor2008_codec,
204+
PForCodec => pfor_codec,
205+
SimdBinaryPackingCodec => simdbinarypacking_codec,
206+
SimdFastPFor128Codec => simdfastpfor128_codec,
207+
SimdFastPFor256Codec => simdfastpfor256_codec,
208+
SimdGroupSimpleCodec => simdgroupsimple_codec,
209+
SimdGroupSimpleRingBufCodec => simdgroupsimple_ringbuf_codec,
210+
SimdNewPForCodec => simdnewpfor_codec,
211+
SimdOptPForCodec => simdoptpfor_codec,
212+
SimdPForCodec => simdpfor_codec,
213+
SimdSimplePForCodec => simdsimplepfor_codec,
214+
Simple16Codec => simple16_codec,
215+
Simple8bCodec => simple8b_codec,
216+
Simple8bRleCodec => simple8b_rle_codec,
217+
Simple9Codec => simple9_codec,
218+
Simple9RleCodec => simple9_rle_codec,
219+
SimplePForCodec => simplepfor_codec,
220+
// SnappyCodec => snappy_codec, // Conditional with #ifdef
221+
StreamVByteCodec => streamvbyte_codec,
222+
VByteCodec => vbyte_codec,
223+
VarIntCodec + Codec64 => varint_codec,
224+
// VarIntG8iuCodec => varintg8iu_codec, // Conditional with #ifdef
225+
VarIntGbCodec => varintgb_codec,
226+
// VsEncodingCodec => vsencoding_codec, // This is leaking memory
227+
}
228+
229+
#[cfg(test)]
230+
mod tests {
231+
use super::*;
232+
233+
// These duplicate the macro-generated tests, but we want to test that
234+
// the macro expansion itself works correctly
235+
236+
#[test]
237+
fn test_32() {
238+
let codec = FastPFor128Codec::new();
239+
let input = vec![1, 2, 3, 4, 5];
240+
let mut output = vec![0; 10];
241+
let mut output2 = vec![0; 10];
242+
let encoded = codec.encode32(&input, &mut output).unwrap();
243+
let encoded2 = codec.encode32(&input, &mut output2).unwrap();
244+
assert_eq!(encoded, encoded2);
245+
246+
let mut decoded = vec![0; 10];
247+
let mut decoded2 = vec![0; 10];
248+
let decoded = codec.decode32(encoded, &mut decoded).unwrap();
249+
let decoded2 = codec.decode32(encoded, &mut decoded2).unwrap();
250+
assert_eq!(decoded, decoded2);
251+
252+
assert_eq!(decoded, input);
253+
}
254+
255+
#[test]
256+
fn test_64() {
257+
let codec = FastPFor128Codec::new();
258+
let input = vec![1, 2, 3, 4, 5];
259+
let mut output = vec![0; 10];
260+
let mut output2 = vec![0; 10];
261+
let encoded = codec.encode64(&input, &mut output).unwrap();
262+
let encoded2 = codec.encode64(&input, &mut output2).unwrap();
263+
assert_eq!(encoded, encoded2);
264+
265+
let mut decoded = vec![0; 10];
266+
let mut decoded2 = vec![0; 10];
267+
let decoded = codec.decode64(encoded, &mut decoded).unwrap();
268+
let decoded2 = codec.decode64(encoded, &mut decoded2).unwrap();
269+
assert_eq!(decoded, decoded2);
270+
271+
assert_eq!(decoded, input);
272+
}
273+
}

0 commit comments

Comments
 (0)