feat: modular restructure of project as a framework (#71)

Rewrote entire structure, broke out functionality into their own crates, made it so that the core functionality exists in the smaller crates, and people can use those to build whatever CLI/tool they want.

This way we are not dependent on just one CLI implementation, you can pick and choose what file types you want to convert and what output targets you want.

The crates can be seen as the SDK, whereas the binary is just an example of a CLI which uses the SDK.

Author: wilhelmagren

.github/workflows/tests.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate coverage report
-        run: cargo llvm-cov --features rayon --workspace --lcov --output-path lcov.info
+        run: cargo llvm-cov --features mock --workspace --lcov --output-path lcov.info
       - name: Upload coverage report
         uses: codecov/codecov-action@v4
         with:

.gitignore

@@ -24,3 +24,4 @@ Cargo.lock
 
 # Python virtualenv for testing parquet files etc.
 .python-venv/
+.venv/

Cargo.toml

@@ -1,65 +1,41 @@
-[package]
-name = "evolution"
-version = "1.1.0"
+[workspace]
+members = [
+    "examples/*",
+    "crates/*",
+]
+resolver = "2"
+
+[workspace.package]
 edition = "2021"
-description = "🦖 Evolve your fixed-length data files into Apache Parquet, fully parallelized!"
+homepage = "https://github.com/firelink-data/evolution"
+repository = "https://github.com/firelink-data/evolution"
 authors = [
     "Ted Hammarlund <TedHammarlund@gmail.com>",
     "Rickard Lundin <rickard@x14.se>",
     "Wilhelm Ågren <wilhelmagren98@gmail.com>",
 ]
-
+license = "MIT"
+version = "1.2.0"
 readme = "README.md"
-license-file = "LICENSE"
-homepage = "https://github.com/firelink-data/evolution"
-repository = "https://github.com/firelink-data/evolution"
-
+categories = [
+    "science",
+    "database",
+    "concurrency",
+    "command-line-interface",
+]
 keywords = [
-    "cli",
     "arrow",
-    "data-engineering",
-    "apache-parquet",
-    "apache-arrow",
+    "parquet",
 ]
-include = [ "**/*.rs", "Cargo.toml", "LICENSE", "README.md" ]
-default-run = "evolution"
-
-[dependencies]
-chrono = "0.4.38"
-crossbeam = "0.8.4"
-colored = "2.0.4"
-env_logger = "0.11.3"
-half = "2.3.1"
-log = "0.4.20"
-num_cpus = "1.16.0"
-rand = "0.8.5"
-serde_json = "1.0.117"
-threadpool = "1.8.1"
-clap = { version = "4.5.4", features = ["default", "derive"] }
-serde = { version = "1.0.202", features = ["derive"] }
-rayon = { version = "1.10.0", optional = true }
-atoi_simd = { version = "0.16.0", optional = true }
-arrow2 = "0.18.0"
-libc = "0.2.154"
-arrow = "51.0.0"
-parquet = "51.0.0"
-arrow-ipc = "51.0.0"
-deltalake =  { version="0.17.3", features = ["datafusion"]}
-datafusion = "38.0.0"
-
-deltalake-core = "0.17.3"
-
-padder = { version = "1.2.0", features = ["serde"] }
-ordered-channel="1.1.0"
-atomic-counter = "1.0.1"
-crossbeam-channel = "0.5.13"
-tracing = "0.1.40"
-tokio = "1.37.0"
-futures = "0.3.30"
-[dev-dependencies]
-glob = "0.3.1"
+description = "Efficiently evolve your old fixed-length data files into modern file formats. "
 
-[features]
-default = [ "rayon" ]
-rayon = [ "dep:rayon", "dep:atoi_simd" ]
-nightly = []
+[workspace.dependencies]
+evolution-builder = { path = "crates/evolution-builder", version = "1.2.0" }
+evolution-common = { path = "crates/evolution-common", version = "1.2.0" }
+evolution-converter = { path = "crates/evolution-converter", version = "1.2.0" }
+evolution-mocker = { path = "crates/evolution-mocker", version = "1.2.0" }
+evolution-parser = { path = "crates/evolution-parser", version = "1.2.0" }
+evolution-schema = { path = "crates/evolution-schema", version = "1.2.0" }
+evolution-slicer = { path = "crates/evolution-slicer", version = "1.2.0" }
+evolution-target = { path = "crates/evolution-target", version = "1.2.0" }
+evolution-writer = { path = "crates/evolution-writer", version = "1.2.0" }

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Firelink Data
+Copyright (c) 2023-2024 Firelink Data
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -0,0 +1,22 @@
+[package]
+name = "evolution-builder"
+description = "Builder implementations for evolution."
+version = { workspace = true }
+edition = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+readme = { workspace = true }
+
+[lib]
+name = "evolution_builder"
+path = "src/lib.rs"
+bench = false
+
+[dependencies]
+evolution-common = { workspace = true }
+evolution-parser = { workspace = true }
+arrow = { version = "51.0.0", default-features = false }
+log = "0.4.21"
+half = "2.4.1"

crates/evolution-builder/Cargo.toml

@@ -0,0 +1,78 @@
+//
+// MIT License
+//
+// Copyright (c) 2023-2024 Firelink Data
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// File created: 2024-05-07
+// Last updated: 2024-10-11
+//
+
+use arrow::array::ArrayRef;
+use evolution_common::error::Result;
+use evolution_common::NUM_BYTES_FOR_NEWLINE;
+
+///
+pub trait Builder: From<Vec<ColumnBuilderRef>> {}
+
+///
+pub type BuilderRef = Box<dyn Builder>;
+
+///
+pub trait ColumnBuilder: Send + Sync {
+    fn try_build_column(&mut self, bytes: &[u8]) -> Result<usize>;
+    fn finish(&mut self) -> (&str, ArrayRef);
+}
+
+///
+pub type ColumnBuilderRef = Box<dyn ColumnBuilder>;
+
+///
+pub struct ParquetBuilder {
+    columns: Vec<ColumnBuilderRef>,
+}
+
+impl ParquetBuilder {
+    ///
+    pub fn try_build_from_slice(&mut self, buffer: &[u8]) -> Result<()> {
+        let mut idx: usize = 0;
+        while idx < buffer.len() {
+            for column in self.columns.iter_mut() {
+                idx += column.try_build_column(&buffer[idx..])?;
+            }
+            idx += NUM_BYTES_FOR_NEWLINE;
+        }
+
+        Ok(())
+    }
+
+    ///
+    pub fn columns(&mut self) -> &mut Vec<ColumnBuilderRef> {
+        &mut self.columns
+    }
+}
+
+impl From<Vec<ColumnBuilderRef>> for ParquetBuilder {
+    fn from(columns: Vec<ColumnBuilderRef>) -> Self {
+        Self { columns }
+    }
+}
+
+impl Builder for ParquetBuilder {}