fixed choice

Author: Rickard Lundin

src/chunked/arrow_converter.rs

@@ -76,8 +76,10 @@ pub(crate) struct Slice2Arrow<'a> {
     pub(crate) fn_line_break_len: FnLineBreakLen,
     pub(crate) masterbuilders: MasterBuilders,
     pub(crate) consistent_counter: ConsistentCounter,
+    pub(crate) threaded_write: (Sender<RecordBatch>,JoinHandle<Result<Stats>>)
 }
 
+
 pub(crate) struct MasterBuilders {
     builders: Vec<Vec<Box<dyn Sync + Send + ColumnBuilder>>>,
     outfile: PathBuf,

src/chunked/mod.rs

@@ -43,7 +43,7 @@ use parquet::errors::{ParquetError, Result};
 pub(crate) mod arrow_converter;
 pub(crate) mod residual_slicer;
 pub(crate) mod self_converter;
-mod threaded_file_output;
+pub(crate) mod threaded_file_output;
 mod trimmer;
 
 pub(crate) struct ChunkAndResidue {

src/chunked/threaded_file_output.rs

@@ -66,11 +66,24 @@ use std::thread::JoinHandle;
 use std::time::{Duration, Instant};
 use thread::spawn;
 use Compression::SNAPPY;
+use crate::cli::Targets;
 
 pub(crate) struct parquet_file_out {
     pub(crate) sender: Option<Sender<RecordBatch>>,
 }
-
+pub(crate) fn output_factory (target: Targets,schema: SchemaRef,_outfile:PathBuf) -> (Sender<RecordBatch>, JoinHandle<Result<Stats>>) {
+    match target {
+        Targets::Parquet => {
+             let mut pfo: Box<dyn arrow_file_output> = Box::new(parquet_file_out { sender: None });
+             pfo.setup(schema, _outfile)
+        }
+        Targets::IPC => {
+            let mut pfo: Box<dyn arrow_file_output> = Box::new(ipc_file_out { sender: None });
+            pfo.setup(schema, _outfile)
+        }
+        Targets::None => {todo!()}
+    }
+}
 impl arrow_file_output for parquet_file_out {
     fn setup(
         &mut self,

src/cli.rs

@@ -38,6 +38,8 @@ use std::fs;
 #[cfg(feature = "rayon")]
 use std::fs::File;
 use std::path::PathBuf;
+use arrow::datatypes::SchemaRef;
+use atomic_counter::ConsistentCounter;
 
 #[cfg(feature = "rayon")]
 use crate::chunked::arrow_converter::{MasterBuilders, Slice2Arrow};
@@ -47,6 +49,7 @@ use crate::chunked::residual_slicer::ResidualSlicer;
 use crate::chunked::self_converter::SampleSliceAggregator;
 #[cfg(feature = "rayon")]
 use crate::chunked::{find_last_nl, line_break_len_cr, Converter as ChunkedConverter, Slicer};
+use crate::chunked::threaded_file_output::output_factory;
 use crate::converter::Converter;
 use crate::error::Result;
 use crate::mocker::Mocker;
@@ -59,6 +62,14 @@ enum Converters {
     None,
 }
 
+#[cfg(feature = "rayon")]
+#[derive(clap::ValueEnum, Clone)]
+pub(crate) enum Targets {
+    Parquet,
+    IPC,
+    None,
+}
+
 #[cfg(feature = "rayon")]
 #[derive(clap::ValueEnum, Clone)]
 enum Slicers {
@@ -77,6 +88,13 @@ enum Slicers {
 pub struct Cli {
     #[command(subcommand)]
     command: Commands,
+    #[arg(
+        long = "target",
+        value_name = "TARGET-ENCODING",
+        action = ArgAction::Set,
+        default_value = "Parquet",
+    )]
+    target: Targets,
 
     /// Set the number of threads (logical cores) to use when multi-threading.
     #[arg(
@@ -89,6 +107,7 @@ pub struct Cli {
     n_threads: usize,
 }
 
+
 #[derive(Subcommand)]
 enum Commands {
     /// Convert a fixed-length file (.flf) to parquet.
@@ -275,12 +294,14 @@ impl Cli {
                             schema.to_path_buf(),
                             n_threads as i16,
                         );
-
+                        let sc=master_builders.schema_factory();
+                        
                         let s2a: Box<Slice2Arrow> = Box::new(Slice2Arrow {
                             fn_line_break: find_last_nl,
                             fn_line_break_len: line_break_len_cr,
                             masterbuilders: master_builders,
                             consistent_counter: ConsistentCounter::new(0),
+                            threaded_write: output_factory(self.target.clone(),sc ,out_file.clone().to_path_buf()),
                         });
                         s2a
                     }