@@ -29,19 +29,18 @@ use std::fs;
2929use std:: fs:: File ;
3030use std:: path:: PathBuf ;
3131
32- use clap:: { Parser , Subcommand } ;
33- use log:: { info} ;
34- use parquet:: arrow:: ArrowWriter ;
35- use crate :: slicers:: { ChunkAndResidue , find_last_nl, line_break_len_cr} ;
36- use crate :: slicers:: old_slicer:: { IN_MAX_CHUNKS , OldSlicer } ;
3732use crate :: converters:: arrow2_converter:: { MasterBuilder , Slice2Arrow2 } ;
3833use crate :: converters:: arrow_converter:: { MasterBuilders , Slice2Arrow } ;
3934use crate :: converters:: self_converter:: SampleSliceAggregator ;
4035use crate :: converters:: Converter ;
41- use crate :: { error, mocker, schema} ;
4236use crate :: dump:: dump;
37+ use crate :: slicers:: old_slicer:: { OldSlicer , IN_MAX_CHUNKS } ;
4338use crate :: slicers:: Slicer ;
44-
39+ use crate :: slicers:: { find_last_nl, line_break_len_cr, ChunkAndResidue } ;
40+ use crate :: { error, mocker, schema} ;
41+ use clap:: { Parser , Subcommand } ;
42+ use log:: info;
43+ use parquet:: arrow:: ArrowWriter ;
4544
4645#[ derive( Parser ) ]
4746#[ command( author, version, about, long_about = None ) ]
@@ -73,7 +72,6 @@ enum Slicers {
7372 New ,
7473}
7574
76-
7775#[ derive( Subcommand ) ]
7876enum Commands {
7977 /// does testing things
@@ -90,10 +88,10 @@ enum Commands {
9088 Convert {
9189 /// Sets schema file
9290
93- #[ clap( value_enum, value_name = "CONVERTER" ) ]
91+ #[ clap( value_enum, value_name = "CONVERTER" ) ]
9492 converter : Converters ,
9593
96- #[ clap( value_enum, value_name = "SLICER" ) ]
94+ #[ clap( value_enum, value_name = "SLICER" ) ]
9795 slicer : Slicers ,
9896
9997 #[ arg( short, long, value_name = "SCHEMA" ) ]
@@ -109,26 +107,27 @@ enum Commands {
109107 } ,
110108 Dump {
111109 /// Sets schema file
112- #[ clap( value_enum, value_name = "CONVERTER" ) ]
110+ #[ clap( value_enum, value_name = "CONVERTER" ) ]
113111 converter : Converters ,
114112 /// Sets input file
115113 #[ arg( short, long, value_name = "IN-FILE" ) ]
116114 in_file : PathBuf ,
117115 } ,
118-
119116}
120117
121118impl Cli {
122-
123- pub fn run < ' a > ( & self , in_buffers : & mut [ ChunkAndResidue ; IN_MAX_CHUNKS ] ) -> Result < ( ) , error:: ExecutionError > {
119+ pub fn run < ' a > (
120+ & self ,
121+ in_buffers : & mut [ ChunkAndResidue ; IN_MAX_CHUNKS ] ,
122+ ) -> Result < ( ) , error:: ExecutionError > {
124123 let n_logical_threads = num_cpus:: get ( ) ;
125124 let mut n_threads: usize = self . n_threads as usize ;
126125
127126 if n_threads > n_logical_threads {
128127 info ! (
129- "you specified to use {} thread, but your CPU only has {} logical threads" ,
130- n_threads, n_logical_threads,
131- ) ;
128+ "you specified to use {} thread, but your CPU only has {} logical threads" ,
129+ n_threads, n_logical_threads,
130+ ) ;
132131 n_threads = n_logical_threads;
133132 }
134133
@@ -139,37 +138,37 @@ impl Cli {
139138
140139 match & self . command {
141140 Some ( Commands :: Mock {
142- schema,
143- file,
144- n_rows,
145- } ) => {
141+ schema,
142+ file,
143+ n_rows,
144+ } ) => {
146145 print ! ( "target file {:?}" , file. as_ref( ) ) ;
147146
148147 mocker:: Mocker :: new (
149148 schema:: FixedSchema :: from_path ( schema. into ( ) ) ,
150149 file. clone ( ) ,
151150 n_threads,
152151 )
153- . generate ( n_rows. unwrap ( ) as usize ) ;
152+ . generate ( n_rows. unwrap ( ) as usize ) ;
154153 Ok ( ( ) )
155154 }
156155
157156 Some ( Commands :: Dump {
158- converter : _,
159- in_file
160- } ) => {
157+ converter : _,
158+ in_file,
159+ } ) => {
161160 let _in_file = fs:: File :: open ( & in_file) . expect ( "bbb" ) ;
162161 dump ( _in_file) ;
163162
164163 Ok ( ( ) )
165164 }
166165 Some ( Commands :: Convert {
167- converter,
168- slicer : _,
169- schema,
170- in_file,
171- out_file
172- } ) => {
166+ converter,
167+ slicer : _,
168+ schema,
169+ in_file,
170+ out_file,
171+ } ) => {
173172 let _in_file = fs:: File :: open ( & in_file) . expect ( "bbb" ) ;
174173
175174 let mut slicer_instance: Box < dyn Slicer > = Box :: new ( OldSlicer {
@@ -178,13 +177,21 @@ impl Cli {
178177
179178 let converter_instance: Box < dyn Converter > = match converter {
180179 Converters :: Arrow => {
181- let mut master_builders = MasterBuilders :: builders_factory ( schema. to_path_buf ( ) , n_threads as i16 , ) ;
180+ let mut master_builders = MasterBuilders :: builders_factory (
181+ schema. to_path_buf ( ) ,
182+ n_threads as i16 ,
183+ ) ;
182184 let writer: ArrowWriter < File > = master_builders. writer_factory ( out_file) ;
183185
184- let s2a: Box < Slice2Arrow > = Box :: new ( Slice2Arrow { writer : writer, fn_line_break : find_last_nl, fn_line_break_len : line_break_len_cr, masterbuilders : master_builders } ) ;
185- // writer.finish().expect("finish");
186+ let s2a: Box < Slice2Arrow > = Box :: new ( Slice2Arrow {
187+ writer : writer,
188+ fn_line_break : find_last_nl,
189+ fn_line_break_len : line_break_len_cr,
190+ masterbuilders : master_builders,
191+ } ) ;
192+ // writer.finish().expect("finish");
186193 s2a
187- } ,
194+ }
188195 Converters :: Arrow2 => {
189196 let _out_file = fs:: OpenOptions :: new ( )
190197 . create ( true )
@@ -193,10 +200,14 @@ impl Cli {
193200 . expect ( "aaa" ) ;
194201
195202 let master_builder = MasterBuilder :: builder_factory ( schema. to_path_buf ( ) ) ;
196- let s2a: Box < Slice2Arrow2 > = Box :: new ( Slice2Arrow2 { file_out : _out_file, fn_line_break : find_last_nl, master_builder } ) ;
203+ let s2a: Box < Slice2Arrow2 > = Box :: new ( Slice2Arrow2 {
204+ file_out : _out_file,
205+ fn_line_break : find_last_nl,
206+ master_builder,
207+ } ) ;
197208
198209 s2a
199- } ,
210+ }
200211
201212 Converters :: None => {
202213 let _out_file = fs:: OpenOptions :: new ( )
@@ -205,24 +216,37 @@ impl Cli {
205216 . open ( out_file)
206217 . expect ( "aaa" ) ;
207218
208- let s3a: Box < SampleSliceAggregator > = Box :: new ( SampleSliceAggregator { file_out : _out_file, fn_line_break : find_last_nl } ) ;
219+ let s3a: Box < SampleSliceAggregator > = Box :: new ( SampleSliceAggregator {
220+ file_out : _out_file,
221+ fn_line_break : find_last_nl,
222+ } ) ;
209223 s3a
210- } ,
224+ }
211225 } ;
212226
213-
214- let r = slicer_instance. slice_and_convert ( converter_instance, in_buffers, _in_file, n_threads as usize ) ;
227+ let r = slicer_instance. slice_and_convert (
228+ converter_instance,
229+ in_buffers,
230+ _in_file,
231+ n_threads as usize ,
232+ ) ;
215233 match r {
216- Ok ( s) => { print ! ( "Operation successful inbytes={} out bytes={} num of rows={}" , s. bytes_in, s. bytes_out, s. num_rows) ; }
217- Err ( x) => { print ! ( "Operation failed: {}" , x) ; }
234+ Ok ( s) => {
235+ print ! (
236+ "Operation successful inbytes={} out bytes={} num of rows={}" ,
237+ s. bytes_in, s. bytes_out, s. num_rows
238+ ) ;
239+ }
240+ Err ( x) => {
241+ print ! ( "Operation failed: {}" , x) ;
242+ }
218243 }
219244
220-
221245 Ok ( ( ) )
222246 }
223247
224- // Ok(()) => todo!(),
225- _ => { Ok ( ( ) ) }
248+ // Ok(()) => todo!(),
249+ _ => Ok ( ( ) ) ,
226250 }
227251 }
228- }
252+ }
0 commit comments