[fix] off-by-one byte index on windows, update tools (#35)

* [fix] off-by-one linebreak index on windows

* [feat] set env vars for windows, fix parquet verify

Author: wilhelmagren

src/converter.rs

@@ -175,7 +175,8 @@ impl Converter {
             let byte_idx_last_line_break = line_break_indices
                 .last()
                 .ok_or("No line breaks found in the read buffer!")?;
-            let n_bytes_left_after_line_break = buffer_capacity - 1 - byte_idx_last_line_break;
+            let n_bytes_left_after_line_break =
+                buffer_capacity - byte_idx_last_line_break - NUM_CHARS_FOR_NEWLINE;
 
             reader.seek_relative(-(n_bytes_left_after_line_break as i64))?;
 
@@ -342,7 +343,7 @@ impl Converter {
         let mut remaining_bytes: usize = bytes_to_read;
         let mut bytes_processed: usize = 0;
         let mut bytes_overlapped: usize = 0;
-        let mut buffer_capacity = self.buffer_size;
+        let mut buffer_capacity: usize = self.buffer_size;
 
         // We wrap the file descriptor in a [`BufReader`] to improve the syscall
         // efficiency of small and repeated I/O calls to the same file.
@@ -386,7 +387,8 @@ impl Converter {
             let byte_idx_last_line_break = line_break_indices
                 .last()
                 .ok_or("No line breaks found in read buffer!")?;
-            let n_bytes_left_after_line_break = buffer_capacity - 1 - byte_idx_last_line_break;
+            let n_bytes_left_after_line_break =
+                buffer_capacity - byte_idx_last_line_break - NUM_CHARS_FOR_NEWLINE;
 
             reader.seek_relative(-(n_bytes_left_after_line_break as i64))?;
 

src/mocker.rs

@@ -373,12 +373,12 @@ impl MockerBuilder {
             None => {
                 if n_rows >= MIN_NUM_ROWS_FOR_MULTITHREADING && multithreaded {
                     let mocker_buffer_size = MOCKER_BUFFER_NUM_ROWS / (n_threads - 1);
-                    info!("Optional field '--buffer-size' not provided.");
-                    info!("Mocker buffer size is now {} rows.", mocker_buffer_size);
+                    #[cfg(debug_assertions)]
+                    debug!("Optional field '--buffer-size' not provided, mocker buffer size is now {} rows.", mocker_buffer_size);
                     mocker_buffer_size
                 } else {
-                    info!("Optional field '--buffer-size' not provided.");
-                    info!("Mocker buffer size is now {} rows.", MOCKER_BUFFER_NUM_ROWS);
+                    #[cfg(debug_assertions)]
+                    debug!("Optional field '--buffer-size' not provided, mocker buffer size is now {} rows.", MOCKER_BUFFER_NUM_ROWS);
                     MOCKER_BUFFER_NUM_ROWS
                 }
             }
@@ -387,9 +387,9 @@ impl MockerBuilder {
         let thread_channel_capacity: usize = match self.thread_channel_capacity {
             Some(c) => c,
             None => {
-                info!("Optional field '--thread-channel-capacity' not provided.");
-                info!(
-                    "Mocker thread channel capacity is now {} messages.",
+                #[cfg(debug_assertions)]
+                debug!(
+                    "Optional field '--thread-channel-capacity' not provided, mocker thread channel capacity is now {} messages.",
                     MOCKER_THREAD_CHANNEL_CAPACITY
                 );
                 MOCKER_THREAD_CHANNEL_CAPACITY
@@ -479,6 +479,6 @@ fn master_thread_write(
         drop(buffer);
     }
 
-    info!("Master thread done, cleaning up resources.");
+    info!("Master thread done writing mocked data!");
     Ok(())
 }

tools/EnableEnvironment.ps1

@@ -0,0 +1,5 @@
+
+.\.python-venv\Scripts\Activate.ps1
+
+$env:RUST_LOG = "DEBUG"
+

tools/verify-parquet.py

@@ -1,9 +1,3 @@
-<<<<<<< HEAD
-#!/usr/bin/env python3
-=======
-#!/usr/bin/python3
->>>>>>> main
-
 import argparse
 import pyarrow.parquet as pq
 
@@ -17,19 +11,13 @@
     parser.add_argument(
         "file", help="The .parquet file to verify.",
     )
-<<<<<<< HEAD
     parser.add_argument(
         "-p", "--print", action="store_true", help="Read the entire parquet file as a table and print it." 
     )
 
     args = parser.parse_args()
     file = args.file
     print_table = args.print
-=======
-
-    args = parser.parse_args()
-    file = args.file
->>>>>>> main
 
     parquet_file = pq.ParquetFile(file)
     print("="*60)
@@ -38,17 +26,10 @@
     print(parquet_file.schema)
     print("="*60)
 
-<<<<<<< HEAD
     if print_table:
         table = pq.read_table(file)
         df = table.to_pandas()
         print(df.head())
         print("="*60)
 
-=======
-    table = pq.read_table(file)
-    df = table.to_pandas()
-    print(df.head())
-    print("="*60)
->>>>>>> main
     print("\nDone!\n")