Skip to content

Commit fca5c30

Browse files
author
Arthur Glowacki
committed
Fixed large stack usage in netcdf class
1 parent 4755c43 commit fca5c30

8 files changed

Lines changed: 40 additions & 40 deletions

File tree

‎src/core/process_whole.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,6 @@ DLL_EXPORT bool optimize_integrated_fit_params(data_struct::Analysis_Job<double>
120120

121121
DLL_EXPORT void generate_optimal_params(data_struct::Analysis_Job<double>* analysis_job);
122122

123-
void load_and_fit_quatification_datasets(data_struct::Analysis_Job<double>* analysis_job, size_t detector_num, std::vector<Quantification_Standard<double>>& standard_element_weights, std::unordered_map<size_t, double>& quant_map);
124-
125123
DLL_EXPORT void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job, std::string roi_file_name, std::map<int, std::map<std::string, data_struct::Fit_Parameters<double>>>& out_roi_fit_params, Callback_Func_Status_Def* status_callback = nullptr);
126124

127125
DLL_EXPORT void optimize_rois(data_struct::Analysis_Job<double>& analysis_job);

‎src/data_struct/fit_parameters.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ void Fit_Parameters<T_real>::from_array_d(const std::vector<double> &arr)
208208
{
209209
for(auto& itr : _params)
210210
{
211-
if (itr.second.opt_array_index != -1 && itr.second.opt_array_index < (int)arr.size())
211+
if (itr.second.opt_array_index != -1 && itr.second.opt_array_index < arr.size())
212212
{
213213
itr.second.value = arr[itr.second.opt_array_index];
214214
}
@@ -224,7 +224,7 @@ void Fit_Parameters<T_real>::from_array(const T_real* arr, size_t arr_size)
224224
//logit_s<<"\n";
225225
for(auto& itr : _params)
226226
{
227-
if (itr.second.opt_array_index != -1 && itr.second.opt_array_index < (int)arr_size)
227+
if (itr.second.opt_array_index != -1 && itr.second.opt_array_index < arr_size)
228228
{
229229
itr.second.value = arr[itr.second.opt_array_index];
230230
}

‎src/io/file/hl_file_io.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@ DLL_EXPORT bool load_and_integrate_spectra_volume(std::string dataset_directory,
474474
}
475475
}
476476

477-
bool has_external_files = hasNetcdf | hasBnpNetcdf | hasHdf | hasXspress;
477+
bool has_external_files = (hasNetcdf || hasBnpNetcdf || hasHdf || hasXspress);
478478

479479
bool ends_in_mca = false;
480480
size_t dlen = dataset_file.length();
@@ -1138,7 +1138,7 @@ DLL_EXPORT bool load_spectra_volume(std::string dataset_directory,
11381138
}
11391139

11401140
// try to load spectra from mda file
1141-
if (false == mda_io.load_spectra_volume(dataset_directory + "mda" + DIR_END_CHAR + dataset_file, detector_num, spectra_volume, hasNetcdf | hasBnpNetcdf | hasHdf | hasXspress, hasNetcdf))
1141+
if (false == mda_io.load_spectra_volume(dataset_directory + "mda" + DIR_END_CHAR + dataset_file, detector_num, spectra_volume, (hasNetcdf || hasBnpNetcdf || hasHdf || hasXspress), hasNetcdf))
11421142
{
11431143
scan_type = STR_SCAN_TYPE_2D_MAP;
11441144
logE << "Load spectra " << dataset_directory + "mda" + DIR_END_CHAR + dataset_file << "\n";

‎src/io/file/netcdf_io.cpp‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ NetCDF_IO<T_real>* NetCDF_IO<T_real>::inst()
100100

101101
template<typename T_real>
102102
size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
103-
std::string path,
103+
const std::string& path,
104104
size_t detector,
105105
data_struct::Spectra_Line<T_real>* spec_line,
106106
size_t line_size,
@@ -117,7 +117,6 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
117117
size_t start[] = {0, 0, 0};
118118
size_t count[] = {1, 1, 1047808};
119119
ptrdiff_t stride[] = {1, 1, 1};
120-
T_real data_in[1][1][1050000] = {0};
121120
size_t spectra_size = 0;
122121
nc_type rh_type;
123122
int rh_ndims = 0;
@@ -200,14 +199,14 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
200199
{
201200
start[0] = m0;
202201
//read header
203-
if( (retval = _nc_get_vars_real(ncid, varid, start, count, stride, &data_in[0][0][0]) ) != 0)
202+
if( (retval = _nc_get_vars_real(ncid, varid, start, count, stride, &_data_in[0][0][0]) ) != 0)
204203
{
205204
logE<< nc_strerror(retval)<<"\n";
206205
nc_close(ncid);
207206
return col_idx;
208207
}
209208

210-
if (data_in[0][0][0] != 21930 || data_in[0][0][1] != -21931)
209+
if (_data_in[0][0][0] != 21930 || _data_in[0][0][1] != -21931)
211210
{
212211
logE<<"NetCDF header [0][0][0] not found! Stopping load : "<<path<<"\n";
213212
nc_close(ncid);
@@ -224,17 +223,17 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
224223
d_idx += 2 * detector;
225224
}
226225

227-
size_t dset_det = size_t(data_in[0][0][d_idx]);
226+
size_t dset_det = size_t(_data_in[0][0][d_idx]);
228227
if (dset_det != detector)
229228
{
230229
logE << "detector not found! "<< dset_det <<" != "<<detector<<" Stopping load : " << path << "\n";
231230
nc_close(ncid);
232231
return (size_t)-1;
233232
}
234233

235-
header_size = data_in[0][0][2];
236-
cols_before_inc = data_in[0][0][8]; //sum all across the first dim looking at value 8
237-
spectra_size = data_in[0][0][20];
234+
header_size = _data_in[0][0][2];
235+
cols_before_inc = _data_in[0][0][8]; //sum all across the first dim looking at value 8
236+
spectra_size = _data_in[0][0][20];
238237

239238
//size_t inc_size = ((header_size + (spectra_size * detector)) + (spectra_size * (MAX_NUM_SUPPORTED_DETECOTRS_PER_COL - detector)) );
240239
size_t inc_size = header_size + (spectra_size * MAX_NUM_SUPPORTED_DETECOTRS_PER_COL);
@@ -250,7 +249,7 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
250249
callback_spectra = new data_struct::Spectra<T_real>(spectra_size);
251250
}
252251

253-
if (data_in[0][0][l] != 13260 || data_in[0][0][l+1] != -13261)
252+
if (_data_in[0][0][l] != 13260 || _data_in[0][0][l+1] != -13261)
254253
{
255254
if(col_idx < spec_cntr -2)
256255
{
@@ -265,8 +264,8 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
265264
}
266265

267266

268-
unsigned short i1 = data_in[0][0][l+ELAPSED_LIVETIME_OFFSET+(detector*8)];
269-
unsigned short i2 = data_in[0][0][l+ELAPSED_LIVETIME_OFFSET+(detector*8)+1];
267+
unsigned short i1 = _data_in[0][0][l+ELAPSED_LIVETIME_OFFSET+(detector*8)];
268+
unsigned short i2 = _data_in[0][0][l+ELAPSED_LIVETIME_OFFSET+(detector*8)+1];
270269
unsigned int ii = i1 | i2<<16;
271270
elapsed_livetime = ((float)ii) * 320e-9f; // need to multiply by this value becuase of the way it is saved
272271
if (ltype == E_load_type::LINE)
@@ -298,8 +297,8 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
298297
callback_spectra->elapsed_livetime(elapsed_livetime);
299298
}
300299

301-
i1 = data_in[0][0][l+ELAPSED_REALTIME_OFFSET+(detector*8)];
302-
i2 = data_in[0][0][l+ELAPSED_REALTIME_OFFSET+(detector*8)+1];
300+
i1 = _data_in[0][0][l+ELAPSED_REALTIME_OFFSET+(detector*8)];
301+
i2 = _data_in[0][0][l+ELAPSED_REALTIME_OFFSET+(detector*8)+1];
303302
ii = i1 | i2<<16;
304303
elapsed_realtime = ((float)ii) * 320e-9f; // need to multiply by this value becuase of the way it is saved
305304
if (ltype == E_load_type::LINE)
@@ -331,8 +330,8 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
331330
callback_spectra->elapsed_realtime(elapsed_realtime);
332331
}
333332

334-
i1 = data_in[0][0][l+INPUT_COUNTS_OFFSET+(detector*8)];
335-
i2 = data_in[0][0][l+INPUT_COUNTS_OFFSET+(detector*8)+1];
333+
i1 = _data_in[0][0][l+INPUT_COUNTS_OFFSET+(detector*8)];
334+
i2 = _data_in[0][0][l+INPUT_COUNTS_OFFSET+(detector*8)+1];
336335
ii = i1 | i2<<16;
337336
input_counts = ((float)ii) / elapsed_livetime;
338337
if (ltype == E_load_type::LINE)
@@ -365,8 +364,8 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
365364
}
366365

367366

368-
i1 = data_in[0][0][l+OUTPUT_COUNTS_OFFSET+(detector*8)];
369-
i2 = data_in[0][0][l+OUTPUT_COUNTS_OFFSET+(detector*8)+1];
367+
i1 = _data_in[0][0][l+OUTPUT_COUNTS_OFFSET+(detector*8)];
368+
i2 = _data_in[0][0][l+OUTPUT_COUNTS_OFFSET+(detector*8)+1];
370369
ii = i1 | i2<<16;
371370
output_counts = ((float)ii) / elapsed_realtime;
372371
if (ltype == E_load_type::LINE)
@@ -405,22 +404,22 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
405404
(*spec_line)[col_idx].recalc_elapsed_livetime();
406405
for (size_t k = 0; k < spectra_size; k++)
407406
{
408-
(*spec_line)[col_idx][k] = data_in[0][0][l+k];
407+
(*spec_line)[col_idx][k] = _data_in[0][0][l+k];
409408
}
410409
}
411410
else if (ltype == E_load_type::INTEGRATED)
412411
{
413412
for (size_t k = 0; k < spectra_size; k++)
414413
{
415-
(*spectra)(k) += data_in[0][0][l+k];
414+
(*spectra)(k) += _data_in[0][0][l+k];
416415
}
417416
}
418417
else if(ltype == E_load_type::CALLBACKF && callback_fun != nullptr)
419418
{
420419
callback_spectra->recalc_elapsed_livetime();
421420
for (size_t k = 0; k < spectra_size; k++)
422421
{
423-
(*callback_spectra)[k] = data_in[0][0][l+k];
422+
(*callback_spectra)[k] = _data_in[0][0][l+k];
424423
}
425424
(*callback_fun)(cur_row, col_idx, max_rows, line_size, detector, callback_spectra, user_data);
426425
}
@@ -458,23 +457,25 @@ size_t NetCDF_IO<T_real>::_load_spectra(E_load_type ltype,
458457
//-----------------------------------------------------------------------------
459458

460459
template<typename T_real>
461-
size_t NetCDF_IO<T_real>::load_spectra_line(std::string path, size_t detector, data_struct::Spectra_Line<T_real>* spec_line)
460+
size_t NetCDF_IO<T_real>::load_spectra_line(const std::string& path, size_t detector, data_struct::Spectra_Line<T_real>* spec_line)
462461
{
462+
463463
return _load_spectra(E_load_type::LINE, path, detector, spec_line, (size_t)-1, nullptr, 0, 0, nullptr, nullptr);
464+
464465
}
465466

466467
//-----------------------------------------------------------------------------
467468

468469
template<typename T_real>
469-
size_t NetCDF_IO<T_real>::load_spectra_line_integrated(std::string path, size_t detector, size_t line_size, data_struct::Spectra<T_real>* spectra)
470+
size_t NetCDF_IO<T_real>::load_spectra_line_integrated(const std::string& path, size_t detector, size_t line_size, data_struct::Spectra<T_real>* spectra)
470471
{
471472
return _load_spectra(E_load_type::INTEGRATED, path, detector, nullptr, line_size, spectra, 0, 0, nullptr, nullptr);
472473
}
473474

474475
//-----------------------------------------------------------------------------
475476

476477
template<typename T_real>
477-
bool NetCDF_IO<T_real>::load_spectra_line_with_callback(std::string path,
478+
bool NetCDF_IO<T_real>::load_spectra_line_with_callback(const std::string& path,
478479
const std::vector<size_t>& detector_num_arr,
479480
int row,
480481
size_t max_rows,
@@ -493,15 +494,14 @@ bool NetCDF_IO<T_real>::load_spectra_line_with_callback(std::string path,
493494
//-----------------------------------------------------------------------------
494495

495496
template<typename T_real>
496-
size_t NetCDF_IO<T_real>::load_scalers_line(std::string path, std::string tag, size_t row, data_struct::Scan_Info<T_real>* scan_info, data_struct::Params_Override<T_real> * params_override)
497+
size_t NetCDF_IO<T_real>::load_scalers_line(const std::string& path, std::string tag, size_t row, data_struct::Scan_Info<T_real>* scan_info, data_struct::Params_Override<T_real> * params_override)
497498
{
498499
std::lock_guard<std::mutex> lock(_mutex);
499500

500501
int ncid = 0, varid = 0, retval = 0;
501502
size_t start[] = {0, 0, 0};
502503
size_t count[] = {1, 1, 1};
503504
ptrdiff_t stride[] = {1, 1, 1};
504-
//T_real data_in[10000][1][11];
505505
T_real *data_in = nullptr;
506506
size_t dim2size[NC_MAX_VAR_DIMS] = {0};
507507
nc_type rh_type;

‎src/io/file/netcdf_io.h‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class DLL_EXPORT NetCDF_IO
6969

7070
static NetCDF_IO* inst();
7171

72-
~NetCDF_IO();
72+
~NetCDF_IO() {};
7373

7474
/**
7575
* @brief load_spectra_line
@@ -78,20 +78,20 @@ class DLL_EXPORT NetCDF_IO
7878
* @param spec_line
7979
* @return the number of spectra loaded. 0 if fail.
8080
*/
81-
size_t load_spectra_line(std::string path, size_t detector, data_struct::Spectra_Line<T_real>* spec_line);
81+
size_t load_spectra_line(const std::string& path, size_t detector, data_struct::Spectra_Line<T_real>* spec_line);
8282

83-
bool load_spectra_line_with_callback(std::string path,
83+
bool load_spectra_line_with_callback(const std::string& path,
8484
const std::vector<size_t>& detector_num_arr,
8585
int row,
8686
size_t max_rows,
8787
size_t max_cols,
8888
data_struct::IO_Callback_Func_Def<T_real> callback_fun,
8989
void* user_data);
9090

91-
size_t load_spectra_line_integrated(std::string path, size_t detector, size_t line_size, data_struct::Spectra<T_real>* spectra);
91+
size_t load_spectra_line_integrated(const std::string& path, size_t detector, size_t line_size, data_struct::Spectra<T_real>* spectra);
9292

9393
// tetramm scalers netcdf file
94-
size_t load_scalers_line(std::string path, std::string tag, size_t row, data_struct::Scan_Info<T_real>* scan_info, data_struct::Params_Override<T_real> * params_override);
94+
size_t load_scalers_line(const std::string& path, std::string tag, size_t row, data_struct::Scan_Info<T_real>* scan_info, data_struct::Params_Override<T_real> * params_override);
9595

9696
private:
9797
NetCDF_IO();
@@ -112,7 +112,7 @@ class DLL_EXPORT NetCDF_IO
112112
}
113113

114114
size_t _load_spectra(E_load_type ltype,
115-
std::string path,
115+
const std::string& path,
116116
size_t detector,
117117
data_struct::Spectra_Line<T_real>* spec_line,
118118
size_t line_size,
@@ -126,6 +126,8 @@ class DLL_EXPORT NetCDF_IO
126126

127127
static std::mutex _mutex;
128128

129+
T_real _data_in[1][1][1050000];
130+
129131
};
130132

131133
}// end namespace file

‎src/support/eigen-git-mirror‎

Submodule eigen-git-mirror updated from 6dbbf0a to e986838

‎src/support/pybind11‎

Submodule pybind11 updated 298 files

‎vcpkg‎

Submodule vcpkg updated 3537 files

0 commit comments

Comments
 (0)