Skip to content

Commit f3c2220

Browse files
author
Daniel Lemire
committed
Major reorg
1 parent c5db7cb commit f3c2220

30 files changed

Lines changed: 814 additions & 561 deletions

‎CMakeLists.txt‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,63 @@
11
cmake_minimum_required(VERSION 3.10)
22
set(CMAKE_C_COMPILER clang)
33
set(CMAKE_CXX_COMPILER clang++)
4-
add_compile_options(-Wall -Wextra -freflection -stdlib=libc++ -std=c++26)
54
project(experimental_json_builder
65
VERSION 0.0.1
76
DESCRIPTION "Serializing gigabytes of JSON per second"
87
HOMEPAGE_URL "https://simdjson.org/"
98
LANGUAGES CXX
109
)
10+
11+
12+
13+
option(SIMDJSON_SERIALIZER_SANITIZE_MEMORY "Sanitize memory" OFF)
14+
15+
if(SIMDJSON_SERIALIZER_SANITIZE_MEMORY)
16+
message(STATUS "Setting the memory sanitizer.")
17+
add_compile_options(
18+
-fsanitize=memory -fno-sanitize-recover=all
19+
)
20+
link_libraries(
21+
-fsanitize=memory -fno-sanitize-recover=all
22+
)
23+
endif()
24+
25+
if(NOT CMAKE_BUILD_TYPE)
26+
if(SIMDJSON_SANITIZE OR SIMDJSON_SANITIZE_UNDEFINED)
27+
message(STATUS "No build type selected and you have enabled the sanitizer, \
28+
default to Debug. Consider setting CMAKE_BUILD_TYPE.")
29+
message(STATUS "Setting debug optimization flag to -O1 to help sanitizer.")
30+
set(CMAKE_CXX_FLAGS_DEBUG "-O1" CACHE STRING "" FORCE)
31+
set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE)
32+
else()
33+
message(STATUS "No build type selected, default to Release")
34+
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
35+
endif()
36+
endif()
37+
1138
if (NOT CMAKE_BUILD_TYPE)
1239
message(STATUS "No build type selected, default to Release")
1340
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
1441
endif()
15-
#set(CMAKE_CXX_STANDARD 23)
16-
#set(CMAKE_CXX_STANDARD_REQUIRED ON)
42+
43+
add_library(simdjson_serialization INTERFACE)
44+
add_library(simdjson::serialization ALIAS simdjson_serialization)
45+
target_compile_options(simdjson_serialization INTERFACE -freflection -stdlib=libc++ -std=c++26)
46+
47+
target_include_directories(
48+
simdjson_serialization
49+
INTERFACE
50+
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
51+
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
52+
)
1753

1854
include(cmake/CPM.cmake)
1955
# include simdjson
2056
CPMAddPackage("gh:simdjson/simdjson@3.9.4")
21-
CPMAddPackage("gh:nlohmann/json@3.10.5")
22-
23-
# Include reflect-cpp
24-
CPMAddPackage(
25-
NAME reflect-cpp
26-
GITHUB_REPOSITORY getml/reflect-cpp
27-
PATCHES reflect-cpp.patch
28-
GIT_TAG v0.13.0
29-
)
3057

31-
# Optionally, specify the C++ compiler directly (here we use clang++)
32-
add_subdirectory(src)
58+
3359
add_subdirectory(examples)
34-
add_subdirectory(benchmarks/src)
60+
add_subdirectory(benchmarks)
3561

3662
enable_testing()
3763
add_subdirectory(tests)

‎benchmarks/CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
CPMAddPackage("gh:nlohmann/json@3.10.5")
2+
3+
# Include reflect-cpp
4+
CPMAddPackage(
5+
NAME reflect-cpp
6+
GITHUB_REPOSITORY getml/reflect-cpp
7+
PATCHES reflect-cpp.patch
8+
GIT_TAG v0.13.0
9+
)
10+
add_subdirectory(simpleparser)
11+
add_subdirectory(src)

reflect-cpp.patch renamed to benchmarks/reflect-cpp.patch

File renamed without changes.

‎benchmarks/simpleparser/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
add_library(simpleparser INTERFACE)
2+
target_include_directories(simpleparser INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
3+
target_link_libraries(simpleparser INTERFACE simdjson::serialization)

src/from_json.hpp renamed to benchmarks/simpleparser/from_json.hpp

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
#pragma once
2-
3-
#include <iostream>
2+
#include "simdjson/json_builder/json_builder.h"
43
#include <string>
5-
#include <unordered_map>
64
#include <vector>
7-
#include <charconv>
85
#include <experimental/meta> // Reflection header
9-
#include "experimental_json_builder.hpp" // We might want to extract the concepts + metaprogramming to a separate header and include here and in the builder
106
#include "json_parser.hpp"
117

12-
namespace experimental_json_builder {
8+
namespace simpleparser {
9+
namespace json_builder {
1310

1411
using JsonValue = json_parser::JsonValue;
1512
using JsonValueType = json_parser::JsonValueType;
@@ -39,16 +36,16 @@ void from_json(const JsonValue& j, std::string& t) {
3936
}
4037

4138
template <class T>
42-
void from_json(const JsonValue& j, std::vector<T>& t) requires ContainerButNotString<std::vector<T>>;
39+
void from_json(const JsonValue& j, std::vector<T>& t) requires simdjson::json_builder::ContainerButNotString<std::vector<T>>;
4340

4441
template <class T>
45-
void from_json(const JsonValue& j, T& obj) requires(UserDefinedType<T> && !ContainerButNotString<T> && !std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !std::is_same_v<T, const char *>) {
42+
void from_json(const JsonValue& j, T& obj) requires(simdjson::json_builder::UserDefinedType<T> && !simdjson::json_builder::ContainerButNotString<T> && !std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !std::is_same_v<T, const char *>) {
4643
if (j.type != JsonValueType::Object) {
4744
throw std::runtime_error("Expected object type for user-defined type value");
4845
}
4946

5047
// Using reflection to iterate over data members
51-
[:expand(std::meta::nonstatic_data_members_of(^T)):] >> [&]<auto dm> {
48+
[:simdjson::json_builder::expand(std::meta::nonstatic_data_members_of(^T)):] >> [&]<auto dm> {
5249
constexpr auto name = std::meta::name_of(dm);
5350
auto it = j.object_value.find(std::string(reinterpret_cast<const char*>(name.data()), name.size()));
5451
if (it != j.object_value.end()) {
@@ -58,7 +55,7 @@ void from_json(const JsonValue& j, T& obj) requires(UserDefinedType<T> && !Conta
5855
}
5956

6057
template <class T>
61-
void from_json(const JsonValue& j, std::vector<T>& t) requires ContainerButNotString<std::vector<T>> {
58+
void from_json(const JsonValue& j, std::vector<T>& t) requires simdjson::json_builder::ContainerButNotString<std::vector<T>> {
6259
if (j.type != JsonValueType::Array) {
6360
throw std::runtime_error("Expected array type for container value");
6461
}
@@ -71,4 +68,5 @@ void from_json(const JsonValue& j, std::vector<T>& t) requires ContainerButNotSt
7168
}
7269

7370

74-
} // namespace experimental_json_builder
71+
} // namespace json_builder
72+
} // namespace simpleparser

src/json_parser.hpp renamed to benchmarks/simpleparser/json_parser.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33
#include <unordered_map>
44
#include <vector>
55
#include <sstream>
6-
#include <charconv>
76
#include <experimental/meta>
87

8+
namespace simpleparser {
99
namespace json_parser {
1010

1111
enum class JsonValueType { Null, Boolean, Number, String, Array, Object };
@@ -216,3 +216,4 @@ class JsonParser {
216216
};
217217

218218
} // namespace json_parser
219+
} // namespace simpleparser

‎benchmarks/src/CMakeLists.txt‎

Lines changed: 4 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,4 @@
1-
cmake_minimum_required(VERSION 3.10)
2-
3-
# Set C++ standard to C++17 or higher if needed
4-
#set(CMAKE_CXX_STANDARD 23)
5-
#set(CMAKE_CXX_STANDARD_REQUIRED ON)
6-
7-
# Optionally, specify the C++ compiler directly (here we use clang++)
8-
set(CMAKE_C_COMPILER clang)
9-
set(CMAKE_CXX_COMPILER clang++)
10-
11-
12-
option(SIMDJSON_SANITIZE_MEMORY "Sanitize memory" OFF)
13-
14-
if(SIMDJSON_SANITIZE_MEMORY)
1+
if(SIMDJSON_SERIALIZER_SANITIZE_MEMORY)
152
message(STATUS "Setting the memory sanitizer.")
163
add_compile_options(
174
-fsanitize=memory -fno-sanitize-recover=all
@@ -21,46 +8,15 @@ if(SIMDJSON_SANITIZE_MEMORY)
218
)
229
endif()
2310

24-
if(NOT CMAKE_BUILD_TYPE)
25-
if(SIMDJSON_SANITIZE OR SIMDJSON_SANITIZE_UNDEFINED)
26-
message(STATUS "No build type selected and you have enabled the sanitizer, \
27-
default to Debug. Consider setting CMAKE_BUILD_TYPE.")
28-
message(STATUS "Setting debug optimization flag to -O1 to help sanitizer.")
29-
set(CMAKE_CXX_FLAGS_DEBUG "-O1" CACHE STRING "" FORCE)
30-
set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE)
31-
else()
32-
message(STATUS "No build type selected, default to Release")
33-
set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
34-
endif()
35-
endif()
36-
37-
# set(CMAKE_OSX_ARCHITECTURES "arm64")
38-
39-
# if(CMAKE_BUILD_TYPE STREQUAL "Debug")
40-
# set(CMAKE_C_FLAGS_DEBUG "-g -fsanitize=address,undefined -fno-omit-frame-pointer")
41-
# set(CMAKE_CXX_FLAGS_DEBUG "-g -fsanitize=address,undefined -fno-omit-frame-pointer")
42-
# else()
43-
# set(CMAKE_C_FLAGS_RELEASE "-O2 -fsanitize=address,undefined")
44-
# set(CMAKE_CXX_FLAGS_RELEASE "-O2 -fsanitize=address,undefined")
45-
# set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O2 -g -fsanitize=address,undefined")
46-
# set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -fsanitize=address,undefined")
47-
#endif()
48-
49-
# Include directories
50-
include_directories(${CMAKE_SOURCE_DIR})
11+
#include_directories(${CMAKE_SOURCE_DIR})
5112

5213
# Add executable targets
5314
add_executable(SerializationBenchmark benchmark_serialization.cpp)
5415
add_executable(SerializationTwitterBenchmark benchmark_serialization_twitter.cpp)
5516

56-
target_compile_options(SerializationBenchmark PRIVATE -freflection -stdlib=libc++ -std=c++26)
57-
target_compile_options(SerializationTwitterBenchmark PRIVATE -freflection -stdlib=libc++ -std=c++26)
58-
59-
60-
# Link sanitizers and libraries
61-
target_link_libraries(SerializationTwitterBenchmark PRIVATE nlohmann_json::nlohmann_json simdjson::simdjson )
6217

63-
target_link_libraries(SerializationBenchmark PRIVATE simdjson-serial nlohmann_json::nlohmann_json simdjson::simdjson)
18+
target_link_libraries(SerializationTwitterBenchmark PRIVATE simpleparser simdjson::serialization nlohmann_json::nlohmann_json simdjson::simdjson)
19+
target_link_libraries(SerializationBenchmark PRIVATE simpleparser simdjson::serialization nlohmann_json::nlohmann_json simdjson::simdjson)
6420

6521
# cpp-reflect may fail to build on LLVM 19 due to this issue
6622
# https://github.com/hanickadot/compile-time-regular-expressions/issues/307

‎benchmarks/src/benchmark_helper.hpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#include "event_counter.h"
44
#include <atomic>
55

6-
inline event_collector& get_collector() {
6+
inline event_collector &get_collector() {
77
static event_collector collector;
88
return collector;
99
}
@@ -12,7 +12,7 @@ template <class function_type>
1212
event_aggregate bench(const function_type &function, size_t min_repeat = 10,
1313
size_t min_time_ns = 1000000000,
1414
size_t max_repeat = 100000) {
15-
event_collector& collector = get_collector();
15+
event_collector &collector = get_collector();
1616
event_aggregate aggregate{};
1717
size_t N = min_repeat;
1818
if (N == 0) {
@@ -35,8 +35,9 @@ event_aggregate bench(const function_type &function, size_t min_repeat = 10,
3535

3636
// Source of the 2 functions below:
3737
// https://github.com/simdutf/simdutf/blob/master/benchmarks/base64/benchmark_base64.cpp
38-
inline void pretty_print(size_t strings, size_t bytes, std::string name, event_aggregate agg) {
39-
event_collector& collector = get_collector();
38+
inline void pretty_print(size_t strings, size_t bytes, std::string name,
39+
event_aggregate agg) {
40+
event_collector &collector = get_collector();
4041
printf("%-60s : ", name.c_str());
4142
printf(" %5.2f MB/s ", bytes * 1000 / agg.elapsed_ns());
4243
printf(" %5.2f Ms/s ", strings * 1000 / agg.elapsed_ns());

‎benchmarks/src/benchmark_reflect_serialization.cpp‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@
44
#include <rfl.hpp>
55
#include <rfl/json.hpp>
66

7-
void bench_reflect_cpp(std::vector<User>& data) {
7+
void bench_reflect_cpp(std::vector<User> &data) {
88
std::string output = rfl::json::write(data);
99
size_t output_volume = output.size();
1010
printf("# output volume: %zu bytes\n", output_volume);
1111

1212
volatile size_t measured_volume = 0;
13-
pretty_print(
14-
1, output_volume, "bench_reflect_cpp",
15-
bench([&data, &measured_volume, &output_volume] () {
16-
std::string output = rfl::json::write(data);
17-
measured_volume = output.size();
18-
if(measured_volume != output_volume) { printf("mismatch\n"); }
19-
})
20-
);
13+
pretty_print(1, output_volume, "bench_reflect_cpp",
14+
bench([&data, &measured_volume, &output_volume]() {
15+
std::string output = rfl::json::write(data);
16+
measured_volume = output.size();
17+
if (measured_volume != output_volume) {
18+
printf("mismatch\n");
19+
}
20+
}));
2121
}

‎benchmarks/src/benchmark_reflect_serialization.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
#define BENCHMARK_REFLECT_SERIALIZATION_TWITTER_HPP
33

44
#include "user_profile.hpp"
5-
void bench_reflect_cpp(std::vector<User>& data);
5+
void bench_reflect_cpp(std::vector<User> &data);
66
#endif

0 commit comments

Comments
 (0)