adding a demo and making things look nicer

Author: Daniel Lemire

CMakeLists.txt

@@ -52,7 +52,7 @@ target_include_directories(
 include(cmake/CPM.cmake)
 CPMAddPackage(
   NAME simdjson
-  GITHUB_REPOSITORY the-moisrex/simdjson-custom-types
+  GITHUB_REPOSITORY simdjson/simdjson
   GIT_TAG builder_development_branch
 ) # include simdjson
 

README.md

@@ -57,7 +57,7 @@ Or use one of the Macros described
 
 By leveraging compile-time reflection, we can support for something as simple as:
 
-```c++
+```cpp
 int main() {
     X s1 = {.a = '1', .b = 10, .c = 0, .d = "test string\n\r\"", .e = {1, 2, 3}, .f = {"ab", "cd", "fg"}};
 
@@ -66,7 +66,31 @@ int main() {
 }
 ```
 
-As of now, we were able to implement this with ~1k lines of code (and that is including Neon + SSE2 optimizations in [json_escaping.hpp](src/json_escaping.hpp)).
+We can make the following example work at compile time:
+```cpp
+struct kid {
+  int age;
+  std::string name;
+  std::vector<std::string> toys;
+};
+
+void demo() {
+  simdjson::padded_string json_str =
+      R"({"age": 12, "name": "John", "toys": ["car", "ball"]})"_padded;
+  simdjson::ondemand::parser parser;
+  auto doc = parser.iterate(json_str);
+  kid k = doc.get<kid>();
+  std::print("I am {} years old\n", k.age);
+  for (const auto &toy : k.toys) {
+    std::print("I have a {}\n", toy);
+  }
+  std::print("My name is {}\n", k.name);
+
+  std::print("My JSON is {}\n", simdjson::json_builder::to_json_string(k));
+}
+```
+
+
 
 The benchmark results show that our serialization speed can be 20/30x faster than [nlohmann](https://github.com/nlohmann/json).
 

benchmarks/src/benchmark_serialization.cpp

@@ -163,8 +163,7 @@ void bench_custom(std::vector<User> &data) {
 }
 
 template <class T> void bench_fast_simpler(std::vector<T> &data) {
-  simdjson::json_builder::StringBuilder b(32 * 1024 *
-                                          1024); // pre-allocate 32 MB
+  simdjson::json_builder::string_builder b;
   simdjson::json_builder::fast_to_json_string(b, data);
   size_t output_volume = b.size();
   b.reset();
@@ -182,6 +181,23 @@ template <class T> void bench_fast_simpler(std::vector<T> &data) {
                }));
 }
 
+
+template <class T> void bench_fast_with_alloc(std::vector<T> &data) {
+  std::string json = simdjson::json_builder::to_json_string(data);
+  size_t output_volume = json.size();
+  printf("# output volume: %zu bytes\n", output_volume);
+
+  volatile size_t measured_volume = 0;
+  pretty_print(data.size(), output_volume, "bench_fast_with_alloc",
+               bench([&data, &measured_volume, &output_volume]() {
+                 std::string json = simdjson::json_builder::to_json_string(data);
+                 measured_volume = json.size();
+                 if (measured_volume != output_volume) {
+                   printf("mismatch\n");
+                 }
+               }));
+}
+
 void bench_nlohmann(std::vector<User> &data) {
   std::string output = nlohmann_serialize(data);
   size_t output_volume = output.size();
@@ -206,8 +222,9 @@ int main() {
 
   std::vector<std::vector<User>> in_array = {test_data};
   bench_nlohmann(test_data);
-  bench_custom(test_data);
+  //bench_custom(test_data);
   bench_fast_simpler(test_data);
+  bench_fast_with_alloc(test_data);
 #if SIMDJSON_BENCH_CPP_REFLECT
   bench_reflect_cpp(test_data);
 #endif

benchmarks/src/benchmark_serialization_twitter.cpp

@@ -19,8 +19,7 @@
 #endif
 
 template <class T> void bench_fast_simpler(T &data) {
-  simdjson::json_builder::StringBuilder b(32 * 1024 *
-                                          1024); // pre-allocate 32 MB
+  simdjson::json_builder::string_builder b;
   simdjson::json_builder::fast_to_json_string(b, data);
   size_t output_volume = b.size();
   b.reset();
@@ -38,6 +37,24 @@ template <class T> void bench_fast_simpler(T &data) {
                }));
 }
 
+
+template <class T> void bench_fast_with_alloc(T &data) {
+  std::string json = simdjson::json_builder::to_json_string(data);
+  size_t output_volume = json.size();
+  printf("# output volume: %zu bytes\n", output_volume);
+
+  volatile size_t measured_volume = 0;
+  pretty_print(sizeof(data), output_volume, "bench_fast_with_alloc",
+               bench([&data, &measured_volume, &output_volume]() {
+                 std::string json = simdjson::json_builder::to_json_string(data);
+
+                 measured_volume = json.size();
+                 if (measured_volume != output_volume) {
+                   printf("mismatch\n");
+                 }
+               }));
+}
+
 void bench_nlohmann(TwitterData &data) {
   std::string output = nlohmann_serialize(data);
   size_t output_volume = output.size();
@@ -143,8 +160,7 @@ void test_correctness(std::string_view json_str) {
   // Now let's do a round-trip
 
   // Serializing the simd_struct back to a string
-  simdjson::json_builder::StringBuilder b(32 * 1024 *
-                                          1024); // pre-allocate 32 MB
+  simdjson::json_builder::string_builder b;
   simdjson::json_builder::fast_to_json_string(b, simd_struct);
   std::string simd_struct_to_json = std::string(b.view());
 
@@ -173,9 +189,10 @@ int main() {
   simpleparser::json_parser::JsonParser parser(json_str);
   auto json_value = parser.parse();
   TwitterData my_struct;
-  simpleparser::json_builder::from_json(json_value, my_struct);
-  bench_fast_simpler(my_struct);
   bench_nlohmann(my_struct);
+  //simpleparser::json_builder::from_json(json_value, my_struct);
+  bench_fast_simpler(my_struct);
+  bench_fast_with_alloc(my_struct);
 #if SIMDJSON_BENCH_CPP_REFLECT
   bench_reflect_cpp(my_struct);
 #endif

examples/CMakeLists.txt

@@ -1,6 +1,9 @@
 add_compile_options(-Wall -Wextra -Weffc++ -Wsign-compare -Wshadow -Wwrite-strings
 -Wpointer-arith -Winit-self -Wconversion -Wno-sign-conversion)
 
+add_executable(demo demo.cpp)
+target_link_libraries(demo PRIVATE simdjson::serialization simdjson::simdjson)
+
 add_executable(example example.cpp)
 target_link_libraries(example PRIVATE simdjson::serialization simdjson::simdjson)
 

examples/example.cpp

@@ -40,7 +40,7 @@ int main() {
                 .i = {1, 2, 3},
                 .z = {.x = 1000}}};
 
-  simdjson::json_builder::StringBuilder sb;
+  simdjson::json_builder::string_builder sb;
   simdjson::json_builder::fast_to_json_string(sb, s1);
   std::cout << sb.c_str() << std::endl;
   return EXIT_SUCCESS;

examples/example2.cpp

@@ -139,19 +139,5 @@ int main() {
   std::cout << my_struct.id << std::endl;
   std::cout << my_struct.name << std::endl;
   std::cout << my_struct.values.size() << std::endl;
-
-  /* simdjson::json_builder::StringBuilder sb;
-  simdjson::json_builder::fast_to_json_string(sb, my_struct);
-  std::cout << sb.c_str() << std::endl;
-
-  std::string json_str_nested =
-      R"({"a":1,"b":10,"c":0,"d":"test string\n\r\"","e":[1,2,3],"f":["ab","cd","fg"],"y":{"g":100,"h":"test string\n\r\"","i":[1,2,3]}})";
-  doc = parser.iterate(json_str_nested);
-
-  X s1 = X(doc);
-  simdjson::json_builder::StringBuilder sb2;
-  simdjson::json_builder::fast_to_json_string(sb2, s1);
-  std::cout << sb2.c_str() << std::endl;*/
-
   return 0;
 }

include/simdjson/json_builder/json_builder.h

@@ -54,7 +54,7 @@ template <typename R> consteval auto expand(R range) {
 
 template <class T>
   requires(ContainerButNotString<T>)
-constexpr void atom(StringBuilder &b, const T &t) {
+constexpr void atom(string_builder &b, const T &t) {
   if (t.size() == 0) {
     b.append_unescaped("[]");
     return;
@@ -72,11 +72,11 @@ template <class T>
   requires(std::is_same_v<T, std::string> ||
            std::is_same_v<T, std::string_view> ||
            std::is_same_v<T, const char *>)
-constexpr void atom(StringBuilder &b, const T &t) {
+constexpr void atom(string_builder &b, const T &t) {
   b.append(t);
 }
 
-template <arithmetic T> constexpr void atom(StringBuilder &b, const T t) {
+template <arithmetic T> constexpr void atom(string_builder &b, const T t) {
   b.append(t);
 }
 
@@ -85,7 +85,7 @@ template <class T>
            !std::is_same_v<T, std::string> &&
            !std::is_same_v<T, std::string_view> &&
            !std::is_same_v<T, const char *>)
-constexpr void atom(StringBuilder &b, const T &t) {
+constexpr void atom(string_builder &b, const T &t) {
   int i = 0;
   b.append('{');
   [:expand(std::meta::nonstatic_data_members_of(^T)):] >> [&]<auto dm> {
@@ -103,7 +103,7 @@ constexpr void atom(StringBuilder &b, const T &t) {
 }
 
 // works for struct
-template <class Z> void fast_to_json_string(StringBuilder &b, const Z &z) {
+template <class Z> void fast_to_json_string(string_builder &b, const Z &z) {
   int i = 0;
   b.append('{');
   [:expand(std::meta::nonstatic_data_members_of(^Z)):] >> [&]<auto dm> {
@@ -123,7 +123,7 @@ template <class Z> void fast_to_json_string(StringBuilder &b, const Z &z) {
 // works for container
 template <class Z>
   requires(ContainerButNotString<Z>)
-void fast_to_json_string(StringBuilder &b, const Z &z) {
+void fast_to_json_string(string_builder &b, const Z &z) {
   if (z.size() == 0) {
     b.append_unescaped("[]");
     return;
@@ -137,6 +137,20 @@ void fast_to_json_string(StringBuilder &b, const Z &z) {
   b.append(']');
 }
 
+template <class Z>
+std::string to_json_string(const Z &z) {
+  string_builder b;
+  fast_to_json_string(b, z);
+  return std::string(b);
+}
+
+template <class Z>
+void to_json(const Z &z, std::string &s) {
+  string_builder b;
+  fast_to_json_string(b, z);
+  s.assign(b.view());
+}
+
 } // namespace json_builder
 
 } // namespace simdjson

include/simdjson/json_builder/string_builder.h

@@ -13,10 +13,9 @@ namespace json_builder {
 template <typename T>
 concept arithmetic = std::is_arithmetic_v<T>;
 
-class StringBuilder final {
+class string_builder final {
 public:
-  // By default, we allocate 1 MB.
-  StringBuilder(size_t initial_capacity = 1048576)
+  string_builder(size_t initial_capacity = 1024)
       : buffer(new char[initial_capacity]), position(0),
         capacity(initial_capacity) {}
 
@@ -85,6 +84,10 @@ class StringBuilder final {
     position += len;
   }
 
+  operator std::string() const { return std::string(view()); }
+
+  operator std::string_view() const { return view(); }
+
   std::string_view view() const {
     return std::string_view(buffer.get(), position);
   }

include/simdjson/json_builder/universal_formatter.h

@@ -12,7 +12,7 @@ namespace json_builder {
 struct universal_formatter {
   constexpr auto parse(auto &ctx) { return ctx.begin(); }
 
-  template <typename T> auto format(T const &t, StringBuilder &sb) const {
+  template <typename T> auto format(T const &t, string_builder &sb) const {
     sb.append('{');
     auto delim = [first = true, &sb]() mutable {
       if (!first) {