CXX-1383 implement composite score collector

Author: iwysiu

benchmark/CMakeLists.txt

@@ -26,9 +26,19 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Gv")
 endif()
 
-add_custom_target(benchmark DEPENDS run_benchmark)
+include_directories(
+    ${CMAKE_INSTALL_PREFIX}/${BSONCXX_HEADER_INSTALL_DIR}
+    ${CMAKE_INSTALL_PREFIX}/${MONGOCXX_HEADER_INSTALL_DIR}
+)
 
-add_library(run_benchmark benchmark_runner.cpp score_recorder.cpp bson_encoding.hpp)
+set(BENCHMARK_LIBRARY
+    benchmark_runner.cpp
+    main.cpp
+    microbench.cpp
+    score_recorder.cpp
+)
 
-target_include_directories(run_benchmark PUBLIC "/usr/local/include/mongocxx/v_noabi" "/usr/local/include/bsoncxx/v_noabi")
-target_link_libraries(run_benchmark mongocxx bsoncxx)
+add_executable(microbenchmarks ${BENCHMARK_LIBRARY})
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+target_link_libraries(microbenchmarks mongocxx bsoncxx Threads::Threads)

benchmark/benchmark_runner.cpp

@@ -12,11 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <chrono>
+#include "benchmark_runner.hpp"
 
-#include <mongocxx/instance.hpp>
 #include "bson/bson_encoding.hpp"
-#include "microbench.hpp"
 #include "multi_doc/bulk_insert.hpp"
 #include "multi_doc/find_many.hpp"
 #include "multi_doc/gridfs_download.hpp"
@@ -26,60 +24,86 @@
 #include "single_doc/run_command.hpp"
 
 namespace benchmark {
-const std::chrono::duration<int, std::milli> mintime{60000};
-const std::chrono::duration<int, std::milli> maxtime{300000};
 
-bool finished_running(const std::chrono::duration<int, std::milli>& curr_time, std::uint32_t iter) {
-    return (curr_time > maxtime || (curr_time > mintime && iter > 100));
-}
+// The task sizes and iteration numbers come from the Driver Perfomance Benchmarking Reference Doc.
+benchmark_runner::benchmark_runner() {
+    using bsoncxx::stdx::make_unique;
 
-void run_microbench(microbench* bench,
-                    bsoncxx::stdx::string_view input_json_filename = bsoncxx::stdx::string_view()) {
-    bench->setup(input_json_filename);
+    // Bson microbenchmarks
+    _microbenches.push_back(make_unique<bson_encoding>(75.31, "extended_bson/flat_bson.json"));
+    _microbenches.push_back(make_unique<bson_encoding>(19.64, "extended_bson/deep_bson.json"));
+    _microbenches.push_back(make_unique<bson_encoding>(57.34, "extended_bson/full_bson.json"));
+    // TODO CXX-1241: Add bson_decoding equivalents.
 
-    for (std::uint32_t iteration = 0; !finished_running(bench->get_execution_time(), iteration);
-         iteration++) {
-        bench->before_task();
+    // Single doc microbenchmarks
+    _microbenches.push_back(make_unique<run_command>());
+    _microbenches.push_back(make_unique<find_one_by_id>("single_and_multi_document/tweet.json"));
+    _microbenches.push_back(
+        make_unique<insert_one>(2.75, 10000, "single_and_multi_document/small_doc.json"));
+    _microbenches.push_back(
+        make_unique<insert_one>(27.31, 10, "single_and_multi_document/large_doc.json"));
 
-        bench->do_task();
+    // Multi doc microbenchmarks
+    _microbenches.push_back(make_unique<find_many>("single_and_multi_document/tweet.json"));
+    _microbenches.push_back(
+        make_unique<bulk_insert>(2.75, 10000, "single_and_multi_document/small_doc.json"));
+    _microbenches.push_back(
+        make_unique<bulk_insert>(27.31, 10, "single_and_multi_document/large_doc.json"));
+    _microbenches.push_back(
+        make_unique<gridfs_upload>("single_and_multi_document/gridfs_large.bin"));
+    _microbenches.push_back(
+        make_unique<gridfs_download>("single_and_multi_document/gridfs_large.bin"));
 
-        bench->after_task();
-    }
-    bench->teardown();
+    // TODO CXX-1378: add parallel microbenchmarks
 }
 
-// this would run the benchmarks and collect the data.
-int main() {
+void benchmark_runner::run_microbenches(benchmark_type tag) {
     mongocxx::instance instance{};
 
-    bson_encoding flat_bson_encode;
-    run_microbench(&flat_bson_encode, "FLAT_BSON.json");
-    bson_encoding deep_bson_encode;
-    run_microbench(&deep_bson_encode, "DEEP_BSON.json");
-    bson_encoding full_bson_encode;
-    run_microbench(&full_bson_encode, "FULL_BSON.json");
-    // TODO CXX-1241: Add bson_decoding equivalents.
+    for (std::unique_ptr<microbench>& bench : _microbenches) {
+        if (tag == benchmark::benchmark_type::all_benchmarks || bench->has_tag(tag)) {
+            bench->run();
+        }
+    }
+}
+
+double benchmark_runner::calculate_average(benchmark_type tag) {
+    std::uint32_t count = 0;
+    double total = 0.0;
+    for (std::unique_ptr<microbench>& bench : _microbenches) {
+        if (bench->has_tag(tag)) {
+            count++;
+            total += bench->get_results().get_score();
+        }
+    }
+    return total / static_cast<double>(count);
+}
+
+double benchmark_runner::calculate_bson_bench_score() {
+    return calculate_average(benchmark_type::bson_bench);
+}
+
+double benchmark_runner::calculate_single_bench_score() {
+    return calculate_average(benchmark_type::single_bench);
+}
+
+double benchmark_runner::calculate_multi_bench_score() {
+    return calculate_average(benchmark_type::multi_bench);
+}
+
+double benchmark_runner::calculate_parallel_bench_score() {
+    return calculate_average(benchmark_type::parallel_bench);
+}
+
+double benchmark_runner::calculate_read_bench_score() {
+    return calculate_average(benchmark_type::read_bench);
+}
+
+double benchmark_runner::calculate_write_bench_score() {
+    return calculate_average(benchmark_type::write_bench);
+}
 
-    run_command run_command_bench;
-    run_microbench(&run_command_bench);
-    find_one_by_id find_one_by_id_bench;
-    run_microbench(&find_one_by_id_bench, "TWEET.json");
-    insert_one small_doc_insert_one(10000);
-    run_microbench(&small_doc_insert_one, "SMALL_DOC.json");
-    insert_one large_doc_insert_one(10);
-    run_microbench(&large_doc_insert_one, "LARGE_DOC.json");
-
-    find_many find_many_bench;
-    run_microbench(&find_many_bench, "TWEET.json");
-    bulk_insert small_doc_bulk_insert{10000};
-    run_microbench(&small_doc_bulk_insert, "SMALL_DOC.json");
-    bulk_insert large_doc_bulk_insert{10};
-    run_microbench(&large_doc_bulk_insert, "LARGE_DOC.json");
-    gridfs_upload gridfs_upload_bench;
-    run_microbench(&gridfs_upload_bench, "GRIDFS_LARGE.bin");
-    gridfs_download gridfs_download_bench;
-    run_microbench(&gridfs_download_bench, "GRIDFS_LARGE.bin");
-
-    // get results from the microbenches...
+double benchmark_runner::calculate_driver_bench_score() {
+    return (calculate_read_bench_score() + calculate_write_bench_score()) / 2.0;
 }
 }

benchmark/benchmark_runner.hpp

@@ -0,0 +1,52 @@
+// Copyright 2017 MongoDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <mongocxx/instance.hpp>
+
+#include "microbench.hpp"
+
+namespace benchmark {
+
+class benchmark_runner {
+    // this would run the benchmarks and collect the data.
+
+   public:
+    // Initialize microbenchmarks with the size in MB of their tasks and, if necessary, number of
+    // iterations if multiple microbenchmarks use a subclass.
+    benchmark_runner();
+
+    void run_microbenches(benchmark_type type = benchmark_type::all_benchmarks);
+
+    double calculate_bson_bench_score();
+
+    double calculate_single_bench_score();
+
+    double calculate_multi_bench_score();
+
+    double calculate_parallel_bench_score();
+
+    double calculate_read_bench_score();
+
+    double calculate_write_bench_score();
+
+    double calculate_driver_bench_score();
+
+   private:
+    double calculate_average(benchmark_type);
+
+    std::vector<std::unique_ptr<microbench>> _microbenches;
+};
+}

benchmark/bson/bson_decoding.hpp

@@ -20,9 +20,13 @@ namespace benchmark {
 
 class bson_decoding : public microbench {
    public:
-    bson_decoding() : microbench{10000} {}
+    bson_decoding() = delete;
 
-    void setup(bsoncxx::stdx::string_view);
+    bson_decoding(double task_size, bsoncxx::stdx::string_view json_file)
+        : microbench{task_size, "bson_decoding"} {
+        _tags.insert(benchmark_type::bson_bench);
+        _json = parse_json_file_to_strings(json_file)[0];
+    }
 
    protected:
     void task();
@@ -31,10 +35,6 @@ class bson_decoding : public microbench {
     std::string _json;
 }
 
-void bson_decoding::setup(bsoncxx::stdx::string_view json_file){
-    _json = parse_json_file_to_strings(json_file)[0];
-}
-
 void bson_decoding::task() {
     for (std::uint32_t i = 0; i < 10000; i++) {
         // TODO CXX-1241: call bson_as_extended json on _json.

benchmark/bson/bson_encoding.hpp

@@ -15,31 +15,52 @@
 #pragma once
 
 #include <bsoncxx/json.hpp>
+#include <bsoncxx/types.hpp>
+
+#include <iostream>
 #include "../microbench.hpp"
 
 namespace benchmark {
 
 class bson_encoding : public microbench {
    public:
     // TODO: need to wait for scoring object to be finished to implement constructor
-    bson_encoding() : microbench{10000} {}
+    bson_encoding() = delete;
 
-    void setup(bsoncxx::stdx::string_view);
+    bson_encoding(double task_size, bsoncxx::stdx::string_view json_file)
+        : microbench{task_size, "bson_encoding"},
+          _json{parse_json_file_to_strings(json_file)[0]},
+          _doc{bsoncxx::from_json(bsoncxx::stdx::string_view{_json})} {
+        _tags.insert(benchmark_type::bson_bench);
+    }
 
    protected:
     void task();
+    void teardown();
 
    private:
+    std::int64_t _x;
+    std::string _s;
     std::string _json;
+    bsoncxx::document::value _doc;
 };
 
-void bson_encoding::setup(bsoncxx::stdx::string_view json_file) {
-    _json = parse_json_file_to_strings(json_file)[0];
-}
-
 void bson_encoding::task() {
-    for (std::uint32_t i = 0; i < 10000; i++) {
-        bsoncxx::document::value doc = bsoncxx::from_json(bsoncxx::stdx::string_view{_json});
+    // bsoncxx::stdx::string_view json_view{_json};
+    // for (std::uint32_t i = 0; i < 10000; i++) {
+    //    bsoncxx::from_json(bsoncxx::stdx::string_view{_json});
+    //}
+    _x = 0;
+    for (auto&& it : _doc.view()) {
+        if (it.type() == bsoncxx::type::k_utf8) {
+            _s = it.get_utf8().value.to_string();
+        }
     }
+
+    // std::cout << _x << std::endl;
+}
+
+void bson_encoding::teardown() {
+    std::cout << _x << std::endl;
 }
 }

benchmark/composite_score.hpp

@@ -0,0 +1,33 @@
+// Copyright 2017 MongoDB Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+
+#include "benchmark_runner.hpp"
+
+using namespace benchmark;
+
+int main() {
+    benchmark_runner runner;
+    runner.run_microbenches();
+
+    std::cout << "BSONBench: " << runner.calculate_bson_bench_score() << " MB/s" << std::endl;
+    std::cout << "SingleBench: " << runner.calculate_single_bench_score() << " MB/s" << std::endl;
+    std::cout << "MultiBench: " << runner.calculate_multi_bench_score() << "MB/s" << std::endl;
+    std::cout << "ParallelBench: " << runner.calculate_parallel_bench_score() << "MB/s"
+              << std::endl;
+    std::cout << "ReadBench: " << runner.calculate_read_bench_score() << "MB/s" << std::endl;
+    std::cout << "WriteBench: " << runner.calculate_write_bench_score() << "MB/s" << std::endl;
+    std::cout << "DriverBench: " << runner.calculate_driver_bench_score() << "MB/s" << std::endl;
+}