Metrics + OpenTelemetry Tracing (#541)

.pkg

@@ -69,15 +69,15 @@
 [boost]
   [email protected]:motis-project/boost.git
   branch=master
-  commit=1c3f21c1fa8b149da89e2f6bcb48b28fff30fa5e
+  commit=60cae66449fa3c9599b2b7d3d5d44c65301ed3a3
 [mimalloc]
   [email protected]:motis-project/mimalloc.git
   branch=master
   commit=0087f000848de31b0090cb6f282348bd2fd3a9b8
 [nigiri]
   [email protected]:motis-project/nigiri.git
   branch=master
-  commit=ada780c39ab11cfd6a1dd6e8b972f6b877f07d1b
+  commit=d6e1c8f5c0a8264e568fa85a34622e105a975315
 [osr]
   [email protected]:motis-project/osr.git
   branch=master
@@ -86,3 +86,11 @@
   [email protected]:triptix-tech/adr.git
   branch=master
   commit=bef3f11a8c1642939e7047bcebf3d2f3528b23a3
+[prometheus-cpp]
+  [email protected]:motis-project/prometheus-cpp.git
+  branch=master
+  commit=e420cd7cf3995a994220b40a36c987ac8e67c0bf
+[opentelemetry-cpp]
+  [email protected]:motis-project/opentelemetry-cpp.git
+  branch=main
+  commit=ec4aef6b17b697052edef5417825ad71947b2ed1

.pkg.lock

@@ -1,6 +1,6 @@
-5392555982345106832
+2672259012793768236
 cista f52a62c4d83377acd398227ab4fcd6c946bdbd70
-zlib fe8e13ffca867612951bc6baf114e5ac8b00f305
+zlib ee0742244d93c4237154ae16c3db42b5f284b442
 boost 60cae66449fa3c9599b2b7d3d5d44c65301ed3a3
 conf f9bf4bd83bf55a2170725707e526cbacc45dcc66
 context 797dd16e2b5e959997ddcd5bdeac4f80931169b6
@@ -18,15 +18,19 @@ guess cecba85dbd6796c92a7f9c4e274849bb6ef03494
 lmdb 9bd01f14f549d8202413c4cd5f49b066b0a22b66
 mimalloc 2a557cafb2e9e7c872358a83a63c62a7e14330b3
 miniz 1edbdece9d71dc65c6ff405572ee37cbdcef7af4
-libressl 39c1bf084d5c179d7bbce7ba902fffbebff0ee15
+libressl 24acd9e710fbe842e863572da9d738715fbc74b8
 net 785b39c08212732e510305f0eef18de70f19b15e
 PEGTL 1c1aa6e650e4d26f10fa398f148ec0cdc5f0808d
 oh d21c30f40e52a83d6dc09bcffd0067598b5ec069
+curl 3358dac21192864ef2ba47c88704d3b8d8203804
+json 410c74782230daaa15054d6ee0975c0607091cb3
+opentelemetry-proto 1624689398a3226c45994d70cb544a1e781dc032
 abseil-cpp ba5240842d352b4b67a32092453a2fe5fe53a62e
 protobuf d8136b9c6a62db6ce09900ecdeb82bb793096cbd
+opentelemetry-cpp ec4aef6b17b697052edef5417825ad71947b2ed1
 unordered_dense c11595a7743d20622637584bddf77243d72ae152
 wyhash 1e012b57fc2227a9e583a57e2eacb3da99816d99
-nigiri ada780c39ab11cfd6a1dd6e8b972f6b877f07d1b
+nigiri d6e1c8f5c0a8264e568fa85a34622e105a975315
 expat 636c9861e8e7c119f3626d1e6c260603ab624516
 libosmium d5cc2a02d997c2b464d37d37c3a75cd9efa23dc4
 protozero 8c9f3fa97c2cfdceef86d0b61818ae98e9328f29
@@ -45,6 +49,7 @@ tbb b3011be5060ec1be43c76d4a8cc80d5550adb31d
 osrm-backend 40975002f31c4f1eaa623880efaf579c6007edf9
 cpptoml 2133029ec819e8398e96fa679993b269f21ff9f2
 ppr 512cfefe0fe7c7a64ef0adefbf7af2adfe5200c4
+prometheus-cpp e420cd7cf3995a994220b40a36c987ac8e67c0bf
 FTXUI dd6a5d371fd7a3e2937bb579955003c54b727233
 tg 20c0f298b8ce58de29a790290f44dca7c4ecc364
 adr bef3f11a8c1642939e7047bcebf3d2f3528b23a3

CMakeLists.txt

@@ -3,6 +3,8 @@ project(motis)
 
 enable_language(CXX C ASM)
 
+set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
+
 if (MSVC)
   # PDB debug information is not supported by buildcache.
   # Store debug info in the object files.
@@ -46,8 +48,10 @@ option(MOTIS_WITH_WEBUI "enable motis-web-ui target" OFF)
 if(NOT DEFINED CMAKE_MSVC_RUNTIME_LIBRARY)
   if(MOTIS_MIMALLOC)
     set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
+    set(protobuf_MSVC_STATIC_RUNTIME OFF)
   else()
     set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+    set(protobuf_MSVC_STATIC_RUNTIME ON)
   endif()
 endif()
 
@@ -239,6 +243,7 @@ target_link_libraries(motis
   ianatzdb-res
   pbf_sdf_fonts_res-res
   tiles_server_res-res
+  opentelemetry_exporter_otlp_http
 )
 target_compile_options(motis PRIVATE ${MOTIS_CXX_FLAGS})
 set_target_properties(motis PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")

CMakePresets.json

@@ -257,4 +257,4 @@
       "configurePreset": "linux-sanitizer"
     }
   ]
-}
+}

base/bootstrap/CMakeLists.txt

@@ -26,4 +26,11 @@ add_library(motis-bootstrap STATIC ${motis-bootstrap-files})
 target_include_directories(motis-bootstrap PUBLIC include)
 target_compile_features(motis-bootstrap PUBLIC cxx_std_17)
 target_compile_options(motis-bootstrap PRIVATE ${MOTIS_CXX_FLAGS})
-target_link_libraries(motis-bootstrap conf utl motis-module ${module-targets})
+target_link_libraries(motis-bootstrap
+  conf
+  utl
+  prometheus-cpp::core
+  opentelemetry_trace
+  motis-module
+  ${module-targets}
+)

base/bootstrap/src/motis_instance.cc

@@ -4,15 +4,22 @@
 #include <exception>
 #include <filesystem>
 #include <iostream>
+#include <memory>
 
+#include "fmt/format.h"
 #include "fmt/ranges.h"
 
+#include "prometheus/registry.h"
+
+#include "opentelemetry/trace/tracer.h"
+
 #include "utl/pipes.h"
 #include "utl/progress_tracker.h"
 #include "utl/raii.h"
 #include "utl/verify.h"
 
 #include "motis/core/common/logging.h"
+#include "motis/core/otel/tracer.h"
 #include "motis/module/context/motis_call.h"
 #include "motis/module/context/motis_publish.h"
 #include "motis/bootstrap/import_files.h"
@@ -25,6 +32,8 @@ using namespace motis::logging;
 namespace motis::bootstrap {
 
 motis_instance::motis_instance() : controller{build_modules()} {
+  emplace_data(to_res_id(global_res_id::METRICS),
+               std::make_shared<prometheus::Registry>());
   for (auto& m : modules_) {
     m->set_shared_data(this);
   }
@@ -66,6 +75,9 @@ void motis_instance::import(module_settings const& module_opt,
                             bool const silent) {
   auto bars = utl::global_progress_bars{silent};
 
+  auto span = motis_tracer->StartSpan("import");
+  auto scope = opentelemetry::trace::Scope{span};
+
   auto dispatcher = import_dispatcher{};
 
   register_import_files(dispatcher);
@@ -103,23 +115,39 @@ void motis_instance::import(module_settings const& module_opt,
 
 void motis_instance::init_modules(module_settings const& module_opt,
                                   unsigned num_threads) {
+  auto outer_span = motis_tracer->StartSpan("init_modules");
+  auto outer_scope = opentelemetry::trace::Scope{outer_span};
+
   for (auto const& module : modules_) {
     if (!module_opt.is_module_active(module->prefix())) {
       continue;
     }
 
+    auto span =
+        motis_tracer->StartSpan(fmt::format("init {}", module->module_name()));
+    auto scope = opentelemetry::trace::Scope{outer_span};
+
     if (!module->import_successful()) {
       LOG(info) << module->module_name() << ": import was not successful";
+      span->SetStatus(opentelemetry::trace::StatusCode::kError,
+                      "import failed");
       continue;
     }
 
     try {
       module->init(registry_);
     } catch (std::exception const& e) {
+      span->AddEvent("exception", {
+                                      {"exception.message", e.what()},
+                                  });
+      span->SetStatus(opentelemetry::trace::StatusCode::kError, "exception");
       LOG(emrg) << "module " << module->module_name()
                 << ": unhandled init error: " << e.what();
       throw;
     } catch (...) {
+      span->AddEvent("exception", {{"exception.type", "unknown"}});
+      span->SetStatus(opentelemetry::trace::StatusCode::kError,
+                      "unknown error");
       LOG(emrg) << "module " << module->module_name()
                 << "unhandled unknown init error";
       throw;

base/core/CMakeLists.txt

@@ -13,4 +13,5 @@ target_link_libraries(motis-core
   geo
   motis-data
   motis-module
+  opentelemetry_api
 )

base/core/include/motis/core/common/timing.h

@@ -5,15 +5,27 @@
 #define MOTIS_START_TIMING(_X) \
   auto _X##_start = std::chrono::steady_clock::now()
 #define MOTIS_STOP_TIMING(_X) auto _X##_stop = std::chrono::steady_clock::now()
+
+#define MOTIS_TIMING_S(_X)                                               \
+  (std::chrono::duration_cast<std::chrono::duration<double>>(_X##_stop - \
+                                                             _X##_start) \
+       .count())
+
 #define MOTIS_TIMING_MS(_X)                                          \
   (std::chrono::duration_cast<std::chrono::milliseconds>(_X##_stop - \
                                                          _X##_start) \
        .count())
+
 #define MOTIS_TIMING_US(_X)                                          \
   (std::chrono::duration_cast<std::chrono::microseconds>(_X##_stop - \
                                                          _X##_start) \
        .count())
 
+#define MOTIS_GET_TIMING_S(_X)                                \
+  (std::chrono::duration_cast<std::chrono::duration<double>>( \
+       std::chrono::steady_clock::now() - _X##_start)         \
+       .count())
+
 #define MOTIS_GET_TIMING_MS(_X)                           \
   (std::chrono::duration_cast<std::chrono::milliseconds>( \
        std::chrono::steady_clock::now() - _X##_start)     \

base/core/include/motis/core/otel/tracer.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <memory>
+
+#include "opentelemetry/trace/tracer.h"
+
+namespace motis {
+
+extern std::shared_ptr<opentelemetry::trace::Tracer> motis_tracer;
+
+}  // namespace motis

base/core/src/otel/tracer.cc

@@ -0,0 +1,8 @@
+#include "motis/core/otel/tracer.h"
+
+namespace motis {
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::shared_ptr<opentelemetry::trace::Tracer> motis_tracer;
+
+}  // namespace motis

base/launcher/include/motis/launcher/http_text_map_carrier.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <type_traits>
+
+#include "boost/beast/http/fields.hpp"
+#include "boost/beast/http/message.hpp"
+
+#include "opentelemetry/context/propagation/global_propagator.h"
+#include "opentelemetry/context/propagation/text_map_propagator.h"
+#include "opentelemetry/trace/propagation/http_trace_context.h"
+
+namespace motis::launcher {
+
+template <typename T>
+struct http_text_map_carrier
+    : public opentelemetry::context::propagation::TextMapCarrier {
+  explicit http_text_map_carrier(T& header) : header_{header} {}
+
+  virtual opentelemetry::nostd::string_view Get(
+      opentelemetry::nostd::string_view key) const noexcept override {
+    return header_[key];
+  }
+
+  virtual void Set(opentelemetry::nostd::string_view key,
+                   opentelemetry::nostd::string_view value) noexcept override {
+    if constexpr (!std::is_const_v<T>) {
+      header_.set(key, value);
+    }
+  }
+
+  T& header_;
+};
+
+}  // namespace motis::launcher

base/launcher/include/motis/launcher/launcher_settings.h

@@ -51,6 +51,7 @@ class launcher_settings : public conf::configuration {
     param(init_, "init", "init operation");
     param(num_threads_, "num_threads", "number of worker threads");
     param(direct_mode_, "direct", "no ctx/multi-threading");
+    param(otlp_http_, "otlp_http", "enable OTLP HTTP exporter");
   }
 
   motis_mode_t mode_{launcher_settings::motis_mode_t::SERVER};
@@ -59,6 +60,7 @@ class launcher_settings : public conf::configuration {
   std::string init_;
   unsigned num_threads_{std::thread::hardware_concurrency()};
   bool direct_mode_{sizeof(void*) >= 8 ? false : true};
+  bool otlp_http_{};
 };
 
 }  // namespace motis::launcher

base/launcher/include/motis/launcher/web_server.h

@@ -5,12 +5,12 @@
 
 #include "boost/asio/io_service.hpp"
 
-#include "motis/module/receiver.h"
+#include "motis/module/controller.h"
 
 namespace motis::launcher {
 
 struct web_server {
-  web_server(boost::asio::io_service&, motis::module::receiver&);
+  web_server(boost::asio::io_service&, motis::module::controller&);
   ~web_server();
 
   web_server(web_server&&) = default;