From 2f1d3cd4c3f7df5b5c66259515665b454c00c4db Mon Sep 17 00:00:00 2001 From: Pavel Shramov Date: Fri, 20 Dec 2024 14:00:10 +0300 Subject: [PATCH] wip: logger: Write logs in separate thread --- src/logger/common.h | 42 ++++++++++++++ src/logger/logger.cc | 53 +++++++++--------- src/logger/thread.h | 128 +++++++++++++++++++++++++++++++++++++++++++ src/logger/util.h | 2 +- 4 files changed, 199 insertions(+), 26 deletions(-) create mode 100644 src/logger/common.h create mode 100644 src/logger/thread.h diff --git a/src/logger/common.h b/src/logger/common.h new file mode 100644 index 00000000..de46df8e --- /dev/null +++ b/src/logger/common.h @@ -0,0 +1,42 @@ +#ifndef _LOGGER_COMMON_H +#define _LOGGER_COMMON_H + +#include "tll/logger.h" +#include "tll/logger/impl.h" +#include "tll/util/refptr.h" +#include "tll/util/time.h" + +#include + +namespace tll::logger { + +struct tll_logger_obj_t : tll::util::refbase_t +{ + const char * name = ""; + void * obj = nullptr; + tll_logger_impl_t * impl = nullptr; + + ~tll_logger_obj_t() + { + if (obj && impl && impl->log_free) + impl->log_free(impl, name, obj); + } + + auto log(tll::time_point ts, tll_logger_level_t level, std::string_view body) + { + return (*impl->log)(ts.time_since_epoch().count(), name, level, body.data(), body.size(), obj); + } +}; + +struct Logger : public tll_logger_t, public tll::util::refbase_t +{ + void destroy(); + + std::mutex lock; + std::string name; + tll::util::refptr_t impl; +}; + +} // namespace tll::logger + +#endif//_LOGGER_COMMON_H diff --git a/src/logger/logger.cc b/src/logger/logger.cc index dd7efb5a..a9dcde0c 100644 --- a/src/logger/logger.cc +++ b/src/logger/logger.cc @@ -9,10 +9,10 @@ #include "tll/logger.h" #include "tll/logger/impl.h" #include "tll/util/refptr.h" +#include "tll/util/size.h" #include "tll/util/string.h" #include "tll/util/time.h" -#include #include #include #include @@ -22,8 +22,9 @@ #include #include +#include "logger/common.h" #include "logger/config.h" - +#include "logger/thread.h" #include "logger/util.h" #ifdef WITH_SPDLOG @@ -32,34 +33,14 @@ namespace tll::logger { -struct tll_logger_obj_t : tll::util::refbase_t -{ - const char * name = ""; - void * obj = nullptr; - tll_logger_impl_t * impl = nullptr; - - ~tll_logger_obj_t() - { - if (obj && impl && impl->log_free) - impl->log_free(impl, name, obj); - } -}; - -struct Logger : public tll_logger_t, public tll::util::refbase_t -{ - void destroy(); - - std::mutex lock; - std::string name; - tll::util::refptr_t impl; -}; - struct logger_context_t { std::shared_mutex lock; typedef std::unique_lock wlock_t; typedef std::shared_lock rlock_t; + std::unique_ptr _thread; + std::map> _loggers; std::map> _levels_prefix; std::map> _levels; @@ -68,6 +49,12 @@ struct logger_context_t static tll_logger_impl_t stdio; tll_logger_impl_t * impl = &stdio; + ~logger_context_t() + { + // Wait for thread to finish if it was spawned + _thread.reset(); + } + Logger * init(std::string_view name) { { @@ -213,6 +200,19 @@ struct logger_context_t int configure(const tll::ConstConfig &cfg) { + auto thread = cfg.getT("thread", false); + if (thread && *thread) { + auto size = cfg.getT("ring-size").value_or(128 * 1024); + //fmt::println(">>> Create new thread, size {}", size); + std::unique_ptr tmp { new Thread() }; + if (tmp->init(size)) + return 0; + + std::swap(_thread, tmp); + if (tmp) + tmp->stop(); + } + if (auto levels = cfg.sub("levels"); levels) { std::set skip; for (auto & [k, v] : levels->browse("**", true)) { @@ -320,7 +320,10 @@ int tll_logger_log(tll_logger_t * l, tll_logger_level_t level, const char * buf, auto ts = tll::time::now(); - return (*impl->impl->log)(ts.time_since_epoch().count(), impl->name, level, buf, len, impl->obj); + if (tll::logger::context._thread) { + return tll::logger::context._thread->push(log, ts, level, {buf, len}); + } else + return log->impl->log(ts, level, {buf, len}); } tll_logger_buf_t * tll_logger_tls_buf() diff --git a/src/logger/thread.h b/src/logger/thread.h new file mode 100644 index 00000000..2229ae15 --- /dev/null +++ b/src/logger/thread.h @@ -0,0 +1,128 @@ +#ifndef _LOGGER_THREAD_H +#define _LOGGER_THREAD_H + +#include "tll/cppring.h" +#include "tll/logger.h" +#include "tll/util/time.h" + +#include +#include +#include + +#include +#include +#include + +#include "logger/common.h" + +namespace tll::logger { + +struct Header +{ + uint16_t level = TLL_LOGGER_DEBUG; + tll::logger::Logger * logger = nullptr; + tll::time_point timestamp; +}; + +struct Thread +{ + std::mutex _lock; + std::shared_ptr _ring; + int _fd = -1; + bool _stop = false; + std::thread _thread; + + ~Thread() + { + stop(); + + if (_thread.joinable()) + _thread.join(); + _thread = {}; + + if (_fd != -1) + close(_fd); + _fd = -1; + } + + int init(size_t size) + { + _fd = eventfd(0, EFD_SEMAPHORE | EFD_NONBLOCK | EFD_CLOEXEC); + if (_fd == -1) + return EINVAL; + _ring.reset(tll::Ring::allocate(size).release()); + if (!_ring) + return EINVAL; + + _thread = std::thread([](Thread * self) { self->run(); }, this); + return 0; + } + + void stop() + { + _stop = true; + eventfd_write(_fd, 1); + } + + void run() + { + std::shared_ptr ring = _ring; + pollfd pfd = { .fd = _fd, .events = POLLIN }; + while (!_stop || !ring->empty()) { + auto r = poll(&pfd, 1, 1000); + if (r == 1) + step(); + else if (r < 0) + return; + else if (_stop) + return; + } + } + + void step() + { + const void * data; + size_t size; + if (_ring->read(&data, &size)) + return; + + eventfd_t v; + eventfd_read(_fd, &v); + + if (size < sizeof(Header)) + return; + + auto header = (const Header *) data; + + std::string_view body((const char *) (header + 1), size - sizeof(Header)); + + { + std::unique_lock lck(header->logger->lock); + header->logger->impl->log(header->timestamp, (tll_logger_level_t) header->level, body); + } + header->logger->unref(); + _ring->shift(); + } + + int push(Logger * log, tll::time_point ts, tll_logger_level_t level, std::string_view body) + { + std::unique_lock lock(_lock); + void * data; + if (_ring->write_begin(&data, sizeof(Header) + body.size())) { + _lock.unlock(); + return log->impl->log(ts, level, body); + } + auto header = (Header *) data; + header->logger = log->ref(); + header->timestamp = ts; + header->level = level; + memcpy(header + 1, body.data(), body.size()); + _ring->write_end(data, sizeof(Header) + body.size()); + eventfd_write(_fd, 1); + return 0; + } +}; + +} // namespace tll::logger + +#endif//_LOGGER_THREAD_H diff --git a/src/logger/util.h b/src/logger/util.h index d396cdce..5fc3ec2a 100644 --- a/src/logger/util.h +++ b/src/logger/util.h @@ -2,7 +2,7 @@ #define _LOGGER_UTIL_H #include "tll/logger.h" -#include "tll/util/conv.h" +#include "tll/conv/base.h" constexpr bool icmp(char a, char b) {