From c9110ed667a20359b47ce94ed3ac8e14c740f519 Mon Sep 17 00:00:00 2001 From: hezhiqiang Date: Sun, 26 Jan 2025 19:33:24 +0800 Subject: [PATCH] FMR --- be/src/util/interval_histogram.cpp | 88 ---------------------- be/src/util/interval_histogram.h | 48 ------------ be/src/util/threadpool.cpp | 22 ++---- be/src/util/threadpool.h | 8 +- be/test/util/interval_histogram_test.cpp | 96 ------------------------ 5 files changed, 8 insertions(+), 254 deletions(-) delete mode 100644 be/src/util/interval_histogram.cpp delete mode 100644 be/src/util/interval_histogram.h delete mode 100644 be/test/util/interval_histogram_test.cpp diff --git a/be/src/util/interval_histogram.cpp b/be/src/util/interval_histogram.cpp deleted file mode 100644 index 0095afad45be0f..00000000000000 --- a/be/src/util/interval_histogram.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/interval_histogram.h" - -#include -#include -#include -#include - -#include "gutil/integral_types.h" - -namespace doris { - -template -IntervalHistogramStat::IntervalHistogramStat(size_t N) : window(N) {} - -template -void IntervalHistogramStat::add(T value) { - std::unique_lock lock(mutex); - if (window.full()) { - window.pop_front(); - } - window.push_back(value); -} - -template -T IntervalHistogramStat::mean() { - std::shared_lock lock(mutex); - if (window.empty()) { - return T(); - } - T sum = std::accumulate(window.begin(), window.end(), T()); - return sum / window.size(); -} - -template -T IntervalHistogramStat::median() { - std::shared_lock lock(mutex); - if (window.empty()) { - return T(); - } - - std::vector sorted(window.begin(), window.end()); - std::sort(sorted.begin(), sorted.end()); - - size_t mid = sorted.size() / 2; - return sorted.size() % 2 == 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; -} - -template -T IntervalHistogramStat::max() { - std::shared_lock lock(mutex); - return *std::max_element(window.begin(), window.end()); -} - -template -T IntervalHistogramStat::min() { - std::shared_lock lock(mutex); - return *std::min_element(window.begin(), window.end()); -} - -template -void IntervalHistogramStat::reduce_size(size_t n) { - std::unique_lock lock(mutex); - for (size_t i = 0; i < n && !window.empty(); ++i) { - window.pop_front(); - } -} - -template class doris::IntervalHistogramStat; -template class doris::IntervalHistogramStat; - -} // namespace doris diff --git a/be/src/util/interval_histogram.h b/be/src/util/interval_histogram.h deleted file mode 100644 index e103974e728b57..00000000000000 --- a/be/src/util/interval_histogram.h +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -namespace doris { - -// A thread-safe interval histogram stat class. -// IntervalHistogramStat will keep a FIXED-SIZE window of values and provide -// statistics like mean, median, max, min. - -template -class IntervalHistogramStat { -public: - explicit IntervalHistogramStat(size_t N); - - void add(T value); - - T mean(); - T median(); - T max(); - T min(); - - void reduce_size(size_t n); // New method to reduce the size of the window - -private: - boost::circular_buffer window; - mutable std::shared_mutex mutex; -}; - -} // namespace doris diff --git a/be/src/util/threadpool.cpp b/be/src/util/threadpool.cpp index 0e7b216601577b..f0f504b4dc218b 100644 --- a/be/src/util/threadpool.cpp +++ b/be/src/util/threadpool.cpp @@ -46,10 +46,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(thread_pool_queue_size, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(thread_pool_max_queue_size, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(thread_pool_max_threads, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(thread_pool_submit_failed, MetricUnit::NOUNIT); -DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(task_execution_time_ns_avg_in_last_1000_times, - MetricUnit::NANOSECONDS); -DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(task_wait_worker_ns_avg_in_last_1000_times, - MetricUnit::NANOSECONDS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(task_execution_time_ns_total, MetricUnit::NANOSECONDS); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(task_wait_worker_time_ns_total, MetricUnit::NANOSECONDS); using namespace ErrorCode; using std::string; @@ -311,8 +309,8 @@ Status ThreadPool::init() { INT_GAUGE_METRIC_REGISTER(_metric_entity, thread_pool_max_threads); INT_GAUGE_METRIC_REGISTER(_metric_entity, thread_pool_queue_size); INT_GAUGE_METRIC_REGISTER(_metric_entity, thread_pool_max_queue_size); - INT_GAUGE_METRIC_REGISTER(_metric_entity, task_execution_time_ns_avg_in_last_1000_times); - INT_GAUGE_METRIC_REGISTER(_metric_entity, task_wait_worker_ns_avg_in_last_1000_times); + INT_COUNTER_METRIC_REGISTER(_metric_entity, task_execution_time_ns_total); + INT_COUNTER_METRIC_REGISTER(_metric_entity, task_wait_worker_time_ns_total); INT_COUNTER_METRIC_REGISTER(_metric_entity, thread_pool_submit_failed); _metric_entity->register_hook("update", [this]() { @@ -327,14 +325,6 @@ Status ThreadPool::init() { thread_pool_queue_size->set_value(get_queue_size()); thread_pool_max_queue_size->set_value(get_max_queue_size()); thread_pool_max_threads->set_value(max_threads()); - task_execution_time_ns_avg_in_last_1000_times->set_value( - _task_execution_time_ns_statistic.mean()); - task_wait_worker_ns_avg_in_last_1000_times->set_value( - _task_wait_worker_time_ns_statistic.mean()); - - // Remove older data from the histogram so that we can see curve could be more smooth. - _task_execution_time_ns_statistic.reduce_size(1); - _task_wait_worker_time_ns_statistic.reduce_size(1); }); return Status::OK(); } @@ -606,7 +596,7 @@ void ThreadPool::dispatch_thread() { DCHECK_EQ(ThreadPoolToken::State::RUNNING, token->state()); DCHECK(!token->_entries.empty()); Task task = std::move(token->_entries.front()); - _task_wait_worker_time_ns_statistic.add(task.submit_time_wather.elapsed_time()); + task_wait_worker_time_ns_total->increment(task.submit_time_wather.elapsed_time()); token->_entries.pop_front(); token->_active_threads++; --_total_queued_tasks; @@ -624,7 +614,7 @@ void ThreadPool::dispatch_thread() { // with this threadpool, and produce a deadlock. task.runnable.reset(); l.lock(); - _task_execution_time_ns_statistic.add(task_execution_time_watch.elapsed_time()); + task_execution_time_ns_total->increment(task_execution_time_watch.elapsed_time()); // Possible states: // 1. The token was shut down while we ran its task. Transition to QUIESCED. // 2. The token has no more queued tasks. Transition back to IDLE. diff --git a/be/src/util/threadpool.h b/be/src/util/threadpool.h index 8c262fdfb46346..73f0ae33c79a79 100644 --- a/be/src/util/threadpool.h +++ b/be/src/util/threadpool.h @@ -40,7 +40,6 @@ #include "agent/cgroup_cpu_ctl.h" #include "common/status.h" -#include "util/interval_histogram.h" #include "util/metrics.h" #include "util/uid_util.h" #include "util/work_thread_pool.hpp" @@ -414,11 +413,8 @@ class ThreadPool { IntGauge* thread_pool_queue_size = nullptr; IntGauge* thread_pool_max_queue_size = nullptr; IntGauge* thread_pool_max_threads = nullptr; - IntGauge* task_execution_time_ns_avg_in_last_1000_times = nullptr; - IntGauge* task_wait_worker_ns_avg_in_last_1000_times = nullptr; - - IntervalHistogramStat _task_execution_time_ns_statistic {1000}; - IntervalHistogramStat _task_wait_worker_time_ns_statistic {1000}; + IntCounter* task_execution_time_ns_total = nullptr; + IntCounter* task_wait_worker_time_ns_total = nullptr; IntCounter* thread_pool_submit_failed = nullptr; }; diff --git a/be/test/util/interval_histogram_test.cpp b/be/test/util/interval_histogram_test.cpp deleted file mode 100644 index be70dbbd3b4dae..00000000000000 --- a/be/test/util/interval_histogram_test.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/interval_histogram.h" - -#include - -#include - -namespace doris { - -TEST(IntervalHistogramStat, SerialTest) { - IntervalHistogramStat stat(5); - - stat.add(10); - stat.add(20); - stat.add(30); - stat.add(40); - stat.add(50); - - EXPECT_EQ(stat.mean(), 30); - EXPECT_EQ(stat.median(), 30); - EXPECT_EQ(stat.max(), 50); - EXPECT_EQ(stat.min(), 10); - - // Make window move forward - stat.add(60); - stat.add(70); - - // window now contains [30, 40, 50, 60, 70] - EXPECT_EQ(stat.mean(), 50); - EXPECT_EQ(stat.median(), 50); - EXPECT_EQ(stat.max(), 70); - EXPECT_EQ(stat.min(), 30); -} - -TEST(IntervalHistogramStatTest, ParallelTest) { - constexpr int thread_count = 10; - constexpr int values_per_thread = 10; - IntervalHistogramStat stat(thread_count * values_per_thread); - - auto add_values = [&stat](int start_value, int count) { - for (int i = 0; i < count; ++i) { - stat.add(start_value + i); - } - }; - - std::vector threads; - for (int i = 0; i < thread_count; ++i) { - threads.emplace_back(add_values, i * values_per_thread, values_per_thread); - } - - for (auto& thread : threads) { - thread.join(); - } - - int total_values = thread_count * values_per_thread; - EXPECT_EQ(stat.mean(), (total_values - 1) / 2); - EXPECT_EQ(stat.max(), total_values - 1); - EXPECT_EQ(stat.min(), 0); - EXPECT_EQ(stat.median(), (total_values - 1) / 2); -} - -TEST(IntervalHistogramStatTest, ReduceSizeTest) { - IntervalHistogramStat stat(5); - - stat.add(10); - stat.add(20); - stat.add(30); - stat.add(40); - stat.add(50); - - stat.reduce_size(2); - - // window now contains [30, 40, 50] - EXPECT_EQ(stat.mean(), 40); - EXPECT_EQ(stat.median(), 40); - EXPECT_EQ(stat.max(), 50); - EXPECT_EQ(stat.min(), 30); -} - -} // namespace doris