From 28d40bacfaf3aca1c46232fea6bee357df9adde3 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 26 Jan 2025 16:38:07 +0800 Subject: [PATCH 1/2] [opt](kerberos) use ticket cache instead of keytab on BE side --- be/CMakeLists.txt | 3 + be/src/common/config.cpp | 3 +- be/src/common/config.h | 2 + be/src/common/kerberos/kerberos_config.cpp | 45 +++ be/src/common/kerberos/kerberos_config.h | 77 ++++ .../common/kerberos/kerberos_ticket_cache.cpp | 304 +++++++++++++++ .../common/kerberos/kerberos_ticket_cache.h | 142 +++++++ .../common/kerberos/kerberos_ticket_mgr.cpp | 178 +++++++++ be/src/common/kerberos/kerberos_ticket_mgr.h | 109 ++++++ be/src/common/kerberos/krb5_interface.h | 74 ++++ .../common/kerberos/krb5_interface_impl.cpp | 153 ++++++++ be/src/common/kerberos/krb5_interface_impl.h | 59 +++ be/src/exec/schema_scanner.cpp | 3 + .../schema_backend_kerberos_ticket_cache.cpp | 97 +++++ .../schema_backend_kerberos_ticket_cache.h} | 43 +-- .../schema_scanner/schema_scanner_helper.cpp | 19 + .../schema_scanner/schema_scanner_helper.h | 4 + be/src/io/file_factory.cpp | 1 + be/src/io/fs/hdfs/hdfs_mgr.cpp | 255 ++++++++++++ be/src/io/fs/hdfs/hdfs_mgr.h | 90 +++++ be/src/io/fs/hdfs_file_reader.cpp | 3 +- be/src/io/fs/hdfs_file_system.cpp | 184 ++------- be/src/io/fs/hdfs_file_system.h | 56 +-- be/src/io/fs/hdfs_file_writer.cpp | 14 +- be/src/io/hdfs_builder.cpp | 109 ++++-- be/src/io/hdfs_builder.h | 23 +- be/src/{util => io}/hdfs_util.cpp | 43 ++- be/src/io/hdfs_util.h | 93 +++++ be/src/runtime/exec_env.h | 11 + be/src/runtime/exec_env_init.cpp | 14 + be/src/util/jni-util.cpp | 11 +- .../common/kerberos/kerberos_config_test.cpp | 60 +++ .../kerberos_ticket_cache_auth_test.cpp | 133 +++++++ .../kerberos/kerberos_ticket_cache_test.cpp | 242 ++++++++++++ .../kerberos/kerberos_ticket_mgr_test.cpp | 141 +++++++ be/test/io/fs/hdfs/hdfs_mgr_test.cpp | 362 ++++++++++++++++++ .../doris/analysis/SchemaTableType.java | 4 +- .../org/apache/doris/catalog/SchemaTable.java | 16 + .../BackendPartitionedSchemaScanNode.java | 2 +- gensrc/thrift/Descriptors.thrift | 3 +- .../kerberos/test_single_hive_kerberos.groovy | 24 +- .../kerberos/test_two_hive_kerberos.groovy | 65 +++- 42 files changed, 2941 insertions(+), 333 deletions(-) create mode 100644 be/src/common/kerberos/kerberos_config.cpp create mode 100644 be/src/common/kerberos/kerberos_config.h create mode 100644 be/src/common/kerberos/kerberos_ticket_cache.cpp create mode 100644 be/src/common/kerberos/kerberos_ticket_cache.h create mode 100644 be/src/common/kerberos/kerberos_ticket_mgr.cpp create mode 100644 be/src/common/kerberos/kerberos_ticket_mgr.h create mode 100644 be/src/common/kerberos/krb5_interface.h create mode 100644 be/src/common/kerberos/krb5_interface_impl.cpp create mode 100644 be/src/common/kerberos/krb5_interface_impl.h create mode 100644 be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.cpp rename be/src/{util/hdfs_util.h => exec/schema_scanner/schema_backend_kerberos_ticket_cache.h} (50%) create mode 100644 be/src/io/fs/hdfs/hdfs_mgr.cpp create mode 100644 be/src/io/fs/hdfs/hdfs_mgr.h rename be/src/{util => io}/hdfs_util.cpp (59%) create mode 100644 be/src/io/hdfs_util.h create mode 100644 be/test/common/kerberos/kerberos_config_test.cpp create mode 100644 be/test/common/kerberos/kerberos_ticket_cache_auth_test.cpp create mode 100644 be/test/common/kerberos/kerberos_ticket_cache_test.cpp create mode 100644 be/test/common/kerberos/kerberos_ticket_mgr_test.cpp create mode 100644 be/test/io/fs/hdfs/hdfs_mgr_test.cpp diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 7d8579845fb07e..4e9e56f7ea3616 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -500,6 +500,9 @@ if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) hadoop_hdfs ) add_definitions(-DUSE_HADOOP_HDFS) + # USE_DORIS_HADOOP_HDFS means use hadoop deps from doris-thirdparty. + # the hadoop deps from doris-thirdparty contains some modification diff from the standard hadoop, such as log interface + add_definitions(-DUSE_DORIS_HADOOP_HDFS) else() add_library(hdfs3 STATIC IMPORTED) set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libhdfs3.a) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index f047071139e831..44eb0fba88d2f2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1153,8 +1153,9 @@ DEFINE_Int32(rocksdb_max_write_buffer_number, "5"); DEFINE_mBool(allow_zero_date, "false"); DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); -DEFINE_mString(kerberos_ccache_path, ""); +DEFINE_mString(kerberos_ccache_path, "/tmp/"); DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf"); +DEFINE_mInt32(kerberos_refresh_interval_second, "3600"); DEFINE_mString(get_stack_trace_tool, "libunwind"); DEFINE_mString(dwarf_location_info_mode, "FAST"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 29080a56defbcf..23a3a4da1c3726 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1193,6 +1193,8 @@ DECLARE_mBool(allow_invalid_decimalv2_literal); DECLARE_mString(kerberos_ccache_path); // set krb5.conf path, use "/etc/krb5.conf" by default DECLARE_mString(kerberos_krb5_conf_path); +// the interval for renew kerberos ticket cache +DECLARE_mInt32(kerberos_refresh_interval_second); // Values include `none`, `glog`, `boost`, `glibc`, `libunwind` DECLARE_mString(get_stack_trace_tool); diff --git a/be/src/common/kerberos/kerberos_config.cpp b/be/src/common/kerberos/kerberos_config.cpp new file mode 100644 index 00000000000000..24827ec6c8b125 --- /dev/null +++ b/be/src/common/kerberos/kerberos_config.cpp @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_config.h" + +#include + +#include "common/config.h" +#include "util/md5.h" + +namespace doris::kerberos { + +KerberosConfig::KerberosConfig() + : _refresh_interval_second(3600), _min_time_before_refresh_second(600) {} + +std::string KerberosConfig::get_hash_code(const std::string& principal, const std::string& keytab) { + return _get_hash_code(principal, keytab); +} + +std::string KerberosConfig::_get_hash_code(const std::string& principal, + const std::string& keytab) { + // use md5(principal + keytab) as hash code + // so that same (principal + keytab) will have same name. + std::string combined = principal + keytab; + Md5Digest digest; + digest.update(combined.c_str(), combined.length()); + digest.digest(); + return digest.hex(); +} + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/kerberos_config.h b/be/src/common/kerberos/kerberos_config.h new file mode 100644 index 00000000000000..af6bf84746328d --- /dev/null +++ b/be/src/common/kerberos/kerberos_config.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" + +namespace doris::kerberos { + +// Configuration class for Kerberos authentication +class KerberosConfig { +public: + // Constructor with default values for refresh intervals + KerberosConfig(); + + // Set the Kerberos principal and keytab file path + void set_principal_and_keytab(const std::string& principal, const std::string& keytab) { + _principal = principal; + _keytab_path = keytab; + } + // Set the path to krb5.conf configuration file + void set_krb5_conf_path(const std::string& path) { _krb5_conf_path = path; } + // Set the interval for refreshing Kerberos tickets (in seconds) + void set_refresh_interval(int32_t interval) { _refresh_interval_second = interval; } + // Set the minimum time before refreshing tickets (in seconds) + void set_min_time_before_refresh(int32_t time) { _min_time_before_refresh_second = time; } + + // Get the Kerberos principal name + const std::string& get_principal() const { return _principal; } + // Get the path to the keytab file + const std::string& get_keytab_path() const { return _keytab_path; } + // Get the path to krb5.conf configuration file + const std::string& get_krb5_conf_path() const { return _krb5_conf_path; } + // Get the ticket refresh interval in seconds + int32_t get_refresh_interval_second() const { return _refresh_interval_second; } + // Get the minimum time before refresh in seconds + int32_t get_min_time_before_refresh_second() const { return _min_time_before_refresh_second; } + + std::string get_hash_code() const { return _get_hash_code(_principal, _keytab_path); } + + // Use principal and keytab to generate a hash code. + static std::string get_hash_code(const std::string& principal, const std::string& keytab); + +private: + static std::string _get_hash_code(const std::string& principal, const std::string& keytab); + +private: + // Kerberos principal name (e.g., "user@REALM.COM") + std::string _principal; + // Path to the Kerberos keytab file + std::string _keytab_path; + // Path to the Kerberos configuration file (krb5.conf) + std::string _krb5_conf_path; + // Interval for refreshing Kerberos tickets (in seconds) + int32_t _refresh_interval_second; + // Minimum time before refreshing tickets (in seconds) + int32_t _min_time_before_refresh_second; +}; + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/kerberos_ticket_cache.cpp b/be/src/common/kerberos/kerberos_ticket_cache.cpp new file mode 100644 index 00000000000000..982dee22a5be7a --- /dev/null +++ b/be/src/common/kerberos/kerberos_ticket_cache.cpp @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_ticket_cache.h" + +#include +#include +#include +#include + +#include "common/config.h" + +namespace doris::kerberos { + +KerberosTicketCache::KerberosTicketCache(const KerberosConfig& config, const std::string& root_path, + std::unique_ptr krb5_interface) + : _config(config), + _ccache_root_dir(root_path), + _krb5_interface(std::move(krb5_interface)) {} + +KerberosTicketCache::~KerberosTicketCache() { + stop_periodic_refresh(); + _cleanup_context(); + if (std::filesystem::exists(_ticket_cache_path)) { + std::filesystem::remove(_ticket_cache_path); + } + LOG(INFO) << "destroy kerberos ticket cache " << _ticket_cache_path + << " with principal: " << _config.get_principal(); +} + +Status KerberosTicketCache::initialize() { + std::lock_guard lock(_mutex); + RETURN_IF_ERROR(_init_ticket_cache_path()); + Status st = _initialize_context(); + LOG(INFO) << "initialized kerberos ticket cache " << _ticket_cache_path + << " with principal: " << _config.get_principal() + << " and keytab: " << _config.get_keytab_path() << ": " << st.to_string(); + return st; +} + +Status KerberosTicketCache::_init_ticket_cache_path() { + std::string cache_file_md5 = "doris_krb_" + _config.get_hash_code(); + + // The path should be with prefix "_ccache_root_dir" + std::filesystem::path full_path = std::filesystem::path(_ccache_root_dir) / cache_file_md5; + full_path = std::filesystem::weakly_canonical(full_path); + std::filesystem::path parent_path = full_path.parent_path(); + + LOG(INFO) << "try creating kerberos ticket path: " << full_path.string() + << " for principal: " << _config.get_principal(); + try { + if (!std::filesystem::exists(parent_path)) { + // Create the parent dir if not exists + std::filesystem::create_directories(parent_path); + } else { + // Delete the ticket cache file if exists + if (std::filesystem::exists(full_path)) { + std::filesystem::remove(full_path); + } + } + + _ticket_cache_path = full_path.string(); + return Status::OK(); + } catch (const std::filesystem::filesystem_error& e) { + return Status::InternalError("Error when setting kerberos ticket cache file: {}, {}", + full_path.native(), e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Exception when setting kerberos ticket cache file: {}, {}", + full_path.native(), e.what()); + } catch (...) { + return Status::InternalError("Unknown error when setting kerberos ticket cache file: {}", + full_path.native()); + } +} + +Status KerberosTicketCache::login() { + std::lock_guard lock(_mutex); + + krb5_keytab keytab = nullptr; + krb5_ccache temp_ccache = nullptr; + try { + // Open the keytab file + RETURN_IF_ERROR( + _krb5_interface->kt_resolve(_context, _config.get_keytab_path().c_str(), &keytab)); + + // init ccache + RETURN_IF_ERROR( + _krb5_interface->cc_resolve(_context, _ticket_cache_path.c_str(), &temp_ccache)); + + // get init creds + krb5_get_init_creds_opt* opts = nullptr; + RETURN_IF_ERROR(_krb5_interface->get_init_creds_opt_alloc(_context, &opts)); + + krb5_creds creds; + Status status = _krb5_interface->get_init_creds_keytab(_context, &creds, _principal, keytab, + 0, // start time + nullptr, // TKT service name + opts); + + if (!status.ok()) { + _krb5_interface->get_init_creds_opt_free(_context, opts); + _krb5_interface->kt_close(_context, keytab); + if (temp_ccache) { + _krb5_interface->cc_close(_context, temp_ccache); + } + return status; + } + + // init ccache file + status = _krb5_interface->cc_initialize(_context, temp_ccache, _principal); + if (!status.ok()) { + _krb5_interface->free_cred_contents(_context, &creds); + _krb5_interface->get_init_creds_opt_free(_context, opts); + _krb5_interface->kt_close(_context, keytab); + _krb5_interface->cc_close(_context, temp_ccache); + return status; + } + + // save ccache + status = _krb5_interface->cc_store_cred(_context, temp_ccache, &creds); + if (!status.ok()) { + _krb5_interface->free_cred_contents(_context, &creds); + _krb5_interface->get_init_creds_opt_free(_context, opts); + _krb5_interface->kt_close(_context, keytab); + _krb5_interface->cc_close(_context, temp_ccache); + return status; + } + + // clean + _krb5_interface->free_cred_contents(_context, &creds); + _krb5_interface->get_init_creds_opt_free(_context, opts); + _krb5_interface->kt_close(_context, keytab); + + // Only set _ccache if everything succeeded + if (_ccache) { + _krb5_interface->cc_close(_context, _ccache); + } + _ccache = temp_ccache; + + } catch (...) { + if (keytab) { + _krb5_interface->kt_close(_context, keytab); + } + if (temp_ccache) { + _krb5_interface->cc_close(_context, temp_ccache); + } + return Status::InternalError("Failed to login with kerberos"); + } + return Status::OK(); +} + +Status KerberosTicketCache::login_with_cache() { + std::lock_guard lock(_mutex); + + // Close existing ccache if any + if (_ccache) { + _krb5_interface->cc_close(_context, _ccache); + _ccache = nullptr; + } + + return _krb5_interface->cc_resolve(_context, _ticket_cache_path.c_str(), &_ccache); +} + +Status KerberosTicketCache::write_ticket_cache() { + std::lock_guard lock(_mutex); + + if (!_ccache) { + return Status::InternalError("No credentials cache available"); + } + + // MIT Kerberos automatically writes to the cache file + // when using the FILE: cache type + return Status::OK(); +} + +Status KerberosTicketCache::refresh_tickets() { + try { + return login(); + } catch (const std::exception& e) { + std::stringstream ss; + ss << "Failed to refresh tickets: " << e.what(); + return Status::InternalError(ss.str()); + } +} + +void KerberosTicketCache::start_periodic_refresh() { + _should_stop_refresh = false; + _refresh_thread = std::make_unique([this]() { + auto refresh_interval = std::chrono::milliseconds( + static_cast(_config.get_refresh_interval_second() * 1000)); + auto sleep_duration = _refresh_thread_sleep_time; + std::chrono::milliseconds accumulated_time(0); + while (!_should_stop_refresh) { + std::this_thread::sleep_for(sleep_duration); + accumulated_time += sleep_duration; + if (accumulated_time >= refresh_interval) { + accumulated_time = std::chrono::milliseconds(0); // Reset accumulated time + Status st = refresh_tickets(); + if (!st.ok()) { + // ignore and continue + LOG(WARNING) << st.to_string(); + } else { + LOG(INFO) << "refresh kerberos ticket cache: " << _ticket_cache_path; + } + } + } + }); +} + +void KerberosTicketCache::stop_periodic_refresh() { + if (_refresh_thread) { + _should_stop_refresh = true; + _refresh_thread->join(); + _refresh_thread.reset(); + } +} + +Status KerberosTicketCache::_initialize_context() { + if (!_config.get_krb5_conf_path().empty()) { + if (setenv("KRB5_CONFIG", _config.get_krb5_conf_path().c_str(), 1) != 0) { + return Status::InvalidArgument("Failed to set KRB5_CONFIG environment variable"); + } + LOG(INFO) << "Using custom krb5.conf: " << _config.get_krb5_conf_path(); + } + + RETURN_IF_ERROR(_krb5_interface->init_context(&_context)); + + return _krb5_interface->parse_name(_context, _config.get_principal().c_str(), &_principal); +} + +void KerberosTicketCache::_cleanup_context() { + if (_principal) { + _krb5_interface->free_principal(_context, _principal); + } + if (_ccache) { + _krb5_interface->cc_close(_context, _ccache); + } + if (_context) { + _krb5_interface->free_context(_context); + } +} + +std::vector KerberosTicketCache::get_ticket_info() { + std::lock_guard lock(_mutex); + std::vector result; + + if (!_ccache || !_context) { + // If no valid cache or context, return empty vector + return result; + } + + krb5_cc_cursor cursor; + if (_krb5_interface->cc_start_seq_get(_context, _ccache, &cursor) != Status::OK()) { + return result; + } + + // Iterate through all credentials in the cache + krb5_creds creds; + while (_krb5_interface->cc_next_cred(_context, _ccache, &cursor, &creds) == Status::OK()) { + KerberosTicketInfo info; + info.principal = _config.get_principal(); + info.keytab_path = _config.get_keytab_path(); + info.start_time = static_cast(creds.times.starttime); + info.expiry_time = static_cast(creds.times.endtime); + info.auth_time = static_cast(creds.times.authtime); + // minus 2, + // one is shared_from_this(), the other is ref in _ticket_caches of KerberosTicketMgr + info.use_count = shared_from_this().use_count() - 2; + + // Get the service principal name + char* service_name = nullptr; + if (_krb5_interface->unparse_name(_context, creds.server, &service_name) == Status::OK()) { + info.service_principal = service_name; + _krb5_interface->free_unparsed_name(_context, service_name); + } else { + info.service_principal = "Unparse Error"; + } + info.cache_path = _ticket_cache_path; + info.refresh_interval_second = _config.get_refresh_interval_second(); + info.hash_code = _config.get_hash_code(); + + result.push_back(std::move(info)); + _krb5_interface->free_cred_contents(_context, &creds); + } + + _krb5_interface->cc_end_seq_get(_context, _ccache, &cursor); + return result; +} + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/kerberos_ticket_cache.h b/be/src/common/kerberos/kerberos_ticket_cache.h new file mode 100644 index 00000000000000..2c2c4f94fe1a35 --- /dev/null +++ b/be/src/common/kerberos/kerberos_ticket_cache.h @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "common/kerberos/kerberos_config.h" +#include "common/kerberos/krb5_interface.h" +#include "common/status.h" + +namespace doris::kerberos { + +// Structure to hold detailed information about a Kerberos ticket cache +struct KerberosTicketInfo { + std::string principal; // Client principal + std::string keytab_path; // Path to keytab file + std::string service_principal; // Service principal this credential is for + std::string cache_path; // Path of ticket cache file + std::string hash_code; // the hash code from config + int64_t start_time; // Unix timestamp in seconds + int64_t expiry_time; // Unix timestamp in seconds + int64_t auth_time; // Unix timestamp in seconds + long use_count; // Reference count of the shared_ptr + long refresh_interval_second; // Refresh interval second +}; + +// Class responsible for managing Kerberos ticket cache, including initialization, +// authentication, and periodic ticket refresh +class KerberosTicketCache : public std::enable_shared_from_this { +public: + // Constructor that takes a Kerberos configuration and an optional KRB5 interface implementation + explicit KerberosTicketCache( + const KerberosConfig& config, const std::string& root_path, + std::unique_ptr krb5_interface = Krb5InterfaceFactory::create()); + + virtual ~KerberosTicketCache(); + + // Prevent copying of ticket cache instances + KerberosTicketCache(const KerberosTicketCache&) = delete; + KerberosTicketCache& operator=(const KerberosTicketCache&) = delete; + + // Initialize the ticket cache by setting up the cache path and Kerberos context + // Logic: Creates cache directory if needed, initializes KRB5 context and principal + virtual Status initialize(); + + // Perform a fresh Kerberos login using the configured principal and keytab + // Logic: Opens keytab, obtains new credentials, and stores them in the cache + virtual Status login(); + + // Attempt to login using existing cached credentials + // Logic: Resolves the existing ticket cache without obtaining new credentials + virtual Status login_with_cache(); + + // Write the current credentials to the ticket cache file + virtual Status write_ticket_cache(); + + // Refresh Kerberos tickets if they're close to expiration or if forced + // Logic: Checks if refresh is needed based on ticket expiration time, + // performs a new login if necessary + virtual Status refresh_tickets(); + + // Start the background thread for periodic ticket refresh + // Logic: Creates a thread that periodically checks and refreshes tickets + virtual void start_periodic_refresh(); + + // Stop the background ticket refresh thread + virtual void stop_periodic_refresh(); + + // Getters for configuration and cache path + virtual const KerberosConfig& get_config() const { return _config; } + virtual const std::string get_ticket_cache_path() const { return _ticket_cache_path; } + + // For testing purposes + void set_refresh_thread_sleep_time(std::chrono::milliseconds sleep_time) { + _refresh_thread_sleep_time = sleep_time; + } + + // For testing purposes + void set_ticket_cache_path(const std::string& mock_path) { _ticket_cache_path = mock_path; } + + // Get detailed information about all credentials in the current ticket cache + virtual std::vector get_ticket_info(); + +private: + // Initialize the ticket cache file path using principal and keytab information + Status _init_ticket_cache_path(); + // Initialize the Kerberos context and principal + Status _initialize_context(); + // Clean up Kerberos resources + void _cleanup_context(); + +private: + // Kerberos configuration containing principal, keytab, and refresh settings + KerberosConfig _config; + // For testing purposes + std::string _ccache_root_dir; + // Path to the ticket cache file + std::string _ticket_cache_path; + // Kerberos context handle + krb5_context _context {nullptr}; + // Credentials cache handle + krb5_ccache _ccache {nullptr}; + // Principal handle + krb5_principal _principal {nullptr}; + + // Thread for periodic ticket refresh + std::unique_ptr _refresh_thread; + // Mutex for thread synchronization + std::mutex _mutex; + // Flag to control refresh thread execution + std::atomic _should_stop_refresh {false}; + // Sleep time between refresh checks (in milliseconds) + std::chrono::milliseconds _refresh_thread_sleep_time {10}; + + // Interface for KRB5 operations + std::unique_ptr _krb5_interface; +}; + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/kerberos_ticket_mgr.cpp b/be/src/common/kerberos/kerberos_ticket_mgr.cpp new file mode 100644 index 00000000000000..3b634d54c1eadd --- /dev/null +++ b/be/src/common/kerberos/kerberos_ticket_mgr.cpp @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_ticket_mgr.h" + +#include +#include + +#include "common/logging.h" +#include "exec/schema_scanner/schema_scanner_helper.h" +#include "service/backend_options.h" +#include "vec/core/block.h" + +namespace doris::kerberos { + +KerberosTicketMgr::KerberosTicketMgr(const std::string& root_path) { + _root_path = root_path; + _start_cleanup_thread(); +} + +KerberosTicketMgr::~KerberosTicketMgr() { + _stop_cleanup_thread(); +} + +void KerberosTicketMgr::_start_cleanup_thread() { + _cleanup_thread = std::make_unique(&KerberosTicketMgr::_cleanup_loop, this); +} + +void KerberosTicketMgr::_stop_cleanup_thread() { + if (_cleanup_thread) { + _should_stop_cleanup_thread = true; + _cleanup_thread->join(); + _cleanup_thread.reset(); + } +} + +void KerberosTicketMgr::_cleanup_loop() { +#ifdef BE_TEST + static constexpr int64_t CHECK_INTERVAL_SECONDS = 1; // For testing purpose +#else + static constexpr int64_t CHECK_INTERVAL_SECONDS = 5; // Check stop flag every 5 seconds +#endif + uint64_t last_cleanup_time = std::time(nullptr); + + while (!_should_stop_cleanup_thread) { + uint64_t current_time = std::time(nullptr); + + // Only perform cleanup if enough time has passed + if (current_time - last_cleanup_time >= _cleanup_interval.count()) { + std::vector keys_to_remove; + { + std::lock_guard lock(_mutex); + for (const auto& entry : _ticket_caches) { + // Check if this is the last reference to the ticket cache + // use_count() == 1 means it is only referenced in the _ticket_caches map + if (entry.second.cache.use_count() == 1) { + LOG(INFO) << "Found unused Kerberos ticket cache for principal: " + << entry.second.cache->get_config().get_principal() + << ", keytab: " + << entry.second.cache->get_config().get_keytab_path(); + keys_to_remove.push_back(entry.first); + } + } + + // Remove entries under lock + for (const auto& key : keys_to_remove) { + LOG(INFO) << "Removing unused Kerberos ticket cache for key: " << key; + _ticket_caches.erase(key); + } + } + + last_cleanup_time = current_time; + } + + // Sleep for a short interval to check stop flag more frequently + std::this_thread::sleep_for(std::chrono::seconds(CHECK_INTERVAL_SECONDS)); + } +} + +Status KerberosTicketMgr::get_or_set_ticket_cache( + const KerberosConfig& config, std::shared_ptr* ticket_cache) { + std::string key = config.get_hash_code(); + + std::lock_guard lock(_mutex); + + // Check if already exists + auto it = _ticket_caches.find(key); + if (it != _ticket_caches.end()) { + *ticket_cache = it->second.cache; + return Status::OK(); + } + + // Create new ticket cache + auto new_ticket_cache = _make_new_ticket_cache(config); + RETURN_IF_ERROR(new_ticket_cache->initialize()); + RETURN_IF_ERROR(new_ticket_cache->login()); + RETURN_IF_ERROR(new_ticket_cache->write_ticket_cache()); + new_ticket_cache->start_periodic_refresh(); + + // Insert into _ticket_caches + KerberosTicketEntry entry {.cache = new_ticket_cache}; + auto [inserted_it, success] = _ticket_caches.emplace(key, std::move(entry)); + if (!success) { + return Status::InternalError("Failed to insert ticket cache into map"); + } + + *ticket_cache = new_ticket_cache; + LOG(INFO) << "create new kerberos ticket cache: " + << inserted_it->second.cache->get_ticket_cache_path(); + return Status::OK(); +} + +std::shared_ptr KerberosTicketMgr::get_ticket_cache( + const std::string& principal, const std::string& keytab_path) { + std::string key = KerberosConfig::get_hash_code(principal, keytab_path); + + std::lock_guard lock(_mutex); + auto it = _ticket_caches.find(key); + if (it != _ticket_caches.end()) { + return it->second.cache; + } + return nullptr; +} + +std::shared_ptr KerberosTicketMgr::_make_new_ticket_cache( + const KerberosConfig& config) { + return std::make_shared(config, _root_path); +} + +std::vector KerberosTicketMgr::get_krb_ticket_cache_info() { + std::vector result; + std::lock_guard lock(_mutex); + + for (const auto& entry : _ticket_caches) { + auto cache_info = entry.second.cache->get_ticket_info(); + result.insert(result.end(), cache_info.begin(), cache_info.end()); + } + + return result; +} + +void KerberosTicketMgr::get_ticket_cache_info_block(vectorized::Block* block, + const cctz::time_zone& ctz) { + TBackend be = BackendOptions::get_local_backend(); + int64_t be_id = be.id; + std::string be_ip = be.host; + std::vector infos = get_krb_ticket_cache_info(); + for (auto& info : infos) { + SchemaScannerHelper::insert_int64_value(0, be_id, block); + SchemaScannerHelper::insert_string_value(1, be_ip, block); + SchemaScannerHelper::insert_string_value(2, info.principal, block); + SchemaScannerHelper::insert_string_value(3, info.keytab_path, block); + SchemaScannerHelper::insert_string_value(4, info.service_principal, block); + SchemaScannerHelper::insert_string_value(5, info.cache_path, block); + SchemaScannerHelper::insert_string_value(6, info.hash_code, block); + SchemaScannerHelper::insert_datetime_value(7, info.start_time, ctz, block); + SchemaScannerHelper::insert_datetime_value(8, info.expiry_time, ctz, block); + SchemaScannerHelper::insert_datetime_value(9, info.auth_time, ctz, block); + SchemaScannerHelper::insert_int64_value(10, info.use_count, block); + SchemaScannerHelper::insert_int64_value(11, info.refresh_interval_second, block); + } +} + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/kerberos_ticket_mgr.h b/be/src/common/kerberos/kerberos_ticket_mgr.h new file mode 100644 index 00000000000000..67c660d0b46db6 --- /dev/null +++ b/be/src/common/kerberos/kerberos_ticket_mgr.h @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cctz/time_zone.h" +#include "common/kerberos/kerberos_config.h" +#include "common/kerberos/kerberos_ticket_cache.h" +#include "common/status.h" + +namespace doris { + +namespace vectorized { +class Block; +} + +namespace kerberos { + +// Structure to hold a ticket cache instance and its last access time +struct KerberosTicketEntry { + std::shared_ptr cache; + std::chrono::steady_clock::time_point last_access_time; +}; + +// Manager class responsible for maintaining multiple Kerberos ticket caches +// and handling their lifecycle, including creation, access, and cleanup +class KerberosTicketMgr { +public: + // Constructor that takes the expiration time for unused ticket caches + explicit KerberosTicketMgr(const std::string& root_path); + + // Get or create a ticket cache for the given Kerberos configuration + // Logic: Checks if cache exists, if not creates new one, initializes it, + // performs login, and starts periodic refresh + Status get_or_set_ticket_cache(const KerberosConfig& config, + std::shared_ptr* ticket_cache); + + Status remove_ticket_cache(const std::string& principal, const std::string& keytab_path); + + // Get the ticket cache object. This is used by HdfsHandler to hold a reference + std::shared_ptr get_ticket_cache(const std::string& principal, + const std::string& keytab_path); + + // Get detailed information about all active Kerberos ticket caches + std::vector get_krb_ticket_cache_info(); + + // Set the cleanup interval for testing purpose + void set_cleanup_interval(std::chrono::seconds interval) { _cleanup_interval = interval; } + + void get_ticket_cache_info_block(vectorized::Block* block, const cctz::time_zone& ctz); + + virtual ~KerberosTicketMgr(); + +protected: + // Prevent copying of ticket manager instances + KerberosTicketMgr(const KerberosTicketMgr&) = delete; + KerberosTicketMgr& operator=(const KerberosTicketMgr&) = delete; + + // Factory method to create new ticket cache instances + // Can be overridden in tests to provide mock implementations + virtual std::shared_ptr _make_new_ticket_cache( + const KerberosConfig& config); + + // Start the cleanup thread + void _start_cleanup_thread(); + // Stop the cleanup thread + void _stop_cleanup_thread(); + // Cleanup thread function + void _cleanup_loop(); + +protected: + // The root dir of ticket caches + std::string _root_path; + // Map storing ticket caches, keyed by MD5 hash of principal and keytab path + std::unordered_map _ticket_caches; + // Mutex for thread-safe access to the ticket cache map + std::mutex _mutex; + + // Cleanup thread related members + std::atomic _should_stop_cleanup_thread {false}; + std::unique_ptr _cleanup_thread; + std::chrono::seconds _cleanup_interval {3600}; // Default to 1 hour +}; + +} // namespace kerberos +} // namespace doris diff --git a/be/src/common/kerberos/krb5_interface.h b/be/src/common/kerberos/krb5_interface.h new file mode 100644 index 00000000000000..550fea7e62a9a7 --- /dev/null +++ b/be/src/common/kerberos/krb5_interface.h @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "common/status.h" + +namespace doris::kerberos { + +// Interface for krb5 operations, can be mocked for testing +class Krb5Interface { +public: + virtual ~Krb5Interface() = default; + + virtual Status init_context(krb5_context* context) = 0; + virtual Status parse_name(krb5_context context, const char* name, + krb5_principal* principal) = 0; + virtual Status kt_resolve(krb5_context context, const char* name, krb5_keytab* keytab) = 0; + virtual Status cc_resolve(krb5_context context, const char* name, krb5_ccache* ccache) = 0; + virtual Status get_init_creds_opt_alloc(krb5_context context, + krb5_get_init_creds_opt** opt) = 0; + virtual Status get_init_creds_keytab(krb5_context context, krb5_creds* creds, + krb5_principal client, krb5_keytab keytab, + krb5_deltat start, const char* in_tkt_service, + krb5_get_init_creds_opt* options) = 0; + virtual Status cc_initialize(krb5_context context, krb5_ccache cache, + krb5_principal principal) = 0; + virtual Status cc_store_cred(krb5_context context, krb5_ccache cache, krb5_creds* creds) = 0; + virtual Status timeofday(krb5_context context, krb5_timestamp* timeret) = 0; + virtual Status cc_start_seq_get(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor) = 0; + virtual Status cc_next_cred(krb5_context context, krb5_ccache cache, krb5_cc_cursor* cursor, + krb5_creds* creds) = 0; + + virtual void cc_end_seq_get(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor) = 0; + virtual void free_principal(krb5_context context, krb5_principal principal) = 0; + virtual void free_cred_contents(krb5_context context, krb5_creds* creds) = 0; + virtual void get_init_creds_opt_free(krb5_context context, krb5_get_init_creds_opt* opt) = 0; + virtual void kt_close(krb5_context context, krb5_keytab keytab) = 0; + virtual void cc_close(krb5_context context, krb5_ccache cache) = 0; + virtual void free_context(krb5_context context) = 0; + virtual const char* get_error_message(krb5_context context, krb5_error_code code) = 0; + virtual void free_error_message(krb5_context context, const char* message) = 0; + virtual Status unparse_name(krb5_context context, krb5_principal principal, char** name) = 0; + virtual void free_unparsed_name(krb5_context context, char* name) = 0; +}; + +// Factory to create Krb5Interface instances +class Krb5InterfaceFactory { +public: + static std::unique_ptr create(); +}; + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/krb5_interface_impl.cpp b/be/src/common/kerberos/krb5_interface_impl.cpp new file mode 100644 index 00000000000000..6ae8d11b6c788d --- /dev/null +++ b/be/src/common/kerberos/krb5_interface_impl.cpp @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/krb5_interface_impl.h" + +namespace doris::kerberos { + +Status Krb5InterfaceImpl::init_context(krb5_context* context) { + krb5_error_code code = krb5_init_context(context); + if (code != 0) { + return Status::InternalError("Failed to initialize krb5 context, error code: {}", code); + } + return Status::OK(); +} + +Status Krb5InterfaceImpl::parse_name(krb5_context context, const char* name, + krb5_principal* principal) { + krb5_error_code code = krb5_parse_name(context, name, principal); + return _check_error(code, context, "Failed to parse principal name"); +} + +Status Krb5InterfaceImpl::kt_resolve(krb5_context context, const char* name, krb5_keytab* keytab) { + krb5_error_code code = krb5_kt_resolve(context, name, keytab); + return _check_error(code, context, "Failed to resolve keytab"); +} + +Status Krb5InterfaceImpl::cc_resolve(krb5_context context, const char* name, krb5_ccache* ccache) { + krb5_error_code code = krb5_cc_resolve(context, name, ccache); + return _check_error(code, context, "Failed to resolve credential cache"); +} + +Status Krb5InterfaceImpl::get_init_creds_opt_alloc(krb5_context context, + krb5_get_init_creds_opt** opt) { + krb5_error_code code = krb5_get_init_creds_opt_alloc(context, opt); + return _check_error(code, context, "Failed to allocate get_init_creds_opt"); +} + +Status Krb5InterfaceImpl::get_init_creds_keytab(krb5_context context, krb5_creds* creds, + krb5_principal client, krb5_keytab keytab, + krb5_deltat start, const char* in_tkt_service, + krb5_get_init_creds_opt* options) { + krb5_error_code code = krb5_get_init_creds_keytab(context, creds, client, keytab, start, + in_tkt_service, options); + return _check_error(code, context, "Failed to get initial credentials"); +} + +Status Krb5InterfaceImpl::cc_initialize(krb5_context context, krb5_ccache cache, + krb5_principal principal) { + krb5_error_code code = krb5_cc_initialize(context, cache, principal); + return _check_error(code, context, "Failed to initialize credential cache"); +} + +Status Krb5InterfaceImpl::cc_store_cred(krb5_context context, krb5_ccache cache, + krb5_creds* creds) { + krb5_error_code code = krb5_cc_store_cred(context, cache, creds); + return _check_error(code, context, "Failed to store credentials"); +} + +Status Krb5InterfaceImpl::timeofday(krb5_context context, krb5_timestamp* timeret) { + krb5_error_code code = krb5_timeofday(context, timeret); + return _check_error(code, context, "Failed to get current time"); +} + +Status Krb5InterfaceImpl::cc_start_seq_get(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor) { + krb5_error_code code = krb5_cc_start_seq_get(context, cache, cursor); + return _check_error(code, context, "Failed to start credential iteration"); +} + +Status Krb5InterfaceImpl::cc_next_cred(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor, krb5_creds* creds) { + krb5_error_code code = krb5_cc_next_cred(context, cache, cursor, creds); + return _check_error(code, context, "Failed to get next credential"); +} + +void Krb5InterfaceImpl::cc_end_seq_get(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor) { + krb5_cc_end_seq_get(context, cache, cursor); +} + +void Krb5InterfaceImpl::free_principal(krb5_context context, krb5_principal principal) { + krb5_free_principal(context, principal); +} + +void Krb5InterfaceImpl::free_cred_contents(krb5_context context, krb5_creds* creds) { + krb5_free_cred_contents(context, creds); +} + +void Krb5InterfaceImpl::get_init_creds_opt_free(krb5_context context, + krb5_get_init_creds_opt* opt) { + krb5_get_init_creds_opt_free(context, opt); +} + +void Krb5InterfaceImpl::kt_close(krb5_context context, krb5_keytab keytab) { + krb5_kt_close(context, keytab); +} + +void Krb5InterfaceImpl::cc_close(krb5_context context, krb5_ccache cache) { + krb5_cc_close(context, cache); +} + +void Krb5InterfaceImpl::free_context(krb5_context context) { + krb5_free_context(context); +} + +const char* Krb5InterfaceImpl::get_error_message(krb5_context context, krb5_error_code code) { + return krb5_get_error_message(context, code); +} + +void Krb5InterfaceImpl::free_error_message(krb5_context context, const char* message) { + krb5_free_error_message(context, message); +} + +Status Krb5InterfaceImpl::unparse_name(krb5_context context, krb5_principal principal, + char** name) { + krb5_error_code code = krb5_unparse_name(context, principal, name); + return _check_error(code, context, "Failed to unparse principal name"); +} + +void Krb5InterfaceImpl::free_unparsed_name(krb5_context context, char* name) { + krb5_free_unparsed_name(context, name); +} + +Status Krb5InterfaceImpl::_check_error(krb5_error_code code, krb5_context context, + const char* message) { + if (code) { + const char* err_message = get_error_message(context, code); + std::string full_message = std::string(message) + ": " + err_message; + free_error_message(context, err_message); + return Status::InternalError(full_message); + } + return Status::OK(); +} + +std::unique_ptr Krb5InterfaceFactory::create() { + return std::make_unique(); +} + +} // namespace doris::kerberos diff --git a/be/src/common/kerberos/krb5_interface_impl.h b/be/src/common/kerberos/krb5_interface_impl.h new file mode 100644 index 00000000000000..916dcaf59dfc8c --- /dev/null +++ b/be/src/common/kerberos/krb5_interface_impl.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/kerberos/krb5_interface.h" + +namespace doris::kerberos { + +class Krb5InterfaceImpl : public Krb5Interface { +public: + Status init_context(krb5_context* context) override; + Status parse_name(krb5_context context, const char* name, krb5_principal* principal) override; + Status kt_resolve(krb5_context context, const char* name, krb5_keytab* keytab) override; + Status cc_resolve(krb5_context context, const char* name, krb5_ccache* ccache) override; + Status get_init_creds_opt_alloc(krb5_context context, krb5_get_init_creds_opt** opt) override; + Status get_init_creds_keytab(krb5_context context, krb5_creds* creds, krb5_principal client, + krb5_keytab keytab, krb5_deltat start, const char* in_tkt_service, + krb5_get_init_creds_opt* options) override; + Status cc_initialize(krb5_context context, krb5_ccache cache, + krb5_principal principal) override; + Status cc_store_cred(krb5_context context, krb5_ccache cache, krb5_creds* creds) override; + Status timeofday(krb5_context context, krb5_timestamp* timeret) override; + Status cc_start_seq_get(krb5_context context, krb5_ccache cache, + krb5_cc_cursor* cursor) override; + Status cc_next_cred(krb5_context context, krb5_ccache cache, krb5_cc_cursor* cursor, + krb5_creds* creds) override; + + void cc_end_seq_get(krb5_context context, krb5_ccache cache, krb5_cc_cursor* cursor) override; + void free_principal(krb5_context context, krb5_principal principal) override; + void free_cred_contents(krb5_context context, krb5_creds* creds) override; + void get_init_creds_opt_free(krb5_context context, krb5_get_init_creds_opt* opt) override; + void kt_close(krb5_context context, krb5_keytab keytab) override; + void cc_close(krb5_context context, krb5_ccache cache) override; + void free_context(krb5_context context) override; + const char* get_error_message(krb5_context context, krb5_error_code code) override; + void free_error_message(krb5_context context, const char* message) override; + Status unparse_name(krb5_context context, krb5_principal principal, char** name) override; + void free_unparsed_name(krb5_context context, char* name) override; + +private: + Status _check_error(krb5_error_code code, krb5_context context, const char* message); +}; + +} // namespace doris::kerberos diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index 6336abd7f0bd35..b106a7bde27aa2 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -28,6 +28,7 @@ #include "exec/schema_scanner/schema_active_queries_scanner.h" #include "exec/schema_scanner/schema_backend_active_tasks.h" +#include "exec/schema_scanner/schema_backend_kerberos_ticket_cache.h" #include "exec/schema_scanner/schema_catalog_meta_cache_stats_scanner.h" #include "exec/schema_scanner/schema_charsets_scanner.h" #include "exec/schema_scanner/schema_collations_scanner.h" @@ -238,6 +239,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaCatalogMetaCacheStatsScanner::create_unique(); case TSchemaTableType::SCH_TABLE_OPTIONS: return SchemaTableOptionsScanner::create_unique(); + case TSchemaTableType::SCH_BACKEND_KERBEROS_TICKET_CACHE: + return SchemaBackendKerberosTicketCacheScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.cpp b/be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.cpp new file mode 100644 index 00000000000000..a37e7c1a76f10b --- /dev/null +++ b/be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.cpp @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_backend_kerberos_ticket_cache.h" + +#include "common/kerberos/kerberos_ticket_mgr.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "vec/common/string_ref.h" +#include "vec/core/block.h" +#include "vec/data_types/data_type_factory.hpp" + +namespace doris { + +std::vector SchemaBackendKerberosTicketCacheScanner::_s_tbls_columns = { + // name, type, size + {"BE_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"BE_IP", TYPE_STRING, sizeof(StringRef), true}, + {"PRINCIPAL", TYPE_STRING, sizeof(StringRef), true}, + {"KEYTAB", TYPE_STRING, sizeof(StringRef), true}, + {"SERVICE_PRINCIPAL", TYPE_STRING, sizeof(StringRef), true}, + {"TICKET_CACHE_PATH", TYPE_STRING, sizeof(StringRef), true}, + {"HASH_CODE", TYPE_STRING, sizeof(StringRef), true}, + {"START_TIME", TYPE_DATETIME, sizeof(int128_t), true}, + {"EXPIRE_TIME", TYPE_DATETIME, sizeof(int128_t), true}, + {"AUTH_TIME", TYPE_DATETIME, sizeof(int128_t), true}, + {"REF_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"REFRESH_INTERVAL_SECOND", TYPE_BIGINT, sizeof(int64_t), true}}; + +SchemaBackendKerberosTicketCacheScanner::SchemaBackendKerberosTicketCacheScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_BACKEND_KERBEROS_TICKET_CACHE) {} + +SchemaBackendKerberosTicketCacheScanner::~SchemaBackendKerberosTicketCacheScanner() {} + +Status SchemaBackendKerberosTicketCacheScanner::start(RuntimeState* state) { + _block_rows_limit = state->batch_size(); + _timezone_obj = state->timezone_obj(); + return Status::OK(); +} + +Status SchemaBackendKerberosTicketCacheScanner::get_next_block_internal(vectorized::Block* block, + bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + + if (_info_block == nullptr) { + _info_block = vectorized::Block::create_unique(); + + for (int i = 0; i < _s_tbls_columns.size(); ++i) { + TypeDescriptor descriptor(_s_tbls_columns[i].type); + auto data_type = + vectorized::DataTypeFactory::instance().create_data_type(descriptor, true); + _info_block->insert(vectorized::ColumnWithTypeAndName( + data_type->create_column(), data_type, _s_tbls_columns[i].name)); + } + + _info_block->reserve(_block_rows_limit); + + ExecEnv::GetInstance()->kerberos_ticket_mgr()->get_ticket_cache_info_block( + _info_block.get(), _timezone_obj); + _total_rows = (int)_info_block->rows(); + } + + if (_row_idx == _total_rows) { + *eos = true; + return Status::OK(); + } + + int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); + vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); + RETURN_IF_ERROR(mblock.add_rows(_info_block.get(), _row_idx, current_batch_rows)); + _row_idx += current_batch_rows; + + *eos = _row_idx == _total_rows; + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/util/hdfs_util.h b/be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.h similarity index 50% rename from be/src/util/hdfs_util.h rename to be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.h index 0e10cc578b429a..ecb6110f5616e5 100644 --- a/be/src/util/hdfs_util.h +++ b/be/src/exec/schema_scanner/schema_backend_kerberos_ticket_cache.h @@ -17,36 +17,35 @@ #pragma once -#include +#include -#include "io/fs/hdfs.h" -#include "io/fs/path.h" +#include "cctz/time_zone.h" +#include "common/status.h" +#include "exec/schema_scanner.h" namespace doris { -class HDFSCommonBuilder; +class RuntimeState; +namespace vectorized { +class Block; +} // namespace vectorized -namespace io { +class SchemaBackendKerberosTicketCacheScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaBackendKerberosTicketCacheScanner); -class HDFSHandle { public: - ~HDFSHandle() {} + SchemaBackendKerberosTicketCacheScanner(); + ~SchemaBackendKerberosTicketCacheScanner() override; - static HDFSHandle& instance(); + Status start(RuntimeState* state) override; + Status get_next_block_internal(vectorized::Block* block, bool* eos) override; - hdfsFS create_hdfs_fs(HDFSCommonBuilder& builder); + static std::vector _s_tbls_columns; private: - HDFSHandle() {} + int _block_rows_limit = 4096; + int _row_idx = 0; + int _total_rows = 0; + std::unique_ptr _info_block = nullptr; + cctz::time_zone _timezone_obj; }; - -// if the format of path is hdfs://ip:port/path, replace it to /path. -// path like hdfs://ip:port/path can't be used by libhdfs3. -Path convert_path(const Path& path, const std::string& namenode); - -std::string get_fs_name(const std::string& path); - -// return true if path_or_fs contains "hdfs://" -bool is_hdfs(const std::string& path_or_fs); - -} // namespace io -} // namespace doris +}; // namespace doris diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.cpp b/be/src/exec/schema_scanner/schema_scanner_helper.cpp index 0fea9d8c39f328..ea8d7ab0211cec 100644 --- a/be/src/exec/schema_scanner/schema_scanner_helper.cpp +++ b/be/src/exec/schema_scanner/schema_scanner_helper.cpp @@ -17,6 +17,7 @@ #include "exec/schema_scanner/schema_scanner_helper.h" +#include "cctz/time_zone.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" @@ -49,6 +50,24 @@ void SchemaScannerHelper::insert_datetime_value(int col_index, const std::vector nullable_column->get_null_map_data().emplace_back(0); } +void SchemaScannerHelper::insert_datetime_value(int col_index, int64_t timestamp, + const cctz::time_zone& ctz, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + auto* nullable_column = assert_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + + std::vector datas(1); + VecDateTimeValue src[1]; + src[0].from_unixtime(timestamp, ctz); + datas[0] = src; + auto data = datas[0]; + assert_cast*>(col_ptr)->insert_data( + reinterpret_cast(data), 0); + nullable_column->get_null_map_data().emplace_back(0); +} + void SchemaScannerHelper::insert_int64_value(int col_index, int64_t int_val, vectorized::Block* block) { vectorized::MutableColumnPtr mutable_col_ptr; diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.h b/be/src/exec/schema_scanner/schema_scanner_helper.h index f7b47ede91bb5d..29e5335112b261 100644 --- a/be/src/exec/schema_scanner/schema_scanner_helper.h +++ b/be/src/exec/schema_scanner/schema_scanner_helper.h @@ -22,6 +22,8 @@ #include #include +#include "cctz/time_zone.h" + // this is a util class which can be used by all shema scanner // all common functions are added in this class. namespace doris { @@ -34,6 +36,8 @@ class SchemaScannerHelper { static void insert_string_value(int col_index, std::string str_val, vectorized::Block* block); static void insert_datetime_value(int col_index, const std::vector& datas, vectorized::Block* block); + static void insert_datetime_value(int col_index, int64_t timestamp, const cctz::time_zone& ctz, + vectorized::Block* block); static void insert_int64_value(int col_index, int64_t int_val, vectorized::Block* block); static void insert_double_value(int col_index, double double_val, vectorized::Block* block); diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 95d537320883e8..4dd43b1094f1cd 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -27,6 +27,7 @@ #include "common/config.h" #include "common/status.h" #include "io/fs/broker_file_system.h" +#include "io/fs/hdfs/hdfs_mgr.h" #include "io/fs/hdfs_file_system.h" #include "io/fs/local_file_system.h" #include "io/fs/multi_table_pipe.h" diff --git a/be/src/io/fs/hdfs/hdfs_mgr.cpp b/be/src/io/fs/hdfs/hdfs_mgr.cpp new file mode 100644 index 00000000000000..c4b4ea42465c05 --- /dev/null +++ b/be/src/io/fs/hdfs/hdfs_mgr.cpp @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/hdfs/hdfs_mgr.h" + +#include +#include + +#include +#include + +#include "common/kerberos/kerberos_ticket_mgr.h" +#include "common/logging.h" +#include "gutil/hash/hash.h" +#include "io/fs/err_utils.h" +#include "io/hdfs_builder.h" +#include "io/hdfs_util.h" +#include "runtime/exec_env.h" +#include "vec/common/string_ref.h" + +namespace doris::io { + +HdfsMgr::HdfsMgr() : _should_stop_cleanup_thread(false) { + _start_cleanup_thread(); +} + +HdfsMgr::~HdfsMgr() { + _stop_cleanup_thread(); +} + +void HdfsMgr::_start_cleanup_thread() { + _cleanup_thread = std::make_unique(&HdfsMgr::_cleanup_loop, this); +} + +void HdfsMgr::_stop_cleanup_thread() { + if (_cleanup_thread) { + _should_stop_cleanup_thread = true; + _cleanup_thread->join(); + _cleanup_thread.reset(); + } +} + +void HdfsMgr::_cleanup_loop() { +#ifdef BE_TEST + static constexpr int64_t CHECK_INTERVAL_SECONDS = 1; // For testing purpose +#else + static constexpr int64_t CHECK_INTERVAL_SECONDS = 5; // Check stop flag every 5 seconds +#endif + uint64_t last_cleanup_time = std::time(nullptr); + + while (!_should_stop_cleanup_thread) { + uint64_t current_time = std::time(nullptr); + + // Only perform cleanup if enough time has passed + if (current_time - last_cleanup_time >= _cleanup_interval_seconds) { + // Collect expired handlers under lock + std::vector> handlers_to_cleanup; + { + std::lock_guard lock(_mutex); + std::vector to_remove; + + // Find expired handlers + for (const auto& entry : _fs_handlers) { + if (current_time - entry.second->last_access_time >= + _instance_timeout_seconds) { + LOG(INFO) << "Found expired HDFS handler, hash_code=" << entry.first + << ", last_access_time=" << entry.second->last_access_time + << ", is_kerberos=" << entry.second->is_kerberos_auth + << ", principal=" << entry.second->principal + << ", fs_name=" << entry.second->fs_name; + to_remove.push_back(entry.first); + handlers_to_cleanup.push_back(entry.second); + } + } + + // Remove expired handlers from map under lock + for (uint64_t hash_code : to_remove) { + _fs_handlers.erase(hash_code); + } + } + + // Cleanup handlers outside lock + for (const auto& handler : handlers_to_cleanup) { + LOG(INFO) << "Start to cleanup HDFS handler" + << ", is_kerberos=" << handler->is_kerberos_auth + << ", principal=" << handler->principal + << ", fs_name=" << handler->fs_name; + + // The kerberos ticket cache will be automatically cleaned up when the last reference is gone + // DO NOT call hdfsDisconnect(), or we will meet "Filesystem closed" + // even if we create a new one + // hdfsDisconnect(handler->hdfs_fs); + + LOG(INFO) << "Finished cleanup HDFS handler" + << ", fs_name=" << handler->fs_name; + } + + handlers_to_cleanup.clear(); + last_cleanup_time = current_time; + } + + // Sleep for a short interval to check stop flag more frequently + std::this_thread::sleep_for(std::chrono::seconds(CHECK_INTERVAL_SECONDS)); + } +} + +Status HdfsMgr::get_or_create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + uint64_t hash_code = _hdfs_hash_code(hdfs_params, fs_name); + + // First check without lock + { + std::lock_guard lock(_mutex); + auto it = _fs_handlers.find(hash_code); + if (it != _fs_handlers.end()) { + LOG(INFO) << "Reuse existing HDFS handler, hash_code=" << hash_code + << ", is_kerberos=" << it->second->is_kerberos_auth + << ", principal=" << it->second->principal << ", fs_name=" << fs_name; + it->second->update_access_time(); + *fs_handler = it->second; + return Status::OK(); + } + } + + // Create new hdfsFS handler outside the lock + LOG(INFO) << "Start to create new HDFS handler, hash_code=" << hash_code + << ", fs_name=" << fs_name; + + std::shared_ptr new_fs_handler; + RETURN_IF_ERROR(_create_hdfs_fs(hdfs_params, fs_name, &new_fs_handler)); + + // Double check with lock before inserting + { + std::lock_guard lock(_mutex); + auto it = _fs_handlers.find(hash_code); + if (it != _fs_handlers.end()) { + // Another thread has created the handler, use it instead + LOG(INFO) << "Another thread created HDFS handler, reuse it, hash_code=" << hash_code + << ", is_kerberos=" << it->second->is_kerberos_auth + << ", principal=" << it->second->principal << ", fs_name=" << fs_name; + it->second->update_access_time(); + *fs_handler = it->second; + return Status::OK(); + } + + // Store the new handler + *fs_handler = new_fs_handler; + _fs_handlers[hash_code] = new_fs_handler; + + LOG(INFO) << "Finished create new HDFS handler, hash_code=" << hash_code + << ", is_kerberos=" << new_fs_handler->is_kerberos_auth + << ", principal=" << new_fs_handler->principal << ", fs_name=" << fs_name; + } + + return Status::OK(); +} + +Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + HDFSCommonBuilder builder; + RETURN_IF_ERROR(create_hdfs_builder(hdfs_params, fs_name, &builder)); + hdfsFS hdfs_fs = hdfsBuilderConnect(builder.get()); + if (hdfs_fs == nullptr) { + return Status::InternalError("failed to connect to hdfs {}: {}", fs_name, hdfs_error()); + } + + bool is_kerberos = builder.is_kerberos(); + *fs_handler = std::make_shared( + hdfs_fs, is_kerberos, is_kerberos ? hdfs_params.hdfs_kerberos_principal : "", + is_kerberos ? hdfs_params.hdfs_kerberos_keytab : "", fs_name, + builder.get_ticket_cache()); + return Status::OK(); +} + +// https://brpc.apache.org/docs/server/basics/ +// According to the brpc doc, JNI code checks stack layout and cannot be run in +// bthreads so create a pthread for creating hdfs connection if necessary. +Status HdfsMgr::_create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + bool is_pthread = bthread_self() == 0; + LOG(INFO) << "create hdfs fs, is_pthread=" << is_pthread << " fs_name=" << fs_name; + if (is_pthread) { // running in pthread + return _create_hdfs_fs_impl(hdfs_params, fs_name, fs_handler); + } + + // running in bthread, switch to a pthread and wait + Status st; + auto btx = bthread::butex_create(); + *(int*)btx = 0; + std::thread t([&] { + st = _create_hdfs_fs_impl(hdfs_params, fs_name, fs_handler); + *(int*)btx = 1; + bthread::butex_wake_all(btx); + }); + std::unique_ptr> defer((int*)0x01, [&t, &btx](...) { + if (t.joinable()) t.join(); + bthread::butex_destroy(btx); + }); + timespec tmout {.tv_sec = std::chrono::system_clock::now().time_since_epoch().count() + 60, + .tv_nsec = 0}; + if (int ret = bthread::butex_wait(btx, 1, &tmout); ret != 0) { + std::string msg = "failed to wait create_hdfs_fs finish. fs_name=" + fs_name; + LOG(WARNING) << msg << " error=" << std::strerror(errno); + st = Status::Error(msg); + } + return st; +} + +uint64_t HdfsMgr::_hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name) { + uint64_t hash_code = 0; + // The specified fsname is used first. + // If there is no specified fsname, the default fsname is used + if (!fs_name.empty()) { + hash_code ^= Fingerprint(fs_name); + } else if (hdfs_params.__isset.fs_name) { + hash_code ^= Fingerprint(hdfs_params.fs_name); + } + + if (hdfs_params.__isset.user) { + hash_code ^= Fingerprint(hdfs_params.user); + } + if (hdfs_params.__isset.hdfs_kerberos_principal) { + hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_principal); + } + if (hdfs_params.__isset.hdfs_kerberos_keytab) { + hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_keytab); + } + if (hdfs_params.__isset.hdfs_conf) { + std::map conf_map; + for (const auto& conf : hdfs_params.hdfs_conf) { + conf_map[conf.key] = conf.value; + } + for (auto& conf : conf_map) { + hash_code ^= Fingerprint(conf.first); + hash_code ^= Fingerprint(conf.second); + } + } + return hash_code; +} + +} // namespace doris::io diff --git a/be/src/io/fs/hdfs/hdfs_mgr.h b/be/src/io/fs/hdfs/hdfs_mgr.h new file mode 100644 index 00000000000000..579f6efcc32484 --- /dev/null +++ b/be/src/io/fs/hdfs/hdfs_mgr.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "io/fs/hdfs.h" +#include "io/hdfs_util.h" + +namespace doris::io { + +// A manager class to handle multiple hdfsFS instances +class HdfsMgr { +public: + HdfsMgr(); + + // Get or create a hdfsFS instance based on the given parameters + Status get_or_create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler); + + virtual ~HdfsMgr(); + +protected: + // For testing purpose + friend class HdfsMgrTest; + size_t get_fs_handlers_size() const { return _fs_handlers.size(); } + bool has_fs_handler(uint64_t hash_code) const { + return _fs_handlers.find(hash_code) != _fs_handlers.end(); + } + void set_instance_timeout_seconds(int64_t timeout_seconds) { + _instance_timeout_seconds = timeout_seconds; + } + void set_cleanup_interval_seconds(int64_t interval_seconds) { + _cleanup_interval_seconds = interval_seconds; + } + uint64_t _hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name); + + virtual Status _create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler); + +private: + HdfsMgr(const HdfsMgr&) = delete; + HdfsMgr& operator=(const HdfsMgr&) = delete; + + // Start the cleanup thread + void _start_cleanup_thread(); + // Stop the cleanup thread + void _stop_cleanup_thread(); + // Cleanup thread function + void _cleanup_loop(); + // Remove kerberos ticket cache if instance is using kerberos auth + void _cleanup_kerberos_ticket(const HdfsHandler& handler); + + Status _create_hdfs_fs(const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler); + +private: + std::mutex _mutex; + std::unordered_map> _fs_handlers; + + std::atomic _should_stop_cleanup_thread; + std::unique_ptr _cleanup_thread; + int64_t _cleanup_interval_seconds = 3600; // Run cleanup every hour + int64_t _instance_timeout_seconds = 86400; // 24 hours timeout +}; + +} // namespace doris::io diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index ac75e2e722b9d7..a5f62bada1a931 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -27,12 +27,11 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/logging.h" #include "io/fs/err_utils.h" -// #include "io/fs/hdfs_file_system.h" +#include "io/hdfs_util.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" #include "util/doris_metrics.h" -#include "util/hdfs_util.h" namespace doris { namespace io { diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp index 09e60917865e7c..d7c1230c6a7d5d 100644 --- a/be/src/io/fs/hdfs_file_system.cpp +++ b/be/src/io/fs/hdfs_file_system.cpp @@ -38,53 +38,25 @@ #include "io/fs/file_reader.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" +#include "io/fs/hdfs/hdfs_mgr.h" #include "io/fs/hdfs_file_reader.h" #include "io/fs/hdfs_file_writer.h" #include "io/fs/local_file_system.h" #include "io/hdfs_builder.h" -#include "util/hdfs_util.h" +#include "io/hdfs_util.h" #include "util/obj_lru_cache.h" #include "util/slice.h" namespace doris { namespace io { -#ifndef CHECK_HDFS_HANDLE -#define CHECK_HDFS_HANDLE(handle) \ - if (!handle) { \ - return Status::IOError("init Hdfs handle error"); \ +#ifndef CHECK_HDFS_HANDLER +#define CHECK_HDFS_HANDLER(handler) \ + if (!handler) { \ + return Status::IOError("init Hdfs handler error"); \ } #endif -// Cache for HdfsFileSystemHandle -class HdfsFileSystemCache { -public: - static int MAX_CACHE_HANDLE; - - static HdfsFileSystemCache* instance() { - static HdfsFileSystemCache s_instance; - return &s_instance; - } - - HdfsFileSystemCache(const HdfsFileSystemCache&) = delete; - const HdfsFileSystemCache& operator=(const HdfsFileSystemCache&) = delete; - - // This function is thread-safe - Status get_connection(const THdfsParams& hdfs_params, const std::string& fs_name, - std::shared_ptr* fs_handle); - -private: - std::mutex _lock; - std::unordered_map> _cache; - - HdfsFileSystemCache() = default; - - uint64 _hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name); - Status _create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, hdfsFS* fs); - void _clean_invalid(); - void _clean_oldest(); -}; - class HdfsFileHandleCache { public: static HdfsFileHandleCache* instance() { @@ -115,7 +87,7 @@ Status HdfsFileHandleCache::get_file(const std::shared_ptr& fs, bool cache_hit; std::string fname = file.string(); RETURN_IF_ERROR(HdfsFileHandleCache::instance()->cache().get_file_handle( - fs->_fs_handle->hdfs_fs, fname, mtime, file_size, false, accessor, &cache_hit)); + fs->_fs_handler->hdfs_fs, fname, mtime, file_size, false, accessor, &cache_hit)); accessor->set_fs(fs); return Status::OK(); @@ -139,7 +111,7 @@ HdfsFileSystem::HdfsFileSystem(const THdfsParams& hdfs_params, std::string id, const std::string& fs_name, RuntimeProfile* profile) : RemoteFileSystem("", std::move(id), FileSystemType::HDFS), _hdfs_params(hdfs_params), - _fs_handle(nullptr), + _fs_handler(nullptr), _profile(profile) { if (fs_name.empty() && _hdfs_params.__isset.fs_name) { _fs_name = _hdfs_params.fs_name; @@ -151,9 +123,9 @@ HdfsFileSystem::HdfsFileSystem(const THdfsParams& hdfs_params, std::string id, HdfsFileSystem::~HdfsFileSystem() = default; Status HdfsFileSystem::connect_impl() { - RETURN_IF_ERROR( - HdfsFileSystemCache::instance()->get_connection(_hdfs_params, _fs_name, &_fs_handle)); - if (!_fs_handle) { + RETURN_IF_ERROR(ExecEnv::GetInstance()->hdfs_mgr()->get_or_create_fs(_hdfs_params, _fs_name, + &_fs_handler)); + if (!_fs_handler) { return Status::IOError("failed to init Hdfs handle with, please check hdfs params."); } return Status::OK(); @@ -167,7 +139,7 @@ Status HdfsFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer, Status HdfsFileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader, const FileReaderOptions& opts) { - CHECK_HDFS_HANDLE(_fs_handle); + CHECK_HDFS_HANDLER(_fs_handler); Path real_path = convert_path(file, _fs_name); FileHandleCache::Accessor accessor; @@ -180,9 +152,9 @@ Status HdfsFileSystem::open_file_internal(const Path& file, FileReaderSPtr* read } Status HdfsFileSystem::create_directory_impl(const Path& dir, bool failed_if_exists) { - CHECK_HDFS_HANDLE(_fs_handle); + CHECK_HDFS_HANDLER(_fs_handler); Path real_path = convert_path(dir, _fs_name); - int res = hdfsCreateDirectory(_fs_handle->hdfs_fs, real_path.string().c_str()); + int res = hdfsCreateDirectory(_fs_handler->hdfs_fs, real_path.string().c_str()); if (res == -1) { return Status::IOError("failed to create directory {}: {}", dir.native(), hdfs_error()); } @@ -210,9 +182,9 @@ Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) { if (!exists) { return Status::OK(); } - CHECK_HDFS_HANDLE(_fs_handle); + CHECK_HDFS_HANDLER(_fs_handler); Path real_path = convert_path(path, _fs_name); - int res = hdfsDelete(_fs_handle->hdfs_fs, real_path.string().c_str(), is_recursive); + int res = hdfsDelete(_fs_handler->hdfs_fs, real_path.string().c_str(), is_recursive); if (res == -1) { return Status::IOError("failed to delete directory {}: {}", path.native(), hdfs_error()); } @@ -220,9 +192,9 @@ Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) { } Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const { - CHECK_HDFS_HANDLE(_fs_handle); + CHECK_HDFS_HANDLER(_fs_handler); Path real_path = convert_path(path, _fs_name); - int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str()); + int is_exists = hdfsExists(_fs_handler->hdfs_fs, real_path.string().c_str()); #ifdef USE_HADOOP_HDFS // when calling hdfsExists() and return non-zero code, // if errno is ENOENT, which means the file does not exist. @@ -243,9 +215,9 @@ Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const { } Status HdfsFileSystem::file_size_impl(const Path& path, int64_t* file_size) const { - CHECK_HDFS_HANDLE(_fs_handle); + CHECK_HDFS_HANDLER(_fs_handler); Path real_path = convert_path(path, _fs_name); - hdfsFileInfo* file_info = hdfsGetPathInfo(_fs_handle->hdfs_fs, real_path.string().c_str()); + hdfsFileInfo* file_info = hdfsGetPathInfo(_fs_handler->hdfs_fs, real_path.string().c_str()); if (file_info == nullptr) { return Status::IOError("failed to get file size of {}: {}", path.native(), hdfs_error()); } @@ -261,11 +233,11 @@ Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vectorhdfs_fs, real_path.c_str(), &numEntries); + hdfsListDirectory(_fs_handler->hdfs_fs, real_path.c_str(), &numEntries); if (hdfs_file_info == nullptr) { return Status::IOError("failed to list files/directors {}: {}", path.native(), hdfs_error()); @@ -289,7 +261,7 @@ Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vectorhdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str()); + int ret = hdfsRename(_fs_handler->hdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str()); if (ret == 0) { LOG(INFO) << "finished to rename file. orig: " << normal_orig_name << ", new: " << normal_new_name; @@ -376,115 +348,5 @@ Status HdfsFileSystem::download_impl(const Path& remote_file, const Path& local_ return local_writer->close(); } -// ************* HdfsFileSystemCache ****************** -int HdfsFileSystemCache::MAX_CACHE_HANDLE = 64; - -Status HdfsFileSystemCache::_create_fs(const THdfsParams& hdfs_params, const std::string& fs_name, - hdfsFS* fs) { - HDFSCommonBuilder builder; - RETURN_IF_ERROR(create_hdfs_builder(hdfs_params, fs_name, &builder)); - hdfsFS hdfs_fs = hdfsBuilderConnect(builder.get()); - if (hdfs_fs == nullptr) { - return Status::IOError("faield to connect to hdfs {}: {}", fs_name, hdfs_error()); - } - *fs = hdfs_fs; - return Status::OK(); -} - -void HdfsFileSystemCache::_clean_invalid() { - std::vector removed_handle; - for (auto& item : _cache) { - if (item.second.use_count() == 1 && item.second->invalid()) { - removed_handle.emplace_back(item.first); - } - } - for (auto& handle : removed_handle) { - _cache.erase(handle); - } -} - -void HdfsFileSystemCache::_clean_oldest() { - uint64_t oldest_time = ULONG_MAX; - uint64 oldest = 0; - for (auto& item : _cache) { - if (item.second.use_count() == 1 && item.second->last_access_time() < oldest_time) { - oldest_time = item.second->last_access_time(); - oldest = item.first; - } - } - _cache.erase(oldest); -} - -Status HdfsFileSystemCache::get_connection(const THdfsParams& hdfs_params, - const std::string& fs_name, - std::shared_ptr* fs_handle) { - uint64 hash_code = _hdfs_hash_code(hdfs_params, fs_name); - { - std::lock_guard l(_lock); - auto it = _cache.find(hash_code); - if (it != _cache.end()) { - std::shared_ptr handle = it->second; - if (!handle->invalid()) { - handle->update_last_access_time(); - *fs_handle = std::move(handle); - return Status::OK(); - } - // fs handle is invalid, erase it. - _cache.erase(it); - LOG(INFO) << "erase the hdfs handle, fs name: " << hdfs_params.fs_name; - } - - // not find in cache, or fs handle is invalid - // create a new one and try to put it into cache - hdfsFS hdfs_fs = nullptr; - RETURN_IF_ERROR(_create_fs(hdfs_params, fs_name, &hdfs_fs)); - if (_cache.size() >= MAX_CACHE_HANDLE) { - _clean_invalid(); - _clean_oldest(); - } - if (_cache.size() < MAX_CACHE_HANDLE) { - auto handle = std::make_shared(hdfs_fs, true); - handle->update_last_access_time(); - *fs_handle = handle; - _cache[hash_code] = std::move(handle); - } else { - *fs_handle = std::make_shared(hdfs_fs, false); - } - } - return Status::OK(); -} - -uint64 HdfsFileSystemCache::_hdfs_hash_code(const THdfsParams& hdfs_params, - const std::string& fs_name) { - uint64 hash_code = 0; - // The specified fsname is used first. - // If there is no specified fsname, the default fsname is used - if (!fs_name.empty()) { - hash_code ^= Fingerprint(fs_name); - } else if (hdfs_params.__isset.fs_name) { - hash_code ^= Fingerprint(hdfs_params.fs_name); - } - - if (hdfs_params.__isset.user) { - hash_code ^= Fingerprint(hdfs_params.user); - } - if (hdfs_params.__isset.hdfs_kerberos_principal) { - hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_principal); - } - if (hdfs_params.__isset.hdfs_kerberos_keytab) { - hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_keytab); - } - if (hdfs_params.__isset.hdfs_conf) { - std::map conf_map; - for (auto& conf : hdfs_params.hdfs_conf) { - conf_map[conf.key] = conf.value; - } - for (auto& conf : conf_map) { - hash_code ^= Fingerprint(conf.first); - hash_code ^= Fingerprint(conf.second); - } - } - return hash_code; -} } // namespace io } // namespace doris diff --git a/be/src/io/fs/hdfs_file_system.h b/be/src/io/fs/hdfs_file_system.h index 74d098004ab236..b52c30f0ce0e88 100644 --- a/be/src/io/fs/hdfs_file_system.h +++ b/be/src/io/fs/hdfs_file_system.h @@ -40,59 +40,7 @@ class THdfsParams; namespace io { struct FileInfo; -class HdfsFileSystemHandle { -public: - HdfsFileSystemHandle(hdfsFS fs, bool cached) - : hdfs_fs(fs), - from_cache(cached), - _create_time(_now()), - _last_access_time(0), - _invalid(false) {} - - ~HdfsFileSystemHandle() { - if (hdfs_fs != nullptr) { - // DO NOT call hdfsDisconnect(), or we will meet "Filesystem closed" - // even if we create a new one - // hdfsDisconnect(hdfs_fs); - } - hdfs_fs = nullptr; - } - - int64_t last_access_time() { return _last_access_time; } - - void update_last_access_time() { - if (from_cache) { - _last_access_time = std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - } - } - - bool invalid() { return _invalid; } - - void set_invalid() { _invalid = true; } - - hdfsFS hdfs_fs; - // When cache is full, and all handlers are in use, HdfsFileSystemCache will return an uncached handler. - // Client should delete the handler in such case. - const bool from_cache; - -private: - // For kerberos authentication, we need to save create time so that - // we can know if the kerberos ticket is expired. - std::atomic _create_time; - // HdfsFileSystemCache try to remove the oldest handler when the cache is full - std::atomic _last_access_time; - // Client will set invalid if error thrown, and HdfsFileSystemCache will not reuse this handler - std::atomic _invalid; - - uint64_t _now() { - return std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - } -}; - +class HdfsHandler; class HdfsFileHandleCache; class HdfsFileSystem final : public RemoteFileSystem { public: @@ -133,7 +81,7 @@ class HdfsFileSystem final : public RemoteFileSystem { RuntimeProfile* profile); const THdfsParams& _hdfs_params; std::string _fs_name; - std::shared_ptr _fs_handle = nullptr; + std::shared_ptr _fs_handler = nullptr; RuntimeProfile* _profile = nullptr; }; } // namespace io diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index fe68d1363d530a..03d7f68adbba8e 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -28,8 +28,8 @@ #include "common/logging.h" #include "io/fs/err_utils.h" #include "io/fs/hdfs_file_system.h" +#include "io/hdfs_util.h" #include "service/backend_options.h" -#include "util/hdfs_util.h" namespace doris { namespace io { @@ -55,7 +55,7 @@ Status HdfsFileWriter::close() { if (_hdfs_file == nullptr) { return Status::OK(); } - int result = hdfsFlush(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file); + int result = hdfsFlush(_hdfs_fs->_fs_handler->hdfs_fs, _hdfs_file); if (result == -1) { std::stringstream ss; ss << "failed to flush hdfs file. " @@ -65,7 +65,7 @@ Status HdfsFileWriter::close() { return Status::InternalError(ss.str()); } - result = hdfsCloseFile(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file); + result = hdfsCloseFile(_hdfs_fs->_fs_handler->hdfs_fs, _hdfs_file); _hdfs_file = nullptr; if (result != 0) { std::string err_msg = hdfs_error(); @@ -89,7 +89,7 @@ Status HdfsFileWriter::appendv(const Slice* data, size_t data_cnt) { const char* p = result.data; while (left_bytes > 0) { int64_t written_bytes = - hdfsWrite(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file, p, left_bytes); + hdfsWrite(_hdfs_fs->_fs_handler->hdfs_fs, _hdfs_file, p, left_bytes); if (written_bytes < 0) { return Status::InternalError("write hdfs failed. namenode: {}, path: {}, error: {}", _hdfs_fs->_fs_name, _path.native(), hdfs_error()); @@ -123,10 +123,10 @@ Status HdfsFileWriter::open() { Status HdfsFileWriter::_open() { _path = convert_path(_path, _hdfs_fs->_fs_name); std::string hdfs_dir = _path.parent_path().string(); - int exists = hdfsExists(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str()); + int exists = hdfsExists(_hdfs_fs->_fs_handler->hdfs_fs, hdfs_dir.c_str()); if (exists != 0) { VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir; - int ret = hdfsCreateDirectory(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str()); + int ret = hdfsCreateDirectory(_hdfs_fs->_fs_handler->hdfs_fs, hdfs_dir.c_str()); if (ret != 0) { std::stringstream ss; ss << "create dir failed. " @@ -138,7 +138,7 @@ Status HdfsFileWriter::_open() { } } // open file - _hdfs_file = hdfsOpenFile(_hdfs_fs->_fs_handle->hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0); + _hdfs_file = hdfsOpenFile(_hdfs_fs->_fs_handler->hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0); if (_hdfs_file == nullptr) { std::stringstream ss; ss << "open file failed. " diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 99ee89596ed9ac..256c4eb4fd3785 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -27,14 +27,16 @@ #include "agent/utils.h" #include "common/config.h" +#include "common/kerberos/kerberos_ticket_mgr.h" #include "common/logging.h" #include "io/fs/hdfs.h" +#include "runtime/exec_env.h" #include "util/string_util.h" #include "util/uid_util.h" namespace doris { -#ifdef USE_HADOOP_HDFS +#ifdef USE_DORIS_HADOOP_HDFS void err_log_message(const char* fmt, ...) { va_list args; va_start(args, fmt); @@ -88,13 +90,13 @@ void va_err_log_message(const char* fmt, va_list ap) { struct hdfsLogger logger = {.errLogMessage = err_log_message, .vaErrLogMessage = va_err_log_message}; -#endif // #ifdef USE_HADOOP_HDFS +#endif // #ifdef USE_DORIS_HADOOP_HDFS Status HDFSCommonBuilder::init_hdfs_builder() { -#ifdef USE_HADOOP_HDFS +#ifdef USE_DORIS_HADOOP_HDFS static std::once_flag flag; std::call_once(flag, []() { hdfsSetLogger(&logger); }); -#endif // #ifdef USE_HADOOP_HDFS +#endif // #ifdef USE_DORIS_HADOOP_HDFS hdfs_builder = hdfsNewBuilder(); if (hdfs_builder == nullptr) { @@ -122,6 +124,43 @@ Status HDFSCommonBuilder::check_krb_params() { return Status::OK(); } +void HDFSCommonBuilder::set_hdfs_conf(const std::string& key, const std::string& val) { + hdfs_conf[key] = val; +} + +std::string HDFSCommonBuilder::get_hdfs_conf_value(const std::string& key, + const std::string& default_val) const { + auto it = hdfs_conf.find(key); + if (it != hdfs_conf.end()) { + return it->second; + } else { + return default_val; + } +} + +void HDFSCommonBuilder::set_hdfs_conf_to_hdfs_builder() { + for (const auto& pair : hdfs_conf) { + hdfsBuilderConfSetStr(hdfs_builder, pair.first.c_str(), pair.second.c_str()); + } +} + +Status HDFSCommonBuilder::set_kerberos_ticket_cache() { + kerberos::KerberosConfig config; + config.set_principal_and_keytab(hdfs_kerberos_principal, hdfs_kerberos_keytab); + config.set_krb5_conf_path(config::kerberos_krb5_conf_path); + config.set_refresh_interval(config::kerberos_refresh_interval_second); + config.set_min_time_before_refresh(600); + kerberos::KerberosTicketMgr* ticket_mgr = ExecEnv::GetInstance()->kerberos_ticket_mgr(); + // the life cycle of string "ticket_cache_file" must be same as hdfs_builder, + // so here we need to use the ticket_cache_file instead of a temp string + RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); + hdfsBuilderSetKerbTicketCachePath(hdfs_builder, ticket_cache->get_ticket_cache_path().c_str()); + hdfsBuilderSetForceNewInstance(hdfs_builder); + LOG(INFO) << "get kerberos ticket path: " << ticket_cache->get_ticket_cache_path() + << " with principal: " << hdfs_kerberos_principal; + return Status::OK(); +} + THdfsParams parse_properties(const std::map& properties) { StringCaseMap prop(properties.begin(), properties.end()); std::vector hdfs_configs; @@ -157,44 +196,42 @@ THdfsParams parse_properties(const std::map& propertie Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_name, HDFSCommonBuilder* builder) { RETURN_IF_ERROR(builder->init_hdfs_builder()); - hdfsBuilderSetNameNode(builder->get(), fs_name.c_str()); - // set kerberos conf - if (hdfsParams.__isset.hdfs_kerberos_keytab) { - builder->kerberos_login = true; - builder->hdfs_kerberos_keytab = hdfsParams.hdfs_kerberos_keytab; -#ifdef USE_HADOOP_HDFS - hdfsBuilderSetKerb5Conf(builder->get(), doris::config::kerberos_krb5_conf_path.c_str()); - hdfsBuilderSetKeyTabFile(builder->get(), hdfsParams.hdfs_kerberos_keytab.c_str()); -#endif - } - if (hdfsParams.__isset.hdfs_kerberos_principal) { - builder->kerberos_login = true; - builder->hdfs_kerberos_principal = hdfsParams.hdfs_kerberos_principal; - hdfsBuilderSetPrincipal(builder->get(), hdfsParams.hdfs_kerberos_principal.c_str()); - } else if (hdfsParams.__isset.user) { - hdfsBuilderSetUserName(builder->get(), hdfsParams.user.c_str()); -#ifdef USE_HADOOP_HDFS - hdfsBuilderSetKerb5Conf(builder->get(), nullptr); - hdfsBuilderSetKeyTabFile(builder->get(), nullptr); -#endif - } - // set other conf + builder->fs_name = fs_name; + hdfsBuilderSetNameNode(builder->get(), builder->fs_name.c_str()); + LOG(INFO) << "set hdfs namenode: " << fs_name; + + std::string auth_type = "simple"; + // First, copy all hdfs conf and set to hdfs builder if (hdfsParams.__isset.hdfs_conf) { + // set other conf for (const THdfsConf& conf : hdfsParams.hdfs_conf) { - hdfsBuilderConfSetStr(builder->get(), conf.key.c_str(), conf.value.c_str()); - LOG(INFO) << "set hdfs config: " << conf.key << ", value: " << conf.value; -#ifdef USE_HADOOP_HDFS - // Set krb5.conf, we should define java.security.krb5.conf in catalog properties - if (strcmp(conf.key.c_str(), "java.security.krb5.conf") == 0) { - hdfsBuilderSetKerb5Conf(builder->get(), conf.value.c_str()); + builder->set_hdfs_conf(conf.key, conf.value); + LOG(INFO) << "set hdfs config key: " << conf.key << ", value: " << conf.value; + if (strcmp(conf.key.c_str(), "hadoop.security.authentication") == 0) { + auth_type = conf.value; } -#endif } + builder->set_hdfs_conf_to_hdfs_builder(); } - if (builder->is_kerberos()) { - RETURN_IF_ERROR(builder->check_krb_params()); + + if (auth_type == "kerberos") { + // set kerberos conf + if (!hdfsParams.__isset.hdfs_kerberos_principal || + !hdfsParams.__isset.hdfs_kerberos_keytab) { + return Status::InvalidArgument("Must set both principal and keytab"); + } + builder->kerberos_login = true; + builder->hdfs_kerberos_principal = hdfsParams.hdfs_kerberos_principal; + builder->hdfs_kerberos_keytab = hdfsParams.hdfs_kerberos_keytab; + RETURN_IF_ERROR(builder->set_kerberos_ticket_cache()); + } else { + if (hdfsParams.__isset.user) { + builder->hadoop_user = hdfsParams.user; + hdfsBuilderSetUserName(builder->get(), builder->hadoop_user.c_str()); + } } - hdfsBuilderConfSetStr(builder->get(), "ipc.client.fallback-to-simple-auth-allowed", "true"); + hdfsBuilderConfSetStr(builder->get(), FALLBACK_TO_SIMPLE_AUTH_ALLOWED.c_str(), + TRUE_VALUE.c_str()); return Status::OK(); } diff --git a/be/src/io/hdfs_builder.h b/be/src/io/hdfs_builder.h index 125f3bed82be41..a03eed8c76663f 100644 --- a/be/src/io/hdfs_builder.h +++ b/be/src/io/hdfs_builder.h @@ -33,7 +33,14 @@ const std::string FS_KEY = "fs.defaultFS"; const std::string USER = "hadoop.username"; const std::string KERBEROS_PRINCIPAL = "hadoop.kerberos.principal"; const std::string KERBEROS_KEYTAB = "hadoop.kerberos.keytab"; -const std::string TICKET_CACHE_PATH = "/tmp/krb5cc_doris_"; +const std::string HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; +const std::string FALLBACK_TO_SIMPLE_AUTH_ALLOWED = "ipc.client.fallback-to-simple-auth-allowed"; +const std::string TRUE_VALUE = "true"; + +namespace kerberos { +class KerberosTicketCache; +class KerberosConfig; +}; // namespace kerberos class HDFSCommonBuilder { friend Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_name, @@ -59,11 +66,25 @@ class HDFSCommonBuilder { bool is_kerberos() const { return kerberos_login; } Status check_krb_params(); + Status set_kerberos_ticket_cache(); + void set_hdfs_conf(const std::string& key, const std::string& val); + std::string get_hdfs_conf_value(const std::string& key, const std::string& default_val) const; + void set_hdfs_conf_to_hdfs_builder(); + + std::shared_ptr get_ticket_cache() { return ticket_cache; } + private: hdfsBuilder* hdfs_builder = nullptr; bool kerberos_login {false}; + + // We should save these info from thrift, + // so that the lifecycle of these will same as hdfs_builder + std::string fs_name; + std::string hadoop_user; std::string hdfs_kerberos_keytab; std::string hdfs_kerberos_principal; + std::shared_ptr ticket_cache; + std::unordered_map hdfs_conf; }; THdfsParams parse_properties(const std::map& properties); diff --git a/be/src/util/hdfs_util.cpp b/be/src/io/hdfs_util.cpp similarity index 59% rename from be/src/util/hdfs_util.cpp rename to be/src/io/hdfs_util.cpp index 794c53f15e4f85..0e163d035f2691 100644 --- a/be/src/util/hdfs_util.cpp +++ b/be/src/io/hdfs_util.cpp @@ -15,36 +15,38 @@ // specific language governing permissions and limitations // under the License. -#include "util/hdfs_util.h" +#include "io/hdfs_util.h" + +#include +#include +#include #include +#include #include "common/logging.h" #include "io/fs/err_utils.h" #include "io/hdfs_builder.h" +#include "vec/common/string_ref.h" -namespace doris { -namespace io { - -HDFSHandle& HDFSHandle::instance() { - static HDFSHandle hdfs_handle; - return hdfs_handle; -} +namespace doris::io { -hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) { - hdfsFS hdfs_fs = hdfsBuilderConnect(hdfs_builder.get()); - if (hdfs_fs == nullptr) { - LOG(WARNING) << "connect to hdfs failed." - << ", error: " << hdfs_error(); - return nullptr; - } - return hdfs_fs; -} +namespace hdfs_bvar { +bvar::LatencyRecorder hdfs_read_latency("hdfs_read"); +bvar::LatencyRecorder hdfs_write_latency("hdfs_write"); +bvar::LatencyRecorder hdfs_create_dir_latency("hdfs_create_dir"); +bvar::LatencyRecorder hdfs_open_latency("hdfs_open"); +bvar::LatencyRecorder hdfs_close_latency("hdfs_close"); +bvar::LatencyRecorder hdfs_flush_latency("hdfs_flush"); +bvar::LatencyRecorder hdfs_hflush_latency("hdfs_hflush"); +bvar::LatencyRecorder hdfs_hsync_latency("hdfs_hsync"); +}; // namespace hdfs_bvar Path convert_path(const Path& path, const std::string& namenode) { std::string fs_path; - if (path.native().starts_with(namenode)) { - // `path` is URI format, remove the namenode part in `path` + if (path.native().find(namenode) != std::string::npos) { + // `path` is uri format, remove the namenode part in `path` + // FIXME(plat1ko): Not robust if `namenode` doesn't appear at the beginning of `path` fs_path = path.native().substr(namenode.size()); } else { fs_path = path; @@ -61,5 +63,4 @@ bool is_hdfs(const std::string& path_or_fs) { return path_or_fs.rfind("hdfs://") == 0; } -} // namespace io -} // namespace doris +} // namespace doris::io diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h new file mode 100644 index 00000000000000..7b9a62adda5642 --- /dev/null +++ b/be/src/io/hdfs_util.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "common/kerberos/kerberos_ticket_cache.h" +#include "common/status.h" +#include "io/fs/hdfs.h" +#include "io/fs/path.h" + +namespace cloud { +class HdfsVaultInfo; +} + +namespace doris { +class HDFSCommonBuilder; +class THdfsParams; + +namespace io { + +class HdfsHandler { +public: + hdfsFS hdfs_fs; + bool is_kerberos_auth; + std::string principal; + std::string keytab_path; + std::string fs_name; + std::atomic last_access_time; + std::shared_ptr ticket_cache; + + HdfsHandler(hdfsFS fs, bool is_kerberos, const std::string& principal_, + const std::string& keytab_path_, const std::string& fs_name_, + std::shared_ptr ticket_cache_) + : hdfs_fs(fs), + is_kerberos_auth(is_kerberos), + principal(principal_), + keytab_path(keytab_path_), + fs_name(fs_name_), + last_access_time(std::time(nullptr)), + ticket_cache(ticket_cache_) {} + + ~HdfsHandler() { + // The ticket_cache will be automatically released when the last reference is gone + // No need to explicitly cleanup kerberos ticket + } + + void update_access_time() { last_access_time = std::time(nullptr); } +}; + +namespace hdfs_bvar { +extern bvar::LatencyRecorder hdfs_read_latency; +extern bvar::LatencyRecorder hdfs_write_latency; +extern bvar::LatencyRecorder hdfs_create_dir_latency; +extern bvar::LatencyRecorder hdfs_open_latency; +extern bvar::LatencyRecorder hdfs_close_latency; +extern bvar::LatencyRecorder hdfs_flush_latency; +extern bvar::LatencyRecorder hdfs_hflush_latency; +extern bvar::LatencyRecorder hdfs_hsync_latency; +}; // namespace hdfs_bvar + +// if the format of path is hdfs://ip:port/path, replace it to /path. +// path like hdfs://ip:port/path can't be used by libhdfs3. +Path convert_path(const Path& path, const std::string& namenode); + +std::string get_fs_name(const std::string& path); + +// return true if path_or_fs contains "hdfs://" +bool is_hdfs(const std::string& path_or_fs); + +} // namespace io +} // namespace doris diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index a6f84e1db8f381..98f8ec04ba450e 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -58,6 +58,7 @@ class WorkloadGroupMgr; struct WriteCooldownMetaExecutors; namespace io { class FileCacheFactory; +class HdfsMgr; } // namespace io namespace segment_v2 { class InvertedIndexSearcherCache; @@ -65,6 +66,10 @@ class InvertedIndexQueryCache; class TmpFileDirs; } // namespace segment_v2 +namespace kerberos { +class KerberosTicketMgr; +} + class WorkloadSchedPolicyMgr; class BfdParser; class BrokerMgr; @@ -254,6 +259,9 @@ class ExecEnv { return _write_cooldown_meta_executors.get(); } + kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; } + io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; } + #ifdef BE_TEST void set_tmp_file_dir(std::unique_ptr tmp_file_dirs) { this->_tmp_file_dirs = std::move(tmp_file_dirs); @@ -475,6 +483,9 @@ class ExecEnv { orc::MemoryPool* _orc_memory_pool = nullptr; arrow::MemoryPool* _arrow_memory_pool = nullptr; + + kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr; + io::HdfsMgr* _hdfs_mgr = nullptr; }; template <> diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 45c229cd7146f7..6a47f2c74b0a1d 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -32,6 +32,7 @@ #include #include "common/config.h" +#include "common/kerberos/kerberos_ticket_mgr.h" #include "common/logging.h" #include "common/status.h" #include "io/cache/block/block_file_cache_factory.h" @@ -103,6 +104,15 @@ #include "vec/sink/delta_writer_v2_pool.h" #include "vec/sink/load_stream_map_pool.h" #include "vec/spill/spill_stream_manager.h" +// clang-format off +// this must after util/brpc_client_cache.h +// /doris/thirdparty/installed/include/brpc/errno.pb.h:69:3: error: expected identifier +// EINTERNAL = 2001, +// ^ +// /doris/thirdparty/installed/include/hadoop_hdfs/hdfs.h:61:19: note: expanded from macro 'EINTERNAL' +// #define EINTERNAL 255 +#include "io/fs/hdfs/hdfs_mgr.h" +// clang-format on #if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) @@ -253,6 +263,8 @@ Status ExecEnv::_init(const std::vector& store_paths, _dns_cache = new DNSCache(); _write_cooldown_meta_executors = std::make_unique(); _spill_stream_mgr = new vectorized::SpillStreamManager(std::move(spill_store_map)); + _kerberos_ticket_mgr = new kerberos::KerberosTicketMgr(config::kerberos_ccache_path); + _hdfs_mgr = new io::HdfsMgr(); _backend_client_cache->init_metrics("backend"); _frontend_client_cache->init_metrics("frontend"); _broker_client_cache->init_metrics("broker"); @@ -700,6 +712,8 @@ void ExecEnv::destroy() { // dns cache is a global instance and need to be released at last SAFE_DELETE(_dns_cache); + SAFE_DELETE(_kerberos_ticket_mgr); + SAFE_DELETE(_hdfs_mgr); SAFE_DELETE(_heap_profiler); diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 01409fb3ea585f..df79ee15339fad 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -32,6 +32,7 @@ #include #include +#include "common/config.h" #include "gutil/strings/substitute.h" #include "util/doris_metrics.h" #include "util/jni_native_method.h" @@ -90,6 +91,10 @@ const std::string GetDorisJNIClasspathOption() { } } +const std::string GetKerb5ConfPath() { + return "-Djava.security.krb5.conf=" + config::kerberos_krb5_conf_path; +} + [[maybe_unused]] void SetEnvIfNecessary() { const auto* doris_home = getenv("DORIS_HOME"); DCHECK(doris_home) << "Environment variable DORIS_HOME is not set."; @@ -103,10 +108,11 @@ const std::string GetDorisJNIClasspathOption() { // LIBHDFS_OPTS const std::string java_opts = getenv("JAVA_OPTS") ? getenv("JAVA_OPTS") : ""; std::string libhdfs_opts = - fmt::format("{} -Djava.library.path={}/lib/hadoop_hdfs/native:{}", java_opts, + fmt::format("{} -Djava.library.path={}/lib/hadoop_hdfs/native:{} ", java_opts, getenv("DORIS_HOME"), getenv("DORIS_HOME") + std::string("/lib")); + libhdfs_opts += fmt::format("{} ", GetKerb5ConfPath()); - setenv("LIBHDFS_OPTS", libhdfs_opts.c_str(), 0); + setenv("LIBHDFS_OPTS", libhdfs_opts.c_str(), 1); } // Only used on non-x86 platform @@ -136,6 +142,7 @@ const std::string GetDorisJNIClasspathOption() { std::istream_iterator()); options.push_back(GetDorisJNIClasspathOption()); } + options.push_back(GetKerb5ConfPath()); std::unique_ptr jvm_options(new JavaVMOption[options.size()]); for (int i = 0; i < options.size(); ++i) { jvm_options[i] = {const_cast(options[i].c_str()), nullptr}; diff --git a/be/test/common/kerberos/kerberos_config_test.cpp b/be/test/common/kerberos/kerberos_config_test.cpp new file mode 100644 index 00000000000000..c27be4bfd312fc --- /dev/null +++ b/be/test/common/kerberos/kerberos_config_test.cpp @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_config.h" + +#include + +namespace doris::kerberos { + +class KerberosConfigTest : public testing::Test { +protected: + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(KerberosConfigTest, DefaultValues) { + KerberosConfig config; + EXPECT_EQ(config.get_principal(), ""); + EXPECT_EQ(config.get_keytab_path(), ""); + EXPECT_EQ(config.get_krb5_conf_path(), ""); + EXPECT_EQ(config.get_refresh_interval_second(), 3600); + EXPECT_EQ(config.get_min_time_before_refresh_second(), 600); +} + +TEST_F(KerberosConfigTest, SetterAndGetter) { + KerberosConfig config; + + // Test principal and keytab + config.set_principal_and_keytab("test_principal", "/path/to/keytab"); + EXPECT_EQ(config.get_principal(), "test_principal"); + EXPECT_EQ(config.get_keytab_path(), "/path/to/keytab"); + + // Test krb5 conf path + config.set_krb5_conf_path("/etc/krb5.conf"); + EXPECT_EQ(config.get_krb5_conf_path(), "/etc/krb5.conf"); + + // Test refresh interval + config.set_refresh_interval(500); + EXPECT_EQ(config.get_refresh_interval_second(), 500); + + // Test min time before refresh + config.set_min_time_before_refresh(800); + EXPECT_EQ(config.get_min_time_before_refresh_second(), 800); +} + +} // namespace doris::kerberos diff --git a/be/test/common/kerberos/kerberos_ticket_cache_auth_test.cpp b/be/test/common/kerberos/kerberos_ticket_cache_auth_test.cpp new file mode 100644 index 00000000000000..399f208a6e4273 --- /dev/null +++ b/be/test/common/kerberos/kerberos_ticket_cache_auth_test.cpp @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "common/kerberos/kerberos_config.h" +#include "common/kerberos/kerberos_ticket_cache.h" +#include "gutil/strings/substitute.h" + +namespace doris::kerberos { + +class KerberosTicketCacheAuthTest : public testing::Test { +public: + void SetUp() override { + // Get Kerberos configuration from environment variables + const char* krb5_conf = std::getenv("KRB5_CONFIG"); + const char* principal = std::getenv("KRB5_TEST_PRINCIPAL"); + const char* keytab = std::getenv("KRB5_TEST_KEYTAB"); + + // Skip the test if any required environment variable is not set + if (!krb5_conf || !principal || !keytab) { + GTEST_SKIP() << "Skipping test because one or more required environment variables are " + "not set:\n" + << " KRB5_CONFIG=" << (krb5_conf ? krb5_conf : "not set") << "\n" + << " KRB5_TEST_PRINCIPAL=" << (principal ? principal : "not set") << "\n" + << " KRB5_TEST_KEYTAB=" << (keytab ? keytab : "not set"); + return; + } + + // Skip if the required files don't exist + if (!std::filesystem::exists(krb5_conf) || !std::filesystem::exists(keytab)) { + GTEST_SKIP() << "Skipping test because required files don't exist:\n" + << " krb5.conf: " + << (std::filesystem::exists(krb5_conf) ? "exists" : "not found") << "\n" + << " keytab: " + << (std::filesystem::exists(keytab) ? "exists" : "not found"); + return; + } + + // Initialize test configuration + _config.set_krb5_conf_path(krb5_conf); + _config.set_principal_and_keytab(principal, keytab); + _config.set_refresh_interval(300); // 5 minutes + _config.set_min_time_before_refresh(600); // 10 minutes + + // Create temporary directory for ticket cache + _temp_dir = std::filesystem::temp_directory_path() / "doris_krb5_test"; + if (std::filesystem::exists(_temp_dir)) { + std::filesystem::remove_all(_temp_dir); + } + std::filesystem::create_directories(_temp_dir); + } + + void TearDown() override { + if (std::filesystem::exists(_temp_dir)) { + std::filesystem::remove_all(_temp_dir); + } + } + +protected: + KerberosConfig _config; + std::filesystem::path _temp_dir; +}; + +// Test real Kerberos authentication +// How to run: +// KRB5_CONFIG=/to/krb5.conf \ +// KRB5_TEST_PRINCIPAL=your_principal \ +// KRB5_TEST_KEYTAB=/to/hdfs.keytab \ +// sh run-be-ut.sh --run --filter=KerberosTicketCacheAuthTest.* +TEST_F(KerberosTicketCacheAuthTest, TestRealAuthentication) { + // If SetUp() skipped the test, we should skip here too + if (testing::Test::HasFatalFailure()) { + return; + } + + std::cout << "Starting real Kerberos authentication test with:" << std::endl; + std::cout << " krb5.conf: " << _config.get_krb5_conf_path() << std::endl; + std::cout << " principal: " << _config.get_principal() << std::endl; + std::cout << " keytab: " << _config.get_keytab_path() << std::endl; + + std::string cache_path; + { + // Create ticket cache instance + KerberosTicketCache ticket_cache(_config, _temp_dir.string()); + + // Initialize the ticket cache + Status status = ticket_cache.initialize(); + ASSERT_TRUE(status.ok()) << "Failed to initialize ticket cache: " << status.to_string(); + + // Perform Kerberos login + status = ticket_cache.login(); + ASSERT_TRUE(status.ok()) << "Failed to perform Kerberos login: " << status.to_string(); + + // Write ticket to cache file + status = ticket_cache.write_ticket_cache(); + ASSERT_TRUE(status.ok()) << "Failed to write ticket cache: " << status.to_string(); + + // Verify that the cache file exists + ASSERT_TRUE(std::filesystem::exists(ticket_cache.get_ticket_cache_path())) + << "Ticket cache file not found at: " << ticket_cache.get_ticket_cache_path(); + cache_path = ticket_cache.get_ticket_cache_path(); + + // Try to login with the cache + status = ticket_cache.login_with_cache(); + ASSERT_TRUE(status.ok()) << "Failed to login with cache: " << status.to_string(); + + std::cout << "Successfully authenticated and created ticket cache at: " + << ticket_cache.get_ticket_cache_path() << std::endl; + } + + // After ticket_cache delete, the cache file should be deleted too + ASSERT_FALSE(std::filesystem::exists(cache_path)) << cache_path; +} + +} // namespace doris::kerberos diff --git a/be/test/common/kerberos/kerberos_ticket_cache_test.cpp b/be/test/common/kerberos/kerberos_ticket_cache_test.cpp new file mode 100644 index 00000000000000..cbd6294fde4e52 --- /dev/null +++ b/be/test/common/kerberos/kerberos_ticket_cache_test.cpp @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_ticket_cache.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace doris::kerberos { + +using testing::_; + +// Mock class +class MockKrb5Interface : public Krb5Interface { +public: + MOCK_METHOD(Status, init_context, (krb5_context*), (override)); + MOCK_METHOD(Status, parse_name, (krb5_context, const char*, krb5_principal*), (override)); + MOCK_METHOD(Status, kt_resolve, (krb5_context, const char*, krb5_keytab*), (override)); + MOCK_METHOD(Status, cc_resolve, (krb5_context, const char*, krb5_ccache*), (override)); + MOCK_METHOD(Status, get_init_creds_opt_alloc, (krb5_context, krb5_get_init_creds_opt**), + (override)); + MOCK_METHOD(Status, get_init_creds_keytab, + (krb5_context, krb5_creds*, krb5_principal, krb5_keytab, krb5_deltat, const char*, + krb5_get_init_creds_opt*), + (override)); + MOCK_METHOD(Status, cc_initialize, (krb5_context, krb5_ccache, krb5_principal), (override)); + MOCK_METHOD(Status, cc_store_cred, (krb5_context, krb5_ccache, krb5_creds*), (override)); + MOCK_METHOD(Status, timeofday, (krb5_context, krb5_timestamp*), (override)); + MOCK_METHOD(Status, cc_start_seq_get, (krb5_context, krb5_ccache, krb5_cc_cursor*), (override)); + MOCK_METHOD(Status, cc_next_cred, (krb5_context, krb5_ccache, krb5_cc_cursor*, krb5_creds*), + (override)); + MOCK_METHOD(void, cc_end_seq_get, (krb5_context, krb5_ccache, krb5_cc_cursor*), (override)); + + MOCK_METHOD(void, free_principal, (krb5_context, krb5_principal), (override)); + MOCK_METHOD(void, free_cred_contents, (krb5_context, krb5_creds*), (override)); + MOCK_METHOD(void, get_init_creds_opt_free, (krb5_context, krb5_get_init_creds_opt*), + (override)); + MOCK_METHOD(void, kt_close, (krb5_context, krb5_keytab), (override)); + MOCK_METHOD(void, cc_close, (krb5_context, krb5_ccache), (override)); + MOCK_METHOD(void, free_context, (krb5_context), (override)); + MOCK_METHOD(const char*, get_error_message, (krb5_context, krb5_error_code), (override)); + MOCK_METHOD(void, free_error_message, (krb5_context, const char*), (override)); + MOCK_METHOD(Status, unparse_name, (krb5_context, krb5_principal, char**), (override)); + MOCK_METHOD(void, free_unparsed_name, (krb5_context, char*), (override)); +}; + +class KerberosTicketCacheTest : public testing::Test { +protected: + void SetUp() override { + _mock_krb5 = std::make_unique>(); + _mock_krb5_ptr = _mock_krb5.get(); + + // Create a temporary directory for testing + _test_dir = std::filesystem::temp_directory_path() / "kerberos_test"; + std::filesystem::create_directories(_test_dir); + + _config.set_principal_and_keytab("test_principal", "/path/to/keytab"); + _config.set_krb5_conf_path("/etc/krb5.conf"); + _config.set_refresh_interval(2); + _config.set_min_time_before_refresh(600); + + _cache = std::make_unique(_config, _test_dir.string(), + std::move(_mock_krb5)); + _cache->set_refresh_thread_sleep_time(std::chrono::milliseconds(1)); + } + + void TearDown() override { + _cache.reset(); + if (std::filesystem::exists(_test_dir)) { + std::filesystem::remove_all(_test_dir); + } + } + + // Helper function to set up basic expectations for initialization + void SetupBasicInitExpectations() { + EXPECT_CALL(*_mock_krb5_ptr, init_context(_)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, parse_name(_, _, _)).WillOnce(testing::Return(Status::OK())); + } + + // Helper function to simulate ticket cache file creation/update + void SimulateTicketCacheFileUpdate(const std::string& cache_path) { + std::ofstream cache_file(cache_path); + cache_file << "mock ticket cache content"; + cache_file.close(); + // Sleep a bit to ensure file timestamps are different + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + void ExpectForLogin(const std::string& cache_path) { + // Setup expectations for login + EXPECT_CALL(*_mock_krb5_ptr, kt_resolve(_, _, _)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_resolve(_, _, _)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_opt_alloc(_, _)) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_keytab(_, _, _, _, _, _, _)) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_initialize(_, _, _)) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_store_cred(_, _, _)) + .WillOnce(testing::DoAll( + testing::Invoke([this, cache_path](krb5_context, krb5_ccache, krb5_creds*) { + SimulateTicketCacheFileUpdate(cache_path); + return Status::OK(); + }))); + + // Cleanup calls + EXPECT_CALL(*_mock_krb5_ptr, free_cred_contents(_, _)).Times(1); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_opt_free(_, _)).Times(1); + EXPECT_CALL(*_mock_krb5_ptr, kt_close(_, _)).Times(1); + } + + // Helper function to get file last write time + std::filesystem::file_time_type GetFileLastWriteTime(const std::string& path) { + return std::filesystem::last_write_time(path); + } + + void printFileTime(const std::filesystem::file_time_type& ftime) { + auto sctp = std::chrono::time_point_cast( + ftime - std::filesystem::file_time_type::clock::now() + + std::chrono::system_clock::now()); + std::time_t cftime = std::chrono::system_clock::to_time_t(sctp); + std::cout << std::put_time(std::localtime(&cftime), "%Y-%m-%d %H:%M:%S") << '\n'; + } + +protected: + KerberosConfig _config; + std::unique_ptr _cache; + MockKrb5Interface* _mock_krb5_ptr; + std::unique_ptr _mock_krb5; + std::filesystem::path _test_dir; +}; + +TEST_F(KerberosTicketCacheTest, Initialize) { + SetupBasicInitExpectations(); + ASSERT_TRUE(_cache->initialize().ok()); + + // Verify that the cache file is created in the test directory + std::string cache_path = _cache->get_ticket_cache_path(); + ASSERT_TRUE(cache_path.find(_test_dir.string()) == 0); +} + +TEST_F(KerberosTicketCacheTest, LoginSuccess) { + SetupBasicInitExpectations(); + ASSERT_TRUE(_cache->initialize().ok()); + + std::string cache_path = _cache->get_ticket_cache_path(); + ExpectForLogin(cache_path); + ASSERT_TRUE(_cache->login().ok()); + ASSERT_TRUE(std::filesystem::exists(cache_path)); +} + +TEST_F(KerberosTicketCacheTest, RefreshTickets) { + SetupBasicInitExpectations(); + ASSERT_TRUE(_cache->initialize().ok()); + + std::string cache_path = _cache->get_ticket_cache_path(); + + // Create initial ticket cache file + SimulateTicketCacheFileUpdate(cache_path); + auto initial_write_time = GetFileLastWriteTime(cache_path); + + // Setup expectations for refresh (login) + EXPECT_CALL(*_mock_krb5_ptr, kt_resolve(_, _, _)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_resolve(_, _, _)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_opt_alloc(_, _)) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_keytab(_, _, _, _, _, _, _)) + .WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_initialize(_, _, _)).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*_mock_krb5_ptr, cc_store_cred(_, _, _)) + .WillOnce(testing::DoAll( + testing::Invoke([this, cache_path](krb5_context, krb5_ccache, krb5_creds*) { + SimulateTicketCacheFileUpdate(cache_path); + return Status::OK(); + }))); + + // Cleanup calls + // Because we forcbly refresh the ticket, the _need_refresh() is not called, + // so it only call free_cred_contents once. + EXPECT_CALL(*_mock_krb5_ptr, free_cred_contents(_, _)).Times(1); + EXPECT_CALL(*_mock_krb5_ptr, get_init_creds_opt_free(_, _)).Times(1); + EXPECT_CALL(*_mock_krb5_ptr, kt_close(_, _)).Times(1); + + ASSERT_TRUE(_cache->refresh_tickets().ok()); + + // Verify that the cache file has been updated + auto new_write_time = GetFileLastWriteTime(cache_path); + ASSERT_GT(new_write_time, initial_write_time); +} + +TEST_F(KerberosTicketCacheTest, PeriodicRefresh) { + SetupBasicInitExpectations(); + ASSERT_TRUE(_cache->initialize().ok()); + + std::string cache_path = _cache->get_ticket_cache_path(); + SimulateTicketCacheFileUpdate(cache_path); + auto initial_write_time = GetFileLastWriteTime(cache_path); + printFileTime(initial_write_time); + + ExpectForLogin(cache_path); + + // Start periodic refresh + _cache->start_periodic_refresh(); + + // Wait for a short time to allow some refresh attempts + // Because the refresh interval is 2s, need larger than 2s + std::this_thread::sleep_for(std::chrono::milliseconds(4000)); + + // Stop periodic refresh + _cache->stop_periodic_refresh(); + + // Verify that the test directory still exists + ASSERT_TRUE(std::filesystem::exists(_test_dir)); + + // Verify that the cache file still exists and has been updated + ASSERT_TRUE(std::filesystem::exists(cache_path)); + auto final_write_time = GetFileLastWriteTime(cache_path); + printFileTime(final_write_time); + ASSERT_GT(final_write_time, initial_write_time); +} + +} // namespace doris::kerberos diff --git a/be/test/common/kerberos/kerberos_ticket_mgr_test.cpp b/be/test/common/kerberos/kerberos_ticket_mgr_test.cpp new file mode 100644 index 00000000000000..a205c9f80f7bff --- /dev/null +++ b/be/test/common/kerberos/kerberos_ticket_mgr_test.cpp @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/kerberos/kerberos_ticket_mgr.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace doris::kerberos { + +// Mock KerberosTicketCache class +class MockKerberosTicketCache : public KerberosTicketCache { +public: + explicit MockKerberosTicketCache(const KerberosConfig& config, const std::string& root_path) + : KerberosTicketCache(config, root_path) {} + + MOCK_METHOD(Status, initialize, (), (override)); + MOCK_METHOD(Status, login, (), (override)); + MOCK_METHOD(Status, write_ticket_cache, (), (override)); + MOCK_METHOD(void, start_periodic_refresh, (), (override)); +}; + +class MockKerberosTicketMgr : public KerberosTicketMgr { +public: + MockKerberosTicketMgr(const std::string& root_path) : KerberosTicketMgr(root_path) {} + + // by calling this, will first set _mock_cache_path in MockKerberosTicketMgr, + // and _mock_cache_path will pass to KerberosTicketCache to set cache's ticket_cache_path. + void set_mock_cache_path(const std::string& cache_path) { _mock_cache_path = cache_path; } + void set_should_succeed(bool val) { _should_succeed = val; } + +protected: + std::shared_ptr _make_new_ticket_cache( + const KerberosConfig& config) override { + auto mock_cache = + std::make_shared>(config, _root_path); + + if (_should_succeed) { + EXPECT_CALL(*mock_cache, initialize()).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*mock_cache, login()).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*mock_cache, write_ticket_cache()).WillOnce(testing::Return(Status::OK())); + EXPECT_CALL(*mock_cache, start_periodic_refresh()).Times(1); + mock_cache->set_ticket_cache_path(_mock_cache_path); + } else { + EXPECT_CALL(*mock_cache, initialize()) + .WillOnce(testing::Return(Status::InternalError("Init failed"))); + } + + return mock_cache; + } + +private: + std::string _mock_cache_path; + bool _should_succeed = true; +}; + +class KerberosTicketMgrTest : public testing::Test { +protected: + void SetUp() override { + // Create a temporary directory for testing + _test_dir = std::filesystem::temp_directory_path() / "kerberos_mgr_test"; + std::filesystem::create_directories(_test_dir); + _mgr = std::make_unique(_test_dir.string()); + } + + void TearDown() override { + _mgr.reset(); + if (std::filesystem::exists(_test_dir)) { + std::filesystem::remove_all(_test_dir); + } + } + + // Helper function to create a config + KerberosConfig create_config(const std::string& principal, const std::string& keytab) { + KerberosConfig config; + config.set_principal_and_keytab(principal, keytab); + config.set_refresh_interval(300); + config.set_min_time_before_refresh(600); + return config; + } + +protected: + std::unique_ptr _mgr; + std::filesystem::path _test_dir; +}; + +TEST_F(KerberosTicketMgrTest, GetOrSetNewCache) { + KerberosConfig config = create_config("test_principal", "/path/to/keytab"); + std::string expected_cache_path = (_test_dir / "test_cache").string(); + _mgr->set_should_succeed(true); + _mgr->set_mock_cache_path(expected_cache_path); + + std::shared_ptr ticket_cache; + doris::Status st = _mgr->get_or_set_ticket_cache(config, &ticket_cache); + ASSERT_TRUE(st.ok()) << st; + ASSERT_EQ(ticket_cache->get_ticket_cache_path(), expected_cache_path); + + // Second call should return the same cache path + std::shared_ptr second_ticket_cache; + ASSERT_TRUE(_mgr->get_or_set_ticket_cache(config, &second_ticket_cache).ok()); + ASSERT_EQ(second_ticket_cache->get_ticket_cache_path(), expected_cache_path); +} + +TEST_F(KerberosTicketMgrTest, GetOrSetCacheFailure) { + KerberosConfig config = create_config("test_principal", "/path/to/keytab"); + + // Test initialization failure + { + _mgr->set_should_succeed(false); + std::shared_ptr ticket_cache; + auto status = _mgr->get_or_set_ticket_cache(config, &ticket_cache); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.to_string().find("Init failed") != std::string::npos); + } + + // Verify no cache was added + ASSERT_TRUE(_mgr->get_ticket_cache(config.get_principal(), config.get_keytab_path()) == + nullptr); +} + +} // namespace doris::kerberos diff --git a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp new file mode 100644 index 00000000000000..7f05428acf015a --- /dev/null +++ b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp @@ -0,0 +1,362 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/hdfs/hdfs_mgr.h" + +#include +#include + +#include +#include + +#include "common/kerberos/kerberos_ticket_cache.h" +#include "common/kerberos/kerberos_ticket_mgr.h" +#include "common/status.h" +#include "io/fs/hdfs.h" + +namespace doris::io { + +using testing::_; +using testing::Return; +using testing::NiceMock; +using testing::DoAll; +using testing::SetArgPointee; + +// Mock KerberosTicketCache class to track its lifecycle +class MockKerberosTicketCache : public kerberos::KerberosTicketCache { +public: + MockKerberosTicketCache(const kerberos::KerberosConfig& config, const std::string& root_path) + : KerberosTicketCache(config, root_path) { + ++instance_count; + } + + ~MockKerberosTicketCache() override { --instance_count; } + + static int get_instance_count() { return instance_count; } + static void reset_instance_count() { instance_count = 0; } + + // Mock methods that would normally interact with the real Kerberos system + Status initialize() override { return Status::OK(); } + Status login() override { return Status::OK(); } + Status write_ticket_cache() override { return Status::OK(); } + +private: + static int instance_count; +}; + +int MockKerberosTicketCache::instance_count = 0; + +// Mock HdfsMgr class to override the actual HDFS operations +class MockHdfsMgr : public HdfsMgr { +public: + MockHdfsMgr() : HdfsMgr() {} + + MOCK_METHOD(Status, _create_hdfs_fs_impl, + (const THdfsParams& hdfs_params, const std::string& fs_name, + std::shared_ptr* fs_handler), + (override)); +}; + +// Mock KerberosTicketMgr class to use MockKerberosTicketCache +class MockKerberosTicketMgr : public kerberos::KerberosTicketMgr { +public: + explicit MockKerberosTicketMgr(const std::string& root_path) : KerberosTicketMgr(root_path) {} + +protected: + std::shared_ptr _make_new_ticket_cache( + const kerberos::KerberosConfig& config) override { + return std::make_shared(config, _root_path); + } +}; + +class HdfsMgrTest : public testing::Test { +protected: + void SetUp() override { + _hdfs_mgr = std::make_unique>(); + + // Set shorter timeout for testing + _hdfs_mgr->set_instance_timeout_seconds(2); + _hdfs_mgr->set_cleanup_interval_seconds(1); + + // Reset MockKerberosTicketCache instance count + MockKerberosTicketCache::reset_instance_count(); + + // Setup default mock behavior + ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillByDefault([](const THdfsParams& params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + *fs_handler = + std::make_shared(nullptr, false, "", "", fs_name, nullptr); + return Status::OK(); + }); + } + + void TearDown() override { _hdfs_mgr.reset(); } + + THdfsParams create_test_params(const std::string& user = "", const std::string& principal = "", + const std::string& keytab = "") { + THdfsParams params; + if (!user.empty()) { + params.__set_user(user); + } + if (!principal.empty()) { + params.__set_hdfs_kerberos_principal(principal); + } + if (!keytab.empty()) { + params.__set_hdfs_kerberos_keytab(keytab); + } + return params; + } + +protected: + std::unique_ptr _hdfs_mgr; +}; + +// Test get_or_create_fs with successful creation +TEST_F(HdfsMgrTest, GetOrCreateFsSuccess) { + THdfsParams params = create_test_params("test_user"); + std::shared_ptr handler; + + // First call should create new handler + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 1); + ASSERT_TRUE(handler != nullptr); + + // Second call with same params should reuse handler + std::shared_ptr handler2; + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler2).ok()); + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 1); + ASSERT_EQ(handler, handler2); +} + +// Test get_or_create_fs with creation failure +TEST_F(HdfsMgrTest, GetOrCreateFsFailure) { + THdfsParams params = create_test_params("test_user"); + std::shared_ptr handler; + + // Setup mock to return error + ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillByDefault(Return(Status::InternalError("Mock error"))); + + // Call should fail + ASSERT_FALSE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); + ASSERT_TRUE(handler == nullptr); +} + +// Test _hdfs_hash_code with different inputs +TEST_F(HdfsMgrTest, HdfsHashCode) { + // Same inputs should produce same hash + THdfsParams params1 = create_test_params("user1", "principal1", "keytab1"); + uint64_t hash1 = _hdfs_mgr->_hdfs_hash_code(params1, "fs1"); + uint64_t hash2 = _hdfs_mgr->_hdfs_hash_code(params1, "fs1"); + ASSERT_EQ(hash1, hash2); + + // Different inputs should produce different hashes + THdfsParams params2 = create_test_params("user2", "principal1", "keytab1"); + THdfsParams params3 = create_test_params("user1", "principal2", "keytab1"); + THdfsParams params4 = create_test_params("user1", "principal1", "keytab2"); + + uint64_t hash3 = _hdfs_mgr->_hdfs_hash_code(params2, "fs1"); + uint64_t hash4 = _hdfs_mgr->_hdfs_hash_code(params3, "fs1"); + uint64_t hash5 = _hdfs_mgr->_hdfs_hash_code(params4, "fs1"); + uint64_t hash6 = _hdfs_mgr->_hdfs_hash_code(params1, "fs2"); + + ASSERT_NE(hash1, hash3); + ASSERT_NE(hash1, hash4); + ASSERT_NE(hash1, hash5); + ASSERT_NE(hash1, hash6); +} + +// Test cleanup of expired handlers +TEST_F(HdfsMgrTest, CleanupExpiredHandlers) { + THdfsParams params = create_test_params("test_user"); + std::shared_ptr handler; + + // Create handler + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 1); + + // Wait for cleanup + std::this_thread::sleep_for(std::chrono::seconds(4)); + + // Handler should be removed + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); + + // Creating new handler should work + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 1); +} + +// Test concurrent access to get_or_create_fs +TEST_F(HdfsMgrTest, ConcurrentAccess) { + const int NUM_THREADS = 10; + std::vector threads; + std::vector> handlers(NUM_THREADS); + + THdfsParams params = create_test_params("test_user"); + + // Create multiple threads accessing get_or_create_fs simultaneously + for (int i = 0; i < NUM_THREADS; i++) { + threads.emplace_back([this, ¶ms, &handlers, i]() { + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handlers[i]).ok()); + }); + } + + // Wait for all threads to complete + for (auto& thread : threads) { + thread.join(); + } + + // Verify all threads got the same handler + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 1); + for (int i = 1; i < NUM_THREADS; i++) { + ASSERT_EQ(handlers[0], handlers[i]); + } +} + +// Test sharing of KerberosTicketCache between handlers +TEST_F(HdfsMgrTest, SharedKerberosTicketCache) { + // Create handlers with same Kerberos credentials + THdfsParams params = create_test_params("user1", "principal1", "keytab1"); + std::shared_ptr handler1; + std::shared_ptr handler2; + + // Create a shared ticket cache that will be used by both handlers + auto shared_ticket_mgr = std::make_shared>("/tmp/kerberos"); + // Set cleanup interval to 1 second for testing + shared_ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); + + // Setup mock to create handlers with Kerberos + ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillByDefault([shared_ticket_mgr](const THdfsParams& params, + const std::string& fs_name, + std::shared_ptr* fs_handler) { + kerberos::KerberosConfig config; + config.set_principal_and_keytab(params.hdfs_kerberos_principal, + params.hdfs_kerberos_keytab); + std::shared_ptr ticket_cache; + RETURN_IF_ERROR(shared_ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); + *fs_handler = std::make_shared( + nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, + fs_name, ticket_cache); + return Status::OK(); + }); + + // Create first handler + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs1", &handler1).ok()); + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); + + // Create second handler with same credentials + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs2", &handler2).ok()); + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); + + // Verify both handlers share the same ticket cache + ASSERT_EQ(handler1->ticket_cache, handler2->ticket_cache); + // ASSERT_EQ(handler1->ticket_cache, shared_ticket_cache); +} + +// Test cleanup of KerberosTicketCache when handlers are destroyed +TEST_F(HdfsMgrTest, KerberosTicketCacheCleanup) { + THdfsParams params = create_test_params("user1", "principal1", "keytab1"); + + // Create a ticket manager that will be used by the handler + auto ticket_mgr = std::make_shared>("/tmp/kerberos"); + // Set cleanup interval to 1 second for testing + ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); + + // Setup mock to create handler with Kerberos + ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + kerberos::KerberosConfig config; + config.set_principal_and_keytab(params.hdfs_kerberos_principal, + params.hdfs_kerberos_keytab); + std::shared_ptr ticket_cache; + + RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); + *fs_handler = std::make_shared( + nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, + fs_name, ticket_cache); + return Status::OK(); + }); + + // Create handler + std::shared_ptr handler; + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); + std::shared_ptr ticket_cache_holder = handler->ticket_cache; + handler.reset(); + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); + + // Wait for cleanup + ticket_cache_holder.reset(); + std::this_thread::sleep_for(std::chrono::seconds(6)); + + // Verify handler and ticket cache are cleaned up + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); +} + +// Test multiple handlers with different Kerberos credentials +TEST_F(HdfsMgrTest, DifferentKerberosCredentials) { + // Create a ticket manager that will be used by both handlers + auto ticket_mgr = std::make_shared>("/tmp/kerberos"); + // Set cleanup interval to 1 second for testing + ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); + + // Setup mock to create handlers with Kerberos + ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) + .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, + std::shared_ptr* fs_handler) { + kerberos::KerberosConfig config; + config.set_principal_and_keytab(params.hdfs_kerberos_principal, + params.hdfs_kerberos_keytab); + std::shared_ptr ticket_cache; + + RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); + *fs_handler = std::make_shared( + nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, + fs_name, ticket_cache); + return Status::OK(); + }); + + // Create handlers with different credentials + // std::cout << "xxx 6 MockKerberosTicketCache::get_instance_count(): " << MockKerberosTicketCache::get_instance_count() << std::endl; + THdfsParams params1 = create_test_params("user1", "principal1", "keytab1"); + THdfsParams params2 = create_test_params("user2", "principal2", "keytab2"); + std::shared_ptr handler1; + std::shared_ptr handler2; + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params1, "test_fs1", &handler1).ok()); + ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params2, "test_fs2", &handler2).ok()); + + // Verify each handler has its own ticket cache + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 2); + ASSERT_NE(handler1->ticket_cache, handler2->ticket_cache); + + // Wait for cleanup + // Also need to reset this 2 temp references + handler1.reset(); + handler2.reset(); + std::this_thread::sleep_for(std::chrono::seconds(6)); + + // Verify all handlers and ticket caches are cleaned up + ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); + + ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); +} + +} // namespace doris::io diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index 69787463bc7dc2..a6a28dacdb7724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -88,7 +88,9 @@ public enum SchemaTableType { SCH_CATALOG_META_CACHE_STATISTICS("CATALOG_META_CACHE_STATISTICS", "CATALOG_META_CACHE_STATISTICS", TSchemaTableType.SCH_CATALOG_META_CACHE_STATISTICS), SCH_TABLE_OPTIONS("TABLE_OPTIONS", "TABLE_OPTIONS", - TSchemaTableType.SCH_TABLE_OPTIONS); + TSchemaTableType.SCH_TABLE_OPTIONS), + SCH_BACKEND_KERBEROS_TICKET_CACHE("BACKEND_KERBEROS_TICKET_CACHE", "BACKEND_KERBEROS_TICKET_CACHE", + TSchemaTableType.SCH_BACKEND_KERBEROS_TICKET_CACHE); private static final String dbName = "INFORMATION_SCHEMA"; private static SelectList fullSelectLists; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 8f12300faea57a..190a4f276bbdbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -573,6 +573,22 @@ public class SchemaTable extends Table { .column("BUCKETS_NUM", ScalarType.createType(PrimitiveType.INT)) .column("PARTITION_NUM", ScalarType.createType(PrimitiveType.INT)) .build())) + .put("backend_kerberos_ticket_cache", + new SchemaTable(SystemIdGenerator.getNextId(), "backend_kerberos_ticket_cache", TableType.SCHEMA, + builder().column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("BE_IP", ScalarType.createStringType()) + .column("PRINCIPAL", ScalarType.createStringType()) + .column("KEYTAB", ScalarType.createStringType()) + .column("SERVICE_PRINCIPAL", ScalarType.createStringType()) + .column("TICKET_CACHE_PATH", ScalarType.createStringType()) + .column("HASH_CODE", ScalarType.createStringType()) + .column("START_TIME", ScalarType.createType(PrimitiveType.DATETIME)) + .column("EXPIRE_TIME", ScalarType.createType(PrimitiveType.DATETIME)) + .column("AUTH_TIME", ScalarType.createType(PrimitiveType.DATETIME)) + .column("REF_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("REFRESH_INTERVAL_SECOND", ScalarType.createType(PrimitiveType.BIGINT)) + .build()) + ) .build(); private boolean fetchAllFe = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java index cbbb2f67565c18..04950b61613a68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java @@ -69,7 +69,7 @@ public class BackendPartitionedSchemaScanNode extends SchemaScanNode { BEACKEND_ID_COLUMN_SET.add("be_id"); BACKEND_TABLE.add("file_cache_statistics"); - BEACKEND_ID_COLUMN_SET.add("be_id"); + BACKEND_TABLE.add("backend_kerberos_ticket_cache"); } public static boolean isBackendPartitionedSchemaTable(String tableName) { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 1cb6f79cf97894..f48923d57b5eb9 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -137,7 +137,8 @@ enum TSchemaTableType { SCH_WORKLOAD_GROUP_RESOURCE_USAGE = 49, SCH_TABLE_PROPERTIES = 50, SCH_FILE_CACHE_STATISTICS = 51, - SCH_CATALOG_META_CACHE_STATISTICS = 52; + SCH_CATALOG_META_CACHE_STATISTICS = 52, + SCH_BACKEND_KERBEROS_TICKET_CACHE = 53; } enum THdfsCompression { diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index a5661517e8787b..addc59d3c30095 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -16,6 +16,16 @@ // under the License. suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + def command = "sudo docker ps" + def process = command.execute() + process.waitFor() + + def output = process.in.text + + def keytab_root_dir = "/keytabs" + + println "Docker containers:" + println output String enabled = context.config.otherConfigs.get("enableKerberosTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") if (enabled != null && enabled.equalsIgnoreCase("true")) { @@ -29,7 +39,7 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", "hadoop.security.auth_to_local" = "RULE:[2:\$1@\$0](.*@LABS.TERADATA.COM)s/@.*// RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// @@ -53,7 +63,7 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab" + "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab" ); """ sql """ switch hms_kerberos_hadoop_err1 """ @@ -92,7 +102,7 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa // "fs.defaultFS" = "hdfs://${externalEnvIp}:8520", // "hadoop.security.authentication" = "kerberos", // "hadoop.kerberos.principal"="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM", - // "hadoop.kerberos.keytab" = "/keytabs/presto-server.keytab", + // "hadoop.kerberos.keytab" = "${keytab_root_dir}/presto-server.keytab", // "hive.metastore.thrift.impersonation.enabled" = true" // "hive.metastore.client.credential-cache.location" = "hive-presto-master-krbcc" // ); @@ -102,5 +112,13 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa // } catch (Exception e) { // logger.error(e.message) // } + + // test information_schema.backend_kerberos_ticket_cache + // switch to a normal catalog + sql "switch internal"; + test { + sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/presto-server.keytab";""" + rowNum 1 + } } } diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 725d570d0e3c85..c081e5a401f683 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -20,6 +20,16 @@ import groovyjarjarantlr4.v4.codegen.model.ExceptionClause import org.junit.Assert; suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + def command = "sudo docker ps" + def process = command.execute() + process.waitFor() + + def output = process.in.text + + def keytab_root_dir = "/keytabs" + + println "Docker containers:" + println output String enabled = context.config.otherConfigs.get("enableKerberosTest") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") if (enabled != null && enabled.equalsIgnoreCase("true")) { @@ -34,7 +44,7 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", - "hadoop.kerberos.keytab" = "/keytabs/hive-presto-master.keytab", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab", "hive.metastore.sasl.enabled " = "true", "hive.metastore.kerberos.principal" = "hive/hadoop-master@LABS.TERADATA.COM" ); @@ -50,7 +60,7 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "hadoop.kerberos.min.seconds.before.relogin" = "5", "hadoop.security.authentication" = "kerberos", "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", - "hadoop.kerberos.keytab" = "/keytabs/other-hive-presto-master.keytab", + "hadoop.kerberos.keytab" = "${keytab_root_dir}/other-hive-presto-master.keytab", "hive.metastore.sasl.enabled " = "true", "hive.metastore.kerberos.principal" = "hive/hadoop-master-2@OTHERREALM.COM", "hadoop.security.auth_to_local" ="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// @@ -75,30 +85,32 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d // 3. write back test case sql """ switch ${hms_catalog_name}; """ - sql """ CREATE DATABASE IF NOT EXISTS `test_krb_hms_db`; """ - sql """ USE `test_krb_hms_db`; """ - sql """ CREATE TABLE IF NOT EXISTS test_krb_hive_tbl (id int, str string, dd date) engine = hive; """ - sql """ INSERT INTO test_krb_hms_db.test_krb_hive_tbl values(1, 'krb1', '2023-05-14') """ - sql """ INSERT INTO test_krb_hms_db.test_krb_hive_tbl values(2, 'krb2', '2023-05-24') """ + sql """ CREATE DATABASE IF NOT EXISTS `write_back_krb_hms_db`; """ + sql """ USE `write_back_krb_hms_db`; """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl""" + sql """ CREATE TABLE test_krb_hive_tbl (id int, str string, dd date) engine = hive; """ + sql """ INSERT INTO write_back_krb_hms_db.test_krb_hive_tbl values(1, 'krb1', '2023-05-14') """ + sql """ INSERT INTO write_back_krb_hms_db.test_krb_hive_tbl values(2, 'krb2', '2023-05-24') """ sql """ switch other_${hms_catalog_name}; """ - sql """ CREATE DATABASE IF NOT EXISTS `test_krb_hms_db`; """ - sql """ USE `test_krb_hms_db`; """ - sql """ CREATE TABLE IF NOT EXISTS test_krb_hive_tbl (id int, str string, dd date) engine = hive; """ - sql """ INSERT INTO test_krb_hms_db.test_krb_hive_tbl values(1, 'krb1', '2023-05-24') """ - sql """ INSERT INTO test_krb_hms_db.test_krb_hive_tbl values(2, 'krb2', '2023-05-24') """ + sql """ CREATE DATABASE IF NOT EXISTS `write_back_krb_hms_db`; """ + sql """ USE `write_back_krb_hms_db`; """ + sql """ DROP TABLE IF EXISTS test_krb_hive_tbl""" + sql """ CREATE TABLE test_krb_hive_tbl (id int, str string, dd date) engine = hive; """ + sql """ INSERT INTO write_back_krb_hms_db.test_krb_hive_tbl values(1, 'krb1', '2023-05-24') """ + sql """ INSERT INTO write_back_krb_hms_db.test_krb_hive_tbl values(2, 'krb2', '2023-05-24') """ - sql """ INSERT INTO ${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ - sql """ INSERT INTO other_${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ - order_qt_q03 """ select * from ${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl """ - order_qt_q04 """ select * from other_${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl """ + sql """ INSERT INTO ${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ + sql """ INSERT INTO other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ + order_qt_q03 """ select * from ${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl """ + order_qt_q04 """ select * from other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl """ // 4. multi thread test Thread thread1 = new Thread(() -> { try { for (int i = 0; i < 100; i++) { sql """ select * from ${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ - sql """ INSERT INTO ${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ + sql """ INSERT INTO ${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ } } catch (Exception e) { log.info(e.getMessage()) @@ -110,7 +122,7 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d try { for (int i = 0; i < 100; i++) { sql """ select * from other_${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ - sql """ INSERT INTO other_${hms_catalog_name}.test_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ + sql """ INSERT INTO other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ } } catch (Exception e) { log.info(e.getMessage()) @@ -123,7 +135,20 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d thread1.join() thread2.join() - sql """drop catalog ${hms_catalog_name};""" - sql """drop catalog other_${hms_catalog_name};""" + + // test information_schema.backend_kerberos_ticket_cache + sql """switch internal""" + test { + sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/hive-presto-master.keytab";""" + rowNum 1 + } + + test { + sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@OTHERREALM.COM" and KEYTAB = "${keytab_root_dir}/other-hive-presto-master.keytab";""" + rowNum 1 + } + + // sql """drop catalog ${hms_catalog_name};""" + // sql """drop catalog other_${hms_catalog_name};""" } } From 04fb4872b1a1fed927e033fc229af343247a97d5 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 10 Feb 2025 12:54:43 +0800 Subject: [PATCH 2/2] fix temp kerberos ticket patch string issue --- be/src/io/hdfs_builder.h | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/io/hdfs_builder.h b/be/src/io/hdfs_builder.h index a03eed8c76663f..320275981ad974 100644 --- a/be/src/io/hdfs_builder.h +++ b/be/src/io/hdfs_builder.h @@ -83,6 +83,7 @@ class HDFSCommonBuilder { std::string hadoop_user; std::string hdfs_kerberos_keytab; std::string hdfs_kerberos_principal; + std::string kerberos_ticket_path; std::shared_ptr ticket_cache; std::unordered_map hdfs_conf; };