From a19cbcf20274f54d4af522b0f531270a44a22fbc Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Wed, 12 Jun 2024 10:13:11 +0800 Subject: [PATCH] Customized HTTP useragent support: - users can set their own userAgent in overlaybd.json - the default value of UA is overlaybd/${version_no}-${commitID} Signed-off-by: Yifan Yuan --- CMake/Findphoton.cmake | 2 +- README.md | 2 + src/config.h | 6 +-- src/example_config/overlaybd.json | 3 +- src/image_service.cpp | 2 +- src/main.cpp | 1 + src/overlaybd/registryfs/registryfs.cpp | 5 ++- src/overlaybd/registryfs/registryfs.h | 6 ++- src/overlaybd/registryfs/registryfs_v2.cpp | 18 ++++++-- src/test/image_service_test.cpp | 50 ++++++++++++++++++++++ src/version.h | 5 +++ 11 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 src/version.h diff --git a/CMake/Findphoton.cmake b/CMake/Findphoton.cmake index 02ce5ccd..c905bd93 100644 --- a/CMake/Findphoton.cmake +++ b/CMake/Findphoton.cmake @@ -5,7 +5,7 @@ set(PHOTON_ENABLE_EXTFS ON) FetchContent_Declare( photon GIT_REPOSITORY https://github.com/alibaba/PhotonLibOS.git - GIT_TAG v0.6.16 + GIT_TAG v0.6.17 ) if(BUILD_TESTING) diff --git a/README.md b/README.md index 0b611bdc..daff6fee 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,8 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`. | prefetchConfig.concurrency | Prefetch concurrency for reloading trace, `16` is default | | certConfig.certFile | The path for SSL/TLS client certificate file | | certConfig.keyFile | The path for SSL/TLS client key file | +| userAgent | customized userAgent to identify HTTP request. default value is package version like 'overlaybd/1.1.14-6c449832' | + > NOTE: `download` is the config for background downloading. After an overlaybd device is lauched, a background task will be running to fetch the whole blobs into local directories. After downloading, I/O requests are directed to local files. Unlike other options, download config is reloaded when a device launching. diff --git a/src/config.h b/src/config.h index 9af34d90..33ce1d80 100644 --- a/src/config.h +++ b/src/config.h @@ -17,12 +17,9 @@ #include #include +#include "version.h" #include "overlaybd/config_util.h" -#define MACROTOSTR(x) #x -#define PRINTMACRO(x) MACROTOSTR(x) -static const char OVERLAYBD_VERSION[] = PRINTMACRO(OVERLAYBD_VER); - namespace ImageConfigNS { const int MAX_LAYER_CNT = 256; @@ -157,6 +154,7 @@ struct GlobalConfig : public ConfigUtils::Config { APPCFG_PARA(logConfig, LogConfig); APPCFG_PARA(prefetchConfig, PrefetchConfig); APPCFG_PARA(certConfig, CertConfig); + APPCFG_PARA(userAgent, std::string, OVERLAYBD_VERSION); }; struct AuthConfig : public ConfigUtils::Config { diff --git a/src/example_config/overlaybd.json b/src/example_config/overlaybd.json index 46791cdf..225bc879 100644 --- a/src/example_config/overlaybd.json +++ b/src/example_config/overlaybd.json @@ -35,5 +35,6 @@ "updateInterval": 60000000 }, "enableAudit": true, - "auditPath": "/var/log/overlaybd-audit.log" + "auditPath": "/var/log/overlaybd-audit.log", + "registryFsVersion": "v2" } diff --git a/src/image_service.cpp b/src/image_service.cpp index 69657532..a5e2cda5 100644 --- a/src/image_service.cpp +++ b/src/image_service.cpp @@ -349,7 +349,7 @@ int ImageService::init() { global_fs.underlay_registryfs = registryfs_creator( {this, &ImageService::reload_auth}, cafile, 30UL * 1000000, - global_conf.certConfig().certFile().c_str(), global_conf.certConfig().keyFile().c_str()); + global_conf.certConfig().certFile().c_str(), global_conf.certConfig().keyFile().c_str(), global_conf.userAgent().c_str()); if (global_fs.underlay_registryfs == nullptr) { LOG_ERROR_RETURN(0, -1, "create registryfs failed."); } diff --git a/src/main.cpp b/src/main.cpp index 652b1a71..0498e4d1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include "version.h" #include "image_file.h" #include "image_service.h" #include diff --git a/src/overlaybd/registryfs/registryfs.cpp b/src/overlaybd/registryfs/registryfs.cpp index 3b2a8fa1..27b52954 100644 --- a/src/overlaybd/registryfs/registryfs.cpp +++ b/src/overlaybd/registryfs/registryfs.cpp @@ -528,9 +528,12 @@ inline IFile *RegistryFSImpl::open(const char *pathname, int) { } IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile, uint64_t timeout, - const char *cert_file, const char *key_file) { + const char *cert_file, const char *key_file, const char *__) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); + if (__ != nullptr) { + LOG_WARN("customized UA is unsupported"); + } return new RegistryFSImpl(callback, caFile ? caFile : "", timeout, cert_file ? cert_file : "", key_file ? key_file : ""); } diff --git a/src/overlaybd/registryfs/registryfs.h b/src/overlaybd/registryfs/registryfs.h index 0bad0324..14a71e7c 100644 --- a/src/overlaybd/registryfs/registryfs.h +++ b/src/overlaybd/registryfs/registryfs.h @@ -32,13 +32,15 @@ photon::fs::IFileSystem *new_registryfs_v1(PasswordCB callback, const char *caFile = nullptr, uint64_t timeout = -1, const char *cert_file = nullptr, - const char *key_file = nullptr); + const char *key_file = nullptr, + const char *__ = nullptr); photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile = nullptr, uint64_t timeout = -1, const char *cert_file = nullptr, - const char *key_file = nullptr); + const char *key_file = nullptr, + const char *customized_ua = nullptr); photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile, std::string &upload_url, diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index d3e04616..cac45a9c 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -13,8 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include "../../version.h" #include "registryfs.h" + #include #include #include @@ -124,12 +126,17 @@ class RegistryFSImpl_v2 : public RegistryFS { } RegistryFSImpl_v2(PasswordCB callback, const char *caFile, uint64_t timeout, - photon::net::TLSContext *ctx) + photon::net::TLSContext *ctx, const char *ua) : m_callback(callback), m_caFile(caFile), m_timeout(timeout), m_tls_ctx(ctx), m_meta_size(kMinimalMetaLife), m_scope_token(kMinimalTokenLife), m_url_info(kMinimalAUrlLife) { m_client = nullptr; + if (ua == nullptr) { + m_useragent = OVERLAYBD_VERSION; + } else { + m_useragent = ua; + } this->refresh_client(); } @@ -286,6 +293,8 @@ class RegistryFSImpl_v2 : public RegistryFS { delete m_client; } m_client = new_http_client(nullptr, m_tls_ctx); + LOG_INFO("set user agent: `", m_useragent); + m_client->set_user_agent(m_useragent); } int refresh_token(const estring &url, estring &token) { @@ -309,6 +318,7 @@ class RegistryFSImpl_v2 : public RegistryFS { PasswordCB m_callback; estring m_accelerate; estring m_caFile; + estring m_useragent; uint64_t m_timeout; photon::net::TLSContext *m_tls_ctx; photon::net::http::Client *m_client; @@ -523,14 +533,14 @@ inline IFile *RegistryFSImpl_v2::open(const char *pathname, int) { } IFileSystem *new_registryfs_v2(PasswordCB callback, const char *caFile, uint64_t timeout, - const char *cert_file, const char *key_file) { + const char *cert_file, const char *key_file, const char *customized_ua) { if (!callback) LOG_ERROR_RETURN(EINVAL, nullptr, "password callback not set"); auto ctx = new_tls_context_from_file(cert_file, key_file); if (!ctx) { LOG_ERRNO_RETURN(0, nullptr, "failed to new tls context"); } - return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout, ctx); + return new RegistryFSImpl_v2(callback, caFile ? caFile : "", timeout, ctx, customized_ua); } class RegistryUploader : public VirtualFile { @@ -727,7 +737,7 @@ class RegistryUploader : public VirtualFile { photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_NONE); DEFER(photon::fini()); m_upload_fs = new RegistryFSImpl_v2({this, &RegistryUploader::load_auth}, - "", m_timeout, m_tls_ctx); + "", m_timeout, m_tls_ctx, nullptr); DEFER({ delete m_upload_fs; }); m_http_client_ts = photon::now; ::posix_memalign(&m_upload_buf, 4096, 1024 * 1024); diff --git a/src/test/image_service_test.cpp b/src/test/image_service_test.cpp index b2a32015..a5b81a17 100644 --- a/src/test/image_service_test.cpp +++ b/src/test/image_service_test.cpp @@ -24,11 +24,16 @@ #include "photon/net/socket.h" #include "photon/io/fd-events.h" #include "photon/net/curl.h" +#include "../version.h" +#include + #include #include "../image_service.cpp" +char *test_ua = nullptr; + photon::net::ISocketServer *new_server(std::string ip, uint16_t port) { auto server = photon::net::new_tcp_socket_server(); server->timeout(1000UL*1000); @@ -112,6 +117,51 @@ TEST(ImageTest, enableMetrics) { delete is; } + +int ua_check_handler(void*, photon::net::http::Request &req, photon::net::http::Response &resp, std::string_view) { + auto ua = req.headers["User-Agent"]; + LOG_DEBUG(VALUE(ua)); + EXPECT_EQ(ua, test_ua); + resp.set_result(200); + LOG_INFO("expected UA: `", test_ua); + std::string str = "success"; + resp.headers.content_length(7); + resp.write((void*)str.data(), str.size()); + return 0; +} + + +TEST(http_client, user_agent) { + auto tcpserver = photon::net::new_tcp_socket_server(); + DEFER(delete tcpserver); + tcpserver->bind(18731); + tcpserver->listen(); + auto server = photon::net::http::new_http_server(); + DEFER(delete server); + server->add_handler({nullptr, &ua_check_handler}); + tcpserver->set_handler(server->get_connection_handler()); + tcpserver->start_loop(); + + test_ua = "mytestUA"; + + std::string target_get = "http://localhost:18731/file"; + auto client = photon::net::http::new_http_client(); + client->set_user_agent(test_ua); + DEFER(delete client); + auto op = client->new_operation(photon::net::http::Verb::GET, target_get); + DEFER(delete op); + op->req.headers.content_length(0); + client->call(op); + EXPECT_EQ(op->status_code, 200); + std::string buf; + buf.resize(op->resp.headers.content_length()); + op->resp.read((void*)buf.data(), op->resp.headers.content_length()); + LOG_DEBUG(VALUE(buf)); + EXPECT_EQ(true, buf == "success"); +} + + + int main(int argc, char** argv) { photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT); DEFER(photon::fini();); diff --git a/src/version.h b/src/version.h new file mode 100644 index 00000000..ff24e280 --- /dev/null +++ b/src/version.h @@ -0,0 +1,5 @@ +#pragma once + +#define MACROTOSTR(x) #x +#define PRINTMACRO(x) MACROTOSTR(x) +static const char OVERLAYBD_VERSION[] = PRINTMACRO(OVERLAYBD_VER);