From d0a377777ad929618443843309988c395625f99d Mon Sep 17 00:00:00 2001 From: linghaihui <75124771@qq.com> Date: Tue, 1 Oct 2024 16:20:35 +0800 Subject: [PATCH] chore: fix some issues --- CHANGELOG.md | 10 +++++ Cargo.toml | 4 +- README.md | 12 +++--- examples/objects.rs | 33 +++++++++------ src/objects.rs | 100 ++++++++++++++++++++++++-------------------- 5 files changed, 94 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51cb10e..a78c02b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [0.1.13] - 2024-10-01 + +### Changed + +- 优化大文件上传，每个线程会尝试 10 次 + +### Fixed + +- 修复`put_object`方法报`不是内存对象`的报错 + ## [0.1.12] - 2024-08-23 ### Changed diff --git a/Cargo.toml b/Cargo.toml index 3bbcaac..c545b6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "qcos" -version = "0.1.12" +version = "0.1.13" edition = "2021" authors = ["bujnlc8 <75124771@qq.com>"] description = "Provide basic interface encapsulation of Tencent Cloud Object Storage (cos)" @@ -9,7 +9,7 @@ license = "MIT" keywords = ["cos", "tengxunyun", "tencentcloud", "qcos"] [dependencies] -reqwest = { version = "0.12", features = ["json","stream", "rustls-tls"] , default-features = false} +reqwest = { version = "0.12", features = ["json", "stream", "rustls-tls"] , default-features = false} serde = {version="1", features = ["derive"]} serde_json = "1.0" chrono = "0.4.38" diff --git a/README.md b/README.md index 465e586..352212c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ - 支持文件直传，推荐 1GB 以下的文件 -- 支持分块传输，设置分块大小和最大上传线程数量 +- 支持分块传输，设置分块大小和最大上传线程数量(基于 tokio::spawn) - 支持显示上传进度条(需开启`progress-bar` feature)，上传方法名称加了`_progress_bar`后缀与不显示进度条的方法区分 @@ -28,8 +28,8 @@ async fn main() { let client = Client::new( "Your secrect id", "Your secrect key", - "bucket name", - "region", + "Bucket name", + "Region", ); let mut acl_header = AclHeader::new(); acl_header.insert_object_x_cos_acl(ObjectAcl::PublicRead); @@ -38,7 +38,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("success"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 分块上传，带进度条 #[cfg(feature = "progress-bar")] @@ -71,12 +71,12 @@ async fn main() { ``` [dependencies] -qcos = "0.1.12" +qcos = "0.1" ``` 如果需要开启显示进度条的方法: ``` [dependencies] -qcos = {version = "0.1.12", features=["progress-bar"]} +qcos = {version = "0.1", features=["progress-bar"]} ``` diff --git a/examples/objects.rs b/examples/objects.rs index 2b35496..1bd611e 100644 --- a/examples/objects.rs +++ b/examples/objects.rs @@ -10,10 +10,10 @@ use qcos::request::ErrNo; #[tokio::main] async fn main() { let client = Client::new( - "your secrect id", - "your secrect key", - "bucket name", - "region", + "Your secrect id", + "Your secrect key", + "Bucket name", + "Region", ); // 普通上传，无权限控制 let file_path = PathBuf::from("Cargo.toml"); @@ -23,7 +23,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 私有权限控制 let mut acl = AclHeader::new(); @@ -39,7 +39,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 分块上传 let res = client @@ -57,7 +57,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 分块上传，带进度条 #[cfg(feature = "progress-bar")] @@ -78,7 +78,16 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); + } + // 上传文件 + let res = client + .put_object(&file_path, "Cargo.toml", Some(mime::TEXT_PLAIN_UTF_8), None) + .await; + if res.error_no == ErrNo::SUCCESS { + println!("SUCCESS"); + } else { + println!("[{}]: {}", res.error_no, res.error_message); } // 直接上传二进制数据 let res = client @@ -92,14 +101,14 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 删除文件 test/Cargo.toml let res = client.delete_object("test/Cargo.toml").await; if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 将对象存储对象名称为Cargo.toml的文件下载到本地，名称为local_Cargo.toml let res = client @@ -108,7 +117,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 将对象存储对象名称为Cargo.toml的文件下载到本地，名称为local_Cargo.toml.1，并显示下载进度条 @@ -120,7 +129,7 @@ async fn main() { if res.error_no == ErrNo::SUCCESS { println!("SUCCESS"); } else { - println!("{}", res.error_message); + println!("[{}]: {}", res.error_no, res.error_message); } // 获取预签名下载URL let url = client.get_presigned_download_url("Cargo.toml", 3600); diff --git a/src/objects.rs b/src/objects.rs index 83fa956..edf85ec 100644 --- a/src/objects.rs +++ b/src/objects.rs @@ -94,17 +94,17 @@ impl client::Client { content_type: Option, acl_header: Option, ) -> Response { - let file = match tokio::fs::File::open(file_path).await { + let buf = match tokio::fs::read(file_path).await { Ok(file) => file, Err(e) => { return Response::new( ErrNo::IO, - format!("打开文件失败: {:?}, {}", file_path, e), + format!("读取文件失败: {:?}, {}", file_path, e), Default::default(), ) } }; - self.put_object_binary(file, key, content_type, acl_header) + self.put_object_binary(buf, key, content_type, acl_header) .await } @@ -128,7 +128,6 @@ impl client::Client { /// acl_header.insert_object_x_cos_acl(ObjectAcl::AuthenticatedRead); /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.put_object_progress_bar(&PathBuf::from("Cargo.toml"), "Cargo.toml", Some(mime::TEXT_PLAIN_UTF_8), Some(acl_header), None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` #[cfg(feature = "progress-bar")] @@ -171,7 +170,7 @@ impl client::Client { /// - storage_class: 存储类型`StorageClassEnum` 默认STANDARD /// - acl_header: 请求控制 /// - part_size: 分片大小，单位bytes，要求1M-1G之间，默认50M - /// - max_threads: 最大上传线程数，默认20 + /// - max_threads: 最大上传线程数，默认20， 每个线程会尝试10次 /// - progress_style: 进度条样式 /// /// # Examples @@ -187,7 +186,6 @@ impl client::Client { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// // 分块传输 /// let res = client.put_big_object_progress_bar(&PathBuf::from("Cargo.toml"), "Cargo.toml", Some(mime::TEXT_PLAIN_UTF_8), Some(StorageClassEnum::STANDARD), Some(acl_header), Some(1024 * 1024 * 100), None, None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` #[cfg(feature = "progress-bar")] @@ -273,21 +271,30 @@ impl client::Client { let pb = multi.add(ProgressBar::new(body.len() as u64)); pb.set_style(sty.clone()); let handle = tokio::spawn(async move { - let resp = this - .clone() - .put_object_part_progress_bar( - &key, - &upload_id, - part_number, - body, - content_type, - acl_header, - pb, - ) - .await; - if resp.error_no != ErrNo::SUCCESS { - // 调用清理 - this.abort_object_part(&key, upload_id.as_str()).await; + let mut resp = Response::default(); + let mut try_times = 10; + while try_times > 0 { + try_times -= 1; + resp = this + .clone() + .put_object_part_progress_bar( + &key, + &upload_id, + part_number, + body.clone(), + content_type.clone(), + acl_header.clone(), + pb.clone(), + ) + .await; + if resp.error_no != ErrNo::SUCCESS { + if try_times == 0 { + // 调用清理 + this.abort_object_part(&key, upload_id.as_str()).await; + } + } else { + break; + } } resp }); @@ -335,7 +342,7 @@ impl client::Client { /// - storage_class: 存储类型`StorageClassEnum` 默认STANDARD /// - acl_header: 请求控制 /// - part_size: 分片大小，单位bytes，要求1M-1G之间，默认50M - /// - max_threads: 最大上传线程数，默认20 + /// - max_threads: 最大上传线程数，默认20， 每个线程会尝试10次 /// /// # Examples /// ``` @@ -350,7 +357,6 @@ impl client::Client { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// // 分块传输 /// let res = client.put_big_object(&PathBuf::from("Cargo.toml"),"Cargo.toml", Some(mime::TEXT_PLAIN_UTF_8), Some(StorageClassEnum::STANDARD), Some(acl_header), Some(1024 * 1024 * 100), None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` pub async fn put_big_object( @@ -428,21 +434,32 @@ impl client::Client { let acl_header = acl_header.clone(); let content_type = content_type.clone(); let handle = tokio::spawn(async move { - let resp = this - .clone() - .put_object_part( - &key, - &upload_id, - part_number, - body, - part_size1, - content_type, - acl_header, - ) - .await; - if resp.error_no != ErrNo::SUCCESS { - // 调用清理 - this.abort_object_part(&key, upload_id.as_str()).await; + // 尝试10次 + let mut try_times = 10; + let mut resp = Response::default(); + while try_times > 0 { + try_times -= 1; + resp = this + .clone() + .put_object_part( + &key, + &upload_id, + part_number, + body.clone(), + part_size1, + content_type.clone(), + acl_header.clone(), + ) + .await; + if resp.error_no != ErrNo::SUCCESS { + // 调用清理 + if try_times == 0 { + this.abort_object_part(&key, upload_id.as_str()).await; + break; + } + } else { + break; + } } resp }); @@ -494,7 +511,6 @@ impl client::Client { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let buffer = tokio::fs::File::open("Cargo.toml").await.unwrap(); /// let res = client.put_object_binary_progress_bar(buffer, "Cargo.toml", 100, Some(mime::TEXT_PLAIN_UTF_8), Some(acl_header), None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` #[cfg(feature = "progress-bar")] @@ -547,7 +563,6 @@ impl client::Client { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let buffer = std::fs::read("Cargo.toml").unwrap(); /// let res = client.put_object_binary(buffer, "Cargo.toml", Some(mime::TEXT_PLAIN_UTF_8), Some(acl_header)).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` pub async fn put_object_binary + Send>( @@ -600,7 +615,6 @@ impl client::Client { /// async { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.delete_object("Cargo.toml").await; - /// assert!(res.error_message.contains("403")) /// }; /// ``` pub async fn delete_object(&self, key: &str) -> Response { @@ -632,7 +646,6 @@ impl client::Client { /// async { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.get_object_binary_progress_bar("Cargo.toml", None, None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` #[cfg(feature = "progress-bar")] @@ -723,7 +736,6 @@ impl client::Client { /// async { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.get_object("Cargo.toml", "Cargo.toml", None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` pub async fn get_object(&self, key: &str, file_name: &str, threads: Option) -> Response { @@ -812,7 +824,6 @@ impl client::Client { /// async { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.get_object_progress_bar("Cargo.toml", "Cargo.toml", None, None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` #[cfg(feature = "progress-bar")] @@ -1172,7 +1183,6 @@ impl client::Client { /// async { /// let client = Client::new("foo", "bar", "qcloudtest-xxx", "ap-guangzhou"); /// let res = client.get_object_binary("Cargo.toml", None).await; - /// assert!(res.error_message.contains("403")); /// }; /// ``` pub async fn get_object_binary(&self, key: &str, threads: Option) -> Response {