Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: skip empty chunks when decoding gz #456

Merged
merged 6 commits into from
Oct 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 38 additions & 31 deletions components/chainhook-cli/src/archive/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,48 +41,55 @@ pub async fn download_tsv_file(config: &Config) -> Result<(), String> {

// Download chunks
let (tx, rx) = flume::bounded(0);
destination_path.push(default_tsv_file_path(&config.network.stacks_network));

let decoder_thread = std::thread::spawn(move || {
let input = ChannelRead::new(rx);
let mut decoder = GzDecoder::new(input);
let mut file = fs::File::create(&destination_path).unwrap();
let mut buffer = [0; 512_000];
loop {
match decoder.read(&mut buffer) {
Ok(0) => break,
Ok(n) => {
if let Err(e) = file.write_all(&buffer[..n]) {
let err = format!("unable to update compressed archive: {}", e.to_string());

if res.status() == reqwest::StatusCode::OK {
destination_path.push(default_tsv_file_path(&config.network.stacks_network));

let decoder_thread = std::thread::spawn(move || {
let mut file = fs::File::create(&destination_path).unwrap();
let input = ChannelRead::new(rx);
let mut decoder = GzDecoder::new(input);
let mut buffer = [0; 512_000];
loop {
match decoder.read(&mut buffer) {
Ok(0) => break,
Ok(n) => {
if let Err(e) = file.write_all(&buffer[..n]) {
let err =
format!("unable to update compressed archive: {}", e.to_string());
return Err(err);
}
}
Err(e) => {
let err = format!("unable to write compressed archive: {}", e.to_string());
return Err(err);
}
}
Err(e) => {
let err = format!("unable to write compressed archive: {}", e.to_string());
return Err(err);
}
}
}
let _ = file.flush();
Ok(())
});

if res.status() == reqwest::StatusCode::OK {
let _ = file.flush();
Ok(())
});
let mut stream = res.bytes_stream();
while let Some(item) = stream.next().await {
let chunk = item.or(Err(format!("Error while downloading file")))?;
let chunk = match item {
Ok(i) => Ok(i),
Err(e) => Err(format!("Error while downloading file {}", e.to_string())),
}?;
if chunk.is_empty() {
continue;
}
tx.send_async(chunk.to_vec())
.await
.map_err(|e| format!("unable to download stacks event: {}", e.to_string()))?;
.map_err(|e| format!("unable to download stacks archive: {}", e.to_string()))?;
}
drop(tx);
}

tokio::task::spawn_blocking(|| decoder_thread.join())
.await
.unwrap()
.unwrap()
.unwrap();
tokio::task::spawn_blocking(|| decoder_thread.join())
.await
.unwrap()
.unwrap()
.unwrap();
}

Ok(())
}
Expand Down
35 changes: 35 additions & 0 deletions docs/chainhook-openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,18 @@
}
},
"additionalProperties": false
},
{
"type": "object",
"required": [
"descriptor"
],
"properties": {
"descriptor": {
"$ref": "#/components/schemas/DescriptorMatchingRule"
}
},
"additionalProperties": false
}
],
"required": [
Expand Down Expand Up @@ -726,6 +738,29 @@
}
]
},
"DescriptorMatchingRule": {
"type": "object",
"required": [
"expression"
],
"properties": {
"expression": {
"type": "string"
},
"range": {
"default": null,
"type": "array",
"items": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"maxItems": 2,
"minItems": 2,
"nullable": true
}
}
},
"HookAction": {
"oneOf": [
{
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.