Merge pull request #66 from originalworks/add-tests-ow_data_provider_cli

Add tests for ow_data_provider_cli

.vscode/settings.json

@@ -3,5 +3,6 @@
         "/ow_data_provider_cli/Cargo.toml",
         "/ow_validator_node/Cargo.toml",
         "/ow_blob_codec/Cargo.toml"
-      ]
+      ],
+      "rust-analyzer.cargo.features": "all"
 }

docker/test-ow_data_provider_cli.yml

@@ -0,0 +1,15 @@
+services:
+  ipfs:
+    image: ipfs/kubo:latest
+    ports:
+      - "5001:5001"
+
+  test_runner:
+    depends_on:
+      - ipfs
+    volumes:
+      - ..:/protocol
+    image: rust:latest
+    working_dir: /protocol/ow_data_provider_cli
+    user: "${UID:-1000}:${GID:-1000}"
+    command: cargo test

ow_data_provider_cli/Cargo.toml

@@ -14,3 +14,6 @@ tokio-util = "0.7.12"
 serde = "1.0.210"
 infer = "0.16.0"
 quick-xml = "0.36.2"
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(rust_analyzer)'] }

ow_data_provider_cli/src/constants.rs

@@ -1,8 +1,14 @@
 use alloy::primitives::{address, Address};
 
 pub const DDEX_SEQUENCER_ADDRESS: Address = address!("B965D10739e19a9158e7f713720B0145D996E370");
+
+#[cfg(any(not(test), rust_analyzer))]
 pub const IPFS_API_BASE_URL: &str = "http://localhost:5001";
 pub const IPFS_API_ADD_FILE: &str = "/api/v0/add";
 pub const OUTPUT_FILES_DIR: &str = "./output_files";
 pub const IPFS_CIDS_ROOT_TAG: &str = "MessageHeader";
 pub const IMAGE_FILE_CID_TAG: &str = "ImageIpfsCid";
+
+#[cfg(all(test, not(rust_analyzer)))]
+pub const IPFS_API_BASE_URL: &str = "http://ipfs:5001";
+pub const IPFS_API_CAT_FILE: &str = "/api/v0/cat";

ow_data_provider_cli/src/errors.rs

@@ -8,7 +8,6 @@ pub enum OwDataProviderCliError {
     InvalidBlobProof(),
     SourcePathIsNotDir(String),
     EmptySourcePathFolder(String),
-    ErrorReadingFile(String),
 }
 
 impl fmt::Display for OwDataProviderCliError {
@@ -29,9 +28,6 @@ impl fmt::Display for OwDataProviderCliError {
             Self::EmptySourcePathFolder(path) => {
                 write!(f, "Folder under provided folder_path is empty: {}", path)
             }
-            Self::ErrorReadingFile(file_path) => {
-                write!(f, "Error while reading file from: {}", file_path)
-            }
         }
     }
 }

ow_data_provider_cli/src/ipfs.rs

@@ -24,22 +24,24 @@ use quick_xml::Writer;
 struct IpfsResponse {
     Hash: String,
 }
-
-struct AttachCidAndSaveInput {
-    image_file_cid: String,
-    xml_file_path: String,
-    output_file_name: String,
+#[derive(Debug)]
+pub struct AssetDirProcessingContext {
+    input_xml_path: String,
+    input_image_path: String,
+    image_cid: String,
+    output_xml_path: String,
+    empty: bool,
 }
 
-async fn file_to_multipart_form(file_path: String) -> Result<multipart::Form, Box<dyn Error>> {
+async fn file_to_multipart_form(file_path: &String) -> Result<multipart::Form, Box<dyn Error>> {
     let file = tokio::fs::File::open(file_path).await?;
     let file_stream = FramedRead::new(file, BytesCodec::new());
     let multipart_stream = multipart::Part::stream(Body::wrap_stream(file_stream));
     let multipart_form = multipart::Form::new().part("file", multipart_stream);
     Ok(multipart_form)
 }
 
-pub async fn pin_file(file_path: String) -> Result<String, Box<dyn Error>> {
+pub async fn pin_file(file_path: &String) -> Result<String, Box<dyn Error>> {
     let multipart_form = file_to_multipart_form(file_path).await?;
     let client = reqwest::Client::new();
 
@@ -54,15 +56,14 @@ pub async fn pin_file(file_path: String) -> Result<String, Box<dyn Error>> {
     Ok(result.Hash)
 }
 
-async fn attach_cid_and_save(input: AttachCidAndSaveInput) -> Result<(), Box<dyn Error>> {
+async fn attach_cid_and_save(input: &AssetDirProcessingContext) -> Result<(), Box<dyn Error>> {
     let mut buf = Vec::new();
 
-    let file = fs::File::open(&input.xml_file_path)?;
+    let file = fs::File::open(&input.input_xml_path)?;
     let reader = BufReader::new(file);
     let mut reader = Reader::from_reader(reader);
 
-    fs::create_dir_all(OUTPUT_FILES_DIR)?;
-    let output_file = fs::File::create(format!("{}/{}", OUTPUT_FILES_DIR, input.output_file_name))?;
+    let output_file = fs::File::create(&input.output_xml_path)?;
     let writer = BufWriter::new(output_file);
     let mut writer = Writer::new(writer);
 
@@ -74,7 +75,7 @@ async fn attach_cid_and_save(input: AttachCidAndSaveInput) -> Result<(), Box<dyn
                 if e == &BytesStart::new(IPFS_CIDS_ROOT_TAG) {
                     let tag = BytesStart::new(IMAGE_FILE_CID_TAG);
                     writer.write_event(Event::Start(tag.to_owned()))?;
-                    writer.write_event(Event::Text(BytesText::new(&input.image_file_cid)))?;
+                    writer.write_event(Event::Text(BytesText::new(&input.image_cid)))?;
                     writer.write_event(Event::End(tag.to_end()))?;
                 }
             }
@@ -97,51 +98,181 @@ async fn attach_cid_and_save(input: AttachCidAndSaveInput) -> Result<(), Box<dyn
 async fn process_asset_folder(
     asset_folder_path: PathBuf,
     folder_index: usize,
-) -> Result<(), Box<dyn Error>> {
+) -> Result<AssetDirProcessingContext, Box<dyn Error>> {
+    let mut file_processing_context = AssetDirProcessingContext {
+        input_xml_path: String::new(),
+        input_image_path: String::new(),
+        output_xml_path: String::new(),
+        image_cid: String::new(),
+        empty: true,
+    };
     if asset_folder_path.is_dir() {
         let asset_files = fs::read_dir(asset_folder_path)?;
-        let mut xml_file_path = String::new();
-        let mut image_file_cid = String::new();
+
         for asset_file in asset_files {
             let asset_path = asset_file?.path();
             if asset_path.is_dir() == false {
-                let kind = infer::get_from_path(&asset_path)?.ok_or_else(|| {
-                    Box::new(OwDataProviderCliError::ErrorReadingFile(
-                        asset_path.to_string_lossy().to_string(),
-                    ))
-                })?;
+                let kind = match infer::get_from_path(&asset_path)? {
+                    Some(v) => v,
+                    None => continue,
+                };
+
                 if kind.mime_type().starts_with("image/") {
-                    image_file_cid = pin_file(asset_path.to_string_lossy().to_string()).await?;
+                    file_processing_context.input_image_path =
+                        asset_path.to_string_lossy().to_string();
+                    file_processing_context.image_cid =
+                        pin_file(&file_processing_context.input_image_path).await?;
                 }
                 if kind.extension() == "xml" {
-                    xml_file_path = asset_path.to_string_lossy().to_string();
+                    file_processing_context.input_xml_path =
+                        asset_path.to_string_lossy().to_string();
                 }
             }
         }
-        attach_cid_and_save(AttachCidAndSaveInput {
-            image_file_cid,
-            xml_file_path,
-            output_file_name: format!("{}.xml", folder_index),
-        })
-        .await?;
+
+        if !file_processing_context.image_cid.is_empty()
+            && !file_processing_context.input_xml_path.is_empty()
+            && !file_processing_context.input_image_path.is_empty()
+        {
+            file_processing_context.output_xml_path =
+                format!("{}/{}.xml", OUTPUT_FILES_DIR, folder_index);
+            file_processing_context.empty = false;
+            attach_cid_and_save(&file_processing_context).await?;
+        }
     }
-    Ok(())
+    Ok(file_processing_context)
 }
 
-pub async fn create_output_files(folder_path: &String) -> Result<(), Box<dyn Error>> {
+pub async fn create_output_files(
+    folder_path: &String,
+) -> Result<Vec<AssetDirProcessingContext>, Box<dyn Error>> {
+    let mut result: Vec<AssetDirProcessingContext> = Vec::new();
+    let output_files_path = Path::new(OUTPUT_FILES_DIR);
+    if output_files_path.is_dir() {
+        fs::remove_dir_all(output_files_path)?;
+    }
+    fs::create_dir_all(output_files_path)?;
     let root_folder_dir = Path::new(&folder_path);
+    let mut empty_folder = true;
 
     if root_folder_dir.is_dir() {
         let asset_folders = fs::read_dir(root_folder_dir)?;
 
         for (index, asset_folder) in asset_folders.into_iter().enumerate() {
             let asset_folder_path = asset_folder?.path();
-            process_asset_folder(asset_folder_path, index).await?;
+            let asset_dir_processing_context =
+                process_asset_folder(asset_folder_path, index).await?;
+            if !asset_dir_processing_context.empty {
+                result.push(asset_dir_processing_context);
+                empty_folder = false;
+            }
         }
     } else {
         return Err(Box::new(OwDataProviderCliError::SourcePathIsNotDir(
             root_folder_dir.to_string_lossy().to_string(),
         )));
     }
-    Ok(())
+    if empty_folder {
+        return Err(Box::new(OwDataProviderCliError::EmptySourcePathFolder(
+            folder_path.to_string(),
+        )));
+    }
+    Ok(result)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::constants::IPFS_API_CAT_FILE;
+
+    fn read_file_tag(path: &Path, tag: &String) -> Result<String, Box<dyn Error>> {
+        let mut reader = Reader::from_file(&path)?;
+        let mut buffer = Vec::new();
+
+        let mut inside_id_tag = false;
+
+        let mut tag_value = String::new();
+
+        loop {
+            match reader.read_event_into(&mut buffer) {
+                Ok(Event::Eof) => break,
+                Ok(Event::Start(ref e)) => {
+                    if e == &BytesStart::new(tag) {
+                        inside_id_tag = true;
+                    }
+                }
+                Ok(Event::Text(ref e)) if inside_id_tag => {
+                    tag_value = String::from_utf8(e.to_vec())?;
+                    break;
+                }
+
+                Err(e) => return Err(Box::new(e)),
+                _ => {}
+            }
+            buffer.clear();
+        }
+        Ok(tag_value)
+    }
+
+    async fn fetch_ipfs_file(cid: &String) -> Result<tokio_util::bytes::Bytes, Box<dyn Error>> {
+        let client = reqwest::Client::new();
+        let response = client
+            .post(format!(
+                "{}{}?arg={}",
+                IPFS_API_BASE_URL, IPFS_API_CAT_FILE, cid
+            ))
+            .send()
+            .await?;
+
+        if response.status() != 200 {
+            panic!("Image CID not found {cid}");
+        }
+        let bytes = response.bytes().await?;
+
+        Ok(bytes)
+    }
+
+    #[tokio::test]
+    async fn pin_image_and_add_tag() -> Result<(), Box<dyn Error>> {
+        let test_folder = "./tests";
+        let processing_context_vec = create_output_files(&test_folder.to_string()).await?;
+
+        let processed_count = processing_context_vec.len();
+
+        assert_eq!(
+            processing_context_vec.len(),
+            2,
+            "Wrong output size. Expected 2, got: {processed_count}"
+        );
+
+        for processing_context in processing_context_vec {
+            let expected_image = fs::read(processing_context.input_image_path)?;
+
+            let fetched_image = fetch_ipfs_file(&processing_context.image_cid).await?;
+            assert_eq!(
+                fetched_image.to_vec(),
+                expected_image,
+                "Fetched image doesn't match"
+            );
+            let output_xml_path = Path::new(&processing_context.output_xml_path);
+            let cid_from_xml = read_file_tag(output_xml_path, &IMAGE_FILE_CID_TAG.to_string())?;
+            assert_eq!(
+                cid_from_xml, processing_context.image_cid,
+                "Missing ipfs cid tag in xml"
+            );
+        }
+
+        Ok(())
+    }
+
+    #[should_panic]
+    #[tokio::test]
+    async fn error_when_empty_directory() {
+        let test_folder = "./tests2";
+        tokio::fs::create_dir_all(test_folder).await.unwrap();
+
+        create_output_files(&test_folder.to_string()).await.unwrap();
+        tokio::fs::remove_dir_all(test_folder).await.unwrap();
+        ()
+    }
 }

ow_data_provider_cli/src/lib.rs

@@ -55,7 +55,6 @@ impl Config {
 }
 
 pub async fn run(config: Config) -> Result<(), Box<dyn Error>> {
-    // let ipfs_cid = ipfs::pin_file("./tests/test.xml".to_string()).await?;
     ipfs::create_output_files(&config.folder_path).await?;
     let private_key_signer: PrivateKeySigner = config
         .private_key