diff --git a/crates/atlaspack/src/atlaspack.rs b/crates/atlaspack/src/atlaspack.rs index c777e5cc0..7e91a4bc0 100644 --- a/crates/atlaspack/src/atlaspack.rs +++ b/crates/atlaspack/src/atlaspack.rs @@ -128,7 +128,7 @@ impl Atlaspack { let asset_graph = match request_result { RequestResult::AssetGraph(result) => { - self.commit_assets(result.graph.assets.as_slice())?; + self.commit_assets(result.graph.get_asset_nodes())?; result.graph } @@ -139,16 +139,20 @@ impl Atlaspack { }) } - fn commit_assets(&self, assets: &[AssetNode]) -> anyhow::Result<()> { + fn commit_assets(&self, assets: Vec<&AssetNode>) -> anyhow::Result<()> { let mut txn = self.db.environment().write_txn()?; - for AssetNode { asset, .. } in assets.iter() { - self.db.put(&mut txn, &asset.id, asset.code.bytes())?; - if let Some(map) = &asset.map { + for asset_node in assets { + self.db.put( + &mut txn, + &asset_node.asset.id, + asset_node.asset.code.bytes(), + )?; + if let Some(map) = &asset_node.asset.map { // TODO: For some reason to_buffer strips data when rkyv was upgraded, so now we use json self.db.put( &mut txn, - &format!("map:{}", asset.id), + &format!("map:{}", asset_node.asset.id), map.to_json()?.as_bytes(), )?; } @@ -186,21 +190,20 @@ mod tests { )?; let assets = vec!["foo", "bar", "baz"]; + let nodes = assets + .iter() + .enumerate() + .map(|(idx, asset)| AssetNode { + asset: Asset { + id: idx.to_string(), + code: Code::from(asset.to_string()), + ..Asset::default() + }, + requested_symbols: HashSet::new(), + }) + .collect::>(); - atlaspack.commit_assets( - &assets - .iter() - .enumerate() - .map(|(idx, asset)| AssetNode { - asset: Asset { - id: idx.to_string(), - code: Code::from(asset.to_string()), - ..Asset::default() - }, - requested_symbols: HashSet::new(), - }) - .collect::>(), - )?; + atlaspack.commit_assets(nodes.iter().collect())?; let txn = db.environment().read_txn()?; for (idx, asset) in assets.iter().enumerate() { diff --git a/crates/atlaspack/src/requests/asset_graph_request.rs b/crates/atlaspack/src/requests/asset_graph_request.rs index 7fc57f40a..6a4d92fae 100644 --- a/crates/atlaspack/src/requests/asset_graph_request.rs +++ b/crates/atlaspack/src/requests/asset_graph_request.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; +use atlaspack_core::asset_graph::propagate_requested_symbols; use indexmap::IndexMap; use pathdiff::diff_paths; use petgraph::graph::NodeIndex; @@ -45,14 +46,14 @@ impl Request for AssetGraphRequest { } struct AssetGraphBuilder { - request_id_to_dep_node_index: HashMap, graph: AssetGraph, visited: HashSet, work_count: u32, request_context: RunRequestContext, sender: ResultSender, receiver: ResultReceiver, - asset_request_to_asset: HashMap, + request_id_to_asset_nx: HashMap, + request_id_to_dependency_nx: HashMap, waiting_asset_requests: HashMap>, } @@ -61,14 +62,14 @@ impl AssetGraphBuilder { let (sender, receiver) = channel(); AssetGraphBuilder { - request_id_to_dep_node_index: HashMap::new(), + request_id_to_dependency_nx: HashMap::new(), graph: AssetGraph::new(), visited: HashSet::new(), work_count: 0, request_context, sender, receiver, - asset_request_to_asset: HashMap::new(), + request_id_to_asset_nx: HashMap::new(), waiting_asset_requests: HashMap::new(), } } @@ -137,16 +138,16 @@ impl AssetGraphBuilder { } fn handle_path_result(&mut self, result: PathRequestOutput, request_id: u64) { - let node = *self - .request_id_to_dep_node_index + let dependency_nx = *self + .request_id_to_dependency_nx .get(&request_id) .expect("Missing node index for request id {request_id}"); - let dep_index = self.graph.dependency_index(node).unwrap(); + let DependencyNode { dependency, requested_symbols, state, - } = &mut self.graph.dependencies[dep_index]; + } = &mut self.graph.get_dependency_node_mut(&dependency_nx).unwrap(); let asset_request = match result { PathRequestOutput::Resolved { @@ -185,16 +186,16 @@ impl AssetGraphBuilder { let id = asset_request.id(); if self.visited.insert(id) { - self.request_id_to_dep_node_index.insert(id, node); + self.request_id_to_dependency_nx.insert(id, dependency_nx); self.work_count += 1; let _ = self .request_context .queue_request(asset_request, self.sender.clone()); - } else if let Some(asset_node_index) = self.asset_request_to_asset.get(&id) { + } else if let Some(asset_node_index) = self.request_id_to_asset_nx.get(&id) { // We have already completed this AssetRequest so we can connect the // Dependency to the Asset immediately - self.graph.add_edge(&node, asset_node_index); - self.propagate_requested_symbols(*asset_node_index, node); + self.graph.add_edge(&dependency_nx, asset_node_index); + self.propagate_requested_symbols(*asset_node_index, dependency_nx); } else { // The AssetRequest has already been kicked off but is yet to // complete. Register this Dependency to be connected once it @@ -203,9 +204,9 @@ impl AssetGraphBuilder { .waiting_asset_requests .entry(id) .and_modify(|nodes| { - nodes.insert(node); + nodes.insert(dependency_nx); }) - .or_insert_with(|| HashSet::from([node])); + .or_insert_with(|| HashSet::from([dependency_nx])); } } @@ -232,27 +233,29 @@ impl AssetGraphBuilder { discovered_assets, dependencies, } = result; - let incoming_dep_node_index = *self - .request_id_to_dep_node_index + let incoming_dependency_nx = *self + .request_id_to_dependency_nx .get(&request_id) .expect("Missing node index for request id {request_id}"); // Connect the incoming DependencyNode to the new AssetNode - let asset_node_index = self.graph.add_asset(incoming_dep_node_index, asset.clone()); + let asset_nx = self.graph.add_asset(asset.clone()); + + self.graph.add_edge(&incoming_dependency_nx, &asset_nx); - self - .asset_request_to_asset - .insert(request_id, asset_node_index); + self.request_id_to_asset_nx.insert(request_id, asset_nx); - let root_asset = (&asset, asset_node_index); + let root_asset = (&asset, asset_nx); let mut added_discovered_assets: HashMap = HashMap::new(); // Attach the "direct" discovered assets to the graph let direct_discovered_assets = get_direct_discovered_assets(&discovered_assets, &dependencies); for discovered_asset in direct_discovered_assets { - let asset_node_index = self + let asset_node_index = self.graph.add_asset(discovered_asset.asset.clone()); + + self .graph - .add_asset(incoming_dep_node_index, discovered_asset.asset.clone()); + .add_edge(&incoming_dependency_nx, &asset_node_index); self.add_asset_dependencies( &discovered_asset.dependencies, @@ -261,25 +264,25 @@ impl AssetGraphBuilder { &mut added_discovered_assets, root_asset, ); - self.propagate_requested_symbols(asset_node_index, incoming_dep_node_index); + self.propagate_requested_symbols(asset_node_index, incoming_dependency_nx); } self.add_asset_dependencies( &dependencies, &discovered_assets, - asset_node_index, + asset_nx, &mut added_discovered_assets, root_asset, ); - self.propagate_requested_symbols(asset_node_index, incoming_dep_node_index); + self.propagate_requested_symbols(asset_nx, incoming_dependency_nx); // Connect any previously discovered Dependencies that were waiting // for this AssetNode to be created if let Some(waiting) = self.waiting_asset_requests.remove(&request_id) { for dep in waiting { - self.graph.add_edge(&dep, &asset_node_index); - self.propagate_requested_symbols(asset_node_index, dep); + self.graph.add_edge(&dep, &asset_nx); + self.propagate_requested_symbols(asset_nx, dep); } } } @@ -333,13 +336,14 @@ impl AssetGraphBuilder { .as_ref() .is_some_and(|key| key == &dependency.specifier); - let dep_node = self.graph.add_dependency(asset_node_index, dependency); + let dep_node = self.graph.add_dependency(dependency); + self.graph.add_edge(&asset_node_index, &dep_node); if dep_to_root_asset { self.graph.add_edge(&dep_node, &root_asset.1); } - // If the dependency points to a dicovered asset then add the asset using the new + // If the dependency points to a discovered asset then add the asset using the new // dep as it's parent if let Some(AssetWithDependencies { asset, @@ -356,7 +360,10 @@ impl AssetGraphBuilder { // This discovered_asset isn't yet in the graph so we'll need to add // it and assign it's dependencies by calling added_discovered_assets // recursively. - let asset_node_index = self.graph.add_asset(dep_node, asset.clone()); + let asset_node_index = self.graph.add_asset(asset.clone()); + + self.graph.add_edge(&dep_node, &asset_node_index); + added_discovered_assets.insert(asset.id.clone(), asset_node_index); self.add_asset_dependencies( @@ -377,20 +384,19 @@ impl AssetGraphBuilder { asset_node_index: NodeIndex, incoming_dep_node_index: NodeIndex, ) { - self.graph.propagate_requested_symbols( - asset_node_index, - incoming_dep_node_index, - &mut |dependency_node_index: NodeIndex, dependency: Arc| { - Self::on_undeferred( - &mut self.request_id_to_dep_node_index, - &mut self.work_count, - &mut self.request_context, - &self.sender, - dependency_node_index, - dependency, - ); - }, - ); + for (dependency_node_index, dependency) in + propagate_requested_symbols(&mut self.graph, asset_node_index, incoming_dep_node_index) + .unwrap() + { + Self::on_undeferred( + &mut self.request_id_to_dependency_nx, + &mut self.work_count, + &mut self.request_context, + &self.sender, + dependency_node_index, + dependency, + ); + } } fn handle_target_request_result(&mut self, result: TargetRequestOutput) { @@ -407,7 +413,7 @@ impl AssetGraphBuilder { dependency: Arc::new(dependency), }; self - .request_id_to_dep_node_index + .request_id_to_dependency_nx .insert(request.id(), dep_node); self.work_count += 1; let _ = self @@ -422,7 +428,7 @@ impl AssetGraphBuilder { /// Once they do have symbols in use, this callback will re-trigger resolution/transformation /// for those files. fn on_undeferred( - request_id_to_dep_node_index: &mut HashMap, + request_id_to_asset_nx: &mut HashMap, work_count: &mut u32, request_context: &mut RunRequestContext, sender: &ResultSender, @@ -433,7 +439,7 @@ impl AssetGraphBuilder { dependency: dependency.clone(), }; - request_id_to_dep_node_index.insert(request.id(), dependency_node_index); + request_id_to_asset_nx.insert(request.id(), dependency_node_index); tracing::debug!( "queueing a path request from on_undeferred, {}", dependency.specifier @@ -491,7 +497,7 @@ mod tests { use std::path::{Path, PathBuf}; use std::sync::Arc; - use atlaspack_core::types::{AtlaspackOptions, Code}; + use atlaspack_core::types::*; use atlaspack_filesystem::in_memory_file_system::InMemoryFileSystem; use atlaspack_filesystem::FileSystem; @@ -513,8 +519,14 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 0); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 0); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 0); + assert_eq!( + asset_graph_request_result + .graph + .get_dependency_nodes() + .len(), + 0 + ); } #[tokio::test(flavor = "multi_thread")] @@ -562,26 +574,28 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 1); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 1); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 1); assert_eq!( asset_graph_request_result .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 1 ); + + let Asset { + file_path, code, .. + }: Asset = asset_graph_request_result + .graph + .get_asset_nodes() + .get(0) + .unwrap() + .asset + .clone(); + + assert_eq!(file_path, temporary_dir.join("entry.js")); assert_eq!( - asset_graph_request_result - .graph - .assets - .get(0) - .unwrap() - .asset - .code, + code, (Code::from( String::from( r#" @@ -664,20 +678,25 @@ mod tests { }; // Entry, 2 assets + helpers file - assert_eq!(asset_graph_request_result.graph.assets.len(), 4); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 4); // Entry, entry to assets (2), assets to helpers (2) - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 5); - assert_eq!( asset_graph_request_result .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 5 ); + + let Asset { file_path, .. }: Asset = asset_graph_request_result + .graph + .get_asset_nodes() + .get(0) + .unwrap() + .asset + .clone(); + + assert_eq!(file_path, temporary_dir.join("entry.js")); } fn setup_core_modules(fs: &InMemoryFileSystem, core_path: &Path) { diff --git a/crates/atlaspack_core/src/asset_graph.rs b/crates/atlaspack_core/src/asset_graph.rs deleted file mode 100644 index 6af56d826..000000000 --- a/crates/atlaspack_core/src/asset_graph.rs +++ /dev/null @@ -1,730 +0,0 @@ -use std::{collections::HashSet, sync::Arc}; - -use petgraph::{ - graph::{DiGraph, NodeIndex}, - visit::EdgeRef, - Direction, -}; -use serde::Serialize; - -use crate::types::{Asset, Dependency}; - -#[derive(Clone, Debug)] -pub struct AssetGraph { - graph: DiGraph, - pub assets: Vec, - pub dependencies: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetNode { - pub asset: Asset, - pub requested_symbols: HashSet, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DependencyNode { - pub dependency: Arc, - pub requested_symbols: HashSet, - pub state: DependencyState, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum AssetGraphNode { - Root, - Entry, - Asset(usize), - Dependency(usize), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetGraphEdge {} - -#[derive(Clone, Debug, PartialEq)] -pub enum DependencyState { - New, - Deferred, - Excluded, - Resolved, -} - -impl PartialEq for AssetGraph { - fn eq(&self, other: &Self) -> bool { - let nodes = self.graph.raw_nodes().iter().map(|n| &n.weight); - let other_nodes = other.graph.raw_nodes().iter().map(|n| &n.weight); - - let edges = self - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - let other_edges = other - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - nodes.eq(other_nodes) - && edges.eq(other_edges) - && self.assets == other.assets - && self.dependencies == other.dependencies - } -} - -impl Default for AssetGraph { - fn default() -> Self { - Self::new() - } -} - -impl AssetGraph { - pub fn new() -> Self { - let mut graph = DiGraph::new(); - - graph.add_node(AssetGraphNode::Root); - - AssetGraph { - graph, - assets: Vec::new(), - dependencies: Vec::new(), - } - } - - pub fn edges(&self) -> Vec { - let raw_edges = self.graph.raw_edges(); - let mut edges = Vec::with_capacity(raw_edges.len() * 2); - - for edge in raw_edges { - edges.push(edge.source().index() as u32); - edges.push(edge.target().index() as u32); - } - - edges - } - - pub fn nodes(&self) -> impl Iterator { - let nodes = self.graph.node_weights(); - - nodes - } - - pub fn add_asset(&mut self, parent_idx: NodeIndex, asset: Asset) -> NodeIndex { - let idx = self.assets.len(); - - self.assets.push(AssetNode { - asset, - requested_symbols: HashSet::default(), - }); - - let asset_idx = self.graph.add_node(AssetGraphNode::Asset(idx)); - - self - .graph - .add_edge(parent_idx, asset_idx, AssetGraphEdge {}); - - asset_idx - } - - pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { - // The root node index will always be 0 - let root_node_index = NodeIndex::new(0); - - let is_library = dependency.env.is_library; - let node_index = self.add_dependency(root_node_index, dependency); - - if is_library { - if let Some(dependency_index) = &self.dependency_index(node_index) { - self.dependencies[*dependency_index] - .requested_symbols - .insert("*".into()); - } - } - - node_index - } - - pub fn add_dependency(&mut self, parent_idx: NodeIndex, dependency: Dependency) -> NodeIndex { - let idx = self.dependencies.len(); - - self.dependencies.push(DependencyNode { - dependency: Arc::new(dependency), - requested_symbols: HashSet::default(), - state: DependencyState::New, - }); - - let dependency_idx = self.graph.add_node(AssetGraphNode::Dependency(idx)); - - self - .graph - .add_edge(parent_idx, dependency_idx, AssetGraphEdge {}); - - dependency_idx - } - - pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { - self - .graph - .add_edge(*parent_idx, *child_idx, AssetGraphEdge {}); - } - - pub fn dependency_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Dependency(idx) => Some(*idx), - _ => None, - } - } - - pub fn asset_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Asset(idx) => Some(*idx), - _ => None, - } - } - - /// Propagates the requested symbols from an incoming dependency to an asset, - /// and forwards those symbols to re-exported dependencies if needed. - /// This may result in assets becoming un-deferred and transformed if they - /// now have requested symbols. - pub fn propagate_requested_symbols)>( - &mut self, - asset_node: NodeIndex, - incoming_dep_node: NodeIndex, - on_undeferred: &mut F, - ) { - let DependencyNode { - requested_symbols, .. - } = &self.dependencies[self.dependency_index(incoming_dep_node).unwrap()]; - - let asset_index = self.asset_index(asset_node).unwrap(); - let AssetNode { - asset, - requested_symbols: asset_requested_symbols, - } = &mut self.assets[asset_index]; - - let mut re_exports = HashSet::::default(); - let mut wildcards = HashSet::::default(); - let star = String::from("*"); - - if requested_symbols.contains(&star) { - // If the requested symbols includes the "*" namespace, we need to include all of the asset's - // exported symbols. - if let Some(symbols) = &asset.symbols { - for sym in symbols { - if asset_requested_symbols.insert(sym.exported.clone()) && sym.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(sym.local.clone()); - } - } - } - - // Propagate to all export * wildcard dependencies. - wildcards.insert(star); - } else { - // Otherwise, add each of the requested symbols to the asset. - for sym in requested_symbols.iter() { - if asset_requested_symbols.insert(sym.clone()) { - if let Some(asset_symbol) = asset - .symbols - .as_ref() - .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) - { - if asset_symbol.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(asset_symbol.local.clone()); - } - } else { - // If symbol wasn't found in the asset or a named re-export. - // This means the symbol is in one of the export * wildcards, but we don't know - // which one yet, so we propagate it to _all_ wildcard dependencies. - wildcards.insert(sym.clone()); - } - } - } - } - - let deps: Vec<_> = self - .graph - .neighbors_directed(asset_node, Direction::Outgoing) - .collect(); - for dep_node in deps { - let dep_index = self.dependency_index(dep_node).unwrap(); - let DependencyNode { - dependency, - requested_symbols, - state, - } = &mut self.dependencies[dep_index]; - - let mut updated = false; - if let Some(symbols) = &dependency.symbols { - for sym in symbols { - if sym.is_weak { - // This is a re-export. If it is a wildcard, add all unmatched symbols - // to this dependency, otherwise attempt to match a named re-export. - if sym.local == "*" { - for wildcard in &wildcards { - if requested_symbols.insert(wildcard.clone()) { - updated = true; - } - } - } else if re_exports.contains(&sym.local) - && requested_symbols.insert(sym.exported.clone()) - { - updated = true; - } - } else if requested_symbols.insert(sym.exported.clone()) { - // This is a normal import. Add the requested symbol. - updated = true; - } - } - } - - // If the dependency was updated, propagate to the target asset if there is one, - // or un-defer this dependency so we transform the requested asset. - // We must always resolve new dependencies to determine whether they have side effects. - if updated || *state == DependencyState::New { - if let Some(resolved) = self - .graph - .edges_directed(dep_node, Direction::Outgoing) - .next() - { - // Avoid infintite loops for self references - if resolved.target() != asset_node { - self.propagate_requested_symbols(resolved.target(), dep_node, on_undeferred); - } - } else { - on_undeferred(dep_node, Arc::clone(dependency)); - } - } - } - } - - pub fn serialize_nodes(&self, max_str_len: usize) -> serde_json::Result> { - let mut nodes: Vec = Vec::new(); - let mut curr_node = String::default(); - - for node in self.nodes() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(idx) => { - let asset = self.assets[*idx].asset.clone(); - - SerializedAssetGraphNode::Asset { value: asset } - } - AssetGraphNode::Dependency(idx) => { - let dependency = self.dependencies[*idx].dependency.clone(); - SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency.id(), - dependency: dependency.as_ref().clone(), - }, - has_deferred: self.dependencies[*idx].state == DependencyState::Deferred, - } - } - }; - - let str = serde_json::to_string(&serialized_node)?; - if curr_node.len() + str.len() < (max_str_len - 3) { - if !curr_node.is_empty() { - curr_node.push(','); - } - curr_node.push_str(&str); - } else { - // Add the existing node now as it has reached the max JavaScript string size - nodes.push(format!("[{curr_node}]")); - curr_node = str; - } - } - - // Add the current node if it did not overflow in size - if curr_node.len() < (max_str_len - 3) { - nodes.push(format!("[{curr_node}]")); - } - - Ok(nodes) - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - -impl std::hash::Hash for AssetGraph { - fn hash(&self, state: &mut H) { - for node in self.graph.node_weights() { - std::mem::discriminant(node).hash(state); - match node { - AssetGraphNode::Asset(idx) => self.assets[*idx].asset.id.hash(state), - AssetGraphNode::Dependency(idx) => self.dependencies[*idx].dependency.id().hash(state), - _ => {} - } - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use serde_json::{json, Value}; - - use crate::types::{Symbol, Target}; - - use super::*; - - type TestSymbol<'a> = (&'a str, &'a str, bool); - fn symbol(test_symbol: &TestSymbol) -> Symbol { - let (local, exported, is_weak) = test_symbol; - Symbol { - local: String::from(*local), - exported: String::from(*exported), - is_weak: is_weak.to_owned(), - ..Symbol::default() - } - } - - fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { - assert_eq!( - graph.dependencies[graph.dependency_index(node_index).unwrap()].requested_symbols, - expected - .into_iter() - .map(|s| s.into()) - .collect::>() - ); - } - - fn add_asset( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - file_path: &str, - ) -> NodeIndex { - let index_asset = Asset { - file_path: PathBuf::from(file_path), - symbols: Some(symbols.iter().map(symbol).collect()), - ..Asset::default() - }; - graph.add_asset(parent_node, index_asset) - } - - fn add_dependency( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - ) -> NodeIndex { - let dep = Dependency { - symbols: Some(symbols.iter().map(symbol).collect()), - ..Dependency::default() - }; - graph.add_dependency(parent_node, dep) - } - - #[test] - fn should_request_entry_asset() { - let mut requested = HashSet::new(); - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); - let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols( - index_asset_node, - entry_dep_node, - &mut |dependency_node_index, _dependency| { - requested.insert(dependency_node_index); - }, - ); - - assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); - assert_requested_symbols(&graph, dep_a_node, vec!["a"]); - } - - #[test] - fn should_propagate_named_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "a" from a.js and "b" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true), ("b", "b", true)], - "library.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be the only requested symbol - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec![]); - } - - #[test] - fn should_propagate_wildcard_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from a.js and "*" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be marked as requested on all deps as wildcards make it - // unclear who the owning dep is - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec!["a"]); - } - - #[test] - fn should_propagate_nested_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from library/index.js - let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let library_reexport_dep_node = - add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // library/index.js re-exports "a" from a.js - let library_asset_node = add_asset( - &mut graph, - library_reexport_dep_node, - vec![("a", "a", true)], - "library/index.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on all deps until the a dep is reached - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - } - - #[test] - fn should_propagate_renamed_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "b" from b.js renamed as "a" - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("b", "a", true)], - "library.js", - ); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "b" should be marked as requested on the b dep - assert_requested_symbols(&graph, b_dep, vec!["b"]); - } - - #[test] - fn should_propagate_namespace_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from stuff.js renamed as "a"" - // export * as a from './stuff.js' - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true)], - "library.js", - ); - let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "*" should be marked as requested on the stuff dep - assert_requested_symbols(&graph, stuff_dep, vec!["*"]); - } - - #[test] - fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { - let mut graph = AssetGraph::new(); - - let entry = graph.add_entry_dependency(Dependency { - specifier: String::from("entry"), - ..Dependency::default() - }); - - let entry_asset = graph.add_asset( - entry, - Asset { - file_path: PathBuf::from("entry"), - ..Asset::default() - }, - ); - - for i in 1..100 { - graph.add_dependency( - entry_asset, - Dependency { - specifier: format!("dependency-{}", i), - ..Dependency::default() - }, - ); - } - - let max_str_len = 10000; - let nodes = graph.serialize_nodes(max_str_len)?; - - assert_eq!(nodes.len(), 7); - - // Assert each string is less than the max size - for node in nodes.iter() { - assert!(node.len() < max_str_len); - } - - // Assert all the nodes are included and in the correct order - let first_entry = serde_json::from_str::(&nodes[0])?; - let first_entry = first_entry.as_array().unwrap(); - - assert_eq!(get_type(&first_entry[0]), json!("root")); - assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); - assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); - - for i in 1..first_entry.len() - 2 { - assert_eq!( - get_dependency(&first_entry[i + 2]), - Some(json!(format!("dependency-{}", i))) - ); - } - - let mut specifier = first_entry.len() - 2; - for node in nodes[1..].iter() { - let entry = serde_json::from_str::(&node)?; - let entry = entry.as_array().unwrap(); - - for value in entry { - assert_eq!( - get_dependency(&value), - Some(json!(format!("dependency-{}", specifier))) - ); - - specifier += 1; - } - } - - Ok(()) - } - - fn get_type(node: &Value) -> Value { - node.get("type").unwrap().to_owned() - } - - fn get_dependency(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("dependency")); - - value - .get("value") - .unwrap() - .get("dependency") - .unwrap() - .get("specifier") - .map(|s| s.to_owned()) - } - - fn get_asset(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("asset")); - - value - .get("value") - .unwrap() - .get("filePath") - .map(|s| s.to_owned()) - } -} diff --git a/crates/atlaspack_core/src/asset_graph/asset_graph.rs b/crates/atlaspack_core/src/asset_graph/asset_graph.rs new file mode 100644 index 000000000..d6ecb5249 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/asset_graph.rs @@ -0,0 +1,472 @@ +use std::{collections::HashSet, sync::Arc}; + +use petgraph::graph::NodeIndex; +use petgraph::stable_graph::StableDiGraph; +use petgraph::visit::EdgeRef; +use petgraph::visit::IntoEdgeReferences; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; + +#[derive(Clone, Debug, PartialEq)] +pub struct AssetNode { + pub asset: Asset, + pub requested_symbols: HashSet, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DependencyNode { + pub dependency: Arc, + pub requested_symbols: HashSet, + pub state: DependencyState, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum DependencyState { + New, + Deferred, + Excluded, + Resolved, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum AssetGraphNode { + Root, + Entry, + Asset(AssetNode), + Dependency(DependencyNode), +} + +impl AssetGraphNode { + pub fn id(&self) -> String { + match self { + AssetGraphNode::Root => "@@root".into(), + AssetGraphNode::Entry => "@@entry".into(), + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.clone(), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id(), + } + } +} + +#[derive(Clone, Debug)] +pub struct AssetGraph { + pub graph: StableDiGraph, + root_node_index: NodeIndex, +} + +impl Default for AssetGraph { + fn default() -> Self { + Self::new() + } +} + +impl AssetGraph { + pub fn new() -> Self { + let mut graph = StableDiGraph::new(); + let root_node_index = graph.add_node(AssetGraphNode::Root); + AssetGraph { + graph, + root_node_index, + } + } + + pub fn edges(&self) -> Vec<(usize, usize)> { + let raw_edges = self.graph.edge_references(); + let mut edges = Vec::<(usize, usize)>::new(); + + for edge in raw_edges { + edges.push((edge.source().index(), edge.target().index())); + } + + edges + } + + pub fn nodes(&self) -> impl Iterator { + self.graph.node_weights() + } + + pub fn nodes_from(&self, node_index: &NodeIndex) -> Vec<&AssetGraphNode> { + let mut result = vec![]; + + for edge in self.graph.edges_directed(*node_index, Direction::Outgoing) { + let target_nx = edge.target(); + let target = self.graph.node_weight(target_nx).unwrap(); + result.push(target); + } + + result + } + + pub fn root_node(&self) -> NodeIndex { + self.root_node_index + } + + pub fn add_asset(&mut self, asset: Asset) -> NodeIndex { + self.graph.add_node(AssetGraphNode::Asset(AssetNode { + asset, + requested_symbols: HashSet::default(), + })) + } + + pub fn get_asset_node(&self, nx: &NodeIndex) -> Option<&AssetNode> { + let value = self.graph.node_weight(*nx)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_node_mut(&mut self, nx: &NodeIndex) -> Option<&mut AssetNode> { + let value = self.graph.node_weight_mut(*nx)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_nodes(&self) -> Vec<&AssetNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Asset(asset) = n else { + continue; + }; + results.push(asset); + } + results + } + + pub fn add_dependency(&mut self, dependency: Dependency) -> NodeIndex { + self + .graph + .add_node(AssetGraphNode::Dependency(DependencyNode { + dependency: Arc::new(dependency), + requested_symbols: HashSet::default(), + state: DependencyState::New, + })) + } + + pub fn get_dependency_node(&self, nx: &NodeIndex) -> Option<&DependencyNode> { + let value = self.graph.node_weight(*nx)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn get_dependency_nodes(&self) -> Vec<&DependencyNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Dependency(dependency) = n else { + continue; + }; + results.push(dependency); + } + results + } + + pub fn get_dependency_node_mut(&mut self, nx: &NodeIndex) -> Option<&mut DependencyNode> { + let value = self.graph.node_weight_mut(*nx)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { + let is_library = dependency.env.is_library; + let dependency_nx = self.add_dependency(dependency); + self.add_edge(&self.root_node_index.clone(), &dependency_nx); + + if is_library { + if let Some(dependency_node) = self.get_dependency_node_mut(&dependency_nx) { + dependency_node.requested_symbols.insert("*".into()); + } + } + + dependency_nx + } + + pub fn add_edge(&mut self, from_nx: &NodeIndex, to_nx: &NodeIndex) { + self.graph.add_edge(*from_nx, *to_nx, ()); + } +} + +impl PartialEq for AssetGraph { + fn eq(&self, other: &Self) -> bool { + let nodes = self.nodes(); + let other_nodes = other.nodes(); + + let edges = self.edges(); + let other_edges = other.edges(); + + nodes.eq(other_nodes) && edges.eq(&other_edges) + } +} + +impl std::hash::Hash for AssetGraph { + fn hash(&self, state: &mut H) { + for node in self.graph.node_weights() { + std::mem::discriminant(node).hash(state); + match node { + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.hash(state), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id().hash(state), + _ => {} + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::types::Symbol; + use crate::types::Target; + + use super::super::propagate_requested_symbols::propagate_requested_symbols; + use super::*; + + type TestSymbol<'a> = (&'a str, &'a str, bool); + fn symbol(test_symbol: &TestSymbol) -> Symbol { + let (local, exported, is_weak) = test_symbol; + Symbol { + local: String::from(*local), + exported: String::from(*exported), + is_weak: is_weak.to_owned(), + ..Symbol::default() + } + } + + fn assert_requested_symbols(graph: &AssetGraph, nx: NodeIndex, expected: Vec<&str>) { + assert_eq!( + graph.get_dependency_node(&nx).unwrap().requested_symbols, + expected + .into_iter() + .map(|s| s.into()) + .collect::>() + ); + } + + fn add_asset( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + file_path: &str, + ) -> NodeIndex { + let index_asset = Asset { + file_path: PathBuf::from(file_path), + symbols: Some(symbols.iter().map(symbol).collect()), + ..Asset::default() + }; + let asset_nid = graph.add_asset(index_asset); + graph.add_edge(&parent_node, &asset_nid); + asset_nid + } + + fn add_dependency( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + ) -> NodeIndex { + let dep = Dependency { + symbols: Some(symbols.iter().map(symbol).collect()), + ..Dependency::default() + }; + let node_index = graph.add_dependency(dep); + graph.add_edge(&parent_node, &node_index); + node_index + } + + #[test] + fn should_request_entry_asset() { + let mut requested = HashSet::new(); + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); + let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, index_asset_node, entry_dep_node).unwrap() + { + requested.insert(dependency_node_index); + } + + assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); + assert_requested_symbols(&graph, dep_a_node, vec!["a"]); + } + + #[test] + fn should_propagate_named_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "a" from a.js and "b" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true), ("b", "b", true)], + "library.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + + let mut requested_deps = Vec::new(); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be the only requested symbol + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec![]); + } + + #[test] + fn should_propagate_wildcard_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from a.js and "*" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + + let mut requested_deps = Vec::new(); + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be marked as requested on all deps as wildcards make it + // unclear who the owning dep is + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec!["a"]); + } + + #[test] + fn should_propagate_nested_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from library/index.js + let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let library_reexport_dep_node = + add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // library/index.js re-exports "a" from a.js + let library_asset_node = add_asset( + &mut graph, + library_reexport_dep_node, + vec![("a", "a", true)], + "library/index.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on all deps until the a dep is reached + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + } + + #[test] + fn should_propagate_renamed_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "b" from b.js renamed as "a" + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("b", "a", true)], + "library.js", + ); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "b" should be marked as requested on the b dep + assert_requested_symbols(&graph, b_dep, vec!["b"]); + } + + #[test] + fn should_propagate_namespace_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from stuff.js renamed as "a"" + // export * as a from './stuff.js' + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true)], + "library.js", + ); + let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "*" should be marked as requested on the stuff dep + assert_requested_symbols(&graph, stuff_dep, vec!["*"]); + } +} diff --git a/crates/atlaspack_core/src/asset_graph/mod.rs b/crates/atlaspack_core/src/asset_graph/mod.rs new file mode 100644 index 000000000..222511364 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/mod.rs @@ -0,0 +1,8 @@ +#[allow(clippy::module_inception)] +mod asset_graph; +mod propagate_requested_symbols; +mod serialize_asset_graph; + +pub use self::asset_graph::*; +pub use self::propagate_requested_symbols::*; +pub use self::serialize_asset_graph::*; diff --git a/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs new file mode 100644 index 000000000..b5768f5f2 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs @@ -0,0 +1,162 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use petgraph::graph::NodeIndex; +use petgraph::visit::EdgeRef; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; +use crate::types::Symbol; + +use super::asset_graph::DependencyState; +use super::asset_graph::{AssetGraph, DependencyNode}; + +const CHAR_STAR: &str = "*"; + +/// Propagates the requested symbols from an incoming dependency to an asset, +/// and forwards those symbols to re-exported dependencies if needed. +/// This may result in assets becoming un-deferred and transformed if they +/// now have requested symbols. +pub fn propagate_requested_symbols( + asset_graph: &mut AssetGraph, + asset_index: NodeIndex, + dependency_index: NodeIndex, +) -> Option)>> { + let mut next = vec![(asset_index, dependency_index)]; + let mut on_undeferred = vec![]; + + while let Some((asset_nx, dependency_nx)) = next.pop() { + let mut dependency_re_exports = HashSet::::default(); + let mut dependency_wildcards = HashSet::::default(); + let mut asset_requested_symbols_buf = HashSet::::default(); + + let dependency_node = asset_graph.get_dependency_node(&dependency_nx).unwrap(); + let asset_node = asset_graph.get_asset_node(&asset_nx).unwrap(); + + if dependency_node.requested_symbols.contains(CHAR_STAR) { + // If the requested symbols includes the "*" namespace, we + // need to include all of the asset's exported symbols. + if let Some(symbols) = &asset_node.asset.symbols { + for sym in symbols { + if !asset_node.requested_symbols.contains(&sym.exported) { + continue; + } + asset_requested_symbols_buf.insert(sym.exported.clone()); + if !sym.is_weak { + continue; + } + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(sym.local.clone()); + } + } + + // Propagate to all export * wildcard dependencies. + dependency_wildcards.insert(CHAR_STAR.to_string()); + } else { + // Otherwise, add each of the requested symbols to the asset. + for sym in dependency_node.requested_symbols.iter() { + if asset_node.requested_symbols.contains(sym) { + continue; + } + asset_requested_symbols_buf.insert(sym.clone()); + + let Some(asset_symbol) = get_symbol_by_name(&asset_node.asset, sym) else { + // If symbol wasn't found in the asset or a named re-export. + // This means the symbol is in one of the export * wildcards, but we don't know + // which one yet, so we propagate it to _all_ wildcard dependencies. + dependency_wildcards.insert(sym.clone()); + continue; + }; + + if !asset_symbol.is_weak { + continue; + } + + // If the asset exports this symbol + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(asset_symbol.local.clone()); + } + } + + // Add dependencies to asset + asset_graph + .get_asset_node_mut(&asset_nx) + .unwrap() + .requested_symbols + .extend(asset_requested_symbols_buf); + + let deps: Vec<_> = asset_graph + .graph + .neighbors_directed(asset_nx, Direction::Outgoing) + .collect(); + + for dependency_nx in deps { + let mut updated = false; + + { + let DependencyNode { + dependency, + requested_symbols, + state: _, + } = asset_graph.get_dependency_node_mut(&dependency_nx).unwrap(); + + if let Some(symbols) = &dependency.symbols { + for sym in symbols { + if sym.is_weak { + // This is a re-export. If it is a wildcard, add all unmatched symbols + // to this dependency, otherwise attempt to match a named re-export. + if sym.local == "*" { + for wildcard in &dependency_wildcards { + if requested_symbols.insert(wildcard.clone()) { + updated = true; + } + } + } else if dependency_re_exports.contains(&sym.local) + && requested_symbols.insert(sym.exported.clone()) + { + updated = true; + } + } else if requested_symbols.insert(sym.exported.clone()) { + // This is a normal import. Add the requested symbol. + updated = true; + } + } + } + } + + let DependencyNode { + dependency, + requested_symbols: _, + state, + } = asset_graph.get_dependency_node(&dependency_nx).unwrap(); + + // If the dependency was updated, propagate to the target asset if there is one, + // or un-defer this dependency so we transform the requested asset. + // We must always resolve new dependencies to determine whether they have side effects. + if updated || *state == DependencyState::New { + let Some(resolved) = asset_graph + .graph + .edges_directed(dependency_nx, Direction::Outgoing) + .next() + else { + on_undeferred.push((dependency_nx, Arc::clone(dependency))); + continue; + }; + if resolved.target() == asset_nx { + continue; + } + next.push((resolved.target(), dependency_nx)) + } + } + } + + Some(on_undeferred) +} + +fn get_symbol_by_name<'a>(asset: &'a Asset, sym: &str) -> Option<&'a Symbol> { + asset + .symbols + .as_ref() + .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) +} diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs new file mode 100644 index 000000000..880aa280f --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs @@ -0,0 +1,237 @@ +use std::collections::HashMap; + +use petgraph::graph::NodeIndex; +use petgraph::visit::IntoNodeReferences; +use serde::Serialize; + +use crate::types::{Asset, Dependency}; + +use super::{AssetGraph, AssetGraphNode, DependencyState}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + pub id: String, + pub dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +} + +#[derive(Debug, Serialize)] +pub struct SerializedAssetGraphResult { + pub edges: Vec, + pub nodes: Vec, +} + +pub fn serialize_asset_graph_sorted( + asset_graph: &AssetGraph, +) -> anyhow::Result { + let mut edges = Vec::<(u32, u32)>::new(); + let mut nodes = Vec::::new(); + + // Track index reassignment after sorting + let mut nx_to_index = HashMap::::new(); + + let mut current_nodes = asset_graph + .graph + .node_references() + .collect::>(); + current_nodes.sort_by(|a, b| a.1.id().cmp(&b.1.id())); + + for (current_node_nx, current_node) in current_nodes { + nx_to_index.insert(current_node_nx, nodes.len() as u32); + nodes.push(match current_node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }) + } + + for (from, to) in asset_graph.edges() { + let Some(new_addr_from) = nx_to_index.get(&(from as u32).into()) else { + panic!(); + }; + + let Some(new_addr_to) = nx_to_index.get(&(to as u32).into()) else { + panic!(); + }; + + edges.push((new_addr_from.clone(), new_addr_to.clone())); + } + + edges.sort_by(|a, b| format!("{}{}", a.0, a.1).cmp(&format!("{}{}", b.0, b.1))); + + let edges = edges + .into_iter() + .flat_map(|v| vec![v.0, v.1]) + .collect::>(); + + Ok(SerializedAssetGraphResult { edges, nodes }) +} + +pub fn serialize_asset_graph( + asset_graph: &AssetGraph, + max_str_len: usize, +) -> serde_json::Result> { + let mut nodes: Vec = Vec::new(); + let mut curr_node = String::default(); + + for node in asset_graph.nodes() { + let serialized_node = match node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }; + + let str = serde_json::to_string(&serialized_node)?; + if curr_node.len() + str.len() < (max_str_len - 3) { + if !curr_node.is_empty() { + curr_node.push(','); + } + curr_node.push_str(&str); + } else { + // Add the existing node now as it has reached the max JavaScript string size + nodes.push(format!("[{curr_node}]")); + curr_node = str; + } + } + + // Add the current node if it did not overflow in size + if curr_node.len() < (max_str_len - 3) { + nodes.push(format!("[{curr_node}]")); + } + + Ok(nodes) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::{json, Value}; + + use super::*; + + #[test] + fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { + let mut graph = AssetGraph::new(); + + let entry = graph.add_entry_dependency(Dependency { + specifier: String::from("entry"), + ..Dependency::default() + }); + + let entry_asset = graph.add_asset(Asset { + file_path: PathBuf::from("entry"), + ..Asset::default() + }); + + graph.add_edge(&entry, &entry_asset); + + for i in 1..100 { + let node_index = graph.add_dependency(Dependency { + specifier: format!("dependency-{}", i), + ..Dependency::default() + }); + graph.add_edge(&entry_asset, &node_index); + } + + let max_str_len = 10000; + let nodes = serialize_asset_graph(&graph, max_str_len)?; + + assert_eq!(nodes.len(), 7); + + // Assert each string is less than the max size + for node in nodes.iter() { + assert!(node.len() < max_str_len); + } + + // Assert all the nodes are included and in the correct order + let first_entry = serde_json::from_str::(&nodes[0])?; + let first_entry = first_entry.as_array().unwrap(); + + assert_eq!(get_type(&first_entry[0]), json!("root")); + assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); + assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); + + for i in 1..first_entry.len() - 2 { + assert_eq!( + get_dependency(&first_entry[i + 2]), + Some(json!(format!("dependency-{}", i))) + ); + } + + let mut specifier = first_entry.len() - 2; + for node in nodes[1..].iter() { + let entry = serde_json::from_str::(&node)?; + let entry = entry.as_array().unwrap(); + + for value in entry { + assert_eq!( + get_dependency(&value), + Some(json!(format!("dependency-{}", specifier))) + ); + + specifier += 1; + } + } + + Ok(()) + } + + fn get_type(node: &Value) -> Value { + node.get("type").unwrap().to_owned() + } + + fn get_dependency(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("dependency")); + + value + .get("value") + .unwrap() + .get("dependency") + .unwrap() + .get("specifier") + .map(|s| s.to_owned()) + } + + fn get_asset(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("asset")); + + value + .get("value") + .unwrap() + .get("filePath") + .map(|s| s.to_owned()) + } +} diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 76ada51ab..35de320cc 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -6,6 +6,8 @@ use std::thread; use anyhow::anyhow; use atlaspack::AtlaspackError; +use atlaspack_core::asset_graph::serialize_asset_graph; +use atlaspack_core::asset_graph::serialize_asset_graph_sorted; use lmdb_js_lite::writer::DatabaseWriter; use lmdb_js_lite::LMDB; use napi::Env; @@ -147,13 +149,10 @@ impl AtlaspackNapi { // not supplied as JavaScript Error types. The JavaScript layer needs to handle conversions deferred.resolve(move |env| match result { Ok(asset_graph) => { - let mut js_object = env.create_object()?; + let result = serialize_asset_graph_sorted(&asset_graph)?; + let js_result = env.to_js_value(&result)?; - js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - js_object - .set_named_property("nodes", asset_graph.serialize_nodes(MAX_STRING_LENGTH)?)?; - - NapiAtlaspackResult::ok(&env, js_object) + NapiAtlaspackResult::ok(&env, js_result) } Err(error) => { let js_object = env.to_js_value(&AtlaspackError::from(&error))?; diff --git a/packages/core/core/src/requests/AssetGraphRequestRust.js b/packages/core/core/src/requests/AssetGraphRequestRust.js index f6dc42424..6cf448801 100644 --- a/packages/core/core/src/requests/AssetGraphRequestRust.js +++ b/packages/core/core/src/requests/AssetGraphRequestRust.js @@ -39,10 +39,6 @@ export function createAssetGraphRequestRust( let options = input.options; let serializedAssetGraph = await rustAtlaspack.buildAssetGraph(); - serializedAssetGraph.nodes = serializedAssetGraph.nodes.flatMap((node) => - JSON.parse(node), - ); - let {assetGraph, changedAssets} = instrument( 'atlaspack_v3_getAssetGraph', () => getAssetGraph(serializedAssetGraph),