From dc71b691c28d2a00d33c73205aa8da7356ca37a0 Mon Sep 17 00:00:00 2001 From: David Alsh Date: Thu, 2 Jan 2025 10:39:56 +0000 Subject: [PATCH 1/2] removed indirection from asset graph --- crates/atlaspack/src/atlaspack.rs | 28 +- .../src/requests/asset_graph_request.rs | 147 ++-- crates/atlaspack_core/src/asset_graph.rs | 730 ------------------ .../src/asset_graph/asset_graph.rs | 465 +++++++++++ crates/atlaspack_core/src/asset_graph/mod.rs | 8 + .../propagate_requested_symbols.rs | 162 ++++ .../src/asset_graph/serialize_asset_graph.rs | 172 +++++ .../node-bindings/src/atlaspack/atlaspack.rs | 8 +- 8 files changed, 928 insertions(+), 792 deletions(-) delete mode 100644 crates/atlaspack_core/src/asset_graph.rs create mode 100644 crates/atlaspack_core/src/asset_graph/asset_graph.rs create mode 100644 crates/atlaspack_core/src/asset_graph/mod.rs create mode 100644 crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs create mode 100644 crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs diff --git a/crates/atlaspack/src/atlaspack.rs b/crates/atlaspack/src/atlaspack.rs index c777e5cc0..4f6dc7bbe 100644 --- a/crates/atlaspack/src/atlaspack.rs +++ b/crates/atlaspack/src/atlaspack.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::Arc; use atlaspack_config::atlaspack_rc_config_loader::{AtlaspackRcConfigLoader, LoadConfigOptions}; -use atlaspack_core::asset_graph::{AssetGraph, AssetNode}; +use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, AssetNode}; use atlaspack_core::config_loader::ConfigLoader; use atlaspack_core::plugin::{PluginContext, PluginLogger, PluginOptions}; use atlaspack_core::types::AtlaspackOptions; @@ -128,7 +128,7 @@ impl Atlaspack { let asset_graph = match request_result { RequestResult::AssetGraph(result) => { - self.commit_assets(result.graph.assets.as_slice())?; + self.commit_assets(result.graph.nodes().collect())?; result.graph } @@ -139,10 +139,13 @@ impl Atlaspack { }) } - fn commit_assets(&self, assets: &[AssetNode]) -> anyhow::Result<()> { + fn commit_assets(&self, assets: Vec<&AssetGraphNode>) -> anyhow::Result<()> { let mut txn = self.db.environment().write_txn()?; - for AssetNode { asset, .. } in assets.iter() { + for asset_node in assets.iter() { + let AssetGraphNode::Asset(AssetNode { asset, .. }) = asset_node else { + continue; + }; self.db.put(&mut txn, &asset.id, asset.code.bytes())?; if let Some(map) = &asset.map { // TODO: For some reason to_buffer strips data when rkyv was upgraded, so now we use json @@ -186,12 +189,11 @@ mod tests { )?; let assets = vec!["foo", "bar", "baz"]; - - atlaspack.commit_assets( - &assets - .iter() - .enumerate() - .map(|(idx, asset)| AssetNode { + let nodes = assets + .iter() + .enumerate() + .map(|(idx, asset)| { + AssetGraphNode::Asset(AssetNode { asset: Asset { id: idx.to_string(), code: Code::from(asset.to_string()), @@ -199,8 +201,10 @@ mod tests { }, requested_symbols: HashSet::new(), }) - .collect::>(), - )?; + }) + .collect::>(); + + atlaspack.commit_assets(nodes.iter().collect())?; let txn = db.environment().read_txn()?; for (idx, asset) in assets.iter().enumerate() { diff --git a/crates/atlaspack/src/requests/asset_graph_request.rs b/crates/atlaspack/src/requests/asset_graph_request.rs index 7fc57f40a..0f737bd5d 100644 --- a/crates/atlaspack/src/requests/asset_graph_request.rs +++ b/crates/atlaspack/src/requests/asset_graph_request.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; +use atlaspack_core::asset_graph::propagate_requested_symbols; use indexmap::IndexMap; use pathdiff::diff_paths; use petgraph::graph::NodeIndex; @@ -141,12 +142,12 @@ impl AssetGraphBuilder { .request_id_to_dep_node_index .get(&request_id) .expect("Missing node index for request id {request_id}"); - let dep_index = self.graph.dependency_index(node).unwrap(); + let DependencyNode { dependency, requested_symbols, state, - } = &mut self.graph.dependencies[dep_index]; + } = &mut self.graph.get_dependency_node_mut(node).unwrap(); let asset_request = match result { PathRequestOutput::Resolved { @@ -238,7 +239,11 @@ impl AssetGraphBuilder { .expect("Missing node index for request id {request_id}"); // Connect the incoming DependencyNode to the new AssetNode - let asset_node_index = self.graph.add_asset(incoming_dep_node_index, asset.clone()); + let asset_node_index = self.graph.add_asset(asset.clone()); + + self + .graph + .add_edge(&incoming_dep_node_index, &asset_node_index); self .asset_request_to_asset @@ -250,9 +255,11 @@ impl AssetGraphBuilder { // Attach the "direct" discovered assets to the graph let direct_discovered_assets = get_direct_discovered_assets(&discovered_assets, &dependencies); for discovered_asset in direct_discovered_assets { - let asset_node_index = self + let asset_node_index = self.graph.add_asset(discovered_asset.asset.clone()); + + self .graph - .add_asset(incoming_dep_node_index, discovered_asset.asset.clone()); + .add_edge(&incoming_dep_node_index, &asset_node_index); self.add_asset_dependencies( &discovered_asset.dependencies, @@ -333,13 +340,14 @@ impl AssetGraphBuilder { .as_ref() .is_some_and(|key| key == &dependency.specifier); - let dep_node = self.graph.add_dependency(asset_node_index, dependency); + let dep_node = self.graph.add_dependency(dependency); + self.graph.add_edge(&asset_node_index, &dep_node); if dep_to_root_asset { self.graph.add_edge(&dep_node, &root_asset.1); } - // If the dependency points to a dicovered asset then add the asset using the new + // If the dependency points to a discovered asset then add the asset using the new // dep as it's parent if let Some(AssetWithDependencies { asset, @@ -356,7 +364,10 @@ impl AssetGraphBuilder { // This discovered_asset isn't yet in the graph so we'll need to add // it and assign it's dependencies by calling added_discovered_assets // recursively. - let asset_node_index = self.graph.add_asset(dep_node, asset.clone()); + let asset_node_index = self.graph.add_asset(asset.clone()); + + self.graph.add_edge(&dep_node, &asset_node_index); + added_discovered_assets.insert(asset.id.clone(), asset_node_index); self.add_asset_dependencies( @@ -377,20 +388,19 @@ impl AssetGraphBuilder { asset_node_index: NodeIndex, incoming_dep_node_index: NodeIndex, ) { - self.graph.propagate_requested_symbols( - asset_node_index, - incoming_dep_node_index, - &mut |dependency_node_index: NodeIndex, dependency: Arc| { - Self::on_undeferred( - &mut self.request_id_to_dep_node_index, - &mut self.work_count, - &mut self.request_context, - &self.sender, - dependency_node_index, - dependency, - ); - }, - ); + for (dependency_node_index, dependency) in + propagate_requested_symbols(&mut self.graph, asset_node_index, incoming_dep_node_index) + .unwrap() + { + Self::on_undeferred( + &mut self.request_id_to_dep_node_index, + &mut self.work_count, + &mut self.request_context, + &self.sender, + dependency_node_index, + dependency, + ); + } } fn handle_target_request_result(&mut self, result: TargetRequestOutput) { @@ -491,6 +501,7 @@ mod tests { use std::path::{Path, PathBuf}; use std::sync::Arc; + use atlaspack_core::asset_graph::{AssetGraphNode, AssetNode}; use atlaspack_core::types::{AtlaspackOptions, Code}; use atlaspack_filesystem::in_memory_file_system::InMemoryFileSystem; use atlaspack_filesystem::FileSystem; @@ -513,8 +524,14 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 0); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 0); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 0); + assert_eq!( + asset_graph_request_result + .graph + .get_dependency_nodes() + .len(), + 0 + ); } #[tokio::test(flavor = "multi_thread")] @@ -562,26 +579,48 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 1); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 1); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 1); assert_eq!( asset_graph_request_result .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 1 ); + + let file_path = asset_graph_request_result + .graph + .nodes_from(&asset_graph_request_result.graph.root_node()) + .iter() + .filter_map(|n| match *n { + AssetGraphNode::Asset(asset_node) => Some(asset_node), + _ => None, + }) + .collect::>() + .get(0) + .unwrap() + .asset + .file_path + .clone(); + + let code = asset_graph_request_result + .graph + .nodes_from(&asset_graph_request_result.graph.root_node()) + .iter() + .filter_map(|n| match *n { + AssetGraphNode::Asset(asset_node) => Some(asset_node), + _ => None, + }) + .collect::>() + .get(0) + .unwrap() + .asset + .code + .clone(); + + assert_eq!(file_path, temporary_dir.join("entry.js")); assert_eq!( - asset_graph_request_result - .graph - .assets - .get(0) - .unwrap() - .asset - .code, + code, (Code::from( String::from( r#" @@ -664,20 +703,32 @@ mod tests { }; // Entry, 2 assets + helpers file - assert_eq!(asset_graph_request_result.graph.assets.len(), 4); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 4); // Entry, entry to assets (2), assets to helpers (2) - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 5); - assert_eq!( asset_graph_request_result .graph - .assets - .get(0) - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 5 ); + + let file_path = asset_graph_request_result + .graph + .nodes_from(&asset_graph_request_result.graph.root_node()) + .iter() + .filter_map(|n| match *n { + AssetGraphNode::Asset(asset_node) => Some(asset_node), + _ => None, + }) + .collect::>() + .get(0) + .unwrap() + .asset + .file_path + .clone(); + + assert_eq!(file_path, temporary_dir.join("entry.js")); } fn setup_core_modules(fs: &InMemoryFileSystem, core_path: &Path) { diff --git a/crates/atlaspack_core/src/asset_graph.rs b/crates/atlaspack_core/src/asset_graph.rs deleted file mode 100644 index 6af56d826..000000000 --- a/crates/atlaspack_core/src/asset_graph.rs +++ /dev/null @@ -1,730 +0,0 @@ -use std::{collections::HashSet, sync::Arc}; - -use petgraph::{ - graph::{DiGraph, NodeIndex}, - visit::EdgeRef, - Direction, -}; -use serde::Serialize; - -use crate::types::{Asset, Dependency}; - -#[derive(Clone, Debug)] -pub struct AssetGraph { - graph: DiGraph, - pub assets: Vec, - pub dependencies: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetNode { - pub asset: Asset, - pub requested_symbols: HashSet, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DependencyNode { - pub dependency: Arc, - pub requested_symbols: HashSet, - pub state: DependencyState, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum AssetGraphNode { - Root, - Entry, - Asset(usize), - Dependency(usize), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetGraphEdge {} - -#[derive(Clone, Debug, PartialEq)] -pub enum DependencyState { - New, - Deferred, - Excluded, - Resolved, -} - -impl PartialEq for AssetGraph { - fn eq(&self, other: &Self) -> bool { - let nodes = self.graph.raw_nodes().iter().map(|n| &n.weight); - let other_nodes = other.graph.raw_nodes().iter().map(|n| &n.weight); - - let edges = self - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - let other_edges = other - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - nodes.eq(other_nodes) - && edges.eq(other_edges) - && self.assets == other.assets - && self.dependencies == other.dependencies - } -} - -impl Default for AssetGraph { - fn default() -> Self { - Self::new() - } -} - -impl AssetGraph { - pub fn new() -> Self { - let mut graph = DiGraph::new(); - - graph.add_node(AssetGraphNode::Root); - - AssetGraph { - graph, - assets: Vec::new(), - dependencies: Vec::new(), - } - } - - pub fn edges(&self) -> Vec { - let raw_edges = self.graph.raw_edges(); - let mut edges = Vec::with_capacity(raw_edges.len() * 2); - - for edge in raw_edges { - edges.push(edge.source().index() as u32); - edges.push(edge.target().index() as u32); - } - - edges - } - - pub fn nodes(&self) -> impl Iterator { - let nodes = self.graph.node_weights(); - - nodes - } - - pub fn add_asset(&mut self, parent_idx: NodeIndex, asset: Asset) -> NodeIndex { - let idx = self.assets.len(); - - self.assets.push(AssetNode { - asset, - requested_symbols: HashSet::default(), - }); - - let asset_idx = self.graph.add_node(AssetGraphNode::Asset(idx)); - - self - .graph - .add_edge(parent_idx, asset_idx, AssetGraphEdge {}); - - asset_idx - } - - pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { - // The root node index will always be 0 - let root_node_index = NodeIndex::new(0); - - let is_library = dependency.env.is_library; - let node_index = self.add_dependency(root_node_index, dependency); - - if is_library { - if let Some(dependency_index) = &self.dependency_index(node_index) { - self.dependencies[*dependency_index] - .requested_symbols - .insert("*".into()); - } - } - - node_index - } - - pub fn add_dependency(&mut self, parent_idx: NodeIndex, dependency: Dependency) -> NodeIndex { - let idx = self.dependencies.len(); - - self.dependencies.push(DependencyNode { - dependency: Arc::new(dependency), - requested_symbols: HashSet::default(), - state: DependencyState::New, - }); - - let dependency_idx = self.graph.add_node(AssetGraphNode::Dependency(idx)); - - self - .graph - .add_edge(parent_idx, dependency_idx, AssetGraphEdge {}); - - dependency_idx - } - - pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { - self - .graph - .add_edge(*parent_idx, *child_idx, AssetGraphEdge {}); - } - - pub fn dependency_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Dependency(idx) => Some(*idx), - _ => None, - } - } - - pub fn asset_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Asset(idx) => Some(*idx), - _ => None, - } - } - - /// Propagates the requested symbols from an incoming dependency to an asset, - /// and forwards those symbols to re-exported dependencies if needed. - /// This may result in assets becoming un-deferred and transformed if they - /// now have requested symbols. - pub fn propagate_requested_symbols)>( - &mut self, - asset_node: NodeIndex, - incoming_dep_node: NodeIndex, - on_undeferred: &mut F, - ) { - let DependencyNode { - requested_symbols, .. - } = &self.dependencies[self.dependency_index(incoming_dep_node).unwrap()]; - - let asset_index = self.asset_index(asset_node).unwrap(); - let AssetNode { - asset, - requested_symbols: asset_requested_symbols, - } = &mut self.assets[asset_index]; - - let mut re_exports = HashSet::::default(); - let mut wildcards = HashSet::::default(); - let star = String::from("*"); - - if requested_symbols.contains(&star) { - // If the requested symbols includes the "*" namespace, we need to include all of the asset's - // exported symbols. - if let Some(symbols) = &asset.symbols { - for sym in symbols { - if asset_requested_symbols.insert(sym.exported.clone()) && sym.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(sym.local.clone()); - } - } - } - - // Propagate to all export * wildcard dependencies. - wildcards.insert(star); - } else { - // Otherwise, add each of the requested symbols to the asset. - for sym in requested_symbols.iter() { - if asset_requested_symbols.insert(sym.clone()) { - if let Some(asset_symbol) = asset - .symbols - .as_ref() - .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) - { - if asset_symbol.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(asset_symbol.local.clone()); - } - } else { - // If symbol wasn't found in the asset or a named re-export. - // This means the symbol is in one of the export * wildcards, but we don't know - // which one yet, so we propagate it to _all_ wildcard dependencies. - wildcards.insert(sym.clone()); - } - } - } - } - - let deps: Vec<_> = self - .graph - .neighbors_directed(asset_node, Direction::Outgoing) - .collect(); - for dep_node in deps { - let dep_index = self.dependency_index(dep_node).unwrap(); - let DependencyNode { - dependency, - requested_symbols, - state, - } = &mut self.dependencies[dep_index]; - - let mut updated = false; - if let Some(symbols) = &dependency.symbols { - for sym in symbols { - if sym.is_weak { - // This is a re-export. If it is a wildcard, add all unmatched symbols - // to this dependency, otherwise attempt to match a named re-export. - if sym.local == "*" { - for wildcard in &wildcards { - if requested_symbols.insert(wildcard.clone()) { - updated = true; - } - } - } else if re_exports.contains(&sym.local) - && requested_symbols.insert(sym.exported.clone()) - { - updated = true; - } - } else if requested_symbols.insert(sym.exported.clone()) { - // This is a normal import. Add the requested symbol. - updated = true; - } - } - } - - // If the dependency was updated, propagate to the target asset if there is one, - // or un-defer this dependency so we transform the requested asset. - // We must always resolve new dependencies to determine whether they have side effects. - if updated || *state == DependencyState::New { - if let Some(resolved) = self - .graph - .edges_directed(dep_node, Direction::Outgoing) - .next() - { - // Avoid infintite loops for self references - if resolved.target() != asset_node { - self.propagate_requested_symbols(resolved.target(), dep_node, on_undeferred); - } - } else { - on_undeferred(dep_node, Arc::clone(dependency)); - } - } - } - } - - pub fn serialize_nodes(&self, max_str_len: usize) -> serde_json::Result> { - let mut nodes: Vec = Vec::new(); - let mut curr_node = String::default(); - - for node in self.nodes() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(idx) => { - let asset = self.assets[*idx].asset.clone(); - - SerializedAssetGraphNode::Asset { value: asset } - } - AssetGraphNode::Dependency(idx) => { - let dependency = self.dependencies[*idx].dependency.clone(); - SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency.id(), - dependency: dependency.as_ref().clone(), - }, - has_deferred: self.dependencies[*idx].state == DependencyState::Deferred, - } - } - }; - - let str = serde_json::to_string(&serialized_node)?; - if curr_node.len() + str.len() < (max_str_len - 3) { - if !curr_node.is_empty() { - curr_node.push(','); - } - curr_node.push_str(&str); - } else { - // Add the existing node now as it has reached the max JavaScript string size - nodes.push(format!("[{curr_node}]")); - curr_node = str; - } - } - - // Add the current node if it did not overflow in size - if curr_node.len() < (max_str_len - 3) { - nodes.push(format!("[{curr_node}]")); - } - - Ok(nodes) - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - -impl std::hash::Hash for AssetGraph { - fn hash(&self, state: &mut H) { - for node in self.graph.node_weights() { - std::mem::discriminant(node).hash(state); - match node { - AssetGraphNode::Asset(idx) => self.assets[*idx].asset.id.hash(state), - AssetGraphNode::Dependency(idx) => self.dependencies[*idx].dependency.id().hash(state), - _ => {} - } - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use serde_json::{json, Value}; - - use crate::types::{Symbol, Target}; - - use super::*; - - type TestSymbol<'a> = (&'a str, &'a str, bool); - fn symbol(test_symbol: &TestSymbol) -> Symbol { - let (local, exported, is_weak) = test_symbol; - Symbol { - local: String::from(*local), - exported: String::from(*exported), - is_weak: is_weak.to_owned(), - ..Symbol::default() - } - } - - fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { - assert_eq!( - graph.dependencies[graph.dependency_index(node_index).unwrap()].requested_symbols, - expected - .into_iter() - .map(|s| s.into()) - .collect::>() - ); - } - - fn add_asset( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - file_path: &str, - ) -> NodeIndex { - let index_asset = Asset { - file_path: PathBuf::from(file_path), - symbols: Some(symbols.iter().map(symbol).collect()), - ..Asset::default() - }; - graph.add_asset(parent_node, index_asset) - } - - fn add_dependency( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - ) -> NodeIndex { - let dep = Dependency { - symbols: Some(symbols.iter().map(symbol).collect()), - ..Dependency::default() - }; - graph.add_dependency(parent_node, dep) - } - - #[test] - fn should_request_entry_asset() { - let mut requested = HashSet::new(); - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); - let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols( - index_asset_node, - entry_dep_node, - &mut |dependency_node_index, _dependency| { - requested.insert(dependency_node_index); - }, - ); - - assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); - assert_requested_symbols(&graph, dep_a_node, vec!["a"]); - } - - #[test] - fn should_propagate_named_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "a" from a.js and "b" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true), ("b", "b", true)], - "library.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be the only requested symbol - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec![]); - } - - #[test] - fn should_propagate_wildcard_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from a.js and "*" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be marked as requested on all deps as wildcards make it - // unclear who the owning dep is - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec!["a"]); - } - - #[test] - fn should_propagate_nested_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from library/index.js - let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let library_reexport_dep_node = - add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // library/index.js re-exports "a" from a.js - let library_asset_node = add_asset( - &mut graph, - library_reexport_dep_node, - vec![("a", "a", true)], - "library/index.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on all deps until the a dep is reached - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - } - - #[test] - fn should_propagate_renamed_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "b" from b.js renamed as "a" - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("b", "a", true)], - "library.js", - ); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "b" should be marked as requested on the b dep - assert_requested_symbols(&graph, b_dep, vec!["b"]); - } - - #[test] - fn should_propagate_namespace_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from stuff.js renamed as "a"" - // export * as a from './stuff.js' - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true)], - "library.js", - ); - let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "*" should be marked as requested on the stuff dep - assert_requested_symbols(&graph, stuff_dep, vec!["*"]); - } - - #[test] - fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { - let mut graph = AssetGraph::new(); - - let entry = graph.add_entry_dependency(Dependency { - specifier: String::from("entry"), - ..Dependency::default() - }); - - let entry_asset = graph.add_asset( - entry, - Asset { - file_path: PathBuf::from("entry"), - ..Asset::default() - }, - ); - - for i in 1..100 { - graph.add_dependency( - entry_asset, - Dependency { - specifier: format!("dependency-{}", i), - ..Dependency::default() - }, - ); - } - - let max_str_len = 10000; - let nodes = graph.serialize_nodes(max_str_len)?; - - assert_eq!(nodes.len(), 7); - - // Assert each string is less than the max size - for node in nodes.iter() { - assert!(node.len() < max_str_len); - } - - // Assert all the nodes are included and in the correct order - let first_entry = serde_json::from_str::(&nodes[0])?; - let first_entry = first_entry.as_array().unwrap(); - - assert_eq!(get_type(&first_entry[0]), json!("root")); - assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); - assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); - - for i in 1..first_entry.len() - 2 { - assert_eq!( - get_dependency(&first_entry[i + 2]), - Some(json!(format!("dependency-{}", i))) - ); - } - - let mut specifier = first_entry.len() - 2; - for node in nodes[1..].iter() { - let entry = serde_json::from_str::(&node)?; - let entry = entry.as_array().unwrap(); - - for value in entry { - assert_eq!( - get_dependency(&value), - Some(json!(format!("dependency-{}", specifier))) - ); - - specifier += 1; - } - } - - Ok(()) - } - - fn get_type(node: &Value) -> Value { - node.get("type").unwrap().to_owned() - } - - fn get_dependency(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("dependency")); - - value - .get("value") - .unwrap() - .get("dependency") - .unwrap() - .get("specifier") - .map(|s| s.to_owned()) - } - - fn get_asset(value: &Value) -> Option { - assert_eq!(get_type(&value), json!("asset")); - - value - .get("value") - .unwrap() - .get("filePath") - .map(|s| s.to_owned()) - } -} diff --git a/crates/atlaspack_core/src/asset_graph/asset_graph.rs b/crates/atlaspack_core/src/asset_graph/asset_graph.rs new file mode 100644 index 000000000..bbd94969c --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/asset_graph.rs @@ -0,0 +1,465 @@ +use std::{collections::HashSet, sync::Arc}; + +use petgraph::graph::NodeIndex; +use petgraph::stable_graph::StableDiGraph; +use petgraph::visit::EdgeRef; +use petgraph::visit::IntoEdgeReferences; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; + +#[derive(Clone, Debug, PartialEq)] +pub struct AssetNode { + pub asset: Asset, + pub requested_symbols: HashSet, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DependencyNode { + pub dependency: Arc, + pub requested_symbols: HashSet, + pub state: DependencyState, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum DependencyState { + New, + Deferred, + Excluded, + Resolved, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum AssetGraphNode { + Root, + Entry, + Asset(AssetNode), + Dependency(DependencyNode), +} + +#[derive(Clone, Debug)] +pub struct AssetGraph { + pub graph: StableDiGraph, + root_node_index: NodeIndex, +} + +impl Default for AssetGraph { + fn default() -> Self { + Self::new() + } +} + +impl AssetGraph { + pub fn new() -> Self { + let mut graph = StableDiGraph::new(); + let root_node_index = graph.add_node(AssetGraphNode::Root); + AssetGraph { + graph, + root_node_index, + } + } + + pub fn edges(&self) -> Vec { + let raw_edges = self.graph.edge_references(); + let mut edges = Vec::new(); + + for edge in raw_edges { + edges.push(edge.source().index() as u32); + edges.push(edge.target().index() as u32); + } + + edges + } + + pub fn nodes(&self) -> impl Iterator { + self.graph.node_weights() + } + + pub fn nodes_from(&self, node_index: &NodeIndex) -> Vec<&AssetGraphNode> { + let mut result = vec![]; + + for edge in self.graph.edges_directed(*node_index, Direction::Outgoing) { + let target_nx = edge.target(); + let target = self.graph.node_weight(target_nx).unwrap(); + result.push(target); + } + + result + } + + pub fn root_node(&self) -> NodeIndex { + self.root_node_index + } + + pub fn add_asset(&mut self, asset: Asset) -> NodeIndex { + self.graph.add_node(AssetGraphNode::Asset(AssetNode { + asset, + requested_symbols: HashSet::default(), + })) + } + + pub fn get_asset_node(&self, id: NodeIndex) -> Option<&AssetNode> { + let value = self.graph.node_weight(id)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_node_mut(&mut self, id: NodeIndex) -> Option<&mut AssetNode> { + let value = self.graph.node_weight_mut(id)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_nodes(&self) -> Vec<&AssetNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Asset(asset) = n else { + continue; + }; + results.push(asset); + } + results + } + + pub fn add_dependency(&mut self, dependency: Dependency) -> NodeIndex { + self + .graph + .add_node(AssetGraphNode::Dependency(DependencyNode { + dependency: Arc::new(dependency), + requested_symbols: HashSet::default(), + state: DependencyState::New, + })) + } + + pub fn get_dependency_node(&self, id: NodeIndex) -> Option<&DependencyNode> { + let value = self.graph.node_weight(id)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn get_dependency_nodes(&self) -> Vec<&DependencyNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Dependency(dependency) = n else { + continue; + }; + results.push(dependency); + } + results + } + + pub fn get_dependency_node_mut(&mut self, id: NodeIndex) -> Option<&mut DependencyNode> { + let value = self.graph.node_weight_mut(id)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { + let is_library = dependency.env.is_library; + let node_index = self.add_dependency(dependency); + self.add_edge(&self.root_node_index.clone(), &node_index); + + if is_library { + if let Some(dependency_node) = self.get_dependency_node_mut(node_index) { + dependency_node.requested_symbols.insert("*".into()); + } + } + + node_index + } + + pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { + self.graph.add_edge(*parent_idx, *child_idx, ()); + } +} + +impl PartialEq for AssetGraph { + fn eq(&self, other: &Self) -> bool { + let nodes = self.nodes(); + let other_nodes = other.nodes(); + + let edges = self.edges(); + let other_edges = other.edges(); + + nodes.eq(other_nodes) && edges.eq(&other_edges) + } +} + +impl std::hash::Hash for AssetGraph { + fn hash(&self, state: &mut H) { + for node in self.graph.node_weights() { + std::mem::discriminant(node).hash(state); + match node { + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.hash(state), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id().hash(state), + _ => {} + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::types::Symbol; + use crate::types::Target; + + use super::super::propagate_requested_symbols::propagate_requested_symbols; + use super::*; + + type TestSymbol<'a> = (&'a str, &'a str, bool); + fn symbol(test_symbol: &TestSymbol) -> Symbol { + let (local, exported, is_weak) = test_symbol; + Symbol { + local: String::from(*local), + exported: String::from(*exported), + is_weak: is_weak.to_owned(), + ..Symbol::default() + } + } + + fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { + assert_eq!( + graph + .get_dependency_node(node_index) + .unwrap() + .requested_symbols, + expected + .into_iter() + .map(|s| s.into()) + .collect::>() + ); + } + + fn add_asset( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + file_path: &str, + ) -> NodeIndex { + let index_asset = Asset { + file_path: PathBuf::from(file_path), + symbols: Some(symbols.iter().map(symbol).collect()), + ..Asset::default() + }; + let asset_nid = graph.add_asset(index_asset); + graph.add_edge(&parent_node, &asset_nid); + asset_nid + } + + fn add_dependency( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + ) -> NodeIndex { + let dep = Dependency { + symbols: Some(symbols.iter().map(symbol).collect()), + ..Dependency::default() + }; + let node_index = graph.add_dependency(dep); + graph.add_edge(&parent_node, &node_index); + node_index + } + + #[test] + fn should_request_entry_asset() { + let mut requested = HashSet::new(); + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); + let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, index_asset_node, entry_dep_node).unwrap() + { + requested.insert(dependency_node_index); + } + + assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); + assert_requested_symbols(&graph, dep_a_node, vec!["a"]); + } + + #[test] + fn should_propagate_named_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "a" from a.js and "b" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true), ("b", "b", true)], + "library.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + + let mut requested_deps = Vec::new(); + + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be the only requested symbol + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec![]); + } + + #[test] + fn should_propagate_wildcard_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from a.js and "*" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + + let mut requested_deps = Vec::new(); + for (dependency_node_index, _) in + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap() + { + requested_deps.push(dependency_node_index); + } + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be marked as requested on all deps as wildcards make it + // unclear who the owning dep is + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec!["a"]); + } + + #[test] + fn should_propagate_nested_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from library/index.js + let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let library_reexport_dep_node = + add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // library/index.js re-exports "a" from a.js + let library_asset_node = add_asset( + &mut graph, + library_reexport_dep_node, + vec![("a", "a", true)], + "library/index.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + propagate_requested_symbols(&mut graph, library_entry_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on all deps until the a dep is reached + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + } + + #[test] + fn should_propagate_renamed_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "b" from b.js renamed as "a" + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("b", "a", true)], + "library.js", + ); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "b" should be marked as requested on the b dep + assert_requested_symbols(&graph, b_dep, vec!["b"]); + } + + #[test] + fn should_propagate_namespace_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node).unwrap(); + + // library.js re-exports "*" from stuff.js renamed as "a"" + // export * as a from './stuff.js' + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true)], + "library.js", + ); + let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); + propagate_requested_symbols(&mut graph, library_asset_node, library_dep_node).unwrap(); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "*" should be marked as requested on the stuff dep + assert_requested_symbols(&graph, stuff_dep, vec!["*"]); + } +} diff --git a/crates/atlaspack_core/src/asset_graph/mod.rs b/crates/atlaspack_core/src/asset_graph/mod.rs new file mode 100644 index 000000000..222511364 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/mod.rs @@ -0,0 +1,8 @@ +#[allow(clippy::module_inception)] +mod asset_graph; +mod propagate_requested_symbols; +mod serialize_asset_graph; + +pub use self::asset_graph::*; +pub use self::propagate_requested_symbols::*; +pub use self::serialize_asset_graph::*; diff --git a/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs new file mode 100644 index 000000000..a287ad0f9 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs @@ -0,0 +1,162 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use petgraph::graph::NodeIndex; +use petgraph::visit::EdgeRef; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; +use crate::types::Symbol; + +use super::asset_graph::DependencyState; +use super::asset_graph::{AssetGraph, DependencyNode}; + +const CHAR_STAR: &str = "*"; + +/// Propagates the requested symbols from an incoming dependency to an asset, +/// and forwards those symbols to re-exported dependencies if needed. +/// This may result in assets becoming un-deferred and transformed if they +/// now have requested symbols. +pub fn propagate_requested_symbols( + asset_graph: &mut AssetGraph, + asset_index: NodeIndex, + dependency_index: NodeIndex, +) -> Option)>> { + let mut next = vec![(asset_index, dependency_index)]; + let mut on_undeferred = vec![]; + + while let Some((asset_index, dependency_index)) = next.pop() { + let mut dependency_re_exports = HashSet::::default(); + let mut dependency_wildcards = HashSet::::default(); + let mut asset_requested_symbols_buf = HashSet::::default(); + + let dependency_node = asset_graph.get_dependency_node(dependency_index).unwrap(); + let asset_node = asset_graph.get_asset_node(asset_index).unwrap(); + + if dependency_node.requested_symbols.contains(CHAR_STAR) { + // If the requested symbols includes the "*" namespace, we + // need to include all of the asset's exported symbols. + if let Some(symbols) = &asset_node.asset.symbols { + for sym in symbols { + if !asset_node.requested_symbols.contains(&sym.exported) { + continue; + } + asset_requested_symbols_buf.insert(sym.exported.clone()); + if !sym.is_weak { + continue; + } + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(sym.local.clone()); + } + } + + // Propagate to all export * wildcard dependencies. + dependency_wildcards.insert(CHAR_STAR.to_string()); + } else { + // Otherwise, add each of the requested symbols to the asset. + for sym in dependency_node.requested_symbols.iter() { + if asset_node.requested_symbols.contains(sym) { + continue; + } + asset_requested_symbols_buf.insert(sym.clone()); + + let Some(asset_symbol) = get_symbol_by_name(&asset_node.asset, sym) else { + // If symbol wasn't found in the asset or a named re-export. + // This means the symbol is in one of the export * wildcards, but we don't know + // which one yet, so we propagate it to _all_ wildcard dependencies. + dependency_wildcards.insert(sym.clone()); + continue; + }; + + if !asset_symbol.is_weak { + continue; + } + + // If the asset exports this symbol + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(asset_symbol.local.clone()); + } + } + + // Add dependencies to asset + asset_graph + .get_asset_node_mut(asset_index) + .unwrap() + .requested_symbols + .extend(asset_requested_symbols_buf); + + let deps: Vec<_> = asset_graph + .graph + .neighbors_directed(asset_index, Direction::Outgoing) + .collect(); + + for dep_node in deps { + let mut updated = false; + + { + let DependencyNode { + dependency, + requested_symbols, + state: _, + } = asset_graph.get_dependency_node_mut(dep_node).unwrap(); + + if let Some(symbols) = &dependency.symbols { + for sym in symbols { + if sym.is_weak { + // This is a re-export. If it is a wildcard, add all unmatched symbols + // to this dependency, otherwise attempt to match a named re-export. + if sym.local == "*" { + for wildcard in &dependency_wildcards { + if requested_symbols.insert(wildcard.clone()) { + updated = true; + } + } + } else if dependency_re_exports.contains(&sym.local) + && requested_symbols.insert(sym.exported.clone()) + { + updated = true; + } + } else if requested_symbols.insert(sym.exported.clone()) { + // This is a normal import. Add the requested symbol. + updated = true; + } + } + } + } + + let DependencyNode { + dependency, + requested_symbols: _, + state, + } = asset_graph.get_dependency_node(dep_node).unwrap(); + + // If the dependency was updated, propagate to the target asset if there is one, + // or un-defer this dependency so we transform the requested asset. + // We must always resolve new dependencies to determine whether they have side effects. + if updated || *state == DependencyState::New { + let Some(resolved) = asset_graph + .graph + .edges_directed(dep_node, Direction::Outgoing) + .next() + else { + on_undeferred.push((dep_node, Arc::clone(dependency))); + continue; + }; + if resolved.target() == asset_index { + continue; + } + next.push((resolved.target(), dep_node)) + } + } + } + + Some(on_undeferred) +} + +fn get_symbol_by_name<'a>(asset: &'a Asset, sym: &str) -> Option<&'a Symbol> { + asset + .symbols + .as_ref() + .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) +} diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs new file mode 100644 index 000000000..94fd0aae6 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs @@ -0,0 +1,172 @@ +use serde::Serialize; + +use crate::types::{Asset, Dependency}; + +use super::{AssetGraph, AssetGraphNode, DependencyState}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + id: String, + dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +} + +pub fn serialize_asset_graph( + asset_graph: &AssetGraph, + max_str_len: usize, +) -> serde_json::Result> { + let mut nodes: Vec = Vec::new(); + let mut curr_node = String::default(); + + for node in asset_graph.nodes() { + let serialized_node = match node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }; + + let str = serde_json::to_string(&serialized_node)?; + if curr_node.len() + str.len() < (max_str_len - 3) { + if !curr_node.is_empty() { + curr_node.push(','); + } + curr_node.push_str(&str); + } else { + // Add the existing node now as it has reached the max JavaScript string size + nodes.push(format!("[{curr_node}]")); + curr_node = str; + } + } + + // Add the current node if it did not overflow in size + if curr_node.len() < (max_str_len - 3) { + nodes.push(format!("[{curr_node}]")); + } + + Ok(nodes) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::{json, Value}; + + use super::*; + + #[test] + fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { + let mut graph = AssetGraph::new(); + + let entry = graph.add_entry_dependency(Dependency { + specifier: String::from("entry"), + ..Dependency::default() + }); + + let entry_asset = graph.add_asset(Asset { + file_path: PathBuf::from("entry"), + ..Asset::default() + }); + + graph.add_edge(&entry, &entry_asset); + + for i in 1..100 { + let node_index = graph.add_dependency(Dependency { + specifier: format!("dependency-{}", i), + ..Dependency::default() + }); + graph.add_edge(&entry_asset, &node_index); + } + + let max_str_len = 10000; + let nodes = serialize_asset_graph(&graph, max_str_len)?; + + assert_eq!(nodes.len(), 7); + + // Assert each string is less than the max size + for node in nodes.iter() { + assert!(node.len() < max_str_len); + } + + // Assert all the nodes are included and in the correct order + let first_entry = serde_json::from_str::(&nodes[0])?; + let first_entry = first_entry.as_array().unwrap(); + + assert_eq!(get_type(&first_entry[0]), json!("root")); + assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); + assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); + + for i in 1..first_entry.len() - 2 { + assert_eq!( + get_dependency(&first_entry[i + 2]), + Some(json!(format!("dependency-{}", i))) + ); + } + + let mut specifier = first_entry.len() - 2; + for node in nodes[1..].iter() { + let entry = serde_json::from_str::(&node)?; + let entry = entry.as_array().unwrap(); + + for value in entry { + assert_eq!( + get_dependency(&value), + Some(json!(format!("dependency-{}", specifier))) + ); + + specifier += 1; + } + } + + Ok(()) + } + + fn get_type(node: &Value) -> Value { + node.get("type").unwrap().to_owned() + } + + fn get_dependency(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("dependency")); + + value + .get("value") + .unwrap() + .get("dependency") + .unwrap() + .get("specifier") + .map(|s| s.to_owned()) + } + + fn get_asset(value: &Value) -> Option { + assert_eq!(get_type(&value), json!("asset")); + + value + .get("value") + .unwrap() + .get("filePath") + .map(|s| s.to_owned()) + } +} diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 76ada51ab..6280fd765 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -6,6 +6,7 @@ use std::thread; use anyhow::anyhow; use atlaspack::AtlaspackError; +use atlaspack_core::asset_graph::serialize_asset_graph; use lmdb_js_lite::writer::DatabaseWriter; use lmdb_js_lite::LMDB; use napi::Env; @@ -150,8 +151,11 @@ impl AtlaspackNapi { let mut js_object = env.create_object()?; js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - js_object - .set_named_property("nodes", asset_graph.serialize_nodes(MAX_STRING_LENGTH)?)?; + + js_object.set_named_property( + "nodes", + serialize_asset_graph(&asset_graph, MAX_STRING_LENGTH)?, + )?; NapiAtlaspackResult::ok(&env, js_object) } From 8c7f48301272859d0aefddea5328d5b5ad391ba3 Mon Sep 17 00:00:00 2001 From: David Alsh Date: Fri, 3 Jan 2025 08:44:20 +0000 Subject: [PATCH 2/2] That's one way to do it --- .../src/asset_graph/asset_graph.rs | 18 +++-- .../src/asset_graph/serialize_asset_graph.rs | 71 ++++++++++++++++++- .../node-bindings/src/atlaspack/atlaspack.rs | 13 ++-- .../src/requests/AssetGraphRequestRust.js | 4 -- 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/crates/atlaspack_core/src/asset_graph/asset_graph.rs b/crates/atlaspack_core/src/asset_graph/asset_graph.rs index 075496eb9..d6ecb5249 100644 --- a/crates/atlaspack_core/src/asset_graph/asset_graph.rs +++ b/crates/atlaspack_core/src/asset_graph/asset_graph.rs @@ -38,6 +38,17 @@ pub enum AssetGraphNode { Dependency(DependencyNode), } +impl AssetGraphNode { + pub fn id(&self) -> String { + match self { + AssetGraphNode::Root => "@@root".into(), + AssetGraphNode::Entry => "@@entry".into(), + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.clone(), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id(), + } + } +} + #[derive(Clone, Debug)] pub struct AssetGraph { pub graph: StableDiGraph, @@ -60,13 +71,12 @@ impl AssetGraph { } } - pub fn edges(&self) -> Vec { + pub fn edges(&self) -> Vec<(usize, usize)> { let raw_edges = self.graph.edge_references(); - let mut edges = Vec::new(); + let mut edges = Vec::<(usize, usize)>::new(); for edge in raw_edges { - edges.push(edge.source().index() as u32); - edges.push(edge.target().index() as u32); + edges.push((edge.source().index(), edge.target().index())); } edges diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs index 94fd0aae6..880aa280f 100644 --- a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs +++ b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs @@ -1,3 +1,7 @@ +use std::collections::HashMap; + +use petgraph::graph::NodeIndex; +use petgraph::visit::IntoNodeReferences; use serde::Serialize; use crate::types::{Asset, Dependency}; @@ -7,13 +11,13 @@ use super::{AssetGraph, AssetGraphNode, DependencyState}; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct SerializedDependency { - id: String, - dependency: Dependency, + pub id: String, + pub dependency: Dependency, } #[derive(Debug, Serialize)] #[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { +pub enum SerializedAssetGraphNode { Root, Entry, Asset { @@ -25,6 +29,67 @@ enum SerializedAssetGraphNode { }, } +#[derive(Debug, Serialize)] +pub struct SerializedAssetGraphResult { + pub edges: Vec, + pub nodes: Vec, +} + +pub fn serialize_asset_graph_sorted( + asset_graph: &AssetGraph, +) -> anyhow::Result { + let mut edges = Vec::<(u32, u32)>::new(); + let mut nodes = Vec::::new(); + + // Track index reassignment after sorting + let mut nx_to_index = HashMap::::new(); + + let mut current_nodes = asset_graph + .graph + .node_references() + .collect::>(); + current_nodes.sort_by(|a, b| a.1.id().cmp(&b.1.id())); + + for (current_node_nx, current_node) in current_nodes { + nx_to_index.insert(current_node_nx, nodes.len() as u32); + nodes.push(match current_node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }) + } + + for (from, to) in asset_graph.edges() { + let Some(new_addr_from) = nx_to_index.get(&(from as u32).into()) else { + panic!(); + }; + + let Some(new_addr_to) = nx_to_index.get(&(to as u32).into()) else { + panic!(); + }; + + edges.push((new_addr_from.clone(), new_addr_to.clone())); + } + + edges.sort_by(|a, b| format!("{}{}", a.0, a.1).cmp(&format!("{}{}", b.0, b.1))); + + let edges = edges + .into_iter() + .flat_map(|v| vec![v.0, v.1]) + .collect::>(); + + Ok(SerializedAssetGraphResult { edges, nodes }) +} + pub fn serialize_asset_graph( asset_graph: &AssetGraph, max_str_len: usize, diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 6280fd765..35de320cc 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -7,6 +7,7 @@ use std::thread; use anyhow::anyhow; use atlaspack::AtlaspackError; use atlaspack_core::asset_graph::serialize_asset_graph; +use atlaspack_core::asset_graph::serialize_asset_graph_sorted; use lmdb_js_lite::writer::DatabaseWriter; use lmdb_js_lite::LMDB; use napi::Env; @@ -148,16 +149,10 @@ impl AtlaspackNapi { // not supplied as JavaScript Error types. The JavaScript layer needs to handle conversions deferred.resolve(move |env| match result { Ok(asset_graph) => { - let mut js_object = env.create_object()?; + let result = serialize_asset_graph_sorted(&asset_graph)?; + let js_result = env.to_js_value(&result)?; - js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - - js_object.set_named_property( - "nodes", - serialize_asset_graph(&asset_graph, MAX_STRING_LENGTH)?, - )?; - - NapiAtlaspackResult::ok(&env, js_object) + NapiAtlaspackResult::ok(&env, js_result) } Err(error) => { let js_object = env.to_js_value(&AtlaspackError::from(&error))?; diff --git a/packages/core/core/src/requests/AssetGraphRequestRust.js b/packages/core/core/src/requests/AssetGraphRequestRust.js index f6dc42424..6cf448801 100644 --- a/packages/core/core/src/requests/AssetGraphRequestRust.js +++ b/packages/core/core/src/requests/AssetGraphRequestRust.js @@ -39,10 +39,6 @@ export function createAssetGraphRequestRust( let options = input.options; let serializedAssetGraph = await rustAtlaspack.buildAssetGraph(); - serializedAssetGraph.nodes = serializedAssetGraph.nodes.flatMap((node) => - JSON.parse(node), - ); - let {assetGraph, changedAssets} = instrument( 'atlaspack_v3_getAssetGraph', () => getAssetGraph(serializedAssetGraph),