Skip to content

Commit

Permalink
Spike: Auto Layer Ordering
Browse files Browse the repository at this point in the history
Prefix layers with sequential numbers starting at `0001_<name>`. To support this it assumes any layers on disk with a number prefix is equivalent i.e. changing the order of layers will not invalidate the cache.

This has the benefit that:

- Layers can be named something semantic without side effects.
- Build and launch layer behavior is guaranteed to be the same (provided `read_env` is called and applied for every layer in main).
  • Loading branch information
schneems committed Jan 10, 2025
1 parent c91cee5 commit b6820b6
Show file tree
Hide file tree
Showing 5 changed files with 144 additions and 28 deletions.
5 changes: 5 additions & 0 deletions commons/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,8 @@ libcnb-test = "=0.26.1"
pretty_assertions = "1"
toml = "0.8"
bullet_stream = "0.3.0"

[features]
# Auto name layers in the order of their creation so that launch and build are resolved in the same order
auto_layer_ordering = []
default = ["auto_layer_ordering"]
18 changes: 15 additions & 3 deletions commons/src/cache/app_cache.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::cache::clean::{lru_clean, FilesWithSize};
use crate::cache::{CacheConfig, CacheError, KeepPath};
use crate::layer::order::ordered_layer_name;
use byte_unit::{AdjustedByte, Byte, UnitType};
use fs_extra::dir::CopyOptions;
use libcnb::build::BuildContext;
Expand Down Expand Up @@ -387,9 +388,15 @@ fn create_layer_name(app_root: &Path, path: &Path) -> Result<LayerName, CacheErr
.collect::<Vec<_>>()
.join("_");

format!("cache_{name}")
let layer_name: LayerName = format!("cache_{name}")
.parse()
.map_err(CacheError::InvalidLayerName)
.map_err(CacheError::InvalidLayerName)?;

if cfg!(feature = "auto_layer_ordering") {
Ok(ordered_layer_name(layer_name))
} else {
Ok(layer_name)
}
}

/// Determines if a cache directory in a layer previously existed or not.
Expand Down Expand Up @@ -420,6 +427,8 @@ fn is_empty_dir(path: &Path) -> bool {

#[cfg(test)]
mod tests {
use crate::layer::order::strip_order_prefix;

use super::*;
use filetime::FileTime;
use libcnb::data::layer_name;
Expand All @@ -429,7 +438,10 @@ mod tests {
fn test_to_layer_name() {
let dir = PathBuf::from_str("muh_base").unwrap();
let layer = create_layer_name(&dir, &dir.join("my").join("input")).unwrap();
assert_eq!(layer_name!("cache_my_input"), layer);
assert_eq!(
layer_name!("cache_my_input").as_str(),
&strip_order_prefix(layer.as_str())
);
}

#[test]
Expand Down
1 change: 1 addition & 0 deletions commons/src/layer.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
mod configure_env_layer;
mod default_env_layer;
pub mod diff_migrate;
pub(crate) mod order;

#[deprecated(note = "Use the struct layer API in the latest libcnb.rs instead")]
pub use self::configure_env_layer::ConfigureEnvLayer;
Expand Down
80 changes: 55 additions & 25 deletions commons/src/layer/diff_migrate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
#![doc = include_str!("fixtures/metadata_migration_example.md")]

use crate::display::SentenceList;
use crate::layer::order::contains_entry_with_name_or_pattern;
use crate::layer::order::ordered_layer_name;
use cache_diff::CacheDiff;
use fs_err::PathExt;
use libcnb::build::BuildContext;
Expand All @@ -53,7 +55,7 @@ use libcnb::layer::{
use magic_migrate::TryMigrate;
use serde::ser::Serialize;
use std::fmt::Debug;
use std::path::PathBuf;
use std::path::{Path, PathBuf};

#[cfg(test)]
use bullet_stream as _;
Expand Down Expand Up @@ -120,6 +122,19 @@ impl DiffMigrateLayer {
B: libcnb::Buildpack,
M: CacheDiff + TryMigrate + Serialize + Debug + Clone,
{
let layer_name = if cfg!(feature = "auto_layer_ordering") {
let layer_name = ordered_layer_name(layer_name);

// Move NNNN_<layer_name> or `<layer_name>` to `NNNN_<layer_name>`
if let Some(prior) = layer_to_path(context, &layer_name)? {
move_layer_path(&prior, &context.layers_dir.join(layer_name.as_str()))
.map_err(LayerError::IoError)?;
}
layer_name
} else {
layer_name
};

let layer_ref = context.cached_layer(
layer_name,
CachedLayerDefinition {
Expand Down Expand Up @@ -166,25 +181,28 @@ impl DiffMigrateLayer {
if let (Some(prior_dir), None) = (
prior_layers
.iter()
.map(|layer_name| is_layer_on_disk(layer_name, context))
.map(|layer_name| layer_to_path(context, layer_name))
.collect::<Result<Vec<Option<PathBuf>>, _>>()?
.iter()
.find_map(std::borrow::ToOwned::to_owned),
is_layer_on_disk(&to_layer, context)?,
layer_to_path(context, &to_layer)?,
) {
let to_dir = context.layers_dir.join(to_layer.as_str());
std::fs::create_dir_all(&to_dir).map_err(LayerError::IoError)?;
std::fs::rename(&prior_dir, &to_dir).map_err(LayerError::IoError)?;
std::fs::rename(
prior_dir.with_extension("toml"),
to_dir.with_extension("toml"),
)
.map_err(LayerError::IoError)?;
move_layer_path(&prior_dir, &context.layers_dir.join(to_layer.as_str()))
.map_err(LayerError::IoError)?;
}
self.cached_layer(to_layer, context, metadata)
}
}

fn move_layer_path(from_dir: &Path, to_dir: &Path) -> Result<(), std::io::Error> {
std::fs::create_dir_all(to_dir)?;
std::fs::rename(from_dir, to_dir)?;
std::fs::rename(
from_dir.with_extension("toml"),
to_dir.with_extension("toml"),
)
}

/// Represents when we want to move contents from one (or more) layer names
///
pub struct LayerRename {
Expand All @@ -195,18 +213,24 @@ pub struct LayerRename {
}

/// Returns Some(PathBuf) when the layer exists on disk
fn is_layer_on_disk<B>(
layer_name: &LayerName,
///
/// Is aware of auto layer-ordering and will convert `0001_my_layer` => `my_layer` and vise-versa
fn layer_to_path<B>(
context: &BuildContext<B>,
layer_name: &LayerName,
) -> libcnb::Result<Option<PathBuf>, B::Error>
where
B: libcnb::Buildpack,
{
let path = context.layers_dir.join(layer_name.as_str());

path.fs_err_try_exists()
.map_err(|error| libcnb::Error::LayerError(LayerError::IoError(error)))
.map(|exists| exists.then_some(path))
if cfg!(feature = "auto_layer_ordering") {
contains_entry_with_name_or_pattern(&context.layers_dir, layer_name.as_str())
.map_err(|error| libcnb::Error::LayerError(LayerError::IoError(error)))
} else {
let path = context.layers_dir.join(layer_name.as_str());
path.fs_err_try_exists()
.map_err(|error| libcnb::Error::LayerError(LayerError::IoError(error)))
.map(|exists| exists.then_some(path))
}
}

/// Standardizes formatting for layer cache clearing behavior
Expand Down Expand Up @@ -292,6 +316,7 @@ where
///
/// - Will only ever contain metadata when the cache is retained.
/// - Will contain a message when the cache is cleared, describing why it was cleared.
#[derive(Debug)]
pub enum Meta<M> {
Message(String),
Data(M),
Expand Down Expand Up @@ -322,6 +347,7 @@ mod tests {
use libcnb::layer::{EmptyLayerCause, InvalidMetadataAction, LayerState, RestoredLayerAction};
use magic_migrate::{migrate_toml_chain, try_migrate_deserializer_chain, Migrate, TryMigrate};
use std::convert::Infallible;

/// Struct for asserting the behavior of `CacheBuddy`
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
#[serde(deny_unknown_fields)]
Expand Down Expand Up @@ -418,9 +444,9 @@ mod tests {
}
));

assert!(context
.layers_dir
.join(old_layer_name.as_str())
assert!(layer_to_path(&context, &old_layer_name)
.unwrap()
.unwrap()
.fs_err_try_exists()
.unwrap());

Expand All @@ -446,10 +472,14 @@ mod tests {
)
.unwrap();

assert!(matches!(result.state, LayerState::Restored { cause: _ }));
assert!(context
.layers_dir
.join(new_layer_name.as_str())
assert!(
matches!(result.state, LayerState::Restored { cause: _ }),
"Does not match {:?}",
result.state
);
assert!(layer_to_path(&context, &new_layer_name)
.unwrap()
.unwrap()
.fs_err_try_exists()
.unwrap());
}
Expand Down
68 changes: 68 additions & 0 deletions commons/src/layer/order.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
use libcnb::data::layer::LayerName;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};

/// As layers are created this is incremented
static LAYER_COUNT: AtomicUsize = AtomicUsize::new(1);
/// Formatting for number of leanding zeros. Max number of layers is 1024 (as of 2025). Four digits allows
/// one buildpack to meet or exceed that value
static DIGITS: usize = 4;

fn prefix(value: usize) -> String {
format!("{value:0DIGITS$}_")
}

fn next_count() -> usize {
LAYER_COUNT.fetch_add(1, Ordering::Acquire)
}

/// Removes the `NNNN_` prefix if there is one
pub(crate) fn strip_order_prefix(name: &str) -> String {
let re = regex::Regex::new(&format!("^\\d{{{DIGITS}}}_"))
.expect("internal code bugs caught by unit tests");
re.replace(name, "").to_string()
}

/// Searches the given dir for an entry with the exact name or any NNNN_ prefix and returns Ok(Some(PathBuf))
pub(crate) fn contains_entry_with_name_or_pattern(
dir: &Path,
name: &str,
) -> Result<Option<PathBuf>, std::io::Error> {
let name = strip_order_prefix(name);

let pattern = format!("^\\d{{{DIGITS}}}_{}$", regex::escape(&name));
let re = regex::Regex::new(&pattern).expect("internal error if this fails to compile");

for entry in fs_err::read_dir(dir)?.flatten() {
if let Some(file_name) = entry.file_name().to_str() {
if file_name == name || re.is_match(file_name) {
return Ok(Some(entry.path().clone()));
}
}
}

Ok(None)
}

/// Gets and increments the next name
///
/// # Panics
///
/// Assumes that prepending a value to a valid layer name is a valid operation
#[must_use]
pub(crate) fn ordered_layer_name(name: LayerName) -> LayerName {
let prefix = prefix(next_count());
prefix_layer_name(&prefix, name)
}

/// # Panics
///
/// Assumes that prepending a value to a valid layer name is a valid operation
#[must_use]
#[allow(clippy::needless_pass_by_value)]
fn prefix_layer_name(prefix: impl AsRef<str>, name: LayerName) -> LayerName {
let prefix = prefix.as_ref();
format!("{prefix}{}", name.as_str())
.parse()
.expect("Prepending to a valid layer name is valid")
}

0 comments on commit b6820b6

Please sign in to comment.