From e817af1893f86fab677ff3536236fb61cb21cd07 Mon Sep 17 00:00:00 2001 From: Liam <33645555+lj3954@users.noreply.github.com> Date: Tue, 4 Jun 2024 18:40:56 -0700 Subject: [PATCH] Add NixOS --- Cargo.lock | 356 ++++++++++++++++++++++++++- quickget_ci/Cargo.toml | 1 + quickget_ci/src/linux.rs | 2 + quickget_ci/src/linux/independent.rs | 96 ++++++++ quickget_ci/src/main.rs | 1 + 5 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 quickget_ci/src/linux/independent.rs diff --git a/Cargo.lock b/Cargo.lock index 64fb554..027d348 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -96,6 +107,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -179,6 +199,27 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.98" @@ -208,6 +249,16 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.4" @@ -277,6 +328,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -326,6 +383,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -367,6 +448,32 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" +[[package]] +name = "deflate64" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -375,6 +482,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -414,6 +522,17 @@ dependencies = [ "xcb", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dlib" version = "0.5.2" @@ -495,6 +614,28 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", +] + +[[package]] +name = "flate2" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -716,6 +857,15 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -883,6 +1033,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.13" @@ -962,6 +1121,26 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "liblzma" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243510a5543c358949902b9e76daec3a32d7b03a43abce823e7c62a1a8360172" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83cf78d20a45b5c0f3c7da2dcac255b230efe7d8684282bd35873164c1491187" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libredox" version = "0.1.3" @@ -988,12 +1167,28 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1071,6 +1266,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.19" @@ -1181,7 +1382,7 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "winapi", ] @@ -1192,6 +1393,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1242,6 +1453,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1273,16 +1490,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", + "serde", ] [[package]] name = "quick_fetcher" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e66a96088e32f1ecb08fcd0d6d433cd33310a44e9a9aaa0f90542e68c9951bf" +checksum = "c88a2fd9eea875a236cc5b1395f1b076c26475d7d0940612cb4eafa4e23f9ca0" dependencies = [ + "bzip2", + "flate2", "futures", "indicatif", + "liblzma", "log", "md-5", "once_cell", @@ -1291,8 +1512,11 @@ dependencies = [ "reqwest-retry", "sha1", "sha2", + "tar", "thiserror", "tokio", + "zip", + "zstd", ] [[package]] @@ -1344,6 +1568,7 @@ dependencies = [ "futures", "log", "once_cell", + "quick-xml 0.31.0", "quickemu-rs", "regex", "reqwest", @@ -1429,6 +1654,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.5" @@ -1720,6 +1954,12 @@ dependencies = [ "digest", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.9" @@ -1774,6 +2014,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "2.0.66" @@ -1827,6 +2073,17 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -1859,6 +2116,25 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + [[package]] name = "tinyvec" version = "1.6.0" @@ -2575,6 +2851,17 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xcb" version = "1.4.0" @@ -2598,6 +2885,69 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "054a8e68b76250b253f671d1268cb7f1ae089ec35e195b2efb2a4e9a836d0621" +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "098d5d7737fb0b70814faa73c17df84f047d38dd31d13bbf2ec3fb354b5abf45" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "rand", + "sha1", + "thiserror", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.1" diff --git a/quickget_ci/Cargo.toml b/quickget_ci/Cargo.toml index d898493..e572a16 100644 --- a/quickget_ci/Cargo.toml +++ b/quickget_ci/Cargo.toml @@ -8,6 +8,7 @@ env_logger = "0.11.3" futures = "0.3.30" log = "0.4.21" once_cell = "1.19.0" +quick-xml = { version = "0.31.0", features = ["serialize"] } quickemu-rs = { path = "../quickemu" } regex = "1.10.4" reqwest = "0.12.4" diff --git a/quickget_ci/src/linux.rs b/quickget_ci/src/linux.rs index 57ec785..4fa60f4 100644 --- a/quickget_ci/src/linux.rs +++ b/quickget_ci/src/linux.rs @@ -1,4 +1,6 @@ mod arch_derivatives; +mod independent; mod ubuntu; +pub(crate) use independent::NixOS; pub(crate) use ubuntu::{Edubuntu, Kubuntu, Lubuntu, Ubuntu, UbuntuBudgie, UbuntuCinnamon, UbuntuKylin, UbuntuMATE, UbuntuServer, UbuntuStudio, UbuntuUnity, Xubuntu}; diff --git a/quickget_ci/src/linux/independent.rs b/quickget_ci/src/linux/independent.rs new file mode 100644 index 0000000..f5a686e --- /dev/null +++ b/quickget_ci/src/linux/independent.rs @@ -0,0 +1,96 @@ +use crate::store_data::{Config, Source, WebSource}; +use crate::utils::capture_page; +use quickemu::config::Arch; +use regex::Regex; +use serde::Deserialize; + +use crate::store_data::Distro; + +const NIX_URL: &str = "https://nix-channels.s3.amazonaws.com/?delimiter=/"; +const NIX_DOWNLOAD_URL: &str = "https://channels.nixos.org"; + +pub struct NixOS; +impl Distro for NixOS { + const NAME: &'static str = "nixos"; + const PRETTY_NAME: &'static str = "NixOS"; + const HOMEPAGE: Option<&'static str> = Some("https://nixos.org/"); + const DESCRIPTION: Option<&'static str> = Some("Linux distribution based on Nix package manager, tool that takes a unique approach to package management and system configuration."); + async fn generate_configs() -> Vec { + let standard_release = Regex::new(r#"nixos-(([0-9]+.[0-9]+|(unstable))(?:-small)?)"#).unwrap(); + let iso_regex = Regex::new(r#"latest-nixos-([^-]+)-([^-]+)-linux.iso"#).unwrap(); + let Some(releases) = capture_page(NIX_URL) + .await + .and_then(|page| quick_xml::de::from_str::(&page).ok()) + else { + return Vec::new(); + }; + + let releases: Vec = releases + .contents + .into_iter() + .map(|r| r.key) + .filter(|r| standard_release.is_match(r)) + .rev() + .take(6) + .map(|r| standard_release.captures(&r).unwrap().get(0).unwrap().as_str().to_string()) + .collect(); + let mut futures = Vec::new(); + for release in releases { + println!("Fetching NixOS release: {}", release); + if let Some(page) = capture_page(&format!("{NIX_URL}&prefix={release}/")) + .await + .and_then(|p| quick_xml::de::from_str::(&p).ok()) + { + let page = page + .contents + .into_iter() + .map(|r| r.key) + .filter(|r| iso_regex.is_match(r) && r.ends_with(".iso")) + .collect::>(); + + futures.append( + &mut page + .into_iter() + .map(|page| { + let capture = iso_regex.captures(&page).unwrap(); + let release = release.clone(); + let name = capture.get(0).map(|n| n.as_str().to_string()); + let edition = capture.get(1).map(|e| e.as_str().to_string()); + let arch: Option = capture.get(2).map(|a| a.as_str().to_string()).try_into().ok(); + tokio::spawn(async move { + let iso = format!("{NIX_DOWNLOAD_URL}/{release}/{}", name?); + let hash = capture_page(&format!("{iso}.sha256")) + .await + .map(|h| h.split_whitespace().next().unwrap().to_string()); + Some(Config { + release: Some(release), + edition: Some(edition?), + arch: arch?, + iso: Some(vec![Source::Web(WebSource::new(iso, hash, None, None))]), + ..Default::default() + }) + }) + }) + .collect(), + ); + }; + } + futures::future::join_all(futures) + .await + .into_iter() + .flatten() + .flatten() + .collect() + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct NixReleases { + contents: Vec, +} +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +struct NixRelease { + key: String, +} diff --git a/quickget_ci/src/main.rs b/quickget_ci/src/main.rs index dfdb65f..0dbb39c 100644 --- a/quickget_ci/src/main.rs +++ b/quickget_ci/src/main.rs @@ -25,6 +25,7 @@ async fn main() { spawn(linux::Edubuntu.to_os()), spawn(linux::Xubuntu.to_os()), spawn(linux::UbuntuCinnamon.to_os()), + spawn(linux::NixOS.to_os()), ]; let distros = futures::future::join_all(futures)