From 88c721f08978de3238bb61f9f7524a562284248d Mon Sep 17 00:00:00 2001 From: Nikolay Arhipov Date: Thu, 4 Apr 2024 18:35:36 +0300 Subject: [PATCH] feat: Improved elf stripping --- README.md | 68 ++++++++++---- src/commands/build.rs | 148 ++++++++++++++++++++----------- src/commands/build/unit_graph.rs | 70 +++++++++++++++ src/meta.rs | 37 +++++--- 4 files changed, 240 insertions(+), 83 deletions(-) create mode 100644 src/commands/build/unit_graph.rs diff --git a/README.md b/README.md index 36105ed..903357c 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ Cargo command to work with Sony PlayStation Vita rust project binaries. -For general guidelines see [vita-rust book](https://vita-rust.github.io/book). +For general guidelines see the [vita-rust book]. ## Requirements -- [VitaSDK](https://vitasdk.org/) must be installed, and `VITASDK` environment variable must point to its location. -- [vitacompanion](https://github.com/devnoname120/vitacompanion) for ftp and command server (uploading and running artifacts) -- [PrincessLog](https://github.com/CelesteBlue-dev/PSVita-RE-tools/tree/master/PrincessLog/build) is required for `cargo vita logs` -- [vita-parse-core](https://github.com/xyzz/vita-parse-core) for `cargo vita coredump parse` +- [VitaSDK] must be installed, and `VITASDK` environment variable must point to its location. +- [vitacompanion] for FTP and command server (uploading and running artifacts) +- [PrincessLog] is required for `cargo vita logs` +- [vita-parse-core] for `cargo vita coredump parse` ## Installation @@ -23,7 +23,7 @@ cargo +nightly install cargo-vita ## Usage -Use the nightly toolchain to build Vita apps (either by using rustup override nightly for the project directory or by adding +nightly in the cargo invocation). +Use the nightly toolchain to build Vita apps (either by using `rustup override nightly` for the project directory or by adding +nightly in the cargo invocation). ``` @@ -74,14 +74,17 @@ title_name = "My application" assets = "static" # Optional, this is the default build_std = "std,panic_unwind" -# Optional, true by default. Will strip debug symbols from the resulting elf when enabled. -strip = true -# Optional, this is the default -vita_strip_flags = ["-g"] # Optional, this is the default vita_make_fself_flags = ["-s"] # Optional, this is the default vita_mksfoex_flags = ["-d", "ATTRIBUTE2=12"] + +[package.metadata.vita.dev] +# Strips symbols from the vita elf in dev profile. Optional, default is false +strip_symbols = true +[package.metadata.vita.release] +# Strips symbols from the vita elf in release profile. Optional, default is true +strip_symbols = true ``` ## Examples @@ -105,13 +108,13 @@ cargo vita logs ## Additional tools -For a better development experience it is recommended to install additional modules on your Vita. +For a better development experience, it is recommended to install the following modules on your Vita. ### vitacompanion -When enabled, this module keeps a FTP server on your Vita running on port `1337`, as well as a TCP command server running on port `1338`. +When enabled, this module keeps an FTP server on your Vita running on port `1337`, as well as a TCP command server running on port `1338`. -- The FTP server allows you to easily upload `vpk` and `eboot` files to your Vita. This is FTP server is used by `cargo-vita` for the following commands and flags: +- The FTP server allows you to easily upload `vpk` and `eboot` files to your Vita. This FTP server is used by `cargo-vita` for the following commands and flags: ```sh # Builds a eboot.bin, and uploads it to ux0:/app/TITLEID/eboot.bin @@ -138,7 +141,7 @@ When enabled, this module keeps a FTP server on your Vita running on port `1337` ### PrincessLog This module allows capturing stdout and stderr from your Vita. -In order to capture the logs you need to start a TCP server on your computer, and configure +In order to capture the logs you need to start a TCP server on your computer and configure PrincessLog to connect to it. For convenience `cargo-vita` provides two commands to work with logs: @@ -149,10 +152,10 @@ For convenience `cargo-vita` provides two commands to work with logs: # Start a TCP server on 0.0.0.0, and print all bytes received via the socket to stdout cargo vita logs ``` - - A command to reconfigure PrincessLog with the new ip/port. This will use + - A command to reconfigure PrincessLog with the new IP/port. This will use the FTP server provided by `vitacompanion` to upload a new config. If an IP address of your machine is not explicitly provided, it will be guessed - using [local-ip-address](https://crates.io/crates/local-ip-address) crate. + using [local-ip-address] crate. When a configuration file is updated, the changes are not applied until Vita is rebooted. ```sh @@ -167,6 +170,32 @@ For convenience `cargo-vita` provides two commands to work with logs: cargo vita logs configure --host-ip-address 10.10.10.10 --kernel-debug ``` +## Notes + +To produce the actual artifact runnable on the device, `cargo-vita` does multiple steps: + +1. Calls `cargo build` to build the code and link it to a `elf` file (using linker from [VitaSDK]) +2. Calls `vita-elf-create` from [VitaSDK] to transform the `elf` into Vita `elf` (`velf`) +3. Calls `vita-make-fself` from [VitaSDK] to sign `velf` into `self` (aka `eboot`). + +The second step of this process requires relocation segments in the elf. +This means, that adding `strip=true` or `strip="symbols"` is not supported for Vita target, +since symbol stripping also strips relocation information. + +To counter this issue, `cargo-vita` can do an additional strip step of the `elf` with `--strip-unneeded` flag, which reduces the binary size without interfering with other steps necessary to produce a runnable binary. + +This step is enabled for release builds and disabled for dev builds by default, but can be configured per-crate via the following section in `Cargo.toml`: + +```toml +[package.metadata.vita.dev] +# Strips symbols from the vita elf in dev profile, default is false +strip_symbols = true +[package.metadata.vita.release] +# Strips symbols from the vita elf in release profile, default is true +strip_symbols = true +``` + + ## License Except where noted (below and/or in individual files), all code in this repository is dual-licensed at your option under either: @@ -174,3 +203,10 @@ Except where noted (below and/or in individual files), all code in this reposito * MIT License ([LICENSE-MIT](LICENSE-MIT) or [http://opensource.org/licenses/MIT](http://opensource.org/licenses/MIT)) * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)) + +[vita-rust book]: https://vita-rust.github.io/book +[VitaSDK]: https://vitasdk.org/ +[vitacompanion]: https://github.com/devnoname120/vitacompanion +[PrincessLog]: https://github.com/CelesteBlue-dev/PSVita-RE-tools/tree/master/PrincessLog/build +[vita-parse-core]: https://github.com/xyzz/vita-parse-core +[local-ip-address]: https://crates.io/crates/local-ip-address diff --git a/src/commands/build.rs b/src/commands/build.rs index 0e6b689..c29437e 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -6,13 +6,13 @@ use std::{ process::{Command, Stdio}, }; -use crate::{check, ftp}; +use crate::{check, commands::build::unit_graph::try_parse_unit_graph, ftp}; use anyhow::{bail, Context}; use cargo_metadata::{camino::Utf8PathBuf, Artifact, Message, Package}; -use clap::{Args, Subcommand}; +use clap::{command, Args, Subcommand}; use colored::Colorize; use either::Either; -use log::info; +use log::{info, warn}; use tee::TeeReader; use walkdir::WalkDir; @@ -20,6 +20,8 @@ use crate::meta::{parse_crate_metadata, PackageMetadata, TitleId, VITA_TARGET}; use super::{ConnectionArgs, Executor, OptionalConnectionArgs, Run}; +mod unit_graph; + #[derive(Args, Debug)] pub struct Build { #[command(subcommand)] @@ -210,39 +212,47 @@ impl<'a> BuildContext<'a> { let rust_flags = env::var("RUSTFLAGS").unwrap_or_default() + " --cfg mio_unsupported_force_poll_poll --cfg mio_unsupported_force_waker_pipe"; - let mut command = Command::new(cargo); + // FIXME: move build-std to .cargo/config.toml, since it is shared by ALL of the crates built, + // but the metadata is per-crate. This still works correctly when building only a single workspace crate. + let (meta, _, _) = parse_crate_metadata(None)?; - if let Ok(path) = env::var("PATH") { - let sdk_path = Path::new(&self.sdk).join("bin"); - let path = format!("{}:{path}", sdk_path.display()); - command.env("PATH", path); - } + let command = || { + let mut command = Command::new(&cargo); - // FIXME: move build-std to env/config.toml, since it is shared by all of the crates built - // This still works correctly when building only a single workspace crate though - let (meta, _, _) = parse_crate_metadata(None)?; + if let Ok(path) = env::var("PATH") { + let sdk_path = Path::new(&self.sdk).join("bin"); + let path = format!("{}:{path}", sdk_path.display()); + command.env("PATH", path); + } + command + .env("RUSTFLAGS", &rust_flags) + .env("TARGET_CC", "arm-vita-eabi-gcc") + .env("TARGET_CXX", "arm-vita-eabi-g++") + .pass_path_env("OPENSSL_LIB_DIR", || self.sdk("arm-vita-eabi").join("lib")) + .pass_path_env("OPENSSL_INCLUDE_DIR", || { + self.sdk("arm-vita-eabi").join("include") + }) + .pass_path_env("PKG_CONFIG_PATH", || { + self.sdk("arm-vita-eabi").join("lib").join("pkgconfig") + }) + .pass_env("PKG_CONFIG_SYSROOT_DIR", || &self.sdk) + .env("VITASDK", &self.sdk) + .arg("build") + .arg("-Z") + .arg(format!("build-std={}", &meta.build_std)) + .arg("--target") + .arg(VITA_TARGET) + .arg("--message-format=json-render-diagnostics") + .args(&self.command.cargo_args); + + command + }; + + let hints = try_parse_unit_graph(command()).ok(); + + let mut command = command(); command - .env("RUSTFLAGS", rust_flags) - .env("TARGET_CC", "arm-vita-eabi-gcc") - .env("TARGET_CXX", "arm-vita-eabi-g++") - .pass_path_env("OPENSSL_LIB_DIR", || self.sdk("arm-vita-eabi").join("lib")) - .pass_path_env("OPENSSL_INCLUDE_DIR", || { - self.sdk("arm-vita-eabi").join("include") - }) - .pass_path_env("PKG_CONFIG_PATH", || { - self.sdk("arm-vita-eabi").join("lib").join("pkgconfig") - }) - .pass_env("PKG_CONFIG_SYSROOT_DIR", || &self.sdk) - .env("VITASDK", &self.sdk) - .arg("build") - .arg("-Z") - .arg(format!("build-std={}", meta.build_std)) - .arg("--target") - .arg(VITA_TARGET) - .arg("--message-format") - .arg("json-render-diagnostics") - .args(&self.command.cargo_args) .stdin(Stdio::inherit()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()); @@ -250,29 +260,49 @@ impl<'a> BuildContext<'a> { info!("{}: {command:?}", "Running cargo".blue()); let mut process = command.spawn().context("Unable to spawn build process")?; - let command_stdout = process.stdout.take().context("Build failed")?; - - let reader = if log::max_level() >= log::LevelFilter::Trace { - Either::Left(BufReader::new(TeeReader::new(command_stdout, io::stdout()))) + let stdout = process.stdout.take().context("Build failed")?; + let stdout = if log::max_level() >= log::LevelFilter::Trace { + Either::Left(BufReader::new(TeeReader::new(stdout, io::stdout()))) } else { - Either::Right(BufReader::new(command_stdout)) + Either::Right(BufReader::new(stdout)) }; - let messages: Vec = Message::parse_stream(reader) - .collect::>() - .context("Unable to parse build stdout")?; + let message_stream = Message::parse_stream(stdout); - let artifacts = messages - .iter() - .rev() - .filter_map(|m| match m { - Message::CompilerArtifact(art) if art.executable.is_some() => Some(art.clone()), - _ => None, - }) - .map(ExecutableArtifact::new) - .collect::>()?; + let mut artifacts = Vec::new(); + + for message in message_stream { + match message.context("Unable to parse cargo output")? { + Message::CompilerArtifact(art) if art.executable.is_some() => { + artifacts.push(ExecutableArtifact::new(art)?); + } + _ => {} + } + } if !process.wait_with_output()?.status.success() { + if let Some(hints) = hints { + if hints.strip_symbols() { + warn!( + "{warn}\n \ + Symbols in elf are required by `{velf}` to create a velf file.\n \ + Please remove `{strip_true}` or `{strip_symbols}` from your Cargo.toml.\n \ + If you want to optimize for the binary size, replace it \ + with `{strip_debug}` to strip debug section.\n \ + If you want to strip the symbol data from the resulting \ + binary, set `{strip_velf}` in `{vita_section}` \ + section of your Cargo.toml, this would strip the symbols from the velf.", + warn = "Stripping symbols from ELF is unsupported.".yellow(), + velf = "vita-elf-create".cyan(), + strip_true = "strip=true".cyan(), + strip_symbols = "strip=\"symbols\"".cyan(), + strip_debug = "strip=\"debuginfo\"".cyan(), + strip_velf = "strip_symbols = true".cyan(), + vita_section = format!("[package.metadata.vita.{}]", hints.profile).cyan() + ); + } + } + bail!("cargo build failed") } @@ -280,21 +310,33 @@ impl<'a> BuildContext<'a> { } fn strip(&self, art: &ExecutableArtifact) -> anyhow::Result<()> { - if !art.meta.strip { - info!("{}", "Skipping elf strip".yellow()); + // Try to guess if the elf was built with debug or release profile. + // This intentionally uses components() instead of as_str() to + // ensure that it works with operating systems that use a reverse slash for paths (Windows) + // as well as it works if for some reason the path to elf is not normalized. + let profile = art + .elf + .components() + .skip_while(|s| s.as_str() != "armv7-sony-vita-newlibeabihf") + .nth(1); + + let is_release = profile.map(|p| p.as_str()) == Some("release"); + + if !art.meta.strip_symbols(is_release) { + info!("{}", "Skipping additional elf strip".yellow()); return Ok(()); } let mut command = Command::new(self.sdk_binary("arm-vita-eabi-strip")); command - .args(&art.meta.vita_strip_flags) + .arg("--strip-unneeded") .arg(&art.elf) .stdin(Stdio::inherit()) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); - info!("{}: {command:?}", "Stripping elf".blue()); + info!("{}: {command:?}", "Stripping symbols from elf".blue()); if !command.status()?.success() { bail!("arm-vita-eabi-strip failed"); diff --git a/src/commands/build/unit_graph.rs b/src/commands/build/unit_graph.rs new file mode 100644 index 0000000..5a44105 --- /dev/null +++ b/src/commands/build/unit_graph.rs @@ -0,0 +1,70 @@ +use std::process::{Command, Stdio}; + +use anyhow::Context; + +pub struct BuildHints { + // Can be "debug" or "release" + pub profile: String, + + // Can be "None", "debuginfo", "symbols", "true" or any invalid value + strip: Option, +} + +impl BuildHints { + pub fn strip_symbols(&self) -> bool { + [Some("symbols"), Some("true")].contains(&self.strip.as_deref()) + } +} + +#[derive(serde::Deserialize)] +struct UnitGraph { + units: Vec, +} + +#[derive(serde::Deserialize)] +struct Unit { + profile: Profile, +} + +#[derive(serde::Deserialize)] +struct Profile { + name: String, + strip: Strip, +} + +#[derive(serde::Deserialize)] +struct Strip { + resolved: Option, +} + +#[derive(serde::Deserialize)] +struct StripResolved { + #[serde(rename = "Named")] + named: Option, +} + +pub fn try_parse_unit_graph(mut command: Command) -> anyhow::Result { + command.args(["-Z", "unstable-options", "--unit-graph"]); + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + + let stdout = command + .output() + .context("Unable to spawn build process")? + .stdout; + let json = serde_json::from_slice::(&stdout).context("Unable to parse json")?; + + let last_unit = json + .units + .into_iter() + .next_back() + .context("No units found")? + .profile; + + Ok(BuildHints { + profile: last_unit.name, + strip: last_unit.strip.resolved.and_then(|s| s.named), + }) +} diff --git a/src/meta.rs b/src/meta.rs index 49ea705..49b4fd5 100644 --- a/src/meta.rs +++ b/src/meta.rs @@ -58,18 +58,10 @@ impl FromStr for TitleId { } } -fn default_strip() -> bool { - true -} - fn default_build_std() -> String { "std,panic_unwind".to_string() } -fn default_vita_strip_flags() -> Vec { - vec!["-g".to_string()] -} - fn default_vita_make_fself_flags() -> Vec { vec!["-s".to_string()] } @@ -78,6 +70,7 @@ fn default_vita_mksfoex_flags() -> Vec { vec!["-d".to_string(), "ATTRIBUTE2=12".to_string()] } + #[derive(Deserialize, Debug)] pub struct PackageMetadata { pub title_id: Option, @@ -85,14 +78,30 @@ pub struct PackageMetadata { pub assets: Option, #[serde(default = "default_build_std")] pub build_std: String, - #[serde(default = "default_strip")] - pub strip: bool, - #[serde(default = "default_vita_strip_flags")] - pub vita_strip_flags: Vec, #[serde(default = "default_vita_make_fself_flags")] pub vita_make_fself_flags: Vec, #[serde(default = "default_vita_mksfoex_flags")] pub vita_mksfoex_flags: Vec, + + #[serde(default)] + pub dev: ProfileMetadata, + #[serde(default)] + pub release: ProfileMetadata, +} + +impl PackageMetadata { + pub fn strip_symbols(&self, release: bool) -> bool { + if release { + self.release.strip_symbols.unwrap_or(true) + } else { + self.dev.strip_symbols.unwrap_or(false) + } + } +} + +#[derive(Deserialize, Debug, Default)] +pub struct ProfileMetadata { + pub strip_symbols: Option, } impl Default for PackageMetadata { @@ -102,10 +111,10 @@ impl Default for PackageMetadata { title_name: None, assets: None, build_std: default_build_std(), - strip: default_strip(), - vita_strip_flags: default_vita_strip_flags(), vita_make_fself_flags: default_vita_make_fself_flags(), vita_mksfoex_flags: default_vita_mksfoex_flags(), + release: ProfileMetadata::default(), + dev: ProfileMetadata::default(), } } }