From ee34ce9525816d293c5c23981e88cac1f9b61123 Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Thu, 7 Dec 2023 09:55:58 -0800 Subject: [PATCH 1/3] feat: domain configuration for icx-proxy (#3471) Add dfx.json, networks.json, and `dfx start` support for the icx-proxy `--domain` parameter. Fixes https://dfinity.atlassian.net/browse/SDK-1327 --- CHANGELOG.md | 12 ++++++ docs/cli-reference/dfx-start.md | 1 + docs/dfx-json-schema.json | 37 ++++++++++++++++ docs/networks-json-schema.json | 40 ++++++++++++++++++ e2e/tests-dfx/start.bash | 42 +++++++++++++++++++ src/dfx-core/src/config/model/dfinity.rs | 10 +++++ .../config/model/local_server_descriptor.rs | 16 ++++++- src/dfx-core/src/json/structure.rs | 2 +- src/dfx-core/src/network/provider.rs | 7 ++++ src/dfx/src/actors/icx_proxy.rs | 8 ++++ src/dfx/src/commands/start.rs | 14 +++++++ 11 files changed, 186 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fae22bc965..59c38087b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ Previously, `dfx ledger top-up` only accepted canister principals. Now it accept A change to `curl --help` output made it so the install script did not detect that the `--proto` and `--tlsv1.2` options are available. +### feat: Added support for icx-proxy `--domain` parameter + +In order to access a local replica through a domain name or domain names, +it's necessary to pass the `--domain` parameter to icx-proxy. dfx now supports +this in configuration and as a parameter to dfx start. You can specify a single +domain or a list of domains in any of the following ways: + +- in networks.json, in `..proxy.domain` +- in dfx.json, in `.networks..proxy.domain` +- in dfx.json, in `.defaults.proxy.domain` +- to dfx start, as `dfx start --domain --domain ...` + # 0.15.2 ### fix: `dfx canister delete ` removes the related entry from the canister id store diff --git a/docs/cli-reference/dfx-start.md b/docs/cli-reference/dfx-start.md index 83a399d66c..3621c67bc2 100644 --- a/docs/cli-reference/dfx-start.md +++ b/docs/cli-reference/dfx-start.md @@ -32,6 +32,7 @@ You can use the following option with the `dfx start` command. | `--host host` | Specifies the host interface IP address and port number to bind the frontend to. The default for the local shared network is `127.0.0.1:4943`, while the default for a project-specific network is '127.0.0.1:8000'. | | `--bitcoin-node host:port` | Specifies the address of a bitcoind node. Implies `--enable-bitcoin`. | | `--artificial-delay milliseconds` | Specifies the delay that an update call should incur. Default: 600ms | +| `--domain domain` | A domain that can be served. Can be specified more than once. These are used for canister resolution [default: localhost] | ## Examples diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index 186ca4c6f7..40f4e890ac 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -527,6 +527,16 @@ } ] }, + "proxy": { + "anyOf": [ + { + "$ref": "#/definitions/ConfigDefaultsProxy" + }, + { + "type": "null" + } + ] + }, "replica": { "anyOf": [ { @@ -646,6 +656,23 @@ } } }, + "ConfigDefaultsProxy": { + "description": "Configuration for icx-proxy.", + "type": "object", + "required": [ + "domain" + ], + "properties": { + "domain": { + "description": "A list of domains that can be served. These are used for canister resolution [default: localhost]", + "allOf": [ + { + "$ref": "#/definitions/SerdeVec_for_String" + } + ] + } + } + }, "ConfigDefaultsReplica": { "title": "Local Replica Configuration", "type": "object", @@ -735,6 +762,16 @@ } ] }, + "proxy": { + "anyOf": [ + { + "$ref": "#/definitions/ConfigDefaultsProxy" + }, + { + "type": "null" + } + ] + }, "replica": { "anyOf": [ { diff --git a/docs/networks-json-schema.json b/docs/networks-json-schema.json index c5d7ef47c7..dd7a239ef3 100644 --- a/docs/networks-json-schema.json +++ b/docs/networks-json-schema.json @@ -105,6 +105,23 @@ } } }, + "ConfigDefaultsProxy": { + "description": "Configuration for icx-proxy.", + "type": "object", + "required": [ + "domain" + ], + "properties": { + "domain": { + "description": "A list of domains that can be served. These are used for canister resolution [default: localhost]", + "allOf": [ + { + "$ref": "#/definitions/SerdeVec_for_String" + } + ] + } + } + }, "ConfigDefaultsReplica": { "title": "Local Replica Configuration", "type": "object", @@ -194,6 +211,16 @@ } ] }, + "proxy": { + "anyOf": [ + { + "$ref": "#/definitions/ConfigDefaultsProxy" + }, + { + "type": "null" + } + ] + }, "replica": { "anyOf": [ { @@ -319,6 +346,19 @@ "application", "verifiedapplication" ] + }, + "SerdeVec_for_String": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] } } } \ No newline at end of file diff --git a/e2e/tests-dfx/start.bash b/e2e/tests-dfx/start.bash index 2ba61ab7b7..52a735bda6 100644 --- a/e2e/tests-dfx/start.bash +++ b/e2e/tests-dfx/start.bash @@ -12,6 +12,48 @@ teardown() { standard_teardown } +@test "icx-proxy domain configuration in string form" { + create_networks_json + jq '.local.proxy.domain="xyz.domain"' "$E2E_NETWORKS_JSON" | sponge "$E2E_NETWORKS_JSON" + + dfx_start + + assert_command ps aux + assert_match "icx-proxy.*--domain xyz.domain" +} + +@test "icx-proxy domain configuration in vector form" { + create_networks_json + jq '.local.proxy.domain=["xyz.domain", "abc.something"]' "$E2E_NETWORKS_JSON" | sponge "$E2E_NETWORKS_JSON" + + dfx_start + + assert_command ps aux + assert_match "icx-proxy.*--domain xyz.domain" + assert_match "icx-proxy.*--domain abc.something" +} + +@test "icx-proxy domain configuration from project defaults" { + dfx_new + define_project_network + + jq '.defaults.proxy.domain=["xyz.domain", "abc.something"]' dfx.json | sponge dfx.json + + dfx_start + + assert_command ps aux + assert_match "icx-proxy.*--domain xyz.domain" + assert_match "icx-proxy.*--domain abc.something" +} + +@test "icx-proxy domain configuration from command-line" { + dfx_start --domain xyz.domain --domain def.somewhere + + assert_command ps aux + assert_match "icx-proxy.*--domain xyz.domain" + assert_match "icx-proxy.*--domain def.somewhere" +} + @test "dfx restarts the replica" { dfx_new hello dfx_start diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index 17c8b1d8a7..f4ffd78cc6 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -57,6 +57,7 @@ const EMPTY_CONFIG_DEFAULTS: ConfigDefaults = ConfigDefaults { bootstrap: None, build: None, canister_http: None, + proxy: None, replica: None, }; @@ -555,6 +556,13 @@ pub struct ConfigDefaultsReplica { pub log_level: Option, } +/// Configuration for icx-proxy. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ConfigDefaultsProxy { + /// A list of domains that can be served. These are used for canister resolution [default: localhost] + pub domain: SerdeVec, +} + // Schemars doesn't add the enum value's docstrings. Therefore the explanations have to be up here. /// # Network Type /// Type 'ephemeral' is used for networks that are regularly reset. @@ -643,6 +651,7 @@ pub struct ConfigLocalProvider { pub canister_http: Option, pub replica: Option, pub playground: Option, + pub proxy: Option, } #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] @@ -667,6 +676,7 @@ pub struct ConfigDefaults { pub bootstrap: Option, pub build: Option, pub canister_http: Option, + pub proxy: Option, pub replica: Option, } diff --git a/src/dfx-core/src/config/model/local_server_descriptor.rs b/src/dfx-core/src/config/model/local_server_descriptor.rs index ff4403247c..6d59829cfa 100644 --- a/src/dfx-core/src/config/model/local_server_descriptor.rs +++ b/src/dfx-core/src/config/model/local_server_descriptor.rs @@ -1,12 +1,14 @@ use crate::config::model::bitcoin_adapter; use crate::config::model::canister_http_adapter::HttpAdapterLogLevel; use crate::config::model::dfinity::{ - to_socket_addr, ConfigDefaultsBitcoin, ConfigDefaultsCanisterHttp, ConfigDefaultsReplica, - ReplicaLogLevel, ReplicaSubnetType, DEFAULT_PROJECT_LOCAL_BIND, DEFAULT_SHARED_LOCAL_BIND, + to_socket_addr, ConfigDefaultsBitcoin, ConfigDefaultsCanisterHttp, ConfigDefaultsProxy, + ConfigDefaultsReplica, ReplicaLogLevel, ReplicaSubnetType, DEFAULT_PROJECT_LOCAL_BIND, + DEFAULT_SHARED_LOCAL_BIND, }; use crate::error::network_config::{ NetworkConfigError, NetworkConfigError::ParseBindAddressFailed, }; +use crate::json::structure::SerdeVec; use slog::{debug, info, Logger}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -30,6 +32,7 @@ pub struct LocalServerDescriptor { pub bitcoin: ConfigDefaultsBitcoin, pub canister_http: ConfigDefaultsCanisterHttp, + pub proxy: ConfigDefaultsProxy, pub replica: ConfigDefaultsReplica, pub scope: LocalNetworkScopeDescriptor, @@ -51,6 +54,7 @@ impl LocalServerDescriptor { bind: String, bitcoin: ConfigDefaultsBitcoin, canister_http: ConfigDefaultsCanisterHttp, + proxy: ConfigDefaultsProxy, replica: ConfigDefaultsReplica, scope: LocalNetworkScopeDescriptor, legacy_pid_path: Option, @@ -61,6 +65,7 @@ impl LocalServerDescriptor { bind_address, bitcoin, canister_http, + proxy, replica, scope, legacy_pid_path, @@ -199,6 +204,13 @@ impl LocalServerDescriptor { }; Self { bitcoin, ..self } } + + pub fn with_proxy_domains(self, domains: Vec) -> LocalServerDescriptor { + let proxy = ConfigDefaultsProxy { + domain: SerdeVec::Many(domains), + }; + Self { proxy, ..self } + } } impl LocalServerDescriptor { diff --git a/src/dfx-core/src/json/structure.rs b/src/dfx-core/src/json/structure.rs index 8bfbd71e62..c838f0eacc 100644 --- a/src/dfx-core/src/json/structure.rs +++ b/src/dfx-core/src/json/structure.rs @@ -4,7 +4,7 @@ use serde::Serialize; use std::fmt::Display; use std::str::FromStr; -#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, PartialEq, Eq)] #[serde(untagged)] pub enum SerdeVec { One(T), diff --git a/src/dfx-core/src/network/provider.rs b/src/dfx-core/src/network/provider.rs index 4d39284ab0..6c12ec4efd 100644 --- a/src/dfx-core/src/network/provider.rs +++ b/src/dfx-core/src/network/provider.rs @@ -96,6 +96,11 @@ fn config_network_to_network_descriptor( .clone() .or_else(|| project_defaults.and_then(|x| x.canister_http.clone())) .unwrap_or_default(); + let proxy = local_provider + .proxy + .clone() + .or_else(|| project_defaults.and_then(|x| x.proxy.clone())) + .unwrap_or_default(); let replica = local_provider .replica .clone() @@ -121,6 +126,7 @@ fn config_network_to_network_descriptor( bind_address, bitcoin, canister_http, + proxy, replica, local_scope, legacy_pid_path, @@ -237,6 +243,7 @@ fn create_shared_network_descriptor( canister_http: None, replica: None, playground: None, + proxy: None, })) } (network_name, None) => { diff --git a/src/dfx/src/actors/icx_proxy.rs b/src/dfx/src/actors/icx_proxy.rs index 74e6aa2ba7..916577767d 100644 --- a/src/dfx/src/actors/icx_proxy.rs +++ b/src/dfx/src/actors/icx_proxy.rs @@ -42,6 +42,9 @@ pub struct IcxProxyConfig { /// run icx-proxy in non-quiet mode pub verbose: bool, + + /// list of domains that can be served (localhost if none specified) + pub domains: Vec, } /// The configuration for the icx_proxy actor. @@ -96,6 +99,7 @@ impl IcxProxy { receiver, fetch_root_key, config.verbose, + config.domains.clone(), ), "Failed to start ICX proxy thread.", )?; @@ -189,6 +193,7 @@ fn icx_proxy_start_thread( receiver: Receiver<()>, fetch_root_key: bool, verbose: bool, + domains: Vec, ) -> DfxResult> { let thread_handler = move || { // Start the process, then wait for the file. @@ -205,6 +210,9 @@ fn icx_proxy_start_thread( let s = format!("{}", url); cmd.args(["--replica", &s]); } + for domain in domains { + cmd.args(["--domain", &domain]); + } if !verbose { cmd.arg("-q"); } diff --git a/src/dfx/src/commands/start.rs b/src/dfx/src/commands/start.rs index 03515f5740..1fdcd36230 100644 --- a/src/dfx/src/commands/start.rs +++ b/src/dfx/src/commands/start.rs @@ -80,6 +80,10 @@ pub struct StartOpts { /// Use old metering. #[arg(long)] use_old_metering: bool, + + /// A list of domains that can be served. These are used for canister resolution [default: localhost] + #[arg(long)] + domain: Vec, } // The frontend webserver is brought up by the bg process; thus, the fg process @@ -146,6 +150,7 @@ pub fn exec( enable_canister_http, artificial_delay, use_old_metering, + domain, }: StartOpts, ) -> DfxResult { if !background { @@ -179,6 +184,7 @@ pub fn exec( bitcoin_node, enable_canister_http, emulator, + domain, )?; let local_server_descriptor = network_descriptor.local_server_descriptor()?; @@ -292,6 +298,8 @@ pub fn exec( .log_level .unwrap_or_default(); + let proxy_domains = local_server_descriptor.proxy.domain.clone().into_vec(); + let replica_config = { let replica_config = ReplicaConfig::new( &state_root, @@ -388,6 +396,7 @@ pub fn exec( bind: address_and_port, replica_urls: vec![], // will be determined after replica starts fetch_root_key: !network_descriptor.is_ic, + domains: proxy_domains, verbose: env.get_verbose_level() > 0, }; @@ -453,6 +462,7 @@ pub fn apply_command_line_parameters( bitcoin_nodes: Vec, enable_canister_http: bool, emulator: bool, + domain: Vec, ) -> DfxResult { if enable_canister_http { warn!( @@ -492,6 +502,10 @@ pub fn apply_command_line_parameters( local_server_descriptor = local_server_descriptor.with_bitcoin_nodes(bitcoin_nodes) } + if !domain.is_empty() { + local_server_descriptor = local_server_descriptor.with_proxy_domains(domain) + } + Ok(NetworkDescriptor { local_server_descriptor: Some(local_server_descriptor), ..network_descriptor From 54bbe174e66626eb4b554eb98a91a0022e5074ee Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Thu, 7 Dec 2023 13:32:22 -0800 Subject: [PATCH 2/3] docs: remove description of manual steps from release doc (#3474) This PR deletes the "MANUAL STEPS" release process description. It's inaccurate and we have not used it in at least two years. The only other change is to move the part about whitelisting the frontend canister with the Motoko playground. There were actually two places that described this, both found in the "MANUAL STEPS" section. --- docs/process/release.adoc | 264 ++------------------------------------ 1 file changed, 11 insertions(+), 253 deletions(-) diff --git a/docs/process/release.adoc b/docs/process/release.adoc index 6c61af5f0e..c3e0d85991 100644 --- a/docs/process/release.adoc +++ b/docs/process/release.adoc @@ -99,12 +99,8 @@ git checkout release- && git pull === Ready At this point, you are ready to build a release candidate. -There are two ways you can build a release candidate: +This is a script-based process. -- Using the <> to automate the build process and skip manual testing. -- Using the <> to build and test the release. - -[[script]] == SCRIPT-BASED release process To use the release script to automate building the release candidate: @@ -125,254 +121,6 @@ For example, if releasing 0.7.0, you would run the following command: After running this command to build the release candidate, follow the steps in <> to complete the release process. -[[manual]] -== MANUAL STEPS release process - -The manual release process provides full instructions for building and testing the release candidate binaries to ensure everything is working before making a release available to internal or external developers. - -=== Build DFX - -Verify that you can build DFX from the `+master+` branch without errors. - -. Verify you're connected to VPN, if necessary. -. Build the `dfx` binary by running the following command: -+ -[source, bash] ----- -cargo clean --release -cargo build --release --locked -export dfx_rc="$(pwd)/target/release/dfx" ----- -+ -The `nix-build` command can take a while to complete. -Wait for it to be done. -These commands create the binary then stores the binary in a shell variable. -. Make sure the `$dfx_rc` variable points to a real file by running the following command: -+ -``` -test -x $dfx_rc \ - && echo 'Please proceed.' \ - || echo 'Cant find executable $dfx_rc'="$dfx_rc" -``` -+ -You should see 'Please proceed' returned. -. Delete the existing `dfx` cache to ensure you're not using a stale binary. -+ -[source, bash] ----- -$dfx_rc cache delete ----- -. Ensure `dfx` and `replica` are not running in the background by running the following command: -+ -[source, bash] ----- -ps -ef | grep -E 'replica|dfx' | grep -v grep ----- -+ -If there are any `replica` or `dfx` processes running, use the `kill` command to terminate them. - -=== Validate the default project - -Verify that you can build, deploy, and call the default `hello_world` project without errors. - -. Generate a default new project and change to the project directory by running the following commands: -+ -[source, bash] ----- -$dfx_rc new hello_world -cd hello_world ----- -. Start the local `replica` as a background process by running the following command: -+ -[source, bash] ----- -$dfx_rc start --clean --background ----- -. Create, build, and install canisters by running the following command: -+ -[source, bash] ----- -$dfx_rc deploy ----- -. Call the canister and verify the result by running the following command: -+ -[source, bash] ----- -$dfx_rc canister call hello_world greet everyone ----- -. Save the canister URLs as shell variables, then print them by running the following commands: -+ -[source, bash] ----- -export hello_world_backend_candid_url="http://localhost:4943/candid?canisterId=$($dfx_rc canister id hello_world_backend)" -export hello_world_frontend_url="http://localhost:4943/?canisterId=$($dfx_rc canister id hello_world_frontend)" ----- -. Open a web browser and clear your cache or switch to Private Browsing/Incognito mode. -. Open the following URL in your web browser: -+ -[source,bash] ----- -echo "$hello_world_frontend_url" ----- -+ -. Verify that you are prompted to type a greeting in a prompt window. -.. Type a greeting, then click *OK* to return the greeting in an alert window. -.. Verify there are no errors in the console by opening the Developer Tools. -+ -For example, in the browser, right-click, then click Inspect and select Console to check for errors and warnings. -Warnings can be ignored. - -. Verify the Candid UI by opening the following URL in your web browser: -+ -[source, bash] ----- -echo "$hello_world_backend_candid_url" ----- -+ -.. Verify UI loads, then test the greet function by entering text and clicking *Call* or clicking *Lucky*, -.. Verify there are no errors in the console by opening the Developer Tools. -+ -For example, in the browser, right-click, then click Inspect and select Console to check for errors and warnings. -Warnings can be ignored. -. Stop the replica by running the following command: -+ -[source, bash] ----- -$dfx_rc stop ----- -. Delete the test project by running the following commands: -+ -[source, bash] ----- -cd .. -rm -rf hello_world ----- - - -=== Whitelist asset canister in Motoko Playground - -If the release includes a new version of the asset canister, then the Motoko Playground needs to have the new asset canister WASM whitelisted. -Otherwise `dfx deploy --playground` will not work for asset canisters. - -Find the new asset canister module hash. It will be listed in `CHANGELOG.md` under `` - `Dependencies` - `Frontend canister`. - -Head over to the link:https://github.com/dfinity/motoko-playground[Motoko Playground repo] and create a PR that adds the asset canister module hash to the whitelist in `service/wasm-utils/lib.rs`. -You can use link:https://github.com/dfinity/motoko-playground/pull/175/files#diff-c8a035da9dcede5539deb0e81164ea50730e3177f56aef747d157406b1ba648dR15-R17[this change] as an example. - -==== Update the version - -. Set the new version in a temporary environment variable. -+ -For example, replace `` with a specific version number: -+ -[source, nix-shell] ----- -export NEW_DFX_VERSION= ----- -. If you're not already there, navigate back to the top-level of the `sdk` repo. -. Enter the sdk `nix` development environment by running the following command: -+ -[source, bash] ----- -nix-shell --option extra-binary-caches https://cache.dfinity.systems ----- -. Create a new branch for your changes by running the following command: -+ -[source, nix-shell] ----- -git switch -c $USER/release-$NEW_DFX_VERSION ----- -. Update the first `version` field in `src/dfx/Cargo.toml` to be equal to `$NEW_DFX_VERSION` -. Apply these changes to `Cargo.lock` by running the following command: -+ -[source, nix-shell] ----- -cargo build ----- -. Append the new version to `public/manifest.json` by appending it to the `versions` list. -+ -For example: -+ -[source, json] ----- -{ - "tags": { - "latest": "0.6.0" - }, - "versions": [ - "0.5.15", - "0.6.0", - "n.n.n" - ] -} ----- -+ -*Ensure* `tags.latest` remains the same. -. Exit `nix-shell` to continue. - -==== Create a pull request and tag - -. Create a pull request with the above changes by running the following commands: -+ -[source, bash] ----- -git add --all -git commit --signoff --message "chore: Release $NEW_DFX_VERSION" -git push origin $USER/release-$NEW_DFX_VERSION ----- -. After pushing, click the link in the console to go to your new branch in GitHub, then click *Create Pull Request*. Change the base branch to `release-`. -. Have the validator review and approve the PR. -. Merge the PR manually (the automerge-squash label only works for PRs to the master branch). -+ -NOTE: Depending on the number of jobs queued up, this step can take 45 to 60 minutes to complete. - -. Switch to the release branch by running the following command: -+ -[source, bash] ----- -git switch release-$NEW_DFX_VERSION ----- -. Set the upstream tracking information for the release branch: -+ -[source, bash] ----- -git branch --set-upstream-to=origin/$NEW_DFX_VERSION $NEW_DFX_VERSION ----- -. Update the release branch: -+ -[source, bash] ----- -git pull ----- -. Create a new tag by running the following command: -+ -[source, bash] ----- -git tag --annotate $NEW_DFX_VERSION --message "Release: $NEW_DFX_VERSION" ----- -. Verify the tag points to the correct version and includes annotation by running the following commands: -+ -[source, bash] ----- -git log -git describe --always ----- -. Push the tag by running the following command: -+ -[source, bash] ----- -git push origin $NEW_DFX_VERSION ----- -+ -The link:../../.github/workflows/publish.yml[publish.yml GitHub workflow] will build the release -and upload to GitHub releases after you push the tag. - -==== Add new frontend canister hash to list of WHITELISTED_WASMS in dfinity/motoko-playground repo -You can do it either by using GitHub UI (https://github.com/dfinity/sdk/actions/workflows/broadcast-frontend-hash.yml) or by running the following command: -[source, bash] ----- -gh workflow run "broadcast-frontend-hash.yml" -f dfx_version= ----- [[post-build]] == Notification and post-build validation @@ -391,6 +139,16 @@ If a release is known to have changes that are not backward-compatible, create a Depending on the change, the notification might need to be posted in more than one topic channel. For example, changes to the external network for onboarded developers are currently posted in link:https://forum.dfinity.org/t/network-status-updates/928[Network status and updates] on the link:https://forum.dfinity.org[DFINITY Developer Forum]. +== Add new frontend canister hash to list of WHITELISTED_WASMS in dfinity/motoko-playground repo + +Once the release is ready to promote, you need to register the new frontent canister hash with the Motoko Playground repo. + +You can do it either by using GitHub UI (https://github.com/dfinity/sdk/actions/workflows/broadcast-frontend-hash.yml) or by running the following command: +[source, bash] +---- +gh workflow run "broadcast-frontend-hash.yml" -f dfx_version= +---- + == Promote a release candidate to production . Verify that release notes and documentation are ready for public consumption. From 09939b05b4fe980d7436d091d298928e19f6f7a2 Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Fri, 8 Dec 2023 08:47:24 -0800 Subject: [PATCH 3/3] docs: replace release process document (#3475) --- docs/process/release.adoc | 259 -------------------------------------- docs/process/release.md | 119 ++++++++++++++++++ 2 files changed, 119 insertions(+), 259 deletions(-) delete mode 100644 docs/process/release.adoc create mode 100644 docs/process/release.md diff --git a/docs/process/release.adoc b/docs/process/release.adoc deleted file mode 100644 index c3e0d85991..0000000000 --- a/docs/process/release.adoc +++ /dev/null @@ -1,259 +0,0 @@ -= Release process - -This document describes the release process for `dfx`, with step-by-step instructions, information about automation, and a checklist. - -== Overview - -Before starting the release process, the team should conduct a brief Go/No-Go release review to evaluate the current state of fixes and features ready to be included in a release candidate. -After reviewing the list of fixes and features, the team will decide whether to proceed with staging a build. - -For now, our release process is driven by the DFINITY foundation SDK team. Future work needs to be done to separate the build and release process to run fully on open systems. - -If the new release is given the Go green light, two people who are familiar with the process—a *driver* and a *validator*—use the steps in this document to stage or promote a release candidate. - -The *validator* should be the person most familiar with the process and be able to assist with debugging or resolving issues if the *driver* building the release runs into trouble. - -A successful release is the result of coordination between automation, manual steps performed by team members, and a validation process. -Our goal is to update this document with the latest information as we iterate to improve the release process. - -=== Participants -- Driver / Primary -- Validator / Secondary - -=== Prerequisites - -Before you begin, verify the following prerequisites for the release process *driver*: - -- You must have a GitHub account and access to the `+dfinity+` organization and core repositories like the `sdk` repository -+ -For information about setting up a GitHub account, see link:https://docs.github.com/en/github/getting-started-with-github/signing-up-for-a-new-github-account[Signin up for a GitHub account] -To get permission for your account to access `+dfinity-lab+` and `+dfinity+` repositories, see link:https://www.notion.so/How-to-Get-Github-Access-68c9ad72b5974fa9bbec003592677d02[How to get GitHub access). -If you run into any issues accessing repositories, you can contact IT using the `#help-it` Slack channel and sending a message to `@firstresponder`. - -- You must have VPN access to the DFINITY VPN. -+ -If you don't have a VPN connection set up for your computer, you'll need to follow the instructions in link:https://www.notion.so/How-to-get-VPN-access-to-our-data-center-services-1c9b123152d740508eec25e7ac982259[How to get VPN access to our data center services]. - -- You must have link:https://github.com/dfinity-lab/dfinity/blob/master/CONTRIBUTING.adoc#install-and-configure-nix[Nix installed and configured]. -+ -NOTE: If you are installing Nix for the first time, building the cache can take hours and might timeout depending on the reliability of your network connection. - -- You must have a registered NPM account and `npm` installed on your system. -+ -If you don't have an account, you can go to the link:https://www.npmjs.com/[NPMJS] website and click *Sign Up* to create one using your `firstname.lastname@dfinity.org` email address. -+ -You will need to verify your email address to complete the registration process. -For your account to be added to the `dfinity` organization, contact IT - -=== Preliminary validation - -Verify the general stability of the master branch before attempting to create a release candidate. - -. Use this link:https://github.com/dfinity-lab/sdk/commits/master[link] -to verify: -.. Is `master` green? -.. Was `master` red recently or flaky? -+ - -image:is-master-green.png[] - -=== Preparation - -. Connect to VPN. -. Open a terminal and `cd` into your local copy of the `sdk` repo. - -=== Creating a New Release Branch - -. Check out the `master` branch and pull merged commits from the remote in your working directory by running the following command: -+ -[source, bash] ----- -git checkout master && git pull ----- - -. Create the release branch. Note that the branch name never includes alpha, beta, and so forth. All of these will be released on the same release branch. -+ -[source, bash] ----- -git switch -c release- && git push ----- -+ -For example, if creating the release branch for 0.7.0, you would run the following command: -+ -[source, bash] ----- -git switch -c release-0.7.0 && git push ----- - -. Edit CHANGELOG.md to remove the "UNRELEASED" note from the version to be released. Commit this to change to the release branch. - -=== Resuming on an Existing Release Branch - -. Check out the `master` branch and pull merged commits from the remote in your working directory by running the following command: -+ -[source, bash] ----- -git checkout release- && git pull ----- - -=== Ready - -At this point, you are ready to build a release candidate. -This is a script-based process. - -== SCRIPT-BASED release process - -To use the release script to automate building the release candidate: - -. Run the following command and substitute `` with the version number for this release candidate: -+ -[source, bash] ----- -./scripts/release.sh ----- -+ -For example, if releasing 0.7.0, you would run the following command: -+ -.... -./scripts/release.sh 0.7.0 -.... -. Follow the prompts displayed to complete the release process. - -After running this command to build the release candidate, follow the steps in <> to complete the release process. - - -[[post-build]] -== Notification and post-build validation - -. Wait for the publish workflow to complete. -. Install the build using the `DFX_VERSION=` environment variable. -. Run through the link:https://sdk.dfinity.org/docs/quickstart/local-quickstart.html[_Quick start - Local development_] steps. -. Run through link:https://sdk.dfinity.org/docs/quickstart/network-quickstart.html#ping-the-network[Check the connection to the network] and link:https://sdk.dfinity.org/docs/quickstart/network-quickstart.html#net-deploy[Register, build, and deploy the application] steps to deploy to the network. -. Notify link:https://app.slack.com/client/T43F9UHS5/CGA566TPV[#eng-sdk] team members that the new build is ready for manual installation and testing. -+ -Remind the SDK and Apps teams to add information about _features and fixes_ for release notes to their issues or PRs and to apply the changelog label to have the information included in the release notes. -. Notify the link:https://forum.dfinity.org[Developer Forum] community if there are breaking changes. -+ -If a release is known to have changes that are not backward-compatible, create a forum post to describe the change and any instructions for migrating to the new release. -+ -Depending on the change, the notification might need to be posted in more than one topic channel. -For example, changes to the external network for onboarded developers are currently posted in link:https://forum.dfinity.org/t/network-status-updates/928[Network status and updates] on the link:https://forum.dfinity.org[DFINITY Developer Forum]. - -== Add new frontend canister hash to list of WHITELISTED_WASMS in dfinity/motoko-playground repo - -Once the release is ready to promote, you need to register the new frontent canister hash with the Motoko Playground repo. - -You can do it either by using GitHub UI (https://github.com/dfinity/sdk/actions/workflows/broadcast-frontend-hash.yml) or by running the following command: -[source, bash] ----- -gh workflow run "broadcast-frontend-hash.yml" -f dfx_version= ----- - -== Promote a release candidate to production - -. Verify that release notes and documentation are ready for public consumption. -. Open the `public/manifest.json` file in a text editor. -. Verify that `dfx deploy --playground` works with an asset canister by e.g. deploying the default project created by `dfx new`. -.. If it doesn't work, make sure the PR created on the Motoko Playground repo is merged and deployed. -. Under the `tags` key, change the version number associated with the `latest` key. -+ -For example: -+ -[source,json] ----- -{ - "tags": { - "latest": "n.n.n" - } -} ----- -. Prepare a PR for the manifest by running the following commands: -+ -[source, bash] ----- -git switch -c /update-n.n.n-latest -git add --all -git commit --message "chore: Update the manifest latest to n.n.n " -git push origin /update-n.n.n-latest ----- -. After pushing, click the link in the console to go to your new branch in GitHub, then click *Create Pull Request*. -. Have the validator review and approve the PR, then merge to `master`. -. Verify the Linux and Darwin (macOS) builds are available for download from \https://download.dfinity.systems/sdk/dfx/n.n.n/architecture/dfx-n.n.n.tar.gz. -+ -Linux—Replace _n.n.n_ with the new version number and _architecture_ with `x86_64-linux`. For example, the following link would download version 0.6.1 for Linux: -+ -\https://download.dfinity.systems/sdk/dfx/0.6.1/x86_64-linux/dfx-0.6.1.tar.gz[] -+ -Darwin (macOS)—Replace _n.n.n_ with the new version number and _architecture_ with `x86_64-darwin`. For example, the following link would download version 0.6.1 for macOS: -+ -\https://download.dfinity.systems/sdk/dfx/0.6.1/x86_64-darwin/dfx-0.6.1.tar.gz[] -+ -CI Hydra: -+ -link:https://hydra.dfinity.systems/jobset/dfinity-ci-build/sdk-release[] - -Add a description and publish the tag for the latest release -https://github.com/dfinity-lab/sdk/releases/new?tag=${NEW_DFX_VERSION} - -== Release documentation - -link:https://github.com/dfinity/docs[Documentation repo] - -. Tag the documentation using `git tag -a -m `. -. Publish the tag on the remote server using `git push origin `. -. Deploy updated documentation using Netlify. - -== Requirements and properties - - - Semi-automation - - Consistent delivery - - Validation - - Rollback - - Guardrails - - Flexibility - -== Build mechanism - -Our build process is described in the `release.nix` derivation. -The `release.nix` derivation mainly invokes the `dfx-release` derivation passing the annotated tag on HEAD (which happens right now to be the stable branch). -The `dfx-release` derivation builds the release binaries and files for each platform and generates a manifest for S3 that includes the tag name. -The release tag allows us to keep a directory structure with all past and upcoming releases in S3. - -== CI - -CI release-related operation is split into two jobsets: - - - Generation and publishing of 'install.sh' and 'manifest.json'. - - Tagging of a commit to release, building and publishing the necessary executables and files for supported platforms. - -== Manifest - -We utilize a manifest to indicate to users (and in particular to our installer and dfx executable) available and supported versions for download. -The manifest allows us to rollback a release or remove a release from the list of supported releases. -See link:../specification/version_management{outfilesuffix}[Version Management] for details on the format of the manifest. - -The manifest is generated when a patch is applied on master by the CI. - -== Installer - -The installer is generated when a patch is applied on the `master` branch by the CI. - -== Changelog - -A candidate changelog is generated automatically using the respective tool (under scripts directory). -Currently, the release notes are updated manually in github. - -== Publishing of artifacts - -We now summarize the release process. -Our first step is to ensure the proper and valid state of the `master` branch. -Next, we update `cargo` and the manifest accordingly. -We then create and push an annotated tag on the `stable` branch, generate the changelog. -The product and SDK team members can then inspect, clarify, and develop the changelog to ensure it is appropriate for public -consumption. -After ensuring the proper artifacts are available in S3, we can now publish them by updating the manifest. - -== TODOs and improvements -. version from the tag -. release stress tests -. valid json test for the manifest diff --git a/docs/process/release.md b/docs/process/release.md new file mode 100644 index 0000000000..2f99161f23 --- /dev/null +++ b/docs/process/release.md @@ -0,0 +1,119 @@ +# IC SDK Release Process + +## Overview + +1. Update the replica version +1. Create the release branch +1. Update the changelog on master +1. Create beta releases +1. Open a PR to update the Portal +1. Create the final release +1. Open a PR to promote the release +1. Open a PR to update the Motoko Playground allow-list +1. Promote the release +1. Post to the forum + +## Details + +### Update the Replica Version + +Before making a new release, try to update the replica to the latest version +by running the [update-replica] workflow. + +### Create the Release Branch + +Create a release branch from `master`, for example `release-0.15.3`. + +This branch will be used to create beta releases as well as the final release. + +### Update the changelog + +Open a PR to master. Roll the changelog by adding a new header for the +new dfx version underneath the "# Unreleased" header. Further changes to dfx +should be added under the "#Unreleased" header, unless they are ported to +the release branch. + +### Create Beta Releases + +1. Check out the release branch. +1. Run the release script, for example `./scripts/release.sh 0.15.3-beta.1` +1. Open a PR from the branch pushed by the previous to the release branch, +obtain CR approval, and merge the PR. + - The release script will wait for you to do this + - It will then create and push a tag + - This triggers the [publish][publish-workflow] workflow +1. Wait for the [publish][publish-workflow] workflow to create the GitHub release +from the last commit on the release branch. +1. Update the GitHub release + - Copy/paste the changelog section for the new version into the release notes + - Set the "Prerelease" flag +1. Announce the release to #eng-sdk + - Post a message like this, linking to the GitHub release notes: + > dfx 0.15.3-beta.1 is available for manual installation and testing. + > + > ```bash + > DFX_VERSION=0.15.3-beta.1 sh -ci "$(curl -fsSL https://internetcomputer.org/install.sh)" + > ``` + > + > See also release notes. +1. Repeat the above steps until ready to promote the latest beta. + +### Open a Draft PR to update the Portal + +You can do this step while the beta releases are being tested. + +- Add a link to the [release-notes-table] +- Update the sdk submodule + +Obtain approval, but do not merge the PR yet. + +This PR is a draft in order to help remind the reviewer not to merge it. + +### Create the Final Release + +Once the beta releases are ready to be promoted: + +1. Check out the release branch +2. Run the release script, for example `./scripts/release.sh 0.15.3` +3. Follow the same steps as for the beta releases + +### Open a PR to promote the release + +1. Create a new branch from the release branch, for example `release-0.15.3-promote`. +1. Update the [version manifest][public-manifest]: + - Set `.tags.latest` to the new dfx version + - Remove the beta releases from the `versions` array +1. Open a PR from this branch to master + +Obtain approval, but do not merge this PR yet. + +### Open a PR to Update the Motoko Playground allow-list + +You can do it either by using GitHub UI ([broadcast-frontend-hash-workflow]) +or by running the following command: + +```bash +gh workflow run "broadcast-frontend-hash.yml" -f dfx_version= +``` + +Obtain approval, but do not merge this PR yet. + +### Promote the release + +You should now have three open, approved PRs: + +- Update the portal +- Promote the release +- Update the Motoko Playground allow-list + +Merge all three PRs. + +### Post to the forum + +Post a message to the forum, linking to the GitHub release notes. + +[broadcast-frontend-hash-workflow]: https://github.com/dfinity/sdk/actions/workflows/broadcast-frontend-hash.yml +[public-manifest]: https://github.com/dfinity/sdk/blob/master/public/manifest.json +[publish-workflow]: https://github.com/dfinity/sdk/blob/master/.github/workflows/publish.yml +[release-notes-table]: https://github.com/dfinity/portal/blob/master/docs/other/updates/release-notes/release-notes.md +[update-replica]: https://github.com/dfinity/sdk/actions/workflows/update-replica-version.yml