diff --git a/core/upgrader/api/spec.did b/core/upgrader/api/spec.did index d66a786b5..655098bbf 100644 --- a/core/upgrader/api/spec.did +++ b/core/upgrader/api/spec.did @@ -134,6 +134,8 @@ type RequestDisasterRecoveryInput = record { arg : blob; // The install mode: Install, Upgrade, or Reinstall. install_mode : InstallMode; + // Should the station be forced to stop if it is unstoppable. + force : bool; }; type InstallMode = variant { diff --git a/core/upgrader/api/src/lib.rs b/core/upgrader/api/src/lib.rs index 7874f3d2a..398b7fc55 100644 --- a/core/upgrader/api/src/lib.rs +++ b/core/upgrader/api/src/lib.rs @@ -113,6 +113,8 @@ pub struct RequestDisasterRecoveryInput { pub arg: Vec, pub install_mode: InstallMode, + + pub force: bool, } #[derive(CandidType, Deserialize, Debug, Clone)] diff --git a/core/upgrader/impl/src/model/disaster_recovery.rs b/core/upgrader/impl/src/model/disaster_recovery.rs index ee66cf0b9..b92c863e4 100644 --- a/core/upgrader/impl/src/model/disaster_recovery.rs +++ b/core/upgrader/impl/src/model/disaster_recovery.rs @@ -81,6 +81,8 @@ pub struct StationRecoveryRequest { pub arg_sha256: Vec, /// Time in nanoseconds since the UNIX epoch when the request was submitted. pub submitted_at: Timestamp, + /// Should the station be forced to stop if the station is unstoppable. + pub force: bool, } impl From for upgrader_api::StationRecoveryRequest { diff --git a/core/upgrader/impl/src/services/disaster_recovery.rs b/core/upgrader/impl/src/services/disaster_recovery.rs index 8a81f35c1..831ec4951 100644 --- a/core/upgrader/impl/src/services/disaster_recovery.rs +++ b/core/upgrader/impl/src/services/disaster_recovery.rs @@ -10,15 +10,18 @@ use crate::{ upgrader_ic_cdk::{api::time, spawn}, }; use candid::Principal; +use ic_cdk::api::management_canister::main::{ + list_canister_snapshots, load_canister_snapshot, take_canister_snapshot, uninstall_code, + CanisterIdRecord, LoadCanisterSnapshotArgs, TakeCanisterSnapshotArgs, +}; use ic_stable_structures::memory_manager::MemoryId; use lazy_static::lazy_static; use orbit_essentials::{api::ServiceResult, utils::sha256_hash}; use crate::{ model::{ - Account, AdminUser, DisasterRecovery, DisasterRecoveryCommittee, InstallMode, - RecoveryEvaluationResult, RecoveryFailure, RecoveryResult, RecoveryStatus, - StationRecoveryRequest, + Account, AdminUser, DisasterRecovery, DisasterRecoveryCommittee, RecoveryEvaluationResult, + RecoveryFailure, RecoveryResult, RecoveryStatus, StationRecoveryRequest, }, StableValue, MEMORY_ID_DISASTER_RECOVERY, MEMORY_MANAGER, TARGET_CANISTER_ID, }; @@ -55,25 +58,18 @@ pub struct DisasterRecoveryReleaser { impl Drop for DisasterRecoveryReleaser { fn drop(&mut self) { - let mut value = self.storage.get(); - let last_recovery_result = self.result .take() .unwrap_or(RecoveryResult::Failure(RecoveryFailure { reason: "Recovery failed with unknown error".to_string(), })); - - value.last_recovery_result = Some(last_recovery_result.clone()); - value.recovery_status = RecoveryStatus::Idle; - self.logger.log(LogEntryType::DisasterRecoveryResult( DisasterRecoveryResultLog { - result: last_recovery_result, + result: last_recovery_result.clone(), }, )); - - self.storage.set(value); + self.storage.set_result(last_recovery_result); } } @@ -81,13 +77,20 @@ impl Drop for DisasterRecoveryReleaser { pub struct DisasterRecoveryStorage {} impl DisasterRecoveryStorage { - pub fn get(&self) -> DisasterRecovery { + fn get(&self) -> DisasterRecovery { STORAGE.with(|storage| storage.borrow().get(&()).unwrap_or_default()) } fn set(&self, value: DisasterRecovery) { STORAGE.with(|storage| storage.borrow_mut().insert((), value)); } + + fn set_result(&self, result: RecoveryResult) { + let mut value = self.get(); + value.last_recovery_result = Some(result); + value.recovery_status = RecoveryStatus::Idle; + self.set(value); + } } #[derive(Clone)] @@ -233,7 +236,7 @@ impl DisasterRecoveryService { installer: Arc, logger: Arc, request: StationRecoveryRequest, - ) { + ) -> Result<(), String> { let mut value = storage.get(); logger.log(LogEntryType::DisasterRecoveryStart( @@ -251,40 +254,62 @@ impl DisasterRecoveryService { if since + DISASTER_RECOVERY_IN_PROGESS_EXPIRATION_NS > time() { logger.log(LogEntryType::DisasterRecoveryInProgress(log)); - return; + return Ok(()); } logger.log(LogEntryType::DisasterRecoveryInProgressExpired(log)); value.recovery_status = RecoveryStatus::Idle; } - let Some(station_canister_id) = - TARGET_CANISTER_ID.with(|id| id.borrow().get(&()).map(|id| id.0)) - else { - value.last_recovery_result = Some(RecoveryResult::Failure(RecoveryFailure { - reason: "Station canister ID not set".to_string(), - })); - storage.set(value); - return; - }; - value.recovery_status = RecoveryStatus::InProgress { since: time() }; storage.set(value); + let station_canister_id = TARGET_CANISTER_ID + .with(|id| id.borrow().get(&()).map(|id| id.0)) + .ok_or("Station canister ID not set")?; + let mut releaser = DisasterRecoveryReleaser { storage: storage.clone(), result: None, logger: logger.clone(), }; - // only stop for upgrade - if request.install_mode == InstallMode::Upgrade { - if let Err(err) = installer.stop(station_canister_id).await { - ic_cdk::print(err); + if let Err(err) = installer.stop(station_canister_id).await { + if request.force { + let existing_snapshots = list_canister_snapshots(CanisterIdRecord { + canister_id: station_canister_id, + }) + .await + .map_err(|(_code, msg)| msg)? + .0; + let snapshot = take_canister_snapshot(TakeCanisterSnapshotArgs { + canister_id: station_canister_id, + replace_snapshot: existing_snapshots + .into_iter() + .next() + .map(|snapshot| snapshot.id), + }) + .await + .map_err(|(_code, msg)| msg)? + .0; + uninstall_code(CanisterIdRecord { + canister_id: station_canister_id, + }) + .await + .map_err(|(_code, msg)| msg)?; + load_canister_snapshot(LoadCanisterSnapshotArgs { + canister_id: station_canister_id, + snapshot_id: snapshot.id, + sender_canister_version: None, + }) + .await + .map_err(|(_code, msg)| msg)?; + } else { + return Err(err); } } - match installer + installer .install( station_canister_id, request.wasm_module, @@ -292,22 +317,12 @@ impl DisasterRecoveryService { request.arg, request.install_mode, ) - .await - { - Ok(_) => { - releaser.result = Some(RecoveryResult::Success); - } - Err(reason) => { - releaser.result = Some(RecoveryResult::Failure(RecoveryFailure { reason })); - } - } + .await?; - // only start for upgrade - if request.install_mode == InstallMode::Upgrade { - if let Err(err) = installer.start(station_canister_id).await { - ic_cdk::print(err); - } - } + installer.start(station_canister_id).await?; + + releaser.result = Some(RecoveryResult::Success); + Ok(()) } pub fn request_recovery( @@ -332,6 +347,7 @@ impl DisasterRecoveryService { arg: request.arg, submitted_at: time(), install_mode: request.install_mode.into(), + force: request.force, }; // check if user had previous recovery request @@ -365,7 +381,11 @@ impl DisasterRecoveryService { let logger = self.logger.clone(); spawn(async move { - Self::do_recovery(storage, installer, logger, *request).await; + if let Err(reason) = + Self::do_recovery(storage.clone(), installer, logger, *request).await + { + storage.set_result(RecoveryResult::Failure(RecoveryFailure { reason })); + } }); } } @@ -489,6 +509,7 @@ mod test { module: vec![4, 5, 6], module_extra_chunks: None, install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }, ); assert!(dr.storage.get().recovery_requests.is_empty()); @@ -501,6 +522,7 @@ mod test { module: vec![4, 5, 6], module_extra_chunks: None, install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }, ); @@ -517,6 +539,7 @@ mod test { module: vec![4, 5, 6], module_extra_chunks: None, install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }, ); @@ -531,6 +554,7 @@ mod test { module: vec![4, 5, 6], module_extra_chunks: None, install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }, ); @@ -567,6 +591,7 @@ mod test { arg: vec![7, 8, 9], arg_sha256: vec![10, 11, 12], submitted_at: 0, + force: false, }; // assert that during install the state is set to InProgress @@ -587,7 +612,8 @@ mod test { logger.clone(), recovery_request.clone(), ) - .await; + .await + .unwrap(); // calls install in Idle state assert_eq!( @@ -620,7 +646,8 @@ mod test { logger.clone(), recovery_request.clone(), ) - .await; + .await + .unwrap(); // does not call install in InProgress state assert_eq!( @@ -646,6 +673,7 @@ mod test { arg: vec![7, 8, 9], arg_sha256: vec![10, 11, 12], submitted_at: 0, + force: false, }; let installer = Arc::new(TestInstaller::default()); @@ -656,17 +684,8 @@ mod test { logger.clone(), recovery_request.clone(), ) - .await; - - assert!(matches!( - storage.get().last_recovery_result, - Some(RecoveryResult::Failure(_)) - )); - - assert!(matches!( - storage.get().recovery_status, - RecoveryStatus::Idle - )); + .await + .unwrap_err(); } #[tokio::test] @@ -687,6 +706,7 @@ mod test { arg: vec![7, 8, 9], arg_sha256: vec![10, 11, 12], submitted_at: 0, + force: false, }; let installer = Arc::new(PanickingTestInstaller::default()); @@ -700,7 +720,8 @@ mod test { logger.clone(), recovery_request.clone(), ) - .await; + .await + .unwrap(); // reset the hook let _ = take_hook(); diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index 5a15ab1a9..7e2dfe84d 100755 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -33,8 +33,8 @@ echo "PocketIC download starting" curl -sLO https://github.com/dfinity/pocketic/releases/download/7.0.0/pocket-ic-x86_64-$PLATFORM.gz || exit 1 gzip -df pocket-ic-x86_64-$PLATFORM.gz mv pocket-ic-x86_64-$PLATFORM pocket-ic -export POCKET_IC_BIN="$(pwd)/pocket-ic" chmod +x pocket-ic +export POCKET_IC_BIN="$(pwd)/pocket-ic" echo "PocketIC download completed" cd ../.. diff --git a/tests/canister/api/spec.did b/tests/canister/api/spec.did index cf72c6cd9..f61bcc395 100644 --- a/tests/canister/api/spec.did +++ b/tests/canister/api/spec.did @@ -12,4 +12,6 @@ service : { validate_number : (input : StoreNumberInput) -> (ValidationResponse); store_number : (input : StoreNumberInput) -> (); get_number : () -> (nat64) query; + noop : () -> (); + expensive : () -> (); }; diff --git a/tests/canister/impl/src/lib.rs b/tests/canister/impl/src/lib.rs index 3f372e0ca..bbf694a4b 100644 --- a/tests/canister/impl/src/lib.rs +++ b/tests/canister/impl/src/lib.rs @@ -1,7 +1,8 @@ use candid::{CandidType, Deserialize, Principal}; use futures::future::join_all; use ic_cdk::api::call::call_raw; -use ic_cdk::{query, update}; +use ic_cdk::api::performance_counter; +use ic_cdk::{id, query, update}; thread_local! { static NUMBER: std::cell::RefCell = const { std::cell::RefCell::new(0) }; @@ -55,6 +56,18 @@ async fn get_number() -> u64 { NUMBER.with(|n| *n.borrow()) } +#[update] +async fn noop() {} + +#[update] +async fn expensive() { + loop { + if performance_counter(0) >= 19_000_000_000 { + ic_cdk::call::<_, ()>(id(), "noop", ((),)).await.unwrap(); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tests/integration/src/disaster_recovery_tests.rs b/tests/integration/src/disaster_recovery_tests.rs index d8db6d22c..4957ce107 100644 --- a/tests/integration/src/disaster_recovery_tests.rs +++ b/tests/integration/src/disaster_recovery_tests.rs @@ -2,23 +2,27 @@ use crate::setup::{ get_canister_wasm, setup_new_env, setup_new_env_with_config, WALLET_ADMIN_USER, }; use crate::utils::{ - add_user, advance_time_to_burn_cycles, await_station_healthy, execute_request, + add_external_canister_call_any_method_permission_and_approval_auto, add_user, + advance_time_to_burn_cycles, await_station_healthy, deploy_test_canister, execute_request, get_account_read_permission, get_account_transfer_permission, get_account_update_permission, - get_core_canister_health_status, get_system_info, get_upgrader_disaster_recovery, - get_upgrader_logs, get_user, set_disaster_recovery, upload_canister_chunks_to_asset_canister, - user_test_id, NNS_ROOT_CANISTER_ID, + get_core_canister_health_status, get_request, get_system_info, get_upgrader_disaster_recovery, + get_upgrader_logs, get_user, set_disaster_recovery, submit_request, + upload_canister_chunks_to_asset_canister, user_test_id, NNS_ROOT_CANISTER_ID, }; use crate::TestEnv; use candid::{Encode, Principal}; use orbit_essentials::api::ApiResult; use orbit_essentials::utils::sha256_hash; +use pocket_ic::management_canister::CanisterStatusResultStatus; use pocket_ic::{query_candid_as, update_candid_as, PocketIc}; use station_api::{ - AddAccountOperationInput, AllowDTO, DisasterRecoveryCommitteeDTO, HealthStatus, - ListAccountsResponse, RequestOperationDTO, RequestOperationInput, RequestPolicyRuleDTO, + AddAccountOperationInput, AllowDTO, CallExternalCanisterOperationInput, CanisterMethodDTO, + DisasterRecoveryCommitteeDTO, HealthStatus, ListAccountsResponse, RequestOperationDTO, + RequestOperationInput, RequestPolicyRuleDTO, RequestStatusDTO, SetDisasterRecoveryOperationInput, SystemInit, SystemInstall, SystemUpgrade, }; use std::collections::BTreeMap; +use std::time::Duration; use upgrader_api::{ Account, AdminUser, DisasterRecoveryCommittee, GetDisasterRecoveryAccountsResponse, GetDisasterRecoveryCommitteeResponse, SetDisasterRecoveryAccountsInput, @@ -363,6 +367,7 @@ fn test_disaster_recovery_flow() { }) .unwrap(), install_mode: upgrader_api::InstallMode::Reinstall, + force: false, }; let bad_request = upgrader_api::RequestDisasterRecoveryInput { @@ -370,6 +375,7 @@ fn test_disaster_recovery_flow() { module_extra_chunks: Some(module_extra_chunks), arg: vec![1, 2, 3], install_mode: upgrader_api::InstallMode::Reinstall, + force: false, }; let res: (ApiResult<()>,) = update_candid_as( @@ -569,6 +575,7 @@ fn test_disaster_recovery_flow_recreates_same_accounts() { })) .unwrap(), install_mode: upgrader_api::InstallMode::Reinstall, + force: false, },), ) .expect("Unexpected failed update call to request disaster recovery"); @@ -737,6 +744,7 @@ fn test_disaster_recovery_flow_reuses_same_upgrader() { })) .unwrap(), install_mode: upgrader_api::InstallMode::Reinstall, + force: false, },), ) .expect("Unexpected failed update call to request disaster recovery"); @@ -813,6 +821,7 @@ fn test_disaster_recovery_in_progress() { }) .unwrap(), install_mode: upgrader_api::InstallMode::Reinstall, + force: false, }; let res: (ApiResult<()>,) = update_candid_as( @@ -906,6 +915,7 @@ fn test_disaster_recovery_install() { }) .unwrap(), install_mode: upgrader_api::InstallMode::Install, + force: false, }; let res: (ApiResult<()>,) = update_candid_as( @@ -940,6 +950,7 @@ fn test_disaster_recovery_upgrade() { module_extra_chunks: Some(module_extra_chunks), arg: Encode!(&station_init_arg).unwrap(), install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }; let res: (ApiResult<()>,) = update_candid_as( @@ -983,6 +994,7 @@ fn test_disaster_recovery_failing() { module_extra_chunks: Some(module_extra_chunks), arg: Encode!(&arg).unwrap(), install_mode: upgrader_api::InstallMode::Upgrade, + force: false, }; let res: (ApiResult<()>,) = update_candid_as( @@ -997,3 +1009,182 @@ fn test_disaster_recovery_failing() { await_disaster_recovery_failure(&env, canister_ids.station, upgrader_id); } + +#[test] +fn test_disaster_recovery_unstoppable() { + let TestEnv { + env, canister_ids, .. + } = setup_new_env(); + + let system_info = get_system_info(&env, WALLET_ADMIN_USER, canister_ids.station); + let upgrader_id = system_info.upgrader_id; + + let test_canister = deploy_test_canister(&env); + + add_external_canister_call_any_method_permission_and_approval_auto( + &env, + canister_ids.station, + WALLET_ADMIN_USER, + ); + + // submit request to call the "expensive" method on the test canister and make the request "Processing" + let execution_method = CanisterMethodDTO { + canister_id: test_canister, + method_name: "expensive".to_string(), + }; + let call_canister_operation = + RequestOperationInput::CallExternalCanister(CallExternalCanisterOperationInput { + validation_method: None, + execution_method, + arg: Some(Encode!(&()).unwrap()), + execution_method_cycles: None, + }); + let call_canister_operation_request = submit_request( + &env, + WALLET_ADMIN_USER, + canister_ids.station, + call_canister_operation.clone(), + ); + // timer's period for processing requests is 5 seconds + env.advance_time(Duration::from_secs(5)); + env.tick(); + let call_request_in_progress = get_request( + &env, + WALLET_ADMIN_USER, + canister_ids.station, + call_canister_operation_request.clone(), + ); + match call_request_in_progress.status { + RequestStatusDTO::Processing { .. } => (), + _ => panic!( + "Unexpected request status: {:?}", + call_request_in_progress.status + ), + }; + + // make disaster recovery upgrade of the station + let station_init_arg = SystemInstall::Upgrade(SystemUpgrade { name: None }); + let new_wasm_module = get_canister_wasm("station"); + let (base_chunk, module_extra_chunks) = + upload_canister_chunks_to_asset_canister(&env, new_wasm_module, 500_000); + let mut disaster_recovery_request = upgrader_api::RequestDisasterRecoveryInput { + module: base_chunk, + module_extra_chunks: Some(module_extra_chunks), + arg: Encode!(&station_init_arg).unwrap(), + install_mode: upgrader_api::InstallMode::Upgrade, + force: false, + }; + let res: (ApiResult<()>,) = update_candid_as( + &env, + upgrader_id, + WALLET_ADMIN_USER, + "request_disaster_recovery", + (disaster_recovery_request.clone(),), + ) + .unwrap(); + res.0.unwrap(); + // start processing the mgmt canister call from upgrader to stop the station + env.tick(); + + // the station should be "Stopping" by now + let station_status = env + .canister_status(canister_ids.station, Some(upgrader_id)) + .unwrap(); + assert!(matches!( + station_status.status, + CanisterStatusResultStatus::Stopping + )); + + // the station can't be stopped yet because it has an open call context + // with a pending down-stream call to the "expensive" method of the test canister + // now we advance time by 5 mins to time out (i.e., fail) the upgrader's call to stop the station + env.advance_time(Duration::from_secs(5 * 60)); + + // disaster recovery of the station fails because the station could not be stopped + await_disaster_recovery_failure(&env, canister_ids.station, upgrader_id); + let dr_status = get_upgrader_disaster_recovery(&env, &upgrader_id, &canister_ids.station); + match dr_status.last_recovery_result { + Some(upgrader_api::RecoveryResult::Failure(err)) => { + assert!(err.reason.contains("Stop canister request timed out")) + } + _ => panic!( + "Unexpected last recovery result: {:?}", + dr_status.last_recovery_result + ), + }; + + // the station should still be "Stopping" + let station_status = env + .canister_status(canister_ids.station, Some(upgrader_id)) + .unwrap(); + assert!(matches!( + station_status.status, + CanisterStatusResultStatus::Stopping + )); + + // force stop in disaster recovery + disaster_recovery_request.force = true; + + // we perform successful disaster recovery with forced stop twice + // to test that snapshots can be taken multiple times + let mut snapshots = env + .list_canister_snapshots(canister_ids.station, Some(upgrader_id)) + .unwrap(); + assert!(snapshots.is_empty()); + for i in 0..2 { + let new_name = format!("recovered-{}", i); + let station_init_arg = SystemInstall::Upgrade(SystemUpgrade { + name: Some(new_name.clone()), + }); + disaster_recovery_request.arg = Encode!(&station_init_arg).unwrap(); + let res: (ApiResult<()>,) = update_candid_as( + &env, + upgrader_id, + WALLET_ADMIN_USER, + "request_disaster_recovery", + (disaster_recovery_request.clone(),), + ) + .unwrap(); + res.0.unwrap(); + // start processing the mgmt canister call from upgrader to stop the station + env.tick(); + + // the station can't be stopped yet because it has an open call context + // with a pending down-stream call to the "expensive" method of the test canister + // now we advance time by 5 mins to time out (i.e., fail) the upgrader's call to stop the station + env.advance_time(Duration::from_secs(5 * 60)); + + // disaster recovery should succeed now when forcing the station to stop + await_disaster_recovery_success(&env, canister_ids.station, upgrader_id); + + // check that a snapshot has been taken + let current_snapshots = env + .list_canister_snapshots(canister_ids.station, Some(upgrader_id)) + .unwrap(); + assert_eq!(current_snapshots.len(), 1); + assert!( + snapshots.is_empty() + || (snapshots.len() == 1 && current_snapshots[0].id != snapshots[0].id) + ); + snapshots = current_snapshots; + + // check new station name set during disaster recovery + let system_info = get_system_info(&env, WALLET_ADMIN_USER, canister_ids.station); + assert_eq!(system_info.name, new_name); + + // the call request will be "Processing" forever since we deleted its call context during disaster recovery + let call_request_in_progress = get_request( + &env, + WALLET_ADMIN_USER, + canister_ids.station, + call_canister_operation_request.clone(), + ); + match call_request_in_progress.status { + RequestStatusDTO::Processing { .. } => (), + _ => panic!( + "Unexpected request status: {:?}", + call_request_in_progress.status + ), + }; + } +} diff --git a/tests/integration/src/utils.rs b/tests/integration/src/utils.rs index 81e1475f9..a1b82a065 100644 --- a/tests/integration/src/utils.rs +++ b/tests/integration/src/utils.rs @@ -890,11 +890,23 @@ pub(crate) fn await_station_healthy(env: &PocketIc, station_id: Principal) { ); } -pub(crate) fn add_external_canister_call_any_method_permission_and_approval( +pub(crate) fn deploy_test_canister(env: &PocketIc) -> Principal { + let test_canister = create_canister(env, WALLET_ADMIN_USER); + let test_canister_wasm = get_canister_wasm("test_canister"); + env.install_canister( + test_canister, + test_canister_wasm, + vec![], + Some(WALLET_ADMIN_USER), + ); + test_canister +} + +pub(crate) fn add_external_canister_call_any_method_permission_and_approval_rule( env: &PocketIc, station_id: Principal, admin_id: Principal, - quorum: station_api::QuorumDTO, + rule: station_api::RequestPolicyRuleDTO, ) { // add the permissions for admins to call any external canister execute_request( @@ -929,8 +941,35 @@ pub(crate) fn add_external_canister_call_any_method_permission_and_approval( validation_method: station_api::ValidationMethodResourceTargetDTO::No, }, ), - rule: station_api::RequestPolicyRuleDTO::Quorum(quorum), + rule, }), ) .expect("Failed to add approval policy to call external canister"); } + +pub(crate) fn add_external_canister_call_any_method_permission_and_approval( + env: &PocketIc, + station_id: Principal, + admin_id: Principal, + quorum: station_api::QuorumDTO, +) { + add_external_canister_call_any_method_permission_and_approval_rule( + env, + station_id, + admin_id, + station_api::RequestPolicyRuleDTO::Quorum(quorum), + ); +} + +pub(crate) fn add_external_canister_call_any_method_permission_and_approval_auto( + env: &PocketIc, + station_id: Principal, + admin_id: Principal, +) { + add_external_canister_call_any_method_permission_and_approval_rule( + env, + station_id, + admin_id, + station_api::RequestPolicyRuleDTO::AutoApproved, + ); +}