From 6dc2740f63f0656ab21d4f800a385298d85a514f Mon Sep 17 00:00:00 2001 From: jonathanfrappier <92055993+jonathanfrappier@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:44:27 -0500 Subject: [PATCH] Add troubleshooting section to 1.16.x docs for Vault 003 exam --- .../docs/troubleshoot/generate-root-token.mdx | 161 ++++++++++++++ .../docs/troubleshoot/lease-issues.mdx | 122 +++++++++++ .../docs/troubleshoot/tune-lease-ttl.mdx | 198 ++++++++++++++++++ website/data/docs-nav-data.json | 25 ++- 4 files changed, 502 insertions(+), 4 deletions(-) create mode 100644 website/content/docs/troubleshoot/generate-root-token.mdx create mode 100644 website/content/docs/troubleshoot/lease-issues.mdx create mode 100644 website/content/docs/troubleshoot/tune-lease-ttl.mdx diff --git a/website/content/docs/troubleshoot/generate-root-token.mdx b/website/content/docs/troubleshoot/generate-root-token.mdx new file mode 100644 index 000000000000..5a69620ed33e --- /dev/null +++ b/website/content/docs/troubleshoot/generate-root-token.mdx @@ -0,0 +1,161 @@ +--- +layout: docs +page_title: Regenerate a Vault root token +description: >- + Regenerate a lost or revoked root token. +--- + +# Regenerate a Vault root token + +Your Vault root token is a special token that gives you access to **all** Vault +operations. Best practice is to enable an appropriate authentication method for +Vault admins once the server is running and revoke the root token. + +For emergency situations where your require a root token, you can use the +[`operator generate-root`](/vault/docs/commands/operator/generate-root) CLI +command and a one-time password (OTP) or Pretty Good Privacy (PGP) to generate +a new root token. + +## Before you start + +- **You need your Vault keys**. If you use auto-unseal, you need your + [recovery](/vault/docs/concepts/seal#recovery-key) keys, otherwise you need + your unseal keys. +- **Identify current key holders**. You must distribute the token nonce to your + unseal/recovery key holders during root token generation. + +## Step 1: Create a root token nonce + +1. Generate a token nonce for your new root token: + + + + + **You need the returned OTP value to decode the new root token**. + + ```shell-session + $ vault operator generate-root -init + + A One-Time-Password has been generated for you and is shown in the OTP field. + You will need this value to decode the resulting root token, so keep it safe. + Nonce 15565c79-cc9e-5e64-b986-8506e7bd1918 + Started true + Progress 0/1 + Complete false + OTP 5JFQaH76Ky2TIuSt4SPvO1CGkx + OTP Length 26 + ``` + + + + + Use the `-pgp-key` option to provide a path to your PGP public key or Keybase + username to encrypt the new root token. **You will need the returned PGP + value to decode the new root token**. + + ```shell-session + $ vault operator generate-root -init -pgp-key=keybase:sethvargo + + Nonce e24dec5e-f1ea-2dfe-ecce-604022006976 + Started true + Progress 0/5 + Complete false + PGP Fingerprint e2f8e2974623ba2a0e933a59c921994f9c27e0ff + ``` + + + + +1. Distribute the nonce to each of your unseal/recovery key holders. + +## Step 2: Establish key quorum with the token nonce + + + + If you use a TTY, the `operator generate-root` command prompts for your key + and automatically completes the nonce value. + + + +1. Have each unseal/recovery key holder run `operator generator-root` with their + key and the distributed nonce value: + + ```shell-session + $ echo ${UNSEAL_OR_RECOVERY_KEY} | vault operator generate-root -nonce=${NONCE_VALUE} - + + Root generation operation nonce: f67f4da3-4ae4-68fb-4716-91da6b609c3e + Unseal Key (will be hidden): + ``` + +1. Vault returns the new, encoded root token to the user who triggers quorum: + + + + + ```shell-session + Nonce f67f4da3-4ae4-68fb-4716-91da6b609c3e + Started true + Progress 5/5 + Complete true + Encoded Token IxJpyqxn3YafOGhqhvP6cQ== + ``` + + + + + + ```shell-session + Nonce e24dec5e-f1ea-2dfe-ecce-604022006976 + Started true + Progress 1/1 + Complete true + PGP Fingerprint e2f8e2974623ba2a0e933a59c921994f9c27e0ff + Encoded Token wcFMA0RVkFtoqzRlARAAI3Ux8kdSpfgXdF9mg... + ``` + + + + +## Step 3: Decode the new root token + +Decode the new root token using OTP or PGP. + + + + +Use `operator generate-root` and the OTP value from nonce generation to decode +the new root token: + +```shell-session +$ vault operator generate-root \ + -decode=${ENCODED_TOKEN} \ + -otp=${NONCE_OTP} + +hvs.XXXXXXXXXXXXXXXXXXXXXXXX +``` + + + + + +Use your PGP credentials and `gpg` or `keybase` to decrypt the new root token. + + +**`gpg`**: + +```shell-session +$ echo ${ENCODED_TOKEN} | base64 --decode | gpg --decrypt + +hvs.XXXXXXXXXXXXXXXXXXXXXXXX +``` + +**`keybase`**: + +```shell-session +$ echo ${ENCODED_TOKEN} | base64 --decode | keybase pgp decrypt + +hvs.XXXXXXXXXXXXXXXXXXXXXXXX +``` + + + \ No newline at end of file diff --git a/website/content/docs/troubleshoot/lease-issues.mdx b/website/content/docs/troubleshoot/lease-issues.mdx new file mode 100644 index 000000000000..1cca28d9e942 --- /dev/null +++ b/website/content/docs/troubleshoot/lease-issues.mdx @@ -0,0 +1,122 @@ +--- +layout: docs +page_title: Lease problems +description: >- + Troubleshoot lease problems in Vault. +--- + +# Troubleshoot lease problems + +Explanations, workarounds, and solutions for common lease problems in Vault. + +## `429 - Too Many Requests` + +### Problem + +Vault returns a `429 - Too Many Requests` response when users try to +authenticate. For example: + + + +```text +Error making API request. + +URL: PUT https://127.0.0.1:61555/v1/auth/userpass/login/foo +Code: 429. Errors: + +* 1 error occurred: + * request path "auth/userpass/login/foo": lease count quota exceeded +``` + + + +### Cause + +Vault returns a `429 - Too Many Requests` response if a new lease request +violates the configured lease quota limit. + +To guard against [lease explosions](/vault/docs/troubleshoot/lease-explosions), +Vault rejects authentication requests if completing the request would violate +the configured lease quota limit. + +### Solution + +1. Correct any client-side errors that may cause excessive lease creation. +1. Determine if your resource needs have changed and complete the + [Protecting Vault with Resource Quotas](/vault/tutorials/operations/resource-quotas) + tutorial to determine new, appropriate defaults. +1. Use the [`vault lease`](/vault/docs/commands/lease) CLI command or + [lease count quota endpoint](/vault/api-docs/system/lease-count-quotas) to + tune your lease count quota. + + + Consider making short-term changes to your lease quotas when you expect a + significant increase in lease creation. For example, when you release a new + feature or complete a marketing push to increase your user base. + + + +## Lease explosion (degraded performance) + +### Problem + +Your Vault nodes are out of memory and unresponsive to new lease requests. + +### Cause + +Clients have caused a lease explosion with consistent, high-volume API requests. + + + + Unchecked lease explosions create cascading denial-of-service issues for the + active node that can result in denial-of-service issues for the entire + cluster. + + + +### Solution + +To resolve a lease explosion, you need to mitigate the problem to stabilize +Vault and provide space for cluster recovery then clean up your Vault +environment. + +1. Mitigate resource stress by adjusting TTL values for your Vault instance: + + Config level | Parameter | Precedence + -------------------- | ---------------------- | ----------- + Database plugin | `ttl` or `default_ttl` | first + Database plugin | `max_ttl` | first + AuthN/secrets plugin | `ttl` or `default_ttl` | second + AuthN/secrets plugin | `max_ttl` | second + Vault | `default_lease_ttl` | last + Vault | `max_lease_ttl` | last + + **Granular TTLs on a role, group, or user level always override plugin and + system-wide TTL values**. + +1. Use firewalls or load balancers to limit API calls to Vault from aberrant + clients and reduce load on the struggling cluster . + +1. Once the cluster stabilizes, check the active node to determine if you can + wait for it to purge leases automatically or if you need to speed up the + process by manually revoking leases. + +1. If the cluster requires manual intervention, confirm you have a recent, valid + snapshots of the cluster. + +1. Once you confirm a valid snapshot of the cluster exists, use + [`vault lease revoke`](/vault/docs/commands/lease/revoke) to manually revoke + the offending leases. + + + + Revoking or forcefully revoking leases is potentially a dangerous operation. + Do not proceed without a valid snapshot. If you have a valid Vault + Enterprise license, consider contacting the + [HashiCorp Customer Support team](https://support.hashicorp.com/) for help. + + + +### Related tutorials + +- [Troubleshoot irrevocable leases](/vault/tutorials/monitoring/troubleshoot-irrevocable-leases) \ No newline at end of file diff --git a/website/content/docs/troubleshoot/tune-lease-ttl.mdx b/website/content/docs/troubleshoot/tune-lease-ttl.mdx new file mode 100644 index 000000000000..a50c3fd03d48 --- /dev/null +++ b/website/content/docs/troubleshoot/tune-lease-ttl.mdx @@ -0,0 +1,198 @@ +--- +layout: docs +page_title: Tune the lease TTL +description: >- + Understand the behavior of time-to-live set on leases. +--- + +# Tune the lease time-to-live (TTL) + +The benefit of using Vault's dynamic secrets engines and auth methods is the +ability to control how long the Vault-managed credentials (leases) remain valid. +Often times, you generate short-lived credentials or tokens to reduce the risk +of unauthorized attacks caused by leaked credentials or tokens. If you do not +explicitly specify the time-to-live (TTL), Vault generates leases with TTL of 32 +days by default. + +For example, you enabled AppRole auth method at `approle`, and create a role +named `read-only` with max lease TTL of **120 days**. + +```shell-session +$ vault write auth/approle/role/read-only token_policies="read-only" \ + token_ttl=90d token_max_ttl=120d +``` + +The command returns a warning about the TTL exceeding the mount's max TTL value. + + + +```plaintext +WARNING! The following warnings were returned from Vault: + + * token_max_ttl is greater than the backend mount's maximum TTL value; + issued tokens' max TTL value will be truncated +``` + + + + +Therefore, it will return a client token with TTL of 768 hours (32 days) instead +of 120 days. + + + +```shell-session +$ vault write auth/approle/login role_id= secret_id= + +WARNING! The following warnings were returned from Vault: + + * TTL of "2880h" exceeded the effective max_ttl of "768h"; TTL value is + capped accordingly + +Key Value +--- ----- +token hvs.CAESIJeVezY3UObHXTvzpI722q0MmaARB1692fT-MmdzcryvGh4KHGh2cy43czViYXVZS3FnSzltWmdVZ3Q0MmFTdkc +token_accessor wXTOvz5xxBi2vvUpTBhemUXr +token_duration 768h +token_renewable true +token_policies ["default" "read-only"] +identity_policies [] +policies ["default" "read-only"] +token_meta_role_name read-only +``` + + + +## Max lease TTL on an auth mount + +You cannot set the TTL for a role to go beyond the max lease TTL set on the +AppRole auth mount (`approle` in this example). The default lease TTL and max +lease TTL are 32 days (768 hours). + +```shell-session +$ vault read sys/auth/approle/tune +``` + +**Output:** + + + +```plaintext +Key Value +--- ----- +default_lease_ttl 768h +description n/a +force_no_cache false +max_lease_ttl 768h +token_type default-service +``` + + + +If the desired max lease TTL is 120 days (2880 hours), update the max lease TTL +on the mount. + +```shell-session +$ vault auth tune -max-lease-ttl=120d approle +``` + +The following command lists all available parameters that you can tune. + +```shell-session +$ vault auth tune -h +``` + +Now, the AppRole will generate a lease with token duration of 120 days (2880 hours). + + + +```shell-session +$ vault write auth/approle/login role_id= secret_id= + +Key Value +--- ----- +token hvs.CAESIOzTpLX4naKw-epzhcb3DneZ9ZuRTx4tKh5mTT1CajLQGh4KHGh2cy5TUFFhY3QzVzdmSTFwQUduOWlrMVRWaUE +token_accessor blc2MGA4EmmqEROzqlotFbqr +token_duration 2880h +token_renewable true +token_policies ["default" "jenkins"] +identity_policies [] +policies ["default" "jenkins"] +token_meta_role_name jenkins +``` + + + + +## Max lease TTL on a secrets mount + +Similar to the AppRole auth method example, you can tune the max lease TTL on +dynamic secrets. + +For example, you enabled database secrets engine at `mongodb` and create a role +named `tester` with max lease TTL of 120 days (2880 hours). When you request a +database credential for the `tester` role, it returns a warning, and its lease +duration is 32 days (768 hours) instead of 120 days. + + + +```shell-session +$ vault read mongodb/creds/tester + +WARNING! The following warnings were returned from Vault: + + * TTL of "2880h" exceeded the effective max_ttl of "768h"; TTL value is + capped accordingly + +Key Value +--- ----- +lease_id mongodb/creds/tester/fVPt15506k3UW9n4pq0kIpBH +lease_duration 768h +lease_renewable true +password Eskkx6yRhAN4--H9WL7B +username v-token-tester-6BtY903qOZBpzYa4yQs8-1724715513 +``` + + + +To set the desired TTL on the role, tune the max lease TTL on the `mongodb` +mount. + +```shell-session +$ vault secrets tune -max-lease-ttl=120d mongodb +``` + +Verify the configured max lease TTL available on the mount. + + + +```shell-session +$ vault read sys/mounts/mongodb/tune + +Key Value +--- ----- +default_lease_ttl 768h +description n/a +force_no_cache false +max_lease_ttl 2880h +``` + + + +The following command lists all available parameters that you can tune. + +```shell-session +$ vault secrets tune -h +``` + +When you introduce Vault into your existing system, the existing applications +may not be able to handle short-lived leases. You can tune the default TTLs +on each mount. + +On the similar note, if the system default of 32 days is too long, you can tune +the default TTL to be shorter to comply with your organization's policy. + +## API + +- [Tune auth method](/vault/api-docs/system/auth#tune-auth-method) +- [Tune mount configuration](/vault/api-docs/system/mounts#tune-mount-configuration) \ No newline at end of file diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index 65e1d7e4bcf6..bb98a5efe16f 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -361,7 +361,7 @@ }, { "title": "TCP", - "routes": [ + "routes": [ { "title": "Overview", "path": "configuration/listener/tcp" @@ -1111,7 +1111,7 @@ "path": "commands/transform/import" } ] - }, + }, { "title": "unwrap", "path": "commands/unwrap" @@ -1990,8 +1990,25 @@ "path": "interoperability-matrix" }, { - "title": "Troubleshoot", - "href": "https://learn.hashicorp.com/tutorials/vault/troubleshooting-vault" + "title": "Troubleshoot Vault", + "routes": [ + { + "title": "Generate a root token", + "path": "troubleshoot/generate-root-token" + }, + { + "title": "Troubleshoot lease errors", + "path": "troubleshoot/lease-issues" + }, + { + "title": "Tune the lease TTL", + "path": "troubleshoot/tune-lease-ttl" + }, + { + "title": "Troubleshooting tutorials", + "href": "https://learn.hashicorp.com/tutorials/vault/troubleshooting-vault" + } + ] }, { "divider": true