diff --git a/src/commands/start.py b/src/commands/start.py index 88e3f652..711e780d 100644 --- a/src/commands/start.py +++ b/src/commands/start.py @@ -13,6 +13,8 @@ from src.common.vault_config import VaultConfig from src.config.networks import AVAILABLE_NETWORKS from src.config.settings import ( + DEFAULT_HASHI_VAULT_JWT_FILE, + DEFAULT_HASHI_VAULT_OIDC_MOUNT, DEFAULT_HASHI_VAULT_PARALLELISM, DEFAULT_MAX_FEE_PER_GAS_GWEI, DEFAULT_METRICS_HOST, @@ -177,6 +179,23 @@ envvar='HASHI_VAULT_TOKEN', help='Authentication token for accessing Hashi vault.', ) +@click.option( + '--hashi-vault-oidc-auth-role', + envvar='HASHI_VAULT_OIDC_AUTH_ROLE', + help='A role to use when authenticating via OID Connect with Hashi vault.', +) +@click.option( + '--hashi-vault-oidc-auth-mount-point', + envvar='HASHI_VAULT_OIDC_AUTH_MOUNT_POINT', + help='A role to use when authenticating via OID Connect with Hashi vault.', + default=DEFAULT_HASHI_VAULT_OIDC_MOUNT, +) +@click.option( + '--hashi-vault-jwt-auth-file', + envvar='HASHI_VAULT_JWT_FILE', + help='A path to file that contains JWT content for dynamically authenticating via OID Connect.', + default=DEFAULT_HASHI_VAULT_JWT_FILE, +) @click.option( '--hashi-vault-key-path', envvar='HASHI_VAULT_KEY_PATH', @@ -252,6 +271,9 @@ def start( hashi_vault_key_path: list[str] | None, hashi_vault_key_prefix: list[str] | None, hashi_vault_token: str | None, + hashi_vault_oidc_auth_role: str | None, + hashi_vault_oidc_auth_mount_point: str | None, + hashi_vault_jwt_auth_file: str | None, hashi_vault_url: str | None, hashi_vault_parallelism: int, hot_wallet_file: str | None, @@ -284,6 +306,9 @@ def start( keystores_password_file=keystores_password_file, remote_signer_url=remote_signer_url, hashi_vault_token=hashi_vault_token, + hashi_vault_auth_mount=hashi_vault_oidc_auth_mount_point, + hashi_vault_auth_role=hashi_vault_oidc_auth_role, + hashi_vault_jwt_file=hashi_vault_jwt_auth_file, hashi_vault_key_paths=hashi_vault_key_path, hashi_vault_key_prefixes=hashi_vault_key_prefix, hashi_vault_parallelism=hashi_vault_parallelism, diff --git a/src/commands/validators_exit.py b/src/commands/validators_exit.py index c6d3019c..e6f86a41 100644 --- a/src/commands/validators_exit.py +++ b/src/commands/validators_exit.py @@ -19,6 +19,8 @@ from src.config.networks import AVAILABLE_NETWORKS from src.config.settings import ( DEFAULT_HASHI_VAULT_ENGINE_NAME, + DEFAULT_HASHI_VAULT_JWT_FILE, + DEFAULT_HASHI_VAULT_OIDC_MOUNT, DEFAULT_HASHI_VAULT_PARALLELISM, settings, ) @@ -110,6 +112,23 @@ class ValidatorExit: help='How much requests to K/V secrets engine to do in parallel.', default=DEFAULT_HASHI_VAULT_PARALLELISM, ) +@click.option( + '--hashi-vault-oidc-auth-role', + envvar='HASHI_VAULT_OIDC_AUTH_ROLE', + help='A role to use when authenticating via OID Connect with Hashi vault.', +) +@click.option( + '--hashi-vault-oidc-auth-mount-point', + envvar='HASHI_VAULT_OIDC_AUTH_MOUNT_POINT', + help='A role to use when authenticating via OID Connect with Hashi vault.', + default=DEFAULT_HASHI_VAULT_OIDC_MOUNT, +) +@click.option( + '--hashi-vault-jwt-auth-file', + envvar='HASHI_VAULT_JWT_FILE', + help='A path to file that contains JWT content for dynamically authenticating via OID Connect.', + default=DEFAULT_HASHI_VAULT_JWT_FILE, +) @click.option( '-v', '--verbose', @@ -144,6 +163,9 @@ def validators_exit( hashi_vault_key_path: list[str] | None, hashi_vault_key_prefix: list[str] | None, hashi_vault_token: str | None, + hashi_vault_oidc_auth_role: str | None, + hashi_vault_oidc_auth_mount_point: str | None, + hashi_vault_jwt_auth_file: str | None, hashi_vault_url: str | None, hashi_vault_engine_name: str, hashi_vault_parallelism: int, @@ -165,6 +187,9 @@ def validators_exit( consensus_endpoints=consensus_endpoints, remote_signer_url=remote_signer_url, hashi_vault_token=hashi_vault_token, + hashi_vault_auth_mount=hashi_vault_oidc_auth_mount_point, + hashi_vault_auth_role=hashi_vault_oidc_auth_role, + hashi_vault_jwt_file=hashi_vault_jwt_auth_file, hashi_vault_key_paths=hashi_vault_key_path, hashi_vault_key_prefixes=hashi_vault_key_prefix, hashi_vault_url=hashi_vault_url, diff --git a/src/config/settings.py b/src/config/settings.py index ef67dabd..cf280e3c 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -21,6 +21,8 @@ DEFAULT_HASHI_VAULT_PARALLELISM = 8 DEFAULT_HASHI_VAULT_ENGINE_NAME = 'secret' +DEFAULT_HASHI_VAULT_JWT_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token' +DEFAULT_HASHI_VAULT_OIDC_MOUNT = 'kubernetes' # pylint: disable-next=too-many-public-methods,too-many-instance-attributes @@ -57,6 +59,9 @@ class Settings(metaclass=Singleton): hashi_vault_url: str | None hashi_vault_engine_name: str hashi_vault_token: str | None + hashi_vault_jwt_file: Path | None + hashi_vault_auth_mount: str | None + hashi_vault_auth_role: str | None hashi_vault_parallelism: int hot_wallet_file: Path hot_wallet_password_file: Path @@ -121,6 +126,9 @@ def set( hashi_vault_url: str | None = None, hashi_vault_engine_name: str = DEFAULT_HASHI_VAULT_ENGINE_NAME, hashi_vault_token: str | None = None, + hashi_vault_jwt_file: str | None = DEFAULT_HASHI_VAULT_JWT_FILE, + hashi_vault_auth_mount: str | None = DEFAULT_HASHI_VAULT_OIDC_MOUNT, + hashi_vault_auth_role: str | None = None, hashi_vault_parallelism: int = DEFAULT_HASHI_VAULT_PARALLELISM, hot_wallet_file: str | None = None, hot_wallet_password_file: str | None = None, @@ -187,6 +195,11 @@ def set( self.hashi_vault_key_paths = hashi_vault_key_paths self.hashi_vault_key_prefixes = hashi_vault_key_prefixes self.hashi_vault_token = hashi_vault_token + self.hashi_vault_jwt_file = ( + Path(hashi_vault_jwt_file) if hashi_vault_jwt_file is not None else None + ) + self.hashi_vault_auth_mount = hashi_vault_auth_mount + self.hashi_vault_auth_role = hashi_vault_auth_role self.hashi_vault_parallelism = hashi_vault_parallelism # hot wallet @@ -310,8 +323,9 @@ def need_deposit_data_file(self) -> bool: ) REMOTE_SIGNER_TIMEOUT = decouple_config('REMOTE_SIGNER_TIMEOUT', cast=int, default=30) -# Hashi vault timeout +# Hashi vault timeouts HASHI_VAULT_TIMEOUT = 10 +HASHI_VAULT_OIDC_LOGIN_TIMEOUT = 60 # Graphql timeout GRAPH_API_TIMEOUT = 10 diff --git a/src/conftest.py b/src/conftest.py index 66a35a26..b20429ab 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -21,7 +21,13 @@ from src.common.vault_config import VaultConfig from src.config.networks import HOLESKY from src.config.settings import settings -from src.test_fixtures.hashi_vault import hashi_vault_url, mocked_hashi_vault # noqa +from src.test_fixtures.hashi_vault import ( # noqa + hashi_vault_url, + jwt_auth_secret, + mocked_hashi_vault, + mocked_hashi_vault_oidc_auth, + mocked_jwt_auth_file, +) from src.test_fixtures.remote_signer import mocked_remote_signer, remote_signer_url from src.validators.keystores.remote import RemoteSignerKeystore from src.validators.signing.tests.oracle_functions import OracleCommittee diff --git a/src/test_fixtures/hashi_vault.py b/src/test_fixtures/hashi_vault.py index ff3cef80..a399f2b8 100644 --- a/src/test_fixtures/hashi_vault.py +++ b/src/test_fixtures/hashi_vault.py @@ -1,11 +1,20 @@ import json +import tempfile +import uuid from functools import partial +from pathlib import Path from typing import Generator import pytest from aioresponses import CallbackResult, aioresponses +@pytest.fixture(scope='session') +def jwt_auth_secret() -> str: + # Generate unique uuid secret for every run + return str(uuid.uuid4()) + + @pytest.fixture def hashi_vault_url() -> str: return 'http://vault:8200' @@ -144,3 +153,198 @@ def _mocked_error_path(url, **kwargs) -> CallbackResult: repeat=True, ) yield + + +@pytest.fixture +def mocked_hashi_vault_oidc_auth( + hashi_vault_url: str, + jwt_auth_secret: str, +) -> Generator: + # Generated via + # eth-staking-smith existing-mnemonic \ + # --chain holesky \ + # --num_validators 2 \ + # --mnemonic 'provide iron update bronze session immense garage want round enhance artefact position make wash analyst skirt float jealous trend spread ginger rapid express tool' + _hashi_vault_pk_sk_mapping = { + 'b05e93c4501233eeb7f1e7b0ee400caaa04608249c4aab61c18e04c675aaf2a0f03808d533c877fbbd57b04927c01ce0': '3eeedd7a6679d2e2036682b6f03ef16105a847321303aec163548aa3fa5e9eeb', + 'aa84894836cb3d897a1a11344920c41c472ed67667fd8a3453e557214442370ffc1d007ae7af67120de00afa068349be': '236f33410e6972a2db36ba3736099396768219b327e18eae49392f153007d468', + } + + # Generated via + # eth-staking-smith existing-mnemonic \ + # --chain holesky \ + # --num_validators 2 \ + # --mnemonic 'route flight verb churn work creek crane hole obscure young shaft area bird border refuse usage flash engage burden retreat drama bamboo profit sense' + _hashi_vault_prefixed_pk_sk_mapping = { + '8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c': '5d88e114821bf871f321399d99fe58cb24d6434b416f112e8e46077e05399dc0', + '8979806d4e5d841758868b208df0dd961c12a0cf044e2de1d18e269ca0ad0308672be2f71d3d5606834764fe5b1d0bc4': '01352aec5cadb78eba6f716570d28b40f24b96c522dac535bc81375ceb54bf0b', + } + + state_auth = dict(logins=0) + + def _mocked_auth_login(duration, url, **kwargs): + if kwargs['json']['jwt'] != jwt_auth_secret: + raise AssertionError('Invalid JWT value passed to a mock') + state_auth['logins'] += 1 + return CallbackResult( + status=200, + body=json.dumps( + { + 'auth': { + 'client_token': str(uuid.uuid4()), + 'accessor': str(uuid.uuid4()), + 'policies': ['default'], + 'metadata': { + 'role': kwargs['json']['role'], + }, + 'lease_duration': duration, + 'renewable': True, + } + } + ), + ) + + def _mocked_auth_failure(url, **kwargs): + return CallbackResult( + status=403, + body=json.dumps( + { + 'errors': ['invalid_token'], + } + ), + ) + + def _mocked_secret_path(data, url, **kwargs) -> CallbackResult: + return CallbackResult( + status=200, + body=json.dumps( + dict( + data=dict( + data=data, + ) + ) + ), # type: ignore + ) + + def _mocked_secrets_list(data, url, **kwargs) -> CallbackResult: + return CallbackResult( + status=200, + body=json.dumps( + dict( + data=dict( + keys=data, + ) + ) + ), # type: ignore + ) + + state_reauth_list = dict(attempt=0) + + def _mocked_secrets_list_reauth(data, url, **kwargs) -> CallbackResult: + state_reauth_list['attempt'] += 1 + if state_reauth_list['attempt'] == 1: + # Yield 403 invalid token on first attempt + return CallbackResult( + status=403, + body=json.dumps( + dict( + errors=['invalid_token'], + ) + ), # type: ignore + ) + else: + # On aubsequent request after reauth, return normal payload + + # This ensures only one request is done + if state_reauth_list['attempt'] != 2: + raise AssertionError('Invalid secret list amount done') + # This ensures login has been called twice + if state_auth['logins'] != 2: + raise AssertionError('Invalid amount of logins was performed') + + return CallbackResult( + status=200, + body=json.dumps(dict(data=dict(keys=data))), # type: ignore + ) + + def _mocked_error_path(url, **kwargs) -> CallbackResult: + return CallbackResult( + status=200, body=json.dumps(dict(errors=list('token not provided'))) # type: ignore + ) + + with aioresponses() as m: + # Successful authentication with lengthy token duration + m.post( + f'{hashi_vault_url}/v1/auth/kubernetes/login', + callback=partial(_mocked_auth_login, 2764800), + repeat=True, + ) + + # Auth with small lease duration to yield is_stale = true + # on the failure leading to reauth + m.post( + f'{hashi_vault_url}/v1/auth/kubernetes_reauth/login', + callback=partial(_mocked_auth_login, 0), + repeat=True, + ) + + # Authentication error + m.post( + f'{hashi_vault_url}/v1/auth/kubernetes_error/login', + callback=_mocked_auth_failure, + repeat=True, + ) + + # Mocked bundled signing keys endpoints + m.get( + f'{hashi_vault_url}/v1/secret/data/ethereum/signing/keystores', + callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping), + repeat=True, + ) + # Mocked bundled signing keys on reauth endpoint + m.get( + f'{hashi_vault_url}/v1/secret/data/ethereum/signing/keystores', + callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping), + repeat=True, + ) + # Mocked prefixed signing keys endpoints + m.add( + f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed1', + callback=partial( + _mocked_secrets_list, list(_hashi_vault_prefixed_pk_sk_mapping.keys()) + ), + repeat=True, + method='LIST', + ) + for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping.items(): + m.get( + f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed1/{_pk}', + callback=partial(_mocked_secret_path, {'value': _sk}), + repeat=True, + ) + + # Mocked prefixed signing keys endpoints with reauth + m.add( + f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed2', + callback=partial( + _mocked_secrets_list_reauth, list(_hashi_vault_prefixed_pk_sk_mapping.keys()) + ), + repeat=True, + method='LIST', + ) + for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping.items(): + m.get( + f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed2/{_pk}', + callback=partial(_mocked_secret_path, {'value': _sk}), + repeat=True, + ) + + yield + + +@pytest.fixture +def mocked_jwt_auth_file(jwt_auth_secret: str) -> Generator: + with tempfile.NamedTemporaryFile('w', delete=False) as jwt_auth: + jwt_auth.write(jwt_auth_secret) + jwt_auth.flush() + yield Path(jwt_auth.name) diff --git a/src/validators/keystores/hashi_vault.py b/src/validators/keystores/hashi_vault.py deleted file mode 100644 index d5419abd..00000000 --- a/src/validators/keystores/hashi_vault.py +++ /dev/null @@ -1,238 +0,0 @@ -import abc -import asyncio -import itertools -import logging -import urllib.parse -from dataclasses import dataclass -from typing import Iterator - -from aiohttp import ClientSession, ClientTimeout -from eth_typing import HexStr -from eth_utils import add_0x_prefix -from web3 import Web3 - -from src.config.settings import HASHI_VAULT_TIMEOUT, settings -from src.validators.keystores.local import Keys, LocalKeystore -from src.validators.typings import BLSPrivkey - -logger = logging.getLogger(__name__) - - -@dataclass -class HashiVaultConfiguration: - token: str - url: str - engine_name: str - key_paths: list[str] - key_prefixes: list[str] - parallelism: int - - @classmethod - def from_settings(cls) -> 'HashiVaultConfiguration': - if not ( - settings.hashi_vault_url is not None - and settings.hashi_vault_token is not None - and ( - settings.hashi_vault_key_paths is not None - or settings.hashi_vault_key_prefixes is not None - ) - ): - raise RuntimeError( - 'All three of URL, token and key path must be specified for hashi vault' - ) - return cls( - token=settings.hashi_vault_token, - url=settings.hashi_vault_url, - engine_name=settings.hashi_vault_engine_name, - key_paths=settings.hashi_vault_key_paths or [], - key_prefixes=settings.hashi_vault_key_prefixes or [], - parallelism=settings.hashi_vault_parallelism, - ) - - def prefix_url(self, keys_prefix: str) -> str: - """An URL for Vault secrets engine location that holds prefixes for keys.""" - keys_prefix = keys_prefix.strip('/') - # URL is used for listing, so it lists metadata - return self.secret_url(keys_prefix, location='metadata') - - def secret_url(self, key_path: str, location: str = 'data') -> str: - return urllib.parse.urljoin( - self.url, - f'/v1/{self.engine_name}/{location}/{key_path}', - ) - - -@dataclass -class HashiVaultKeysLoader(metaclass=abc.ABCMeta): - config: HashiVaultConfiguration - input_iter: Iterator[str] - - def session(self) -> ClientSession: - return ClientSession( - timeout=ClientTimeout(HASHI_VAULT_TIMEOUT), - headers={'X-Vault-Token': self.config.token}, - ) - - @staticmethod - def merge_keys_responses(keys_responses: list[Keys], merged_keys: Keys) -> None: - """Merge keys objects, proactively searching for duplicate keys to prevent - potential slashing.""" - for keys in keys_responses: - for pk, sk in keys.items(): - if pk in merged_keys: - logger.error('Duplicate validator key %s found in hashi vault', pk) - raise RuntimeError('Found duplicate key in path') - merged_keys[pk] = sk - - @abc.abstractmethod - async def load(self, merged_keys: Keys) -> None: - """Populate merged_keys structure with validator keys from given loader.""" - raise NotImplementedError - - -class HashiVaultBundledKeysLoader(HashiVaultKeysLoader): - async def load(self, merged_keys: Keys) -> None: - """Load all the key bundles from input locations.""" - while key_chunk := list(itertools.islice(self.input_iter, self.config.parallelism)): - async with self.session() as session: - keys_responses = await asyncio.gather( - *[ - self._load_bundled_hashi_vault_keys( - session=session, - secret_url=self.config.secret_url(key_path), - ) - for key_path in key_chunk - ] - ) - self.merge_keys_responses(keys_responses, merged_keys) - - @staticmethod - async def _load_bundled_hashi_vault_keys(session: ClientSession, secret_url: str) -> Keys: - """ - Load public and private keys from hashi vault - K/V secret engine. - - All public and private keys must be stored as hex string with or without 0x prefix. - """ - keys = [] - logger.info('Will load validator keys from %s', secret_url) - - response = await session.get(secret_url) - response.raise_for_status() - - key_data = await response.json() - - if 'data' not in key_data: - logger.error('Failed to retrieve keys from hashi vault') - for error in key_data.get('errors', []): - logger.error('hashi vault error: %s', error) - raise RuntimeError('Can not retrieve validator signing keys from hashi vault') - - for pk, sk in key_data['data']['data'].items(): - sk_bytes = Web3.to_bytes(hexstr=sk) - keys.append((add_0x_prefix(HexStr(pk)), BLSPrivkey(sk_bytes))) - validator_keys = Keys(dict(keys)) - return validator_keys - - -class HashiVaultPrefixedKeysLoader(HashiVaultKeysLoader): - async def load(self, merged_keys: Keys) -> None: - """Discover all the keys under given prefix. Then, load the keys into merged structure.""" - prefix_leaf_location_tuples = [] - while prefix_chunk := list(itertools.islice(self.input_iter, self.config.parallelism)): - async with self.session() as session: - prefix_leaf_location_tuples += await asyncio.gather( - *[ - self._find_prefixed_hashi_vault_keys( - session=session, - prefix=prefix_path, - prefix_url=self.config.prefix_url(prefix_path), - ) - for prefix_path in prefix_chunk - ] - ) - - # Flattened list of prefix, pubkey tuples - keys_paired_with_prefix: list[tuple[str, str]] = sum( - prefix_leaf_location_tuples, - [], - ) - prefixed_keys_iter = iter(keys_paired_with_prefix) - while prefixed_chunk := list(itertools.islice(prefixed_keys_iter, self.config.parallelism)): - async with self.session() as session: - keys_responses = await asyncio.gather( - *[ - self._load_prefixed_hashi_vault_key( - session=session, - secret_url=self.config.secret_url(f'{key_prefix}/{key_path}'), - ) - for (key_prefix, key_path) in prefixed_chunk - ] - ) - self.merge_keys_responses(keys_responses, merged_keys) - - @staticmethod - async def _find_prefixed_hashi_vault_keys( - session: ClientSession, prefix: str, prefix_url: str - ) -> list[tuple[str, str]]: - """ - Discover public keys under prefix in hashi vault K/V secret engine - - All public keys must be a final chunk of the secret path without 0x prefix, - all secret keys are stored under these paths with arbitrary secret dictionary - key, and secret value with or without 0x prefix. - """ - logger.info('Will discover validator keys in %s', prefix_url) - response = await session.request(method='LIST', url=prefix_url) - response.raise_for_status() - key_paths = await response.json() - if 'data' not in key_paths: - logger.error('Failed to discover keys in hashi vault') - for error in key_paths.get('errors', []): - logger.error('hashi vault error: %s', error) - raise RuntimeError('Can not discover validator public keys from hashi vault') - discovered_keys = key_paths['data']['keys'] - return list(zip([prefix] * len(discovered_keys), discovered_keys)) - - @staticmethod - async def _load_prefixed_hashi_vault_key(session: ClientSession, secret_url: str) -> Keys: - logger.info('Will load keys from %s', secret_url) - response = await session.get(url=secret_url) - response.raise_for_status() - key_data = await response.json() - if 'data' not in key_data: - logger.error('Failed to retrieve keys from hashi vault') - for error in key_data.get('errors', []): - logger.error('hashi vault error: %s', error) - raise RuntimeError('Can not retrieve validator signing keys from hashi vault') - # Last chunk of URL is a public key - pk = add_0x_prefix(HexStr(secret_url.strip('/').split('/')[-1])) - if len(key_data['data']['data']) > 1: - raise RuntimeError( - f'Invalid multi-value secret at path {secret_url}, ' - 'should only contain single value', - ) - sk = list(key_data['data']['data'].values())[0] - sk_bytes = Web3.to_bytes(hexstr=sk) - return Keys({pk: BLSPrivkey(sk_bytes)}) - - -class HashiVaultKeystore(LocalKeystore): - @staticmethod - async def load() -> 'HashiVaultKeystore': - """Extracts private keys from the keystores.""" - hashi_vault_config = HashiVaultConfiguration.from_settings() # noqa: NEW100 - - merged_keys = Keys({}) - - for loader_class, input_iter in { - HashiVaultBundledKeysLoader: iter(hashi_vault_config.key_paths), - HashiVaultPrefixedKeysLoader: iter(hashi_vault_config.key_prefixes), - }.items(): - loader = loader_class( - config=hashi_vault_config, - input_iter=input_iter, - ) - await loader.load(merged_keys) - - return HashiVaultKeystore(merged_keys) diff --git a/src/validators/keystores/hashi_vault/__init__.py b/src/validators/keystores/hashi_vault/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/validators/keystores/hashi_vault/auth.py b/src/validators/keystores/hashi_vault/auth.py new file mode 100644 index 00000000..f04060dc --- /dev/null +++ b/src/validators/keystores/hashi_vault/auth.py @@ -0,0 +1,87 @@ +import logging +from calendar import timegm +from dataclasses import dataclass +from datetime import datetime, timezone + +from aiohttp.client import ClientSession, ClientTimeout + +from src.config.settings import HASHI_VAULT_OIDC_LOGIN_TIMEOUT, HASHI_VAULT_TIMEOUT + +from .config import HashiVaultConfiguration + +logger = logging.getLogger(__name__) + + +def utc_timestamp() -> int: + return timegm(datetime.now(tz=timezone.utc).timetuple()) + + +@dataclass +class HashiVaultToken: + value: str + valid_until: int + + def is_stale(self) -> bool: + """Check if supposed lease duration end is later than current time.""" + return self.valid_until != -1 and self.valid_until <= utc_timestamp() + + def session(self) -> ClientSession: + return ClientSession( + timeout=ClientTimeout(HASHI_VAULT_TIMEOUT), + headers={'X-Vault-Token': self.value}, + ) + + +async def acquire_vault_token(config: HashiVaultConfiguration) -> HashiVaultToken: + """Acquires Vault token, either from static config or dynamically via OIDC""" + if config.token is not None: + return HashiVaultToken( + value=config.token, + valid_until=-1, + ) + + if ( + config.auth_role is not None + and config.jwt_file is not None + and config.mount_point is not None + ): + if not config.jwt_file.exists() or not config.jwt_file.is_file(): + raise RuntimeError( + f'JWT token file path {config.jwt_file} must point to an existing file' + ) + login_url = f'{config.url}/v1/auth/{config.mount_point}/login' + async with ClientSession( + timeout=ClientTimeout(total=HASHI_VAULT_OIDC_LOGIN_TIMEOUT), + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + ) as session: + now = utc_timestamp() + jwt = config.jwt_file.read_text(encoding='utf-8').strip() + params = { + 'role': config.auth_role, + 'jwt': jwt, + } + response = await session.post(url=login_url, json=params) + + if response.status != 200: + try: + token_data = await response.json() + except ValueError: + # Non-json response + logger.error('Non-JSON response on auth, not logging error for safety') + raise + for error in token_data.get('errors', []): + logger.debug('Got Hashi Vault error when authenticating: %s', error) + raise RuntimeError('Can not authenticate with Hashi Vault via OID connect') + + token_data = await response.json() + client_token = token_data['auth']['client_token'] + lease_duration = token_data['auth']['lease_duration'] + return HashiVaultToken( + valid_until=now + lease_duration, + value=client_token, + ) + + raise RuntimeError('Invalid Hashi Vault auth config') diff --git a/src/validators/keystores/hashi_vault/client.py b/src/validators/keystores/hashi_vault/client.py new file mode 100644 index 00000000..896a94ff --- /dev/null +++ b/src/validators/keystores/hashi_vault/client.py @@ -0,0 +1,88 @@ +import logging +from asyncio import Event, ensure_future +from dataclasses import dataclass + +from aiohttp.client import ClientSession + +from .auth import HashiVaultToken, acquire_vault_token +from .config import HashiVaultConfiguration + +HASHI_VAULT_AUTH_MAX_RETRIES = 3 +logger = logging.getLogger(__name__) + + +@dataclass +class HashiVaultClient: + """Client instance with automated token refresh.""" + + config: HashiVaultConfiguration + token: HashiVaultToken + has_auth: Event + session: ClientSession + + @classmethod + async def from_hashi_vault_config(cls, config: HashiVaultConfiguration) -> 'HashiVaultClient': + hashi_vault_config = HashiVaultConfiguration.from_settings() + token = await acquire_vault_token(hashi_vault_config) + has_auth = Event() + # After token is acquired, event is not initially guarding anything + # It will be used only in case of 403 + has_auth.set() + return cls( + config=config, + token=token, + has_auth=has_auth, + session=token.session(), + ) + + async def request(self, url: str, method: str) -> dict: + """Perform Vault request and parse errors.""" + retry = 0 + while retry < HASHI_VAULT_AUTH_MAX_RETRIES: + if not self.has_auth.is_set(): + # Parallel authentication + await self.has_auth.wait() + response = await self.session.request( + method=method, + url=url, + ) + if response.status == 403 and self.token.is_stale(): + # Attempt to re-authenticate via OIDC + # This will not work with static token, + # we assume if static gets 403 response then + # fail + if not self.has_auth.is_set(): + self.has_auth.clear() + self.token = await acquire_vault_token(self.config) + await self.session.close() + self.session = self.token.session() + self.has_auth.set() + else: + # Already authenticating from parallel request, + # wait until its done + await self.has_auth.wait() + continue + + response_data = await response.json() + # Common response format handling + if 'data' not in response_data or response.status != 200: + logger.error('Failed to discover keys in hashi vault') + errors = response_data.get('errors', []) + if not isinstance(errors, list): + errors = [ + errors, + ] + for error in errors: + logger.error('hashi vault error: %s', error) + raise RuntimeError('Can not retrieve key data from hashi vault') + + return response_data['data'] + + raise RuntimeError( + 'Failed authenticating with ' + f'Hashi Vault more than {HASHI_VAULT_AUTH_MAX_RETRIES} times' + ) + + def __del__(self) -> None: + # Cleanup session when client is not used any more + ensure_future(self.session.close()) diff --git a/src/validators/keystores/hashi_vault/config.py b/src/validators/keystores/hashi_vault/config.py new file mode 100644 index 00000000..923055d3 --- /dev/null +++ b/src/validators/keystores/hashi_vault/config.py @@ -0,0 +1,68 @@ +import urllib.parse +from dataclasses import dataclass +from pathlib import Path + +from src.config.settings import settings + + +@dataclass +class HashiVaultConfiguration: # pylint: disable=too-many-instance-attributes + token: str | None + + jwt_file: Path | None + auth_role: str | None + mount_point: str | None + + url: str + engine_name: str + key_paths: list[str] + key_prefixes: list[str] + parallelism: int + + @classmethod + def from_settings(cls) -> 'HashiVaultConfiguration': + if not ( + settings.hashi_vault_url is not None + and ( + settings.hashi_vault_token is not None + or ( + settings.hashi_vault_auth_role is not None + and settings.hashi_vault_auth_mount is not None + and settings.hashi_vault_jwt_file is not None + ) + ) + and ( + settings.hashi_vault_key_paths is not None + or settings.hashi_vault_key_prefixes is not None + ) + ): + raise RuntimeError( + 'All three of URL, key path or prefix, ' + 'and either token or OIDC config for it, ' + 'must be specified for hashi vault' + ) + return cls( + url=settings.hashi_vault_url, + engine_name=settings.hashi_vault_engine_name, + key_paths=settings.hashi_vault_key_paths or [], + key_prefixes=settings.hashi_vault_key_prefixes or [], + parallelism=settings.hashi_vault_parallelism, + # Static auth token + token=settings.hashi_vault_token, + # OID connect auth params + jwt_file=settings.hashi_vault_jwt_file, + auth_role=settings.hashi_vault_auth_role, + mount_point=settings.hashi_vault_auth_mount, + ) + + def secret_url(self, key_path: str, location: str = 'data') -> str: + return urllib.parse.urljoin( + self.url, + f'/v1/{self.engine_name}/{location}/{key_path}', + ) + + def prefix_url(self, keys_prefix: str) -> str: + """An URL for Vault secrets engine location that holds prefixes for keys.""" + keys_prefix = keys_prefix.strip('/') + # URL is used for listing, so it lists metadata + return self.secret_url(keys_prefix, location='metadata') diff --git a/src/validators/keystores/hashi_vault/loader.py b/src/validators/keystores/hashi_vault/loader.py new file mode 100644 index 00000000..a21a1db7 --- /dev/null +++ b/src/validators/keystores/hashi_vault/loader.py @@ -0,0 +1,170 @@ +import abc +import asyncio +import itertools +import logging +from dataclasses import dataclass +from typing import Iterator + +from eth_typing import HexStr +from eth_utils import add_0x_prefix +from web3 import Web3 + +from src.validators.keystores.local import Keys, LocalKeystore +from src.validators.typings import BLSPrivkey + +from .client import HashiVaultClient +from .config import HashiVaultConfiguration + +logger = logging.getLogger(__name__) + + +@dataclass +class HashiVaultKeysLoader(metaclass=abc.ABCMeta): + config: HashiVaultConfiguration + input_iter: Iterator[str] + + @staticmethod + def merge_keys_responses(keys_responses: list[Keys], merged_keys: Keys) -> None: + """Merge keys objects, proactively searching for duplicate keys to prevent + potential slashing.""" + for keys in keys_responses: + for pk, sk in keys.items(): + if pk in merged_keys: + logger.error('Duplicate validator key %s found in hashi vault', pk) + raise RuntimeError('Found duplicate key in path') + merged_keys[pk] = sk + + @abc.abstractmethod + async def load(self, client: HashiVaultClient, merged_keys: Keys) -> None: + """Populate merged_keys structure with validator keys from given loader.""" + raise NotImplementedError + + +class HashiVaultBundledKeysLoader(HashiVaultKeysLoader): + async def load(self, client: HashiVaultClient, merged_keys: Keys) -> None: + """Load all the key bundles from input locations.""" + while key_chunk := list(itertools.islice(self.input_iter, self.config.parallelism)): + keys_responses = await asyncio.gather( + *[ + self._load_bundled_hashi_vault_keys( + client=client, + secret_url=self.config.secret_url(key_path), + ) + for key_path in key_chunk + ] + ) + self.merge_keys_responses(keys_responses, merged_keys) + + @staticmethod + async def _load_bundled_hashi_vault_keys(client: HashiVaultClient, secret_url: str) -> Keys: + """ + Load public and private keys from hashi vault + K/V secret engine. + + All public and private keys must be stored as hex string with or without 0x prefix. + """ + keys = [] + logger.info('Will load validator keys from %s', secret_url) + + response = await client.request( + secret_url, + method='GET', + ) + for pk, sk in response['data'].items(): + sk_bytes = Web3.to_bytes(hexstr=sk) + keys.append((add_0x_prefix(HexStr(pk)), BLSPrivkey(sk_bytes))) + validator_keys = Keys(dict(keys)) + return validator_keys + + +class HashiVaultPrefixedKeysLoader(HashiVaultKeysLoader): + async def load(self, client: HashiVaultClient, merged_keys: Keys) -> None: + """Discover all the keys under given prefix. Then, load the keys into merged structure.""" + prefix_leaf_location_tuples = [] + while prefix_chunk := list(itertools.islice(self.input_iter, self.config.parallelism)): + prefix_leaf_location_tuples += await asyncio.gather( + *[ + self._find_prefixed_hashi_vault_keys( + client=client, + prefix=prefix_path, + prefix_url=self.config.prefix_url(prefix_path), + ) + for prefix_path in prefix_chunk + ] + ) + + # Flattened list of prefix, pubkey tuples + keys_paired_with_prefix: list[tuple[str, str]] = sum( + prefix_leaf_location_tuples, + [], + ) + prefixed_keys_iter = iter(keys_paired_with_prefix) + while prefixed_chunk := list(itertools.islice(prefixed_keys_iter, self.config.parallelism)): + keys_responses = await asyncio.gather( + *[ + self._load_prefixed_hashi_vault_key( + client=client, + secret_url=self.config.secret_url(f'{key_prefix}/{key_path}'), + ) + for (key_prefix, key_path) in prefixed_chunk + ] + ) + self.merge_keys_responses(keys_responses, merged_keys) + + @staticmethod + async def _find_prefixed_hashi_vault_keys( + client: HashiVaultClient, prefix: str, prefix_url: str + ) -> list[tuple[str, str]]: + """ + Discover public keys under prefix in hashi vault K/V secret engine + + All public keys must be a final chunk of the secret path without 0x prefix, + all secret keys are stored under these paths with arbitrary secret dictionary + key, and secret value with or without 0x prefix. + """ + logger.info('Will discover validator keys in %s', prefix_url) + response = await client.request(method='LIST', url=prefix_url) + discovered_keys = response['keys'] + return list(zip([prefix] * len(discovered_keys), discovered_keys)) + + @staticmethod + async def _load_prefixed_hashi_vault_key(client: HashiVaultClient, secret_url: str) -> Keys: + logger.info('Will load keys from %s', secret_url) + response = await client.request( + method='GET', + url=secret_url, + ) + # Last chunk of URL is a public key + pk = add_0x_prefix(HexStr(secret_url.strip('/').split('/')[-1])) + if len(response['data']) > 1: + raise RuntimeError( + f'Invalid multi-value secret at path {secret_url}, ' + 'should only contain single value', + ) + sk = list(response['data'].values())[0] + sk_bytes = Web3.to_bytes(hexstr=sk) + return Keys({pk: BLSPrivkey(sk_bytes)}) + + +class HashiVaultKeystore(LocalKeystore): + @staticmethod + async def load() -> 'HashiVaultKeystore': + """Extracts private keys from the keystores.""" + hashi_vault_config = HashiVaultConfiguration.from_settings() + merged_keys = Keys({}) + + for loader_class, input_list in { + HashiVaultBundledKeysLoader: hashi_vault_config.key_paths, + HashiVaultPrefixedKeysLoader: hashi_vault_config.key_prefixes, + }.items(): + if len(input_list) == 0: + continue + input_iter = iter(input_list) + loader = loader_class( + config=hashi_vault_config, + input_iter=input_iter, + ) + client = await HashiVaultClient.from_hashi_vault_config(hashi_vault_config) + await loader.load(client, merged_keys) + + return HashiVaultKeystore(merged_keys) diff --git a/src/validators/keystores/load.py b/src/validators/keystores/load.py index f9e01c7a..1cf8f45f 100644 --- a/src/validators/keystores/load.py +++ b/src/validators/keystores/load.py @@ -2,7 +2,7 @@ from src.config.settings import settings from src.validators.keystores.base import BaseKeystore -from src.validators.keystores.hashi_vault import HashiVaultKeystore +from src.validators.keystores.hashi_vault.loader import HashiVaultKeystore from src.validators.keystores.local import LocalKeystore from src.validators.keystores.remote import RemoteSignerKeystore diff --git a/src/validators/keystores/tests/test_hashi_vault.py b/src/validators/keystores/tests/test_hashi_vault.py index 79de23cf..d4b76510 100644 --- a/src/validators/keystores/tests/test_hashi_vault.py +++ b/src/validators/keystores/tests/test_hashi_vault.py @@ -1,10 +1,10 @@ import pytest -from aiohttp.client import ClientSession from src.config.settings import settings -from src.validators.keystores.hashi_vault import ( +from src.validators.keystores.hashi_vault.client import HashiVaultClient +from src.validators.keystores.hashi_vault.config import HashiVaultConfiguration +from src.validators.keystores.hashi_vault.loader import ( HashiVaultBundledKeysLoader, - HashiVaultConfiguration, HashiVaultKeystore, HashiVaultPrefixedKeysLoader, ) @@ -23,13 +23,16 @@ async def test_hashi_vault_bundled_keystores_loading( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 1 - config = HashiVaultConfiguration.from_settings() + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None - async with ClientSession() as session: - keystore = await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( - session=session, - secret_url=config.secret_url('ethereum/signing/keystores'), - ) + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) + keystore = await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( + client=client, + secret_url=config.secret_url('ethereum/signing/keystores'), + ) assert len(keystore) == 2 @@ -45,14 +48,18 @@ async def test_hashi_vault_prefixed_keystores_finding( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 1 + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) - async with ClientSession() as session: - keystores_prefixes = await HashiVaultPrefixedKeysLoader._find_prefixed_hashi_vault_keys( - session=session, - prefix='ethereum/signing/prefixed1', - prefix_url=config.prefix_url('ethereum/signing/prefixed1'), - ) + keystores_prefixes = await HashiVaultPrefixedKeysLoader._find_prefixed_hashi_vault_keys( + client=client, + prefix='ethereum/signing/prefixed1', + prefix_url=config.prefix_url('ethereum/signing/prefixed1'), + ) assert len(keystores_prefixes) == 2 @pytest.mark.usefixtures('mocked_hashi_vault') @@ -67,15 +74,18 @@ async def test_hashi_vault_prefixed_keystores_loading( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 1 - config = HashiVaultConfiguration.from_settings() + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None - async with ClientSession() as session: - keystore = await HashiVaultPrefixedKeysLoader._load_prefixed_hashi_vault_key( - session=session, - secret_url=config.secret_url( - 'ethereum/signing/prefixed1/8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c', - ), - ) + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) + keystore = await HashiVaultPrefixedKeysLoader._load_prefixed_hashi_vault_key( + client=client, + secret_url=config.secret_url( + 'ethereum/signing/prefixed1/8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c', + ), + ) assert list(keystore.keys()) == [ '0x8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c' ] @@ -91,7 +101,14 @@ async def test_hashi_vault_keystores_not_configured( settings.hashi_vault_key_path = None settings.hashi_vault_parallelism = 1 - with pytest.raises(RuntimeError, match='URL, token and key path must be specified'): + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + + with pytest.raises( + RuntimeError, + match='All three of URL, key path or prefix, and either token or OIDC config for it, must be specified for hashi vault', + ): await HashiVaultConfiguration.from_settings() @pytest.mark.usefixtures('mocked_hashi_vault') @@ -106,15 +123,17 @@ async def test_hashi_vault_bundled_keystores_inaccessible( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 1 - with pytest.raises( - RuntimeError, match='Can not retrieve validator signing keys from hashi vault' - ): - config = HashiVaultConfiguration.from_settings() - async with ClientSession() as session: - await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( - session=session, - secret_url=config.secret_url('ethereum/inaccessible/keystores'), - ) + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + + config = HashiVaultConfiguration.from_settings() + with pytest.raises(RuntimeError, match='Can not retrieve key data from hashi vault'): + client = await HashiVaultClient.from_hashi_vault_config(config) + await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( + client=client, + secret_url=config.secret_url('ethereum/inaccessible/keystores'), + ) @pytest.mark.usefixtures('mocked_hashi_vault') async def test_hashi_vault_bundled_keystores_parallel( @@ -131,13 +150,18 @@ async def test_hashi_vault_bundled_keystores_parallel( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 2 + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) loader = HashiVaultBundledKeysLoader( config=config, input_iter=iter(settings.hashi_vault_key_paths), ) keys = {} - await loader.load(keys) + await loader.load(client, keys) assert len(keys) == 4 @@ -155,14 +179,19 @@ async def test_hashi_vault_bundled_keystores_sequential( ] settings.hashi_vault_parallelism = 1 + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) loader = HashiVaultBundledKeysLoader( config=config, input_iter=iter(settings.hashi_vault_key_paths), ) keys = {} - await loader.load(keys) + await loader.load(client, keys) assert len(keys) == 4 @@ -180,6 +209,10 @@ async def test_hashi_vault_duplicates_parallel( ] settings.hashi_vault_parallelism = 2 + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + keystore = HashiVaultKeystore({}) with pytest.raises(RuntimeError, match='Found duplicate key in path'): await keystore.load() @@ -195,13 +228,16 @@ async def test_hashi_vault_keystores_loading_custom_engine_name( settings.hashi_vault_key_paths = [] settings.hashi_vault_parallelism = 1 - config = HashiVaultConfiguration.from_settings() + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None - async with ClientSession() as session: - keystore = await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( - session=session, - secret_url=config.secret_url('ethereum/signing/keystores'), - ) + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) + keystore = await HashiVaultBundledKeysLoader._load_bundled_hashi_vault_keys( + client=client, + secret_url=config.secret_url('ethereum/signing/keystores'), + ) assert len(keystore) == 2 @@ -217,13 +253,18 @@ async def test_hashi_vault_keystores_prefixed_loader( settings.hashi_vault_key_prefixes = [] settings.hashi_vault_parallelism = 1 + settings.hashi_vault_auth_mount = None + settings.hashi_vault_auth_role = None + settings.hashi_vault_jwt_file = None + config = HashiVaultConfiguration.from_settings() + client = await HashiVaultClient.from_hashi_vault_config(config) loader = HashiVaultPrefixedKeysLoader( config=config, input_iter=iter(['ethereum/signing/prefixed1']) ) keystore = {} - await loader.load(keystore) + await loader.load(client, keystore) assert len(keystore) == 2 @@ -248,3 +289,107 @@ async def test_hashi_vault_load_bundled_and_prefixed( keystore = HashiVaultKeystore({}) keys = await keystore.load() assert len(keys) == 8 + + @pytest.mark.usefixtures('mocked_hashi_vault_oidc_auth') + async def test_hashi_vault_oidc_auth_success( + self, + hashi_vault_url: str, + mocked_jwt_auth_file: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_engine_name = 'secret' + settings.hashi_vault_token = None + settings.hashi_vault_key_paths = [ + 'ethereum/signing/keystores', + ] + settings.hashi_vault_key_prefixes = [ + 'ethereum/signing/prefixed1', + ] + settings.hashi_vault_parallelism = 2 + + settings.hashi_vault_auth_mount = 'kubernetes' + settings.hashi_vault_auth_role = 'GoodIAM' + settings.hashi_vault_jwt_file = mocked_jwt_auth_file + + keystore = HashiVaultKeystore({}) + keys = await keystore.load() + assert len(keys) == 4 + + @pytest.mark.usefixtures('mocked_hashi_vault_oidc_auth') + async def test_hashi_vault_oidc_auth_error( + self, + hashi_vault_url: str, + mocked_jwt_auth_file: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_engine_name = 'secret' + settings.hashi_vault_token = None + settings.hashi_vault_key_paths = [ + 'ethereum/signing/keystores', + ] + settings.hashi_vault_key_prefixes = [ + 'ethereum/signing/prefixed1', + ] + settings.hashi_vault_parallelism = 2 + + settings.hashi_vault_auth_mount = 'kubernetes_error' + settings.hashi_vault_auth_role = 'BadIAM' + settings.hashi_vault_jwt_file = mocked_jwt_auth_file + + keystore = HashiVaultKeystore({}) + with pytest.raises( + RuntimeError, match='Can not authenticate with Hashi Vault via OID connect' + ): + await keystore.load() + + @pytest.mark.usefixtures('mocked_hashi_vault_oidc_auth') + async def test_hashi_vault_oidc_auth_reauth_success( + self, + hashi_vault_url: str, + mocked_jwt_auth_file: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_engine_name = 'secret' + settings.hashi_vault_token = None + settings.hashi_vault_key_paths = [ + 'ethereum/signing/keystores', + ] + settings.hashi_vault_key_prefixes = [ + 'ethereum/signing/prefixed2', + ] + settings.hashi_vault_parallelism = 2 + + # Special endpoint where token stale check is always true + settings.hashi_vault_auth_mount = 'kubernetes_reauth' + settings.hashi_vault_auth_role = 'ReauthIAM' + settings.hashi_vault_jwt_file = mocked_jwt_auth_file + + keystore = HashiVaultKeystore({}) + keys = await keystore.load() + assert len(keys) == 4 + + @pytest.mark.usefixtures('mocked_hashi_vault_oidc_auth') + async def test_hashi_vault_oidc_auth_reauth_error( + self, + hashi_vault_url: str, + mocked_jwt_auth_file: str, + ): + settings.hashi_vault_url = hashi_vault_url + settings.hashi_vault_engine_name = 'secret' + settings.hashi_vault_token = None + settings.hashi_vault_key_paths = [ + 'ethereum/signing/keystores', + ] + settings.hashi_vault_key_prefixes = [ + 'ethereum/signing/prefixed2', + ] + settings.hashi_vault_parallelism = 2 + + # Special endpoint where token stale check is always false + settings.hashi_vault_auth_mount = 'kubernetes' + settings.hashi_vault_auth_role = 'ReauthIAM' + settings.hashi_vault_jwt_file = mocked_jwt_auth_file + + keystore = HashiVaultKeystore({}) + with pytest.raises(RuntimeError, match='Can not retrieve key data from hashi vault'): + await keystore.load()