Support of dynamic OIDC/JWT auth for Hashi Vault

src/commands/start.py

@@ -13,6 +13,8 @@
 from src.common.vault_config import VaultConfig
 from src.config.networks import AVAILABLE_NETWORKS
 from src.config.settings import (
+    DEFAULT_HASHI_VAULT_JWT_FILE,
+    DEFAULT_HASHI_VAULT_OIDC_MOUNT,
     DEFAULT_HASHI_VAULT_PARALLELISM,
     DEFAULT_MAX_FEE_PER_GAS_GWEI,
     DEFAULT_METRICS_HOST,
@@ -177,6 +179,23 @@
     envvar='HASHI_VAULT_TOKEN',
     help='Authentication token for accessing Hashi vault.',
 )
+@click.option(
+    '--hashi-vault-oidc-auth-role',
+    envvar='HASHI_VAULT_OIDC_AUTH_ROLE',
+    help='A role to use when authenticating via OID Connect with Hashi vault.',
+)
+@click.option(
+    '--hashi-vault-oidc-auth-mount-point',
+    envvar='HASHI_VAULT_OIDC_AUTH_MOUNT_POINT',
+    help='A role to use when authenticating via OID Connect with Hashi vault.',
+    default=DEFAULT_HASHI_VAULT_OIDC_MOUNT,
+)
+@click.option(
+    '--hashi-vault-jwt-auth-file',
+    envvar='HASHI_VAULT_JWT_FILE',
+    help='A path to file that contains JWT content for dynamically authenticating via OID Connect.',
+    default=DEFAULT_HASHI_VAULT_JWT_FILE,
+)
 @click.option(
     '--hashi-vault-key-path',
     envvar='HASHI_VAULT_KEY_PATH',
@@ -252,6 +271,9 @@ def start(
     hashi_vault_key_path: list[str] | None,
     hashi_vault_key_prefix: list[str] | None,
     hashi_vault_token: str | None,
+    hashi_vault_oidc_auth_role: str | None,
+    hashi_vault_oidc_auth_mount_point: str | None,
+    hashi_vault_jwt_auth_file: str | None,
     hashi_vault_url: str | None,
     hashi_vault_parallelism: int,
     hot_wallet_file: str | None,
@@ -284,6 +306,9 @@ def start(
         keystores_password_file=keystores_password_file,
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
+        hashi_vault_auth_mount=hashi_vault_oidc_auth_mount_point,
+        hashi_vault_auth_role=hashi_vault_oidc_auth_role,
+        hashi_vault_jwt_file=hashi_vault_jwt_auth_file,
         hashi_vault_key_paths=hashi_vault_key_path,
         hashi_vault_key_prefixes=hashi_vault_key_prefix,
         hashi_vault_parallelism=hashi_vault_parallelism,

src/commands/validators_exit.py

@@ -19,6 +19,8 @@
 from src.config.networks import AVAILABLE_NETWORKS
 from src.config.settings import (
     DEFAULT_HASHI_VAULT_ENGINE_NAME,
+    DEFAULT_HASHI_VAULT_JWT_FILE,
+    DEFAULT_HASHI_VAULT_OIDC_MOUNT,
     DEFAULT_HASHI_VAULT_PARALLELISM,
     settings,
 )
@@ -110,6 +112,23 @@ class ValidatorExit:
     help='How much requests to K/V secrets engine to do in parallel.',
     default=DEFAULT_HASHI_VAULT_PARALLELISM,
 )
+@click.option(
+    '--hashi-vault-oidc-auth-role',
+    envvar='HASHI_VAULT_OIDC_AUTH_ROLE',
+    help='A role to use when authenticating via OID Connect with Hashi vault.',
+)
+@click.option(
+    '--hashi-vault-oidc-auth-mount-point',
+    envvar='HASHI_VAULT_OIDC_AUTH_MOUNT_POINT',
+    help='A role to use when authenticating via OID Connect with Hashi vault.',
+    default=DEFAULT_HASHI_VAULT_OIDC_MOUNT,
+)
+@click.option(
+    '--hashi-vault-jwt-auth-file',
+    envvar='HASHI_VAULT_JWT_FILE',
+    help='A path to file that contains JWT content for dynamically authenticating via OID Connect.',
+    default=DEFAULT_HASHI_VAULT_JWT_FILE,
+)
 @click.option(
     '-v',
     '--verbose',
@@ -144,6 +163,9 @@ def validators_exit(
     hashi_vault_key_path: list[str] | None,
     hashi_vault_key_prefix: list[str] | None,
     hashi_vault_token: str | None,
+    hashi_vault_oidc_auth_role: str | None,
+    hashi_vault_oidc_auth_mount_point: str | None,
+    hashi_vault_jwt_auth_file: str | None,
     hashi_vault_url: str | None,
     hashi_vault_engine_name: str,
     hashi_vault_parallelism: int,
@@ -165,6 +187,9 @@ def validators_exit(
         consensus_endpoints=consensus_endpoints,
         remote_signer_url=remote_signer_url,
         hashi_vault_token=hashi_vault_token,
+        hashi_vault_auth_mount=hashi_vault_oidc_auth_mount_point,
+        hashi_vault_auth_role=hashi_vault_oidc_auth_role,
+        hashi_vault_jwt_file=hashi_vault_jwt_auth_file,
         hashi_vault_key_paths=hashi_vault_key_path,
         hashi_vault_key_prefixes=hashi_vault_key_prefix,
         hashi_vault_url=hashi_vault_url,

src/config/settings.py

@@ -21,6 +21,8 @@
 
 DEFAULT_HASHI_VAULT_PARALLELISM = 8
 DEFAULT_HASHI_VAULT_ENGINE_NAME = 'secret'
+DEFAULT_HASHI_VAULT_JWT_FILE = '/var/run/secrets/kubernetes.io/serviceaccount/token'
+DEFAULT_HASHI_VAULT_OIDC_MOUNT = 'kubernetes'
 
 
 # pylint: disable-next=too-many-public-methods,too-many-instance-attributes
@@ -57,6 +59,9 @@ class Settings(metaclass=Singleton):
     hashi_vault_url: str | None
     hashi_vault_engine_name: str
     hashi_vault_token: str | None
+    hashi_vault_jwt_file: Path | None
+    hashi_vault_auth_mount: str | None
+    hashi_vault_auth_role: str | None
     hashi_vault_parallelism: int
     hot_wallet_file: Path
     hot_wallet_password_file: Path
@@ -121,6 +126,9 @@ def set(
         hashi_vault_url: str | None = None,
         hashi_vault_engine_name: str = DEFAULT_HASHI_VAULT_ENGINE_NAME,
         hashi_vault_token: str | None = None,
+        hashi_vault_jwt_file: str | None = DEFAULT_HASHI_VAULT_JWT_FILE,
+        hashi_vault_auth_mount: str | None = DEFAULT_HASHI_VAULT_OIDC_MOUNT,
+        hashi_vault_auth_role: str | None = None,
         hashi_vault_parallelism: int = DEFAULT_HASHI_VAULT_PARALLELISM,
         hot_wallet_file: str | None = None,
         hot_wallet_password_file: str | None = None,
@@ -187,6 +195,11 @@ def set(
         self.hashi_vault_key_paths = hashi_vault_key_paths
         self.hashi_vault_key_prefixes = hashi_vault_key_prefixes
         self.hashi_vault_token = hashi_vault_token
+        self.hashi_vault_jwt_file = (
+            Path(hashi_vault_jwt_file) if hashi_vault_jwt_file is not None else None
+        )
+        self.hashi_vault_auth_mount = hashi_vault_auth_mount
+        self.hashi_vault_auth_role = hashi_vault_auth_role
         self.hashi_vault_parallelism = hashi_vault_parallelism
 
         # hot wallet
@@ -310,8 +323,9 @@ def need_deposit_data_file(self) -> bool:
 )
 REMOTE_SIGNER_TIMEOUT = decouple_config('REMOTE_SIGNER_TIMEOUT', cast=int, default=30)
 
-# Hashi vault timeout
+# Hashi vault timeouts
 HASHI_VAULT_TIMEOUT = 10
+HASHI_VAULT_OIDC_LOGIN_TIMEOUT = 60
 
 # Graphql timeout
 GRAPH_API_TIMEOUT = 10

src/conftest.py

@@ -21,7 +21,13 @@
 from src.common.vault_config import VaultConfig
 from src.config.networks import HOLESKY
 from src.config.settings import settings
-from src.test_fixtures.hashi_vault import hashi_vault_url, mocked_hashi_vault  # noqa
+from src.test_fixtures.hashi_vault import (  # noqa
+    hashi_vault_url,
+    jwt_auth_secret,
+    mocked_hashi_vault,
+    mocked_hashi_vault_oidc_auth,
+    mocked_jwt_auth_file,
+)
 from src.test_fixtures.remote_signer import mocked_remote_signer, remote_signer_url
 from src.validators.keystores.remote import RemoteSignerKeystore
 from src.validators.signing.tests.oracle_functions import OracleCommittee

src/test_fixtures/hashi_vault.py

@@ -1,11 +1,20 @@
 import json
+import tempfile
+import uuid
 from functools import partial
+from pathlib import Path
 from typing import Generator
 
 import pytest
 from aioresponses import CallbackResult, aioresponses
 
 
+@pytest.fixture(scope='session')
+def jwt_auth_secret() -> str:
+    # Generate unique uuid secret for every run
+    return str(uuid.uuid4())
+
+
 @pytest.fixture
 def hashi_vault_url() -> str:
     return 'http://vault:8200'
@@ -144,3 +153,198 @@ def _mocked_error_path(url, **kwargs) -> CallbackResult:
             repeat=True,
         )
         yield
+
+
+@pytest.fixture
+def mocked_hashi_vault_oidc_auth(
+    hashi_vault_url: str,
+    jwt_auth_secret: str,
+) -> Generator:
+    # Generated via
+    # eth-staking-smith existing-mnemonic \
+    #   --chain holesky \
+    #   --num_validators 2 \
+    #   --mnemonic 'provide iron update bronze session immense garage want round enhance artefact position make wash analyst skirt float jealous trend spread ginger rapid express tool'
+    _hashi_vault_pk_sk_mapping = {
+        'b05e93c4501233eeb7f1e7b0ee400caaa04608249c4aab61c18e04c675aaf2a0f03808d533c877fbbd57b04927c01ce0': '3eeedd7a6679d2e2036682b6f03ef16105a847321303aec163548aa3fa5e9eeb',
+        'aa84894836cb3d897a1a11344920c41c472ed67667fd8a3453e557214442370ffc1d007ae7af67120de00afa068349be': '236f33410e6972a2db36ba3736099396768219b327e18eae49392f153007d468',
+    }
+
+    # Generated via
+    # eth-staking-smith existing-mnemonic \
+    #   --chain holesky \
+    #   --num_validators 2 \
+    #   --mnemonic 'route flight verb churn work creek crane hole obscure young shaft area bird border refuse usage flash engage burden retreat drama bamboo profit sense'
+    _hashi_vault_prefixed_pk_sk_mapping = {
+        '8b09379ca969e8283a42a09285f430e8bd58c70bb33b44397ae81dac01b1403d0f631f156d211b6931a1c6284e2e469c': '5d88e114821bf871f321399d99fe58cb24d6434b416f112e8e46077e05399dc0',
+        '8979806d4e5d841758868b208df0dd961c12a0cf044e2de1d18e269ca0ad0308672be2f71d3d5606834764fe5b1d0bc4': '01352aec5cadb78eba6f716570d28b40f24b96c522dac535bc81375ceb54bf0b',
+    }
+
+    state_auth = dict(logins=0)
+
+    def _mocked_auth_login(duration, url, **kwargs):
+        if kwargs['json']['jwt'] != jwt_auth_secret:
+            raise AssertionError('Invalid JWT value passed to a mock')
+        state_auth['logins'] += 1
+        return CallbackResult(
+            status=200,
+            body=json.dumps(
+                {
+                    'auth': {
+                        'client_token': str(uuid.uuid4()),
+                        'accessor': str(uuid.uuid4()),
+                        'policies': ['default'],
+                        'metadata': {
+                            'role': kwargs['json']['role'],
+                        },
+                        'lease_duration': duration,
+                        'renewable': True,
+                    }
+                }
+            ),
+        )
+
+    def _mocked_auth_failure(url, **kwargs):
+        return CallbackResult(
+            status=403,
+            body=json.dumps(
+                {
+                    'errors': ['invalid_token'],
+                }
+            ),
+        )
+
+    def _mocked_secret_path(data, url, **kwargs) -> CallbackResult:
+        return CallbackResult(
+            status=200,
+            body=json.dumps(
+                dict(
+                    data=dict(
+                        data=data,
+                    )
+                )
+            ),  # type: ignore
+        )
+
+    def _mocked_secrets_list(data, url, **kwargs) -> CallbackResult:
+        return CallbackResult(
+            status=200,
+            body=json.dumps(
+                dict(
+                    data=dict(
+                        keys=data,
+                    )
+                )
+            ),  # type: ignore
+        )
+
+    state_reauth_list = dict(attempt=0)
+
+    def _mocked_secrets_list_reauth(data, url, **kwargs) -> CallbackResult:
+        state_reauth_list['attempt'] += 1
+        if state_reauth_list['attempt'] == 1:
+            # Yield 403 invalid token on first attempt
+            return CallbackResult(
+                status=403,
+                body=json.dumps(
+                    dict(
+                        errors=['invalid_token'],
+                    )
+                ),  # type: ignore
+            )
+        else:
+            # On aubsequent request after reauth, return normal payload
+
+            # This ensures only one request is done
+            if state_reauth_list['attempt'] != 2:
+                raise AssertionError('Invalid secret list amount done')
+            # This ensures login has been called twice
+            if state_auth['logins'] != 2:
+                raise AssertionError('Invalid amount of logins was performed')
+
+            return CallbackResult(
+                status=200,
+                body=json.dumps(dict(data=dict(keys=data))),  # type: ignore
+            )
+
+    def _mocked_error_path(url, **kwargs) -> CallbackResult:
+        return CallbackResult(
+            status=200, body=json.dumps(dict(errors=list('token not provided')))  # type: ignore
+        )
+
+    with aioresponses() as m:
+        # Successful authentication with lengthy token duration
+        m.post(
+            f'{hashi_vault_url}/v1/auth/kubernetes/login',
+            callback=partial(_mocked_auth_login, 2764800),
+            repeat=True,
+        )
+
+        # Auth with small lease duration to yield is_stale = true
+        # on the failure leading to reauth
+        m.post(
+            f'{hashi_vault_url}/v1/auth/kubernetes_reauth/login',
+            callback=partial(_mocked_auth_login, 0),
+            repeat=True,
+        )
+
+        # Authentication error
+        m.post(
+            f'{hashi_vault_url}/v1/auth/kubernetes_error/login',
+            callback=_mocked_auth_failure,
+            repeat=True,
+        )
+
+        # Mocked bundled signing keys endpoints
+        m.get(
+            f'{hashi_vault_url}/v1/secret/data/ethereum/signing/keystores',
+            callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping),
+            repeat=True,
+        )
+        # Mocked bundled signing keys on reauth endpoint
+        m.get(
+            f'{hashi_vault_url}/v1/secret/data/ethereum/signing/keystores',
+            callback=partial(_mocked_secret_path, _hashi_vault_pk_sk_mapping),
+            repeat=True,
+        )
+        # Mocked prefixed signing keys endpoints
+        m.add(
+            f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed1',
+            callback=partial(
+                _mocked_secrets_list, list(_hashi_vault_prefixed_pk_sk_mapping.keys())
+            ),
+            repeat=True,
+            method='LIST',
+        )
+        for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping.items():
+            m.get(
+                f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed1/{_pk}',
+                callback=partial(_mocked_secret_path, {'value': _sk}),
+                repeat=True,
+            )
+
+        # Mocked prefixed signing keys endpoints with reauth
+        m.add(
+            f'{hashi_vault_url}/v1/secret/metadata/ethereum/signing/prefixed2',
+            callback=partial(
+                _mocked_secrets_list_reauth, list(_hashi_vault_prefixed_pk_sk_mapping.keys())
+            ),
+            repeat=True,
+            method='LIST',
+        )
+        for _pk, _sk in _hashi_vault_prefixed_pk_sk_mapping.items():
+            m.get(
+                f'{hashi_vault_url}/v1/secret/data/ethereum/signing/prefixed2/{_pk}',
+                callback=partial(_mocked_secret_path, {'value': _sk}),
+                repeat=True,
+            )
+
+        yield
+
+
+@pytest.fixture
+def mocked_jwt_auth_file(jwt_auth_secret: str) -> Generator:
+    with tempfile.NamedTemporaryFile('w', delete=False) as jwt_auth:
+        jwt_auth.write(jwt_auth_secret)
+        jwt_auth.flush()
+        yield Path(jwt_auth.name)