Skip to content

Commit

Permalink
Add logging relation to kserve-controller (#250)
Browse files Browse the repository at this point in the history
* Add logging relation to kserve-controller

Use LogForwarder from loki_push_api library to forwards all logs from
workload container to Loki. This requires to use ops 2.9 or newer, so it
was needed to be updated.

fixes: #236

* bump chisme to 0.4.2
rgildein authored Jul 11, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 2876ed2 commit cb0e103
Showing 10 changed files with 2,880 additions and 88 deletions.
2,750 changes: 2,750 additions & 0 deletions charms/kserve-controller/lib/charms/loki_k8s/v1/loki_push_api.py

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions charms/kserve-controller/metadata.yaml
Original file line number Diff line number Diff line change
@@ -63,3 +63,6 @@ requires:
interface: kubernetes_manifest
service-accounts:
interface: kubernetes_manifest
logging:
interface: loki_push_api
optional: true
3 changes: 3 additions & 0 deletions charms/kserve-controller/requirements-integration.in
Original file line number Diff line number Diff line change
@@ -7,3 +7,6 @@ lightkube
pytest-operator
requests
tenacity
# Pin to >=0.4.0 because the reusable test infrastructure is on that version and above
# This prevents pip-compile from trying to pin an earlier version
charmed-kubeflow-chisme>=0.4.0
136 changes: 66 additions & 70 deletions charms/kserve-controller/requirements-integration.txt
Original file line number Diff line number Diff line change
@@ -4,29 +4,27 @@
#
# pip-compile requirements-integration.in
#
aiohttp==3.8.6
aiohttp==3.9.5
# via -r requirements-integration.in
aiosignal==1.3.1
# via aiohttp
anyio==4.0.0
# via httpcore
appnope==0.1.4
# via ipython
asttokens==2.4.0
anyio==4.4.0
# via httpx
asttokens==2.4.1
# via stack-data
async-timeout==4.0.3
# via aiohttp
attrs==23.1.0
attrs==23.2.0
# via
# aiohttp
# jsonschema
backcall==0.2.0
# via ipython
bcrypt==4.0.1
bcrypt==4.1.3
# via paramiko
cachetools==5.3.1
cachetools==5.3.3
# via google-auth
certifi==2023.7.22
certifi==2024.7.4
# via
# httpcore
# httpx
@@ -36,47 +34,45 @@ cffi==1.16.0
# via
# cryptography
# pynacl
charmed-kubeflow-chisme==0.2.0
charmed-kubeflow-chisme==0.4.2
# via -r requirements-integration.in
charset-normalizer==3.3.0
# via
# aiohttp
# requests
cryptography==41.0.4
charset-normalizer==3.3.2
# via requests
cryptography==42.0.8
# via paramiko
decorator==5.1.1
# via
# ipdb
# ipython
deepdiff==6.2.1
# via charmed-kubeflow-chisme
exceptiongroup==1.1.3
exceptiongroup==1.2.1
# via
# anyio
# pytest
executing==2.0.0
executing==2.0.1
# via stack-data
frozenlist==1.4.0
frozenlist==1.4.1
# via
# aiohttp
# aiosignal
google-auth==2.23.3
google-auth==2.31.0
# via kubernetes
h11==0.14.0
# via httpcore
httpcore==0.18.0
httpcore==1.0.5
# via httpx
httpx==0.25.0
httpx==0.27.0
# via lightkube
hvac==1.2.1
hvac==2.3.0
# via juju
idna==3.4
idna==3.7
# via
# anyio
# httpx
# requests
# yarl
importlib-resources==6.1.0
importlib-resources==6.4.0
# via jsonschema
iniconfig==2.0.0
# via pytest
@@ -86,32 +82,33 @@ ipython==8.12.3
# via ipdb
jedi==0.19.1
# via ipython
jinja2==3.1.2
jinja2==3.1.4
# via
# -r requirements-integration.in
# charmed-kubeflow-chisme
# pytest-operator
jsonschema==4.17.3
# via serialized-data-interface
juju==3.2.2
juju==3.5.0.0
# via
# -r requirements-integration.in
# charmed-kubeflow-chisme
# pytest-operator
kubernetes==28.1.0
kubernetes==30.1.0
# via juju
lightkube==0.14.0
lightkube==0.15.3
# via
# -r requirements-integration.in
# charmed-kubeflow-chisme
lightkube-models==1.28.1.4
lightkube-models==1.30.0.8
# via lightkube
macaroonbakery==1.3.1
macaroonbakery==1.3.4
# via juju
markupsafe==2.1.3
markupsafe==2.1.5
# via jinja2
matplotlib-inline==0.1.6
matplotlib-inline==0.1.7
# via ipython
multidict==6.0.4
multidict==6.0.5
# via
# aiohttp
# yarl
@@ -121,47 +118,47 @@ oauthlib==3.2.2
# via
# kubernetes
# requests-oauthlib
ops==2.14.0
ops==2.14.1
# via
# charmed-kubeflow-chisme
# serialized-data-interface
ordered-set==4.1.0
# via deepdiff
packaging==23.2
# via pytest
paramiko==2.12.0
packaging==24.1
# via
# juju
# pytest
paramiko==3.4.0
# via juju
parso==0.8.3
parso==0.8.4
# via jedi
pexpect==4.8.0
pexpect==4.9.0
# via ipython
pickleshare==0.7.5
# via ipython
pkgutil-resolve-name==1.3.10
# via jsonschema
pluggy==1.3.0
pluggy==1.5.0
# via pytest
prompt-toolkit==3.0.39
prompt-toolkit==3.0.47
# via ipython
protobuf==3.20.3
protobuf==5.27.2
# via macaroonbakery
ptyprocess==0.7.0
# via pexpect
pure-eval==0.2.2
# via stack-data
pyasn1==0.5.0
pyasn1==0.6.0
# via
# juju
# pyasn1-modules
# rsa
pyasn1-modules==0.3.0
pyasn1-modules==0.4.0
# via google-auth
pycparser==2.21
pycparser==2.22
# via cffi
pygments==2.16.1
pygments==2.18.0
# via ipython
pyhcl==0.4.5
# via hvac
pymacaroons==0.13.0
# via macaroonbakery
pynacl==1.5.0
@@ -173,19 +170,19 @@ pyrfc3339==1.1
# via
# juju
# macaroonbakery
pyrsistent==0.19.3
pyrsistent==0.20.0
# via jsonschema
pytest==7.4.2
pytest==8.2.2
# via
# pytest-asyncio
# pytest-operator
pytest-asyncio==0.21.1
pytest-asyncio==0.21.2
# via pytest-operator
pytest-operator==0.29.0
pytest-operator==0.35.0
# via -r requirements-integration.in
python-dateutil==2.8.2
python-dateutil==2.9.0.post0
# via kubernetes
pytz==2023.3.post1
pytz==2024.1
# via pyrfc3339
pyyaml==6.0.1
# via
@@ -195,19 +192,19 @@ pyyaml==6.0.1
# ops
# pytest-operator
# serialized-data-interface
requests==2.31.0
requests==2.32.3
# via
# -r requirements-integration.in
# hvac
# kubernetes
# macaroonbakery
# requests-oauthlib
# serialized-data-interface
requests-oauthlib==1.3.1
requests-oauthlib==2.0.0
# via kubernetes
rsa==4.9
# via google-auth
ruamel-yaml==0.17.35
ruamel-yaml==0.18.6
# via charmed-kubeflow-chisme
ruamel-yaml-clib==0.2.8
# via ruamel-yaml
@@ -218,17 +215,15 @@ six==1.16.0
# asttokens
# kubernetes
# macaroonbakery
# paramiko
# pymacaroons
# python-dateutil
sniffio==1.3.0
sniffio==1.3.1
# via
# anyio
# httpcore
# httpx
stack-data==0.6.3
# via ipython
tenacity==8.2.3
tenacity==8.5.0
# via
# -r requirements-integration.in
# charmed-kubeflow-chisme
@@ -238,29 +233,30 @@ tomli==2.0.1
# pytest
toposort==1.10
# via juju
traitlets==5.11.2
traitlets==5.14.3
# via
# ipython
# matplotlib-inline
typing-extensions==4.8.0
typing-extensions==4.12.2
# via
# anyio
# ipython
# typing-inspect
typing-inspect==0.9.0
# via juju
urllib3==1.26.17
urllib3==2.2.2
# via
# kubernetes
# requests
wcwidth==0.2.8
wcwidth==0.2.13
# via prompt-toolkit
websocket-client==1.6.4
websocket-client==1.8.0
# via
# kubernetes
# ops
websockets==8.1
websockets==12.0
# via juju
yarl==1.9.2
yarl==1.9.4
# via aiohttp
zipp==3.17.0
zipp==3.19.2
# via importlib-resources
8 changes: 7 additions & 1 deletion charms/kserve-controller/requirements-unit.txt
Original file line number Diff line number Diff line change
@@ -19,6 +19,8 @@ charmed-kubeflow-chisme==0.2.0
# -r requirements.in
charset-normalizer==3.3.0
# via requests
cosl==0.0.12
# via -r requirements.in
coverage==7.3.2
# via -r requirements-unit.in
deepdiff==6.2.1
@@ -57,11 +59,12 @@ lightkube-models==1.28.1.4
# lightkube
markupsafe==2.1.3
# via jinja2
ops==2.14.0
ops==2.14.1
# via
# -r requirements-unit.in
# -r requirements.in
# charmed-kubeflow-chisme
# cosl
# serialized-data-interface
ordered-set==4.1.0
# via deepdiff
@@ -82,6 +85,7 @@ pytest-mock==3.11.1
pyyaml==6.0.1
# via
# -r requirements.in
# cosl
# lightkube
# ops
# serialized-data-interface
@@ -104,6 +108,8 @@ tenacity==8.2.3
# via charmed-kubeflow-chisme
tomli==2.0.1
# via pytest
typing-extensions==4.12.2
# via cosl
urllib3==2.0.6
# via requests
websocket-client==1.6.4
2 changes: 2 additions & 0 deletions charms/kserve-controller/requirements.in
Original file line number Diff line number Diff line change
@@ -4,3 +4,5 @@ lightkube-models
ops
pyyaml==6.0.1
serialized-data-interface
# from loki_k8s.v1.loki_push_api.py
cosl
8 changes: 7 additions & 1 deletion charms/kserve-controller/requirements.txt
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@ charmed-kubeflow-chisme==0.2.0
# via -r requirements.in
charset-normalizer==3.3.0
# via requests
cosl==0.0.12
# via -r requirements.in
deepdiff==6.2.1
# via charmed-kubeflow-chisme
exceptiongroup==1.1.3
@@ -48,10 +50,11 @@ lightkube-models==1.28.1.4
# lightkube
markupsafe==2.1.3
# via jinja2
ops==2.14.0
ops==2.14.1
# via
# -r requirements.in
# charmed-kubeflow-chisme
# cosl
# serialized-data-interface
ordered-set==4.1.0
# via deepdiff
@@ -62,6 +65,7 @@ pyrsistent==0.19.3
pyyaml==6.0.1
# via
# -r requirements.in
# cosl
# lightkube
# ops
# serialized-data-interface
@@ -82,6 +86,8 @@ sniffio==1.3.0
# httpx
tenacity==8.2.3
# via charmed-kubeflow-chisme
typing-extensions==4.12.2
# via cosl
urllib3==2.0.6
# via requests
websocket-client==1.6.4
3 changes: 3 additions & 0 deletions charms/kserve-controller/src/charm.py
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
GatewayRelationMissingError,
GatewayRequirer,
)
from charms.loki_k8s.v1.loki_push_api import LogForwarder
from charms.resource_dispatcher.v0.kubernetes_manifests import (
KubernetesManifest,
KubernetesManifestRequirerWrapper,
@@ -163,6 +164,8 @@ def __init__(self, *args):
self._rbac_proxy_container_name = "kube-rbac-proxy"
self.rbac_proxy_container = self.unit.get_container(self._rbac_proxy_container_name)

self._logging = LogForwarder(charm=self)

@property
def _context(self):
"""Returns a dictionary containing context to be used for rendering."""
48 changes: 32 additions & 16 deletions charms/kserve-controller/tests/integration/test_charm.py
Original file line number Diff line number Diff line change
@@ -15,6 +15,10 @@
import tenacity
import yaml
from charmed_kubeflow_chisme.kubernetes import KubernetesResourceHandler
from charmed_kubeflow_chisme.testing import (
assert_logging,
deploy_and_assert_grafana_agent,
)
from lightkube.core.exceptions import ApiError
from lightkube.models.meta_v1 import ObjectMeta
from lightkube.resources.core_v1 import (
@@ -189,7 +193,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
config={"kind": "ingress"},
trust=True,
)
await ops_test.model.add_relation("istio-pilot", "istio-ingressgateway")
await ops_test.model.integrate("istio-pilot", "istio-ingressgateway")
await ops_test.model.wait_for_idle(
["istio-pilot", "istio-ingressgateway"],
raise_on_blocked=False,
@@ -212,7 +216,7 @@ async def test_build_and_deploy(ops_test: OpsTest):
application_name=APP_NAME,
trust=True,
)
await ops_test.model.add_relation("istio-pilot", "kserve-controller")
await ops_test.model.integrate("istio-pilot", APP_NAME)

# issuing dummy update_status just to trigger an event
async with ops_test.fast_forward():
@@ -224,6 +228,11 @@ async def test_build_and_deploy(ops_test: OpsTest):
)
assert ops_test.model.applications[APP_NAME].units[0].workload_status == "active"

# Deploying grafana-agent-k8s and add all relations
await deploy_and_assert_grafana_agent(
ops_test.model, APP_NAME, metrics=False, dashboard=False, logging=True
)


@pytest.fixture()
def test_namespace(lightkube_client: lightkube.Client):
@@ -304,6 +313,12 @@ def assert_inf_svc_state():
assert_inf_svc_state()


async def test_logging(ops_test: OpsTest):
"""Test logging is defined in relation data bag."""
app = ops_test.model.applications[APP_NAME]
await assert_logging(app)


# # Remove the InferenceService deployed in RawDeployment mode
# lightkube_client.delete(
# inference_service_resource, name=inf_svc_name, namespace=rawdeployment_mode_namespace
@@ -350,15 +365,13 @@ async def test_deploy_knative_dependencies(ops_test: OpsTest):
)

# Relate kserve-controller and knative-serving
await ops_test.model.add_relation("knative-serving", "kserve-controller")
await ops_test.model.integrate("knative-serving", APP_NAME)

# Change deployment mode to Serverless
await ops_test.model.applications["kserve-controller"].set_config(
{"deployment-mode": "serverless"}
)
await ops_test.model.applications[APP_NAME].set_config({"deployment-mode": "serverless"})

await ops_test.model.wait_for_idle(
["kserve-controller"],
[APP_NAME],
raise_on_blocked=False,
status="active",
timeout=90 * 10,
@@ -446,11 +459,11 @@ async def test_configmap_changes_with_config(
lightkube_client (lightkube.Client): The Lightkube client to interact with Kubernetes.
ops_test (OpsTest): The Juju OpsTest fixture to interact with the deployed model.
"""
await ops_test.model.applications["kserve-controller"].set_config(
await ops_test.model.applications[APP_NAME].set_config(
{"custom_images": '{"configmap__batcher": "custom:1.0"}'} # noqa: E501
)
await ops_test.model.wait_for_idle(
apps=["kserve-controller"], status="active", raise_on_blocked=True, timeout=300
apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=300
)
inferenceservice_config = lightkube_client.get(
ConfigMap, CONFIGMAP_NAME, namespace=ops_test.model_name
@@ -461,7 +474,10 @@ async def test_configmap_changes_with_config(
async def test_relate_to_object_store(ops_test: OpsTest):
"""Test if the charm can relate to minio and stay in Active state"""
await ops_test.model.deploy(
OBJECT_STORAGE_CHARM_NAME, channel="ckf-1.7/stable", config=OBJECT_STORAGE_CONFIG
OBJECT_STORAGE_CHARM_NAME,
channel="ckf-1.7/stable",
config=OBJECT_STORAGE_CONFIG,
trust=True,
)
await ops_test.model.wait_for_idle(
apps=[OBJECT_STORAGE_CHARM_NAME],
@@ -470,7 +486,7 @@ async def test_relate_to_object_store(ops_test: OpsTest):
raise_on_error=False,
timeout=600,
)
await ops_test.model.relate(OBJECT_STORAGE_CHARM_NAME, CHARM_NAME)
await ops_test.model.integrate(OBJECT_STORAGE_CHARM_NAME, CHARM_NAME)
await ops_test.model.wait_for_idle(
apps=[CHARM_NAME],
status="active",
@@ -510,10 +526,10 @@ async def test_deploy_resource_dispatcher(ops_test: OpsTest):
idle_period=60,
)

await ops_test.model.relate(
await ops_test.model.integrate(
f"{CHARM_NAME}:service-accounts", f"{RESOURCE_DISPATCHER_CHARM_NAME}:service-accounts"
)
await ops_test.model.relate(
await ops_test.model.integrate(
f"{CHARM_NAME}:secrets", f"{RESOURCE_DISPATCHER_CHARM_NAME}:secrets"
)

@@ -553,8 +569,8 @@ async def test_blocked_on_invalid_config(ops_test: OpsTest):
Args:
ops_test (OpsTest): The Juju OpsTest fixture to interact with the deployed model.
"""
await ops_test.model.applications["kserve-controller"].set_config({"custom_images": "{"})
await ops_test.model.applications[APP_NAME].set_config({"custom_images": "{"})
await ops_test.model.wait_for_idle(
apps=["kserve-controller"], status="blocked", raise_on_blocked=False, timeout=300
apps=[APP_NAME], status="blocked", raise_on_blocked=False, timeout=300
)
assert ops_test.model.applications["kserve-controller"].units[0].workload_status == "blocked"
assert ops_test.model.applications[APP_NAME].units[0].workload_status == "blocked"
7 changes: 7 additions & 0 deletions charms/kserve-controller/tests/unit/test_charm.py
Original file line number Diff line number Diff line change
@@ -91,6 +91,13 @@ def mocked_lightkube_client(mocker, mocked_resource_handler):
yield mocked_resource_handler.lightkube_client


def test_log_forwarding(harness):
"""Test LogForwarder initialization."""
with patch("charm.LogForwarder") as mock_logging:
harness.begin()
mock_logging.assert_called_once_with(charm=harness.charm)


def test_events(harness, mocked_resource_handler, mocker):
harness.begin()
on_event = mocker.patch("charm.KServeControllerCharm._on_event")

0 comments on commit cb0e103

Please sign in to comment.