From e300128a3c920eadef5994513e67d393f86380f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Tue, 18 Feb 2025 17:03:35 +0100 Subject: [PATCH 1/4] Add data scheme for /update service --- .../ooniprobe/routers/v1/probe_services.py | 25 +++++++++++++++++-- .../ooniprobe/tests/test_probe_auth.py | 6 ++--- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index bc44deac..eed72a45 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -1,7 +1,7 @@ import logging from datetime import datetime, timezone, timedelta import time -from typing import List +from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Response @@ -131,7 +131,28 @@ def probe_register_post( class ProbeUpdate(BaseModel): - pass + """ + The original format of this comes from: + https://github.com/ooni/orchestra/blob/master/registry/registry/handler/registry.go#L25 + """ + probe_cc : Optional[str] = None + probe_asn : Optional[str] = None + platform : Optional[str] = None + + software_name : Optional[str] = None + software_version : Optional[str] = None + supported_tests : Optional[List[str]] = None + + network_type : Optional[str] = None + available_bandwidth : Optional[str] = None + language : Optional[str] = None + + token : Optional[str] = None + + probe_family : Optional[str] = None + probe_id : Optional[str] = None + + password : Optional[str] = None class ProbeUpdateResponse(BaseModel): diff --git a/ooniapi/services/ooniprobe/tests/test_probe_auth.py b/ooniapi/services/ooniprobe/tests/test_probe_auth.py index 8fe65ac3..274dea93 100644 --- a/ooniapi/services/ooniprobe/tests/test_probe_auth.py +++ b/ooniapi/services/ooniprobe/tests/test_probe_auth.py @@ -1,4 +1,4 @@ -from typing import Dict +from typing import Dict, Any from ooniprobe.common import auth from fastapi.testclient import TestClient @@ -58,14 +58,14 @@ def test_update(client: TestClient, jwt_encryption_key): assert json["status"] == "ok" -def _get_update_data() -> Dict[str, str]: +def _get_update_data() -> Dict[str, Any]: return { "probe_cc": "IT", "probe_asn": "AS1234", "platform": "android", "software_name": "ooni-testing", "software_version": "0.0.1", - "supported_tests": "web_connectivity", + "supported_tests": ["web_connectivity"], "network_type": "wifi", "available_bandwidth": "100", "language": "en", From f86466046b00968b4a10a03a885fddd2b139fd78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Feb 2025 12:57:20 +0100 Subject: [PATCH 2/4] Add metrics for ooni probe login --- .../ooniprobe/routers/v1/probe_services.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index eed72a45..d05c64b5 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -4,6 +4,8 @@ from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Response +from prometheus_client import Counter, Info +from enum import Enum from ...common.dependencies import get_settings from ...common.routers import BaseModel @@ -15,6 +17,11 @@ log = logging.getLogger(__name__) +class Metrics: + PROBE_LOGIN = Counter( + "probe_login_requests", "Requests made to the probe login endpoint", + labelnames=["state", "detail", "login"] + ) class ProbeLogin(BaseModel): # Allow None username and password @@ -43,24 +50,28 @@ def probe_login_post( # TODO: We have to find a way to explicitly log metrics with prometheus. # We're currently using the instrumentator default metrics, like http response counts # Maybe using the same exporter as the instrumentator? + try: dec = decode_jwt(token, audience="probe_login", key=settings.jwt_encryption_key) registration_time = dec["iat"] + log.info("probe login: successful") - # metrics.incr("probe_login_successful") + Metrics.PROBE_LOGIN.labels(login = 'standard', detail="ok", state="successful").inc() + except jwt.exceptions.MissingRequiredClaimError: log.info("probe login: invalid or missing claim") - # metrics.incr("probe_login_failed") + Metrics.PROBE_LOGIN.labels(login = 'standard', detail="invalid_or_missing_claim", state="failed").inc() + raise HTTPException(status_code=401, detail="Invalid credentials") except jwt.exceptions.InvalidSignatureError: log.info("probe login: invalid signature") - # metrics.incr("probe_login_failed") + Metrics.PROBE_LOGIN.labels(login = 'standard', detail="invalid_signature", state="failed").inc() + raise HTTPException(status_code=401, detail="Invalid credentials") except jwt.exceptions.DecodeError: - # Not a JWT token: treat it as a "legacy" login - # return jerror("Invalid or missing credentials", code=401) log.info("probe login: legacy login successful") - # metrics.incr("probe_legacy_login_successful") + Metrics.PROBE_LOGIN.labels(login = 'legacy', detail="ok", state="successful").inc() + registration_time = None exp = datetime.now(timezone.utc) + timedelta(days=7) From 4857fe364f7b19a3f7cabf2fd36cd10ceaebcc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Feb 2025 13:33:55 +0100 Subject: [PATCH 3/4] Add probe update metadata logging --- .../src/ooniprobe/routers/v1/probe_services.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index d05c64b5..ece2d247 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -23,6 +23,10 @@ class Metrics: labelnames=["state", "detail", "login"] ) + PROBE_UPDATE_INFO = Info( + "probe_update_info", "Information reported in the probe update endpoint", + ) + class ProbeLogin(BaseModel): # Allow None username and password # to deliver informational 401 error when they're missing @@ -173,4 +177,17 @@ class ProbeUpdateResponse(BaseModel): @router.put("/update/{client_id}", tags=["ooniprobe"]) def probe_update_post(probe_update: ProbeUpdate) -> ProbeUpdateResponse: log.info("update successful") + + # Log update metadata into prometheus + probe_update_dict = probe_update.model_dump(exclude_none=True) + + # Info doesn't allows list, if we have a list we have to convert it + # to string + if probe_update_dict['supported_tests'] is not None: + tests = probe_update_dict['supported_tests'] + tests_str = ";".join(tests) + probe_update_dict['supported_tests'] = tests_str + + Metrics.PROBE_UPDATE_INFO.info(probe_update_dict) + return ProbeUpdateResponse(status="ok") From 3a0185115e32714c2d3b62f5ead4a2147133c4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20D=C3=ADaz?= Date: Wed, 19 Feb 2025 13:36:50 +0100 Subject: [PATCH 4/4] remove done todo --- .../ooniprobe/src/ooniprobe/routers/v1/probe_services.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index ece2d247..63043868 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -51,9 +51,6 @@ def probe_login_post( raise HTTPException(status_code=401, detail="Missing credentials") token = probe_login.username - # TODO: We have to find a way to explicitly log metrics with prometheus. - # We're currently using the instrumentator default metrics, like http response counts - # Maybe using the same exporter as the instrumentator? try: dec = decode_jwt(token, audience="probe_login", key=settings.jwt_encryption_key)