From de8eb4092ba967900c9aae2e646290a3ae729312 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Tue, 13 Dec 2022 16:06:48 +0100 Subject: [PATCH 1/3] README: typos --- README.md | 154 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 89 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 359a2b77..ba07bd13 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,129 @@ # Promfetcher -Promfetcher was made for [cloud foundry](https://cloudfoundry.org) and the idea behind is to give ability to fetch metrics from all app instances in a cloud foundry environment. +Promfetcher was made for [Cloud Foundry] in order to expose [OpenMetrics] from all instances +of an App in a Cloud Foundry environment. -User can retrieve is metrics by simply call `/v1/apps/[org_name]/[space_name]/[app_name]/metrics` or by route url `/v1/apps/metrics?route_url=my.route.com` which will merge all metrics from app(s) instances and add labels: +User can retrieve the metrics by simply calling `/v1/apps/${org_name}/${space_name}/${app_name}/metrics`, +or by the application route URL through `/v1/apps/metrics?route_url=my.route.com`, +which will merge all the metrics from an App instances, +then add the following labels (similar to what can be found in the variable [`VCAP_APPLICATION`]): -- `organization_id` -- `space_id` -- `app_id` -- `organization_name` -- `space_name` -- `app_name` -- `index` - app instance index -- `instance_id` - the same as index -- `instance` - real container address +- `organization_id` - The GUID identifying the org where the app is deployed. +- `organization_name` - The human-readable name of the org where the app is deployed. +- `space_id` - The GUID identifying the space where the app is deployed. +- `space_name` - The human-readable name of the space where the app is deployed. +- `app_id` - The GUID identifying the app. +- `app_name` - The name assigned to the app when it was pushed. +- `index` - The index number of the app instance. +- `instance_id` - (same as the `index`) +- `instance` - The real IP address and port of the container running the App instance. -It also a service broker for cloud foundry to be able to set metrics endpoint for a particular which not use `/metrics` by default. +It is also a Cloud Foundry [service broker] able to expose an endpoint containing some system metrics +for applications without one. -## Example +[Cloud Foundry]: https://cloudfoundry.org +[service broker]: https://docs.cloudfoundry.org/services/overview.html +[`VCAP_APPLICATION`]: https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html#VCAP-APPLICATION -Metrics from app instance 0: +## Usage -``` -go_memstats_mspan_sys_bytes{} 65536 -``` +### Set up -Metrics from app instance 1: +On Cloud Foundry, you should deploy it through its corresponding [BOSH release]: +https://github.com/orange-cloudfoundry/promfetcher-release -``` -go_memstats_mspan_sys_bytes{} 5600 -``` +[BOSH release]: https://bosh.io/releases/ -become: +### Standard endpoint -``` -go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="0",instance_id="0",instance="172.76.112.90:61038"} 65536 -go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="1",instance_id="1",instance="172.76.112.91:61010"} 65536 -``` +If your Apps metrics are available on the `/metrics` path (as per [OpenMetrics] recommendations), +you have nothing else to do and you can retrieve App instances metrics by simply calling one of: + +- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/metrics` +- `promfetcher.example.net/v1/apps/{app_id}/metrics` +- `promfetcher.example.net/v1/apps/metrics?app="[org_name]/[space_name]/[app_name]"` +- `promfetcher.example.net/v1/apps/metrics?app="[app_id]"` +- `promfetcher.example.net/v1/apps/{route.url.com}/metrics` +- `promfetcher.example.net/v1/apps/metrics?route_url="[route.url.com]"` + +To retrieve only the metrics exposed from your application (without the Promfetcher sugar coating), +use `/only-app-metrics` instead of `/metrics`, i.e.: + +- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/only-app-metrics` +- `promfetcher.example.net/v1/apps/only-app-metrics?app="[app_id]"` -## How to use ? +### Setting a custom endpoint -## If metrics available on `/metrics` on your app +Add to your querystring the parameter `metric_path={/my-metrics/endpoint}`, i.e.: -You have nothing to do, you can retrieve app instances metrics by simply call one of: +- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/metrics?metric_path=/my-metrics/endpoint` -- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/metrics](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/metrics) -- [my.promfetcher.com/v1/apps/\[app_id\]/metrics](my.promfetcher.com/v1/apps/{app_id}/metrics) -- [my.promfetcher.com/v1/apps/metrics?app="\[org_name\]/\[space_name\]/\[app_name\]"](my.promfetcher.com/v1/apps/metrics?app="\[org_name\]/\[space_name\]/\[app_name\]") -- [my.promfetcher.com/v1/apps/metrics?app="\[app_id\]"](my.promfetcher.com/v1/apps/metrics?app="\[app_id\]") -- [my.promfetcher.com/v1/apps/\[route.url.com\]/metrics](my.promfetcher.com/v1/apps/{route.url.com}/metrics) -- [my.promfetcher.com/v1/apps/metrics?route_url="\[route.url.com\]"](my.promfetcher.com/v1/apps/metrics?route_url="\[route.url.com\]") +### Pass HTTP headers to the App -## Set a different endpoint +If you do a request with headers, they are all passed to the App. -Add url param `metric_path=/my-metrics/endpoint`, e.g.: +This is useful for authentication purpose, for example: -- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/metrics?metric_path=/my-metrics/endpoint](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/metrics?metric_path=/my-metrics/endpoint) +1. I have an App with metrics on `/metrics` which is protected with HTTP Basic Auth +2. You can perform curl: `curl https://username:password@promfetcher.example.net/v1/apps/my-app/metrics` +3. HTTP Basic Auth headers are passed to the App, and you can retrieve the information + (note that Promfetcher does not store any data) -## Pass http headers to app, useful for authentication +## Under the hood -If you do a request with headers, they are all passed to app. +### How does it work? -This is useful for authentication purpose, example on basic auth +Promfetcher only needs [gorouter] and will read route table from it. -1. I have an app with metrics on `/metrics` but it is protected with basic auth `foo`/`bar` -2. You can perform curl: `curl https://foo:bar@my.promfetcher.com/v1/apps/my-app/metrics` -3. Basic auth header are passed to app and you can retrieve information (note that promfetcher do not store anything) +When asking metrics for an App, Promfetcher will asynchronously call all App instances +(provided by the gorouter routing table) metrics endpoint and merge them together with new labels. -## Retrieving only metrics from your app and not those from external +[gorouter]: https://github.com/cloudfoundry/gorouter -Use `/only-app-metrics` instead of `/metrics`, e.g.: +Example, given an App with metrics from instance 0: -- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/only-app-metrics](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/only-app-metrics) -- [my.promfetcher.com/v1/apps/only-app-metrics?app="\[app_id\]"](my.promfetcher.com/v1/apps/only-app-metrics?app="\[app_id\]") +``` +go_memstats_mspan_sys_bytes{} 65536 +``` + +And metrics from App instance 1: -## How it works ? +``` +go_memstats_mspan_sys_bytes{} 5600 +``` + +Promfetcher will merge them to: + +``` +go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="0",instance_id="0",instance="172.76.112.90:61038"} 65536 +go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="1",instance_id="1",instance="172.76.112.91:61010"} 65536 +``` -Promfetcher only needs [gorouter](https://github.com/cloudfoundry/gorouter) and will read route table from it. -When asking metrics for an app, promfetcher will call async all app instance (gave by routing table from gorouter) metrics endpoint and merge them together with new labels. +### Graceful shutdown -## How to deploy ? +Upon receiving `SIGINT`, `SIGTERM` or `SIGUSR1`, Promfetcher will stop listening to new connections +and will wait up to 15 seconds to let the processing transactions a chance to finish before exiting. -You should deploy it with boshrelease associated with: https://github.com/orange-cloudfoundry/promfetcher-release +### Health Check -## Metrics +The default Promfetcher [Health Check] is of "port" type on `8080`. -Promfetcher expose metrics on `/metrics`: +Promfetcher answers with an HTTP 200 status if healthy and HTTP 503 otherwise. -- `promfetch_metric_fetch_failed_total`: Number of non fetched metrics without be an normal error. -- `promfetch_metric_fetch_success_total`: Number of fetched metrics succeeded for an app (app instance call are summed). -- `promfetch_latest_time_scrape_route`: Last time that route has been scraped in seconds. -- `promfetch_scrape_route_failed_total`: Number of non fetched metrics without be an normal error. +[Health Check]: https://docs.cloudfoundry.org/devguide/deploy-apps/healthchecks.html -## Graceful shutdown +The administrator can send a `SIGUSR1` to force an unhealthy status in addition to stop it gracefully. -Promfetcher when receiving a SIGINT or SIGTERM or SIGUSR1 signal will stop listening new connections and will wait to finish opened requests before stopping. If opened requests are not finished after 15 seconds the server will be hard closed. +### Promfetcher's internal metrics -User can +Promfetcher metrics are exposed on the [OpenMetrics] standard path `/metrics` and contains the following: -## Health Check +[//]: # (curl -s https://promfetcher.example.net/metrics | sed -n 's/^# HELP \(promfetch_[^ ]*\)/- `\1`:/p') -Health check is available by default on port 8080. If promfetcher is not healthy or not yet healthy it will respond a 503 error, if not it will respond a 200. +- `promfetch_metric_fetch_failed_total`: Number of non-fetched metrics without be a normal error. +- `promfetch_metric_fetch_success_total`: Number of fetched metrics succeeded for an App (App instances calls are summed). +- `promfetch_latest_time_scrape_route`: Last time that route has been scraped, in seconds. +- `promfetch_scrape_route_failed_total`: Number of non-fetched metrics without be an normal error. -User can send a `USR1` signal on promfetcher to set unhealthy on health check in addition to stop gracefully. +[OpenMetrics]: https://github.com/OpenObservability/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md From 82206db009ca574161d6e299686136782ea1aa70 Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Thu, 15 Dec 2022 14:01:42 +0100 Subject: [PATCH 2/3] change the Accept header sent to the exporters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a new version of the API endpoint (`/v2`). This new version change the `Accept` HTTP header sent to the exporters requested (from `application/openmetrics-text` … to `text/plain`) because the Go library we use does not seems to support all OpenMetrics specificities ^1. The requests sent to the exporters contains a new HTTP header `X-Promfetcher-API-version` with the Promfetchers API version. ^1: Java actuator does respect the OpenMetrics specifications; the double quotes in the comments is not supported for example: > # HELP process_cpu_usage The \"recent cpu usage\" for the Java Virtual Machine process See also: [1]: https://github.com/prometheus/client_golang/issues/829#issuecomment-752753238 [2]: https://github.com/prometheus/common/issues/214 --- api/init.go | 27 ++++++++++++++++++++++----- api/metrics.go | 12 ++++++++++++ models/api.go | 4 ++++ scrapers/scrape.go | 12 +++++++++--- 4 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 models/api.go diff --git a/api/init.go b/api/init.go index 7bcddf4d..56b99f9d 100644 --- a/api/init.go +++ b/api/init.go @@ -22,20 +22,37 @@ func Register(rtr *mux.Router, metFetcher *fetchers.MetricsFetcher, broker *Brok } handlerMetrics := handlers.CompressHandler(http.HandlerFunc(api.metrics)) - rtr.Handle("/v1/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics). + handlerOnlyAppMetrics := handlers.CompressHandler(forceOnlyForApp(http.HandlerFunc(api.metrics))) + + // API v1: deprecated + routerApiV1 := rtr.PathPrefix("/v1").Subrouter() + routerApiV1.Handle("/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics). Methods(http.MethodGet) - rtr.Handle("/v1/apps/metrics", handlerMetrics). + routerApiV1.Handle("/apps/metrics", handlerMetrics). Methods(http.MethodGet) - handlerOnlyAppMetrics := handlers.CompressHandler(forceOnlyForApp(http.HandlerFunc(api.metrics))) + routerApiV1.Handle("/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics). + Methods(http.MethodGet) + + routerApiV1.Handle("/apps/only-app-metrics", handlerOnlyAppMetrics). + Methods(http.MethodGet) + + // API v2 + routerApiV2 := rtr.PathPrefix("/v2").Subrouter() + routerApiV2.Handle("/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics). + Methods(http.MethodGet) + + routerApiV2.Handle("/apps/metrics", handlerMetrics). + Methods(http.MethodGet) - rtr.Handle("/v1/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics). + routerApiV2.Handle("/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics). Methods(http.MethodGet) - rtr.Handle("/v1/apps/only-app-metrics", handlerOnlyAppMetrics). + routerApiV2.Handle("/apps/only-app-metrics", handlerOnlyAppMetrics). Methods(http.MethodGet) + // non-API routes rtr.NewRoute().MatcherFunc(func(req *http.Request, m *mux.RouteMatch) bool { return strings.HasPrefix(req.URL.Path, "/broker/v2") }).Handler(http.StripPrefix("/broker", broker.Handler())) diff --git a/api/metrics.go b/api/metrics.go index 793961f8..de06f847 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -3,15 +3,27 @@ package api import ( "fmt" "net/http" + "regexp" "strings" "github.com/gorilla/mux" + "github.com/orange-cloudfoundry/promfetcher/models" "github.com/prometheus/common/expfmt" "github.com/orange-cloudfoundry/promfetcher/errors" ) func (a Api) metrics(w http.ResponseWriter, req *http.Request) { + // extract the API version from the requested path (ie: /v2) + // and set it to an HTTP header + apiVersion := regexp.MustCompile("/v([0-9]+)(?:/|$)").FindStringSubmatch(req.URL.Path) + if len(apiVersion) == 2 { + req.Header.Set(models.XPromfetcherApiVersion, apiVersion[1]) + } else { + // default to v1 + req.Header.Set(models.XPromfetcherApiVersion, "1") + } + appIdOrPathOrName, ok := mux.Vars(req)["appIdOrPathOrName"] if !ok { appIdOrPathOrName = req.URL.Query().Get("app") diff --git a/models/api.go b/models/api.go new file mode 100644 index 00000000..3e60c15f --- /dev/null +++ b/models/api.go @@ -0,0 +1,4 @@ +package models + +//XPromfetcherApiVersion name of the HTTP header corresponding to the Promfetcher API version +const XPromfetcherApiVersion = `X-Promfetcher-API-version` diff --git a/scrapers/scrape.go b/scrapers/scrape.go index 5b72a358..b4fcc415 100644 --- a/scrapers/scrape.go +++ b/scrapers/scrape.go @@ -12,10 +12,9 @@ import ( "github.com/orange-cloudfoundry/promfetcher/clients" "github.com/orange-cloudfoundry/promfetcher/errors" "github.com/orange-cloudfoundry/promfetcher/models" + "github.com/prometheus/common/expfmt" ) -const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1` - type Scraper struct { backendFactory *clients.BackendFactory db *gorm.DB @@ -69,7 +68,14 @@ func (s Scraper) Scrape(route *models.Route, metricPathDefault string, headers h req.Header[k] = v } } - req.Header.Add("Accept", acceptHeader) + // Prometheus parser is not OpenMetrics compliant + // See: prometheus/common issues: 214, 829 + req.Header.Set("Accept", string(expfmt.FmtText)) + // keep the OpenMetrics accept HTTP header for the /v1 endpoint + if req.Header.Get(models.XPromfetcherApiVersion) == "1" { + req.Header.Set(models.XPromfetcherApiVersion, "1") + req.Header.Set("Accept", `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`) + } req.Header.Add("Accept-Encoding", "gzip") req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", fmt.Sprintf("%f", (30*time.Second).Seconds())) req.Header.Set("X-Forwarded-Proto", scheme) From b22754bcd5a084c191d0cab415e876d3b286fdbc Mon Sep 17 00:00:00 2001 From: Gilles Miraillet Date: Wed, 11 Jan 2023 18:07:31 +0100 Subject: [PATCH 3/3] Changes `Accept` header function of api version used. - v1: `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1` - v2: `text/plain; version=0.0.4; charset=utf-8` --- api/metrics.go | 20 +++++++++----------- models/api.go | 4 ---- scrapers/scrape.go | 9 --------- 3 files changed, 9 insertions(+), 24 deletions(-) delete mode 100644 models/api.go diff --git a/api/metrics.go b/api/metrics.go index de06f847..86dda5a7 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -7,23 +7,12 @@ import ( "strings" "github.com/gorilla/mux" - "github.com/orange-cloudfoundry/promfetcher/models" "github.com/prometheus/common/expfmt" "github.com/orange-cloudfoundry/promfetcher/errors" ) func (a Api) metrics(w http.ResponseWriter, req *http.Request) { - // extract the API version from the requested path (ie: /v2) - // and set it to an HTTP header - apiVersion := regexp.MustCompile("/v([0-9]+)(?:/|$)").FindStringSubmatch(req.URL.Path) - if len(apiVersion) == 2 { - req.Header.Set(models.XPromfetcherApiVersion, apiVersion[1]) - } else { - // default to v1 - req.Header.Set(models.XPromfetcherApiVersion, "1") - } - appIdOrPathOrName, ok := mux.Vars(req)["appIdOrPathOrName"] if !ok { appIdOrPathOrName = req.URL.Query().Get("app") @@ -47,6 +36,15 @@ func (a Api) metrics(w http.ResponseWriter, req *http.Request) { _, onlyAppMetrics := req.URL.Query()["only_from_app"] headersMetrics := make(http.Header) + // extract the API version from the requested path (ie: /v2) + // and set it to an HTTP header + apiVersion := regexp.MustCompile("/v([0-9]+)(?:/|$)").FindStringSubmatch(req.URL.Path) + if len(apiVersion) == 2 && apiVersion[1] != "1" { + headersMetrics.Set("Accept", string(expfmt.FmtText)) + } else { + headersMetrics.Set("Accept", `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`) + } + auth := req.Header.Get("Authorization") if auth != "" { headersMetrics.Set("Authorization", auth) diff --git a/models/api.go b/models/api.go deleted file mode 100644 index 3e60c15f..00000000 --- a/models/api.go +++ /dev/null @@ -1,4 +0,0 @@ -package models - -//XPromfetcherApiVersion name of the HTTP header corresponding to the Promfetcher API version -const XPromfetcherApiVersion = `X-Promfetcher-API-version` diff --git a/scrapers/scrape.go b/scrapers/scrape.go index b4fcc415..75e02fed 100644 --- a/scrapers/scrape.go +++ b/scrapers/scrape.go @@ -12,7 +12,6 @@ import ( "github.com/orange-cloudfoundry/promfetcher/clients" "github.com/orange-cloudfoundry/promfetcher/errors" "github.com/orange-cloudfoundry/promfetcher/models" - "github.com/prometheus/common/expfmt" ) type Scraper struct { @@ -68,14 +67,6 @@ func (s Scraper) Scrape(route *models.Route, metricPathDefault string, headers h req.Header[k] = v } } - // Prometheus parser is not OpenMetrics compliant - // See: prometheus/common issues: 214, 829 - req.Header.Set("Accept", string(expfmt.FmtText)) - // keep the OpenMetrics accept HTTP header for the /v1 endpoint - if req.Header.Get(models.XPromfetcherApiVersion) == "1" { - req.Header.Set(models.XPromfetcherApiVersion, "1") - req.Header.Set("Accept", `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`) - } req.Header.Add("Accept-Encoding", "gzip") req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", fmt.Sprintf("%f", (30*time.Second).Seconds())) req.Header.Set("X-Forwarded-Proto", scheme)