Merge pull request #41 from orange-cloudfoundry/rda

adds /v2/ endpoint that changes Accept header sent to reuested exporters to `text/plain`

README.md

@@ -1,105 +1,129 @@
 # Promfetcher
 
-Promfetcher was made for [cloud foundry](https://cloudfoundry.org) and the idea behind is to give ability to fetch metrics from all app instances in a cloud foundry environment.
+Promfetcher was made for [Cloud Foundry] in order to expose [OpenMetrics] from all instances
+of an App in a Cloud Foundry environment.
 
-User can retrieve is metrics by simply call `/v1/apps/[org_name]/[space_name]/[app_name]/metrics` or by route url `/v1/apps/metrics?route_url=my.route.com` which will merge all metrics from app(s) instances and add labels:
+User can retrieve the metrics by simply calling `/v1/apps/${org_name}/${space_name}/${app_name}/metrics`,
+or by the application route URL through `/v1/apps/metrics?route_url=my.route.com`,
+which will merge all the metrics from an App instances,
+then add the following labels (similar to what can be found in the variable [`VCAP_APPLICATION`]):
 
-- `organization_id`
-- `space_id`
-- `app_id`
-- `organization_name`
-- `space_name`
-- `app_name`
-- `index` - app instance index
-- `instance_id` - the same as index
-- `instance` - real container address
+- `organization_id` - The GUID identifying the org where the app is deployed.
+- `organization_name` - The human-readable name of the org where the app is deployed.
+- `space_id` - The GUID identifying the space where the app is deployed.
+- `space_name` - The human-readable name of the space where the app is deployed.
+- `app_id` - The GUID identifying the app.
+- `app_name` - The name assigned to the app when it was pushed.
+- `index` - The index number of the app instance.
+- `instance_id` - (same as the `index`)
+- `instance` - The real IP address and port of the container running the App instance.
 
-It also a service broker for cloud foundry to be able to set metrics endpoint for a particular which not use `/metrics` by default.
+It is also a Cloud Foundry [service broker] able to expose an endpoint containing some system metrics
+for applications without one.
 
-## Example
+[Cloud Foundry]: https://cloudfoundry.org
+[service broker]: https://docs.cloudfoundry.org/services/overview.html
+[`VCAP_APPLICATION`]: https://docs.cloudfoundry.org/devguide/deploy-apps/environment-variable.html#VCAP-APPLICATION
 
-Metrics from app instance 0:
+## Usage
 
-```
-go_memstats_mspan_sys_bytes{} 65536
-```
+### Set up
 
-Metrics from app instance 1:
+On Cloud Foundry, you should deploy it through its corresponding [BOSH release]:
+https://github.com/orange-cloudfoundry/promfetcher-release
 
-```
-go_memstats_mspan_sys_bytes{} 5600
-```
+[BOSH release]: https://bosh.io/releases/
 
-become:
+### Standard endpoint
 
-```
-go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="0",instance_id="0",instance="172.76.112.90:61038"} 65536
-go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="1",instance_id="1",instance="172.76.112.91:61010"} 65536
-```
+If your Apps metrics are available on the `/metrics` path (as per [OpenMetrics] recommendations),
+you have nothing else to do and you can retrieve App instances metrics by simply calling one of:
+
+- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/metrics`
+- `promfetcher.example.net/v1/apps/{app_id}/metrics`
+- `promfetcher.example.net/v1/apps/metrics?app="[org_name]/[space_name]/[app_name]"`
+- `promfetcher.example.net/v1/apps/metrics?app="[app_id]"`
+- `promfetcher.example.net/v1/apps/{route.url.com}/metrics`
+- `promfetcher.example.net/v1/apps/metrics?route_url="[route.url.com]"`
+
+To retrieve only the metrics exposed from your application (without the Promfetcher sugar coating),
+use `/only-app-metrics` instead of `/metrics`, i.e.:
+
+- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/only-app-metrics`
+- `promfetcher.example.net/v1/apps/only-app-metrics?app="[app_id]"`
 
-## How to use ?
+### Setting a custom endpoint
 
-## If metrics available on `/metrics` on your app
+Add to your querystring the parameter `metric_path={/my-metrics/endpoint}`, i.e.:
 
-You have nothing to do, you can retrieve app instances metrics by simply call one of:
+- `promfetcher.example.net/v1/apps/{org_name}/{space_name}/{app_name}/metrics?metric_path=/my-metrics/endpoint`
 
-- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/metrics](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/metrics)
-- [my.promfetcher.com/v1/apps/\[app_id\]/metrics](my.promfetcher.com/v1/apps/{app_id}/metrics)
-- [my.promfetcher.com/v1/apps/metrics?app="\[org_name\]/\[space_name\]/\[app_name\]"](my.promfetcher.com/v1/apps/metrics?app="\[org_name\]/\[space_name\]/\[app_name\]")
-- [my.promfetcher.com/v1/apps/metrics?app="\[app_id\]"](my.promfetcher.com/v1/apps/metrics?app="\[app_id\]")
-- [my.promfetcher.com/v1/apps/\[route.url.com\]/metrics](my.promfetcher.com/v1/apps/{route.url.com}/metrics)
-- [my.promfetcher.com/v1/apps/metrics?route_url="\[route.url.com\]"](my.promfetcher.com/v1/apps/metrics?route_url="\[route.url.com\]")
+### Pass HTTP headers to the App
 
-## Set a different endpoint
+If you do a request with headers, they are all passed to the App.
 
-Add url param `metric_path=/my-metrics/endpoint`, e.g.:
+This is useful for authentication purpose, for example:
 
-- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/metrics?metric_path=/my-metrics/endpoint](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/metrics?metric_path=/my-metrics/endpoint)
+1. I have an App with metrics on `/metrics` which is protected with HTTP Basic Auth
+2. You can perform curl: `curl https://username:[email protected]/v1/apps/my-app/metrics`
+3. HTTP Basic Auth headers are passed to the App, and you can retrieve the information
+   (note that Promfetcher does not store any data)
 
-## Pass http headers to app, useful for authentication
+## Under the hood
 
-If you do a request with headers, they are all passed to app.
+### How does it work?
 
-This is useful for authentication purpose, example on basic auth
+Promfetcher only needs [gorouter] and will read route table from it.
 
-1. I have an app with metrics on `/metrics` but it is protected with basic auth `foo`/`bar`
-2. You can perform curl: `curl https://foo:[email protected]/v1/apps/my-app/metrics`
-3. Basic auth header are passed to app and you can retrieve information (note that promfetcher do not store anything)
+When asking metrics for an App, Promfetcher will asynchronously call all App instances
+(provided by the gorouter routing table) metrics endpoint and merge them together with new labels.
 
-## Retrieving only metrics from your app and not those from external
+[gorouter]: https://github.com/cloudfoundry/gorouter
 
-Use `/only-app-metrics` instead of `/metrics`, e.g.:
+Example, given an App with metrics from instance 0:
 
-- [my.promfetcher.com/v1/apps/\[org_name\]/\[space_name\]/\[app_name\]/only-app-metrics](my.promfetcher.com/v1/apps/{org_name}/{space_name}/{app_name}/only-app-metrics)
-- [my.promfetcher.com/v1/apps/only-app-metrics?app="\[app_id\]"](my.promfetcher.com/v1/apps/only-app-metrics?app="\[app_id\]")
+```
+go_memstats_mspan_sys_bytes{} 65536
+```
+
+And metrics from App instance 1:
 
-## How it works ?
+```
+go_memstats_mspan_sys_bytes{} 5600
+```
+
+Promfetcher will merge them to:
+
+```
+go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="0",instance_id="0",instance="172.76.112.90:61038"} 65536
+go_memstats_mspan_sys_bytes{organization_id="7d66c7e7-196a-40e5-a259-f5afaf6a56f4",space_id="2ac205af-e18f-49a9-9a8b-48ef2bab2292",app_id="621617db-9dd9-4211-8848-b245f3ea16b2",organization_name="system",space_name="tools",app_name="app",index="1",instance_id="1",instance="172.76.112.91:61010"} 65536
+```
 
-Promfetcher only needs [gorouter](https://github.com/cloudfoundry/gorouter) and will read route table from it.
 
-When asking metrics for an app, promfetcher will call async all app instance (gave by routing table from gorouter) metrics endpoint and merge them together with new labels.
+### Graceful shutdown
 
-## How to deploy ?
+Upon receiving `SIGINT`, `SIGTERM` or `SIGUSR1`, Promfetcher will stop listening to new connections
+and will wait up to 15 seconds to let the processing transactions a chance to finish before exiting.
 
-You should deploy it with boshrelease associated with: https://github.com/orange-cloudfoundry/promfetcher-release
+### Health Check
 
-## Metrics
+The default Promfetcher [Health Check] is of "port" type on `8080`.
 
-Promfetcher expose metrics on `/metrics`:
+Promfetcher answers with an HTTP 200 status if healthy and HTTP 503 otherwise.
 
-- `promfetch_metric_fetch_failed_total`: Number of non fetched metrics without be an normal error.
-- `promfetch_metric_fetch_success_total`: Number of fetched metrics succeeded for an app (app instance call are summed).
-- `promfetch_latest_time_scrape_route`: Last time that route has been scraped in seconds.
-- `promfetch_scrape_route_failed_total`: Number of non fetched metrics without be an normal error.
+[Health Check]: https://docs.cloudfoundry.org/devguide/deploy-apps/healthchecks.html
 
-## Graceful shutdown
+The administrator can send a `SIGUSR1` to force an unhealthy status in addition to stop it gracefully.
 
-Promfetcher when receiving a SIGINT or SIGTERM or SIGUSR1 signal will stop listening new connections and will wait to finish opened requests before stopping. If opened requests are not finished after 15 seconds the server will be hard closed.
+### Promfetcher's internal metrics
 
-User can
+Promfetcher metrics are exposed on the [OpenMetrics] standard path `/metrics` and contains the following:
 
-## Health Check
+[//]: # (curl -s https://promfetcher.example.net/metrics | sed -n 's/^# HELP \(promfetch_[^ ]*\)/- `\1`:/p')
 
-Health check is available by default on port 8080. If promfetcher is not healthy or not yet healthy it will respond a 503 error, if not it will respond a 200.
+- `promfetch_metric_fetch_failed_total`: Number of non-fetched metrics without be a normal error.
+- `promfetch_metric_fetch_success_total`: Number of fetched metrics succeeded for an App (App instances calls are summed).
+- `promfetch_latest_time_scrape_route`: Last time that route has been scraped, in seconds.
+- `promfetch_scrape_route_failed_total`: Number of non-fetched metrics without be an normal error.
 
-User can send a `USR1` signal on promfetcher to set unhealthy on health check in addition to stop gracefully.
+[OpenMetrics]: https://github.com/OpenObservability/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md

api/init.go

@@ -22,20 +22,37 @@ func Register(rtr *mux.Router, metFetcher *fetchers.MetricsFetcher, broker *Brok
 	}
 
 	handlerMetrics := handlers.CompressHandler(http.HandlerFunc(api.metrics))
-	rtr.Handle("/v1/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics).
+	handlerOnlyAppMetrics := handlers.CompressHandler(forceOnlyForApp(http.HandlerFunc(api.metrics)))
+
+	// API v1: deprecated
+	routerApiV1 := rtr.PathPrefix("/v1").Subrouter()
+	routerApiV1.Handle("/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics).
 		Methods(http.MethodGet)
 
-	rtr.Handle("/v1/apps/metrics", handlerMetrics).
+	routerApiV1.Handle("/apps/metrics", handlerMetrics).
 		Methods(http.MethodGet)
 
-	handlerOnlyAppMetrics := handlers.CompressHandler(forceOnlyForApp(http.HandlerFunc(api.metrics)))
+	routerApiV1.Handle("/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics).
+		Methods(http.MethodGet)
+
+	routerApiV1.Handle("/apps/only-app-metrics", handlerOnlyAppMetrics).
+		Methods(http.MethodGet)
+
+	// API v2
+	routerApiV2 := rtr.PathPrefix("/v2").Subrouter()
+	routerApiV2.Handle("/apps/{appIdOrPathOrName:.*}/metrics", handlerMetrics).
+		Methods(http.MethodGet)
+
+	routerApiV2.Handle("/apps/metrics", handlerMetrics).
+		Methods(http.MethodGet)
 
-	rtr.Handle("/v1/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics).
+	routerApiV2.Handle("/apps/{appIdOrPathOrName:.*}/only-app-metrics", handlerOnlyAppMetrics).
 		Methods(http.MethodGet)
 
-	rtr.Handle("/v1/apps/only-app-metrics", handlerOnlyAppMetrics).
+	routerApiV2.Handle("/apps/only-app-metrics", handlerOnlyAppMetrics).
 		Methods(http.MethodGet)
 
+	// non-API routes
 	rtr.NewRoute().MatcherFunc(func(req *http.Request, m *mux.RouteMatch) bool {
 		return strings.HasPrefix(req.URL.Path, "/broker/v2")
 	}).Handler(http.StripPrefix("/broker", broker.Handler()))

api/metrics.go

@@ -3,6 +3,7 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"regexp"
 	"strings"
 
 	"github.com/gorilla/mux"
@@ -35,6 +36,15 @@ func (a Api) metrics(w http.ResponseWriter, req *http.Request) {
 	_, onlyAppMetrics := req.URL.Query()["only_from_app"]
 
 	headersMetrics := make(http.Header)
+	// extract the API version from the requested path (ie: /v2)
+	// and set it to an HTTP header
+	apiVersion := regexp.MustCompile("/v([0-9]+)(?:/|$)").FindStringSubmatch(req.URL.Path)
+	if len(apiVersion) == 2 && apiVersion[1] != "1" {
+		headersMetrics.Set("Accept", string(expfmt.FmtText))
+	} else {
+		headersMetrics.Set("Accept", `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`)
+	}
+
 	auth := req.Header.Get("Authorization")
 	if auth != "" {
 		headersMetrics.Set("Authorization", auth)

scrapers/scrape.go

@@ -14,8 +14,6 @@ import (
 	"github.com/orange-cloudfoundry/promfetcher/models"
 )
 
-const acceptHeader = `application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1`
-
 type Scraper struct {
 	backendFactory *clients.BackendFactory
 	db             *gorm.DB
@@ -69,7 +67,6 @@ func (s Scraper) Scrape(route *models.Route, metricPathDefault string, headers h
 			req.Header[k] = v
 		}
 	}
-	req.Header.Add("Accept", acceptHeader)
 	req.Header.Add("Accept-Encoding", "gzip")
 	req.Header.Set("X-Prometheus-Scrape-Timeout-Seconds", fmt.Sprintf("%f", (30*time.Second).Seconds()))
 	req.Header.Set("X-Forwarded-Proto", scheme)