Merge pull request #91 from YANG-DB/otel-demo-integration

add OTEL demo dashboards integration for sample tutorial and Observability

integrations/observability/otel-demo/assets/aliases.json

@@ -0,0 +1,61 @@
+{
+  "endpoint": "/_aliases",
+  "payload": {
+    "actions": [
+      {
+        "add": {
+          "index": "ss4o-metrics-*",
+          "alias": "ss4o_metrics"
+        }
+      },
+      {
+        "add": {
+          "index": "otel-metrics-*",
+          "alias": "ss4o_metrics"
+        }
+      },
+      {
+        "add": {
+          "index": "ss4o-traces-*",
+          "alias": "ss4o_traces"
+        }
+      },
+      {
+        "add": {
+          "index": "jaeger-span-*",
+          "alias": "ss4o_traces"
+        }
+      },
+      {
+        "add": {
+          "index": "otel-v1-apm-span-*",
+          "alias": "ss4o_traces"
+        }
+      },
+      {
+        "add": {
+          "index": "jaeger-service-*",
+          "alias": "ss4o_services"
+        }
+      },
+      {
+        "add": {
+          "index": "otel-v1-apm-service-map",
+          "alias": "ss4o_services"
+        }
+      },
+      {
+        "add": {
+          "index": "otel-events-*",
+          "alias": "ss4o_logs"
+        }
+      },
+      {
+        "add": {
+          "index": "ss4o_logs-*",
+          "alias": "ss4o_logs"
+        }
+      }
+    ]
+  }
+}

integrations/observability/otel-demo/assets/datasources.json

@@ -0,0 +1,12 @@
+{
+  "endpoint": "/_plugins/_query/_datasources",
+  "payload": [
+    {
+      "name": "prometheus",
+      "connector": "prometheus",
+      "properties": {
+        "prometheus.uri": "http://prometheus:9090"
+      }
+    }
+  ]
+}

integrations/observability/otel-demo/assets/demo-langing-page-1.0.0.ndjson

@@ -0,0 +1,3 @@
+{"attributes":{"description":"OTEL demo landing page","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"OTEL demo landing page","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"OTEL demo landing page\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"markdown\":\"\\n![](https://raw.githubusercontent.com/opensearch-project/.github/main/profile/banner.jpg)\\n# OpenSearch Observability OTEL Demo\\n\\nWelcome to the [OpenSearch](https://opensearch.org/docs/latest) OpenTelemetry [Demo](https://opentelemetry.io/docs/demo/) documentation, which covers how to install and run the demo, and some scenarios you can use to view OpenTelemetry in action.\\n\\n## Purpose\\nThe purpose of this demo is to demonstrate the different capabilities of OpenSearch Observability to investigate and reflect your system.\\n\\n![](../../../.github/img/DemoFlow.png)\\n\\n### Services\\n[OTEL DEMO](https://opentelemetry.io/docs/demo/services/) Describes the list of services that are composing the Astronomy Shop.\\n\\nThe main services that are open to user interactions:\\n\\n- [Dashboards](https://observability.playground.opensearch.org/)\\n\\n- [Demo Proxy](https://observability.playground.demo-proxy.opensearch.org/)\\n\\n- [Demo loader](https://observability.playground.demo-loader.opensearch.org/)\\n\\n- [Demo feature-flag](https://observability.playground.demo-feature-flag.opensearch.org/)\\n\\n---\\n\\n### Screenshots\\n\\n_**The Shop**_\\n\\n\\n![Shop](https://opentelemetry.io/docs/demo/screenshots/frontend-1.png)\\n\\n\\n\\n_**The load generator**_\\n\\n\\n![](https://opentelemetry.io/docs/demo/screenshots/load-generator-ui.png)\\n\\n---\\n\\n### [Integrations](http://localhost:5601/app/integrations#/available) \\n![](https://github.com/opensearch/opensearch-catalog/blob/otel-demo-integration/integrations/observability/otel-demo/static/dashboard1.png?raw=true)\\n\\nThe integration service is a list of pre-canned assets that are loaded in a combined manner to allow users the ability for simple and automatic way to discover and review their services topology.\\n\\nThese [OTEL demo](https://github.com/opensearch-project/opensearch-catalog/pull/91) integrations contain the following assets:\\n- components & index template mapping\\n   - [traces mapping](http://localhost:5601/app/opensearch_index_management_dashboards#/create-template/ss4o_traces_template)\\n   - [logs mapping](http://localhost:5601/app/opensearch_index_management_dashboards#/create-template/ss4o_logs_template)\\n   - [metrics mapping](http://localhost:5601/app/opensearch_index_management_dashboards#/create-template/ss4o_metrics_template)\\n\\n- Index Patterns\\n    - [traces](http://localhost:5601/app/management/opensearch-dashboards/indexPatterns/)\\n    - [logs](http://localhost:5601/app/management/opensearch-dashboards/indexPatterns/)\\n    - [metrics](http://localhost:5601/app/management/opensearch-dashboards/indexPatterns/)\\n\\n- Index Aliases\\n    - [traces](http://localhost:5601/app/opensearch_index_management_dashboards#/aliases)\\n    - [logs](http://localhost:5601/app/opensearch_index_management_dashboards#/aliases)\\n    - [metrics](http://localhost:5601/app/opensearch_index_management_dashboards#/aliases)\\n\\n- datasources\\n     - [Prometheus](http://localhost:5601/app/datasources#/manage/prometheus) \\n- data-stream & indices\\n     - [Traces / Data-prepper](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/otel-v1-apm-span-000001) \\n     - [Traces / Jaeger](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/jaeger-span-2023-12-13) \\n     - [Services / Data-prepper](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/otel-v1-apm-service-map) \\n     - [Services / Jaeger](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/jaeger-service-2023-12-13) \\n     - [Metrics / Data-prepper ](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/otel-metrics-2023.12.13) \\n     - [Log-Events / Data-prepper ](http://localhost:5601/app/opensearch_index_management_dashboards#/index-detail/otel-events-2023.12.13) \\n\\n\\n- dashboards\\n   - [OTEL Architecture](http://localhost:5601/app/dashboards#/view/67e37e40-f750-11ed-b6d0-850581e4a72d)\\n   - [System-Wide AMP](http://localhost:5601/app/dashboards#/view/b0d09c20-9893-11ee-8bb8-69dd3b5541dd)\\n---\\n### Ingestion\\nThe ingestion capabilities for OpenSearch is to be able to support multiple pipelines:\\n- [Data-Prepper](https://github.com/opensearch-project/data-prepper/) is an OpenSearch ingestion project that allows ingestion of OTEL standard signals using Otel-Collector\\n- [Jaeger](https://opensearch.org/docs/latest/observing-your-data/trace/trace-analytics-jaeger/) is an ingestion framework which has a build in capability for pushing OTEL signals into OpenSearch\\n- [Fluent-Bit](https://docs.fluentbit.io/manual/pipeline/outputs/opensearch) is an ingestion framework which has a build in capability for pushing OTEL signals into OpenSearch\\n\",\"openLinksInNewTab\":false}}"},"id":"dd4bebe0-f66a-11ed-9518-f5d5eb1d70bf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-12-13T03:47:42.872Z","version":"WzEzNCw3XQ=="}
+{"attributes":{"description":"OTEL demo landing page","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"0e0c418a-81f8-4d85-8ba7-8d8ef6e2b1d7\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"0e0c418a-81f8-4d85-8ba7-8d8ef6e2b1d7\",\"version\":\"2.11.0\",\"panelRefName\":\"panel_0\"}]","timeRestore":false,"title":"OTEL demo landing page","version":1},"id":"e66e2da0-f66a-11ed-9518-f5d5eb1d70bf","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"dd4bebe0-f66a-11ed-9518-f5d5eb1d70bf","name":"panel_0","type":"visualization"}],"type":"dashboard","updated_at":"2023-12-13T03:54:43.163Z","version":"WzEzNSw3XQ=="}
+{"exportedCount":2,"missingRefCount":0,"missingReferences":[]}

integrations/observability/otel-demo/assets/otel-architecture-1.0.0.ndjson

@@ -0,0 +1,3 @@
+{"attributes":{"description":"OTEL Astronomy Demo Application architecture","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"otel-architecture","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"otel-architecture\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"# OTEL Astronomy Demo Application\\n\\nThe following diagram presents the OTEL Astronomy shop services architecture:\\n\\n![](https://github.com/opensearch-project/opentelemetry-demo/raw/main/.github/img/docker-services-topology.png)\\n\\n\\n### Trace Collectors\\nGaining a macro-level perspective on incoming data, such as sample counts and cardinality, is essential for comprehending the collector’s internal dynamics. However, when delving into the details, the interconnections can become complex. The Collector Data Flow Dashboard aims to demonstrate the capabilities of the OpenTelemetry demo application, offering a solid foundation for users to build upon.\\n\\nMonitoring data flow through the OpenTelemetry Collector is crucial for several reasons.\\n - All services are traces in all the development languages\\n - Auto instrumented\\n - Manual spans and attributes\\n - Span events\\n - Span links\\n\\nTrace Headers are propagated across all services (**Context propagation**)\\n\\n\\n### Metric Collectors\\nCollecting all the KPI information into Prometheus time series storage including:\\n - runtime metrics\\n - HTTP / gRPC latency distribution\\n\\n### Data Flow Overview\\nCollector Data Flow Dashboard provides valuable guidance on which metrics to monitor. Users can tailor their own dashboard variations by adding necessary metrics specific to their use cases, such as memory_delimiter processor or other data flow indicators. This demo dashboard serves as a starting point, enabling users to explore diverse usage scenarios and adapt the tool to their unique monitoring needs.\\n\\n![](https://github.com/opensearch-project/opentelemetry-demo/blob/main/.github/img/DemoFlow.png?raw=true)\\n\\nThe diagram below provides an overview of the system components, showcasing the configuration derived from the OpenTelemetry Collector (otelcol) configuration file utilized by the OpenTelemetry demo application. Additionally, it highlights the observability data (traces and metrics) flow within the system.\\n\\n#### Simple purchase use case\\nThis flow diagram shows the trace evolution from the user selecting a purchased item going through different backend services until reaching the storage database.\\n![](img/DemoFlow.png)\\n\\n#### Services\\n- [All Services](http://localhost:5601/app/observability-traces#/services)\\n  ![](https://raw.githubusercontent.com/YANG-DB/opensearch-catalog/otel-demo-integration/integrations/observability/otel-demo/static/services.png)\\n \\n - [accountingservice](http://localhost:5601/app/observability-traces#/services/accountingservice)\\n     - [measurements ](http://localhost:5601/app/dashboards#/view/b0d09c20-9893-11ee-8bb8-69dd3b5541dd?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15d,to:now))&_a=(description:'disk%20utilization%0Acpu%20utilization%0Ajvm.heap%0Anetwork',filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'8ca59f40-92e3-11ee-8253-ff1f9ce273b1',key:serviceName.keyword,negate:!f,params:(query:recommendationservice),type:phrase),query:(match_phrase:(serviceName.keyword:recommendationservice)))),fullScreenMode:!f,options:(hidePanelTitles:!f,useMargins:!t),panels:!((embeddableConfig:(),gridData:(h:15,i:'4de730c8-c5cd-45a1-8df4-1988e8e10f2e',w:24,x:0,y:0),id:'86922340-98b9-11ee-8bb8-69dd3b5541dd',panelIndex:'4de730c8-c5cd-45a1-8df4-1988e8e10f2e',type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'9278872d-ac7e-4d01-8608-923cd1e2c867',w:24,x:24,y:0),id:'6dca4bd0-98b9-11ee-8bb8-69dd3b5541dd',panelIndex:'9278872d-ac7e-4d01-8608-923cd1e2c867',type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'223eb983-bf4e-4896-9c9e-c3e85f5bfecd',w:24,x:0,y:15),id:c92ac860-98b9-11ee-8bb8-69dd3b5541dd,panelIndex:'223eb983-bf4e-4896-9c9e-c3e85f5bfecd',type:visualization,version:'2.11.0'),(embeddableConfig:(hidePanelTitles:!f),gridData:(h:15,i:'1d66438e-7d9c-4850-9316-a46a69a74688',w:24,x:24,y:15),id:d8ed8300-98b9-11ee-8bb8-69dd3b5541dd,panelIndex:'1d66438e-7d9c-4850-9316-a46a69a74688',title:services.cpu.time,type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'8504673d-3fdd-4ad7-8edd-d68f60431302',w:24,x:0,y:30),id:'952a8320-98c3-11ee-8bb8-69dd3b5541dd',panelIndex:'8504673d-3fdd-4ad7-8edd-d68f60431302',type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:c56a2e16-34a9-4ab7-8786-53f217afc7de,w:24,x:24,y:30),id:b36d3f30-98c3-11ee-8bb8-69dd3b5541dd,panelIndex:c56a2e16-34a9-4ab7-8786-53f217afc7de,type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'3a2051d7-348d-4d44-b037-fbcce680d83d',w:24,x:0,y:45),id:e4d3c5c0-98c4-11ee-8bb8-69dd3b5541dd,panelIndex:'3a2051d7-348d-4d44-b037-fbcce680d83d',type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'12ccbeba-d6c9-4a0d-a295-7aecfd32a2ca',w:24,x:24,y:45),id:'033888c0-98c5-11ee-8bb8-69dd3b5541dd',panelIndex:'12ccbeba-d6c9-4a0d-a295-7aecfd32a2ca',type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:f705b89b-d364-45d4-b632-af5df69b90ad,w:24,x:0,y:60),id:'164cb850-98c5-11ee-8bb8-69dd3b5541dd',panelIndex:f705b89b-d364-45d4-b632-af5df69b90ad,type:visualization,version:'2.11.0'),(embeddableConfig:(),gridData:(h:15,i:'4d6aede1-5a70-4f12-b4ad-4eb896b5f550',w:24,x:24,y:60),id:'3e2fe6d0-98c5-11ee-8bb8-69dd3b5541dd',panelIndex:'4d6aede1-5a70-4f12-b4ad-4eb896b5f550',type:visualization,version:'2.11.0')),query:(language:kuery,query:''),timeRestore:!f,title:SystemWide-APM,viewMode:edit))\\n - [adservice](http://localhost:5601/app/observability-traces#/services/adservice)\\n - [cartservice](http://localhost:5601/app/observability-traces#/services/cartservice)\\n - [checkoutservice](http://localhost:5601/app/observability-traces#/services/checkoutservice)\\n - [currencyservice](http://localhost:5601/app/observability-traces#/services/currencyservice)\\n - [emailservice](http://localhost:5601/app/observability-traces#/services/emailservice)\\n - [featureflagservice](http://localhost:5601/app/observability-traces#/services/featureflagservice)\\n - [frauddetectionservice](http://localhost:5601/app/observability-traces#/services/frauddetectionservice)\\n - [frontend](http://localhost:5601/app/observability-traces#/services/frontend)\\n - [frauddetectionservice](http://localhost:5601/app/observability-traces#/services/frauddetectionservice)\\n - [loadgenerator](http://localhost:5601/app/observability-traces#/services/loadgenerator)\\n      - [measurements ](http://localhost:5601/app/dashboards#/view/57a35630-9936-11ee-8bb8-69dd3b5541dd?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:'2023-12-04T18:51:48.732Z',to:'2023-12-05T00:50:48.914Z'))&_a=(description:'',filters:!(),fullScreenMode:!f,options:(hidePanelTitles:!f,useMargins:!t),query:(language:kuery,query:''),timeRestore:!f,title:loadgenerators-services,viewMode:view))\\n - [paymentservice](http://localhost:5601/app/observability-traces#/services/paymentservice)\\n - [productcatalogservice](http://localhost:5601/app/observability-traces#/services/productcatalogservice)\\n - [quoteservice](http://localhost:5601/app/observability-traces#/services/quoteservice)\\n - [recommendationservice](http://localhost:5601/app/observability-traces#/services/recommendationservice)\\n-  [shippingservice](http://localhost:5601/app/observability-traces#/services/shippingservice)\\n---\\n## OTEL collector pipelines\\n\\n### Logs\\n![](https://github.com/opensearch-project/opentelemetry-demo/blob/main/src/otelcollector/img/logs-pipe.png?raw=true)\\n\\n### Metrics\\n![](https://github.com/opensearch-project/opentelemetry-demo/blob/main/src/otelcollector/img/metrics-pipe.png?raw=true)\\n\\n### Traces\\n![](https://github.com/opensearch-project/opentelemetry-demo/blob/main/src/otelcollector/img/traces-pipe.png?raw=true)\\n\\n\\n---\\n## Reference\\n**_OTEL Demo info_**\\n- [architecture](https://opentelemetry.io/docs/demo/architecture/)\\n- [collector-data-flow-dashboard](https://opentelemetry.io/docs/demo/collector-data-flow-dashboard/)\\n- [services](https://opentelemetry.io/docs/demo/services/)\\n\\n**_OTEL Demo repository_**\\n - [OpenSearch: OpenTelemetry community demo](https://github.com/opensearch-project/opentelemetry-demo)\\n\"}}"},"id":"5c297aa0-f750-11ed-b6d0-850581e4a72d","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-12-14T22:44:55.303Z","version":"WzE3Nyw4XQ=="}
+{"attributes":{"description":"OTEL Astronomy Demo Application architecture","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"cc8d389f-6ab0-4590-bd7d-140ed04a28b1\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"cc8d389f-6ab0-4590-bd7d-140ed04a28b1\",\"version\":\"2.11.0\",\"panelRefName\":\"panel_0\"}]","timeRestore":false,"title":"otel-demo-architecture","version":1},"id":"67e37e40-f750-11ed-b6d0-850581e4a72d","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"5c297aa0-f750-11ed-b6d0-850581e4a72d","name":"panel_0","type":"visualization"}],"type":"dashboard","updated_at":"2023-12-12T21:39:40.356Z","version":"WzEwMSw2XQ=="}
+{"exportedCount":2,"missingRefCount":0,"missingReferences":[]}