Otel demo

logs.js

@@ -0,0 +1,26 @@
+const {Resource} = require('@opentelemetry/resources')
+const {
+    LoggerProvider,
+    SimpleLogRecordProcessor
+} = require('@opentelemetry/sdk-logs');
+const {SemanticResourceAttributes} = require('@opentelemetry/semantic-conventions')
+const { OTLPLogExporter } = require('@opentelemetry/exporter-logs-otlp-http');
+
+
+module.exports = (serviceName) => {
+
+    const resource = new Resource({
+        [ SemanticResourceAttributes.SERVICE_NAME ]: serviceName
+    });
+
+    const loggerProvider = new LoggerProvider({
+        resource
+    });
+
+    loggerProvider.addLogRecordProcessor(new SimpleLogRecordProcessor(new OTLPLogExporter({
+        url: process.env.HF_VAR_OPT_URL + ':4318/v1/logs',
+        keepAlive: true,
+    })));
+
+    return loggerProvider.getLogger('default');
+}

metrics-definition.js

@@ -0,0 +1,25 @@
+const meter = process.env.HF_VAR_ENABLE_OTEL === "1" ? require("./metrics.js")("hyperflow-job-executor"): undefined;
+
+
+const cpuMetric = meter.createObservableGauge('cpu-usage', {
+    description: 'CPU usage',
+    unit: 'Percentage'
+});
+
+const memoryMetric = meter.createObservableGauge('memory-usage', {
+    description: 'Memory usage',
+    unit: 'Bytes'
+});
+
+const cTimeMetric = meter.createObservableGauge('ctime', {
+    description: 'cTime',
+    unit: 'ms'
+});
+
+
+// Export constants using module.exports
+module.exports = {
+    cpuMetric: cpuMetric,
+    memoryMetric: memoryMetric,
+    cTimeMetric: cTimeMetric,
+};

metrics.js

@@ -0,0 +1,25 @@
+const {PeriodicExportingMetricReader, ConsoleMetricExporter, MeterProvider} = require('@opentelemetry/sdk-metrics')
+const {Resource} = require('@opentelemetry/resources')
+const {SemanticResourceAttributes} = require('@opentelemetry/semantic-conventions')
+const {OTLPMetricExporter} = require('@opentelemetry/exporter-metrics-otlp-http')
+
+
+module.exports = (serviceName) => {
+
+    const exporter = new OTLPMetricExporter({
+        url: process.env.HF_VAR_OPT_URL+':4318/v1/metrics'
+    })
+    const consoleReader = new PeriodicExportingMetricReader(
+        {exporter: exporter,
+            exportIntervalMillis: 3000}
+    )
+    const provider = new MeterProvider({
+        readers: [consoleReader],
+        resource: new Resource({
+            [SemanticResourceAttributes.SERVICE_NAME]:
+            serviceName,
+        }),
+    });
+
+    return provider.getMeter("hyperflow-job-executor");
+}

monitoring-local-setup/README.md

@@ -0,0 +1,35 @@
+# HyperFlow Job executor monitoring
+
+## Set required env variables to
+- HF_VAR_ENABLE_TRACING: 1
+- HF_VAR_ENABLE_OTEL: 1
+- HF_VAR_OPT_URL: http://localhost
+
+## Run docker compose
+docker compose up -d
+
+## Check if all containers work
+Opensearch might fail to start due to low memory, if it happens check:
+https://stackoverflow.com/questions/51445846/elasticsearch-max-virtual-memory-areas-vm-max-map-count-65530-is-too-low-inc
+
+Verify if dataprepper properly initialized connection with sink (opensearch) by checking logs
+
+## Run workflow locally
+
+## Open opensearch dashboards
+
+Navigate to
+http://localhost:5601/
+
+login using credentials 
+admin
+Hyperflow1!
+
+Go to Dashboards Management -> Index Patterns
+
+create index patterns
+- hyperflow_traces
+- hyperflow_metrics
+- hyperflow_logs
+
+Go to Discover and choose one of new index patterns as source

monitoring-local-setup/data-prepper-config.yaml

@@ -0,0 +1 @@
+ssl: false

monitoring-local-setup/docker-compose.yml

@@ -0,0 +1,74 @@
+version: '3'
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    command: [ "--config=/etc/otel-collector-config.yaml" ]
+    ports:
+      - "1888:1888"   # pprof extension
+      - "13133:13133" # health_check extension
+      - "4317:4317"        # OTLP gRPC receiver
+      - "4318:4318"        # OTLP HTTP receiver
+      - "55670:55679" # zpages extension
+      - "14250:14250"
+    networks:
+      - backend
+
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch
+      - discovery.seed_hosts=opensearch
+      - cluster.initial_cluster_manager_nodes=opensearch
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Hyperflow1!
+      - plugins.security.disabled=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+    ports:
+      - 9200:9200
+      - 9600:9600
+    networks:
+      - backend
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["http://opensearch:9200"]'
+      DISABLE_SECURITY_DASHBOARDS_PLUGIN: "true"
+    networks:
+      - backend
+  data-prepper:
+    container_name: dataprepper
+    image: opensearchproject/data-prepper:latest
+    volumes:
+      - ./pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
+      - ./data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
+    ports:
+      - "21890:21890"
+      - "21891:21891"
+    networks:
+      - backend
+    depends_on:
+      - opensearch
+volumes:
+  opensearch-data:
+  opensearch-data1:
+
+networks:
+  backend:

monitoring-local-setup/otel-collector-config.yaml

@@ -0,0 +1,41 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:4318"
+
+processors:
+  batch:
+
+
+exporters:
+  otlp/traces:
+    endpoint: dataprepper:21890
+    tls:
+      insecure: true
+      insecure_skip_verify: true
+  otlp/metrics:
+    endpoint: dataprepper:21891
+    tls:
+      insecure: true
+      insecure_skip_verify: true
+  otlp/logs:
+    endpoint: dataprepper:21892
+    tls:
+      insecure: true
+      insecure_skip_verify: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/traces]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/metrics]
+    logs:
+      receivers: [otlp]
+      exporters: [otlp/logs]
+

monitoring-local-setup/pipelines.yaml

@@ -0,0 +1,69 @@
+entry-pipeline:
+  delay: "100"
+  source:
+    otel_trace_source:
+      ssl: false
+  sink:
+    - pipeline:
+        name: "raw-pipeline"
+    - pipeline:
+        name: "service-map-pipeline"
+raw-pipeline:
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  processor:
+    - otel_trace_raw:
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: "Hyperflow1!"
+        index_type: custom
+        index: hyperflow_traces
+        ssl: false
+service-map-pipeline:
+  delay: "100"
+  source:
+    pipeline:
+      name: "entry-pipeline"
+  processor:
+    - service_map_stateful:
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: "Hyperflow1!"
+        index_type: trace-analytics-service-map
+        ssl: false
+
+
+metrics-pipeline:
+  source:
+    otel_metrics_source:
+      ssl: false
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: "Hyperflow1!"
+        index_type: custom
+        index: hyperflow_metrics
+        ssl: false
+
+logs-pipeline:
+  source:
+    otel_logs_source:
+      ssl: false
+  sink:
+    - opensearch:
+        hosts: ["http://opensearch:9200"]
+        insecure: true
+        username: admin
+        password: "Hyperflow1!"
+        index: hyperflow_logs
+        ssl: false
+