[Ceph] Add Integration Package with OSD Tree data stream

packages/ceph/_dev/build/docs/README.md

@@ -6,20 +6,21 @@
 
 Use the Ceph integration to:
 
-- Collect metrics related to the cluster health, Object Storage Daemons (OSD) performance and Object Storage Daemons (OSD) pool stats.
+- Collect metrics related to the cluster health, Object Storage Daemons (OSD) performance, Object Storage Daemons (OSD) pool stats and Object Storage Daemons (OSD) tree.
 - Create visualizations to monitor, measure and analyze the usage trend and key data, and derive business insights.
 - Create alerts to reduce the MTTD and also the MTTR by referencing relevant logs when troubleshooting an issue.
 
 ## Data streams
 
 The Ceph integration collects metrics data.
 
-Metrics give you insight into the statistics of the Ceph. The Metric data streams collected by the Ceph integration are `cluster_health`, `osd_performance` and `osd_pool_stats`, so that the user can monitor and troubleshoot the performance of the Ceph instance.
+Metrics give you insight into the statistics of the Ceph. The Metric data streams collected by the Ceph integration are `cluster_health`, `osd_performance`, `osd_pool_stats` and `osd_tree`, so that the user can monitor and troubleshoot the performance of the Ceph instance.
 
 Data stream:
 - `cluster_health`: Represents information related to the health of the cluster.
 - `osd_performance`: Tracks Object Storage Daemons (OSD) performance.
 - `osd_pool_stats`: Represents information related to client I/O rates.
+- `osd_tree`: Represents information related to structure of the Object Storage Daemons (OSD) tree.
 
 Note:
 - Users can monitor and see the metrics inside the ingested documents for Ceph in the `logs-*` index pattern from `Discover`.
@@ -107,4 +108,12 @@ This is the `osd_pool_stats` data stream. This data stream collects metrics rela
 
 {{event "osd_pool_stats"}}
 
-{{fields "osd_pool_stats"}}
+{{fields "osd_pool_stats"}}
+
+### OSD Tree
+
+This is the `osd_tree` data stream. This data stream collects metrics related to Object Storage Daemon (OSD) tree id, name, status, exists, crush_weight, etc.
+
+{{event "osd_tree"}}
+
+{{fields "osd_tree"}}

packages/ceph/_dev/deploy/docker/files/config.yml

@@ -26,3 +26,12 @@ rules:
       - status_code: 200
         body: |-
           {"failed":[],"finished":[{"command":"osd pool stats format=json","outb":"[{\"pool_name\":\"device_health_metrics\",\"pool_id\":1,\"recovery\":{},\"recovery_rate\":{},\"client_io_rate\":{\"read_bytes_sec\":6622518,\"write_bytes_sec\":6622518,\"read_op_per_sec\":11,\"write_op_per_sec\":11}},{\"pool_name\":\"elastic\",\"pool_id\":9,\"recovery\":{},\"recovery_rate\":{},\"client_io_rate\":{}}]","outs":""}],"has_failed":false,"id":"140222622573952","is_finished":true,"is_waiting":false,"running":[],"state":"success","waiting":[]}
+  - path: /request
+    query_params:
+      wait: 1
+    methods: ['POST']
+    request_body: '{"format":"json","prefix":"osd tree"}'
+    responses:
+      - status_code: 200
+        body: |-
+          {"failed":[],"finished":[{"command":"osd tree format=json","outb":"{\"nodes\":[{\"id\":0,\"device_class\":\"hdd\",\"name\":\"osd.0\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0.0194854736328125,\"depth\":2,\"pool_weights\":{},\"exists\":1,\"status\":\"up\",\"reweight\":1,\"primary_affinity\":1},{\"id\":-7,\"name\":\"node02\",\"type\":\"host\",\"type_id\":1,\"pool_weights\":{},\"children\":[2]},{\"id\":2,\"device_class\":\"hdd\",\"name\":\"osd.2\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0.0194854736328125,\"depth\":2,\"pool_weights\":{},\"exists\":1,\"status\":\"up\",\"reweight\":1,\"primary_affinity\":1},{\"id\":-9,\"name\":\"node03\",\"type\":\"host\",\"type_id\":1,\"pool_weights\":{},\"children\":[3]},{\"id\":3,\"device_class\":\"hdd\",\"name\":\"osd.3\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0.0194854736328125,\"depth\":2,\"pool_weights\":{},\"exists\":1,\"status\":\"up\",\"reweight\":1,\"primary_affinity\":1},{\"id\":-5,\"name\":\"node04\",\"type\":\"host\",\"type_id\":1,\"pool_weights\":{},\"children\":[1]},{\"id\":1,\"device_class\":\"hdd\",\"name\":\"osd.1\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0.0194854736328125,\"depth\":2,\"pool_weights\":{},\"exists\":1,\"status\":\"up\",\"reweight\":1,\"primary_affinity\":1}],\"stray\":[{\"id\":4,\"name\":\"osd.4\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0,\"depth\":0,\"exists\":1,\"status\":\"destroyed\",\"reweight\":0,\"primary_affinity\":1},{\"id\":5,\"name\":\"osd.5\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0,\"depth\":0,\"exists\":1,\"status\":\"destroyed\",\"reweight\":0,\"primary_affinity\":1}]}\n","outs":""}],"has_failed":false,"id":"140222986406304","is_finished":true,"is_waiting":false,"running":[],"state":"success","waiting":[]}

packages/ceph/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.4.0"
+  changes:
+    - description: Ceph integration package with "osd_tree" data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5174
 - version: "0.3.0"
   changes:
     - description: Ceph integration package with "osd_pool_stats" data stream.

packages/ceph/data_stream/osd_tree/_dev/test/pipeline/test-common-config.yml

@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event

packages/ceph/data_stream/osd_tree/_dev/test/pipeline/test-osd-tree-metrics-1.log

@@ -0,0 +1 @@
+{"id":-1,"name":"default","type":"root","type_id":11,"children":[-5,-9,-7,-3]}

packages/ceph/data_stream/osd_tree/_dev/test/pipeline/test-osd-tree-metrics-1.log-expected.json

@@ -0,0 +1,36 @@
+{
+    "expected": [
+        {
+            "ceph": {
+                "osd_tree": {
+                    "children": [
+                        "-5",
+                        "-9",
+                        "-7",
+                        "-3"
+                    ],
+                    "node_osd_id": -1,
+                    "node_osd_name": "default",
+                    "type": {
+                        "id": 11,
+                        "name": "root"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.5.1"
+            },
+            "event": {
+                "kind": "metric",
+                "module": "ceph",
+                "original": "{\"id\":-1,\"name\":\"default\",\"type\":\"root\",\"type_id\":11,\"children\":[-5,-9,-7,-3]}",
+                "type": [
+                    "info"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/ceph/data_stream/osd_tree/_dev/test/pipeline/test-osd-tree-metrics-2.log

@@ -0,0 +1 @@
+{"id":0,"device_class":"hdd","name":"osd.0","type":"osd","type_id":0,"crush_weight":0.0194854736328125,"depth":2,"pool_weights":{},"exists":1,"status":"up","reweight":1,"primary_affinity":1}

packages/ceph/data_stream/osd_tree/_dev/test/pipeline/test-osd-tree-metrics-2.log-expected.json

@@ -0,0 +1,39 @@
+{
+    "expected": [
+        {
+            "ceph": {
+                "osd_tree": {
+                    "crush_weight": 0.0194854736328125,
+                    "depth": 2,
+                    "device_class": "hdd",
+                    "exists": true,
+                    "node_osd_id": 0,
+                    "node_osd_name": "osd.0",
+                    "primary_affinity": {
+                        "count": 1
+                    },
+                    "reweight": 1,
+                    "status": "up",
+                    "type": {
+                        "id": 0,
+                        "name": "osd"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.5.1"
+            },
+            "event": {
+                "kind": "metric",
+                "module": "ceph",
+                "original": "{\"id\":0,\"device_class\":\"hdd\",\"name\":\"osd.0\",\"type\":\"osd\",\"type_id\":0,\"crush_weight\":0.0194854736328125,\"depth\":2,\"pool_weights\":{},\"exists\":1,\"status\":\"up\",\"reweight\":1,\"primary_affinity\":1}",
+                "type": [
+                    "info"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/ceph/data_stream/osd_tree/_dev/test/system/test-default-config.yml

@@ -0,0 +1,7 @@
+input: httpjson
+service: ceph
+vars:
+  hostname: http://{{Hostname}}:{{Port}}
+data_stream:
+  vars:
+    preserve_original_event: true

packages/ceph/data_stream/osd_tree/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,49 @@
+config_version: 2
+interval: {{period}}
+{{#if api_user}}
+{{#if api_secret}}
+auth.basic.user: {{api_user}}
+auth.basic.password: {{api_secret}}
+{{/if}}
+{{/if}}
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.url: {{hostname}}/request?wait=1
+response.split:
+  target: body.finished
+  ignore_empty_value: true
+  transforms:
+    - set:
+        target: body.outb
+        value: '[[.body.outb]]'
+        value_type: json
+  split:
+    target: body.outb.nodes
+    ignore_empty_value: true
+request.transforms:
+  - set:
+      target: body.prefix
+      value: "osd tree"
+  - set:
+      target: body.format
+      value: "json"
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+- add_fields:
+    target: service
+    fields:
+      address: {{hostname}}
+{{#if processors}}
+{{processors}}
+{{/if}}

packages/ceph/data_stream/osd_tree/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,123 @@
+---
+description: Pipeline for processing Ceph OSD Tree metrics.
+processors:
+  - set:
+      field: ecs.version
+      value: 8.5.1
+  - set:
+      field: event.type
+      value: [info]
+  - set:
+      field: event.kind
+      value: metric
+  - set:
+      field: event.module
+      value: ceph
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+  - json:
+      field: event.original
+      target_field: json
+      on_failure:
+        - append:
+            field: error.message
+            value: '{{{_ingest.on_failure_message}}}'
+  - convert:
+      field: json.exists
+      type: string
+      ignore_missing: true
+  - set:
+      field: json.exists
+      value: true
+      if: ctx.json?.exists == "1"
+  - set:
+      field: json.exists
+      value: false
+      if: ctx.json?.exists == "0"
+  - convert:
+      field: json.children
+      type: string
+      ignore_missing: true
+  - rename: 
+      field: json.children
+      target_field: ceph.osd_tree.children
+      ignore_missing: true
+  - rename: 
+      field: json.crush_weight
+      target_field: ceph.osd_tree.crush_weight
+      ignore_missing: true
+  - rename: 
+      field: json.depth
+      target_field: ceph.osd_tree.depth
+      ignore_missing: true
+  - rename: 
+      field: json.device_class
+      target_field: ceph.osd_tree.device_class
+      ignore_missing: true
+  - rename: 
+      field: json.exists
+      target_field: ceph.osd_tree.exists
+      ignore_missing: true
+  - rename: 
+      field: json.id
+      target_field: ceph.osd_tree.node_osd_id
+      ignore_missing: true
+  - rename: 
+      field: json.name
+      target_field: ceph.osd_tree.node_osd_name
+      ignore_missing: true
+  - rename: 
+      field: json.primary_affinity
+      target_field: ceph.osd_tree.primary_affinity.count
+      ignore_missing: true
+  - rename: 
+      field: json.reweight
+      target_field: ceph.osd_tree.reweight
+      ignore_missing: true
+  - rename: 
+      field: json.status
+      target_field: ceph.osd_tree.status
+      ignore_missing: true
+  - rename: 
+      field: json.type
+      target_field: ceph.osd_tree.type.name
+      ignore_missing: true
+  - rename: 
+      field: json.type_id
+      target_field: ceph.osd_tree.type.id
+      ignore_missing: true
+  - script:
+      description: Drops null/empty values recursively.
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+  - remove:
+      field: event.original
+      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_missing: true
+  - remove:
+      field:
+        - json
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"
+  - append:
+      field: event.kind
+      value: pipeline_error
+      allow_duplicates: false

packages/ceph/data_stream/osd_tree/fields/base-fields.yml

@@ -0,0 +1,18 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
+- name: tags
+  type: keyword
+  description: List of keywords used to tag each event.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/ceph/data_stream/osd_tree/fields/ecs.yml

@@ -0,0 +1,22 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.agent_id_status
+- external: ecs
+  name: event.created
+- external: ecs
+  name: event.dataset
+- external: ecs
+  name: event.ingested
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.module
+- external: ecs
+  name: event.original
+- external: ecs
+  name: event.type
+- external: ecs
+  name: service.address