[AWS] Move usage lightweight module config into integration

packages/aws/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.17.3"
+  changes:
+    - description: Move usage metrics config from beats to integrations
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3764
+    - description: Move dynamodb metrics config from beats to integrations
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3764
 - version: "1.17.2"
   changes:
     - description: Improve support for event.original field from upstream forwarders.

packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs

@@ -32,4 +32,44 @@ tags_filter: {{tags_filter}}
 {{/if}}
 {{#if proxy_url }}
 proxy_url: {{proxy_url}}
-{{/if}}
+{{/if}}
+metrics:
+- namespace: AWS/DynamoDB
+  resource_type: dynamodb
+  statistic: ["Average"]
+  name:
+  - SuccessfulRequestLatency
+  - OnlineIndexPercentageProgress
+  - ProvisionedWriteCapacityUnits
+  - ProvisionedReadCapacityUnits
+  - ConsumedReadCapacityUnits
+  - ConsumedWriteCapacityUnits
+  - ReplicationLatency
+  - TransactionConflict
+  - AccountProvisionedReadCapacityUtilization
+  - AccountProvisionedWriteCapacityUtilization
+- namespace: AWS/DynamoDB
+  resource_type: dynamodb
+  statistic: ["Sum"]
+  name:
+  - SystemErrors
+  - ConsumedReadCapacityUnits
+  - ConsumedWriteCapacityUnits
+  - ConditionalCheckFailedRequests
+  - PendingReplicationCount
+  - TransactionConflict
+  - ReadThrottleEvents
+  - ThrottledRequests
+  - WriteThrottleEvents
+- namespace: AWS/DynamoDB
+  resource_type: dynamodb
+  statistic: ["Maximum"]
+  name:
+  - SuccessfulRequestLatency
+  - ReplicationLatency
+  - AccountMaxReads
+  - AccountMaxTableLevelReads
+  - AccountMaxTableLevelWrites
+  - AccountMaxWrites
+  - MaxProvisionedTableReadCapacityUtilization
+  - MaxProvisionedTableWriteCapacityUtilization

packages/aws/data_stream/dynamodb/fields/package-fields.yml

@@ -13,7 +13,3 @@
       type: object
       description: |
         Metric dimensions.
-    - name: '*.metrics.*.*'
-      type: object
-      description: |
-        Metrics that returned from Cloudwatch API query.

packages/aws/data_stream/dynamodb/sample_event.json

@@ -1,59 +1,78 @@
 {
-    "@timestamp": "2020-05-28T17:17:08.666Z",
+    "@timestamp": "2022-07-25T21:53:00.000Z",
     "agent": {
-        "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b",
-        "id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
-        "name": "MacBook-Elastic.local",
+        "name": "docker-fleet-agent",
+        "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864",
         "type": "metricbeat",
-        "version": "8.0.0"
+        "ephemeral_id": "64a12b83-a4f1-487c-8d2c-9581fda6ca2a",
+        "version": "8.3.2"
     },
-    "event": {
-        "dataset": "aws.dynamodb",
-        "module": "aws",
-        "duration": 10266182336
-    },
-    "service": {
-        "type": "aws"
-    },
-    "ecs": {
-        "version": "1.5.0"
+    "elastic_agent": {
+        "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864",
+        "version": "8.3.2",
+        "snapshot": false
     },
     "cloud": {
+        "provider": "aws",
+        "region": "eu-central-1",
         "account": {
             "name": "elastic-beats",
             "id": "428152502467"
-        },
-        "provider": "aws",
-        "region": "eu-central-1"
+        }
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "service": {
+        "type": "aws"
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "metrics",
+        "dataset": "aws.dynamodb"
+    },
+    "metricset": {
+        "period": 300000,
+        "name": "dynamodb"
+    },
+    "event": {
+        "duration": 10586366300,
+        "agent_id_status": "verified",
+        "ingested": "2022-07-25T21:57:51Z",
+        "module": "aws",
+        "dataset": "aws.dynamodb"
     },
     "aws": {
-        "dimensions": {
-            "TableName": "TryDaxTable3"
+        "cloudwatch": {
+            "namespace": "AWS/DynamoDB"
         },
         "dynamodb": {
             "metrics": {
-                "ProvisionedWriteCapacityUnits": {
-                    "avg": 1
+                "AccountProvisionedWriteCapacityUtilization": {
+                    "avg": 0.01
+                },
+                "MaxProvisionedTableWriteCapacityUtilization": {
+                    "max": 0.01
                 },
-                "ProvisionedReadCapacityUnits": {
-                    "avg": 1
+                "MaxProvisionedTableReadCapacityUtilization": {
+                    "max": 0.01
                 },
-                "ConsumedWriteCapacityUnits": {
-                    "avg": 0,
-                    "sum": 0
+                "AccountMaxTableLevelReads": {
+                    "max": 40000
                 },
-                "ConsumedReadCapacityUnits": {
-                    "avg": 0,
-                    "sum": 0
+                "AccountMaxReads": {
+                    "max": 80000
+                },
+                "AccountProvisionedReadCapacityUtilization": {
+                    "avg": 0.01
+                },
+                "AccountMaxWrites": {
+                    "max": 80000
+                },
+                "AccountMaxTableLevelWrites": {
+                    "max": 40000
                 }
             }
-        },
-        "cloudwatch": {
-            "namespace": "AWS/DynamoDB"
         }
-    },
-    "metricset": {
-        "name": "dynamodb",
-        "period": 300000
     }
 }

packages/aws/data_stream/usage/agent/stream/stream.yml.hbs

@@ -1,4 +1,4 @@
-metricsets: ["usage"]
+metricsets: ["cloudwatch"]
 period: {{period}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}
@@ -32,4 +32,7 @@ tags_filter: {{tags_filter}}
 {{/if}}
 {{#if proxy_url }}
 proxy_url: {{proxy_url}}
-{{/if}}
+{{/if}}
+metrics:
+- namespace: AWS/Usage
+  statistic: ["Sum"]

packages/aws/data_stream/usage/fields/package-fields.yml

@@ -13,7 +13,3 @@
       type: object
       description: |
         Metric dimensions.
-    - name: '*.metrics.*.*'
-      type: object
-      description: |
-        Metrics that returned from Cloudwatch API query.

packages/aws/data_stream/usage/sample_event.json

@@ -1,5 +1,40 @@
 {
-    "@timestamp": "2020-05-28T17:58:30.929Z",
+    "@timestamp": "2022-07-25T20:50:00.000Z",
+    "agent": {
+        "name": "docker-fleet-agent",
+        "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864",
+        "type": "metricbeat",
+        "ephemeral_id": "6bab70d4-84d9-411d-887c-f144d4244e78",
+        "version": "8.3.2"
+    },
+    "elastic_agent": {
+        "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864",
+        "version": "8.3.2",
+        "snapshot": false
+    },
+    "cloud": {
+        "provider": "aws",
+        "region": "eu-north-1",
+        "account": {
+            "name": "elastic-beats",
+            "id": "428152502467"
+        }
+    },
+    "ecs": {
+        "version": "8.0.0"
+    },
+    "service": {
+        "type": "aws"
+    },
+    "data_stream": {
+        "namespace": "default",
+        "type": "metrics",
+        "dataset": "aws.usage"
+    },
+    "metricset": {
+        "period": 60000,
+        "name": "cloudwatch"
+    },
     "aws": {
         "usage": {
             "metrics": {
@@ -13,39 +48,16 @@
         },
         "dimensions": {
             "Type": "API",
-            "Resource": "GetMetricData",
+            "Resource": "ListMetrics",
             "Service": "CloudWatch",
             "Class": "None"
         }
     },
     "event": {
-        "duration": 1191329839,
-        "dataset": "aws.usage",
-        "module": "aws"
-    },
-    "service": {
-        "type": "aws"
-    },
-    "ecs": {
-        "version": "1.5.0"
-    },
-    "cloud": {
-        "provider": "aws",
-        "region": "eu-north-1",
-        "account": {
-            "name": "elastic-beats",
-            "id": "428152502467"
-        }
-    },
-    "metricset": {
-        "name": "usage",
-        "period": 60000
-    },
-    "agent": {
-        "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b",
-        "id": "12f376ef-5186-4e8b-a175-70f1140a8f30",
-        "name": "MacBook-Elastic.local",
-        "type": "metricbeat",
-        "version": "8.0.0"
+        "duration": 1432082500,
+        "agent_id_status": "verified",
+        "ingested": "2022-07-25T20:51:19Z",
+        "module": "aws",
+        "dataset": "aws.usage"
     }
 }