elastic · narph · Aug 5, 2021 · Jul 26, 2021 · Aug 2, 2021 · Aug 2, 2021
diff --git a/packages/azure/_dev/build/build.yml b/packages/azure/_dev/build/build.yml
@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]
diff --git a/packages/azure/_dev/build/docs/README.md b/packages/azure/_dev/build/docs/README.md
@@ -7,7 +7,7 @@ There are several requirements before using the integration since the logs will
    * to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export
    * to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub
 
-The module contains the following filesets:
+The package contains the following data streams:
 
 ### activitylogs
 Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription.
@@ -21,6 +21,9 @@ Will retrieve azure Active Directory sign-in logs. The sign-ins report provides
 ### auditlogs 
 Will retrieve azure Active Directory audit logs. The audit logs provide traceability through logs for all changes done by various features within Azure AD. Examples of audit logs include changes made to any resources within Azure AD like adding or removing users, apps, groups, roles and policies.
 
+### springcloudlogs 
+Will retrieve Azure Spring Cloud system and  application logs.
+
 ### Credentials
 
 `eventhub` :

diff --git a/packages/azure/_dev/build/docs/springcloudlogs.md b/packages/azure/_dev/build/docs/springcloudlogs.md
@@ -0,0 +1,19 @@
+## Logs
+
+The azure logs integration retrieves different types of log data from Azure.
+There are several requirements before using the integration since the logs will actually be read from azure event hubs.
+
+   * the logs have to be exported first to the event hub https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-create-kafka-enabled
+   * to export activity logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/azure-monitor/platform/activity-log-export
+   * to export audit and sign-in logs to event hubs users can follow the steps here https://docs.microsoft.com/en-us/azure/active-directory/reports-monitoring/tutorial-azure-monitor-stream-logs-to-event-hub
+
+
+Azure Spring Cloud logs provide system and application information for Azure Spring Cloud resources.
+
+### springcloudlogs
+
+This is the `springcloudlogs` data stream of the Azure Logs package. It will collect any Spring Cloud logs that have been streamed through an azure event hub.
+
+{{event "springcloudlogs"}}
+
+{{fields "springcloudlogs"}}
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.0"
+  changes:
+    - description: Add spring cloud logs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1355
 - version: '0.6.2'
   changes:
     - description: update to ECS 1.11.0

diff --git a/packages/azure/data_stream/springcloudlogs/_dev/test/pipeline/test-common-config.yml b/packages/azure/data_stream/springcloudlogs/_dev/test/pipeline/test-common-config.yml
@@ -0,0 +1,5 @@
+dynamic_fields:
+  event.ingested: ".*"
+fields:
+  tags:
+    - preserve_original_event
diff --git a/packages/azure/data_stream/springcloudlogs/_dev/test/pipeline/test-springcloudlogs-raw.log b/packages/azure/data_stream/springcloudlogs/_dev/test/pipeline/test-springcloudlogs-raw.log
@@ -0,0 +1 @@
+{ "time": "2021-07-01T19:30:30.535404056Z", "LogFormat": "RAW", "resourceId": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", "operationName": "Microsoft.AppPlatform/Spring/logs", "category": "ApplicationConsole", "level": "Informational", "location": "westus2", "properties": {"Log":"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\n","Stream":"stdout","AppName":"admin-server","InstanceName":"admin-server-default-12-8459d44f68-g4b5f","ServiceId":"c41fd000b1a5450eb234039376da26de","ServiceName":"hm-sc-petclinic"}}
diff --git a/...data_stream/springcloudlogs/_dev/test/pipeline/test-springcloudlogs-raw.log-expected.json b/...data_stream/springcloudlogs/_dev/test/pipeline/test-springcloudlogs-raw.log-expected.json
@@ -0,0 +1,50 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2021-07-01T19:30:30.535Z",
+            "azure": {
+                "resource": {
+                    "group": "SA-HEMANT",
+                    "id": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC",
+                    "name": "HM-SC-PETCLINIC",
+                    "provider": "MICROSOFT.APPPLATFORM/SPRING"
+                },
+                "springcloudlogs": {
+                    "category": "ApplicationConsole",
+                    "event_category": "Administrative",
+                    "log_format": "RAW",
+                    "operation_name": "Microsoft.AppPlatform/Spring/logs",
+                    "properties": {
+                        "app_name": "admin-server",
+                        "instance_name": "admin-server-default-12-8459d44f68-g4b5f",
+                        "service_id": "c41fd000b1a5450eb234039376da26de",
+                        "service_name": "hm-sc-petclinic",
+                        "stream": "stdout"
+                    }
+                },
+                "subscription_id": "EDD63B67-0BA2-4837-A4EB-CD484E9FF623"
+            },
+            "cloud": {
+                "provider": "azure"
+            },
+            "ecs": {
+                "version": "1.10.0"
+            },
+            "event": {
+                "action": "Microsoft.AppPlatform/Spring/logs",
+                "kind": "event",
+                "original": "{ \"time\": \"2021-07-01T19:30:30.535404056Z\", \"LogFormat\": \"RAW\", \"resourceId\": \"/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC\", \"operationName\": \"Microsoft.AppPlatform/Spring/logs\", \"category\": \"ApplicationConsole\", \"level\": \"Informational\", \"location\": \"westus2\", \"properties\": {\"Log\":\"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\\n\",\"Stream\":\"stdout\",\"AppName\":\"admin-server\",\"InstanceName\":\"admin-server-default-12-8459d44f68-g4b5f\",\"ServiceId\":\"c41fd000b1a5450eb234039376da26de\",\"ServiceName\":\"hm-sc-petclinic\"}}"
+            },
+            "geo": {
+                "name": "westus2"
+            },
+            "log": {
+                "level": "Informational"
+            },
+            "message": "2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\n",
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}
diff --git a/packages/azure/data_stream/springcloudlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/springcloudlogs/agent/stream/azure-eventhub.yml.hbs
@@ -0,0 +1,33 @@
+{{#if connection_string}}
+connection_string: {{connection_string}}
+{{/if}}
+{{#if eventhub}}
+eventhub: {{eventhub}}
+storage_account_container: filebeat-springcloudlogs-{{eventhub}}
+{{/if}}
+{{#if consumer_group}}
+consumer_group: {{consumer_group}}
+{{/if}}
+{{#if storage_account}}
+storage_account: {{storage_account}}
+{{/if}}
+{{#if storage_account_key}}
+storage_account_key: {{storage_account_key}}
+{{/if}}
+{{#if resource_manager_endpoint}}
+resource_manager_endpoint: {{resource_manager_endpoint}}
+{{/if}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
diff --git a/packages/azure/data_stream/springcloudlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/springcloudlogs/agent/stream/log.yml.hbs
@@ -0,0 +1,19 @@
+paths:
+  {{#each paths as |path i|}}
+- {{path}}
+  {{/each}}
+exclude_files: [".gz$"]
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
diff --git a/...azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml b/...azure/data_stream/springcloudlogs/elasticsearch/ingest_pipeline/azure-shared-pipeline.yml
@@ -0,0 +1,91 @@
+---
+description: Pipeline for parsing azure activity logs.
+processors:
+  - set:
+      field: cloud.provider
+      value: azure
+  - grok:
+      field: azure.resource_id
+      patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/namespaces/%{NAMESPACE:azure.resource.namespace}/authorizationRules/%{RULE:azure.resource.authorization_rule}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+        GROUPID: .+
+        PROVIDERNAME: .+
+        NAMESPACE: .+
+        RULE: .+
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+      - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+        GROUPID: .+
+        PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+/([A-Za-z])\w+.
+        NAME: ((?!AUTHORIZATIONRULES).)*$
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+        - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}/providers/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+        GROUPID: .+
+        PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+        NAME: .+
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+      - /providers/%{PROVIDER:azure.resource.provider}
+      - /PROVIDERS/%{PROVIDER:azure.resource.provider}
+      pattern_definitions:
+        PROVIDER: .+
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}
+        - /subscriptions/%{SUBID:azure.subscription_id}/providers/%{PROVIDERNAME:azure.resource.provider}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+        PROVIDERNAME: ([A-Za-z])\w+.([A-Za-z])\w+\/([A-Za-z][^\/])\w+
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}
+        - /subscriptions/%{SUBID:azure.subscription_id}/resourceGroups/%{GROUPID:azure.resource.group}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+        GROUPID: .+
+      ignore_failure: true
+  - grok:
+      field: azure.resource_id
+      if: 'ctx.azure?.subscription_id == null'
+      patterns:
+        - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}
+        - /subscriptions/%{SUBID:azure.subscription_id}
+      pattern_definitions:
+        SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1}
+      ignore_failure: true
+  - rename:
+      field: azure.resource_id
+      target_field: azure.resource.id
+      ignore_missing: true
+  - lowercase:
+      field: event.outcome
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{ "time": "2021-07-01T19:30:30.535404056Z", "LogFormat": "RAW", "resourceId": "/SUBSCRIPTIONS/EDD63B67-0BA2-4837-A4EB-CD484E9FF623/RESOURCEGROUPS/SA-HEMANT/PROVIDERS/MICROSOFT.APPPLATFORM/SPRING/HM-SC-PETCLINIC", "operationName": "Microsoft.AppPlatform/Spring/logs", "category": "ApplicationConsole", "level": "Informational", "location": "westus2", "properties": {"Log":"2021-07-01 19:30:30.535 INFO 1 --- [oundedElastic-9] c.c.c.ConfigServicePropertySourceLocator : Located environment: name=admin-server, profiles=[mysql], label=null, version=638a1af7fc8d331d7eb26a571275e954632717e8, state=null\n","Stream":"stdout","AppName":"admin-server","InstanceName":"admin-server-default-12-8459d44f68-g4b5f","ServiceId":"c41fd000b1a5450eb234039376da26de","ServiceName":"hm-sc-petclinic"}}