Merge pull request #49 from fabiogermann/master

Fixed exception in logstash and added ECS translation config

Author: bitsofinfo

2082_filter_section_h_extract_stopwatch.conf

@@ -24,15 +24,15 @@ filter {
     # micro -> milli
     ruby {
       code => "
-          event_date_milliseconds = (event.get('event_date_microseconds') / 1000.0)
+          event_date_milliseconds = (event.get('event_date_microseconds').to_i / 1000.0)
           event.set('event_date_milliseconds', event_date_milliseconds)
         "
     }
 
     # milli -> seconds
     ruby {
       code => "
-          event_date_seconds = (event.get('event_date_milliseconds') / 1000.0)
+          event_date_seconds = (event.get('event_date_milliseconds').to_i / 1000.0)
           event.set('event_date_seconds', event_date_seconds)
         "
     }

2600_filter_ecs.conf

@@ -0,0 +1,139 @@
+filter {
+  if [type] == "mod_security" {
+
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    # Align logs with Elastic ECS
+    #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    mutate {
+      # Rename to existing ECS fields
+      rename => ["fqdn", "[host][hostname]"]
+      rename => ["requestHeaders.User-Agent", "[user_agent][original]"]
+      rename => ["httpMethod", "[http][request][method]"]
+      rename => ["requestedUri", "[url][path]"]
+      rename => ["sourceIp", "[client][ip]"]
+      rename => ["sourcePort", "[client][port]"]
+      rename => ["program", "[service][type]"]
+      rename => ["destIp", "[destination][ip]"]
+      rename => ["destPort", "[destination][port]"]
+      rename => ["uniqueId", "[tracing][trace][id]"]
+
+      # Rename to custom ECS fields
+      rename => ["requestHeaders", "[http][request][header]"]
+      rename => ["responseHeaders", "[http][response][header]"]
+      rename => ["auditLogTrailer", "[apache][mod_security][audit_log_trailer]"]
+      rename => ["rawSectionA", "[apache][mod_security][section][a]"]
+      rename => ["rawSectionB", "[apache][mod_security][section][b]"]
+      rename => ["rawSectionF", "[apache][mod_security][section][f]"]
+      rename => ["rawSectionH", "[apache][mod_security][section][h]"]
+      rename => ["matchedRules", "[apache][mod_security][matched_rule][raw]"]
+      rename => ["secRuleIds", "[apache][mod_security][matched_rule][id]"]
+    }
+
+    grok {
+      match => [ "incomingProtocol", "(HTTP\W)(%{NUMBER:http.version})" ]
+    }
+
+    grok {
+      match => [ "responseStatus", "(%{NUMBER:http.response.status_code:long} %{DATA})" ]
+    }
+
+    useragent { # workaround until ECS support is available: https://github.com/logstash-plugins/logstash-filter-useragent/issues/56
+      source => "[user_agent][original]"
+      target => "ua_tmp"
+
+      add_field => {
+        "[user_agent][device][name]" => "%{[ua_tmp][device]}"
+        "[user_agent][os][name]" => "%{[ua_tmp][os_name]}"
+        "[user_agent][name]" => "%{[ua_tmp][name]}"
+      }
+    }
+
+    # OS version ECS compatibility
+    if [ua_tmp][os_major] {
+      mutate {
+        add_field => {
+          "[user_agent][os][version]" => "%{[ua_tmp][os_major]}"
+        }
+      }
+
+      if [ua_tmp][os_minor] {
+        mutate {
+          replace => {
+            "[user_agent][os][version]" => "%{[user_agent][os][version]}.%{[ua_tmp][os_minor]}"
+          }
+        }
+
+        if [ua_tmp][os_patch] {
+          mutate {
+            replace => {
+              "[user_agent][os][version]" => "%{[user_agent][os][version]}.%{[ua_tmp][os_patch]}"
+            }
+          }
+
+          if [ua_tmp][os_build] {
+            mutate {
+              replace => {
+                "[user_agent][os][version]" => "%{[user_agent][os][version]}.%{[ua_tmp][os_build]}"
+              }
+            }
+          }
+        }
+      }
+
+      mutate {
+        add_field => {
+          "[user_agent][os][full]" => "%{[user_agent][os][name]} %{[user_agent][os][version]}"
+        }
+      }
+    }
+
+    # User agent version ECS compatibility
+    if [ua_tmp][major] {
+      mutate {
+        add_field => {
+          "[user_agent][version]" => "%{[ua_tmp][major]}"
+        }
+      }
+
+      if [ua_tmp][minor] {
+        mutate {
+          replace => {
+            "[user_agent][version]" => "%{[user_agent][version]}.%{[ua_tmp][minor]}"
+          }
+        }
+
+        if [ua_tmp][patch] {
+          mutate {
+            replace => {
+              "[user_agent][version]" => "%{[user_agent][version]}.%{[ua_tmp][patch]}"
+            }
+          }
+
+          if [ua_tmp][build] {
+            mutate {
+              replace => {
+                "[user_agent][version]" => "%{[user_agent][version]}.%{[ua_tmp][build]}"
+              }
+            }
+          }
+        }
+      }
+    }
+
+    mutate {
+      remove_field => ["ua_tmp"]
+    }
+
+#    geoip {
+#      source => "[client][ip]"
+#      target => "[client][geo]"
+#      database => "/var/lib/GeoIP/GeoIP2-City.mmdb"
+#    }
+#    geoip {
+#      source => "[client][ip]"
+#      target => "[client][as]"
+#      database => "/var/lib/GeoIP/GeoLite2-ASN.mmdb"
+#    }
+  }
+}