From 5505d4ea5128183d670e67e6fd5e120a27032f33 Mon Sep 17 00:00:00 2001 From: Paul Czarkowski Date: Mon, 30 Sep 2013 09:45:14 -0500 Subject: [PATCH] updated for 1.2 --- recipes/central-syslog/index.md | 5 +- recipes/central-syslog/logstash-indexer.conf | 146 ++++-------------- .../central-syslog/logstash-indexer_NAT.conf | 38 ++--- recipes/central-syslog/syslog-server.conf | 33 ++++ 4 files changed, 79 insertions(+), 143 deletions(-) diff --git a/recipes/central-syslog/index.md b/recipes/central-syslog/index.md index 74c2f09..b8c1742 100644 --- a/recipes/central-syslog/index.md +++ b/recipes/central-syslog/index.md @@ -26,14 +26,15 @@ If you have a large set of logs to slurp, you may want to set `maxmemory 500mb` # prerequisites -* This recipe requires logstash 1.1.1 or newer. +* This recipe requires logstash 1.2 or newer. * This recipe assumes a standard rsyslog format ( PRI prefix not needed, but it does yield richer results ) * This recipe assumes you have a logstash-indexer running redis for queueing. # Syslog Server - File Input -The config on your syslog server should look like below. Pretty simple stuff, we're just declaring a file input and pushing to redis. I chose not to do any filtering here as I want logstash to simply act as an agent on this server. +The config on your syslog server should look like below. Pretty simple stuff, we're just declaring a file input and pushing to redis. The filter section here is optional, but if your syslog server has plenty of free CPU it's probably +worth doing. {% include_code syslog-server.conf %} diff --git a/recipes/central-syslog/logstash-indexer.conf b/recipes/central-syslog/logstash-indexer.conf index 4ef1302..36133c7 100644 --- a/recipes/central-syslog/logstash-indexer.conf +++ b/recipes/central-syslog/logstash-indexer.conf @@ -1,129 +1,45 @@ input { redis { - host => "127.0.0.1" + host => "127.0.0.1" data_type => "list" - type => "redis" - key => "logstash" - message_format => "json_event" + type => "redis" + key => "logstash" + threads => 5 } } filter { - # Check if syslog message has PRI using grep. If so then : - # strip the syslog PRI part and create facility and severity fields. - # the original syslog message is saved in field %{syslog_raw_message}. - # the extracted PRI is available in the %{syslog_pri} field. - # - # You get %{syslog_facility_code} and %{syslog_severity_code} fields. - # You also get %{syslog_facility} and %{syslog_severity} fields if the - # use_labels option is set True (the default) on syslog_pri filter. - grep { - type => "syslog" - match => ["@message","<\d+>"] - add_tag => "has_pri" - drop => false + if [type] == "syslog" { + if ! [processed] { + grok { + pattern => + [ + "<%{NUMBER:syslog_pri}> %{SYSLOGBASE2} %{GREEDYDATA:message_remainder}", + "%{SYSLOGBASE2} %{GREEDYDATA:message_remainder}" + ] + add_tag => "match_syslog" + } + if "match_syslog" in [tags] { + syslog_pri { + syslog_pri_field_name => "syslog_pri" + } + date { + match => ["SYSLOGTIMESTAMP","MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"] + } + mutate { + replace => ["message", "%{message_remainder}"] + } + mutate { + remove => ["message_remainder"] + add_field => ["processed","true"] + remove_tag => ["match_syslog"] + } + } + } } - grok { - type => "syslog" - tags => [ "has_pri" ] - pattern => [ "<%{POSINT:syslog_pri}>%{SPACE}%{GREEDYDATA:message_remainder}" ] - add_tag => "got_syslog_pri" - add_field => [ "syslog_raw_message", "%{@message}" ] - } - syslog_pri { - type => "syslog" - tags => [ "got_syslog_pri" ] - } - mutate { - type => "syslog" - tags => [ "got_syslog_pri" ] - replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog" - tags => [ "got_syslog_pri" ] - remove => [ "message_remainder" ] - } - # strip the syslog timestamp and force event timestamp to be the same. - # the original string is saved in field %{syslog_timestamp}. - # the original logstash input timestamp is saved in field %{received_at}. - grok { - type => "syslog" - pattern => [ "%{SYSLOGTIMESTAMP:syslog_timestamp}%{SPACE}%{GREEDYDATA:message_remainder}" ] - add_tag => "got_syslog_timestamp" - add_field => [ "received_at", "%{@timestamp}" ] - } - mutate { - type => "syslog" - tags => [ "got_syslog_timestamp" ] - replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog" - tags => [ "got_syslog_timestamp" ] - remove => [ "message_remainder" ] - } - date { - type => "syslog" - tags => [ "got_syslog_timestamp" ] - # season to taste for your own syslog format(s) - syslog_timestamp => [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ] - } - # strip the host field from the syslog line. - # the extracted host field becomes the logstash %{@source_host} metadata - # and is also available in the filed %{syslog_hostname}. - # the original logstash source_host is saved in field %{logstash_source}. - grok { - type => "syslog" - pattern => [ "%{SYSLOGHOST:syslog_hostname}%{SPACE}%{GREEDYDATA:message_remainder}" ] - add_tag => "got_syslog_host" - add_field => [ "logstash_source", "%{@source_host}" ] - } - mutate { - type => "syslog" - tags => [ "got_syslog_host" ] - replace => [ "@source_host", "%{syslog_hostname}" ] - replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # message_remainder no longer needed. - type => "syslog" - tags => [ "got_syslog_host" ] - remove => [ "message_remainder" ] - } - - - # strip the program and optional pid field from the syslog line. - # available in the field %{syslog_program} and %{syslog_pid}. - grok { - type => "syslog" - pattern => [ "%{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:%{SPACE}%{GREEDYDATA:message_remainder}" ] - add_tag => "got_syslog_program" - } - mutate { - type => "syslog" - tags => [ "got_syslog_program" ] - replace => [ "@message", "%{message_remainder}" ] - } - mutate { - # message_remainder no longer needed. - type => "syslog" - tags => [ "got_syslog_program" ] - remove => [ "message_remainder" ] - } - - ## Any extra processing you wish to do should be done here before - ## closing filter stanza and proceeding to output stanzas. - ## See logstash-indexer-NAT.conf example. - } output { elasticsearch { - type => "syslog" - # Uncomment below if you wish syslog messages to have their own ES index. - # index => "logstash-syslog-%{+YYYY.MM.dd}" } } \ No newline at end of file diff --git a/recipes/central-syslog/logstash-indexer_NAT.conf b/recipes/central-syslog/logstash-indexer_NAT.conf index f353d07..65a5fe5 100644 --- a/recipes/central-syslog/logstash-indexer_NAT.conf +++ b/recipes/central-syslog/logstash-indexer_NAT.conf @@ -3,30 +3,16 @@ # Slip this into the logstash-indexer.conf just before the filter stanza is closed. - grep { - type => "syslog" - match => ["@message","^RULE"] - add_tag => "is_Linux_NAT" - drop => false - } - kv { - type => "syslog" - tags => [ "is_Linux_NAT" ] - prefix => "nat_" - } - grok { - type => "syslog" - tags => [ "is_Linux_NAT" ] - pattern => [ "^RULE %{NUMBER:nat_Rule} -- %{DATA:nat_Action} %{GREEDYDATA:message_remainder}" ] - } - mutate { - type => "syslog" - tags => [ "is_Linux_NAT" ] - replace => [ "@message", "NAT - %{nat_Action} -- %{nat_SRC}:%{nat_SPT} -> %{nat_DST}:%{nat_DPT}" ] - } - mutate { - # XXX must not be combined with replacement which uses same field - type => "syslog" - tags => [ "is_Linux_NAT" ] - remove => [ "message_remainder" ] +if [type] == "syslog" { + if [message] =~ /^RULE/ { + kv { + prefix => "nat_" + } + grok { + pattern => [ "^RULE %{NUMBER:nat_Rule} -- %{DATA:nat_Action} " ] + } + mutate { + replace => [ "message", "NAT - %{nat_Action} -- %{nat_SRC}:%{nat_SPT} -> %{nat_DST}:%{nat_DPT}" ] + } } +} diff --git a/recipes/central-syslog/syslog-server.conf b/recipes/central-syslog/syslog-server.conf index 6655370..bb0aa84 100644 --- a/recipes/central-syslog/syslog-server.conf +++ b/recipes/central-syslog/syslog-server.conf @@ -10,6 +10,39 @@ input { } } +filter { + if [type] == "syslog" { + if ! [processed] { + grok { + pattern => + [ + "<%{NUMBER:syslog_pri}> %{SYSLOGBASE2} %{GREEDYDATA:message_remainder}", + "%{SYSLOGBASE2} %{GREEDYDATA:message_remainder}" + ] + add_tag => "match_syslog" + } + if "match_syslog" in [tags] { + syslog_pri { + syslog_pri_field_name => "syslog_pri" + } + date { + match => ["SYSLOGTIMESTAMP","MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"] + } + mutate { + replace => ["message", "%{message_remainder}"] + } + mutate { + remove => ["message_remainder"] + add_field => ["processed","true"] + remove_tag => ["match_syslog"] + } + } + } + } +} + + + output { redis { # change below to the hostname or ip address of your redis server. can add more than one redis host.