Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updated for 1.2 #78

Open
wants to merge 1 commit into
base: gh-pages
Choose a base branch
from
Open

updated for 1.2 #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions recipes/central-syslog/index.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,15 @@ If you have a large set of logs to slurp, you may want to set `maxmemory 500mb`

# prerequisites

* This recipe requires logstash 1.1.1 or newer.
* This recipe requires logstash 1.2 or newer.
* This recipe assumes a standard rsyslog format ( PRI prefix not needed, but it does yield richer results )
* This recipe assumes you have a logstash-indexer running redis for queueing.


# Syslog Server - File Input

The config on your syslog server should look like below. Pretty simple stuff, we're just declaring a file input and pushing to redis. I chose not to do any filtering here as I want logstash to simply act as an agent on this server.
The config on your syslog server should look like below. Pretty simple stuff, we're just declaring a file input and pushing to redis. The filter section here is optional, but if your syslog server has plenty of free CPU it's probably
worth doing.

{% include_code syslog-server.conf %}

Expand Down
146 changes: 31 additions & 115 deletions recipes/central-syslog/logstash-indexer.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,129 +1,45 @@
input {
redis {
host => "127.0.0.1"
host => "127.0.0.1"
data_type => "list"
type => "redis"
key => "logstash"
message_format => "json_event"
type => "redis"
key => "logstash"
threads => 5
}
}

filter {
# Check if syslog message has PRI using grep. If so then :
# strip the syslog PRI part and create facility and severity fields.
# the original syslog message is saved in field %{syslog_raw_message}.
# the extracted PRI is available in the %{syslog_pri} field.
#
# You get %{syslog_facility_code} and %{syslog_severity_code} fields.
# You also get %{syslog_facility} and %{syslog_severity} fields if the
# use_labels option is set True (the default) on syslog_pri filter.
grep {
type => "syslog"
match => ["@message","<\d+>"]
add_tag => "has_pri"
drop => false
if [type] == "syslog" {
if ! [processed] {
grok {
pattern =>
[
"<%{NUMBER:syslog_pri}> %{SYSLOGBASE2} %{GREEDYDATA:message_remainder}",
"%{SYSLOGBASE2} %{GREEDYDATA:message_remainder}"
]
add_tag => "match_syslog"
}
if "match_syslog" in [tags] {
syslog_pri {
syslog_pri_field_name => "syslog_pri"
}
date {
match => ["SYSLOGTIMESTAMP","MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"]
}
mutate {
replace => ["message", "%{message_remainder}"]
}
mutate {
remove => ["message_remainder"]
add_field => ["processed","true"]
remove_tag => ["match_syslog"]
}
}
}
}
grok {
type => "syslog"
tags => [ "has_pri" ]
pattern => [ "<%{POSINT:syslog_pri}>%{SPACE}%{GREEDYDATA:message_remainder}" ]
add_tag => "got_syslog_pri"
add_field => [ "syslog_raw_message", "%{@message}" ]
}
syslog_pri {
type => "syslog"
tags => [ "got_syslog_pri" ]
}
mutate {
type => "syslog"
tags => [ "got_syslog_pri" ]
replace => [ "@message", "%{message_remainder}" ]
}
mutate {
# XXX must not be combined with replacement which uses same field
type => "syslog"
tags => [ "got_syslog_pri" ]
remove => [ "message_remainder" ]
}
# strip the syslog timestamp and force event timestamp to be the same.
# the original string is saved in field %{syslog_timestamp}.
# the original logstash input timestamp is saved in field %{received_at}.
grok {
type => "syslog"
pattern => [ "%{SYSLOGTIMESTAMP:syslog_timestamp}%{SPACE}%{GREEDYDATA:message_remainder}" ]
add_tag => "got_syslog_timestamp"
add_field => [ "received_at", "%{@timestamp}" ]
}
mutate {
type => "syslog"
tags => [ "got_syslog_timestamp" ]
replace => [ "@message", "%{message_remainder}" ]
}
mutate {
# XXX must not be combined with replacement which uses same field
type => "syslog"
tags => [ "got_syslog_timestamp" ]
remove => [ "message_remainder" ]
}
date {
type => "syslog"
tags => [ "got_syslog_timestamp" ]
# season to taste for your own syslog format(s)
syslog_timestamp => [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
}
# strip the host field from the syslog line.
# the extracted host field becomes the logstash %{@source_host} metadata
# and is also available in the filed %{syslog_hostname}.
# the original logstash source_host is saved in field %{logstash_source}.
grok {
type => "syslog"
pattern => [ "%{SYSLOGHOST:syslog_hostname}%{SPACE}%{GREEDYDATA:message_remainder}" ]
add_tag => "got_syslog_host"
add_field => [ "logstash_source", "%{@source_host}" ]
}
mutate {
type => "syslog"
tags => [ "got_syslog_host" ]
replace => [ "@source_host", "%{syslog_hostname}" ]
replace => [ "@message", "%{message_remainder}" ]
}
mutate {
# message_remainder no longer needed.
type => "syslog"
tags => [ "got_syslog_host" ]
remove => [ "message_remainder" ]
}


# strip the program and optional pid field from the syslog line.
# available in the field %{syslog_program} and %{syslog_pid}.
grok {
type => "syslog"
pattern => [ "%{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?:%{SPACE}%{GREEDYDATA:message_remainder}" ]
add_tag => "got_syslog_program"
}
mutate {
type => "syslog"
tags => [ "got_syslog_program" ]
replace => [ "@message", "%{message_remainder}" ]
}
mutate {
# message_remainder no longer needed.
type => "syslog"
tags => [ "got_syslog_program" ]
remove => [ "message_remainder" ]
}

## Any extra processing you wish to do should be done here before
## closing filter stanza and proceeding to output stanzas.
## See logstash-indexer-NAT.conf example.

}

output {
elasticsearch {
type => "syslog"
# Uncomment below if you wish syslog messages to have their own ES index.
# index => "logstash-syslog-%{+YYYY.MM.dd}"
}
}
38 changes: 12 additions & 26 deletions recipes/central-syslog/logstash-indexer_NAT.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,16 @@
# Slip this into the logstash-indexer.conf just before the filter stanza is closed.


grep {
type => "syslog"
match => ["@message","^RULE"]
add_tag => "is_Linux_NAT"
drop => false
}
kv {
type => "syslog"
tags => [ "is_Linux_NAT" ]
prefix => "nat_"
}
grok {
type => "syslog"
tags => [ "is_Linux_NAT" ]
pattern => [ "^RULE %{NUMBER:nat_Rule} -- %{DATA:nat_Action} %{GREEDYDATA:message_remainder}" ]
}
mutate {
type => "syslog"
tags => [ "is_Linux_NAT" ]
replace => [ "@message", "NAT - %{nat_Action} -- %{nat_SRC}:%{nat_SPT} -> %{nat_DST}:%{nat_DPT}" ]
}
mutate {
# XXX must not be combined with replacement which uses same field
type => "syslog"
tags => [ "is_Linux_NAT" ]
remove => [ "message_remainder" ]
if [type] == "syslog" {
if [message] =~ /^RULE/ {
kv {
prefix => "nat_"
}
grok {
pattern => [ "^RULE %{NUMBER:nat_Rule} -- %{DATA:nat_Action} " ]
}
mutate {
replace => [ "message", "NAT - %{nat_Action} -- %{nat_SRC}:%{nat_SPT} -> %{nat_DST}:%{nat_DPT}" ]
}
}
}
33 changes: 33 additions & 0 deletions recipes/central-syslog/syslog-server.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,39 @@ input {
}
}

filter {
if [type] == "syslog" {
if ! [processed] {
grok {
pattern =>
[
"<%{NUMBER:syslog_pri}> %{SYSLOGBASE2} %{GREEDYDATA:message_remainder}",
"%{SYSLOGBASE2} %{GREEDYDATA:message_remainder}"
]
add_tag => "match_syslog"
}
if "match_syslog" in [tags] {
syslog_pri {
syslog_pri_field_name => "syslog_pri"
}
date {
match => ["SYSLOGTIMESTAMP","MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"]
}
mutate {
replace => ["message", "%{message_remainder}"]
}
mutate {
remove => ["message_remainder"]
add_field => ["processed","true"]
remove_tag => ["match_syslog"]
}
}
}
}
}



output {
redis {
# change below to the hostname or ip address of your redis server. can add more than one redis host.
Expand Down