diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cac5bed..210a5f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,9 @@ -# This is a basic workflow to help you get started with Actions - name: Test Matrix # Controls when the workflow will run on: - # Triggers the workflow on push or pull request events but only for the "master" branch push: - branches: [ "master" ] + branches: [ "master", "develop" ] pull_request: branches: [ "master" ] @@ -20,7 +17,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest'] - perl: [ '5.40', '5.38', '5.36', '5.34', '5.32', '5.30', '5.28', '5.26', '5.24', '5.22', '5.20', '5.18', '5.16', '5.14' ] + perl: [ '5.40', '5.38', '5.36', '5.34', '5.32', '5.30', '5.28', '5.26', '5.24', '5.22', '5.20', '5.18', '5.16' ] name: Perl ${{ matrix.perl }} on ${{ matrix.os }} steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index ae2b0f8..0c3fe25 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ Makefile.old .cvsignore pm_to_blib MYMETA.* +nytprof* diff --git a/README.mkdn b/README.mkdn index cdc98b5..b0cc596 100644 --- a/README.mkdn +++ b/README.mkdn @@ -4,7 +4,7 @@ Parse::Syslog::Line - Simple syslog line parser # VERSION -version 6.0 +version 6.1 # SYNOPSIS @@ -251,14 +251,6 @@ The changes to the API and fields returned are as follows: - `NormalizeToUTC` is **deprecated**, every log now returns `datetime_utc` - `OutputTimeZone` is **deprecated**, use `TimeMomentFormatString` - **Field Changes** - - `date_raw` - - **Removed** from the fields returned, use `datetime_raw`. - - - `datetime_obj` - - **Removed** from the fields returned as we dropped support for [DateTime](https://metacpan.org/pod/DateTime). - - `datetime_utc` Present in every document, use this for portability. @@ -455,6 +447,10 @@ whitespace will be assumed to be a continuation of the previous line. It is not exported by default. +## psl\_enable\_sdata + +Call this to turn on all the Structured Data Parsing Options + ## preamble\_priority Takes the Integer portion of the syslog messsage and returns diff --git a/benchmarks/00-defaults.pl b/benchmarks/00-defaults.pl index 47484b4..5a84b72 100644 --- a/benchmarks/00-defaults.pl +++ b/benchmarks/00-defaults.pl @@ -1,10 +1,11 @@ #!perl -use v5.14; +use v5.16; use warnings; use Dumbbench; use Const::Fast; use Parse::Syslog::Line; +psl_enable_sdata(); use FindBin; use lib "$FindBin::Bin/../t/lib"; @@ -21,7 +22,7 @@ my ($test) = @_; @copy = @msgs unless @copy and $last ne $test; $last=$test; - parse_syslog_line(shift @copy); + my $doc = parse_syslog_line(shift @copy); }; my $bench = Dumbbench->new( diff --git a/benchmarks/01-parse.pl b/benchmarks/01-parse.pl index 372aa32..baaee3a 100644 --- a/benchmarks/01-parse.pl +++ b/benchmarks/01-parse.pl @@ -1,6 +1,6 @@ #!perl -use v5.14; +use v5.16; use warnings; use Benchmark qw/timethese cmpthese/; use Const::Fast; diff --git a/benchmarks/02-benchsmarter.pl b/benchmarks/02-benchsmarter.pl index b19a1fd..98109e6 100644 --- a/benchmarks/02-benchsmarter.pl +++ b/benchmarks/02-benchsmarter.pl @@ -1,6 +1,6 @@ #!perl -use v5.14; +use v5.16; use warnings; use Const::Fast; use Dumbbench; diff --git a/benchmarks/xt-date-parsing-iso.pl b/benchmarks/xt-date-parsing-iso.pl index 7eb0679..84bf5d9 100644 --- a/benchmarks/xt-date-parsing-iso.pl +++ b/benchmarks/xt-date-parsing-iso.pl @@ -1,6 +1,6 @@ #!/usr/bin/perl # -use v5.14; +use v5.16; use warnings; use Benchmark qw(cmpthese); diff --git a/benchmarks/xt-date-parsing-legacy.pl b/benchmarks/xt-date-parsing-legacy.pl index d73583f..2cd579c 100644 --- a/benchmarks/xt-date-parsing-legacy.pl +++ b/benchmarks/xt-date-parsing-legacy.pl @@ -1,6 +1,6 @@ #!/usr/bin/perl # -use v5.14; +use v5.16; use warnings; use Benchmark qw(cmpthese); diff --git a/bin/parse-syslog-line.pl b/bin/parse-syslog-line.pl index 964224d..db31e18 100644 --- a/bin/parse-syslog-line.pl +++ b/bin/parse-syslog-line.pl @@ -1,14 +1,14 @@ #!perl # PODNAME: parse-syslog-line.pl # ABSTRACT: Parse a syslog message and display the structured data -use v5.14; +use v5.16; use warnings; use Data::Printer; use Getopt::Long::Descriptive; use Pod::Usage; use JSON::MaybeXS; -use Parse::Syslog::Line qw( parse_syslog_line ); +use Parse::Syslog::Line; use YAML::XS; my $enc; @@ -94,9 +94,9 @@ =head1 EXAMPLES =cut # Configure +psl_enable_sdata() if $opt->sdata; $Parse::Syslog::Line::PruneEmpty = !$opt->empty; $Parse::Syslog::Line::PruneRaw = !$opt->raw; -$Parse::Syslog::Line::AutoDetectJSON = $Parse::Syslog::Line::AutoDetectKeyValues = $opt->sdata; $Parse::Syslog::Line::RFC5424StructuredData = !$opt->disable_rfc_sdata; $Parse::Syslog::Line::RFC5424StructuredDataStrict = $opt->strict_rfc_sdata; diff --git a/cpanfile b/cpanfile index 5967177..6c2177e 100644 --- a/cpanfile +++ b/cpanfile @@ -7,6 +7,7 @@ requires "Data::Printer" => "0"; requires "English" => "0"; requires "Exporter" => "0"; requires "Getopt::Long::Descriptive" => "0"; +requires "Hash::Merge::Simple" => "0"; requires "JSON::MaybeXS" => "0"; requires "Module::Load" => "0"; requires "Module::Loaded" => "0"; @@ -15,7 +16,7 @@ requires "Pod::Usage" => "0"; requires "Ref::Util" => "0"; requires "Time::Moment" => "0"; requires "YAML::XS" => "0"; -requires "perl" => "5.014"; +requires "perl" => "v5.16.0"; requires "warnings" => "0"; recommends "Cpanel::JSON::XS" => "0"; @@ -34,16 +35,15 @@ on 'test' => sub { requires "Test::Deep" => "0"; requires "Test::MockTime" => "0"; requires "Test::More" => "0"; - requires "YAML" => "0"; requires "bignum" => "0"; requires "lib" => "0"; - requires "perl" => "5.014"; + requires "perl" => "v5.16.0"; requires "strict" => "0"; }; on 'configure' => sub { requires "ExtUtils::MakeMaker" => "0"; - requires "perl" => "v5.14.0"; + requires "perl" => "v5.16.0"; }; on 'develop' => sub { diff --git a/dist.ini b/dist.ini index 5597ca1..669b001 100644 --- a/dist.ini +++ b/dist.ini @@ -12,6 +12,7 @@ exclude_filename = cpanfile exclude_filename = dist.ini exclude_filename = weaver.ini exclude_filename = weaver-ci.ini +exclude_filename = cpanfile [ExecDir] dir = bin [PruneCruft] diff --git a/lib/Parse/Syslog/Line.pm b/lib/Parse/Syslog/Line.pm index 2438557..7eaba45 100644 --- a/lib/Parse/Syslog/Line.pm +++ b/lib/Parse/Syslog/Line.pm @@ -2,22 +2,23 @@ package Parse::Syslog::Line; -use v5.14; +use v5.16; use warnings; use Carp; use Const::Fast; use English qw(-no_match_vars); use Exporter; -use JSON::MaybeXS qw( decode_json ); -# RECOMMEND PREREQ: Cpanel::JSON::XS -use Module::Load qw( load ); -use Module::Loaded qw( is_loaded ); -use POSIX qw( strftime tzset ); -use Ref::Util qw( is_arrayref ); +use Hash::Merge::Simple qw( dclone_merge ); +use JSON::MaybeXS qw( decode_json ); +use Module::Load qw( load ); +use Module::Loaded qw( is_loaded ); +use POSIX qw( strftime tzset ); +use Ref::Util qw( is_arrayref ); use Time::Moment; +# RECOMMEND PREREQ: Cpanel::JSON::XS -our $VERSION = '6.0'; +our $VERSION = '6.1'; # Default for Handling Parsing our $DateParsing = 1; @@ -167,10 +168,11 @@ Optional Exports: =cut our @ISA = qw(Exporter); -our @EXPORT = qw(parse_syslog_line); +our @EXPORT = qw(parse_syslog_line psl_enable_sdata); our @EXPORT_OK = qw( parse_syslog_line parse_syslog_lines + psl_enable_sdata preamble_priority preamble_facility %LOG_FACILITY %LOG_PRIORITY get_syslog_timezone set_syslog_timezone @@ -186,7 +188,7 @@ our %EXPORT_TAGS = ( const my %RE => ( IPv4 => qr/(?>(?:[0-9]{1,3}\.){3}[0-9]{1,3})/, preamble => qr/^\<(\d+)\>(\d{0,2}(?=\s))?\s*/, - date => qr/ + date_named_capture => qr/ (?:(?\d{4})\s)? # Option Year: YYYY (? # Whole String (?[A-Za-z]{3}) # Month: Mmm @@ -201,6 +203,21 @@ const my %RE => ( )? ) /x, + date => qr/ + (?:(\d{4})\s)? # Option Year: YYYY --> 1 + ( # Whole String -------> 2 + ([A-Za-z]{3}) # Month: Mmm -------> 3 + \s+ + ([0-9]+) # Day: DD ------------> 4 + \s+ + ([0-9]{1,2}) # Hour: HH -----------> 5 + :([0-9]{2}) # Minute: MM ---------> 6 + :([0-9]{2}) # Second: SS ---------> 7 + (?:\. # Time: .DDD millisecond or .DDDDDD microsecond resolution + ((?:[0-9]{3}){1,2}) # Highprecision --> 8 + )? + ) + /x, date_long => qr/ (?:[0-9]{4}\s+)? # Year: Because, Cisco ([.*])? # Cisco adds a * for no ntp, and a . for configured but out of sync @@ -236,9 +253,9 @@ const my %RE => ( (?[0-9]{2}) ) )/x, - host => qr/\s*(?[^:\s]+)\s+/, + host => qr/\s*([^:\s]+)\s+/, cisco_detection => qr/\s*[0-9]*:\s+/, - program_raw => qr/\s*([^\[][^:]+)(?:|\s-)\s+/, + program_raw => qr/\s*([^\[][^:]+)(:|\s-)\s+/, program_name => qr/(.[^\[\(\ ]*)(.*)/, program_sub => qr/(?>\(([^\)]+)\))/, program_pid => qr/(?>\[([^\]]+)\])/, @@ -502,14 +519,6 @@ need to build it yourself. =over 2 -=item C - -B from the fields returned, use C. - -=item C - -B from the fields returned as we dropped support for L. - =item C Present in every document, use this for portability. @@ -690,15 +699,6 @@ A convenient function which sets the syslog timezone to UTC. =cut -my %_empty_msg = map { $_ => undef } qw( - preamble priority priority_int facility facility_int - datetime_raw date time epoch tz - datetime_local datetime_str datetime_utc - host_raw host domain - program_raw program_name program_pid program_sub -); - - my $SYSLOG_TIMEZONE = ''; sub parse_syslog_line { @@ -710,7 +710,7 @@ sub parse_syslog_line { state $DisableWarnings = $ENV{PARSE_SYSLOG_LINE_QUIET} || $ENV{TEST_ACTIVE} || $ENV{TEST2_ACTIVE}; # Initialize everything to undef - my %msg = $PruneEmpty ? () : %_empty_msg; + my %msg = (); $msg{message_raw} = $raw_string unless $PruneRaw; # Lines that begin with a space aren't syslog messages, skip @@ -735,11 +735,20 @@ sub parse_syslog_line { # Handle Date/Time my %date = (); if( $raw_string =~ s/^$RE{date}//o) { - $msg{datetime_raw} = $+{date}; - $msg{datetime_raw} .= " $+{year}" - if $+{year}; + # Positional + $msg{datetime_raw} = $2; + $msg{datetime_raw} .= " $1" if $1; # Copy into the date hash - %date = %+; + %date = ( + year => $1, + date => $2, + month => $3, + day => $4, + hour => $5, + minute => $6, + second => $7, + highprecision => $8, + ); $date{month_abbr} = 1; } elsif( $raw_string =~ s/^$RE{date_iso8601}//o) { @@ -788,7 +797,7 @@ sub parse_syslog_line { $date{year} = 1900 + (localtime)[5]; } $tm = eval { Time::Moment->new(%date) }; - if ( $tm ) { + if ( defined $tm ) { # Check that the timestamp isn't more than 1 day in the future $tm = Time::Moment->new(%date, year => $date{year} - 1) if !$has_year && $tm->epoch > (time + 86400); @@ -827,7 +836,7 @@ sub parse_syslog_line { # # Host Information: if( $raw_string =~ s/^$RE{host}//o ) { - my $hostStr = $+{host}; + my $hostStr = $1; my($ip) = ($hostStr =~ /($RE{IPv4})/o); if( defined $ip && length $ip ) { $msg{host_raw} = $hostStr; @@ -841,9 +850,9 @@ sub parse_syslog_line { } } # Check for relayed logs, grab the origin - while( $raw_string =~ /^(?:\s*[0-9]+\s+)?(?$RE{date_iso8601}|$RE{date})\s+$RE{host}/go ) { - $msg{origin} = $+{host}; - $msg{origin_date} = $+{any_date}; + while( $raw_string =~ /^(?:\s*[0-9]+\s+)?$RE{date_iso8601}\s+$RE{host}/go ) { + $msg{origin} = $2; + $msg{origin_date} = $1; $raw_string = substr($raw_string,pos($raw_string)); } @@ -868,8 +877,8 @@ sub parse_syslog_line { my $progsep = ':'; if( $ExtractProgram ) { if( $raw_string =~ s/^$RE{program_raw}//o ) { - $progsep = $+{program_sep}; $msg{program_raw} = $1; + $progsep = $2 || ''; my $progStr = join ' ', grep {!exists $INT_PRIORITY{$_}} split /\s+/, $msg{program_raw}; if( $progStr =~ /^$RE{program_name}/o ) { $msg{program_name} = $1; @@ -954,14 +963,15 @@ sub parse_syslog_line { $CpanelJSONXSWarning = 1; } eval { - $msg{SDATA} = decode_json(substr($msg{content},$pos)); + my $json = decode_json(substr($msg{content},$pos)); + $msg{SDATA} = $msg{SDATA} ? dclone_merge($json,$msg{SDATA}) : $json; 1; } or do { my $err = $@; $msg{_json_error} = sprintf "Failed to decode json: %s", $err; }; } - if( !$msg{SDATA} && $AutoDetectKeyValues && $msg{content} =~ /(?:^|\s)[a-zA-Z\.0-9\-_]+=\S+/ ) { + if( $AutoDetectKeyValues && $msg{content} =~ /(?:^|\s)[a-zA-Z\.0-9\-_]+=\S+/ ) { my %sdata = (); while( $msg{content} =~ /$RE{kvdata}/og ) { my ($k,$v) = ($1,$2); @@ -982,14 +992,16 @@ sub parse_syslog_line { $sdata{$k} = $v; } } - $msg{SDATA} = \%sdata if keys %sdata; + if ( %sdata ) { + $msg{SDATA} = $msg{SDATA} ? dclone_merge(\%sdata,$msg{SDATA}) : \%sdata; + } } if( $PruneRaw ) { delete $msg{$_} for grep { $_ =~ /_raw$/ } keys %msg; } if( $PruneEmpty ) { - delete $msg{$_} for grep { !defined $msg{$_} || !length $msg{$_} } keys %msg; + delete $msg{$_} for grep { !length $msg{$_} } keys %msg; } if( @PruneFields ) { no warnings; @@ -1058,6 +1070,18 @@ It is not exported by default. } +=head2 psl_enable_sdata + +Call this to turn on all the Structured Data Parsing Options + +=cut + +sub psl_enable_sdata { + $AutoDetectJSON = 1; + $AutoDetectKeyValues = 1; + $RFC5424StructuredData = 1; +} + =head2 preamble_priority Takes the Integer portion of the syslog messsage and returns diff --git a/profile/01-minimal.pl b/profile/01-minimal.pl index 5d27073..e3ee4d2 100644 --- a/profile/01-minimal.pl +++ b/profile/01-minimal.pl @@ -1,9 +1,10 @@ #!perl -use strict; +use v5.16; use warnings; use Parse::Syslog::Line; use Time::HiRes qw(gettimeofday tv_interval); +psl_enable_sdata(); use FindBin; use lib "$FindBin::Bin/../t/lib"; diff --git a/t/00-load.t b/t/00-load.t index 8a43acd..0bfb0eb 100644 --- a/t/00-load.t +++ b/t/00-load.t @@ -1,13 +1,16 @@ #!perl +use v5.16; +use warnings; + use FindBin; use Test::More; use lib "$FindBin::Bin/lib"; use test::Data; -diag( "Testing Parse::Syslog::Line $Parse::Syslog::Line::VERSION, Perl $], $^X" ); use_ok( 'Parse::Syslog::Line' ); +diag( "Testing Parse::Syslog::Line $Parse::Syslog::Line::VERSION, Perl $], $^X" ); my $tests = get_test_data(); is( ref $tests, 'HASH', "test::Data::get_tests() loads a hash" ); diff --git a/t/01-parse.t b/t/01-parse.t index b18b60f..c3233cd 100644 --- a/t/01-parse.t +++ b/t/01-parse.t @@ -1,6 +1,6 @@ #!perl -use strict; +use v5.16; use warnings; use FindBin; @@ -19,11 +19,8 @@ use_utc_syslog(); # this avoids HTTP::Date weirdnes with dates "in the future" Test::MockTime::set_fixed_time("2018-12-01T00:00:00Z"); -# Datetime Fields -my @dtfields = qw/date datetime_local datetime_str datetime_utc epoch time tz/; subtest "Basic Functionality Test" => sub { - my @delete = qw(datetime_obj); my $TESTS = get_test_data(); foreach my $file (sort keys %{ $TESTS }) { my $test = $TESTS->{$file}; @@ -36,11 +33,9 @@ subtest "Basic Functionality Test" => sub { ${"Parse::Syslog::Line::$k"} = $test->{options}{$k}; } } - my $msg = parse_syslog_line($test->{string}); - delete $msg->{$_} for grep { exists $msg->{$_} } @delete; - delete $test->{expected}{$_} for grep { exists $test->{expected}{$_} } @delete; + my $msg = normalize_test_result(parse_syslog_line($test->{string})); is_deeply( $msg, $test->{expected}, "$file - $test->{name}" ) - || diag( YAML::XS::Dump($msg) ); + || diag(YAML::XS::Dump($msg)); # Restore Defaults if( keys %restore ) { foreach my $k ( keys %restore ) { @@ -53,38 +48,36 @@ subtest "Basic Functionality Test" => sub { subtest 'Disable Program Extraction' => sub { local $Parse::Syslog::Line::ExtractProgram = 0; - my @delete = (); my $TESTS = get_test_data(); foreach my $file (sort keys %{ $TESTS }) { my $test = $TESTS->{$file}; # Skip tests with specific options next if exists $test->{options}; - my $msg = parse_syslog_line($test->{string}); + my $msg = normalize_test_result(parse_syslog_line($test->{string})); my %expected = %{ $test->{expected} }; - delete $msg->{$_} for @delete; - $expected{$_} = undef for qw(program_name program_sub program_pid); + delete $expected{$_} for qw(program_name program_sub program_pid); if( $msg->{content} && $expected{program_raw} ) { - my $expected_program = $expected{program_raw}; + my $expected_program = delete $expected{program_raw}; my $content = delete $msg->{content}; my $expected_content = delete $expected{content}; like( $content, qr/\Q$expected_program\E(\s-|:)\s\Q$expected_content\E/, "Content correct in $file" ); } - undef($expected{program_raw}); - is_deeply( $msg, \%expected, "$file - $test->{name} (no extract program)" ) || diag( YAML::XS::Dump($msg) ); } }; subtest 'Custom parser' => sub { - sub parse_func { my ($date) = @_; $date //= " "; my $modified = "[$date]"; return $modified; } + # Datetime Fields + my @delete = qw/datetime_local datetime_utc tz/; + my @undef = qw/date datetime_str epoch time/; local $Parse::Syslog::Line::FmtDate = \&parse_func; @@ -94,11 +87,10 @@ subtest 'Custom parser' => sub { # Skip tests with specific options next if exists $test->{options}; my %resp = %{ $test->{expected} }; - foreach my $part (@dtfields) { - $resp{$part} = undef; - } + delete $resp{$_} for @delete; + undef($resp{$_}) for @undef; $resp{date} = "[" . $resp{datetime_raw} . "]"; - my $msg = parse_syslog_line($test->{string}); + my $msg = normalize_test_result(parse_syslog_line($test->{string})); is_deeply( $msg, \%resp, "FmtDate " . $test->{name} ) || diag( YAML::XS::Dump($msg) ); } diff --git a/t/02-functions.t b/t/02-functions.t index 1557e06..cce66b5 100644 --- a/t/02-functions.t +++ b/t/02-functions.t @@ -1,6 +1,6 @@ #!perl -use strict; +use v5.16; use warnings; use Test::More; diff --git a/t/03-datetime-calculations.t b/t/03-datetime-calculations.t index 7502755..4f52f78 100644 --- a/t/03-datetime-calculations.t +++ b/t/03-datetime-calculations.t @@ -1,6 +1,6 @@ #!perl -use strict; +use v5.16; use warnings; use bignum; @@ -13,7 +13,7 @@ use DateTime; use DateTime::TimeZone; use Time::Moment; use Storable qw(dclone); -use YAML (); +use YAML::XS (); use Parse::Syslog::Line qw/:with_timezones/; @@ -122,7 +122,7 @@ subtest 'Millisecond resolution' => sub { $got, superhashof($exp), $case_name, - ) || diag YAML::Dump $got; + ) || diag YAML::XS::Dump $got; } }; diff --git a/t/bin/create_test_entry.pl b/t/bin/create-test-entry.pl similarity index 90% rename from t/bin/create_test_entry.pl rename to t/bin/create-test-entry.pl index 368e8a4..8c47ec7 100755 --- a/t/bin/create_test_entry.pl +++ b/t/bin/create-test-entry.pl @@ -1,6 +1,6 @@ #!perl # -use strict; +use v5.16; use warnings; use CLI::Helpers qw(:all); @@ -9,7 +9,7 @@ use Getopt::Long::Descriptive; use Path::Tiny qw(path); use Test::MockTime; -use YAML (); +use YAML::XS (); # We're in t/bin so ../../lib is the dist lib use lib "$FindBin::Bin/../../lib"; @@ -45,7 +45,7 @@ # Only read YAML Data return unless $p->is_file and $p->stringify =~ /\.yaml/; # Reading is fatal if it fails, that's cool - my $contents = YAML::LoadFile( $p->stringify ); + my $contents = YAML::XS::LoadFile( $p->stringify ); # Generate the Test Case in a child to isolate test options if ( my $pid = fork() ) { while( wait() != -1 ) {} @@ -84,16 +84,16 @@ sub generate_test_data { # Generate a Test ID my $id_str = $entry->{string}; - $id_str .= YAML::Dump( $entry->{options} ) if $entry->{options}; + $id_str .= YAML::XS::Dump( $entry->{options} ) if $entry->{options}; my $id = $args{id} || md5_hex($id_str); output({clear => 1, color=>'cyan'}, $entry->{string}); - output({indent => 1}, split /\r?\n/, YAML::Dump($entry->{expected})); + output({indent => 1}, split /\r?\n/, YAML::XS::Dump($entry->{expected})); return unless $opt->noconfirm or confirm("Does this look correct?"); $entry->{name} ||= prompt("What name would you give this test? ", default => $id); my $file = $dataDir->child("${id}.yaml"); - YAML::DumpFile( $file->absolute->stringify, $entry ); + YAML::XS::DumpFile( $file->absolute->stringify, $entry ); output({color=>'green'}, sprintf "Created %s for test: %s", $file->stringify, $entry->{name}, diff --git a/t/bin/dump-test-messages.pl b/t/bin/dump-test-messages.pl new file mode 100644 index 0000000..6e8a198 --- /dev/null +++ b/t/bin/dump-test-messages.pl @@ -0,0 +1,23 @@ +#!perl +# +use v5.16; +use warnings; + +use FindBin; +use YAML::XS (); +use lib "$FindBin::Bin/../lib"; +use test::Data; + +my $match = shift; +my $tests = get_test_data("$FindBin::Bin/.."); +foreach my $file ( sort keys %{ $tests } ) { + my $t = $tests->{$file}; + my $msg = $t->{string}; + if ( $match ) { + say $msg if index(lc($t->{name}), lc($match)) >= 0; + } + else { + say $msg; + } +} + diff --git a/t/data/064eaadcacdcfe59b91cf280f1a25bc9.yaml b/t/data/064eaadcacdcfe59b91cf280f1a25bc9.yaml index 1db99ae..87478c4 100644 --- a/t/data/064eaadcacdcfe59b91cf280f1a25bc9.yaml +++ b/t/data/064eaadcacdcfe59b91cf280f1a25bc9.yaml @@ -18,9 +18,7 @@ expected: priority: info priority_int: 6 program_name: scsitarget.ispfct.targetReset - program_pid: ~ program_raw: '[filer-201: scsitarget.ispfct.targetReset:notice]' - program_sub: ~ time: 11:28:13 tz: Z name: NetApp Filer Logs diff --git a/t/data/0cf67252f51bc98c6302cf2d11832db2.yaml b/t/data/0cf67252f51bc98c6302cf2d11832db2.yaml index cc0b4a4..2942e8b 100644 --- a/t/data/0cf67252f51bc98c6302cf2d11832db2.yaml +++ b/t/data/0cf67252f51bc98c6302cf2d11832db2.yaml @@ -19,9 +19,7 @@ expected: priority: err priority_int: 3 program_name: '%LINK-3-UPDOWN' - program_pid: ~ program_raw: '%LINK-3-UPDOWN' - program_sub: ~ time: 14:58:58 tz: Z name: Cisco NTP No Sync diff --git a/t/data/0e03a69469bdfe47db4dff53681ae434.yaml b/t/data/0e03a69469bdfe47db4dff53681ae434.yaml index 197ef78..a669140 100644 --- a/t/data/0e03a69469bdfe47db4dff53681ae434.yaml +++ b/t/data/0e03a69469bdfe47db4dff53681ae434.yaml @@ -8,20 +8,13 @@ expected: datetime_utc: 2018-01-01T09:09:36Z domain: company.tld epoch: 1514797776 - facility: ~ - facility_int: ~ host: hostname host_raw: hostname.company.tld message: '%ETHPORT-5-IF_DOWN_CFG_CHANGE: Interface Ethernet121/1/1 is down(Config change)' message_raw: '2018-01-01T11:09:36+02:00 hostname.company.tld : 2018 Jan 1 11:09:36.290 CET: %ETHPORT-5-IF_DOWN_CFG_CHANGE: Interface Ethernet121/1/1 is down(Config change)' ntp: ok - preamble: ~ - priority: ~ - priority_int: ~ program_name: '%ETHPORT-5-IF_DOWN_CFG_CHANGE' - program_pid: ~ program_raw: '%ETHPORT-5-IF_DOWN_CFG_CHANGE' - program_sub: ~ time: 11:09:36 tz: +0200 name: Cisco NX-OS diff --git a/t/data/21a3db207b78320769f2cb316dd03f60.yaml b/t/data/21a3db207b78320769f2cb316dd03f60.yaml index cdafe6f..e303495 100644 --- a/t/data/21a3db207b78320769f2cb316dd03f60.yaml +++ b/t/data/21a3db207b78320769f2cb316dd03f60.yaml @@ -6,7 +6,6 @@ expected: datetime_raw: Jan 1 00:10:02 datetime_str: 2018-01-01T00:10:02Z datetime_utc: 2018-01-01T00:10:02Z - domain: ~ epoch: 1514765402 facility: local7 facility_int: 184 @@ -19,9 +18,7 @@ expected: priority: warn priority_int: 4 program_name: '%C4K_EBM-4-HOSTFLAPPING' - program_pid: ~ program_raw: '%C4K_EBM-4-HOSTFLAPPING' - program_sub: ~ time: 00:10:02 tz: Z name: Cisco Catalyst diff --git a/t/data/3a7ef75494efe41176b9d57a06517a54.yaml b/t/data/3a7ef75494efe41176b9d57a06517a54.yaml index da49da5..cea2ac8 100644 --- a/t/data/3a7ef75494efe41176b9d57a06517a54.yaml +++ b/t/data/3a7ef75494efe41176b9d57a06517a54.yaml @@ -10,19 +10,12 @@ expected: datetime_utc: 2015-09-30T11:26:06.779373Z domain: ~ epoch: 1443612366.779373 - facility: ~ - facility_int: ~ host: my-host host_raw: my-host message: 'my-script.pl: {"lunchTime":1443612366.442}' message_raw: '2015-09-30T06:26:06.779373-05:00 my-host my-script.pl: {"lunchTime":1443612366.442}' - preamble: ~ - priority: ~ - priority_int: ~ program_name: my-script.pl - program_pid: ~ program_raw: my-script.pl - program_sub: ~ time: 06:26:06.779373 tz: -0500 name: AutoDetectJSON diff --git a/t/data/3ca7dfaa06c1fd7138e9d7fd75ca49e7.yaml b/t/data/3ca7dfaa06c1fd7138e9d7fd75ca49e7.yaml index 0a23650..de0ee0d 100644 --- a/t/data/3ca7dfaa06c1fd7138e9d7fd75ca49e7.yaml +++ b/t/data/3ca7dfaa06c1fd7138e9d7fd75ca49e7.yaml @@ -6,21 +6,13 @@ expected: datetime_raw: Jan 1 00:00:00 datetime_str: 2018-01-01T00:00:00Z datetime_utc: 2018-01-01T00:00:00Z - domain: ~ epoch: 1514764800 - facility: ~ - facility_int: ~ host: 11.22.33.44 host_raw: 11.22.33.44 message: 'dhcpd: DHCPINFORM from 172.16.2.137 via vlan3' message_raw: 'Jan 1 00:00:00 11.22.33.44 dhcpd: DHCPINFORM from 172.16.2.137 via vlan3' - preamble: ~ - priority: ~ - priority_int: ~ program_name: dhcpd - program_pid: ~ program_raw: dhcpd - program_sub: ~ time: 00:00:00 tz: Z name: 'IP without Hostname, no preamble' diff --git a/t/data/3e119d58d6ea3cf34beabad4f28999f8.yaml b/t/data/3e119d58d6ea3cf34beabad4f28999f8.yaml index ad7c5f4..299fcb2 100644 --- a/t/data/3e119d58d6ea3cf34beabad4f28999f8.yaml +++ b/t/data/3e119d58d6ea3cf34beabad4f28999f8.yaml @@ -24,15 +24,10 @@ expected: datetime_utc: 2018-09-15T00:00:05Z domain: ~ epoch: 1536969605 - facility: ~ - facility_int: ~ host: alma9 host_raw: alma9 message: 'podman[1286826]: 2024-09-15 00:00:05.516700379 -0700 PDT m=+0.050887542 container health_status c12740d991d21bbea2989adffbbecf471eb28bd2d416587eff9c4b94e43fdb92 (image=docker.io/plexinc/pms-docker:latest, name=plex, health_status=healthy, org.opencontainers.image.created=2024-09-11T16:25:53.185Z, org.opencontainers.image.ref.name=ubuntu, org.opencontainers.image.revision=c8e17655441744e88dd3b45f96f824052fda79a3, org.opencontainers.image.title=Plex Media Server, org.opencontainers.image.vendor=Plex, GmbH, org.opencontainers.image.licenses=NOASSERTION, org.opencontainers.image.url=https://github.com/plexinc/plex-media-server, org.opencontainers.image.version=1.41.0.8992-8463ad060, org.opencontainers.image.description="The Plex Media Server", puppet_resource_flags=eyJuZXR3b3JrIj0+Imhvc3QiLCAidm9sdW1lIj0+WyJwbGV4bWVkaWE6L21lZGlhIiwgInBsZXh0cmFuc2NvZGU6L3RyYW5zY29kZSIsICIvaG9tZS9jb250YWluZXJzL3BsZXg6L2NvbmZpZyIsICIvbWVkaWEvbW92aWVzOi9tZWRpYS9tb3ZpZXMiLCAiL21lZGlhL211c2ljOi9tZWRpYS9tdXNpYyIsICIvbWVkaWEvdHY6L21lZGlhL3R2Il0sICJkZXZpY2UiPT5bIi9kZXYvZHJpOi9kZXYvZHJpIl19, org.opencontainers.image.source=https://github.com/plexinc/pms-docker)' message_raw: 'Sep 15 00:00:05 alma9 podman[1286826]: 2024-09-15 00:00:05.516700379 -0700 PDT m=+0.050887542 container health_status c12740d991d21bbea2989adffbbecf471eb28bd2d416587eff9c4b94e43fdb92 (image=docker.io/plexinc/pms-docker:latest, name=plex, health_status=healthy, org.opencontainers.image.created=2024-09-11T16:25:53.185Z, org.opencontainers.image.ref.name=ubuntu, org.opencontainers.image.revision=c8e17655441744e88dd3b45f96f824052fda79a3, org.opencontainers.image.title=Plex Media Server, org.opencontainers.image.vendor=Plex, GmbH, org.opencontainers.image.licenses=NOASSERTION, org.opencontainers.image.url=https://github.com/plexinc/plex-media-server, org.opencontainers.image.version=1.41.0.8992-8463ad060, org.opencontainers.image.description="The Plex Media Server", puppet_resource_flags=eyJuZXR3b3JrIj0+Imhvc3QiLCAidm9sdW1lIj0+WyJwbGV4bWVkaWE6L21lZGlhIiwgInBsZXh0cmFuc2NvZGU6L3RyYW5zY29kZSIsICIvaG9tZS9jb250YWluZXJzL3BsZXg6L2NvbmZpZyIsICIvbWVkaWEvbW92aWVzOi9tZWRpYS9tb3ZpZXMiLCAiL21lZGlhL211c2ljOi9tZWRpYS9tdXNpYyIsICIvbWVkaWEvdHY6L21lZGlhL3R2Il0sICJkZXZpY2UiPT5bIi9kZXYvZHJpOi9kZXYvZHJpIl19, org.opencontainers.image.source=https://github.com/plexinc/pms-docker)' - preamble: ~ - priority: ~ - priority_int: ~ program_name: podman program_pid: 1286826 program_raw: 'podman[1286826]' diff --git a/t/data/4209c3f669b2a1ae81a0db4e8f4c5dd5.yaml b/t/data/4209c3f669b2a1ae81a0db4e8f4c5dd5.yaml index b4f702c..55b00ef 100644 --- a/t/data/4209c3f669b2a1ae81a0db4e8f4c5dd5.yaml +++ b/t/data/4209c3f669b2a1ae81a0db4e8f4c5dd5.yaml @@ -6,12 +6,9 @@ expected: datetime_raw: Jan 1 08:15:00 datetime_str: 2018-01-01T08:15:00Z datetime_utc: 2018-01-01T08:15:00Z - domain: ~ epoch: 1514794500 facility: cron facility_int: 72 - host: ~ - host_raw: ~ message: '/usr/sbin/cron[73991]: (root) CMD (/usr/libexec/atrun)' message_raw: '<78>Jan 1 08:15:00 /usr/sbin/cron[73991]: (root) CMD (/usr/libexec/atrun)' preamble: 78 diff --git a/t/data/4b437e53956899136043f93759f79c16.yaml b/t/data/4b437e53956899136043f93759f79c16.yaml new file mode 100644 index 0000000..bb40235 --- /dev/null +++ b/t/data/4b437e53956899136043f93759f79c16.yaml @@ -0,0 +1,37 @@ +--- +expected: + SDATA: + bar: + x: 2 + y: 2 + foo: + x: 1 + lvl: 2 + x: 'new key, present' + content: 'some content {"foo": {"x":"loses to sdata"},"x":"new key, present"}' + date: 2018-07-26 + datetime_local: 2018-07-26T15:50:14Z + datetime_raw: Jul 26 15:50:14 + datetime_str: 2018-07-26T15:50:14Z + datetime_utc: 2018-07-26T15:50:14Z + domain: ~ + epoch: 1532620214 + facility: local4 + facility_int: 160 + host: ether + host_raw: ether + message: 'janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"foo":{"x":"loses to sdata"},"x":"new key, present"}' + message_raw: '<163>Jul 26 15:50:14 ether janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"foo":{"x":"loses to sdata"},"x":"new key, present"}' + preamble: 163 + priority: err + priority_int: 3 + program_name: janus + program_pid: 4291 + program_raw: 'janus foobar[4291]' + program_sub: foobar + time: 15:50:14 + tz: Z +name: 'SDATA Precedence, SDATA beats JSON' +options: + AutoDetectJSON: 1 +string: '<163>Jul 26 15:50:14 ether janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"foo":{"x":"loses to sdata"},"x":"new key, present"}' diff --git a/t/data/4bab0b1bd6e18f35fcee6fecf1522030.yaml b/t/data/4bab0b1bd6e18f35fcee6fecf1522030.yaml index 173f2da..09a688d 100644 --- a/t/data/4bab0b1bd6e18f35fcee6fecf1522030.yaml +++ b/t/data/4bab0b1bd6e18f35fcee6fecf1522030.yaml @@ -18,9 +18,7 @@ expected: priority: info priority_int: 6 program_name: logger - program_pid: ~ program_raw: info logger - program_sub: ~ time: 10:55:37 tz: Z name: F5 includes level diff --git a/t/data/4cfcff8c1f4bf03d14995286bf9d438f.yaml b/t/data/4cfcff8c1f4bf03d14995286bf9d438f.yaml index d916777..1d57ff3 100644 --- a/t/data/4cfcff8c1f4bf03d14995286bf9d438f.yaml +++ b/t/data/4cfcff8c1f4bf03d14995286bf9d438f.yaml @@ -24,19 +24,12 @@ expected: datetime_utc: 2018-05-20T06:40:46Z domain: ~ epoch: 1526798446 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'kernel: iptables - ACTION=outbound IN= OUT=eth0 SRC=148.251.14.68 DST=71.198.187.165 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=57140 DF PROTO=TCP SPT=47070 DPT=55233 WINDOW=14600 RES=0x00 SYN URGP=0' message_raw: 'May 20 06:40:46 janus kernel: iptables - ACTION=outbound IN= OUT=eth0 SRC=148.251.14.68 DST=71.198.187.165 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=57140 DF PROTO=TCP SPT=47070 DPT=55233 WINDOW=14600 RES=0x00 SYN URGP=0' - preamble: ~ - priority: ~ - priority_int: ~ program_name: kernel - program_pid: ~ program_raw: kernel - program_sub: ~ time: 06:40:46 tz: Z name: iptables entry with key/values diff --git a/t/data/5e8983b3b4b6d1c566af1448590c8586.yaml b/t/data/5e8983b3b4b6d1c566af1448590c8586.yaml new file mode 100644 index 0000000..97f9b90 --- /dev/null +++ b/t/data/5e8983b3b4b6d1c566af1448590c8586.yaml @@ -0,0 +1,37 @@ +--- +expected: + SDATA: + bar: + x: 2 + y: 2 + foo: + x: 1 + lvl: 2 + x: extra json + content: 'some content {"x": "extra json"}' + date: 2018-07-26 + datetime_local: 2018-07-26T15:50:14Z + datetime_raw: Jul 26 15:50:14 + datetime_str: 2018-07-26T15:50:14Z + datetime_utc: 2018-07-26T15:50:14Z + domain: ~ + epoch: 1532620214 + facility: local4 + facility_int: 160 + host: ether + host_raw: ether + message: 'janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"x":"extra json"}' + message_raw: '<163>Jul 26 15:50:14 ether janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"x":"extra json"}' + preamble: 163 + priority: err + priority_int: 3 + program_name: janus + program_pid: 4291 + program_raw: 'janus foobar[4291]' + program_sub: foobar + time: 15:50:14 + tz: Z +name: Cumulative SDATA +options: + AutoDetectJSON: 1 +string: '<163>Jul 26 15:50:14 ether janus foobar[4291]: [lvl=2] [foo x=1] some content [bar x=2 y=2] {"x":"extra json"}' diff --git a/t/data/67b8ab574fc3a9a9fd6a0bdaf1231b14.yaml b/t/data/67b8ab574fc3a9a9fd6a0bdaf1231b14.yaml index 9cb15c6..d878c0a 100644 --- a/t/data/67b8ab574fc3a9a9fd6a0bdaf1231b14.yaml +++ b/t/data/67b8ab574fc3a9a9fd6a0bdaf1231b14.yaml @@ -8,19 +8,12 @@ expected: datetime_utc: 2015-11-13T13:40:01Z domain: ~ epoch: 1447422001 - facility: ~ - facility_int: ~ host: ether host_raw: ether message: 'rsyslogd-2177: imuxsock begins to drop messages from pid 17840 due to rate-limiting' message_raw: '2015 Nov 13 13:40:01 ether rsyslogd-2177: imuxsock begins to drop messages from pid 17840 due to rate-limiting' - preamble: ~ - priority: ~ - priority_int: ~ program_name: rsyslogd-2177 - program_pid: ~ program_raw: rsyslogd-2177 - program_sub: ~ time: 13:40:01 tz: Z name: Year with old date diff --git a/t/data/6f833459bced8cdc42950602d7798680.yaml b/t/data/6f833459bced8cdc42950602d7798680.yaml index 9eeefe9..9d2c5a0 100644 --- a/t/data/6f833459bced8cdc42950602d7798680.yaml +++ b/t/data/6f833459bced8cdc42950602d7798680.yaml @@ -13,19 +13,12 @@ expected: datetime_utc: 2018-05-20T06:40:53Z domain: ~ epoch: 1526798453 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'sudo: brad : TTY=pts/3 ; PWD=/home/brad/code/CPAN/eris-logging ; USER=root ; COMMAND=/bin/grep pam /var/log/messages' message_raw: 'May 20 06:40:53 janus sudo: brad : TTY=pts/3 ; PWD=/home/brad/code/CPAN/eris-logging ; USER=root ; COMMAND=/bin/grep pam /var/log/messages' - preamble: ~ - priority: ~ - priority_int: ~ program_name: sudo - program_pid: ~ program_raw: sudo - program_sub: ~ time: 06:40:53 tz: Z name: crond message with key/value detection enabled diff --git a/t/data/7af843bd9c3dad1a054d79ac3f3589c3.yaml b/t/data/7af843bd9c3dad1a054d79ac3f3589c3.yaml index 37b6b66..7a6eeb3 100644 --- a/t/data/7af843bd9c3dad1a054d79ac3f3589c3.yaml +++ b/t/data/7af843bd9c3dad1a054d79ac3f3589c3.yaml @@ -8,15 +8,10 @@ expected: datetime_utc: 2018-05-19T22:03:29Z domain: ~ epoch: 1526767409 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'sshd[10173]: pam_unix(sshd:session): session opened for user brad by (uid=0)' message_raw: 'May 19 22:03:29 janus sshd[10173]: pam_unix(sshd:session): session opened for user brad by (uid=0)' - preamble: ~ - priority: ~ - priority_int: ~ program_name: sshd program_pid: 10173 program_raw: 'sshd[10173]' diff --git a/t/data/83fa7359bc8057bac3f2c4acc452740d.yaml b/t/data/83fa7359bc8057bac3f2c4acc452740d.yaml index 962a5eb..24e06e3 100644 --- a/t/data/83fa7359bc8057bac3f2c4acc452740d.yaml +++ b/t/data/83fa7359bc8057bac3f2c4acc452740d.yaml @@ -8,15 +8,10 @@ expected: datetime_utc: 2018-01-01T00:00:00Z domain: ~ epoch: 1514764800 - facility: ~ - facility_int: ~ host: example host_raw: example message: 'syslogd 1.2.3: restart (remote reception).' message_raw: 'Jan 1 00:00:00 example syslogd 1.2.3: restart (remote reception).' - preamble: ~ - priority: ~ - priority_int: ~ program_name: syslogd program_pid: ~ program_raw: syslogd 1.2.3 diff --git a/t/data/862f7d0ef6acea8dd7fdedbdbae41592.yaml b/t/data/862f7d0ef6acea8dd7fdedbdbae41592.yaml index 3cf6f30..44d1fc4 100644 --- a/t/data/862f7d0ef6acea8dd7fdedbdbae41592.yaml +++ b/t/data/862f7d0ef6acea8dd7fdedbdbae41592.yaml @@ -69,9 +69,7 @@ expected: priority: info priority_int: 6 program_name: RT_FLOW - program_pid: ~ program_raw: RT_FLOW - program_sub: ~ time: 03:38:26 tz: Z version: 1 diff --git a/t/data/86a4626fee7782296c09fa39a08ff2d1.yaml b/t/data/86a4626fee7782296c09fa39a08ff2d1.yaml index c6eb6ce..8dc007c 100644 --- a/t/data/86a4626fee7782296c09fa39a08ff2d1.yaml +++ b/t/data/86a4626fee7782296c09fa39a08ff2d1.yaml @@ -8,15 +8,10 @@ expected: datetime_utc: 2016-11-19T19:50:01.749659Z domain: ~ epoch: 1479585001.749659 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'CROND[14400]: (root) CMD (/usr/lib64/sa/sa1 1 1)' message_raw: '2016-11-19T20:50:01.749659+01:00 janus CROND[14400]: (root) CMD (/usr/lib64/sa/sa1 1 1)' - preamble: ~ - priority: ~ - priority_int: ~ program_name: CROND program_pid: 14400 program_raw: 'CROND[14400]' diff --git a/t/data/8e44959d30eba57a6feb950e286c977b.yaml b/t/data/8e44959d30eba57a6feb950e286c977b.yaml index d1a5399..dd52f7d 100644 --- a/t/data/8e44959d30eba57a6feb950e286c977b.yaml +++ b/t/data/8e44959d30eba57a6feb950e286c977b.yaml @@ -18,9 +18,7 @@ expected: priority: err priority_int: 3 program_name: '%ASA-3-313001' - program_pid: ~ program_raw: '%ASA-3-313001' - program_sub: ~ time: 18:39:00 tz: Z name: Cisco ASA diff --git a/t/data/8f4e966ac5cadf8171739d472b814c61.yaml b/t/data/8f4e966ac5cadf8171739d472b814c61.yaml index d916777..1d57ff3 100644 --- a/t/data/8f4e966ac5cadf8171739d472b814c61.yaml +++ b/t/data/8f4e966ac5cadf8171739d472b814c61.yaml @@ -24,19 +24,12 @@ expected: datetime_utc: 2018-05-20T06:40:46Z domain: ~ epoch: 1526798446 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'kernel: iptables - ACTION=outbound IN= OUT=eth0 SRC=148.251.14.68 DST=71.198.187.165 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=57140 DF PROTO=TCP SPT=47070 DPT=55233 WINDOW=14600 RES=0x00 SYN URGP=0' message_raw: 'May 20 06:40:46 janus kernel: iptables - ACTION=outbound IN= OUT=eth0 SRC=148.251.14.68 DST=71.198.187.165 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=57140 DF PROTO=TCP SPT=47070 DPT=55233 WINDOW=14600 RES=0x00 SYN URGP=0' - preamble: ~ - priority: ~ - priority_int: ~ program_name: kernel - program_pid: ~ program_raw: kernel - program_sub: ~ time: 06:40:46 tz: Z name: iptables entry with key/values diff --git a/t/data/90b344da741283aed6558f915732b421.yaml b/t/data/90b344da741283aed6558f915732b421.yaml index a1e0983..9612611 100644 --- a/t/data/90b344da741283aed6558f915732b421.yaml +++ b/t/data/90b344da741283aed6558f915732b421.yaml @@ -18,9 +18,7 @@ expected: priority: info priority_int: 6 program_name: scsitarget.ispfct.targetReset - program_pid: ~ program_raw: '[filer-201 scsitarget.ispfct.targetReset:notice]' - program_sub: ~ time: 11:28:13 tz: Z name: NetApp Filer Alt1 diff --git a/t/data/90ff8b49f7fe0526b986d116f4e0c43f.yaml b/t/data/90ff8b49f7fe0526b986d116f4e0c43f.yaml index cab13ea..8c67f56 100644 --- a/t/data/90ff8b49f7fe0526b986d116f4e0c43f.yaml +++ b/t/data/90ff8b49f7fe0526b986d116f4e0c43f.yaml @@ -18,9 +18,7 @@ expected: priority: alert priority_int: 1 program_name: '%ASA-3-313001' - program_pid: ~ program_raw: '%ASA-3-313001' - program_sub: ~ time: 18:39:00 tz: Z name: Cisco ASA Alt diff --git a/t/data/a3f74b54b664d2f0903f2cb8a110b4cd.yaml b/t/data/a3f74b54b664d2f0903f2cb8a110b4cd.yaml index ba69231..b50705d 100644 --- a/t/data/a3f74b54b664d2f0903f2cb8a110b4cd.yaml +++ b/t/data/a3f74b54b664d2f0903f2cb8a110b4cd.yaml @@ -8,15 +8,10 @@ expected: datetime_utc: 2018-05-19T22:05:47Z domain: ~ epoch: 1526767547 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'postfix/smtps/smtpd[11513]: disconnect from unknown[139.28.175.150]' message_raw: 'May 19 22:05:47 janus postfix/smtps/smtpd[11513]: disconnect from unknown[139.28.175.150]' - preamble: ~ - priority: ~ - priority_int: ~ program_name: postfix program_pid: 11513 program_raw: 'postfix/smtps/smtpd[11513]' diff --git a/t/data/adb23949c66b74145bc0f8084e7307a1.yaml b/t/data/adb23949c66b74145bc0f8084e7307a1.yaml index ab48ea1..c2d15be 100644 --- a/t/data/adb23949c66b74145bc0f8084e7307a1.yaml +++ b/t/data/adb23949c66b74145bc0f8084e7307a1.yaml @@ -18,9 +18,7 @@ expected: priority: info priority_int: 6 program_name: scsitarget.ispfct.targetReset - program_pid: ~ program_raw: '[filer-201:scsitarget.ispfct.targetReset:notice]' - program_sub: ~ time: 11:28:13 tz: Z name: NetApp Filer Alt2 diff --git a/t/data/bb8856f6a2f36d1ff8f747b8f280b3a6.yaml b/t/data/bb8856f6a2f36d1ff8f747b8f280b3a6.yaml index 4dea8be..afdaba7 100644 --- a/t/data/bb8856f6a2f36d1ff8f747b8f280b3a6.yaml +++ b/t/data/bb8856f6a2f36d1ff8f747b8f280b3a6.yaml @@ -19,9 +19,7 @@ expected: priority: notice priority_int: 5 program_name: '%CRYPTO-5-IPSEC_SETUP_FAILURE' - program_pid: ~ program_raw: '%CRYPTO-5-IPSEC_SETUP_FAILURE' - program_sub: ~ time: 19:12:19 tz: Z name: Cisco Date Insanity diff --git a/t/data/c06ac2c4540f4e87bd5bb3c70ac39cb0.yaml b/t/data/c06ac2c4540f4e87bd5bb3c70ac39cb0.yaml index 93d5d11..e3f81fc 100644 --- a/t/data/c06ac2c4540f4e87bd5bb3c70ac39cb0.yaml +++ b/t/data/c06ac2c4540f4e87bd5bb3c70ac39cb0.yaml @@ -6,7 +6,6 @@ expected: datetime_raw: Jan 1 12:22:26 datetime_str: 2018-01-01T12:22:26Z datetime_utc: 2018-01-01T12:22:26Z - domain: ~ epoch: 1514809346 facility: local7 facility_int: 184 @@ -19,9 +18,7 @@ expected: priority: notice priority_int: 5 program_name: '%SYS-5-CONFIG_I' - program_pid: ~ program_raw: '%SYS-5-CONFIG_I' - program_sub: ~ time: 12:22:26 tz: Z name: Cisco NTP Unconfigured diff --git a/t/data/c1352d43776bbf2acfb45344101cd74b.yaml b/t/data/c1352d43776bbf2acfb45344101cd74b.yaml index 9b106f5..cfbaac9 100644 --- a/t/data/c1352d43776bbf2acfb45344101cd74b.yaml +++ b/t/data/c1352d43776bbf2acfb45344101cd74b.yaml @@ -8,15 +8,10 @@ expected: datetime_utc: 2021-02-18T17:07:26Z domain: example.com epoch: 1613668046 - facility: ~ - facility_int: ~ host: webproxy101 host_raw: webproxy101.example.com message: '(squid-1)[58016]: 10.1.2.3 - - [18/Feb/2021:17:07:26 +0000] "CONNECT hcaptcha.com:443 HTTP/1.1" 200 3879 "-" "-" TCP_TUNNEL:HIER_DIRECT' message_raw: '2021 Feb 18 17:07:26 webproxy101.example.com (squid-1)[58016]: 10.1.2.3 - - [18/Feb/2021:17:07:26 +0000] "CONNECT hcaptcha.com:443 HTTP/1.1" 200 3879 "-" "-" TCP_TUNNEL:HIER_DIRECT' - preamble: ~ - priority: ~ - priority_int: ~ program_name: (squid-1) program_pid: 58016 program_raw: '(squid-1)[58016]' diff --git a/t/data/cb6569d15b64b8bcc085cf770bc29c4c.yaml b/t/data/cb6569d15b64b8bcc085cf770bc29c4c.yaml index 9eeefe9..9d2c5a0 100644 --- a/t/data/cb6569d15b64b8bcc085cf770bc29c4c.yaml +++ b/t/data/cb6569d15b64b8bcc085cf770bc29c4c.yaml @@ -13,19 +13,12 @@ expected: datetime_utc: 2018-05-20T06:40:53Z domain: ~ epoch: 1526798453 - facility: ~ - facility_int: ~ host: janus host_raw: janus message: 'sudo: brad : TTY=pts/3 ; PWD=/home/brad/code/CPAN/eris-logging ; USER=root ; COMMAND=/bin/grep pam /var/log/messages' message_raw: 'May 20 06:40:53 janus sudo: brad : TTY=pts/3 ; PWD=/home/brad/code/CPAN/eris-logging ; USER=root ; COMMAND=/bin/grep pam /var/log/messages' - preamble: ~ - priority: ~ - priority_int: ~ program_name: sudo - program_pid: ~ program_raw: sudo - program_sub: ~ time: 06:40:53 tz: Z name: crond message with key/value detection enabled diff --git a/t/data/ce263146c04caf101b8593bf39a89376.yaml b/t/data/ce263146c04caf101b8593bf39a89376.yaml index 55462e9..8217b82 100644 --- a/t/data/ce263146c04caf101b8593bf39a89376.yaml +++ b/t/data/ce263146c04caf101b8593bf39a89376.yaml @@ -8,7 +8,7 @@ expected: error_type: conversion_failed internal_log_rate_limit: 'true' stage: processing - _json_error: "Failed to decode json: '\"' expected, at character offset 1 (before \"component_kind=\"tran...\") at /home/brad/code/CPAN/Parse-Syslog-Line/lib/Parse/Syslog/Line.pm line 957.\n" + _json_error: "Failed to decode json: '\"' expected, at character offset 1 (before \"component_kind=\"tran...\") at /Volumes/Code/CPAN/Parse-Syslog-Line/t/bin/../../lib/Parse/Syslog/Line.pm line 966.\n" content: '2024-11-06T00:11:42.563896Z ERROR transform{component_kind="transform" component_id=caddy_parser component_type=remap}: vector::internal_events::remap: Mapping failed with event. error="function call error for \"parse_json\" at (4:25): unable to parse json: expected value at line 1 column 1" error_type="conversion_failed" stage="processing" internal_log_rate_limit=true' date: 2018-11-05 datetime_local: 2018-11-05T16:11:42Z @@ -17,15 +17,10 @@ expected: datetime_utc: 2018-11-05T16:11:42Z domain: ~ epoch: 1541434302 - facility: ~ - facility_int: ~ host: alma9 host_raw: alma9 message: 'vector[3086855]: 2024-11-06T00:11:42.563896Z ERROR transform{component_kind="transform" component_id=caddy_parser component_type=remap}: vector::internal_events::remap: Mapping failed with event. error="function call error for \"parse_json\" at (4:25): unable to parse json: expected value at line 1 column 1" error_type="conversion_failed" stage="processing" internal_log_rate_limit=true' message_raw: 'Nov 5 16:11:42 alma9 vector[3086855]: 2024-11-06T00:11:42.563896Z ERROR transform{component_kind="transform" component_id=caddy_parser component_type=remap}: vector::internal_events::remap: Mapping failed with event. error="function call error for \"parse_json\" at (4:25): unable to parse json: expected value at line 1 column 1" error_type="conversion_failed" stage="processing" internal_log_rate_limit=true' - preamble: ~ - priority: ~ - priority_int: ~ program_name: vector program_pid: 3086855 program_raw: 'vector[3086855]' diff --git a/t/data/e86919071a958d4f24bc13751c2cab47.yaml b/t/data/e86919071a958d4f24bc13751c2cab47.yaml index ee27219..56c4023 100644 --- a/t/data/e86919071a958d4f24bc13751c2cab47.yaml +++ b/t/data/e86919071a958d4f24bc13751c2cab47.yaml @@ -18,9 +18,7 @@ expected: priority: err priority_int: 3 program_name: dhcpd - program_pid: ~ program_raw: dhcpd - program_sub: ~ time: 00:00:00 tz: Z name: Dotted hostname diff --git a/t/data/f94cae02dd7ef975df7bfd365109690c.yaml b/t/data/f94cae02dd7ef975df7bfd365109690c.yaml index ef29c5d..4f90edf 100644 --- a/t/data/f94cae02dd7ef975df7bfd365109690c.yaml +++ b/t/data/f94cae02dd7ef975df7bfd365109690c.yaml @@ -6,7 +6,6 @@ expected: datetime_raw: Jan 1 00:00:00 datetime_str: 2018-01-01T00:00:00Z datetime_utc: 2018-01-01T00:00:00Z - domain: ~ epoch: 1514764800 facility: user facility_int: 8 @@ -18,9 +17,7 @@ expected: priority: err priority_int: 3 program_name: dhcpd - program_pid: ~ program_raw: dhcpd - program_sub: ~ time: 00:00:00 tz: Z name: IP as Hostname diff --git a/t/data/f9f366cc1bd18536713766bb68d51c77.yaml b/t/data/f9f366cc1bd18536713766bb68d51c77.yaml index 3cf6f30..44d1fc4 100644 --- a/t/data/f9f366cc1bd18536713766bb68d51c77.yaml +++ b/t/data/f9f366cc1bd18536713766bb68d51c77.yaml @@ -69,9 +69,7 @@ expected: priority: info priority_int: 6 program_name: RT_FLOW - program_pid: ~ program_raw: RT_FLOW - program_sub: ~ time: 03:38:26 tz: Z version: 1 diff --git a/t/lib/test/Data.pm b/t/lib/test/Data.pm index 3693b9f..3aa812f 100644 --- a/t/lib/test/Data.pm +++ b/t/lib/test/Data.pm @@ -1,6 +1,6 @@ package test::Data; -use strict; +use v5.16; use warnings; use Exporter; @@ -11,17 +11,19 @@ use Storable qw(dclone); use YAML::XS (); our @ISA = qw(Exporter); -our @EXPORT = qw(get_test_data); -our @EXPORT_OK = qw(get_test_data); +our @EXPORT = qw(get_test_data normalize_test_result); +our @EXPORT_OK = qw(get_test_data normalize_test_result); our %EXPORT_TAGS = (); +my @DELETE_KEYS = qw( datetime_obj ); my %TESTS; sub _load_tests { + my ($path) = @_; return if %TESTS; my $has_json = eval { load 'JSON::MaybeXS'; 1; }; - my $bin = path("$FindBin::Bin"); + my $bin = path($path || "$FindBin::Bin"); my $parent = $bin->parent(); my $dir = $parent->child(t => 'data'); @@ -49,6 +51,7 @@ sub _load_tests { local $_ = $test->{$check}; die "Validating $check failed!" unless $checks{$check}->(); } + $test->{expected} = normalize_test_result($test->{expected}); $TESTS{$name} = $test; 1; } or do { @@ -62,8 +65,20 @@ sub _load_tests { } sub get_test_data { - _load_tests(); + _load_tests(@_); return dclone(\%TESTS); } +sub normalize_test_result { + my ($doc) = @_; + + # Remove some items + delete $doc->{$_} for @DELETE_KEYS; + + # Trim the JSON Error + $doc->{_json_error} =~ s{\s+at\s+\S+\s+line\s+\d+\.$}{} if $doc->{_json_error}; + + return $doc; +} + 1;