diff --git a/MANIFEST b/MANIFEST index 0bc6605d13..40dc255483 100644 --- a/MANIFEST +++ b/MANIFEST @@ -127,6 +127,7 @@ lib/Thruk/Utils/CLI/Command.pm lib/Thruk/Utils/CLI/Contact.pm lib/Thruk/Utils/CLI/Cron.pm lib/Thruk/Utils/CLI/Downtimetask.pm +lib/Thruk/Utils/CLI/Filesystem.pm lib/Thruk/Utils/CLI/Find.pm lib/Thruk/Utils/CLI/Graph.pm lib/Thruk/Utils/CLI/Host.pm @@ -150,6 +151,8 @@ lib/Thruk/Utils/Encode.pm lib/Thruk/Utils/External.pm lib/Thruk/Utils/Filter.pm lib/Thruk/Utils/IO.pm +lib/Thruk/Utils/IO/LocalFS.pm +lib/Thruk/Utils/IO/Mysql.pm lib/Thruk/Utils/LMD.pm lib/Thruk/Utils/Log.pm lib/Thruk/Utils/Menu.pm @@ -2835,6 +2838,25 @@ t/scenarios/cli_api/t/thruk_cmd_cli.t t/scenarios/cli_api/test.sh t/scenarios/cli_api/thruk.conf t/scenarios/cli_api/thruk_local.conf +t/scenarios/cluster_db_e2e/docker-compose.yml +t/scenarios/cluster_db_e2e/Makefile +t/scenarios/cluster_db_e2e/omd/Dockerfile +t/scenarios/cluster_db_e2e/omd/playbook.yml +t/scenarios/cluster_db_e2e/omd/test.cfg +t/scenarios/cluster_db_e2e/README +t/scenarios/cluster_db_e2e/scale +t/scenarios/cluster_db_e2e/t/300-controller_cluster.t +t/scenarios/cluster_db_e2e/t/300-controller_rest_v1.t +t/scenarios/cluster_db_e2e/t/300-controller_tac.t +t/scenarios/cluster_db_e2e/t/local/cluster.t +t/scenarios/cluster_db_e2e/thruk/1.rpt +t/scenarios/cluster_db_e2e/thruk/1.tbp +t/scenarios/cluster_db_e2e/thruk/1.tsk +t/scenarios/cluster_db_e2e/thruk/Dockerfile +t/scenarios/cluster_db_e2e/thruk/dot_thruk +t/scenarios/cluster_db_e2e/thruk/playbook.yml +t/scenarios/cluster_db_e2e/thruk/test.cfg +t/scenarios/cluster_db_e2e/thruk/thruk_cluster.conf t/scenarios/cluster_e2e/docker-compose.yml t/scenarios/cluster_e2e/Makefile t/scenarios/cluster_e2e/omd/Dockerfile diff --git a/docs/documentation/cluster.asciidoc b/docs/documentation/cluster.asciidoc index 7b8b7314a4..2feb0c0355 100644 --- a/docs/documentation/cluster.asciidoc +++ b/docs/documentation/cluster.asciidoc @@ -6,9 +6,8 @@ title: Cluster Setup == Thruk Clustering {% include new_since.ad version="2.24" %} Clustered setups provide high-availability and performance improvements at the -price of higher complexity. All Thruk nodes in a cluster must use shared -storage for their `etc_path` and `var_path` while the `tmp_path` should remain -local. +price of higher complexity. All Thruk nodes in a cluster must use some kind of +shared storage or database for `var_path`. === Setup @@ -19,6 +18,23 @@ local. - Enable cluster with `cluster_enabled=1` +==== Shared Filesystem +All Thruk nodes in a cluster must use shared storage for their `etc_path` +and `var_path` while the `tmp_path` remains local. + +This can be either implemented by mounting for example a NFS share or by using +a shared database. + +==== Shared Database +{% include new_since.ad version="3.22" %} +Instead of a shared filesystem, you can use a shared database for the `var_path`. +This is especially useful when you have a database cluster already in place. + +In order to use a mysql/mariadb database, you have to fill in a connection +string in the `var_path_db`. + + var_path_db = mysql://user:password@hostname/databasename + ==== Static Cluster Having a fixed number of cluster nodes will be set if you configure multiple cluster_nodes with fixed hostnames like: diff --git a/docs/documentation/configuration.asciidoc b/docs/documentation/configuration.asciidoc index a87e9f7088..922b5e8a0f 100644 --- a/docs/documentation/configuration.asciidoc +++ b/docs/documentation/configuration.asciidoc @@ -708,6 +708,15 @@ ex.: var_path = ./var +=== var_path_db + +Use database for storing files in var_path. See link:cluster.html[Clustering]. + +ex.: + + var_path_db = mysql://user:password@hostname/databasename + + === tmp_path Path to a temporary directory. Defaults to /tmp if not set and usually diff --git a/docs/documentation/faq.asciidoc b/docs/documentation/faq.asciidoc index 0816782b29..0551fe4c34 100644 --- a/docs/documentation/faq.asciidoc +++ b/docs/documentation/faq.asciidoc @@ -251,14 +251,14 @@ the shipping installer: #> /usr/share/thruk/script/install_puppeteer.sh -This will install puppeteer into /var/lib/thruk/puppeteer +This will install puppeteer into /var/lib/thruk/local/puppeteer It requires node and npm to be installed. Thruk will use the system chromium if it is installed befor running the puppet installer. If no chromium is installed, puppeteer will download chromium -into /var/lib/thruk/puppeteer/chromium. +into /var/lib/thruk/local/puppeteer/chromium. You can disable downloading chromium by setting this into the environment before running the installer. diff --git a/lib/Thruk.pm b/lib/Thruk.pm index 3b2f14494d..5a82079c4f 100644 --- a/lib/Thruk.pm +++ b/lib/Thruk.pm @@ -702,7 +702,7 @@ sub _create_secret_file { return unless (Thruk::Base->mode() eq 'FASTCGI' || Thruk::Base->mode() eq 'DEVSERVER'); my $var_path = $config->{'var_path'} || die("no var path!"); my $secretfile = $var_path.'/secret.key'; - return if -s $secretfile; + return if Thruk::Utils::IO::file_not_empty($secretfile); require Thruk::Utils::Crypt; my $digest = Thruk::Utils::Crypt::random_uuid([time()]); Thruk::Utils::IO::write($secretfile, $digest); diff --git a/lib/Thruk/Backend/Manager.pm b/lib/Thruk/Backend/Manager.pm index 2133d12fa2..a0962328b8 100644 --- a/lib/Thruk/Backend/Manager.pm +++ b/lib/Thruk/Backend/Manager.pm @@ -3123,10 +3123,10 @@ sub caching_query { } # simply remove all files older than 24h my $yesterday = time() - 86400; - for my $file (glob($cache_file."/*.cache")) { - my @stat = stat($file); + for my $file (@{Thruk::Utils::IO::find_files($cache_file, '\.cache$')}) { + my @stat = Thruk::Utils::IO::stat($file); if($stat[9] < $yesterday) { - unlink($file); + Thruk::Utils::IO::unlink($file); } } } else { diff --git a/lib/Thruk/Backend/Provider/Mysql.pm b/lib/Thruk/Backend/Provider/Mysql.pm index 709d9bab6a..fda9e40a59 100644 --- a/lib/Thruk/Backend/Provider/Mysql.pm +++ b/lib/Thruk/Backend/Provider/Mysql.pm @@ -90,8 +90,35 @@ sub new { if(!defined $options->{'peer_key'}) { confess('please provide peer_key'); } + my($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock) = _parse_connection_string($options->{'peer'}); + my $self = { + 'dbhost' => $dbhost, + 'dbport' => $dbport, + 'dbname' => $dbname, + 'dbuser' => $dbuser, + 'dbpass' => $dbpass, + 'dbsock' => $dbsock, + 'peer_config' => $options, + 'verbose' => 0, + }; + bless $self, $class; + + return $self; +} + +########################################################## + +=head2 _parse_connection_string + + _parse_connection_string($str) + +parse and return connection string + +=cut +sub _parse_connection_string { + my($connection_string) = @_; my($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock); - if($options->{'peer'} =~ m/^mysql:\/\/(.*?)(|:.*?)@([^:]+)(|:.*?)\/([^\/]*?)$/mx) { + if($connection_string =~ m/^mysql:\/\/(.*?)(|:.*?)@([^:]+)(|:.*?)\/([^\/]*?)$/mx) { $dbuser = $1; $dbpass = $2; $dbhost = $3; @@ -106,20 +133,7 @@ sub new { } else { die('Mysql connection must match this form: mysql://user:password@host:port/dbname'); } - - my $self = { - 'dbhost' => $dbhost, - 'dbport' => $dbport, - 'dbname' => $dbname, - 'dbuser' => $dbuser, - 'dbpass' => $dbpass, - 'dbsock' => $dbsock, - 'peer_config' => $options, - 'verbose' => 0, - }; - bless $self, $class; - - return $self; + return($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock); } ########################################################## @@ -1514,7 +1528,6 @@ sub _check_lock { # check if there is already a update / import running my $skip = 0; - my $cache_version = 1; eval { $dbh->do('LOCK TABLES `'.$prefix.'_status` READ') unless $c->config->{'logcache_pxc_strict_mode'}; my @pids = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 2 LIMIT 1')}; @@ -1524,10 +1537,6 @@ sub _check_lock { $skip = 1; } } - my @versions = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 4 LIMIT 1')}; - if(scalar @versions > 0 and $versions[0]) { - $cache_version = $versions[0]; - } }; $dbh->do('UNLOCK TABLES') unless $c->config->{'logcache_pxc_strict_mode'}; if($@) { diff --git a/lib/Thruk/Config.pm b/lib/Thruk/Config.pm index 763d98f86e..d4b5b55704 100644 --- a/lib/Thruk/Config.pm +++ b/lib/Thruk/Config.pm @@ -487,6 +487,7 @@ sub set_default_config { # set var dir $config->{'var_path'} = $config->{'home'}.'/var' unless defined $config->{'var_path'}; $config->{'var_path'} =~ s|/$||mx; + $Thruk::Utils::IO::var_path = $config->{'var_path'}; if(!defined $config->{'etc_path'}) { if($ENV{'THRUK_CONFIG'}) { @@ -995,9 +996,8 @@ return secret_key sub secret_key { my $config = &get_config(); my $secret_file = $config->{'var_path'}.'/secret.key'; - return unless -s $secret_file; - my $secret_key = Thruk::Utils::IO::read($secret_file); - chomp($secret_key); + my $secret_key = Thruk::Utils::IO::saferead($secret_file); + chomp($secret_key) if defined $secret_key; return($secret_key); } diff --git a/lib/Thruk/Controller/Rest/V1/broadcast.pm b/lib/Thruk/Controller/Rest/V1/broadcast.pm index a41375faff..36d312e7c9 100644 --- a/lib/Thruk/Controller/Rest/V1/broadcast.pm +++ b/lib/Thruk/Controller/Rest/V1/broadcast.pm @@ -27,8 +27,8 @@ Thruk::Controller::rest_v1::register_rest_path_v1('GET', qr%^/thruk/broadcasts?$ sub _rest_get_thruk_broadcast { my($c, undef, $file) = @_; require Thruk::Utils::Broadcast; - $file = '*' unless $file; - my @files = glob($c->config->{'var_path'}.'/broadcast/'.$file.'.json'); + $file = '.*' unless $file; + my @files = @{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/broadcast/', $file.'\.json$')}; my $broadcasts = Thruk::Controller::rest_v1::load_json_files($c, { files => \@files, authorization_callback => $c->user->check_user_roles('authorized_for_broadcasts') ? undef : \&Thruk::Utils::Broadcast::is_authorized_for_broadcast, @@ -41,7 +41,7 @@ sub _rest_get_thruk_broadcast { $b->{'text'} = Thruk::Utils::Filter::replace_macros($b->{'text'}, $b->{'frontmatter'}); } - if($file eq '*') { + if($file eq '.*') { return($broadcasts); } @@ -80,7 +80,7 @@ sub _rest_get_thruk_broadcast { }); } - if($file ne '*') { + if($file ne '.*') { return($broadcasts->[0]); } } @@ -115,7 +115,7 @@ sub _rest_get_thruk_broadcast_new { if(!$file) { $file = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'.json', localtime); my $x = 1; - while(-e $c->config->{'var_path'}.'/broadcast/'.$file) { + while(Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/broadcast/'.$file)) { $file = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'_'.$x.'.json', localtime); $x++; } diff --git a/lib/Thruk/Controller/broadcast.pm b/lib/Thruk/Controller/broadcast.pm index 1a6082bc1e..bc9df0c6fb 100644 --- a/lib/Thruk/Controller/broadcast.pm +++ b/lib/Thruk/Controller/broadcast.pm @@ -104,7 +104,7 @@ sub index { if($id eq 'new') { $id = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'.json', localtime); my $x = 1; - while(-e $c->config->{'var_path'}.'/broadcast/'.$id) { + while(Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/broadcast/'.$id)) { $id = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'_'.$x.'.json', localtime); $x++; } diff --git a/lib/Thruk/Controller/extinfo.pm b/lib/Thruk/Controller/extinfo.pm index 9c99eb51cb..cb6249a468 100644 --- a/lib/Thruk/Controller/extinfo.pm +++ b/lib/Thruk/Controller/extinfo.pm @@ -343,8 +343,8 @@ sub _process_recurring_downtimes_page { my $old_file; if($nr && !$failed) { $old_file = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk'; - if(-s $old_file) { - my $old_rd = Thruk::Utils::read_data_file($old_file); + my $old_rd = Thruk::Utils::read_data_file($old_file); + if($old_rd) { if(Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $old_rd) != 2) { $failed = 1; } else { @@ -381,8 +381,8 @@ sub _process_recurring_downtimes_page { } for my $nr (@{$numbers}) { my $file = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk'; - if(-s $file) { - my $old_rd = Thruk::Utils::read_data_file($file); + my $old_rd = Thruk::Utils::read_data_file($file); + if($old_rd) { if(Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $old_rd) != 2) { Thruk::Utils::set_message( $c, { style => 'success_message', msg => 'no such downtime!' }); } else { @@ -420,7 +420,8 @@ sub _process_recurring_downtimes_page_edit { $c->stash->{can_edit} = 1; if($nr) { my $file = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk'; - if(-s $file) { + my $exists = Thruk::Utils::IO::saferead($file); + if(defined $exists) { $c->stash->{rd} = Thruk::Utils::RecurringDowntimes::read_downtime($c, $file, undef, undef, undef, undef, undef, undef, undef, 0); my $perms = Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $c->stash->{rd}); # check cmd permission for this downtime diff --git a/lib/Thruk/Controller/login.pm b/lib/Thruk/Controller/login.pm index 68d08d69be..234727d98e 100644 --- a/lib/Thruk/Controller/login.pm +++ b/lib/Thruk/Controller/login.pm @@ -359,6 +359,7 @@ sub _clean_failed_logins { } my $timeout = time() - (30 * 86400); Thruk::Utils::IO::mkdir($dir); +# TODO: ... opendir( my $dh, $dir) or die "can't opendir '$dir': $!"; for my $entry (readdir($dh)) { next if $entry eq '.' or $entry eq '..'; diff --git a/lib/Thruk/Controller/remote.pm b/lib/Thruk/Controller/remote.pm index 26d917f3d3..c631b6ac6c 100644 --- a/lib/Thruk/Controller/remote.pm +++ b/lib/Thruk/Controller/remote.pm @@ -100,12 +100,10 @@ sub index { if($body) { if(ref $body eq 'File::Temp') { my $file = $body->filename(); - if($file and -e $file) { - my $msg = Thruk::Utils::IO::read($file); - unlink($file); - _error($msg); - return $c->render("text" => 'OK'); - } + my $msg = Thruk::Utils::IO::saferead($file); + unlink($file); + _error($msg) if $msg; + return $c->render("text" => 'OK'); } if(ref $body eq 'FileHandle') { while(<$body>) { diff --git a/lib/Thruk/Controller/rest_v1.pm b/lib/Thruk/Controller/rest_v1.pm index 721a11d66d..02b65a1544 100644 --- a/lib/Thruk/Controller/rest_v1.pm +++ b/lib/Thruk/Controller/rest_v1.pm @@ -1749,8 +1749,8 @@ sub load_json_files { my($c, $options) = @_; my $list = []; for my $file (@{$options->{'files'}}) { - next unless -e $file; my $data = Thruk::Utils::IO::json_lock_retrieve($file); + next unless $data; $data->{'file'} = $file; $data->{'file'} =~ s%.*?([^/]+)\.\w+$%$1%mx; if($options->{'pre_process_callback'}) { @@ -1849,8 +1849,15 @@ sub _rest_get_thruk_jobs { my($c, undef, $job) = @_; require Thruk::Utils::External; + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}."/jobs"); + my %dirs; + for my $f (@{$files}) { + my $d = Thruk::Base::dirname($f); + $dirs{$d} = 1; + } + my $data = []; - for my $dir (glob($c->config->{'var_path'}."/jobs/*/.")) { + for my $dir (sort keys %dirs) { if($dir =~ m%/([^/]+)/\.$%mx) { my $id = $1; next if $job && $job ne $id; @@ -1890,7 +1897,7 @@ sub _rest_get_thruk_sessions { my $min5 = time() - (5*60); my $uniq = {}; my $uniq5min = {}; - for my $file (sort glob($c->config->{'var_path'}."/sessions/*")) { + for my $file (sort @{Thruk::Utils::IO::find_files($c->config->{'var_path'}."/sessions/")}) { $total_number++; my $session_data = Thruk::Utils::CookieAuth::retrieve_session(config => $c->config, file => $file); next unless $session_data; diff --git a/lib/Thruk/Metrics.pm b/lib/Thruk/Metrics.pm index ba7d427188..a17a6ca5a0 100644 --- a/lib/Thruk/Metrics.pm +++ b/lib/Thruk/Metrics.pm @@ -35,11 +35,7 @@ sub register { sub get_all { my($self) = @_; $self->store(); - if(!-s $self->{'file'}) { - return({}); - } - my $data = Thruk::Utils::IO::json_lock_retrieve($self->{'file'}); - return($data); + return(Thruk::Utils::IO::json_lock_retrieve($self->{'file'}) // {}); } ############################################## @@ -72,12 +68,6 @@ sub store { $self->_save_help() if $self->{'save_help'}; return if scalar @{$self->{'store'}} == 0; my $data = {}; - if(!-s $self->{'file'}) { - $self->_apply_data($data); - Thruk::Utils::IO::json_store($self->{'file'}, $data, { pretty => 1 }); - $self->{'store'} = []; - return; - } my($fh, $lock_fh); eval { ($fh, $lock_fh) = Thruk::Utils::IO::file_lock($self->{'file'}); @@ -87,7 +77,7 @@ sub store { $self->{'store'} = []; }; my $err = $@; - Thruk::Utils::IO::file_unlock($self->{'file'}, $fh, $lock_fh) if($fh || $lock_fh); + Thruk::Utils::IO::file_unlock($self->{'file'}, $fh, $lock_fh); confess($err) if $err; return; } @@ -95,10 +85,7 @@ sub store { ############################################## sub _save_help { my($self) = @_; - if(!-s $self->{'help_file'}) { - Thruk::Utils::IO::json_store($self->{'help_file'}, {}, { pretty => 1 }); - } - my $help = Thruk::Utils::IO::json_lock_retrieve($self->{'help_file'}); + my $help = Thruk::Utils::IO::json_lock_retrieve($self->{'help_file'}) // {}; for my $key (keys %{$self->{'help'}}) { $help->{$key} = $self->{'help'}->{$key}; } diff --git a/lib/Thruk/Utils.pm b/lib/Thruk/Utils.pm index 7a471b6935..ace1ec551f 100644 --- a/lib/Thruk/Utils.pm +++ b/lib/Thruk/Utils.pm @@ -1089,11 +1089,8 @@ sub get_user_data { } confess("username not allowed") if Thruk::Base::check_for_nasty_filename($username); - my $user_data = {}; - my $file = $c->config->{'var_path'}."/users/".$username; - if(-s $file) { - $user_data = read_data_file($file); - } + my $file = $c->config->{'var_path'}."/users/".$username; + my $user_data = Thruk::Utils::IO::json_retrieve($file) // {}; # remove null entries from site_panel_bookmarks if($user_data->{'site_panel_bookmarks'}) { @@ -1170,8 +1167,7 @@ sub get_global_user_data { my($c) = @_; my $file = $c->config->{'var_path'}."/global_user_data"; - return {} unless -s $file; - return read_data_file($file); + return(Thruk::Utils::IO::json_retrieve($file) // {}); } @@ -1847,13 +1843,9 @@ sub absolute_url { $host =~ s/\/$//mx; # remove last / $fullpath =~ s/\?.*$//mx; $fullpath =~ s/^\///mx; - my($path,$file) = ('', ''); + my $path = ''; if($fullpath =~ m/^(.+)\/(.*)$/mx) { $path = $1; - $file = $2; - } - else { - $file = $fullpath; } $path =~ s/^\///mx; # remove first / @@ -4124,20 +4116,13 @@ removes all files from folder which are older than $max_age and match given $fil sub clean_old_folder_files { my($folder, $filter, $max_age) = @_; - return unless -d $folder; - my $timeout = time() - $max_age; - _debug2("checking for old files in ".$folder." older than: ".(scalar localtime($timeout))); - opendir( my $dh, $folder) || die "can't opendir '$folder': $!"; - for my $entry (readdir($dh)) { - next if $entry eq '.' or $entry eq '..'; - next if $entry !~ m/$filter/mx; - - my $file = $folder.'/'.$entry; - my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($file); + _debug2("checking for old files in ".$folder." older than: ".(scalar localtime($timeout)).($filter ? " matching: ".$filter : "")); + for my $file (@{Thruk::Utils::IO::find_files($folder, $filter)}) { + my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = Thruk::Utils::IO::stat($file); if($mtime && $mtime < $timeout) { _debug2("removing old file: ".$file); - unlink($file); + Thruk::Utils::IO::unlink($file); } } diff --git a/lib/Thruk/Utils/APIKeys.pm b/lib/Thruk/Utils/APIKeys.pm index 0eea59ceb6..6267b3aaf6 100644 --- a/lib/Thruk/Utils/APIKeys.pm +++ b/lib/Thruk/Utils/APIKeys.pm @@ -44,7 +44,7 @@ sub get_keys { my $keys = []; my $folder = $c->config->{'var_path'}.'/api_keys'; - for my $file (glob($folder.'/*')) { + for my $file (@{Thruk::Utils::IO::find_files($folder)}) { my $basename = Thruk::Base::basename($file); next unless $basename =~ $hashed_key_file_regex; next if $basename =~ /\.stats$/mx; @@ -153,7 +153,7 @@ sub create_key { if(defined $roles) { $data->{'roles'} = $roles; } - die("hash collision") if -e $file; + die("hash collision") if Thruk::Utils::IO::file_exists($file); Thruk::Utils::IO::json_lock_store($file, $data, { pretty => 1 }); return($privatekey, $hashed_key, $file); @@ -259,8 +259,8 @@ return key for given file sub read_key { my($config, $file) = @_; confess("no file") unless $file; - return unless -r $file; - my $data = Thruk::Utils::IO::json_lock_retrieve($file); + my $data = Thruk::Utils::IO::json_retrieve($file); + return unless $data; my $hashed_key = Thruk::Base::basename($file); my $type; if($hashed_key =~ m%\.([^\.]+)$%gmx) { @@ -272,14 +272,9 @@ sub read_key { $data->{'digest'} = $type; $data->{'superuser'} = 1 if delete $data->{'system'}; # migrate system keys delete $data->{'force_user'} unless $data->{'superuser'}; - if(-s $file.'.stats') { - my $stats = {}; - eval { - $stats = Thruk::Utils::IO::json_lock_retrieve($file.'.stats'); - }; - _debug("failed to read stats file: ".$@) if $@; - $data = { %{$stats}, %{$data} } if $stats; - } + my $stats = Thruk::Utils::IO::json_retrieve($file.'.stats'); + _debug("failed to read stats file: ".$@) if $@; + $data = { %{$stats}, %{$data} } if $stats; return($data); } diff --git a/lib/Thruk/Utils/Broadcast.pm b/lib/Thruk/Utils/Broadcast.pm index 3deaa2a6b8..f869a76615 100644 --- a/lib/Thruk/Utils/Broadcast.pm +++ b/lib/Thruk/Utils/Broadcast.pm @@ -32,7 +32,7 @@ sub get_broadcasts { my $list = []; my $now = time(); - my @files = glob($c->config->{'var_path'}.'/broadcast/*.json'); + my @files = @{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/broadcast/', '\.json$')}; return([]) unless scalar @files > 0; my $user_data = Thruk::Utils::get_user_data($c); @@ -240,7 +240,7 @@ sub update_dismiss { my $clean_delay = $now - (86400 * 10); for my $file (keys %{$data->{'broadcast'}->{'read'}}) { my $ts = $data->{'broadcast'}->{'read'}->{$file}; - if(!-e $c->config->{'var_path'}.'/broadcast/'.$file && $ts < $clean_delay) { + if(!Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/broadcast/'.$file) && $ts < $clean_delay) { delete $data->{'broadcast'}->{'read'}->{$file}; } } diff --git a/lib/Thruk/Utils/CLI/Apikey.pm b/lib/Thruk/Utils/CLI/Apikey.pm index 2f5fd4c137..45b5469a37 100644 --- a/lib/Thruk/Utils/CLI/Apikey.pm +++ b/lib/Thruk/Utils/CLI/Apikey.pm @@ -129,7 +129,7 @@ sub _print_key { if($hashed_key) { $file = sprintf("%s/api_keys/%s.%s", $c->config->{'var_path'}, $hashed_key, $digest_name); } - if((!$file || !-e $file) && -e $key) { + if((!$file || !Thruk::Utils::IO::file_exists($file)) && Thruk::Utils::IO::file_exists($key)) { $file = $key; } @@ -142,7 +142,7 @@ sub _print_key { if(!$file) { push @{$res}, { 'name' => 'info', 'value' => "wrong key format" }; } - elsif(!-e $file) { + elsif(!Thruk::Utils::IO::file_exists($file)) { push @{$res}, { 'name' => 'info', 'value' => "unable to read key: ".$! }; } elsif(!$data) { diff --git a/lib/Thruk/Utils/CLI/Downtimetask.pm b/lib/Thruk/Utils/CLI/Downtimetask.pm index bdfb36dc6a..780597e5de 100644 --- a/lib/Thruk/Utils/CLI/Downtimetask.pm +++ b/lib/Thruk/Utils/CLI/Downtimetask.pm @@ -123,7 +123,8 @@ sub _handle_file { my $nr = $file; $file = $c->config->{'var_path'}.'/downtimes/'.$file.'.tsk'; - if(!-s $file) { + my $data = Thruk::Utils::IO::saferead($file); + if(!defined $data) { _error("cannot read %s: %s", $file, $!); return("", 1); } @@ -318,9 +319,8 @@ sub _auto_fix_downtimes { require Thruk::Utils::RecurringDowntimes; my $fixed = 0; - my @files = glob($c->config->{'var_path'}.'/downtimes/*.tsk'); - for my $dfile (@files) { - next unless -f $dfile; + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/downtimes/', '\.tsk$'); + for my $dfile (@{$files}) { my $d = Thruk::Utils::RecurringDowntimes::read_downtime($c, $dfile); next unless $d; next unless $d->{'fixable'}; diff --git a/lib/Thruk/Utils/CLI/Filesystem.pm b/lib/Thruk/Utils/CLI/Filesystem.pm new file mode 100644 index 0000000000..82269b2097 --- /dev/null +++ b/lib/Thruk/Utils/CLI/Filesystem.pm @@ -0,0 +1,187 @@ +package Thruk::Utils::CLI::Filesystem; + +=head1 NAME + +Thruk::Utils::CLI::Filesystem - CLI module to synchronize var filesystem + +=head1 DESCRIPTION + +This module provides a CLI interface to synchronize the var filesystem. + +=head1 SYNOPSIS + + Usage: thruk [globaloptions] filesystem + +=head1 OPTIONS + +=over 4 + +=item B + + print help and exit + +=item B + + Available commands are: + + - list list all files + - cat print file content to STDOUT + - sync sync files from db to fs or the other way round + - import [options] alias for "sync fs db" + - export [options] alias for "sync db fs" + - drop removes the var_path database + + options: + + --delete delete files on target which do not exist on source + + +=back + +=cut + +use warnings; +use strict; +use Getopt::Long (); + +use Thruk::Utils::CLI (); +use Thruk::Utils::IO (); +use Thruk::Utils::Log qw/:all/; + +############################################## +# no backends required for this command +our $skip_backends = 1; + +############################################## + +=head1 METHODS + +=head2 cmd + + cmd([ $options ]) + +=cut +sub cmd { + my($c, $action, $commandoptions, $data) = @_; + $c->stats->profile(begin => "_cmd_filesystem($action)"); + + if(!$c->check_user_roles('authorized_for_admin')) { + return("ERROR - authorized_for_admin role required\n", 1); + } + + if(! $c->config->{'var_path_db'}) { + return("ERROR - var_path_db must be enabled\n", 1); + } + + # parse options + my $opts = { + delete => undef, + }; + Getopt::Long::Configure('pass_through'); + Getopt::Long::GetOptionsFromArray($commandoptions, + "delete" => \$opts->{'delete'}, + ) || do { + return(Thruk::Utils::CLI::get_submodule_help(__PACKAGE__)); + }; + + # cache actions + my $command = shift @{$commandoptions} || 'help'; + my($rc, $out) = (3, 'UNKNOWN command'); + if($command eq 'cat') { + ($rc, $out) = _cmd_cat($c, $commandoptions); + } elsif($command eq 'list') { + ($rc, $out) = _cmd_list($c); + } elsif($command eq 'import') { + ($rc, $out) = _cmd_sync($c, ['fs', 'db', @{$commandoptions}], $opts); + } elsif($command eq 'export') { + ($rc, $out) = _cmd_sync($c, ['db', 'fs', @{$commandoptions}], $opts); + } elsif($command eq 'sync') { + ($rc, $out) = _cmd_sync($c, $commandoptions, $opts); + } elsif($command eq 'drop') { + ($rc, $out) = _cmd_drop($c); + } else { + return(Thruk::Utils::CLI::get_submodule_help(__PACKAGE__)); + } + + $data->{'rc'} = $rc; + $data->{'output'} = $out; + + $c->stats->profile(end => "_cmd_filesystem($action)"); + return($data); +} + +############################################## +sub _cmd_list { + my($c) = @_; + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}); + my $out = join("\n", sort @{$files})."\n"; + return(0, $out); +} + +############################################## +sub _cmd_cat { + my($c, $commandoptions) = @_; + my $file = shift @{$commandoptions}; + if(!$file) { + return(1, "ERROR - missing file\n"); + } + my $content = Thruk::Utils::IO::saferead($file); + if(!defined $content) { + return(1, "ERROR - not read file: $!\n"); + } + return(0, $content); +} + +############################################## +sub _cmd_drop { + my($c) = @_; + + Thruk::Utils::IO::handle_io("_drop_tables", 0, $c->config->{'var_path'}, \@_); + + return(0, "OK - tables dropped\n"); +} + +############################################## +sub _cmd_sync { + my($c, $commandoptions, $opts) = @_; + my $from = shift @{$commandoptions}; + my $to = shift @{$commandoptions}; + if(!$from || !$to) { + return(3, "usage: filesystem sync \n"); + } + + my $action; + $action = 'export' if $from eq 'db'; + $action = 'import' if $from eq 'fs'; + if(!$action) { + return(3, "usage: filesystem sync \n"); + } + if($action eq 'import' && $to ne 'db') { + return(3, "usage: filesystem sync \n"); + } + if($action eq 'export' && $to ne 'fs') { + return(3, "usage: filesystem sync \n"); + } + + Thruk::Utils::IO::sync_db_fs($c, $from, $to, $opts); + + return(0, ""); +} + +############################################## + +=head1 EXAMPLES + +List all files: + + %> thruk filesystem list + +Display specific file content: + + %> thruk filesystem cat VAR::/cluster/nodes + +=cut + +############################################## + +1; diff --git a/lib/Thruk/Utils/Cache.pm b/lib/Thruk/Utils/Cache.pm index c85b3bce31..84e058fdb7 100644 --- a/lib/Thruk/Utils/Cache.pm +++ b/lib/Thruk/Utils/Cache.pm @@ -172,17 +172,16 @@ update cache from file =cut sub _update { my($self) = @_; - if(-s $self->{'_cachefile'}) { - my @stat = stat(_); - if(!$self->{'_stat'}->[9] || $stat[9] != $self->{'_stat'}->[9]) { - $self->{'_data'} = $self->_retrieve(); - $self->{'_stat'} = \@stat; - return; - } - } else { - # did not exist before, so create an empty cache - $self->_store(); + + my @stat = Thruk::Utils::IO::stat($self->{'_cachefile'}); + if(!$self->{'_stat'}->[9] || $stat[9] != $self->{'_stat'}->[9]) { + $self->{'_data'} = $self->_retrieve(); + $self->{'_stat'} = \@stat; + return; } + + # did not exist before, so create an empty cache + $self->_store(); return; } @@ -197,7 +196,7 @@ store cache to disk =cut sub _store { my($self) = @_; - Thruk::Utils::IO::json_lock_store($self->{'_cachefile'}, $self->{'_data'}); + Thruk::Utils::IO::json_lock_store($self->{'_cachefile'}, $self->{'_data'} // {}); my @stat = stat($self->{'_cachefile'}) or die("cannot stat ".$self->{'_cachefile'}.": ".$!); $self->{'_stat'} = \@stat; return; diff --git a/lib/Thruk/Utils/Cluster.pm b/lib/Thruk/Utils/Cluster.pm index c6312882c6..f054dd6fa2 100644 --- a/lib/Thruk/Utils/Cluster.pm +++ b/lib/Thruk/Utils/Cluster.pm @@ -189,7 +189,6 @@ removes ourself from the cluster statefile sub unregister { my($self) = @_; return unless $Thruk::Globals::NODE_ID; - return unless -s $self->{'localstate'}; Thruk::Utils::IO::json_lock_patch($self->{'localstate'}, { $Thruk::Globals::NODE_ID => { pids => { $$ => undef }, @@ -231,9 +230,7 @@ return 1 if a cluster is configured =cut sub is_clustered { my($self) = @_; - return 0 if !$self->{'config'}->{'cluster_enabled'}; - return 1 if scalar keys %{$self->{nodes_by_url}} > 1; - return 1 if scalar @{$self->{'config'}->{'cluster_nodes'}} > 1; + return 1 if $self->{'config'}->{'cluster_enabled'}; return 0; } @@ -565,14 +562,9 @@ sub _replace_url_macros { ########################################################## sub _cleanup_jobs_folder { my($self) = @_; - my $keep = time() - 600; - my $jobs_path = $self->{'config'}->{'var_path'}.'/cluster/jobs'; - for my $file (glob($jobs_path.'/*')) { - my @stat = stat($file); - if($stat[9] && $stat[9] < $keep) { - unlink($file); - } - } + + Thruk::Utils::clean_old_folder_files($self->{'config'}->{'var_path'}.'/cluster/jobs', undef, 600); + return; } diff --git a/lib/Thruk/Utils/CookieAuth.pm b/lib/Thruk/Utils/CookieAuth.pm index b3ed390cd8..dd4b767af2 100644 --- a/lib/Thruk/Utils/CookieAuth.pm +++ b/lib/Thruk/Utils/CookieAuth.pm @@ -213,40 +213,37 @@ sub clean_session_files { my $fake_session_timeout = time() - 600; Thruk::Utils::IO::mkdir($sdir); my $sessions_by_user = {}; - opendir( my $dh, $sdir) or die "can't opendir '$sdir': $!"; - for my $entry (readdir($dh)) { - next if $entry eq '.' or $entry eq '..'; - $total++; - my $file = $sdir.'/'.$entry; + for my $file (@{Thruk::Utils::IO::find_files($sdir)}) { my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, - $atime,$mtime,$ctime,$blksize,$blocks) = stat($file); - if($mtime) { - if($mtime < $timeout) { + $atime,$mtime,$ctime,$blksize,$blocks) = Thruk::Utils::IO::stat($file); + next unless $mtime; + $total++; + + if($mtime < $timeout) { + my $data; + eval { + $data = Thruk::Utils::IO::json_lock_retrieve($file); + }; + _warn($@) if $@; + _audit_log("session", "session timeout hit, removing session file", $data->{'username'}//'?', $file, 0); + Thruk::Utils::IO::unlink($file); + $cleaned++; + } + elsif($mtime < $fake_session_timeout) { + eval { my $data; eval { $data = Thruk::Utils::IO::json_lock_retrieve($file); }; _warn($@) if $@; - _audit_log("session", "session timeout hit, removing session file", $data->{'username'}//'?', $entry, 0); - unlink($file); - $cleaned++; - } - elsif($mtime < $fake_session_timeout) { - eval { - my $data; - eval { - $data = Thruk::Utils::IO::json_lock_retrieve($file); - }; - _warn($@) if $@; - if($data && $data->{'fake'}) { - _audit_log("session", "short session timeout hit, removing session file", $data->{'username'}//'?', $entry, 0); - unlink($file); - $cleaned++; - } elsif(defined $data->{'username'}) { - $sessions_by_user->{$data->{'username'}}->{$file} = $mtime; - } - }; - } + if($data && $data->{'fake'}) { + _audit_log("session", "short session timeout hit, removing session file", $data->{'username'}//'?', $file, 0); + Thruk::Utils::IO::unlink($file); + $cleaned++; + } elsif(defined $data->{'username'}) { + $sessions_by_user->{$data->{'username'}}->{$file} = $mtime; + } + }; } } @@ -262,7 +259,7 @@ sub clean_session_files { my $entry = $file; $entry =~ s|^.*/||gmx; _audit_log("session", "max session reached, cleaning old session", $user, $entry, 0); - unlink($file); + Thruk::Utils::IO::unlink($file); $cleaned++; $num--; } else { @@ -417,9 +414,10 @@ sub retrieve_session { my $sdir = $config->{'var_path'}.'/sessions'; $sessionfile = $sdir.'/'.$hashed_key.'.'.$digest_name; + my @stat = Thruk::Utils::IO::stat($sessionfile); + return unless $stat[9]; + my $data; - return unless -e $sessionfile; - my @stat = stat(_); eval { $data = Thruk::Utils::IO::json_lock_retrieve($sessionfile); }; diff --git a/lib/Thruk/Utils/External.pm b/lib/Thruk/Utils/External.pm index 27c18f2cb0..39c24a1982 100644 --- a/lib/Thruk/Utils/External.pm +++ b/lib/Thruk/Utils/External.pm @@ -76,6 +76,7 @@ sub cmd { return $parent_res if $is_parent; _init_child_process($c, $dir, $id, $conf); +# TODO: $cmd = $cmd.'; echo $? > '.$dir."/rc" unless $conf->{'no_shell'}; exec($cmd) or exit(1); # just to be sure } @@ -181,6 +182,7 @@ sub perl { my $res = $c->res->finalize; $c->finalize_request($res); Thruk::Utils::IO::write($dir."/result.dat", $res->[2]->[0]); +# TODO: check $c->stash->{'file_name'} = "result.dat"; $c->stash->{'file_name_meta'} = { code => $res->[0], @@ -188,8 +190,9 @@ sub perl { }; } # rendered output, ex.: from return $c->render(json => $json); - elsif($conf->{'render'} && $c->{'rendered'} && !$c->stash->{'last_redirect_to'} && -e $dir."/perl_res") { + elsif($conf->{'render'} && $c->{'rendered'} && !$c->stash->{'last_redirect_to'} && Thruk::Utils::IO::file_exists($dir."/perl_res")) { local $c->stash->{'job_conf'}->{'clean'} = undef; +# TODO: check $c->stash->{'file_name'} = "perl_res"; $c->stash->{'file_name_meta'} = { code => $c->res->code(), @@ -274,9 +277,9 @@ sub is_running { my($c, $id, $nouser) = @_; confess("got no id") unless $id; - my $dir = $c->config->{'var_path'}."/jobs/".$id; - if(!$nouser && -f $dir."/user" ) { - my $user = Thruk::Utils::IO::read($dir."/user"); + my $dir = $c->config->{'var_path'}."/jobs/".$id; + my $user = Thruk::Utils::IO::saferead($dir."/user"); + if(!$nouser && defined $user) { chomp($user); confess('no remote_user') unless defined $c->stash->{'remote_user'}; return unless $user eq $c->stash->{'remote_user'}; @@ -299,9 +302,9 @@ sub cancel { my($c, $id, $nouser) = @_; confess("got no id") unless $id; - my $dir = $c->config->{'var_path'}."/jobs/".$id; - if(!$nouser && -f $dir."/user" ) { - my $user = Thruk::Utils::IO::read($dir."/user"); + my $dir = $c->config->{'var_path'}."/jobs/".$id; + my $user = Thruk::Utils::IO::saferead($dir."/user"); + if(!$nouser && defined $user) { chomp($user); confess('no remote_user') unless defined $c->stash->{'remote_user'}; return unless $user eq $c->stash->{'remote_user'}; @@ -313,8 +316,9 @@ sub cancel { chomp($pid); # is it running on this node? - if(-s $dir."/hostname") { - my @hosts = Thruk::Utils::IO::read_as_list($dir."/hostname"); + my $hosts = Thruk::Utils::IO::saferead($dir."/hostname"); + if(defined $hosts) { + my @hosts = split(/\n/mx, $hosts); if($hosts[0] ne $Thruk::Globals::NODE_ID) { $c->cluster->run_cluster($hosts[0], 'Thruk::Utils::External::cancel', [$c, $id, $nouser]); return _is_running($c, $dir); @@ -353,8 +357,10 @@ sub read_job { my $job_dir = $c->config->{'var_path'}.'/jobs/'.$id; - my $start = -e $job_dir.'/start' ? (stat(_))[9] : 0; - my $end = -e $job_dir.'/rc' ? (stat(_))[9] : 0; + my @stat = Thruk::Utils::IO::stat($job_dir.'/start'); + my $start = $stat[9] || 0; + @stat = Thruk::Utils::IO::stat($job_dir.'/end'); + my $end = $stat[9] || 0; my $rc = Thruk::Utils::IO::saferead($job_dir.'/rc') // ''; # 0 is OK, everything else is an error (exit code) my $res = Thruk::Utils::IO::saferead($job_dir.'/perl_res') // ''; my $out = Thruk::Utils::IO::saferead($job_dir.'/stdout') // ''; @@ -440,8 +446,7 @@ sub get_status { my @end = Time::HiRes::stat($dir."/stdout"); $end[9] = time() unless defined $end[9]; $time = $end[9] - $start[9]; - } elsif(-f $dir."/status") { - $percent = Thruk::Utils::IO::read($dir."/status"); + } elsif($percent = Thruk::Utils::IO::read($dir."/status")) { chomp($percent); } @@ -452,7 +457,7 @@ sub get_status { chomp($forward) if defined $forward; my $show_output; - if(-f $dir."/show_output") { + if(defined Thruk::Utils::IO::saferead($dir."/show_output")) { $show_output = 1; } @@ -509,9 +514,9 @@ sub get_result { my($c, $id, $nouser) = @_; confess("got no id") unless $id; - my $dir = $c->config->{'var_path'}."/jobs/".$id; - if(!$nouser && -f $dir."/user") { - my $user = Thruk::Utils::IO::read($dir."/user"); + my $dir = $c->config->{'var_path'}."/jobs/".$id; + my $user = Thruk::Utils::IO::saferead($dir."/user"); + if(!$nouser && defined $user) { chomp($user); confess('no remote_user') unless defined $c->stash->{'remote_user'}; return unless $user eq $c->stash->{'remote_user'}; @@ -532,19 +537,19 @@ sub get_result { $err =~ s|^\s*\n||gmx; # dev ino mode nlink uid gid rdev size atime mtime ctime blksize blocks - my @start = Time::HiRes::stat($dir.'/start'); + my @start = Thruk::Utils::IO::stat($dir.'/start'); my @end; my $retries = 10; while($retries > 0) { - if(-f $dir."/killed") { - @end = Time::HiRes::stat($dir."/killed"); + if(Thruk::Utils::IO::file_exists($dir."/killed")) { + @end = Thruk::Utils::IO::stat($dir."/killed"); $killed = "job has been killed"; - } elsif(-f $dir."/stdout") { - @end = Time::HiRes::stat($dir."/stdout"); - } elsif(-f $dir."/stderr") { - @end = Time::HiRes::stat($dir."/stderr"); - } elsif(-f $dir."/rc") { - @end = Time::HiRes::stat($dir."/rc"); + } elsif(Thruk::Utils::IO::file_exists($dir."/stdout")) { + @end = Thruk::Utils::IO::stat($dir."/stdout"); + } elsif(Thruk::Utils::IO::file_exists($dir."/stderr")) { + @end = Thruk::Utils::IO::stat($dir."/stderr"); + } elsif(Thruk::Utils::IO::file_exists($dir."/rc")) { + @end = Thruk::Utils::IO::stat($dir."/rc"); } if(!defined $end[9]) { sleep(1); @@ -563,6 +568,7 @@ sub get_result { my $time = $end[9] - $start[9]; +# TODO: ... my $stash = -f $dir."/stash" ? Storable::retrieve($dir."/stash") : undef; my $rc = Thruk::Utils::IO::saferead($dir."/rc") // -1; @@ -572,29 +578,25 @@ sub get_result { chomp($perl_res) if defined $perl_res; my $profiles = []; - for my $p (glob($dir."/profile.log*")) { + for my $p (@{Thruk::Utils::IO::find_files($dir, "profile.log.*")}) { my $text = Thruk::Utils::IO::read($p); chomp($text); my $htmlfile = $p; $htmlfile =~ s/\.log\./.html./gmx; my $jsonfile = $p; $jsonfile =~ s/\.log\./.json./gmx; - my $totals; - if(-f $jsonfile) { - $totals = Thruk::Utils::IO::json_lock_retrieve($jsonfile); - } + my $totals = Thruk::Utils::IO::json_lock_retrieve($jsonfile); push @{$profiles}, { name => "Job ".$id, time => $end[9] // $start[9], - html => -e $htmlfile ? Thruk::Utils::IO::read($htmlfile) : undef, + html => Thruk::Utils::IO::saferead($htmlfile), text => $text, totals => $totals ? $totals->{'totals'} : undef, }; my $dbfile = $p; $dbfile =~ s/\.log\./.db./gmx; - if(-f $dbfile) { - push @{$profiles}, Thruk::Utils::IO::json_lock_retrieve($dbfile); - } + my $dbprofile = Thruk::Utils::IO::json_lock_retrieve($dbfile); + push @{$profiles}, $dbprofile if $dbprofile; } return($out,$err,$time,$dir,$stash,$rc,$profiles,$start[9],$end[9],$perl_res,$killed); @@ -801,7 +803,7 @@ sub save_profile { my $nr = 0; my $base = $dir.'/profile.log.'.$$; - while(-e $base.'.'.$nr) { + while(Thruk::Utils::IO::file_exists($base.'.'.$nr)) { $nr++; } my $file = $base.'.'.$nr; @@ -1039,37 +1041,50 @@ sub cleanup_job_folders { my($total, $removed) = (0, 0); my $max_age = time() - 3600; # keep them for one hour my $max_age_dead = time() - (86400*3); # clean broken jobs after 3 days - for my $olddir (glob($c->config->{'var_path'}."/jobs/*")) { + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}."/jobs"); + + my %dirs; + for my $f (@{$files}) { + my $d = Thruk::Base::dirname($f); + $dirs{$d} = 1; + } + + for my $olddir (sort keys %dirs) { $total++; - if(-f $olddir.'/rc') { - my @stat = stat($olddir.'/rc'); + my @stat = Thruk::Utils::IO::stat($olddir.'/rc'); + if($stat[9]) { if($stat[9] < $max_age) { remove_job_dir($olddir); $removed++; - if($verbose && -d $olddir.'/.') { + if($verbose && scalar @{Thruk::Utils::IO::find_files($olddir.'/')} > 0) { _warn("unable to remove job folder: %s", $olddir); } } + next; } - elsif(-f $olddir.'/start') { - my @stat = stat($olddir.'/start'); + + @stat = Thruk::Utils::IO::stat($olddir.'/start'); + if($stat[9]) { if($stat[9] < $max_age_dead) { remove_job_dir($olddir); $removed++; - if($verbose && -d $olddir.'/.') { + if($verbose && scalar @{Thruk::Utils::IO::find_files($olddir.'/')} > 0) { _warn("unable to remove job folder: %s", $olddir); } } + next; } - else { - my @stat = stat($olddir.'/'); + + @stat = Thruk::Utils::IO::stat($olddir.'/'); + if($stat[9]) { if($stat[9] && $stat[9] < $max_age_dead) { remove_job_dir($olddir); $removed++; - if($verbose && -d $olddir.'/.') { + if($verbose && scalar @{Thruk::Utils::IO::find_files($olddir.'/')} > 0) { _warn("unable to remove job folder: %s", $olddir); } } + next; } } @@ -1088,8 +1103,7 @@ remove job folder and all files =cut sub remove_job_dir { my($dir) = @_; - unlink(glob($dir."/*")); - rmdir($dir); + Thruk::Utils::IO::remove_folder($dir); return; } @@ -1112,8 +1126,9 @@ sub _is_running { $pid = Thruk::Utils::IO::untaint($pid); # fetch status from remote node - if(-s $dir."/hostname") { - my @hosts = Thruk::Utils::IO::read_as_list($dir."/hostname"); + my $hosts = Thruk::Utils::IO::saferead($dir."/hostname"); + if(defined $hosts) { + my @hosts = split(/\n/mx, $hosts); if($hosts[0] ne $Thruk::Globals::NODE_ID) { confess('clustered _is_running requires $c') unless $c; my $cluster = $c->cluster; @@ -1233,11 +1248,18 @@ sub _clean_unstorable_refs { ############################################## sub _reconnect { my($c) = @_; - return unless $c->db(); - $c->db->reconnect() or do { - _error("reconnect failed: %s", $@); - kill($$); - }; + + my $hdl = $Thruk::Utils::IO::var_db; + if($hdl && $hdl ne "-1") { + $hdl->reconnect(); + } + + if($c->db()) { + $c->db->reconnect() or do { + _error("reconnect failed: %s", $@); + kill($$); + }; + } return; } diff --git a/lib/Thruk/Utils/IO.pm b/lib/Thruk/Utils/IO.pm index 6bf4fa2e86..9e2c3ed6f2 100644 --- a/lib/Thruk/Utils/IO.pm +++ b/lib/Thruk/Utils/IO.pm @@ -13,22 +13,21 @@ IO Utilities Collection for Thruk use warnings; use strict; use Carp qw/confess longmess/; -use Cpanel::JSON::XS (); -use Cwd qw/abs_path/; -use Errno qw(EEXIST); -use Fcntl qw/:DEFAULT :flock :mode SEEK_SET/; -use File::Copy qw/move copy/; use IO::Select (); use IPC::Open3 qw/open3/; use POSIX ":sys_wait_h"; use Scalar::Util 'blessed'; -use Time::HiRes qw/sleep gettimeofday tv_interval/; +use Time::HiRes qw/gettimeofday tv_interval/; use Thruk::Base (); +use Thruk::Config 'noautoload'; use Thruk::Timer qw/timing_breakpoint/; +use Thruk::Utils::IO::LocalFS (); use Thruk::Utils::Log qw/:all/; $Thruk::Utils::IO::MAX_LOCK_RETRIES = 20; +$Thruk::Utils::IO::var_path = undef; +$Thruk::Utils::IO::var_db = undef; ############################################## eval { @@ -49,17 +48,8 @@ close filehandle and ensure permissions and ownership =cut sub close { - my($fh, $filename, $just_close) = @_; - my $t1 = [gettimeofday]; - my $rc = CORE::close($fh); - confess("cannot write to $filename: $!") unless $rc; - ensure_permissions('file', $filename) unless $just_close; - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return $rc; + my(undef, $filename) = @_; + return(handle_io("close", 1, $filename, \@_)); } ############################################## @@ -75,19 +65,10 @@ create folder and ensure permissions and ownership sub mkdir { my(@dirs) = @_; - my $t1 = [gettimeofday]; - for my $dirname (@dirs) { - if(!CORE::mkdir($dirname)) { - my $err = $!; - confess("failed to create ".$dirname.": ".$err) unless -d $dirname; - } - ensure_permissions('dir', $dirname); + handle_io("mkdir", 0, $dirname, [$dirname]); } - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; return 1; } @@ -103,14 +84,7 @@ create folder recursive sub mkdir_r { for my $dirname (@_) { - $dirname =~ s|\/\.?$||gmx; - next if -d $dirname.'/.'; - my $path = ''; - for my $part (split/(\/)/mx, $dirname) { - $path .= $part; - next if $path eq ''; - &mkdir($path) unless -d $path.'/.'; - } + handle_io("mkdir_r", 0, $dirname, [$dirname]); } return 1; } @@ -127,17 +101,7 @@ read file and return content sub read { my($path) = @_; - my $t1 = [gettimeofday]; - - open(my $fh, '<', $path) || die "Can't open file ".$path.": ".$!; - local $/ = undef; - my $content = <$fh>; - CORE::close($fh); - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - return($content); + return(handle_io("read", 0, $path, \@_)); } ############################################## @@ -151,8 +115,8 @@ read file and return decoded content =cut sub read_decoded { - require Thruk::Utils::Encode; - return Thruk::Utils::Encode::decode_any(&read(@_)); + my($path) = @_; + return(handle_io("read_decoded", 0, $path, \@_)); } ############################################## @@ -167,18 +131,7 @@ read file and return content or undef in case it cannot be read sub saferead { my($path) = @_; - my $t1 = [gettimeofday]; - - open(my $fh, '<', $path) || return; - local $/ = undef; - my $content = <$fh>; - CORE::close($fh); - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return($content); + return(handle_io("saferead", 0, $path, \@_)); } ############################################## @@ -192,8 +145,8 @@ safe read file and return decoded content =cut sub saferead_decoded { - require Thruk::Utils::Encode; - return Thruk::Utils::Encode::decode_any(&saferead(@_)); + my($path) = @_; + return(handle_io("saferead_decoded", 0, $path, \@_)); } ############################################## @@ -208,18 +161,7 @@ read file and return content as array sub read_as_list { my($path) = @_; - my $t1 = [gettimeofday]; - - my @res; - open(my $fh, '<', $path) || die "Can't open file ".$path.": ".$!; - chomp(@res = <$fh>); - CORE::close($fh); - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return(@res); + return(handle_io("read_as_list", 0, $path, \@_)); } ############################################## @@ -234,18 +176,7 @@ read file and return content as array, return empty list if open fails sub saferead_as_list { my($path) = @_; - my $t1 = [gettimeofday]; - - my @res; - open(my $fh, '<', $path) || return(@res); - chomp(@res = <$fh>); - CORE::close($fh); - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return(@res); + return(handle_io("saferead_as_list", 0, $path, \@_)); } ############################################## @@ -259,246 +190,131 @@ creates file and ensure permissions =cut sub write { - my($path,$content,$mtime,$append) = @_; - my $t1 = [gettimeofday]; + my($path) = @_; + return(handle_io("write", 0, $path, \@_)); +} - my $mode = $append ? '>>' : '>'; - open(my $fh, $mode, $path) or confess('cannot create file '.$path.': '.$!); - print $fh $content; - &close($fh, $path) or confess("cannot close file ".$path.": ".$!); - if(Time::HiRes->can('utime')) { - Time::HiRes::utime($mtime, $mtime, $path) if $mtime; - } else { - utime($mtime, $mtime, $path) if $mtime; - } +############################################## - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; +=head2 unlink - return 1; + unlink($path) + +remove file + +=cut + +sub unlink { + my(@paths) = @_; + for my $p (@paths) { + handle_io("unlink", 0, $p, [$p]); + } + return; } ############################################## -=head2 ensure_permissions +=head2 file_exists - ensure_permissions($mode, $path) + file_exists($path) -ensure permissions and ownership +returns true if the file exists =cut -sub ensure_permissions { - my($mode, $path) = @_; - return if defined $ENV{'THRUK_NO_TOUCH_PERM'}; +sub file_exists { + my($path) = @_; + return(handle_io("file_exists", 0, $path, \@_)); +} - require Thruk::Config; - my $config = Thruk::Config::get_config(); +############################################## - confess("need a path") unless defined $path; - return unless -e $path; +=head2 file_not_empty - my @stat = stat(_); - my $cur = sprintf "%04o", S_IMODE($stat[2]); + file_not_empty($path) - # set modes - if($mode eq 'file') { - if($cur ne $config->{'mode_file'}) { - chmod(oct($config->{'mode_file'}), $path) || _warn("failed to ensure permissions (0660/$cur) with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); - } - } - elsif($mode eq 'dir') { - if($cur ne $config->{'mode_dir'}) { - chmod(oct($config->{'mode_dir'}), $path) || _warn("failed to ensure permissions (0770/$cur) with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); - } - } - else { - chmod($mode, $path) || _warn("failed to ensure permissions (".$mode.") with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); - } +returns true if the file exists and is not empty - # change owner too if we are root - my $uid = -1; - if($> == 0) { - $uid = $ENV{'THRUK_USER_ID'} or confess('no user id!'); - } +=cut - # change group - chown($uid, $ENV{'THRUK_GROUP_ID'}, $path) if defined $ENV{'THRUK_GROUP_ID'}; - return; +sub file_not_empty { + my($path) = @_; + return(handle_io("file_not_empty", 0, $path, \@_)); } ############################################## -=head2 file_rlock +=head2 stat - file_rlock($file) + stat($path) -locks given file in shared / readonly mode. Returns filehandle. +returns stat of file =cut -sub file_rlock { - my($file) = @_; - confess("no file") unless $file; - my $t1 = [gettimeofday]; - alarm(10); - local $SIG{'ALRM'} = sub { confess("timeout while trying to shared flock: ".$file."\n"._fuser($file)); }; +sub stat { + my($path) = @_; + return(handle_io("stat", 0, $path, \@_)); +} - my $fh; - my $retrys = 0; - my $err; - while($retrys < 3) { - undef $fh; - eval { - alarm(10); - sysopen($fh, $file, O_RDONLY) or confess("cannot open file ".$file.": ".$!); - flock($fh, LOCK_SH) or confess 'Cannot lock_sh '.$file.': '.$!; - }; - $err = $@; - alarm(0); - if(!$err && $fh) { - last; - } - $retrys++; - sleep(0.5); - } - alarm(0); +############################################## - if($err) { - die("failed to shared flock $file: $err"); - } +=head2 rmdir - if($retrys > 0) { - _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; - } + rmdir($path) - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_lock'} += $elapsed if $c; +remove empty folder + +=cut - return($fh); +sub rmdir { + my($path) = @_; + return(handle_io("rmdir", 0, $path, \@_)); } ############################################## -=head2 file_lock +=head2 ensure_permissions - file_lock($file) + ensure_permissions($mode, $path) -locks given file in read/write mode. Returns locked filehandle and lock file handle. +ensure permissions and ownership =cut -sub file_lock { - my($file, $mode) = @_; - confess("no file") unless $file; - if($mode && $mode eq 'sh') { return file_rlock($file); } +sub ensure_permissions { + my(undef, $path) = @_; + return if defined $ENV{'THRUK_NO_TOUCH_PERM'}; + return(handle_io("ensure_permissions", 1, $path, \@_)); +} - my $t1 = [gettimeofday]; - alarm(20); - local $SIG{'ALRM'} = sub { confess("timeout while trying to excl. flock: ".$file."\n"._fuser($file)); }; - - # we can only lock files in existing folders - my $basename = $file; - if($basename !~ m|^[\.\/]|mx) { $basename = './'.$basename; } - $basename =~ s%/[^/]*$%%gmx; - if(!-d $basename.'/.') { - require Thruk::Config; - my $config = Thruk::Config::get_config(); - my $match = sprintf("^(\Q%s\E|\Q%s\E)", $config->{'var_path'}, $config->{'tmp_path'}); - if($basename =~ m/$match/mx) { - mkdir_r($basename); - } else { - confess("cannot lock $file in non-existing folder: ".$!); - } - } +############################################## - my $lock_file = $file.'.lock'; - my $lock_fh; - my $locked = 0; - my $old_inode = (stat($lock_file))[1]; - my $retrys = 0; - while(1) { - $old_inode = (stat($lock_file))[1] unless $old_inode; - if(sysopen($lock_fh, $lock_file, O_RDWR|O_EXCL|O_CREAT, 0660)) { - last; - } - # check for orphaned locks - if($!{EEXIST} && $old_inode) { - sleep(0.3); - if(sysopen($lock_fh, $lock_file, O_RDWR, 0660) && flock($lock_fh, LOCK_EX|LOCK_NB)) { - my $new_inode = (stat($lock_fh))[1]; - if($new_inode && $new_inode == $old_inode) { - $retrys++; - if($retrys > $Thruk::Utils::IO::MAX_LOCK_RETRIES) { - # lock seems to be orphaned, continue normally unless in test mode - confess("got orphaned lock") if $ENV{'TEST_RACE'}; - $locked = 1; - _warn("recovered orphaned lock for ".$file) unless $ENV{'TEST_IO_NOWARNINGS'}; - last; - } - next; - } - if($new_inode && $new_inode != $old_inode) { - $retrys = 0; - undef $old_inode; - } - } else { - $retrys++; - if($retrys > $Thruk::Utils::IO::MAX_LOCK_RETRIES) { - unlink($lock_file); - # we have to move and copy the file itself, otherwise - # the orphaned process may overwrite the file - # and the later flock() might hang again - copy($file, $file.'.copy') or confess("cannot copy file $file: $!"); - move($file, $file.'.orphaned') or confess("cannot move file $file to .orphaned: $!"); - move($file.'.copy', $file) or confess("cannot move file ".$file.".copy: $!"); - unlink($file.'.orphaned'); - _warn("removed orphaned lock for ".$file) unless $ENV{'TEST_IO_NOWARNINGS'}; - $retrys = 0; # start over... - } - } - } - sleep(0.1); - } - if(!$locked) { - flock($lock_fh, LOCK_EX) || confess('Cannot lock_ex '.$lock_file.': '.$!."\n"._fuser($lock_file)); - } +=head2 file_rlock - my $fh; - $retrys = 0; - my $err; - while($retrys < 3) { - alarm(10); - undef $fh; - eval { - sysopen($fh, $file, O_RDWR|O_CREAT) || confess("cannot open file ".$file.": ".$!); - flock($fh, LOCK_EX) || confess('Cannot lock_ex '.$file.': '.$!."\n"._fuser($file)); - }; - $err = $@; - alarm(0); - if(!$err && $fh) { - last; - } - $retrys++; - sleep(0.5); - } - alarm(0); + file_rlock($file) - if($err) { - die("failed to lock $file: $err"); - } +locks given file in shared / readonly mode. Returns filehandle. - if($retrys > 0) { - _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; - } +=cut +sub file_rlock { + my($file) = @_; + return(handle_io("file_rlock", 0, $file, \@_)); +} - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_lock'} += $elapsed if $c; +############################################## - return($fh, $lock_fh); +=head2 file_lock + + file_lock($file) + +locks given file in read/write mode. Returns locked filehandle and lock file handle. + +=cut + +sub file_lock { + my($file) = @_; + return(handle_io("file_lock", 0, $file, \@_)); } ############################################## @@ -512,11 +328,8 @@ unlocks file lock previously with file_lock exclusivly. Returns nothing. =cut sub file_unlock { - my($file, $fh, $lock_fh) = @_; - flock($fh, LOCK_UN) if $fh; - unlink($file.'.lock'); - flock($lock_fh, LOCK_UN); - return; + my($file) = @_; + return(handle_io("file_unlock", 0, $file, \@_)); } ############################################## @@ -540,64 +353,8 @@ $options can be { =cut sub json_store { - my($file, $data, $options) = @_; - - if(defined $options && ref $options ne 'HASH') { - confess("json_store options have been changed to hash."); - } - - if($options->{'skip_config'}) { - $options->{'skip_ensure_permissions'} = 1; - $options->{'skip_validate'} = 1; - } - - my $json = Cpanel::JSON::XS->new->utf8; - $json = $json->pretty if $options->{'pretty'}; - $json = $json->canonical; # keys will be randomly ordered otherwise - $json = $json->convert_blessed; - - my $write_out; - if($options->{'changed_only'}) { - $write_out = $json->encode($data); - if(defined $options->{'compare_data'}) { - return 1 if $options->{'compare_data'} eq $write_out; - } - elsif(-f $file) { - my $old = &read($file); - return 1 if $old eq $write_out; - } - } - - my $t1 = [gettimeofday]; - - my $tmpfile = $options->{'tmpfile'} // $file.'.new'; - open(my $fh, '>', $tmpfile) or confess('cannot write file '.$tmpfile.': '.$!); - print $fh ($write_out || $json->encode($data)) or confess('cannot write file '.$tmpfile.': '.$!); - if($options->{'skip_ensure_permissions'}) { - CORE::close($fh) || confess("cannot close file ".$tmpfile.": ".$!); - } else { - &close($fh, $tmpfile) || confess("cannot close file ".$tmpfile.": ".$!); - } - - if(!$options->{'skip_validate'}) { - require Thruk::Config; - my $config = Thruk::Config::get_config(); - if($config->{'thruk_author'}) { - eval { - my $test = $json->decode(&read($tmpfile)); - }; - confess("json_store failed to write a valid file $tmpfile: ".$@) if $@; - } - } - - - move($tmpfile, $file) or confess("cannot replace $file with $tmpfile: $!"); - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return 1; + my($file) = @_; + return(handle_io("json_store", 0, $file, \@_)); } ############################################## @@ -611,16 +368,8 @@ stores data json encoded. options are passed to json_store. =cut sub json_lock_store { - my($file, $data, $options) = @_; - my($fh, $lock_fh); - eval { - ($fh, $lock_fh) = file_lock($file); - json_store($file, $data, $options); - }; - my $err = $@; - file_unlock($file, $fh, $lock_fh) if($fh || $lock_fh); - confess($err) if $err; - return 1; + my($file) = @_; + return(handle_io("json_lock_store", 0, $file, \@_)); } ############################################## @@ -634,44 +383,8 @@ retrieve json data =cut sub json_retrieve { - my($file, $fh, $lock_fh) = @_; - confess("got no filehandle") unless defined $fh; - - our $jsonreader; - if(!$jsonreader) { - $jsonreader = Cpanel::JSON::XS->new->utf8; - $jsonreader->relaxed(); - } - - my $t1 = [gettimeofday]; - - seek($fh, 0, SEEK_SET) or die "Cannot seek ".$file.": $!\n"; - - my $data; - my $content; - eval { - local $/ = undef; - $content = scalar <$fh>; - $data = $jsonreader->decode($content); - }; - my $err = $@; - if($err) { - # try to unlock - flock($fh, LOCK_UN); - if($lock_fh) { - eval { - file_unlock($file, $fh, $lock_fh); - }; - } - confess("error while reading $file: ".$err); - } - - my $elapsed = tv_interval($t1); - my $c = $Thruk::Globals::c || undef; - $c->stash->{'total_io_time'} += $elapsed if $c; - - return($data, $content) if wantarray; - return $data; + my($file) = @_; + return(handle_io("json_retrieve", 0, $file, \@_)); } ############################################## @@ -686,18 +399,7 @@ retrieve json data sub json_lock_retrieve { my($file) = @_; - return unless -s $file; - my($data, $fh); - eval { - $fh = file_rlock($file); - $data = json_retrieve($file, $fh); - CORE::close($fh) or die("cannot close file ".$file.": ".$!); - undef $fh; # closing the file removes the lock - }; - my $err = $@; - flock($fh, LOCK_UN) if $fh; - confess($err) if $err; - return $data; + return(handle_io("json_lock_retrieve", 0, $file, \@_)); } ############################################## @@ -711,17 +413,8 @@ update json data with locking. options are passed to json_store. =cut sub json_lock_patch { - my($file, $patch_data, $options) = @_; - my($fh, $lock_fh, $data); - eval { - ($fh, $lock_fh) = file_lock($file); - $options->{'lock_fh'} = $lock_fh; - $data = json_patch($file, $fh, $patch_data, $options); - }; - my $err = $@; - file_unlock($file, $fh, $lock_fh) if($fh || $lock_fh); - confess($err) if $err; - return $data; + my($file) = @_; + return(handle_io("json_lock_patch", 0, $file, \@_)); } ############################################## @@ -735,25 +428,191 @@ update json data. options are passed to json_store. =cut sub json_patch { - my($file, $fh, $patch_data, $options) = @_; - if(defined $options && ref $options ne 'HASH') { - confess("json_store options have been changed to hash."); + my($file) = @_; + return(handle_io("json_patch", 0, $file, \@_)); +} + +######################################## + +=head2 touch + + touch($file) + +create file if not exists and update timestamp + +=cut +sub touch { + my($file) = @_; + return(handle_io("touch", 0, $file, \@_)); +} + +################################################### + +=head2 find_files + + find_files($folder, $pattern, $skip_symlinks) + +return list of files for folder and pattern (symlinks will optionally be skipped) + +=cut + +sub find_files { + my($dir) = @_; + my $files = handle_io("find_files", 0, $dir, \@_); + if($Thruk::Utils::IO::var_path) { + my $var_path = $Thruk::Utils::IO::var_path; + $files = [map { my $f = $_; $f =~ s=^VAR::=$var_path=mx; $f; } @{$files}]; } - confess("got no filehandle") unless defined $fh; - my($data, $content); - if(-s $file) { - ($data, $content) = json_retrieve($file, $fh, $options->{'lock_fh'}); - } else { - if(!$options->{'allow_empty'}) { - confess("attempt to patch empty file without allow_empty option: $file"); + return($files); +} + +################################################### + +=head2 remove_folder + + remove_folder($folder) + +recursively remove folder and all files + +=cut +sub remove_folder { + my($dir) = @_; + handle_io("remove_folder", 0, $dir, \@_); + return; +} + +################################################### + +=head2 handle_io + + handle_io($method, $idx, $path, $args) + +wrapper to io functions + +=cut +sub handle_io { + my($method, $idx, $path, $args) = @_; + if($Thruk::Utils::IO::var_path && !$ENV{'THRUK_FORCE_LOCAL_VAR_PATH'}) { + my $var_path = $Thruk::Utils::IO::var_path; + if($path !~ m=/local/=mx && $path =~ s=^$var_path=VAR::=mx) { + my $hdl = ($Thruk::Utils::IO::var_db //= _init_var_db()); + if($hdl && $hdl ne "-1") { + my @arg_copy = @{$args}; # required to not override source references + $arg_copy[$idx] = $path; + return($hdl->$method(@arg_copy)); + } } - ($data, $content) = ({}, ""); } - $data = merge_deep($data, $patch_data); - $options->{'changed_only'} = 1; - $options->{'compare_data'} = $content; - json_store($file, $data, $options); - return $data; + my $f = \&{"Thruk::Utils::IO::LocalFS::".$method}; + return(&{$f}(@{$args})); +} + +######################################## +sub _init_var_db { + my $config = Thruk::Config::get_config(); + return unless $config; + return -1 unless $config->{'var_path_db'}; + if(!defined $Thruk::Utils::IO::var_db) { + if($config->{'var_path_db'} =~ m|^mysql://|mx) { + require Thruk::Utils::IO::Mysql; + $Thruk::Utils::IO::var_db = Thruk::Utils::IO::Mysql->new($config->{'var_path_db'}); + } else { + die("unknown var_path_db type"); + } + } + return $Thruk::Utils::IO::var_db; +} + +############################################## + +=head2 sync_db_fs + + sync_db_fs($c, $from, $to, $opts) + +sync files to database or back + +=cut +sub sync_db_fs { + my($c, $from, $to, $opts) = @_; + + if(!$from || !$to) { + die("usage: filesystem sync "); + } + + my $action; + $action = 'export' if $from eq 'db'; + $action = 'import' if $from eq 'fs'; + if(!$action) { + die("usage: filesystem sync "); + } + if($action eq 'import' && $to ne 'db') { + die("usage: filesystem sync "); + } + if($action eq 'export' && $to ne 'fs') { + die("usage: filesystem sync "); + } + + local $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'} = 1 if $from eq 'fs'; + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}); + delete $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'}; + + for my $file (sort @{$files}) { + _debugs("writing %s %s:", $to eq 'db' ? 'to db' : 'to local fs', $file); + local $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'} = 1 if $from eq 'fs'; + my @stat = Thruk::Utils::IO::stat($file); + my $content = Thruk::Utils::IO::saferead($file); + delete $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'}; + + local $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'} = 1 if $to eq 'fs'; + my $dir = Thruk::Base::dirname($file); + Thruk::Utils::IO::mkdir_r($dir); + Thruk::Utils::IO::write($file, $content, $stat[9]); + _debug(" OK"); + delete $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'}; + } + + # remove all files which do not exist on source + if($opts->{'delete'}) { + local $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'} = 1 if $to eq 'fs'; + my $destfiles = Thruk::Utils::IO::find_files($c->config->{'var_path'}); + my $existing = Thruk::Base::array2hash($files); + + for my $file (sort @{$destfiles}) { + _debug("removing %s:", $file); + Thruk::Utils::IO::unlink($file) if !defined $existing->{$file}; + } + delete $ENV{'THRUK_FORCE_LOCAL_VAR_PATH'}; + } + + return; +} + +######################################## + +=head2 realpath + + realpath($file) + +return realpath of this file + +=cut +sub realpath { + return(Thruk::Utils::IO::LocalFS::realpath(@_)); +} + +######################################## + +=head2 untaint + + untaint($var) + +return untainted variable + +=cut +sub untaint { + my($v) = @_; + if($v && $v =~ /\A(.*)\z/msx) { $v = $1; } + return($v); } ############################################## @@ -1005,50 +864,6 @@ sub _cmd_old { # end REMOVE... } -######################################## - -=head2 untaint - - untaint($var) - -return untainted variable - -=cut -sub untaint { - my($v) = @_; - if($v && $v =~ /\A(.*)\z/msx) { $v = $1; } - return($v); -} - -######################################## - -=head2 realpath - - realpath($file) - -return realpath of this file - -=cut -sub realpath { - my($file) = @_; - return(abs_path($file)); -} - -######################################## - -=head2 touch - - touch($file) - -create file if not exists and update timestamp - -=cut -sub touch { - my($file) = @_; - &write($file, "", Time::HiRes::time(), 1); - return; -} - ############################################## =head2 merge_deep @@ -1166,60 +981,6 @@ sub get_memory_usage { return($rsize); } -################################################### - -=head2 find_files - - find_files($folder, $pattern, $skip_symlinks) - -return list of files for folder and pattern (symlinks will be skipped) - -=cut - -sub find_files { - my($dir, $match, $skip_symlinks) = @_; - my @files; - $dir =~ s/\/$//gmxo; - - # symlinks - if($skip_symlinks && -l $dir) { - return([]); - } - # not a directory? - if(!-d $dir."/.") { - if(defined $match) { - return([]) unless $dir =~ m/$match/mx; - } - return([$dir]); - } - - my @tmpfiles; - opendir(my $dh, $dir."/.") or confess("cannot open directory $dir: $!"); - while(my $file = readdir $dh) { - next if $file eq '.'; - next if $file eq '..'; - push @tmpfiles, $file; - } - closedir $dh; - - for my $file (@tmpfiles) { - # follow sub directories - if(-d sprintf("%s/%s/.", $dir, $file)) { - push @files, @{find_files($dir."/".$file, $match, $skip_symlinks)}; - } else { - # if its a file, make sure it matches our pattern - if(defined $match) { - my $test = $dir."/".$file; - next unless $test =~ m/$match/mx; - } - - push @files, $dir."/".$file; - } - } - - return \@files; -} - ############################################## =head2 all_perl_files @@ -1239,7 +1000,7 @@ sub all_perl_files { push @files, $file; next; } - my $content = &read($file); + my $content = &saferead($file) // ''; if($content =~ m%\#\!(/usr|)/bin/perl%mx || $content =~ m|\Qexec perl -x\E|mx) { push @files, $file; diff --git a/lib/Thruk/Utils/IO/LocalFS.pm b/lib/Thruk/Utils/IO/LocalFS.pm new file mode 100644 index 0000000000..29a9861647 --- /dev/null +++ b/lib/Thruk/Utils/IO/LocalFS.pm @@ -0,0 +1,936 @@ +package Thruk::Utils::IO::LocalFS; + +=head1 NAME + +Thruk::Utils::IO::LocalFS - Store files in local filesystem + +=head1 DESCRIPTION + +Store files in local filesystem + +=cut + +use warnings; +use strict; +use Carp qw/confess/; +use Cpanel::JSON::XS (); +use Cwd qw/abs_path/; +use Errno qw(EEXIST); +use Fcntl qw/:DEFAULT :flock :mode SEEK_SET/; +use File::Copy qw/move copy/; +use Time::HiRes qw/sleep gettimeofday tv_interval/; + +use Thruk::Utils::IO (); +use Thruk::Utils::Log qw/:all/; + +############################################## +=head1 METHODS + +=head2 close + + close($fh, $filename, $just_close) + +close filehandle and ensure permissions and ownership + +=cut +sub close { + my($fh, $filename, $just_close) = @_; + my $t1 = [gettimeofday]; + my $rc = CORE::close($fh); + confess("cannot write to $filename: $!") unless $rc; + ensure_permissions('file', $filename) unless $just_close; + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return $rc; +} + +############################################## + +=head2 mkdir + + mkdir($dirname) + +create folder and ensure permissions and ownership + +=cut +sub mkdir { + my(@dirs) = @_; + + my $t1 = [gettimeofday]; + + for my $dirname (@dirs) { + if(!CORE::mkdir($dirname)) { + my $err = $!; + confess("failed to create ".$dirname.": ".$err) unless -d $dirname; + } + ensure_permissions('dir', $dirname); + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return 1; +} + +############################################## + +=head2 mkdir_r + + mkdir_r($dirname) + +create folder recursive + +=cut +sub mkdir_r { + for my $dirname (@_) { + $dirname =~ s|\/\.?$||gmx; + next if -d $dirname.'/.'; + my $path = ''; + for my $part (split/(\/)/mx, $dirname) { + $path .= $part; + next if $path eq ''; + &mkdir($path) unless -d $path.'/.'; + } + } + return 1; +} + +############################################## + +=head2 read + + read($path) + +read file and return content + +=cut +sub read { + my($path) = @_; + my $t1 = [gettimeofday]; + + open(my $fh, '<', $path) || die "Can't open file ".$path.": ".$!; + local $/ = undef; + my $content = <$fh>; + CORE::close($fh); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return($content); +} + +############################################## + +=head2 read_decoded + + read_decoded($path) + +read file and return decoded content + +=cut +sub read_decoded { + require Thruk::Utils::Encode; + return Thruk::Utils::Encode::decode_any(&read(@_)); +} + +############################################## + +=head2 saferead + + saferead($path) + +read file and return content or undef in case it cannot be read + +=cut +sub saferead { + my($path) = @_; + my $t1 = [gettimeofday]; + + open(my $fh, '<', $path) || return; + local $/ = undef; + my $content = <$fh>; + CORE::close($fh); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return($content); +} + +############################################## + +=head2 saferead_decoded + + saferead_decoded($path) + +safe read file and return decoded content + +=cut +sub saferead_decoded { + require Thruk::Utils::Encode; + return Thruk::Utils::Encode::decode_any(&saferead(@_)); +} + +############################################## + +=head2 read_as_list + + read_as_list($path) + +read file and return content as array + +=cut +sub read_as_list { + my($path) = @_; + my $t1 = [gettimeofday]; + + my @res; + open(my $fh, '<', $path) || die "Can't open file ".$path.": ".$!; + chomp(@res = <$fh>); + CORE::close($fh); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return(@res); +} + +############################################## + +=head2 saferead_as_list + + saferead_as_list($path) + +read file and return content as array, return empty list if open fails + +=cut +sub saferead_as_list { + my($path) = @_; + my $t1 = [gettimeofday]; + + my @res; + open(my $fh, '<', $path) || return(@res); + chomp(@res = <$fh>); + CORE::close($fh); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return(@res); +} + +############################################## + +=head2 write + + write($path, $content, [ $mtime ], [ $append ]) + +creates file and ensure permissions + +=cut +sub write { + my($path,$content,$mtime,$append) = @_; + my $t1 = [gettimeofday]; + + my $mode = $append ? '>>' : '>'; + open(my $fh, $mode, $path) or confess('cannot create file '.$path.': '.$!); + print $fh $content; + &close($fh, $path) or confess("cannot close file ".$path.": ".$!); + if(Time::HiRes->can('utime')) { + Time::HiRes::utime($mtime, $mtime, $path) if $mtime; + } else { + utime($mtime, $mtime, $path) if $mtime; + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 unlink + + unlink($path) + +remove file + +=cut +sub unlink { + my($path) = @_; + my $t1 = [gettimeofday]; + + CORE::unlink($path); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 ensure_permissions + + ensure_permissions($mode, $path) + +ensure permissions and ownership + +=cut +sub ensure_permissions { + my($mode, $path) = @_; + return if defined $ENV{'THRUK_NO_TOUCH_PERM'}; + + require Thruk::Config; + my $config = Thruk::Config::get_config(); + + confess("need a path") unless defined $path; + return unless -e $path; + + my @stat = CORE::stat(_); + my $cur = sprintf "%04o", S_IMODE($stat[2]); + + # set modes + if($mode eq 'file') { + if($cur ne $config->{'mode_file'}) { + chmod(oct($config->{'mode_file'}), $path) || _warn("failed to ensure permissions (0660/$cur) with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); + } + } + elsif($mode eq 'dir') { + if($cur ne $config->{'mode_dir'}) { + chmod(oct($config->{'mode_dir'}), $path) || _warn("failed to ensure permissions (0770/$cur) with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); + } + } + else { + chmod($mode, $path) || _warn("failed to ensure permissions (".$mode.") with uid: ".$>." - ".$<." for ".$path.": ".$!."\n".`ls -dn '$path'`); + } + + # change owner too if we are root + my $uid = -1; + if($> == 0) { + $uid = $ENV{'THRUK_USER_ID'} or confess('no user id!'); + } + + # change group + chown($uid, $ENV{'THRUK_GROUP_ID'}, $path) if defined $ENV{'THRUK_GROUP_ID'}; + return; +} + +############################################## + +=head2 file_rlock + + file_rlock($file) + +locks given file in shared / readonly mode. Returns filehandle. + +=cut +sub file_rlock { + my($file) = @_; + confess("no file") unless $file; + my $t1 = [gettimeofday]; + + alarm(10); + local $SIG{'ALRM'} = sub { confess("timeout while trying to shared flock: ".$file."\n"._fuser($file)); }; + + my $fh; + my $retrys = 0; + my $err; + while($retrys < 3) { + undef $fh; + eval { + alarm(10); + sysopen($fh, $file, O_RDONLY) or confess("cannot open file ".$file.": ".$!); + flock($fh, LOCK_SH) or confess 'Cannot lock_sh '.$file.': '.$!; + }; + $err = $@; + alarm(0); + if(!$err && $fh) { + last; + } + $retrys++; + sleep(0.5); + } + alarm(0); + + if($err) { + die("failed to shared flock $file: $err"); + } + + if($retrys > 0) { + _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_lock'} += $elapsed if $c; + + return($fh); +} + +############################################## + +=head2 file_lock + + file_lock($file) + +locks given file in read/write mode. Returns locked filehandle and lock file handle. + +=cut +sub file_lock { + my($file, $mode) = @_; + confess("no file") unless $file; + if($mode && $mode eq 'sh') { return file_rlock($file); } + + my $t1 = [gettimeofday]; + alarm(20); + local $SIG{'ALRM'} = sub { confess("timeout while trying to excl. flock: ".$file."\n"._fuser($file)); }; + + # we can only lock files in existing folders + my $basename = $file; + if($basename !~ m|^[\.\/]|mx) { $basename = './'.$basename; } + $basename =~ s%/[^/]*$%%gmx; + if(!-d $basename.'/.') { + require Thruk::Config; + my $config = Thruk::Config::get_config(); + my $match = sprintf("^(\Q%s\E|\Q%s\E)", $config->{'var_path'}, $config->{'tmp_path'}); + if($basename =~ m/$match/mx) { + mkdir_r($basename); + } else { + confess("cannot lock $file in non-existing folder: ".$!); + } + } + + my $lock_file = $file.'.lock'; + my $lock_fh; + my $locked = 0; + my $old_inode = (CORE::stat($lock_file))[1]; + my $retrys = 0; + while(1) { + $old_inode = (CORE::stat($lock_file))[1] unless $old_inode; + if(sysopen($lock_fh, $lock_file, O_RDWR|O_EXCL|O_CREAT, 0660)) { + last; + } + # check for orphaned locks + if($!{EEXIST} && $old_inode) { + sleep(0.3); + if(sysopen($lock_fh, $lock_file, O_RDWR, 0660) && flock($lock_fh, LOCK_EX|LOCK_NB)) { + my $new_inode = (CORE::stat($lock_fh))[1]; + if($new_inode && $new_inode == $old_inode) { + $retrys++; + if($retrys > $Thruk::Utils::IO::MAX_LOCK_RETRIES) { + # lock seems to be orphaned, continue normally unless in test mode + confess("got orphaned lock") if $ENV{'TEST_RACE'}; + $locked = 1; + _warn("recovered orphaned lock for ".$file) unless $ENV{'TEST_IO_NOWARNINGS'}; + last; + } + next; + } + if($new_inode && $new_inode != $old_inode) { + $retrys = 0; + undef $old_inode; + } + } else { + $retrys++; + if($retrys > $Thruk::Utils::IO::MAX_LOCK_RETRIES) { + CORE::unlink($lock_file); + # we have to move and copy the file itself, otherwise + # the orphaned process may overwrite the file + # and the later flock() might hang again + copy($file, $file.'.copy') or confess("cannot copy file $file: $!"); + move($file, $file.'.orphaned') or confess("cannot move file $file to .orphaned: $!"); + move($file.'.copy', $file) or confess("cannot move file ".$file.".copy: $!"); + CORE::unlink($file.'.orphaned'); + _warn("removed orphaned lock for ".$file) unless $ENV{'TEST_IO_NOWARNINGS'}; + $retrys = 0; # start over... + } + } + } + sleep(0.1); + } + if(!$locked) { + flock($lock_fh, LOCK_EX) || confess('Cannot lock_ex '.$lock_file.': '.$!."\n"._fuser($lock_file)); + } + + my $fh; + $retrys = 0; + my $err; + while($retrys < 3) { + alarm(10); + undef $fh; + eval { + sysopen($fh, $file, O_RDWR|O_CREAT) || confess("cannot open file ".$file.": ".$!); + flock($fh, LOCK_EX) || confess('Cannot lock_ex '.$file.': '.$!."\n"._fuser($file)); + }; + $err = $@; + alarm(0); + if(!$err && $fh) { + last; + } + $retrys++; + sleep(0.5); + } + alarm(0); + + if($err) { + die("failed to lock $file: $err"); + } + + if($retrys > 0) { + _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_lock'} += $elapsed if $c; + + return($fh, $lock_fh); +} + +############################################## + +=head2 file_unlock + + file_unlock($file, $fh, $lock_fh) + +unlocks file lock previously with file_lock exclusivly. Returns nothing. + +=cut +sub file_unlock { + my($file, $fh, $lock_fh) = @_; + flock($fh, LOCK_UN) if $fh; + CORE::unlink($file.'.lock'); + flock($lock_fh, LOCK_UN); + return; +} + +############################################## + +=head2 file_exists + + file_exists($path) + +returns true if the file exists + +=cut + +sub file_exists { + my($path) = @_; + my $t1 = [gettimeofday]; + + my $rc = -f $path; + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return $rc; +} + +############################################## + +=head2 file_not_empty + + file_not_empty($path) + +returns true if the file exists and is not empty + +=cut + +sub file_not_empty { + my($path) = @_; + my $t1 = [gettimeofday]; + + my $rc = -s $path; + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return $rc; +} + +############################################## + +=head2 stat + + stat($path) + +returns stat of file + +=cut + +sub stat { + my($path) = @_; + my $t1 = [gettimeofday]; + + my @rc = Time::HiRes::stat($path); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return(@rc); +} + +############################################## + +=head2 rmdir + + rmdir($path) + +remove empty folder + +=cut + +sub rmdir { + my($path) = @_; + return(rmdir($path)); +} + +############################################## + +=head2 json_store + + json_store($file, $data, $options) + +stores data json encoded + +$options can be { + pretty => 0/1, # don't write json into a single line and use human readable intendation + tmpfile => # use this tmpfile while writing new contents + changed_only => 0/1, # only write the file if it has changed + compare_data => "...", # use this string to compare for changed content + skip_ensure_permissions => 0/1 # skip running ensure_permissions after write + skip_validate => 0/1 # skip file validation (author only) + skip_config => 0/1 # skip all steps which reqire thruk config +} + +=cut +sub json_store { + my($file, $data, $options) = @_; + + if(defined $options && ref $options ne 'HASH') { + confess("json_store options have been changed to hash."); + } + + if($options->{'skip_config'}) { + $options->{'skip_ensure_permissions'} = 1; + $options->{'skip_validate'} = 1; + } + + my $json = Cpanel::JSON::XS->new->utf8; + $json = $json->pretty if $options->{'pretty'}; + $json = $json->canonical; # keys will be randomly ordered otherwise + $json = $json->convert_blessed; + + my $write_out; + if($options->{'changed_only'}) { + $write_out = $json->encode($data); + if(defined $options->{'compare_data'}) { + return 1 if $options->{'compare_data'} eq $write_out; + } + elsif(-f $file) { + my $old = &read($file); + return 1 if $old eq $write_out; + } + } + + my $t1 = [gettimeofday]; + + my $tmpfile = $options->{'tmpfile'} // $file.'.new'; + open(my $fh, '>', $tmpfile) or confess('cannot write file '.$tmpfile.': '.$!); + print $fh ($write_out || $json->encode($data)) or confess('cannot write file '.$tmpfile.': '.$!); + if($options->{'skip_ensure_permissions'}) { + CORE::close($fh) || confess("cannot close file ".$tmpfile.": ".$!); + } else { + &close($fh, $tmpfile) || confess("cannot close file ".$tmpfile.": ".$!); + } + + if(!$options->{'skip_validate'}) { + require Thruk::Config; + my $config = Thruk::Config::get_config(); + if($config->{'thruk_author'}) { + eval { + my $test = $json->decode(&read($tmpfile)); + }; + confess("json_store failed to write a valid file $tmpfile: ".$@) if $@; + } + } + + + move($tmpfile, $file) or confess("cannot replace $file with $tmpfile: $!"); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 json_lock_store + + json_lock_store($file, $data, [$options]) + +stores data json encoded. options are passed to json_store. + +=cut +sub json_lock_store { + my($file, $data, $options) = @_; + my($fh, $lock_fh); + eval { + ($fh, $lock_fh) = file_lock($file); + json_store($file, $data, $options); + }; + my $err = $@; + file_unlock($file, $fh, $lock_fh) if($fh || $lock_fh); + confess($err) if $err; + return 1; +} + +############################################## + +=head2 json_retrieve + + json_retrieve($file, $fh, [$lock_fh]) + +retrieve json data + +=cut +sub json_retrieve { + my($file, $fh, $lock_fh) = @_; + + our $jsonreader; + if(!$jsonreader) { + $jsonreader = Cpanel::JSON::XS->new->utf8; + $jsonreader->relaxed(); + } + + my $t1 = [gettimeofday]; + + my($data, $content, $err); + if(!$fh) { + eval { + $content = &saferead($file); + $data = $jsonreader->decode($content) if $content; + }; + $err = $@; + } else { + seek($fh, 0, SEEK_SET) or die "Cannot seek ".$file.": $!\n"; + eval { + local $/ = undef; + $content = scalar <$fh>; + $data = $jsonreader->decode($content); + }; + $err = $@; + } + if($err) { + # try to unlock + flock($fh, LOCK_UN); + if($lock_fh) { + eval { + file_unlock($file, $fh, $lock_fh); + }; + } + confess("error while reading $file: ".$err); + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return($data, $content) if wantarray; + return $data; +} + +############################################## + +=head2 json_lock_retrieve + + json_lock_retrieve($file) + +retrieve json data + +=cut +sub json_lock_retrieve { + my($file) = @_; + return unless -s $file; + my($data, $fh); + eval { + $fh = file_rlock($file); + $data = json_retrieve($file, $fh); + CORE::close($fh) or die("cannot close file ".$file.": ".$!); + undef $fh; # closing the file removes the lock + }; + my $err = $@; + flock($fh, LOCK_UN) if $fh; + confess($err) if $err; + return $data; +} + +############################################## + +=head2 json_lock_patch + + json_lock_patch($file, $patch_data, [$options]) + +update json data with locking. options are passed to json_store. + +=cut +sub json_lock_patch { + my($file, $patch_data, $options) = @_; + my($fh, $lock_fh, $data); + eval { + ($fh, $lock_fh) = file_lock($file); + $options->{'lock_fh'} = $lock_fh; + $data = json_patch($file, $fh, $patch_data, $options); + }; + my $err = $@; + file_unlock($file, $fh, $lock_fh) if($fh || $lock_fh); + confess($err) if $err; + return $data; +} + +############################################## + +=head2 json_patch + + json_patch($file, $fh, $patch_data, [$options]) + +update json data. options are passed to json_store. + +=cut +sub json_patch { + my($file, $fh, $patch_data, $options) = @_; + if(defined $options && ref $options ne 'HASH') { + confess("json_store options have been changed to hash."); + } + confess("got no filehandle") unless defined $fh; + my($data, $content); + if(-s $file) { + ($data, $content) = json_retrieve($file, $fh, $options->{'lock_fh'}); + } else { + if(!$options->{'allow_empty'}) { + confess("attempt to patch empty file without allow_empty option: $file"); + } + ($data, $content) = ({}, ""); + } + $data = Thruk::Utils::IO::merge_deep($data, $patch_data); + $options->{'changed_only'} = 1; + $options->{'compare_data'} = $content; + json_store($file, $data, $options); + return $data; +} + +######################################## + +=head2 touch + + touch($file) + +create file if not exists and update timestamp + +=cut +sub touch { + my($file) = @_; + &write($file, "", Time::HiRes::time(), 1); + return; +} + +################################################### + +=head2 find_files + + find_files($folder, $pattern, $skip_symlinks) + +return list of files for folder and pattern (symlinks will be skipped) + +=cut +sub find_files { + my($dir, $match, $skip_symlinks) = @_; + my @files; + $dir =~ s/\/$//gmxo; + + # symlinks + if($skip_symlinks && -l $dir) { + return([]); + } + # not a directory? + if(!-d $dir."/.") { + if(defined $match) { + return([]) unless $dir =~ m/$match/mx; + } + return([$dir]); + } + + my @tmpfiles; + opendir(my $dh, $dir."/.") or confess("cannot open directory $dir: $!"); + while(my $file = readdir $dh) { + next if $file eq '.'; + next if $file eq '..'; + push @tmpfiles, $file; + } + closedir $dh; + + for my $file (@tmpfiles) { + # follow sub directories +# TODO: remove? + if(-d sprintf("%s/%s/.", $dir, $file)) { + push @files, @{find_files($dir."/".$file, $match, $skip_symlinks)}; + } else { + # if its a file, make sure it matches our pattern + if(defined $match) { + my $test = $dir."/".$file; + next unless $test =~ m/$match/mx; + } + + push @files, $dir."/".$file; + } + } + + return \@files; +} + +################################################### + +=head2 remove_folder + + remove_folder($folder) + +recursively remove folder and all files + +=cut +sub remove_folder { + my($dir) = @_; + my @files = &find_files($dir); + CORE::unlink(@files); + CORE::rmdir($dir); + return; +} + +######################################## + +=head2 realpath + + realpath($file) + +return realpath of this file + +=cut +sub realpath { + my($file) = @_; + return(abs_path($file)); +} + +############################################## + +1; diff --git a/lib/Thruk/Utils/IO/Mysql.pm b/lib/Thruk/Utils/IO/Mysql.pm new file mode 100644 index 0000000000..56f87dfbee --- /dev/null +++ b/lib/Thruk/Utils/IO/Mysql.pm @@ -0,0 +1,873 @@ +package Thruk::Utils::IO::Mysql; + +=head1 NAME + +Thruk::Utils::IO::Mysql - Store files in a mysql database + +=head1 DESCRIPTION + +Store files in a mysql database + +=cut + +use warnings; +use strict; +use Carp qw/confess/; +use Cpanel::JSON::XS (); +use Errno (); +use Time::HiRes qw/gettimeofday tv_interval/; + +use Thruk::Backend::Provider::Mysql (); +use Thruk::Config 'noautoload'; +use Thruk::Utils::IO (); +use Thruk::Utils::Log qw/:all/; + +############################################## +# TODO: +# - check reports log files +# - check cron.log +# - check why mtime has no milliseconds +############################################## +=head1 METHODS + +=head2 new + + new($connection) + +=cut +sub new { + my($class, $connection) = @_; + my $hdl = Thruk::Backend::Provider::Mysql->new({ + options => { + peer_key => 'db_storage_driver', + peer => $connection, + }, + }); + + my $config = Thruk::Config::get_config(); + my $self = { + 'class' => $hdl, + 'use_locks' => $config->{'logcache_pxc_strict_mode'}, + }; + bless $self, $class; + $self->_create_tables_if_not_exist(); + return $self; +} + +############################################## + +=head2 reconnect + +recreate database connection + +=cut +sub reconnect { + my($self) = @_; + return($self->_disconnect()); +} + +############################################## + +=head2 _disconnect + +close database connection + +=cut +sub _disconnect { + my($self) = @_; + return($self->{'class'}->_disconnect()); +} + +############################################## + +=head2 _dbh + +try to connect to database and return database handle + +=cut +sub _dbh { + my($self) = @_; + return($self->{'class'}->_dbh); +} + +############################################## +# returns 1 if tables have been newly created or undef if already exist +sub _create_tables_if_not_exist { + my($self) = @_; + + return if $self->_tables_exist(); + + _debug2("creating tables"); + $self->_create_tables(); + + my $c = $Thruk::Globals::c; + Thruk::Utils::IO::sync_db_fs($c, 'fs', 'db') if $c; + return 1; +} + +############################################## +# returns 1 if tables exist, undef if not +sub _tables_exist { + my($self) = @_; + + # check if our tables exist + my $dbh = $self->_dbh; + my @tables = @{$dbh->selectcol_arrayref('SHOW TABLES LIKE "files"')}; + if(scalar @tables >= 1) { + return 1; + } + + return; +} + +############################################## +# returns 1 if tables exist, undef if not +sub _drop_tables { + my($self) = @_; + my $dbh = $self->_dbh; + $dbh->do("DROP TABLE IF EXISTS `files`") || confess $dbh->errstr; + return; +} + +############################################## +sub _create_tables { + my($self) = @_; + my $dbh = $self->_dbh; + my @statements = ( + "DROP TABLE IF EXISTS `files`", + "CREATE TABLE `files` ( + path varchar(255) NOT NULL, + mtime decimal(14,3) DEFAULT NULL, + permission varchar(5), + content LONGTEXT, + PRIMARY KEY (path) + ) DEFAULT CHARSET=utf8 COLLATE=utf8_bin", + ); + for my $stm (@statements) { + $dbh->do($stm) || confess $dbh->errstr; + } + $dbh->commit || confess $dbh->errstr; + return; +} + +############################################## + +=head2 close + +not implemented + +=cut +sub close { + return 0; +} + +############################################## + +=head2 mkdir + +not implemented + +=cut +sub mkdir { + return 1; +} + +############################################## + +=head2 mkdir_r + +not implemented + +=cut +sub mkdir_r { + return 1; +} + +############################################## + +=head2 read + + read($path) + +read file and return content + +=cut +sub read { + my($self, $path) = @_; + my $data = $self->saferead($path); + return($data) if defined $data; + die("no such file: ".$path); +} + +############################################## + +=head2 read_decoded + + read_decoded($path) + +read file and return decoded content + +=cut +sub read_decoded { + require Thruk::Utils::Encode; + return Thruk::Utils::Encode::decode_any(&read(@_)); +} + +############################################## + +=head2 saferead + + saferead($path) + +read file and return content or undef in case it cannot be read + +=cut +sub saferead { + my($self, $path) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + my $data = $dbh->selectcol_arrayref("SELECT content FROM files WHERE path = ".$dbh->quote($path)." LIMIT 1"); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return($data->[0]) if scalar @{$data} > 0; + return; +} + +############################################## + +=head2 saferead_decoded + + saferead_decoded($path) + +safe read file and return decoded content + +=cut +sub saferead_decoded { + require Thruk::Utils::Encode; + return Thruk::Utils::Encode::decode_any(&saferead(@_)); +} + +############################################## + +=head2 read_as_list + + read_as_list($path) + +read file and return content as array + +=cut +sub read_as_list { + my($self, $path) = @_; + + my $data = $self->read($path); + return(split/\n/mx, $data); +} + +############################################## + +=head2 saferead_as_list + + saferead_as_list($path) + +read file and return content as array, return empty list if open fails + +=cut +sub saferead_as_list { + my($self, $path) = @_; + my $data = $self->saferead($path); + return(split/\n/mx, $data); +} + +############################################## + +=head2 write + + write($path, $content, [ $mtime ], [ $append ]) + +creates file and ensure permissions + +=cut +sub write { + my($self,$path,$content,$mtime,$append) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + + if($mtime) { + $mtime = 0 + $mtime; + } else { + $mtime = Time::HiRes::time(); + } + + $content = $dbh->quote($content); + if($append) { + $dbh->do("INSERT INTO files (path,content,mtime)" + ." VALUES(".$dbh->quote($path).", ".$content.", ".$mtime.")" + ." ON DUPLICATE KEY UPDATE mtime=".$mtime.",content=CONCAT(content, ".$content.")"); + } else { + $dbh->do("INSERT INTO files (path,content,mtime)" + ." VALUES(".$dbh->quote($path).", ".$content.", ".$mtime.")" + ." ON DUPLICATE KEY UPDATE mtime=".$mtime.",content=".$content); + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 unlink + + unlink($path) + +remove file + +=cut +sub unlink { + my($self, $path) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + $dbh->do("DELETE FROM files WHERE path = ".$dbh->quote($path)); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 file_exists + + file_exists($path) + +returns true if the file exists + +=cut + +sub file_exists { + my($self, $path) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + my $data = $dbh->selectcol_arrayref("SELECT path FROM files WHERE path = ".$dbh->quote($path)." LIMIT 1"); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return(1) if scalar @{$data} > 0; + ## no critic + $! = &Errno::ENODATA; + ## use critic + return(0); +} + +############################################## + +=head2 file_not_empty + + file_not_empty($path) + +returns true if the file exists and is not empty + +=cut + +sub file_not_empty { + my($self, $path) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + my $data = $dbh->selectcol_arrayref("SELECT length(content) FROM files WHERE path = ".$dbh->quote($path)." AND content != '' LIMIT 1"); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + return($data->[0]) if scalar @{$data} > 0; + ## no critic + $! = &Errno::ENODATA; + ## use critic + return(0); +} + +############################################## + +=head2 stat + + stat($path) + +returns (incomolete) stat of file. Basically only the mtime is returned + +=cut + +sub stat { + my($self, $path) = @_; + my $t1 = [gettimeofday]; + + my $dbh = $self->_dbh; + my $data = $dbh->selectcol_arrayref("SELECT mtime FROM files WHERE path = ".$dbh->quote($path)." LIMIT 1"); + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$ctime,$blksize,$blocks); + + my $mtime = 0; + $mtime = ($data->[0]) if scalar @{$data} > 0; + return(( + $dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size, + $atime,0+$mtime,$ctime,$blksize,$blocks, + )); +} + +############################################## + +=head2 rmdir + + rmdir($path) + +remove empty folder + +=cut + +sub rmdir { + return; +} + +############################################## + +=head2 ensure_permissions + + ensure_permissions($mode, $path) + +ensure permissions and ownership + +=cut +sub ensure_permissions { + my($self, $mode, $path) = @_; + return if defined $ENV{'THRUK_NO_TOUCH_PERM'}; + + if($mode eq 'file') { + my $config = Thruk::Config::get_config(); + $mode = $config->{'mode_file'}; + } + elsif($mode eq 'dir') { + my $config = Thruk::Config::get_config(); + $mode = $config->{'mode_dir'}; + } + + my $dbh = $self->_dbh; + $dbh->do("UPDATE files SET permission = ".$dbh->quote($mode)." WHERE path = ".$dbh->quote($path)); + + return; +} + +############################################## + +=head2 file_rlock + + file_rlock($file) + +locks files table in shared / readonly mode. Returns nothing + +=cut +sub file_rlock { + my($self, $file) = @_; + + return unless $self->{'use_locks'}; + + my $t1 = [gettimeofday]; + + alarm(10); + local $SIG{'ALRM'} = sub { confess("timeout while trying to shared flock: ".$file."\n"); }; + + my $retrys = 0; + my $err; + while($retrys < 3) { + eval { + alarm(10); + $self->_dbh->do('LOCK TABLES files READ') || confess('Cannot lock_sh file: '.$file.': '.$!); + }; + $err = $@; + alarm(0); + if(!$err) { + last; + } + $retrys++; + Time::HiRes::sleep(0.5); + } + alarm(0); + + if($err) { + die("failed to shared flock $file: $err"); + } + + if($retrys > 0) { + _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_lock'} += $elapsed if $c; + + return; +} + +############################################## + +=head2 file_lock + + file_lock($file) + +locks files table in read/write mode. Returns nothing + +=cut +sub file_lock { + my($self, $file, $mode) = @_; + + return unless $self->{'use_locks'}; + + if($mode && $mode eq 'sh') { return $self->file_rlock($file); } + + my $t1 = [gettimeofday]; + alarm(20); + local $SIG{'ALRM'} = sub { confess("timeout while trying to excl. flock: ".$file."\n"); }; + + my $retrys = 0; + my $err; + while($retrys < 3) { + alarm(10); + eval { + $self->_dbh->do('LOCK TABLES files WRITE') || confess('Cannot lock_ex file: '.$file.': '.$!); + }; + $err = $@; + alarm(0); + if(!$err) { + last; + } + $retrys++; + Time::HiRes::sleep(0.5); + } + alarm(0); + + if($err) { + die("failed to lock $file: $err"); + } + + if($retrys > 0) { + _warn("got lock for ".$file." after ".$retrys." retries") unless $ENV{'TEST_IO_NOWARNINGS'}; + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_lock'} += $elapsed if $c; + + return; +} + +############################################## + +=head2 file_unlock + + file_unlock($file) + +unlocks tables previously locked with file_lock exclusivly. Returns nothing. + +=cut +sub file_unlock { + my($self) = @_; + + return unless $self->{'use_locks'}; + + eval { + $self->_dbh->do('UNLOCK TABLES') unless $self->{'use_locks'}; + }; + my $err = $@; + if($err) { + _debug($err); + return; + } + + return; +} + +############################################## + +=head2 json_store + + json_store($file, $data, $options) + +stores data json encoded + +$options can be { + pretty => 0/1, # don't write json into a single line and use human readable intendation + tmpfile => # use this tmpfile while writing new contents + changed_only => 0/1, # only write the file if it has changed + compare_data => "...", # use this string to compare for changed content + skip_ensure_permissions => 0/1 # skip running ensure_permissions after write + skip_validate => 0/1 # skip file validation (author only) + skip_config => 0/1 # skip all steps which reqire thruk config +} + +=cut +sub json_store { + my($self, $file, $data, $options) = @_; + + if(defined $options && ref $options ne 'HASH') { + confess("json_store options have been changed to hash."); + } + + if($options->{'skip_config'}) { + $options->{'skip_ensure_permissions'} = 1; + $options->{'skip_validate'} = 1; + } + + my $json = Cpanel::JSON::XS->new->utf8; + $json = $json->pretty if $options->{'pretty'}; + $json = $json->canonical; # keys will be randomly ordered otherwise + $json = $json->convert_blessed; + + my $write_out; + if($options->{'changed_only'}) { + $write_out = $json->encode($data); + if(defined $options->{'compare_data'}) { + return 1 if $options->{'compare_data'} eq $write_out; + } + else { + my $old = $self->read($file) // ''; + return 1 if $old eq $write_out; + } + } + + my $t1 = [gettimeofday]; + my $mtime = Time::HiRes::time(); + + my $dbh = $self->_dbh; + my $content = $dbh->quote($write_out || $json->encode($data)); + $dbh->do("INSERT INTO files (path,content,mtime) VALUES(".$dbh->quote($file).", ".$content.", ".$mtime.") ON DUPLICATE KEY UPDATE mtime=".$mtime.",content=".$content); + + if(!$options->{'skip_validate'}) { + my $config = Thruk::Config::get_config(); + if($config->{'thruk_author'}) { + eval { + my $test = $json->decode($self->read($file)); + }; + my $err = $@; + confess("json_store failed to write a valid file $file: ".$err) if $err; + } + } + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return 1; +} + +############################################## + +=head2 json_lock_store + + json_lock_store($file, $data, [$options]) + +stores data json encoded. options are passed to json_store. + +=cut +sub json_lock_store { + my($self, $file, $data, $options) = @_; + eval { + $self->file_lock($file); + $self->json_store($file, $data, $options); + }; + my $err = $@; + $self->file_unlock(); + confess($err) if $err; + return 1; +} + +############################################## + +=head2 json_retrieve + + json_retrieve($file) + +retrieve json data + +=cut +sub json_retrieve { + my($self, $file) = @_; + + our $jsonreader; + if(!$jsonreader) { + $jsonreader = Cpanel::JSON::XS->new->utf8; + $jsonreader->relaxed(); + } + + my $t1 = [gettimeofday]; + + my($data, $content); + $content = $self->saferead($file); + $data = $jsonreader->decode($content) if $content; + + my $elapsed = tv_interval($t1); + my $c = $Thruk::Globals::c || undef; + $c->stash->{'total_io_time'} += $elapsed if $c; + + return($data, $content) if wantarray; + return $data; +} + +############################################## + +=head2 json_lock_retrieve + + json_lock_retrieve($file) + +retrieve json data + +=cut +sub json_lock_retrieve { + my($self, $file) = @_; + my($data); + eval { + $self->file_rlock($file); + $data = $self->json_retrieve($file); + }; + my $err = $@; + $self->file_unlock(); + confess($err) if $err; + return $data; +} + +############################################## + +=head2 json_lock_patch + + json_lock_patch($file, $patch_data, [$options]) + +update json data with locking. options are passed to json_store. + +=cut +sub json_lock_patch { + my($self, $file, $patch_data, $options) = @_; + my($data); + eval { + $self->file_lock($file); + $data = $self->json_patch($file, undef, $patch_data, $options); + }; + my $err = $@; + $self->file_unlock(); + confess($err) if $err; + return $data; +} + +############################################## + +=head2 json_patch + + json_patch($file, unused, $patch_data, [$options]) + +update json data. options are passed to json_store. + +=cut +sub json_patch { + my($self, $file, undef, $patch_data, $options) = @_; + if(defined $options && ref $options ne 'HASH') { + confess("json_store options have been changed to hash."); + } + + my($data, $content) = $self->json_retrieve($file); + if(!defined $data) { + if(!$options->{'allow_empty'}) { + confess("attempt to patch empty file without allow_empty option: $file"); + } + ($data, $content) = ({}, ""); + } + $data = Thruk::Utils::IO::merge_deep($data, $patch_data); + $options->{'changed_only'} = 1; + $options->{'compare_data'} = $content; + $self->json_store($file, $data, $options); + return $data; +} + +######################################## + +=head2 touch + + touch($file) + +create file if not exists and update timestamp + +=cut +sub touch { + my($self, $file) = @_; + $self->write($file, "", Time::HiRes::time(), 1); + return; +} + +################################################### + +=head2 find_files + + find_files($folder, $pattern) + +return list of files for folder and pattern + +=cut +sub find_files { + #my($self, $dir, $match, $skip_symlinks) = @_; + my($self, $dir, $match, undef) = @_; + + my @files; + $dir =~ s/\/$//gmxo; + $dir =~ s/'//gmxo; + +# TODO: should not list sub folders... + my $dbh = $self->_dbh; + my @res = @{$dbh->selectall_arrayref("SELECT path FROM files WHERE path LIKE '".$dir."/%'", { Slice => {} })}; + for my $r (@res) { + my $file = $r->{'path'}; + # if its a file, make sure it matches our pattern + if(defined $match) { + next unless $file =~ m/$match/mx; + } + + push @files, $file; + } + + return \@files; +} + + +################################################### + +=head2 remove_folder + + remove_folder($folder) + +recursively remove folder and all files + +=cut +sub remove_folder { + my($self, $dir) = @_; + + $dir =~ s/\/$//gmxo; + $dir =~ s/'//gmxo; + + my $dbh = $self->_dbh; + $dbh->do("DELETE FROM files WHERE path LIKE '".$dir."/%'"); + + return; +} + +############################################## + +1; diff --git a/lib/Thruk/Utils/LMD.pm b/lib/Thruk/Utils/LMD.pm index 2a20d841f7..3bdfbe7d63 100644 --- a/lib/Thruk/Utils/LMD.pm +++ b/lib/Thruk/Utils/LMD.pm @@ -38,7 +38,7 @@ sub check_proc { my $lmd_dir = $config->{'tmp_path'}.'/lmd'; my $logfile = $lmd_dir.'/lmd.log'; - my $size = -s $logfile; + my $size = Thruk::Utils::IO::file_not_empty($logfile); my $keep = $config->{'lmd_rotate_keep_logs'} || 3; my $rotatesize = ($config->{'lmd_rotate_size'} || 20 ) *1024*1024; # rotate logfile if its more than 20mb my $pid; @@ -297,7 +297,7 @@ check if pidfile exists and contains a valid pid, returns zero or the actual pid =cut sub check_pid { my($file) = @_; - return 0 unless -s $file; + return 0 unless Thruk::Utils::IO::file_not_empty($file); my $pid = Thruk::Utils::IO::read($file); if($pid =~ m/^(\d+)\s*$/mx) { $pid = $1; diff --git a/lib/Thruk/Utils/Log.pm b/lib/Thruk/Utils/Log.pm index a8593128e7..ef654d5fc2 100644 --- a/lib/Thruk/Utils/Log.pm +++ b/lib/Thruk/Utils/Log.pm @@ -448,7 +448,7 @@ sub _init_logging { my($log4perl_conf); if($config) { if(Thruk::Base->mode() eq 'FASTCGI' || $ENV{'THRUK_JOB_DIR'} || $ENV{'THRUK_CRON'} || $ENV{'THRUK_AUTH_SCRIPT'}) { - if(defined $config->{'log4perl_conf'} && ! -s $config->{'log4perl_conf'} ) { + if(defined $config->{'log4perl_conf'} && ! Thruk::Utils::IO::file_not_empty($config->{'log4perl_conf'}) ) { die("\n\n*****\nfailed to load log4perl config: ".$config->{'log4perl_conf'}.": ".$!."\n*****\n\n"); } $log4perl_conf = $config->{'log4perl_conf'} || ($config->{'home'}//Thruk::Config::home()).'/log4perl.conf'; @@ -456,7 +456,7 @@ sub _init_logging { } my($log, $target); - if(defined $log4perl_conf && -s $log4perl_conf) { + if(defined $log4perl_conf && Thruk::Utils::IO::file_not_empty($log4perl_conf)) { $log = _get_file_logger($log4perl_conf, $config); $target = "file"; } else { diff --git a/lib/Thruk/Utils/RecurringDowntimes.pm b/lib/Thruk/Utils/RecurringDowntimes.pm index 7f12798cbf..dcb6d13b17 100644 --- a/lib/Thruk/Utils/RecurringDowntimes.pm +++ b/lib/Thruk/Utils/RecurringDowntimes.pm @@ -144,9 +144,8 @@ sub get_downtimes_list { my $default_rd = get_default_recurring_downtime($c, $host, $service); my $downtimes = []; my $reinstall_required = 0; - my @files = glob($c->config->{'var_path'}.'/downtimes/*.tsk'); - for my $dfile (@files) { - next unless -f $dfile; + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/downtimes/', '\.tsk$'); + for my $dfile (@{$files}) { $reinstall_required++ if $dfile !~ m/\/\d+\.tsk$/mx; my $d = read_downtime($c, $dfile, $default_rd, $authhosts, $authservices, $authhostgroups, $authservicegroups, $host, $service, $auth, $backendfilter, $hosts, $services, $hostgroups, $servicegroups); push @{$downtimes}, $d if $d; @@ -257,6 +256,7 @@ sub read_downtime { } my $d = Thruk::Utils::read_data_file($dfile); + return unless $d; $d->{'file'} = $dfile; $d->{'file'} =~ s|^.*/||gmx; $d->{'file'} =~ s|\.tsk$||gmx; @@ -553,7 +553,9 @@ sub get_data_file_name { $nr = 1; } - while(-f $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk') { + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/downtimes/', '\.tsk$'); + $files = Thruk::Base::array2hash($files); + while(defined $files->{$c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk'}) { $nr++; } diff --git a/lib/Thruk/Utils/SelfCheck.pm b/lib/Thruk/Utils/SelfCheck.pm index b56646c36d..4fccc7ce5b 100644 --- a/lib/Thruk/Utils/SelfCheck.pm +++ b/lib/Thruk/Utils/SelfCheck.pm @@ -151,6 +151,7 @@ sub _filesystem_checks { for my $fs (['var path', $c->config->{'var_path'}], ['tmp path', $c->config->{'tmp_path'}], ) { +# TODO: check... if(!-e $fs->[1]) { $details .= sprintf(" - %s %s does not exist: %s\n", $fs->[0], $fs->[1], $!); $rc = 2; @@ -185,8 +186,9 @@ sub _logfile_checks { $c->config->{'log4perl_logfile_in_use'}, ) { next unless $log; # may not be set - next unless -e $log; # may not exist either + next unless Thruk::Utils::IO::file_exists($log); # may not exist either # count errors +# TODO: won't work with DB my @out = split(/\n/mx, Thruk::Utils::IO::cmd("grep 'ERROR' $log")); $details .= sprintf(" - %s: ", $log); if(scalar @out == 0) { @@ -475,8 +477,9 @@ sub _lmd_checks { } for my $log ($c->config->{'tmp_path'}.'/lmd/lmd.log') { - next unless -e $log; # may not exist either + next unless Thruk::Utils::IO::file_exists($log); # may not exist either # count errors +# TODO: won't work with DB my @out = split(/\n/mx, Thruk::Utils::IO::cmd("grep 'Panic:' $log")); $details .= sprintf(" - %s: ", $log); if(scalar @out == 0) { diff --git a/plugins/plugins-available/agents/lib/Thruk/Utils/Agents.pm b/plugins/plugins-available/agents/lib/Thruk/Utils/Agents.pm index d2dfc786e6..8dde40ffb7 100644 --- a/plugins/plugins-available/agents/lib/Thruk/Utils/Agents.pm +++ b/plugins/plugins-available/agents/lib/Thruk/Utils/Agents.pm @@ -4,7 +4,6 @@ use warnings; use strict; use Carp qw/confess/; use Cpanel::JSON::XS qw/decode_json/; -use File::Copy qw/move/; use Monitoring::Config::Object (); use Thruk::Controller::conf (); @@ -765,10 +764,12 @@ sub migrate_hostname { # rename data file my $df1 = $c->config->{'var_path'}.'/agents/hosts/'.$hostname.'.json'; my $df2 = $c->config->{'var_path'}.'/agents/hosts/'.$old_host.'.json'; - if(-f $df2 && !-f $df1) { - move($df2, $df1); + my $d1 = Thruk::Utils::IO::saferead($df1); + my $d2 = Thruk::Utils::IO::saferead($df2); + if(defined $d2 && !defined $d1) { + Thruk::Utils::IO::write($df1, $d2); } - unlink($df2); + Thruk::Utils::IO::unlink($df2); return; } diff --git a/plugins/plugins-available/business_process/lib/Thruk/BP/Components/BP.pm b/plugins/plugins-available/business_process/lib/Thruk/BP/Components/BP.pm index f8701fb869..65f00fcab9 100644 --- a/plugins/plugins-available/business_process/lib/Thruk/BP/Components/BP.pm +++ b/plugins/plugins-available/business_process/lib/Thruk/BP/Components/BP.pm @@ -76,13 +76,14 @@ sub new { bless $self, $class; $self->set_file($c, $file); - if($editmode && -e $self->{'editfile'}) { $file = $self->{'editfile'}; } - if(-s $file) { + if($editmode && Thruk::Utils::IO::file_exists($self->{'editfile'})) { $file = $self->{'editfile'}; } + my $test = Thruk::Utils::IO::saferead($file); + if(defined $test) { $bpdata = Thruk::Utils::IO::json_lock_retrieve($file); return unless $bpdata; return unless $bpdata->{'name'}; } - if(!-e $self->{'file'}) { + if(!Thruk::Utils::IO::file_exists($self->{'file'})) { $self->{'draft'} = 1; } @@ -168,11 +169,13 @@ sub load_runtime_data { my($self) = @_; my $file = $self->{'datafile'}; - if($self->{'editmode'} and -s $self->{'datafile'}.'.edit') { + my $test = Thruk::Utils::IO::saferead($self->{'datafile'}.'.edit'); + if($self->{'editmode'} && defined $test) { $file = $self->{'datafile'}.'.edit'; } - return unless -s $file; + $test = Thruk::Utils::IO::saferead($file); + return unless defined $test; my $data = Thruk::Utils::IO::json_lock_retrieve($file); for my $key (@stateful_keys) { @@ -509,10 +512,10 @@ sub commit { } } - if(-e $self->{'file'} && ! -e $self->{'backupfile'}) { + if(Thruk::Utils::IO::file_exists($self->{'file'}) && !Thruk::Utils::IO::file_exists($self->{'backupfile'})) { copy($self->{'file'}, $self->{'backupfile'}) or die('cannot backup to '.$self->{'backupfile'}.': '.$!); } - if(-e $self->{'editfile'}) { + if(Thruk::Utils::IO::file_exists($self->{'editfile'})) { copy($self->{'editfile'}, $self->{'file'}) or die('cannot commit changes to '.$self->{'file'}.': '.$!); } @@ -540,7 +543,7 @@ revert business process to latest backup sub revert { my ( $self, $c ) = @_; - if(!-e $self->{'backupfile'}) { + if(!Thruk::Utils::IO::file_exists($self->{'backupfile'})) { return; } diff --git a/plugins/plugins-available/business_process/lib/Thruk/BP/Utils.pm b/plugins/plugins-available/business_process/lib/Thruk/BP/Utils.pm index 76c0b5c582..647b97b8c2 100644 --- a/plugins/plugins-available/business_process/lib/Thruk/BP/Utils.pm +++ b/plugins/plugins-available/business_process/lib/Thruk/BP/Utils.pm @@ -191,7 +191,7 @@ sub next_free_bp_file { my $base_folder = bp_base_folder($c); Thruk::Utils::IO::mkdir_r($c->config->{'var_path'}.'/bp'); Thruk::Utils::IO::mkdir_r($base_folder); - while(-e $base_folder.'/'.$num.'.tbp' || -e $c->config->{'var_path'}.'/bp/'.$num.'.tbp.edit') { + while(Thruk::Utils::IO::file_exists($base_folder.'/'.$num.'.tbp') || Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/bp/'.$num.'.tbp.edit')) { $num++; } return($base_folder.'/'.$num.'.tbp', $num); @@ -285,7 +285,7 @@ sub save_bp_objects { Thruk::Utils::IO::write($filename, Encode::encode_utf8($text)); my $new_hex = Thruk::Utils::Crypt::hexdigest(Thruk::Utils::IO::read($filename)); - my $old_hex = -f $file ? Thruk::Utils::Crypt::hexdigest(Thruk::Utils::IO::read($file)) : ''; + my $old_hex = Thruk::Utils::Crypt::hexdigest(Thruk::Utils::IO::saferead($file) || ''); # check if something changed if($new_hex eq $old_hex) { @@ -398,13 +398,13 @@ sub clean_orphaned_edit_files { $threshold = 86400 unless defined $threshold; my $base_folder = bp_base_folder($c); for my $pattern (qw/edit runtime/) { - my @files = glob($c->config->{'var_path'}.'/bp/*.tbp.'.$pattern); - for my $file (@files) { + my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/bp/', '\.tbp\.'.$pattern); + for my $file (@{$files}) { $file =~ m/\/(\d+)\.tbp\.$pattern/mx; - if($1 && !-e $base_folder.'/'.$1.'.tbp') { - my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($file); + if($1 && !Thruk::Utils::IO::file_exists($base_folder.'/'.$1.'.tbp')) { + my($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = Thruk::Utils::IO::stat($file); next if $mtime > (time() - $threshold); - unlink($file); + Thruk::Utils::IO::unlink($file); } } } @@ -469,7 +469,7 @@ sub get_custom_functions { my $functions = []; my @files = glob(bp_base_folder($c).'/*.pm'); for my $filename (@files) { - next unless -s $filename; + next unless Thruk::Utils::IO::file_not_empty($filename); my $f = _parse_custom_functions($filename, 'function$'); push @{$functions}, @{$f}; } @@ -492,7 +492,7 @@ sub get_custom_filter { my $functions = []; my @files = glob(bp_base_folder($c).'/*.pm'); for my $filename (@files) { - next unless -s $filename; + next unless Thruk::Utils::IO::file_not_empty($filename); my $f = _parse_custom_functions($filename, 'filter$'); push @{$functions}, @{$f}; } @@ -863,9 +863,13 @@ runs index update if necessary. =cut sub check_update_index { my($c, $bps) = @_; - if($ENV{'THRUK_CRON'} || !-s $c->config->{'var_path'}.'/bp/.index' || (stat(_))[9] < time()-300) { + return unless $ENV{'THRUK_CRON'}; + + my @stat = Thruk::Utils::IO::stat($c->config->{'var_path'}.'/bp/.index'); + if(!$stat[9] || $stat[9] < time()-300) { _update_index($c, $bps); } + return; } diff --git a/plugins/plugins-available/business_process/lib/Thruk/Controller/bp.pm b/plugins/plugins-available/business_process/lib/Thruk/Controller/bp.pm index 4cec4ff2dd..e759b55c6a 100644 --- a/plugins/plugins-available/business_process/lib/Thruk/Controller/bp.pm +++ b/plugins/plugins-available/business_process/lib/Thruk/Controller/bp.pm @@ -104,7 +104,7 @@ sub index { my $host_templates = []; my $service_templates = []; # simple / fast template grep - if($c->stash->{'objects_templates_file'} and -e $c->stash->{'objects_templates_file'}) { + if($c->stash->{'objects_templates_file'} && Thruk::Utils::IO::file_exists($c->stash->{'objects_templates_file'})) { my $lasttype; open(my $fh, '<', $c->stash->{'objects_templates_file'}) or die("failed to open ".$c->stash->{'objects_templates_file'}.": ".$!); while(my $line = <$fh>) { @@ -196,7 +196,7 @@ sub index { unlink($bp->{'editfile'}); unlink($bp->{'datafile'}.'.edit'); Thruk::Utils::set_message( $c, { style => 'success_message', msg => 'changes canceled' }); - if(-e $bp->{'file'}) { + if(Thruk::Utils::IO::file_exists($bp->{'file'})) { return $c->redirect_to($c->stash->{'url_prefix'}."cgi-bin/bp.cgi?action=details&bp=".$id); } else { return $c->redirect_to($c->stash->{'url_prefix'}."cgi-bin/bp.cgi"); diff --git a/plugins/plugins-available/business_process/lib/Thruk/Utils/CLI/Bp.pm b/plugins/plugins-available/business_process/lib/Thruk/Utils/CLI/Bp.pm index 64a3ae3a67..8f8e9fa9ad 100644 --- a/plugins/plugins-available/business_process/lib/Thruk/Utils/CLI/Bp.pm +++ b/plugins/plugins-available/business_process/lib/Thruk/Utils/CLI/Bp.pm @@ -255,13 +255,13 @@ sub cmd { # merge spool files into one file if($spoolfile) { - my @files = glob($spoolfile.".*"); + my @files = @{Thruk::Utils::IO::find_files($spoolfile)}; if(scalar @files > 0) { for my $file (@files) { my $cont = Thruk::Utils::IO::read($file); Thruk::Utils::IO::write($spoolfile, $cont, undef, 1); + Thruk::Utils::IO::unlink($file); } - unlink(@files); } my $file = $spoolfile.'.ok'; sysopen(my $t,$file,O_WRONLY|O_CREAT|O_NONBLOCK|O_NOCTTY) || die("cannot create $file: $!"); diff --git a/plugins/plugins-available/conf/lib/Monitoring/Config.pm b/plugins/plugins-available/conf/lib/Monitoring/Config.pm index ebca6df3dc..d7ca8258fd 100644 --- a/plugins/plugins-available/conf/lib/Monitoring/Config.pm +++ b/plugins/plugins-available/conf/lib/Monitoring/Config.pm @@ -286,7 +286,7 @@ sub commit { my @new_files; my %new_index; for my $f (@{$self->{'files'}}) { - if(!$f->{'deleted'} || -f $f->{'path'}) { + if(!$f->{'deleted'} || Thruk::Utils::IO::file_exists($f->{'path'})) { push @new_files, $f; $new_index{$f->{'display'}} = $f; $new_index{$f->{'path'}} = $f; @@ -1572,7 +1572,7 @@ sub _set_config { my $core_conf = $self->{'config'}->{'core_conf'}; if(defined $ENV{'OMD_ROOT'} && -d $ENV{'OMD_ROOT'}."/version/." - && ! -s $core_conf + && ! Thruk::Utils::IO::file_not_empty($core_conf) && scalar(@{Thruk::Base::list($self->{'config'}->{'obj_dir'})}) == 0 && scalar(@{Thruk::Base::list($self->{'config'}->{'obj_file'})}) == 0) { my $newest = $self->_newest_file( @@ -2531,7 +2531,7 @@ sub remote_file_sync { } for my $f (@{$self->{'files'}}) { - next unless -f $f->{'path'}; + next unless Thruk::Utils::IO::file_exists($f->{'path'}); $files->{$f->{'display'}} = { 'mtime' => $f->{'mtime'}, 'hex' => $f->{'hex'}, diff --git a/plugins/plugins-available/conf/lib/Thruk/Controller/conf.pm b/plugins/plugins-available/conf/lib/Thruk/Controller/conf.pm index 46bdffb89e..741d5cc843 100644 --- a/plugins/plugins-available/conf/lib/Thruk/Controller/conf.pm +++ b/plugins/plugins-available/conf/lib/Thruk/Controller/conf.pm @@ -623,7 +623,7 @@ sub _process_users_page { ($name, $alias) = split(/\ \-\ /mx,$user, 2); $profile_file = $c->config->{'var_path'}."/users/".$name; $c->stash->{'profile_file'} = $profile_file; - $c->stash->{'profile_file_exists'} = -e $profile_file ? 1 : 0; + $c->stash->{'profile_file_exists'} = Thruk::Utils::IO::file_exists($profile_file) ? 1 : 0; } # save changes to user @@ -1379,7 +1379,7 @@ sub _process_tools_page { my $tools = _get_tools($c); $c->stash->{'tools'} = $tools; my $ignore_file = $c->config->{'var_path'}.'/conf_tools_ignore'; - my $ignores = -s $ignore_file ? Thruk::Utils::IO::json_lock_retrieve($ignore_file) : {}; + my $ignores = Thruk::Utils::IO::json_lock_retrieve($ignore_file) // {}; $c->stash->{'tool'} = $tool; $c->stats->profile(begin => "tool: ".$tool); @@ -1604,7 +1604,7 @@ sub _htpasswd_password { # check old password first? if($has_minus_v && $oldpassword) { my $cmd = [$htpasswd]; - push @{$cmd}, '-c' unless -s $c->config->{'Thruk::Plugin::ConfigTool'}->{'htpasswd'}; + push @{$cmd}, '-c' unless Thruk::Utils::IO::file_not_empty($c->config->{'Thruk::Plugin::ConfigTool'}->{'htpasswd'}); push @{$cmd}, '-i' if $has_minus_i; push @{$cmd}, '-b' if !$has_minus_i; push @{$cmd}, '-v'; @@ -1617,7 +1617,7 @@ sub _htpasswd_password { } my $cmd = [$htpasswd]; - push @{$cmd}, '-c' unless -s $c->config->{'Thruk::Plugin::ConfigTool'}->{'htpasswd'}; + push @{$cmd}, '-c' unless Thruk::Utils::IO::file_not_empty($c->config->{'Thruk::Plugin::ConfigTool'}->{'htpasswd'}); push @{$cmd}, '-i' if $has_minus_i; push @{$cmd}, '-b' if !$has_minus_i; push @{$cmd}, $c->config->{'Thruk::Plugin::ConfigTool'}->{'htpasswd'}; @@ -1945,7 +1945,7 @@ sub _object_revert { my($c, $obj) = @_; my $id = $obj->get_id(); - if(-e $obj->{'file'}->{'path'}) { + if(Thruk::Utils::IO::file_exists($obj->{'file'}->{'path'})) { my $oldobj; my $tmpfile = Monitoring::Config::File->new($obj->{'file'}->{'path'}, undef, $c->{'obj_db'}->{'coretype'}); $tmpfile->update_objects(); @@ -2386,7 +2386,7 @@ sub _file_editor { $c->stash->{'file_link'} = $filename; $c->stash->{'line'} = $c->req->parameters->{'line'} || 1; $c->stash->{'file_content'} = ''; - if(-f $filename) { + if(Thruk::Utils::IO::file_exists($filename)) { my $content = Thruk::Utils::IO::read($filename); $c->stash->{'file_content'} = decode_utf8($content); } diff --git a/plugins/plugins-available/conf/lib/Thruk/Utils/Conf.pm b/plugins/plugins-available/conf/lib/Thruk/Utils/Conf.pm index 85bd579b74..1f3efab289 100644 --- a/plugins/plugins-available/conf/lib/Thruk/Utils/Conf.pm +++ b/plugins/plugins-available/conf/lib/Thruk/Utils/Conf.pm @@ -463,10 +463,7 @@ replace block in config file sub replace_block { my($file, $string, $start, $end) = @_; - my $content = ""; - if(-f $file) { - $content = Thruk::Utils::IO::read($file); - } + my $content = Thruk::Utils::IO::saferead($file) // ""; ## no critic unless($content =~ s/$start.*?$end/$string/sxi) { @@ -557,7 +554,7 @@ sub get_cgi_user_list { } # add users from profiles - my @profiles = glob($c->config->{'var_path'}."/users/*"); + my @profiles = @{Thruk::Utils::IO::find_files($c->config->{'var_path'}."/users/")}; for my $profile (@profiles) { $profile =~ s/^.*\///gmx; $all_contacts->{$profile} = { name => $profile } unless defined $all_contacts->{$profile}; @@ -621,8 +618,7 @@ read htpasswd file sub read_htpasswd { my ( $file ) = @_; my $htpasswd = {}; - return $htpasswd unless -f $file; - my $content = Thruk::Utils::IO::read($file); + my $content = Thruk::Utils::IO::saferead($file) // ''; for my $line (split/\n/mx, $content) { my($user,$hash) = split/:/mx, $line; next unless defined $hash; @@ -708,11 +704,11 @@ sub get_model_retention { my $var_path = $c->config->{'var_path'}; my $file = $c->config->{'var_path'}."/obj_retention.".$backend.".".$user_id.".dat"; - if(! -f $file) { + if(!Thruk::Utils::IO::file_exists($file)) { $file = $c->config->{'var_path'}."/obj_retention.".$backend.".dat"; } - if(! -f $file) { + if(!Thruk::Utils::IO::file_exists($file)) { $c->stats->profile(end => "get_model_retention($backend)"); return 1 if $model->cache_exists($backend); return; diff --git a/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm b/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm index ba41a6fca4..634563b8e3 100644 --- a/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm +++ b/plugins/plugins-available/node-control/lib/Thruk/NodeControl/Utils.pm @@ -161,7 +161,7 @@ sub get_server { $facts->{'last_error'} =~ s/\s+at\s+.*(Utils|HTTP)\.pm\s+line\s+\d+\.//gmx if $facts->{'last_error'}; # gather available logs - my @logs = glob($c->config->{'var_path'}.'/node_control/'.$peer->{'key'}.'_*.log'); + my @logs = @{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/node_control/'.$peer->{'key'}, '_.*\.log$')}; @logs = map { my $l = $_; $l =~ s/^.*\///gmx; $l =~ s/\.log$//gmx; $l =~ s/^$peer->{'key'}_//gmx; $l; } @logs; my $logs = Thruk::Base::array2hash(\@logs); for my $l (sort keys %{$logs}) { @@ -318,7 +318,7 @@ sub update_runtime_data { sub _ansible_get_facts { my($c, $peer, $refresh) = @_; my $file = $c->config->{'var_path'}.'/node_control/'.$peer->{'key'}.'.json'; - if(!$refresh && -e $file) { + if(!$refresh && Thruk::Utils::IO::file_exists($file)) { return(Thruk::Utils::IO::json_lock_retrieve($file)); } if(defined $refresh && !$refresh) { @@ -1189,7 +1189,7 @@ sub config { my $file = $c->config->{'var_path'}.'/node_control/_conf.json'; my $var; - if(-e $file) { + if(Thruk::Utils::IO::file_exists($file)) { $var = Thruk::Utils::IO::json_lock_retrieve($file); } diff --git a/plugins/plugins-available/panorama/lib/Thruk/Controller/panorama.pm b/plugins/plugins-available/panorama/lib/Thruk/Controller/panorama.pm index b5bd87f977..0377b64baf 100644 --- a/plugins/plugins-available/panorama/lib/Thruk/Controller/panorama.pm +++ b/plugins/plugins-available/panorama/lib/Thruk/Controller/panorama.pm @@ -765,7 +765,7 @@ sub _task_upload { } my $newlocation = $folder.'/'.$filename; - if(-s $newlocation && !_check_media_permissions($c, $newlocation)) { + if(Thruk::Utils::IO::file_not_empty($newlocation) && !_check_media_permissions($c, $newlocation)) { # must be text/html result, otherwise extjs form result handler dies $c->stash->{text} = Thruk::Utils::Filter::json_encode({ 'msg' => 'Only administrator/panorama_view_media_manager roles may overwrite existing files.', success => Cpanel::JSON::XS::false }); return; @@ -938,7 +938,7 @@ sub _task_load_dashboard { require MIME::Base64; my $usercontent_folder = $c->stash->{'usercontent_folder'}.'/'; for my $file (sort keys %{$data->{'usercontent'}}) { - my $size = -s $usercontent_folder.$file; + my $size = Thruk::Utils::IO::file_not_empty($usercontent_folder.$file); next if $c->config->{'demo_mode'}; next if $size && !_check_media_permissions($c, $usercontent_folder.$file); # overwrite only if user is allowed to my $content = MIME::Base64::decode_base64($data->{'usercontent'}->{$file}); @@ -3055,8 +3055,8 @@ sub _task_dashboard_list { # add last_used data for my $d (@{$dashboards}) { $d->{'last_used'} = 0; - for my $file (glob($c->config->{'var_path'}.'/panorama/'.$d->{'nr'}.'.tab.*runtime')) { - my @stat = stat($file); + for my $file (@{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/panorama/', $d->{'nr'}.'\.tab\..*runtime$')}) { + my @stat = Thruk::Utils::IO::stat($file); $d->{'last_used'} = $stat[9] if $d->{'last_used'} < $stat[9]; } } @@ -3175,9 +3175,10 @@ sub _task_dashboard_restore { my $dashboard = Thruk::Utils::Panorama::load_dashboard($c, $nr, 1); my $permission = Thruk::Utils::Panorama::is_authorized_for_dashboard($c, $nr, $dashboard); if($permission >= ACCESS_READWRITE && !$dashboard->{'scripted'}) { - die("no such dashboard") unless -e $c->{'panorama_etc'}.'/'.$nr.'.tab'; - die("no such restore point") unless -e $c->{'panorama_var'}.'/'.$nr.'.tab.'.$timestamp.".".$mode; - unlink($c->{'panorama_etc'}.'/'.$nr.'.tab'); + die("no such dashboard") unless Thruk::Utils::IO::file_exists($c->{'panorama_etc'}.'/'.$nr.'.tab'); + die("no such restore point") unless Thruk::Utils::IO::file_exists($c->{'panorama_var'}.'/'.$nr.'.tab.'.$timestamp.".".$mode); + Thruk::Utils::IO::unlink($c->{'panorama_etc'}.'/'.$nr.'.tab'); +# TODO: ... copy($c->{'panorama_var'}.'/'.$nr.'.tab.'.$timestamp.".".$mode, $c->{'panorama_etc'}.'/'.$nr.'.tab'); } my $json = {}; @@ -3638,8 +3639,9 @@ sub _add_json_dashboard_timestamps { my $nr = $tab; $nr =~ s/^pantab_//gmx; $json->{'dashboard_ts'} = {}; +# TODO: move to var my $file = $c->{'panorama_etc'}.'/'.$nr.'.tab'; - if($nr eq "0" && !-s $file) { + if($nr eq "0" && !Thruk::Utils::IO::file_not_empty($file)) { $file = $c->config->{'plugin_path'}.'/plugins-enabled/panorama/0.tab'; } my @stat = stat($file); @@ -3654,7 +3656,7 @@ sub _add_json_dashboard_timestamps { $json->{'dashboard_ts'}->{$tab} = $stat[9] if defined $stat[9]; } my $maintfile = Thruk::Utils::Panorama::get_maint_file($c, $nr); - if(-e $maintfile) { + if(Thruk::Utils::IO::file_exists($maintfile)) { my $maintenance = Thruk::Utils::IO::json_lock_retrieve($maintfile); $json->{'maintenance'}->{$tab} = $maintenance->{'maintenance'}; } else { diff --git a/plugins/plugins-available/panorama/lib/Thruk/Utils/CLI/Panorama.pm b/plugins/plugins-available/panorama/lib/Thruk/Utils/CLI/Panorama.pm index 412c733355..5254bcab15 100644 --- a/plugins/plugins-available/panorama/lib/Thruk/Utils/CLI/Panorama.pm +++ b/plugins/plugins-available/panorama/lib/Thruk/Utils/CLI/Panorama.pm @@ -39,6 +39,7 @@ use Cpanel::JSON::XS qw/encode_json decode_json/; use Getopt::Long (); use Thruk::Utils::CLI (); +use Thruk::Utils::IO (); use Thruk::Utils::Log qw/:all/; ############################################## @@ -91,7 +92,7 @@ sub cmd { my $nr = shift @{$commandoptions}; return(Thruk::Utils::CLI::get_submodule_help(__PACKAGE__)) unless defined $nr; my $file; - if(-e $nr) { + if(Thruk::Utils::IO::file_exists($nr)) { $file = $nr; $nr = -1; } @@ -108,7 +109,7 @@ sub cmd { return(Thruk::Utils::CLI::get_submodule_help(__PACKAGE__)) if scalar @{$commandoptions} == 0; for my $nr (@{$commandoptions}) { my $file; - if(-e $nr) { + if(Thruk::Utils::IO::file_exists($nr)) { $file = $nr; $nr = -1; } diff --git a/plugins/plugins-available/panorama/lib/Thruk/Utils/Panorama.pm b/plugins/plugins-available/panorama/lib/Thruk/Utils/Panorama.pm index ee075ea9d4..22b5b8ed14 100644 --- a/plugins/plugins-available/panorama/lib/Thruk/Utils/Panorama.pm +++ b/plugins/plugins-available/panorama/lib/Thruk/Utils/Panorama.pm @@ -215,13 +215,13 @@ sub load_dashboard { return if $nr !~ m/^\-?[a-zA-Z_\-\d]+$/gmx; # startpage can be overridden, only load original file if there is none in etc/ - if($nr eq "0" && !-s $file) { + if($nr eq "0" && !Thruk::Utils::IO::file_not_empty($file)) { $file = $c->config->{'plugin_path'}.'/plugins-enabled/panorama/0.tab'; } set_is_admin($c); - return unless -s $file; + return unless Thruk::Utils::IO::file_not_empty($file); my $dashboard; my $scripted = 0; if(-x $file) { @@ -282,11 +282,8 @@ sub load_dashboard { $dashboard->{'file_version'} = DASHBOARD_FILE_VERSION; # merge runtime data - my $runtime = {}; my $runtimefile = get_runtime_file($c, $nr); - if(-s $runtimefile) { - $runtime = Thruk::Utils::read_data_file($runtimefile, $c); - } + my $runtime = Thruk::Utils::read_data_file($runtimefile, $c) // {}; for my $tab (keys %{$runtime}) { next if !defined $dashboard->{$tab}; for my $key (keys %{$runtime->{$tab}}) { @@ -311,7 +308,7 @@ sub load_dashboard { # check for maintenance mode my $maintfile = get_maint_file($c, $nr); - if(-e $maintfile) { + if(Thruk::Utils::IO::file_exists($maintfile)) { my $maintenance = Thruk::Utils::IO::json_lock_retrieve($maintfile); $dashboard->{'maintenance'} = $maintenance->{'maintenance'}; } @@ -350,7 +347,7 @@ sub save_dashboard { # find next free number $nr = $c->config->{'Thruk::Plugin::Panorama'}->{'new_files_start_at'} || 1; $file = $c->config->{'etc_path'}.'/panorama/'.$nr.'.tab'; - while(-e $file) { + while(Thruk::Utils::IO::file_exists($file)) { $nr++; $file = $c->config->{'etc_path'}.'/panorama/'.$nr.'.tab'; } @@ -460,7 +457,7 @@ sub is_authorized_for_dashboard { return ACCESS_OWNER if $c->stash->{'is_admin'}; # does that dashboard already exist? - if(-s $file) { + if(Thruk::Utils::IO::file_not_empty($file)) { $dashboard = load_dashboard($c, $nr, 1) unless $dashboard; if($dashboard->{'user'} eq $c->stash->{'remote_user'}) { return ACCESS_READONLY if $c->stash->{'readonly'}; @@ -542,7 +539,7 @@ sub move_dashboard { my $oldnr = $dashboard->{'id'}; $oldnr =~ s/^pantab_//gmx; - if(-e $c->config->{'etc_path'}.'/panorama/'.$newfile) { + if(Thruk::Utils::IO::file_exists($c->config->{'etc_path'}.'/panorama/'.$newfile)) { Thruk::Utils::set_message($c, 'fail_message', 'Renaming dashboard failed, '.$newfile.' does already exist.'); return($dashboard); } diff --git a/plugins/plugins-available/reports2/lib/Thruk/Controller/reports2.pm b/plugins/plugins-available/reports2/lib/Thruk/Controller/reports2.pm index 593456808e..634ab525ff 100644 --- a/plugins/plugins-available/reports2/lib/Thruk/Controller/reports2.pm +++ b/plugins/plugins-available/reports2/lib/Thruk/Controller/reports2.pm @@ -159,7 +159,7 @@ sub index { } } - if(!$ENV{'OMD_ROOT'} && !-d $c->config->{'var_path'}."/puppeteer" && !Thruk::Utils::has_node_module($c, 'puppeteer')) { + if(!$ENV{'OMD_ROOT'} && !-d $c->config->{'var_path'}."/local/puppeteer" && !Thruk::Utils::has_node_module($c, 'puppeteer')) { $c->stash->{'puppeteer'} = 0; } @@ -506,7 +506,8 @@ sub report_email { Thruk::Utils::set_message( $c, { style => 'success_message', msg => '\'to\' address missing' }); } - $c->stash->{size} = -s $c->config->{'var_path'}.'/reports/'.$r->{'nr'}.'.dat'; + my $test = Thruk::Utils::IO::saferead($c->config->{'var_path'}.'/reports/'.$r->{'nr'}.'.dat'); + $c->stash->{size} = $test ? length($test) : 0; if($r->{'var'}->{'attachment'} && (!$r->{'var'}->{'ctype'} || $r->{'var'}->{'ctype'} ne 'html2pdf')) { $c->stash->{attach} = $r->{'var'}->{'attachment'}; } else { @@ -514,9 +515,8 @@ sub report_email { } if(defined $r->{'params'}->{'pdf'} && $r->{'params'}->{'pdf'} eq 'no') { my $attachment = $c->config->{'var_path'}.'/reports/'.$r->{'nr'}.'.html'; - if(-s $attachment) { - $c->stash->{size} = -s $attachment; - } + my $test = Thruk::Utils::IO::saferead($attachment); + $c->stash->{size} = $test ? length($test) : 0; } $c->stash->{subject} = $r->{'subject'} || 'Report: '.$r->{'name'}; $c->stash->{r} = $r; diff --git a/plugins/plugins-available/reports2/lib/Thruk/Utils/CLI/Report.pm b/plugins/plugins-available/reports2/lib/Thruk/Utils/CLI/Report.pm index f66adec2b3..50e91209d7 100644 --- a/plugins/plugins-available/reports2/lib/Thruk/Utils/CLI/Report.pm +++ b/plugins/plugins-available/reports2/lib/Thruk/Utils/CLI/Report.pm @@ -170,7 +170,7 @@ sub _cmd_report { _debug("finished creating report"); if(defined $report_file and $report_file eq '-2') { return($Thruk::Utils::Reports::error // "[".$nr.".rpt] report is running on another node already\n", 0); - } elsif(defined $report_file and -f $report_file) { + } elsif(defined $report_file && Thruk::Utils::IO::file_exists($report_file)) { ## no critic return(sprintf("[%s.rpt] report calculated successfully in %.1fs\n", $nr, $options->{'var'}->{'end_time'}-$options->{'var'}->{'start_time'}), 0) if(-t 0 || $ENV{'THRUK_CRON'}); # avoid pdf being printed to logfile ## use critic diff --git a/plugins/plugins-available/reports2/lib/Thruk/Utils/Reports.pm b/plugins/plugins-available/reports2/lib/Thruk/Utils/Reports.pm index c32d465e8a..2e2eef1388 100644 --- a/plugins/plugins-available/reports2/lib/Thruk/Utils/Reports.pm +++ b/plugins/plugins-available/reports2/lib/Thruk/Utils/Reports.pm @@ -46,7 +46,7 @@ sub get_report_list { $c->stats->profile(begin => "Utils::Reports::get_report_list(".($number_filter//'all').")"); my $reports = []; - for my $rfile (glob($c->config->{'var_path'}.'/reports/*.rpt')) { + for my $rfile (@{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/reports/', '\.rpt$')}) { if($rfile =~ m/\/(\d+)\.rpt/mx) { my $nr = $1; next if $number_filter && $nr ne $number_filter; @@ -109,15 +109,15 @@ sub report_show { } my $report_file = $c->config->{'var_path'}.'/reports/'.$nr.'.dat'; - if($refresh || ! -f $report_file) { + if($refresh || !Thruk::Utils::IO::file_exists($report_file)) { generate_report($c, $nr); } - if(defined $report_file and -f $report_file) { + if(defined $report_file && Thruk::Utils::IO::file_exists($report_file)) { $c->stash->{'template'} = 'passthrough.tt'; if($c->req->parameters->{'html'}) { my $html_file = $c->config->{'var_path'}.'/reports/'.$nr.'.html'; - if(!-e $html_file) { + if(!Thruk::Utils::IO::file_exists($html_file)) { $html_file = $c->config->{'var_path'}.'/reports/'.$nr.'.dat'; } my $report_text = decode_utf8(Thruk::Utils::IO::read($html_file)); @@ -138,6 +138,7 @@ sub report_show { } } my $fh; +# TODO: ... if($report->{'var'}->{'ctype'} eq 'text/html') { open($fh, '<', $c->config->{'var_path'}.'/reports/'.$nr.'.html'); } else { @@ -207,7 +208,8 @@ sub report_send { my $attachment; if($skip_generate) { $attachment = $c->config->{'var_path'}.'/reports/'.$report->{'nr'}.'.dat'; - if(!-s $attachment) { + my $test = Thruk::Utils::IO::saferead($attachment); + if(! defined $test) { Thruk::Utils::set_message( $c, 'fail_message', 'report not yet generated' ); return $c->redirect_to('reports2.cgi'); } @@ -307,7 +309,8 @@ sub report_send { # url reports as html if(defined $report->{'params'}->{'pdf'} && $report->{'params'}->{'pdf'} eq 'no') { $attachment = $c->config->{'var_path'}.'/reports/'.$report->{'nr'}.'.html'; - if(!-s $attachment) { + my $test = Thruk::Utils::IO::saferead($attachment); + if(! defined $test) { $attachment = $c->config->{'var_path'}.'/reports/'.$report->{'nr'}.'.dat'; } my $ctype = 'text/html'; @@ -335,7 +338,8 @@ sub report_send { ); } - if($report->{'var'}->{'json_file'} && -e $report->{'var'}->{'json_file'}) { + if($report->{'var'}->{'json_file'} && Thruk::Utils::IO::file_exists($report->{'var'}->{'json_file'})) { +# TODO: wont't work with DB $msg->attach(Type => 'text/json', Path => $report->{'var'}->{'json_file'}, Filename => encode_utf8(Thruk::Base::basename($report->{'var'}->{'json_file'})), @@ -403,7 +407,7 @@ sub report_save { Thruk::Utils::IO::mkdir($c->config->{'var_path'}.'/reports/'); my $file = $c->config->{'var_path'}.'/reports/'.$nr.'.rpt'; my $old_report; - if($nr ne 'new' && -f $file) { + if($nr ne 'new' && Thruk::Utils::IO::file_exists($file)) { $old_report = read_report_file($c, $nr); return unless defined $old_report; return if $old_report->{'readonly'}; @@ -508,10 +512,10 @@ sub generate_report { # just wait till its finished and return while($options->{'var'}->{'is_running'}) { sleep 1; - return unless -f $report_file; # report may have been deleted meanwhile + return unless Thruk::Utils::IO::file_exists($report_file); # report may have been deleted meanwhile $options = read_report_file($c, $nr); } - if(-e $attachment) { + if(Thruk::Utils::IO::file_exists($attachment)) { return $attachment; } } @@ -551,8 +555,7 @@ sub generate_report { # empty logfile my $logfile = $c->config->{'var_path'}.'/reports/'.$nr.'.log'; - open(my $fh, '>', $logfile); - Thruk::Utils::IO::close($fh, $logfile); + Thruk::Utils::IO::unlink($logfile); # check for exposed custom variables my $allowed = Thruk::Utils::get_exposed_custom_vars($c->config); @@ -678,7 +681,7 @@ sub generate_report { } # set error if not already set - if(!-f $attachment && !$Thruk::Utils::Reports::error) { + if(!Thruk::Utils::IO::file_exists($attachment) && !$Thruk::Utils::Reports::error) { $Thruk::Utils::Reports::error = Thruk::Utils::IO::read($logfile); } _error($Thruk::Utils::Reports::error); @@ -715,18 +718,20 @@ sub generate_report { } if($c->stash->{'debug_info'}) { - my $debug_file = Thruk::Action::AddDefaults::save_debug_information_to_tmp_file($c); - if($debug_file) { + my $debug_tmp_file = Thruk::Action::AddDefaults::save_debug_information_to_tmp_file($c); + if($debug_tmp_file) { my $rpt_debug_file = $c->config->{'var_path'}.'/reports/'.$nr.'.dbg'; - if(-s $debug_file > 1000000) { - Thruk::Utils::IO::cmd("gzip $debug_file >/dev/null 2>&1"); - if(!-s $debug_file && -s $debug_file.'.gz') { + if(Thruk::Utils::IO::file_not_empty($debug_tmp_file) > 1000000) { + Thruk::Utils::IO::cmd("gzip $debug_tmp_file >/dev/null 2>&1"); + if(!Thruk::Utils::IO::file_not_empty($debug_tmp_file) && Thruk::Utils::IO::file_not_empty($debug_tmp_file.'.gz')) { $rpt_debug_file = $c->config->{'var_path'}.'/reports/'.$nr.'.dbg.gz'; - move($debug_file.'.gz', $rpt_debug_file); + $debug_tmp_file = $debug_tmp_file.'.gz'; } - } else { - move($debug_file, $rpt_debug_file); } + my $dbg = Thruk::Utils::IO::saferead($debug_tmp_file); + Thruk::Utils::IO::write($rpt_debug_file, $dbg); + Thruk::Utils::IO::unlink($debug_tmp_file); + my $patch = {}; Thruk::Utils::IO::json_lock_patch($report_file, { var => { debug_file => $rpt_debug_file } }, { pretty => 1 }); } @@ -1113,10 +1118,10 @@ remove any tmp files from this report =cut sub clean_report_tmp_files { my($c, $nr) = @_; - unlink $c->config->{'var_path'}.'/reports/'.$nr.'.dat'; - unlink $c->config->{'var_path'}.'/reports/'.$nr.'.log'; - unlink $c->config->{'var_path'}.'/reports/'.$nr.'.html'; - unlink $c->config->{'var_path'}.'/reports/'.$nr.'.dbg'; + Thruk::Utils::IO::unlink($c->config->{'var_path'}.'/reports/'.$nr.'.dat'); + Thruk::Utils::IO::unlink($c->config->{'var_path'}.'/reports/'.$nr.'.log'); + Thruk::Utils::IO::unlink($c->config->{'var_path'}.'/reports/'.$nr.'.html'); + Thruk::Utils::IO::unlink($c->config->{'var_path'}.'/reports/'.$nr.'.dbg'); return; } @@ -1236,8 +1241,8 @@ returns list ($running, $waiting) sub get_running_reports_number { my($c) = @_; my $index_file = $c->config->{'var_path'}.'/reports/.index'; - return(0,0) unless -s $index_file; my $index = Thruk::Utils::IO::json_lock_retrieve($index_file); + return(0,0) unless defined $index; my $running = 0; my $waiting = 0; for my $nr (keys %{$index}) { @@ -1308,7 +1313,7 @@ sub store_report_data { # find next free number $nr = 1; $file = $c->config->{'var_path'}.'/reports/'.$nr.'.rpt'; - while(-e $file) { + while(Thruk::Utils::IO::file_exists($file)) { $nr++; $file = $c->config->{'var_path'}.'/reports/'.$nr.'.rpt'; } @@ -1368,7 +1373,7 @@ sub read_report_file { return $c->detach('/error/index/99'); } my $file = $c->config->{'var_path'}.'/reports/'.$nr.'.rpt'; - unless(-f $file) { + unless(Thruk::Utils::IO::file_exists($file)) { _error("report does not exist: $!\n"); $c->stash->{errorMessage} = "report does not exist"; $c->stash->{errorDescription} = "please make sure this report exists."; @@ -1411,8 +1416,7 @@ sub read_report_file { # add some runtime information my $rfile = $c->config->{'var_path'}.'/reports/'.$nr.'.dat'; - $report->{'var'}->{'file_exists'} = 0; - $report->{'var'}->{'file_exists'} = 1 if -f $rfile; + $report->{'var'}->{'file_exists'} = Thruk::Utils::IO::file_exists($rfile) ? 1 : 0; $report->{'var'}->{'is_running'} = 0 unless defined $report->{'var'}->{'is_running'}; $report->{'var'}->{'start_time'} = 0 unless defined $report->{'var'}->{'start_time'}; $report->{'var'}->{'end_time'} = 0 unless defined $report->{'var'}->{'end_time'}; @@ -1484,19 +1488,20 @@ sub read_report_file { $report->{'var'}->{'profile'} = $profile; $needs_save = 1; } - if($report->{'var'}->{'debug_file'} && !-e $report->{'var'}->{'debug_file'}) { + if($report->{'var'}->{'debug_file'} && !Thruk::Utils::IO::file_exists($report->{'var'}->{'debug_file'})) { delete $report->{'var'}->{'debug_file'}; $needs_save = 1; } - if($report->{'var'}->{'json_file'} && !-e $report->{'var'}->{'json_file'}) { + if($report->{'var'}->{'json_file'} && !Thruk::Utils::IO::file_exists($report->{'var'}->{'json_file'})) { delete $report->{'var'}->{'json_file'}; $needs_save = 1; } # failed? $report->{'failed'} = 0; - if(-s $log) { - $report->{'error'} = Thruk::Utils::IO::read($log); + my $logdata = Thruk::Utils::IO::saferead($log); + if(defined $logdata) { + $report->{'error'} = $logdata; # strip performance debug output $report->{'error'} =~ s%^\[.*INFO.*Req:.*$%%gmx; @@ -1616,6 +1621,7 @@ sub _get_report_cmd { $thruk_bin = $nice.' -n '.$niceval.' '.$thruk_bin; } $numbers = Thruk::Base::list($numbers); +# TODO: fix my $cmd = sprintf("cd %s && %s '%s report \"%s\"' >/dev/null 2>%s/reports/%d.log", $c->config->{'project_root'}, $c->config->{'thruk_shell'}, @@ -1755,10 +1761,7 @@ sub _convert_to_pdf { } # write out result - open(my $fh, '>', $htmlfile); - binmode $fh; - print $fh $reportdata; - Thruk::Utils::IO::close($fh, $htmlfile); + Thruk::Utils::IO::write($htmlfile, $reportdata); if($htmlonly) { Thruk::Utils::IO::touch($attachment); @@ -1766,22 +1769,24 @@ sub _convert_to_pdf { return; } +# TODO: my $cmd = $c->config->{home}.'/script/html2pdf.sh "file://'.abs_path($htmlfile).'" "'.$attachment.'.pdf" "'.$logfile.'" "'.($is_report//0).'"'; _debug("converting to pdf: ".$cmd); my $out = Thruk::Utils::IO::cmd($cmd.' 2>&1'); - if(!-e $attachment.'.pdf') { - my $error = Thruk::Utils::IO::read($logfile); + if(!Thruk::Utils::IO::file_exists($attachment.'.pdf')) { + my $error = Thruk::Utils::IO::saferead($logfile); if($error eq "") { $error = $out; } if($error =~ m/internal\/modules\/cjs\/loader\.js/mx) { $error =~ s/^.*?internal\/modules\/cjs\/loader(\.js:\d+|:\d*)\s*throw\s*err;\s*\^\s*Error:/Node Error:/sgmx; # remove useless info from node errors Thruk::Utils::IO::write($logfile, $error); } else { - Thruk::Utils::IO::write($logfile, $error, undef, 1) unless -s $logfile; + Thruk::Utils::IO::write($logfile, $error, undef, 1) unless Thruk::Utils::IO::file_not_empty($logfile); } die('report failed: '.$error); } +# TODO: move($attachment.'.pdf', $attachment) or die('move '.$attachment.'.pdf to '.$attachment.' failed: '.$!); Thruk::Utils::IO::ensure_permissions('file', $attachment); @@ -1856,9 +1861,8 @@ returns nothing =cut sub check_for_waiting_reports { my($c) = @_; - my $index_file = $c->config->{'var_path'}.'/reports/.index'; - return unless -s $index_file; - my $index = Thruk::Utils::IO::json_lock_retrieve($index_file); + my $index = Thruk::Utils::IO::json_lock_retrieve($c->config->{'var_path'}.'/reports/.index'); + return unless $index; for my $nr (keys %{$index}) { if($index->{$nr}->{'is_waiting'}) { generate_report_background($c, $nr, undef, undef, 1); diff --git a/script/grafana_export.sh b/script/grafana_export.sh index d79cab63ce..b8b7a2d048 100755 --- a/script/grafana_export.sh +++ b/script/grafana_export.sh @@ -38,15 +38,15 @@ else export XDG_CACHE_HOME=$XDG_CACHE_HOME fi if [ -z "$NODE_PATH" ] && [ -d "/var/lib/thruk/puppeteer/node_modules" ]; then - export NODE_PATH="/var/lib/thruk/puppeteer/node_modules" + export NODE_PATH="/var/lib/thruk/local/puppeteer/node_modules" if [ -z "$PUPPETEER_EXECUTABLE_PATH" ]; then - export PUPPETEER_EXECUTABLE_PATH=$(ls -1 /var/lib/thruk/puppeteer/chromium/chrome/*/chrome*/chrome 2>/dev/null | head -n 1) + export PUPPETEER_EXECUTABLE_PATH=$(ls -1 /var/lib/thruk/local/puppeteer/chromium/chrome/*/chrome*/chrome 2>/dev/null | head -n 1) fi if [ -z "$PUPPETEER_EXECUTABLE_PATH" -a -x /usr/bin/chromium ]; then export PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium fi - if [ -d "/var/lib/thruk/puppeteer/node" ]; then - NODE="/var/lib/thruk/puppeteer/node/bin/node" + if [ -d "/var/lib/thruk/local/puppeteer/node" ]; then + NODE="/var/lib/thruk/local/puppeteer/node/bin/node" fi fi diff --git a/script/html2pdf.sh b/script/html2pdf.sh index 047e5dcb51..9d841ff063 100755 --- a/script/html2pdf.sh +++ b/script/html2pdf.sh @@ -37,16 +37,16 @@ else export XDG_CONFIG_HOME=/tmp/puppeteer.cache.$(id -u) export XDG_CACHE_HOME=$XDG_CACHE_HOME fi -if [ -z "$NODE_PATH" ] && [ -d "/var/lib/thruk/puppeteer/node_modules" ]; then - export NODE_PATH="/var/lib/thruk/puppeteer/node_modules" +if [ -z "$NODE_PATH" ] && [ -d "/var/lib/thruk/local/puppeteer/node_modules" ]; then + export NODE_PATH="/var/lib/thruk/local/puppeteer/node_modules" if [ -z "$PUPPETEER_EXECUTABLE_PATH" ]; then - export PUPPETEER_EXECUTABLE_PATH=$(ls -1 /var/lib/thruk/puppeteer/chromium/chrome/*/chrome*/chrome 2>/dev/null | head -n 1) + export PUPPETEER_EXECUTABLE_PATH=$(ls -1 /var/lib/thruk/local/puppeteer/chromium/chrome/*/chrome*/chrome 2>/dev/null | head -n 1) fi if [ -z "$PUPPETEER_EXECUTABLE_PATH" -a -x /usr/bin/chromium ]; then export PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium fi - if [ -d "/var/lib/thruk/puppeteer/node" ]; then - NODE="/var/lib/thruk/puppeteer/node/bin/node" + if [ -d "/var/lib/thruk/local/puppeteer/node" ]; then + NODE="/var/lib/thruk/local/puppeteer/node/bin/node" fi fi diff --git a/script/install_puppeteer.sh b/script/install_puppeteer.sh index cdb4d9e4d1..9b92e13d35 100755 --- a/script/install_puppeteer.sh +++ b/script/install_puppeteer.sh @@ -2,7 +2,7 @@ # # usage: ./install_puppeteer.sh # -# installs puppeteer into /var/lib/thruk/puppeteer so it can be used to render PDFs +# installs puppeteer into /var/lib/thruk/local/puppeteer so it can be used to render PDFs # ########################################################### @@ -13,7 +13,7 @@ PUPPETEER_VERSION=21.11.0 # the last one with node 16 support # target folder DEST=$1 if [ -z "$DEST" ]; then - DEST=/var/lib/thruk/puppeteer + DEST=/var/lib/thruk/local/puppeteer fi NPM="npm" diff --git a/script/thruk_auth b/script/thruk_auth index bc287351ee..01e828a3e3 100755 --- a/script/thruk_auth +++ b/script/thruk_auth @@ -156,7 +156,7 @@ sub process { # direct login by secret/api key if($extra->[2] && $extra->[2] =~ m/[a-zA-Z0-9_]+/mx) { - my $secret_key = Thruk::Utils::IO::read($secret_file) if -s $secret_file; + my $secret_key = Thruk::Utils::IO::saferead($secret_file); chomp($secret_key) if $secret_key; if($secret_key && $extra->[2] eq $secret_key) { if(!$extra->[3]) { diff --git a/t/090-io.t b/t/090-io.t index 82cdaa2212..312bb9ffd6 100644 --- a/t/090-io.t +++ b/t/090-io.t @@ -19,24 +19,26 @@ my $c = TestUtils::get_c(); use_ok("Thruk::Utils::IO"); +my $srcfolders = "lib/ plugins/plugins-available/*/lib"; my $cmds = [ - "grep -nr 'close\(' lib/ plugins/plugins-available/*/lib", "better use Thruk::Utils::IO::close", - "grep -nr 'mkdir\(' lib/ plugins/plugins-available/*/lib", "better use Thruk::Utils::IO::mkdir", - "grep -nr 'chown\(' lib/ plugins/plugins-available/*/lib", "better use Thruk::Utils::IO::ensure_permissions", - "grep -nr 'chmod\(' lib/ plugins/plugins-available/*/lib", "better use Thruk::Utils::IO::ensure_permissions", - "grep -Pnr 'sleep\\(\\d+\\.' lib/ plugins/plugins-available/*/lib", "better use Time::HiRes::sleep directly", - "grep -nr 'File::Slurp' t/ lib/ plugins/plugins-available/*/lib", "better use Thruk::Utils::IO::read", + "grep -nr 'close\(' $srcfolders", "better use Thruk::Utils::IO::close", + "grep -nr 'mkdir\(' $srcfolders", "better use Thruk::Utils::IO::mkdir", + "grep -nr 'chown\(' $srcfolders", "better use Thruk::Utils::IO::ensure_permissions", + "grep -nr 'chmod\(' $srcfolders", "better use Thruk::Utils::IO::ensure_permissions", + "grep -Enr 'sleep\\([0-9]+\\.' $srcfolders", "better use Time::HiRes::sleep directly", + "grep -nr 'File::Slurp' t/ $srcfolders", "better use Thruk::Utils::IO::read", ]; +my $iocmd = "grep -Enr -- '(\\-s|\\-f|\\-d|unlink\\(|stat\\(|opendir\\(|open\\(|move\\(|glob\\() ' $srcfolders"; # find all close / mkdirs not ensuring permissions my @fails; while(scalar @{$cmds} > 0) { my $cmd = shift @{$cmds}; my $desc = shift @{$cmds}; - open(my $ph, '-|', $cmd.' 2>&1') or die('cmd '.$cmd.' failed: '.$!); - ok($ph, 'cmd '.$cmd.' started'); - while(<$ph>) { - my $line = $_; + ok(1, $cmd); + my($rc, $out) = Thruk::Utils::IO::cmd($cmd); + ok($rc == 0, "rc: $rc"); + for my $line (split m/\n/mx, $out) { chomp($line); $line =~ s|//|/|gmx; @@ -46,6 +48,7 @@ while(scalar @{$cmds} > 0) { next if $line =~ m|STDOUT|mx; next if $line =~ m|POSIX::close|mx; next if $line =~ m|Thruk/Utils/IO\.pm:|mx; + next if $line =~ m|Thruk/Utils/IO/LocalFS\.pm:|mx; next if $line =~ m|Thruk::Utils::IO::close|mx; next if $line =~ m|Thruk::Utils::IO::mkdir|mx; next if $line =~ m|CORE::|mx; @@ -57,14 +60,44 @@ while(scalar @{$cmds} > 0) { next if $line =~ m|\->close|mx; next if $line =~ m|Time::HiRes|mx; - push @fails, $desc." in\n".$line; + fail($desc." in\n".$line); } - close($ph); - ok($? == 0, "exit code is: ".$?." (cmd: ".$cmd.")"); } -for my $fail (sort @fails) { - fail($fail); +{ + ok(1, $iocmd); + my($rc, $out) = Thruk::Utils::IO::cmd($iocmd); + ok($rc == 0, "rc: $rc"); + for my $line (split m/\n/mx, $out) { + chomp($line); + $line =~ s|//|/|gmx; + next if $line =~ m|:\s*\#|mx; + next if $line =~ m|/tmp/|mx; + next if $line =~ m|tmp_path|mx; + next if $line =~ m|/local/|mx; + next if $line =~ m|pidfile|mx; + next if $line =~ m|logcache|mx; + next if $line =~ m|thruk_local.conf|mx; + next if $line =~ m|thruk.conf|mx; + next if $line =~ m|thruk_local.d|mx; + next if $line =~ m|\$addon|mx; + next if $line =~ m|/version|mx; + next if $line =~ m|usercontent|mx; + next if $line =~ m|/root/|mx; + next if $line =~ m|project_root|mx; + next if $line =~ m|script/|mx; + next if $line =~ m|scriptfolder|mx; + next if $line =~ m|plugin_enabled_dir|mx; + next if $line =~ m|/plugins-available/|mx; + next if $line =~ m|spool folder|mx; + next if $line =~ m|route_file|mx; + next if $line =~ m|_info|mx; + next if $line =~ m|\Qlib/Monitoring/Config\E|mx; + next if $line =~ m|\Qlib/Thruk/Utils/LMD\E|mx; + next if $line =~ m|\Qlib/Thruk/Utils/IO/LocalFS.pm\E|mx; + + fail("direct file access in\n".$line); + } } my($rc, $output) = Thruk::Utils::IO::cmd('ls -la'); diff --git a/t/092-backticks.t b/t/092-backticks.t index 48d5b18347..a52835ee80 100644 --- a/t/092-backticks.t +++ b/t/092-backticks.t @@ -20,12 +20,14 @@ for my $cmd (@{$cmds}) { while(my $line = <$ph>) { chomp($line); $line =~ s/'.*?'//gmx; + $line =~ s=/+=/=gmx; next if $line =~ m/nasty\ chars/mx; next if $line =~ m/\Qcontains invalid characters\E/mx; next if $line =~ m/NO\ CRITIC\ BACKTICKS/mx; $line =~ s/\#.*$//gmx; next if($filter && $line !~ m%$filter%mx); next if $line =~ m%\QThruk/Utils/IO.pm:\E%mx; + next if $line =~ m%\QThruk/Utils/IO/LocalFS.pm:\E%mx; next if $line =~ m/(CREATE|ALTER|TRUNCATE|OPTIMIZE|DROP|LOCK)\ TABLE/mx; next if $line =~ m/LEFT\ JOIN/mx; next if $line =~ m/INSERT\ INTO/mx; diff --git a/t/data/user_overrides/thruk.conf b/t/data/user_overrides/thruk.conf index 1656e23399..fda87e5346 100644 --- a/t/data/user_overrides/thruk.conf +++ b/t/data/user_overrides/thruk.conf @@ -22,7 +22,7 @@ user_password_min_length = 5 name = extra type = livestatus - peer = 127.0.0.2:12345 + peer = 127.0.0.1:12345 diff --git a/t/scenarios/_common/Makefile.common b/t/scenarios/_common/Makefile.common index ce0913a017..ec24c2775f 100644 --- a/t/scenarios/_common/Makefile.common +++ b/t/scenarios/_common/Makefile.common @@ -15,6 +15,8 @@ else endif export COMPOSE_HTTP_TIMEOUT=180 +.PHONY: thruk + wait_start: $(THRUK) cache clean for x in $$(seq $(STARTUPWAIT)); do \ diff --git a/t/scenarios/_common/ansible/roles/ssh_site_login/tasks/main.yml b/t/scenarios/_common/ansible/roles/ssh_site_login/tasks/main.yml index 02a8e98d41..138d870c6c 100644 --- a/t/scenarios/_common/ansible/roles/ssh_site_login/tasks/main.yml +++ b/t/scenarios/_common/ansible/roles/ssh_site_login/tasks/main.yml @@ -6,7 +6,10 @@ state: present # hack because ansibles takes over 10seconds, even if it does not install anything when: not passwd_path.stat.exists -- shell: sudo su - {{ site }} -c "mkdir -p .ssh && chmod 700 .ssh && ssh-keygen -t ed25519 -f .ssh/id_ed25519 -N '' && cp .ssh/id_ed25519.pub .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" +- name: create ssh keys + shell: sudo su - {{ site }} -c "mkdir -p .ssh && chmod 700 .ssh && ssh-keygen -t ed25519 -f .ssh/id_ed25519 -N '' && cp .ssh/id_ed25519.pub .ssh/authorized_keys && chmod 600 .ssh/authorized_keys" + args: + creates: /omd/sites/{{ site }}/.ssh/id_ed25519.pub - name: create .ssh/config copy: content: "Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n LogLevel QUIET\n" diff --git a/t/scenarios/_common/base_images/rocky-nightly/Dockerfile b/t/scenarios/_common/base_images/rocky-nightly/Dockerfile index d812685e4d..9f611569a0 100644 --- a/t/scenarios/_common/base_images/rocky-nightly/Dockerfile +++ b/t/scenarios/_common/base_images/rocky-nightly/Dockerfile @@ -13,5 +13,7 @@ RUN omd config demo set GRAFANA on; omd start demo grafana; omd stop demo; omd c # improve startup time of influxdb after first launch RUN omd config demo set INFLUXDB on; omd start demo influxdb; omd stop demo; omd config demo set INFLUXDB off +RUN echo -e "#!/bin/bash\n\nansible-playbook -i localhost, \"/provision/playbook.yml\" -c local -e SITENAME=\$SITENAME\n" > /root/ansible_provision.sh; chmod 755 /root/ansible_provision.sh + # show version string RUN echo "omd-labs-rocky:nightly:omd version: $(omd version -b)" diff --git a/t/scenarios/cluster_db_e2e/.gitignore b/t/scenarios/cluster_db_e2e/.gitignore new file mode 100644 index 0000000000..643afdac90 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/.gitignore @@ -0,0 +1 @@ +_run diff --git a/t/scenarios/cluster_db_e2e/Makefile b/t/scenarios/cluster_db_e2e/Makefile new file mode 100644 index 0000000000..a1ad5047a9 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/Makefile @@ -0,0 +1,24 @@ +include ../_common/Makefile.common + +export THRUK_TEST_AUTH = omdadmin:omd +export PLACK_TEST_EXTERNALSERVER_URI = http://127.0.0.3:60080/demo/ + +wait_start: + for x in $$(seq $(STARTUPWAIT)); do \ + if [ $$(docker compose logs | grep "failed=" | grep -v "failed=0" | wc -l) -gt 0 ]; then $(MAKE) wait_start_failed; exit 1; fi; \ + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo thruk cluster ping >/dev/null 2>&1; \ + if [ $$(docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo thruk cluster status | grep -c OK) -eq 3 ]; then break; else sleep 1; fi; \ + if [ $$x -eq $(STARTUPWAIT) ]; then $(MAKE) wait_start_failed; exit 1; fi; \ + done + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo thruk cluster ping + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo thruk cluster status + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo thruk r -m POST /thruk/cluster/heartbeat + +wait_start_failed_extra: + -curl -kv http://127.0.0.3:60080/demo/thruk/cgi-bin/login.cgi + +extra_test: + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo /usr/local/bin/local_test.sh 0 $(filter-out $@,$(MAKECMDGOALS)) + +extra_test_verbose: + docker compose exec $(DOCKER_COMPOSE_TTY) --user root thruk sudo -iu demo /usr/local/bin/local_test.sh 1 $(filter-out $@,$(MAKECMDGOALS)) diff --git a/t/scenarios/cluster_db_e2e/README b/t/scenarios/cluster_db_e2e/README new file mode 100644 index 0000000000..530df7e82e --- /dev/null +++ b/t/scenarios/cluster_db_e2e/README @@ -0,0 +1,14 @@ +Entry Point +=========== + + - http://127.0.0.3:60080/demo/thruk/ (traefik loadbalancer) + - Login is omdadmin / omd + +Setup +===== + + - 2 clustered thruk nodes based on the thruk/ folder + - 1 traefik load balancer + - 1 omd site used as backend for the thruk nodes + also used as mysql database + - use "./scale " to change number of thruk containers diff --git a/t/scenarios/cluster_db_e2e/docker-compose.yml b/t/scenarios/cluster_db_e2e/docker-compose.yml new file mode 100644 index 0000000000..f64a6c2a5f --- /dev/null +++ b/t/scenarios/cluster_db_e2e/docker-compose.yml @@ -0,0 +1,55 @@ +services: + lb: + image: 'traefik:v3.2' + command: + - "--api=true" + - "--api.dashboard=true" + - "--api.insecure=true" + - "--providers.docker=true" + - "--log.level=DEBUG" + - "--accesslog" + - "--entrypoints.web.address=:80" + - "--providers.docker.exposedbydefault=false" + ports: + - "127.0.0.3:60080:80" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - thruk + labels: + - "traefik.enable=true" + - "traefik.http.routers.dashboard.entrypoints=web" + - "traefik.http.routers.dashboard.rule=(PathPrefix(`/api`) || PathPrefix(`/dashboard`))" + - "traefik.http.routers.dashboard.service=api@internal" + + omd: + build: omd/ + environment: + - TZ=Europe/Berlin + volumes: + - ../../../:/thruk:ro + - ./omd:/provision:ro + ports: + - "127.0.0.3:60557:6557" + + thruk: + build: thruk/ + deploy: + replicas: 3 + ports: + - "80" + environment: + - TZ=Europe/Berlin + labels: + - "traefik.enable=true" + - "traefik.http.routers.thruk.entrypoints=web" + - "traefik.http.routers.thruk.service=thruk" + - "traefik.http.routers.thruk.rule=PathPrefix(`/demo`)" + - "traefik.http.services.thruk.loadbalancer.server.port=80" + - "traefik.http.services.thruk.loadbalancer.healthcheck.port=80" + - "traefik.http.services.thruk.loadbalancer.healthcheck.path=/demo/thruk/cgi-bin/remote.cgi?lb_ping" + - "traefik.http.services.thruk.loadbalancer.healthcheck.interval=3s" + volumes: + - ../../../:/thruk:ro + - .:/test:ro + - ./thruk:/provision:ro diff --git a/t/scenarios/cluster_db_e2e/omd/Dockerfile b/t/scenarios/cluster_db_e2e/omd/Dockerfile new file mode 100644 index 0000000000..b90ebd7a6b --- /dev/null +++ b/t/scenarios/cluster_db_e2e/omd/Dockerfile @@ -0,0 +1,5 @@ +FROM local/thruk-labs-rocky:nightly + +COPY playbook.yml /root/ansible_dropin/ +ENV ANSIBLE_ROLES_PATH=/thruk/t/scenarios/_common/ansible/roles +COPY test.cfg /root/ diff --git a/t/scenarios/cluster_db_e2e/omd/playbook.yml b/t/scenarios/cluster_db_e2e/omd/playbook.yml new file mode 100644 index 0000000000..487b523834 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/omd/playbook.yml @@ -0,0 +1,51 @@ +--- +- hosts: all + roles: + - role: common + tasks: + - name: "omd config change" + shell: "omd config demo change" + args: + stdin: | + APACHE_MODE=own + PNP4NAGIOS=off + LIVESTATUS_TCP=on + MYSQL=on + - name: copy naemon example.cfg + copy: + src: /omd/sites/demo/share/doc/naemon/example.cfg + dest: /omd/sites/demo/etc/naemon/conf.d/example.cfg + owner: demo + group: demo + - name: copy naemon test.cfg + copy: + src: /root/test.cfg + dest: /omd/sites/demo/etc/naemon/conf.d/test.cfg + owner: demo + group: demo + - shell: echo "testkey" > /omd/sites/demo/var/thruk/secret.key + - file: + path: /omd/sites/demo/var/thruk/secret.key + mode: 0600 + owner: demo + group: demo + - name: enable mysql networking + lineinfile: + dest: /opt/omd/sites/demo/.my.cnf + line: "#skip-networking" + regexp: '^\#?skip-networking' + - name: change mysql bind address + lineinfile: + dest: /opt/omd/sites/demo/.my.cnf + line: "bind-address = 0.0.0.0" + regexp: 'bind-address.*=' + - name: start mysql + shell: omd start demo mysql + - name: create mysql table + shell: sudo su - demo -c 'echo "CREATE DATABASE IF NOT EXISTS thruk_var_fs" | mysql' + - name: create mysql user + shell: sudo su - demo -c 'echo "CREATE USER IF NOT EXISTS \"thruk\"@\"%\" IDENTIFIED BY \"thruk\"" | mysql' + - name: grant mysql privileges + shell: sudo su - demo -c 'echo "GRANT ALL PRIVILEGES ON \`thruk_var_fs\`.* TO \"thruk\"@\"%\"" | mysql' + - name: stop mysql + shell: omd stop demo mysql diff --git a/t/scenarios/cluster_db_e2e/omd/test.cfg b/t/scenarios/cluster_db_e2e/omd/test.cfg new file mode 100644 index 0000000000..78c47c6e0f --- /dev/null +++ b/t/scenarios/cluster_db_e2e/omd/test.cfg @@ -0,0 +1,12 @@ +define servicegroup { + servicegroup_name Http Check + alias Http Checks + members localhost,Http +} + +define hostgroup { + hostgroup_name Everything + alias Just all hosts + members * +} + diff --git a/t/scenarios/cluster_db_e2e/scale b/t/scenarios/cluster_db_e2e/scale new file mode 100755 index 0000000000..e205a6fc16 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/scale @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "x$1" = "x" ]; then + echo "usage: $0 " + exit 2 +fi + +docker compose up -d --scale=thruk=$1 diff --git a/t/scenarios/cluster_db_e2e/t/300-controller_cluster.t b/t/scenarios/cluster_db_e2e/t/300-controller_cluster.t new file mode 100644 index 0000000000..9fc240f78a --- /dev/null +++ b/t/scenarios/cluster_db_e2e/t/300-controller_cluster.t @@ -0,0 +1,41 @@ +use warnings; +use strict; +use Test::More; + +BEGIN { + plan tests => 56; + + use lib('t'); + require TestUtils; + import TestUtils; +} + +TestUtils::set_test_user_token(); + +# since the cluster is round-robin, this should trigger an update on each node +for my $x (1..3) { + TestUtils::test_page( + 'url' => '/thruk/r/thruk/cluster/heartbeat', + 'post' => {}, + 'waitfor' => 'heartbeat\ send', + ); +} + +# since the cluster is round-robin, this should trigger an update on each node +for my $x (1..3) { + TestUtils::test_page( + 'url' => '/thruk/r/thruk/cluster/heartbeat', + 'post' => {}, + 'like' => ['heartbeat send'], + ); +} + +TestUtils::test_page( + 'url' => '/thruk/cgi-bin/extinfo.cgi?type=4&cluster=1', + 'like' => ['Performance Information', 'Cluster Status', ']+"ok"[^>]*>'], +); + +TestUtils::test_page( + 'url' => '/thruk/cgi-bin/proxy.cgi/e0364/demo/thruk/cgi-bin/tac.cgi', + 'like' => ['Tactical Monitoring Overview'], +); diff --git a/t/scenarios/cluster_db_e2e/t/300-controller_rest_v1.t b/t/scenarios/cluster_db_e2e/t/300-controller_rest_v1.t new file mode 100644 index 0000000000..158423ef97 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/t/300-controller_rest_v1.t @@ -0,0 +1,364 @@ +use warnings; +use strict; +use Cpanel::JSON::XS qw/decode_json/; +use Test::More; +use URI::Escape qw/uri_escape/; + +use Thruk::Utils::IO (); + +BEGIN { + plan skip_all => 'backends required' if(!-s 'thruk_local.conf' and !defined $ENV{'PLACK_TEST_EXTERNALSERVER_URI'}); + plan tests => 588; +} + +BEGIN { + use lib('t'); + require TestUtils; + import TestUtils; +} +BEGIN { use_ok 'Thruk::Controller::rest_v1' } + +TestUtils::set_test_user_token(); +my($host,$service) = TestUtils::get_test_service(); + +my $list_pages = [ + '/', + '/v1/', + '/index', + '/sites', + '/config/diff', + '/config/precheck', + '/config/files', + '/config/objects', + '/config/fullobjects', + '/commands', + '/comments', + '/contactgroups', + '/contacts', + '/downtimes', + '/hostgroups', + '/hosts', + '/hosts/availability', + '/hosts/'.uri_escape($host), + '/hosts/'.uri_escape($host).'/services', + '/hosts/outages', + '/hosts/'.uri_escape($host).'/outages', + '/logs', + '/alerts', + '/notifications', + '/processinfo', + '/servicegroups', + '/services', + '/services/availability', + '/services/outages', + '/services/'.uri_escape($host).'/'.uri_escape($service), + '/services/'.uri_escape($host).'/'.uri_escape($service).'/outages', + '/timeperiods', + '/lmd/sites', + '/thruk/bp', + '/thruk/cluster', + '/thruk/recurring_downtimes', + '/thruk/jobs', + '/thruk/panorama', + '/thruk/reports', + '/thruk/broadcasts', + '/thruk/sessions', + '/thruk/users', + '/thruk/api_keys', + '/thruk/logcache/stats', +]; + +my $hash_pages = [ + '/checks/stats', + '/hosts/stats', + '/hosts/totals', + '/hosts/'.uri_escape($host).'/availability', + '/processinfo/stats', + '/services/stats', + '/services/totals', + '/services/'.uri_escape($host).'/'.uri_escape($service).'/availability', + '/thruk', + '/thruk/config', + '/thruk/stats', + '/thruk/metrics', + '/thruk/whoami', +]; + +# get config from rest endpoint +my $config = {}; +{ + my $page = TestUtils::test_page( + 'url' => '/thruk/r/thruk/config', + 'content_type' => 'application/json; charset=utf-8', + ); + $config = decode_json($page->{'content'}); +} + +for my $url (@{$list_pages}) { + SKIP: { + skip "skipped, logcache is disabled ", 8 if ($url =~ m/logcache/mx && !$config->{'logcache'}); + + if($url =~ m/logs/mx) { + $url = $url.'?limit=100'; + } + + my $page = TestUtils::test_page( + 'url' => '/thruk/r'.$url, + 'content_type' => 'application/json; charset=utf-8', + ); + my $data = decode_json($page->{'content'}); + is(ref $data, 'ARRAY', "json result is an array: ".$url); + }; +} + +for my $url (@{$hash_pages}) { + my $page = TestUtils::test_page( + 'url' => '/thruk/r'.$url, + 'content_type' => 'application/json; charset=utf-8', + ); + my $data = decode_json($page->{'content'}); + is(ref $data, 'HASH', "json result is a hash: ".$url); +} + +################################################################################ +my $content = Thruk::Utils::IO::read(__FILE__); +my($paths, $keys, $docs) = Thruk::Controller::rest_v1::get_rest_paths(); +for my $p (sort keys %{$paths}) { + if($paths->{$p}->{'GET'}) { + next if $p =~ m%<%mx; + next if $p =~ m%heartbeat%mx; + next if $p =~ m%/editor%mx; + next if $p =~ m%/nc/%mx; + next if $p =~ m%/node-control/%mx; + if($content !~ m%$p%mx) { + fail("missing test case for ".$p); + } + } +} + +################################################################################ +# check if there is a doc entry for every registered path and method +for my $registered (@{$Thruk::Controller::rest_v1::rest_paths}) { + my($method, $regex) = @{$registered}; + my $found = 0; + for my $p (sort keys %{$paths}) { + my $available_methods = $paths->{$p}; + $p =~ s/<[^>]*>/0/gmx; + if($p =~ $regex && $available_methods->{$method}) { + $found = 1; + } + } + if(!$found) { + fail("missing documentation for $method $regex"); + } +} + +################################################################################ +# make sure PUT requests are handled like POST +TestUtils::test_page( + 'url' => '/thruk/r/thruk/reports', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'PUT', + 'post' => {}, + 'like' => ['invalid report template'], + 'fail' => 1, +); + +################################################################################ +{ + my $c = TestUtils::get_c(); + _set_params($c, {'test' => 1}); + my $filter = Thruk::Controller::rest_v1::_livestatus_filter($c); + my $expect = [{ 'test' => { '=' => 1 }}]; + is_deeply($filter, $expect, "simple livestatus filter"); + + _set_params($c, { q => 'host = "test" and time > 1 and time < 10'}); + $filter = Thruk::Controller::rest_v1::_livestatus_filter($c); + $expect = [[{ + '-and' => [ + { 'host_name' => { '=' => 'test' } }, + { 'time' => { '>' => '1' } }, + { 'time' => { '<' => '10' } } + ] + }]]; + is_deeply($filter, $expect, "simple livestatus filter"); + + _set_params($c, { q => '_CITY = "Munich" and rta > 1'}); + $filter = Thruk::Controller::rest_v1::_livestatus_filter($c); + $expect = [[{ + '-and' => [ + { '_CITY' => { '=' => 'Munich' } }, + { 'rta' => { '>' => '1' } }, + ] + }]]; + is_deeply($filter, $expect, "simple livestatus filter"); +}; + +################################################################################ +# test query filter +{ + TestUtils::test_page( + 'url' => '/thruk/r/logs?q=***host_name = "test" AND time > 1 AND time < 10***', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['\[\]'], + ); +}; + +################################################################################ +# test query filter II +{ + TestUtils::test_page( + 'url' => '/thruk/r/logs?q=***host_name = "test" AND (time > 1 AND time < 10)***', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['\[\]'], + ); +}; + +################################################################################ +# test query filter when the filtered item is not in the columns list +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name&state[ne]=5', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['name'], + ); +}; + +################################################################################ +# test query filter when the filtered item is not in the columns list II +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name&q=***state >= 0***', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['name'], + ); +}; + +################################################################################ +# test query filter when the filtered item is not in the columns list III +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name&state[gte]=0', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['name'], + ); +}; + +################################################################################ +# test sorting empty result set +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?q=***(groups>="does not exist")***&sort=_UNKNOWN_CUSTOM_VAR', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['\[\]'], + ); +}; + +################################################################################ +# test columns when no column given +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?q=***(name != "does not exist")***', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['"state"'], + ); +}; + +################################################################################ +# test count(*) with no matches +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?state=-1&columns=count(*)', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['count\(\*\)', '0'], + ); +}; + +################################################################################ +# test aggregation with renamed labels +{ + TestUtils::test_page( + 'url' => '/thruk/r/thruk/sessions?columns=count(*):renamed_label&active=-99', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['"renamed_label" : 0'], + ); +}; + +################################################################################ +# normal query with renamed labels +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name:renamed_label&limit=1', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['"renamed_label"'], + ); +}; + +################################################################################ +# normal query with unknown columns +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name,contacts,UNKNOWN', + 'content_type' => 'application/json; charset=utf-8', + 'method' => 'GET', + 'like' => ['"contacts"', '"UNKNOWN"'], + 'unlike' => ['"contacts" : null,'], + ); +}; + +################################################################################ +# csv output +{ + TestUtils::test_page( + 'url' => '/thruk/r/csv/hosts?columns=name,contacts', + 'content_type' => 'text/plain; charset=utf-8', + 'method' => 'GET', + 'like' => ['name;contacts'], + 'unlike' => ['ARRAY'], + ); +}; + +################################################################################ +# csv output +{ + TestUtils::test_page( + 'url' => '/thruk/r/xls/hosts?columns=name,contacts', + 'content_type' => 'application/x-msexcel', + 'method' => 'GET', + 'like' => ['Arial1'], + 'unlike' => ['ARRAY'], + ); +}; + +################################################################################ +# peer_name / peer_key +{ + TestUtils::test_page( + 'url' => '/thruk/r/hosts?columns=name,peer_name,peer_key&name='.$host, + 'method' => 'GET', + 'like' => [$host, 'peer_name', 'peer_key', 'name'], + 'unlike' => ['ARRAY'], + ); +}; + +################################################################################ +sub _set_params { + my($c, $params) = @_; + for my $key (keys %{$c->req->parameters}) { + delete $c->req->parameters->{$key}; + } + for my $key (keys %{$params}) { + $c->req->parameters->{$key} = $params->{$key}; + } +} +################################################################################ diff --git a/t/scenarios/cluster_db_e2e/t/300-controller_tac.t b/t/scenarios/cluster_db_e2e/t/300-controller_tac.t new file mode 100644 index 0000000000..9bdc43c1a6 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/t/300-controller_tac.t @@ -0,0 +1,41 @@ +use warnings; +use strict; +use Cpanel::JSON::XS qw/decode_json/; +use Test::More; + +BEGIN { + plan skip_all => 'backends required' if(!-s 'thruk_local.conf' and !defined $ENV{'PLACK_TEST_EXTERNALSERVER_URI'}); + plan tests => 20; +} + +BEGIN { + use lib('t'); + require TestUtils; + import TestUtils; +} +BEGIN { use_ok 'Thruk::Controller::tac' } + +my $pages = [ + '/thruk/cgi-bin/tac.cgi', +]; + +for my $url (@{$pages}) { + TestUtils::test_page( + 'url' => $url, + 'like' => 'Tactical Monitoring Overview', + ); +} + +# json pages +$pages = [ + '/thruk/cgi-bin/tac.cgi?view_mode=json', +]; + +for my $url (@{$pages}) { + my $page = TestUtils::test_page( + 'url' => $url, + 'content_type' => 'application/json; charset=utf-8', + ); + my $data = decode_json($page->{'content'}); + is(ref $data, 'HASH', "json result is an array: ".$url); +} diff --git a/t/scenarios/cluster_db_e2e/t/local/cluster.t b/t/scenarios/cluster_db_e2e/t/local/cluster.t new file mode 100644 index 0000000000..2c82e45e98 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/t/local/cluster.t @@ -0,0 +1,93 @@ +use warnings; +use strict; +use Test::More; + +BEGIN { + use lib('t'); + require TestUtils; + import TestUtils; +} + +plan tests => 80; + +########################################################### +# verify that we use the correct thruk binary +TestUtils::test_command({ + cmd => '/bin/bash -c "type thruk"', + like => ['/\/thruk\/script\/thruk/'], +}) or BAIL_OUT("wrong thruk path"); + +########################################################### +# thruk cluster commands +TestUtils::test_command({ cmd => '/bin/mv .thruk .thruk.off' }); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk r -m POST /thruk/cluster/heartbeat', + like => ['/heartbeat send/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk r /thruk/cluster', + like => ['/"node_url"/', '/"last_error" : "",/', '/"response_time" : 0./'], +}); + +########################################################### +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster status', + like => ['/OK/', '/nodes online/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster ping', + like => ['/heartbeat send/'], +}); + +########################################################### +# maint mode +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster maint', + like => ['/OK/', '/set\ into\ maintenance\ mode/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster status', + like => ['/OK/', '/nodes online/', '/MAINT/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env omd stop', + like => ['/Stopping.*OK/'], + errlike => ['/.*/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env omd umount', + like => ['/Cleaning\ up\ temp\ filesystem/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env omd start', + like => ['/Preparing\ tmp\ directory/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env curl -s "http://localhost/demo/thruk/cgi-bin/remote.cgi?lb_ping"', + like => ['/MAINTENANCE/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster ping', + like => ['/heartbeat send/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster status', + like => ['/OK/', '/nodes online/', '/MAINT/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster unmaint', + like => ['/OK\ \-\ removed\ maintenance\ mode/'], +}); +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster status', + like => ['/OK/', '/nodes online/'], + unlike => ['/MAINT/'], +}); + +########################################################### +TestUtils::test_command({ + cmd => '/usr/bin/env thruk cluster restart', + like => ['/all cluster nodes restarted/'], +}); +TestUtils::test_command({ cmd => '/bin/mv .thruk.off .thruk' }); +########################################################### diff --git a/t/scenarios/cluster_db_e2e/thruk/1.rpt b/t/scenarios/cluster_db_e2e/thruk/1.rpt new file mode 100644 index 0000000000..e60d672280 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/1.rpt @@ -0,0 +1,47 @@ +{ + "backends" : [], + "cc" : "", + "desc" : "Example Description", + "failed_backends" : "cancel", + "is_public" : 0, + "name" : "Test Report", + "params" : { + "assumeinitialstates" : "yes", + "breakdown" : "days", + "decimals" : "2", + "details_max_level" : "-1", + "graph_min_sla" : "90", + "host" : "localhost", + "includesoftstates" : "no", + "initialassumedhoststate" : "0", + "language" : "en", + "mail_max_level" : "-1", + "max_outages_pages" : "1", + "max_pnp_sources" : "1", + "max_worst_pages" : "1", + "rpttimeperiod" : "", + "sla" : "98", + "t1" : 1526456460, + "t2" : 1526542860, + "timeperiod" : "last24hours", + "unavailable" : [ + "down", + "unreachable" + ] + }, + "send_types" : [ + { + "cust" : "* * * * *", + "day" : "1", + "hour" : "0", + "minute" : "0", + "month_day" : "1st_Monday", + "type" : "cust", + "week_day" : "" + } + ], + "template" : "sla_host.tt", + "to" : "", + "user" : "omdadmin", + "var" : {} +} diff --git a/t/scenarios/cluster_db_e2e/thruk/1.tbp b/t/scenarios/cluster_db_e2e/thruk/1.tbp new file mode 100644 index 0000000000..39852c448a --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/1.tbp @@ -0,0 +1,82 @@ +{ + "filter" : [], + "name" : "Test Business Process", + "nodes" : [ + { + "contactgroups" : [], + "contacts" : [], + "create_obj" : 1, + "depends" : [ + "node2", + "node5" + ], + "filter" : [], + "function" : "worst()", + "id" : "node1", + "label" : "Test Business Process" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [ + "node3", + "node4" + ], + "filter" : [], + "function" : "worst()", + "id" : "node2", + "label" : "Web" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [], + "filter" : [], + "function" : "status('localhost', 'Http', '=')", + "id" : "node3", + "label" : "none" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [], + "filter" : [], + "function" : "status('localhost', 'Https', '=')", + "id" : "node4", + "label" : "none" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [ + "node6", + "node7" + ], + "filter" : [], + "function" : "worst()", + "id" : "node5", + "label" : "App" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [], + "filter" : [], + "function" : "status('localhost', 'Users', '=')", + "id" : "node6", + "label" : "none" + }, + { + "contactgroups" : [], + "contacts" : [], + "depends" : [], + "filter" : [], + "function" : "status('localhost', 'Disk /', '=')", + "id" : "node7", + "label" : "none" + } + ], + "rankDir" : "TB", + "state_type" : "both", + "template" : "" +} diff --git a/t/scenarios/cluster_db_e2e/thruk/1.tsk b/t/scenarios/cluster_db_e2e/thruk/1.tsk new file mode 100644 index 0000000000..70411ff7d7 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/1.tsk @@ -0,0 +1,28 @@ +{ + "backends" : [], + "childoptions" : 0, + "comment" : "automatic downtime", + "duration" : "1", + "fixed" : "1", + "flex_range" : "1", + "host" : [ + "localhost" + ], + "hostgroup" : [], + "schedule" : [ + { + "cust" : "* * * * *", + "day" : "1", + "hour" : "0", + "minute" : "0", + "month_day" : "1st_Monday", + "type" : "cust", + "week_day" : "" + } + ], + "service" : [ + "Https Cert" + ], + "servicegroup" : [], + "target" : "service" +} diff --git a/t/scenarios/cluster_db_e2e/thruk/Dockerfile b/t/scenarios/cluster_db_e2e/thruk/Dockerfile new file mode 100644 index 0000000000..3153bae2d9 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/Dockerfile @@ -0,0 +1,10 @@ +FROM local/thruk-labs-rocky:nightly + +COPY playbook.yml /root/ansible_dropin/ +ENV ANSIBLE_ROLES_PATH=/thruk/t/scenarios/_common/ansible/roles +COPY test.cfg /root/ +COPY thruk_cluster.conf /root/ +COPY dot_thruk /root/ +COPY 1.tbp /root/ +COPY 1.rpt /root/ +COPY 1.tsk /root/ diff --git a/t/scenarios/cluster_db_e2e/thruk/dot_thruk b/t/scenarios/cluster_db_e2e/thruk/dot_thruk new file mode 100644 index 0000000000..78a2ae58e4 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/dot_thruk @@ -0,0 +1,2 @@ +export THRUK_PERFORMANCE_DEBUG=1 +export TEST_AUTHOR=1 diff --git a/t/scenarios/cluster_db_e2e/thruk/playbook.yml b/t/scenarios/cluster_db_e2e/thruk/playbook.yml new file mode 100644 index 0000000000..0b4fefdacc --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/playbook.yml @@ -0,0 +1,89 @@ +--- +- hosts: all + roles: + - role: common + - role: local_tests + - role: thruk_developer + tasks: +# - shell: "crond" + - name: "omd config change" + shell: "omd config demo change" + args: + stdin: | + APACHE_MODE=own + - shell: rm /omd/sites/demo/etc/naemon/conf.d/*.cfg + - name: "create secret.key" + shell: echo "testkey" > /omd/sites/demo/var/thruk/secret.key + - name: "set permissions on secret.key" + file: + path: /omd/sites/demo/var/thruk/secret.key + mode: 0600 + owner: demo + group: demo + - shell: ln -sfn /thruk/support/thruk_templates.cfg /omd/sites/demo/etc/naemon/conf.d/thruk_templates.cfg + - copy: + src: /root/test.cfg + dest: /omd/sites/demo/etc/naemon/conf.d/test.cfg + owner: demo + group: demo + - copy: + src: /root/thruk_cluster.conf + dest: /omd/sites/demo/etc/thruk/thruk_local.d/cluster.conf + owner: demo + group: demo + - copy: + src: /root/dot_thruk + dest: /omd/sites/demo/.thruk + owner: demo + group: demo + - name: add example business process + copy: + src: /root/1.tbp + dest: /omd/sites/demo/etc/thruk/bp/1.tbp + owner: demo + group: demo + - name: create reports folder + file: + path: /omd/sites/demo/var/thruk/reports + state: directory + owner: demo + group: demo + mode: 0770 + - name: add example report + copy: + src: /root/1.rpt + dest: /omd/sites/demo/var/thruk/reports/1.rpt + owner: demo + group: demo + - name: create downtimes folder + file: + path: /omd/sites/demo/var/thruk/downtimes + state: directory + owner: demo + group: demo + mode: 0770 + - name: add example recurring downtime + copy: + src: /root/1.tsk + dest: /omd/sites/demo/var/thruk/downtimes/1.tsk + owner: demo + group: demo + - name: wait for omd livestatus to become available + wait_for: + host: omd + port: 6557 + - name: "wait for http://omd/demo/ to come up" + uri: + url: "http://omd/demo/thruk/cgi-bin/remote.cgi" + status_code: 200 + register: result + until: result.status == 200 + retries: 180 + delay: 1 + - name: "thruk filesystem import" + shell: sudo su - demo -c "thruk filesystem import" + - name: "thruk cron install" + shell: sudo su - demo -c "thruk cron install" +# TODO: fix +# - name: "thruk bp commit" +# shell: sudo su - demo -c "thruk bp commit" diff --git a/t/scenarios/cluster_db_e2e/thruk/test.cfg b/t/scenarios/cluster_db_e2e/thruk/test.cfg new file mode 100644 index 0000000000..d6f9c88790 --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/test.cfg @@ -0,0 +1,329 @@ +define timeperiod { + timeperiod_name 24x7 + alias 24 Hours A Day, 7 Days A Week + monday 00:00-24:00 + tuesday 00:00-24:00 + wednesday 00:00-24:00 + thursday 00:00-24:00 + friday 00:00-24:00 + saturday 00:00-24:00 + sunday 00:00-24:00 +} + +define contact { + name generic-contact + host_notification_commands notify-host-by-email + host_notification_options d,u,r,f,s + host_notification_period 24x7 + register 0 + service_notification_commands notify-service-by-email + service_notification_options w,u,c,r,f,s + service_notification_period 24x7 +} + +define contact { + contact_name naemonadmin + alias Naemon Admin + use generic-contact + email naemon@localhost +} + +define contactgroup { + contactgroup_name admins + alias Naemon Administrators + members naemonadmin +} + +define command { + command_name notify-host-by-email + command_line /bin/true +} + +define command { + command_name notify-service-by-email + command_line /bin/true +} + +define host { + name generic-host + event_handler_enabled 1 + flap_detection_enabled 1 + notification_period 24x7 + notifications_enabled 1 + process_perf_data 1 + register 0 + retain_nonstatus_information 1 + retain_status_information 1 +} + +define command { + command_name check-host-alive + command_line $USER1$/check_ping -H $HOSTADDRESS$ -w 3000.0,80% -c 5000.0,100% -p 3 +} + +define host { + name linux-server + use generic-host + check_command check-host-alive + check_interval 5 + check_period 24x7 + contact_groups admins + max_check_attempts 10 + notification_interval 120 + notification_options d,u,r + notification_period 24x7 + register 0 + retry_interval 1 +} + +define host { + host_name localhost + alias localhost + address 127.0.0.1 + use linux-server +} + +define hostgroup { + hostgroup_name linux-servers + alias Linux Servers + members localhost +} + +define hostgroup { + hostgroup_name test + alias Test Servers + members test +} + +define service { + name generic-service + active_checks_enabled 1 + check_freshness 0 + check_interval 10 + check_period 24x7 + contact_groups admins + event_handler_enabled 1 + flap_detection_enabled 1 + is_volatile 0 + max_check_attempts 3 + notification_interval 60 + notification_options w,u,c,r + notification_period 24x7 + notifications_enabled 1 + obsess_over_service 1 + passive_checks_enabled 1 + process_perf_data 1 + register 0 + retain_nonstatus_information 1 + retain_status_information 1 + retry_interval 2 +} + +define service { + name local-service + use generic-service + check_interval 5 + max_check_attempts 4 + register 0 + retry_interval 1 +} + +define command { + command_name check_ping + command_line $USER1$/check_ping -H $HOSTADDRESS$ -w $ARG1$ -c $ARG2$ -p 3 +} + +define service { + service_description PING + host_name localhost + use local-service + check_command check_ping!100.0,20%!500.0,60% +} + +define command { + command_name check_local_disk + command_line $USER1$/check_disk -w $ARG1$ -c $ARG2$ -p $ARG3$ +} + +define service { + service_description Root Partition + host_name localhost + use local-service + check_command check_local_disk!2%!1%!/ +} + +define command { + command_name check_local_users + command_line $USER1$/check_users -w $ARG1$ -c $ARG2$ +} + +define service { + service_description Current Users + host_name localhost + use local-service + check_command check_local_users!20!50 +} + +define command { + command_name check_local_procs + command_line $USER1$/check_procs -w $ARG1$ -c $ARG2$ -s $ARG3$ +} + +define service { + service_description Total Processes + host_name localhost + use local-service + check_command check_local_procs!250!400!RSZDT +} + +define command { + command_name check_local_load + command_line $USER1$/check_load -w $ARG1$ -c $ARG2$ +} + +define service { + service_description Current Load + host_name localhost + use local-service + check_command check_local_load!5.0,4.0,3.0!10.0,6.0,4.0 +} + +define command { + command_name check_http + command_line $USER1$/check_http -I $HOSTADDRESS$ $ARG1$ +} + +define service { + service_description HTTP + host_name localhost + use local-service + check_command check_http!-u /naemon/ -e 404 +} + +define command { + command_name check_dummy + command_line printf $ARG1$ +} + +define command { + command_name check_dummy2 + command_line printf $ARG1$ && exit $ARG2$ +} + +define service { + service_description Example Check + host_name localhost + use local-service + check_command check_dummy!'$SERVICEDESC$|x=5$USER5$10$USER5$20$USER5$0$USER5$50' +} + +define host { + host_name test + alias test + address 127.0.0.2 + use linux-server +} + +define service { + service_description ok + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!ok!0 +} + +define service { + service_description ok_downtime + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!ok!0 +} + +define service { + service_description warning + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!warning!1 +} + +define service { + service_description critical + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!critical!2 +} + +define service { + service_description unknown + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!unknown!3 +} + +define service { + service_description pending + host_name test + use local-service + active_checks_enabled 0 + check_command check_dummy2!pending!0 +} + +define service { + service_description critical_downtime + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!critical!2 +} + +define service { + service_description critical_ack + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!critical!2 +} + +define service { + service_description warning_downtime + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!warning!1 +} + +define service { + service_description warning_ack + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!warning!1 +} + +define service { + service_description warning_ack_downtime + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!warning!1 +} + +define service { + service_description unknown_downtime + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!unknown!3 +} + +define service { + service_description unknown_ack + host_name test + use local-service + max_check_attempts 1 + check_command check_dummy2!unknown!3 +} diff --git a/t/scenarios/cluster_db_e2e/thruk/thruk_cluster.conf b/t/scenarios/cluster_db_e2e/thruk/thruk_cluster.conf new file mode 100644 index 0000000000..a89193576c --- /dev/null +++ b/t/scenarios/cluster_db_e2e/thruk/thruk_cluster.conf @@ -0,0 +1,20 @@ +var_path_db = mysql://thruk:thruk@omd/thruk_var_fs + + + + name = OMD + type = http + + peer = http://omd/demo + fallback_peer = omd:6557 + auth = testkey + + + + + + result_backend = OMD + + +cluster_enabled = 1 +cluster_heartbeat_interval = 60 diff --git a/t/scenarios/cluster_e2e/docker-compose.yml b/t/scenarios/cluster_e2e/docker-compose.yml index 503db0b077..bd1ade97db 100644 --- a/t/scenarios/cluster_e2e/docker-compose.yml +++ b/t/scenarios/cluster_e2e/docker-compose.yml @@ -1,6 +1,6 @@ services: lb: - image: 'traefik:latest' + image: 'traefik:v3.2' command: - "--api=true" - "--api.dashboard=true" diff --git a/t/scenarios/cluster_fixed_e2e/docker-compose.yml b/t/scenarios/cluster_fixed_e2e/docker-compose.yml index cc9ebe16bd..5e2bc8bec5 100644 --- a/t/scenarios/cluster_fixed_e2e/docker-compose.yml +++ b/t/scenarios/cluster_fixed_e2e/docker-compose.yml @@ -3,7 +3,7 @@ networks: services: lb: - image: 'traefik:latest' + image: 'traefik:v3.2' command: - "--api=true" - "--api.dashboard=true" diff --git a/thruk.conf b/thruk.conf index ec0f9d878a..c15f47fa36 100644 --- a/thruk.conf +++ b/thruk.conf @@ -838,6 +838,9 @@ locked_message = account is locked, please contact an adminis # Set timeout after which a node is removed from the cluster. #cluster_node_stale_timeout = 120 +# use database for storing files in var_path +#var_path_db = mysql://user:password@hostname/databasename + ###################################### # BACKENDS # set logging of backend in verbose mode. This only