WIP: add database storage

for var filesystem mainly

MANIFEST

@@ -127,6 +127,7 @@ lib/Thruk/Utils/CLI/Command.pm
 lib/Thruk/Utils/CLI/Contact.pm
 lib/Thruk/Utils/CLI/Cron.pm
 lib/Thruk/Utils/CLI/Downtimetask.pm
+lib/Thruk/Utils/CLI/Filesystem.pm
 lib/Thruk/Utils/CLI/Find.pm
 lib/Thruk/Utils/CLI/Graph.pm
 lib/Thruk/Utils/CLI/Host.pm
@@ -150,6 +151,8 @@ lib/Thruk/Utils/Encode.pm
 lib/Thruk/Utils/External.pm
 lib/Thruk/Utils/Filter.pm
 lib/Thruk/Utils/IO.pm
+lib/Thruk/Utils/IO/LocalFS.pm
+lib/Thruk/Utils/IO/Mysql.pm
 lib/Thruk/Utils/LMD.pm
 lib/Thruk/Utils/Log.pm
 lib/Thruk/Utils/Menu.pm
@@ -2835,6 +2838,25 @@ t/scenarios/cli_api/t/thruk_cmd_cli.t
 t/scenarios/cli_api/test.sh
 t/scenarios/cli_api/thruk.conf
 t/scenarios/cli_api/thruk_local.conf
+t/scenarios/cluster_db_e2e/docker-compose.yml
+t/scenarios/cluster_db_e2e/Makefile
+t/scenarios/cluster_db_e2e/omd/Dockerfile
+t/scenarios/cluster_db_e2e/omd/playbook.yml
+t/scenarios/cluster_db_e2e/omd/test.cfg
+t/scenarios/cluster_db_e2e/README
+t/scenarios/cluster_db_e2e/scale
+t/scenarios/cluster_db_e2e/t/300-controller_cluster.t
+t/scenarios/cluster_db_e2e/t/300-controller_rest_v1.t
+t/scenarios/cluster_db_e2e/t/300-controller_tac.t
+t/scenarios/cluster_db_e2e/t/local/cluster.t
+t/scenarios/cluster_db_e2e/thruk/1.rpt
+t/scenarios/cluster_db_e2e/thruk/1.tbp
+t/scenarios/cluster_db_e2e/thruk/1.tsk
+t/scenarios/cluster_db_e2e/thruk/Dockerfile
+t/scenarios/cluster_db_e2e/thruk/dot_thruk
+t/scenarios/cluster_db_e2e/thruk/playbook.yml
+t/scenarios/cluster_db_e2e/thruk/test.cfg
+t/scenarios/cluster_db_e2e/thruk/thruk_cluster.conf
 t/scenarios/cluster_e2e/docker-compose.yml
 t/scenarios/cluster_e2e/Makefile
 t/scenarios/cluster_e2e/omd/Dockerfile

docs/documentation/cluster.asciidoc

@@ -6,9 +6,8 @@ title: Cluster Setup
 == Thruk Clustering
 {% include new_since.ad version="2.24" %}
 Clustered setups provide high-availability and performance improvements at the
-price of higher complexity. All Thruk nodes in a cluster must use shared
-storage for their `etc_path` and `var_path` while the `tmp_path` should remain
-local.
+price of higher complexity. All Thruk nodes in a cluster must use some kind of
+shared storage or database for `var_path`.
 
 
 === Setup
@@ -19,6 +18,23 @@ local.
     - Enable cluster with `cluster_enabled=1`
 
 
+==== Shared Filesystem
+All Thruk nodes in a cluster must use shared storage for their `etc_path`
+and `var_path` while the `tmp_path` remains local.
+
+This can be either implemented by mounting for example a NFS share or by using
+a shared database.
+
+==== Shared Database
+{% include new_since.ad version="3.22" %}
+Instead of a shared filesystem, you can use a shared database for the `var_path`.
+This is especially useful when you have a database cluster already in place.
+
+In order to use a mysql/mariadb database, you have to fill in a connection
+string in the `var_path_db`.
+
+    var_path_db = mysql://user:password@hostname/databasename
+
 ==== Static Cluster
 Having a fixed number of cluster nodes will be set if you configure multiple
 cluster_nodes with fixed hostnames like:

docs/documentation/configuration.asciidoc

@@ -708,6 +708,15 @@ ex.:
   var_path = ./var
 
 
+=== var_path_db
+
+Use database for storing files in var_path. See link:cluster.html[Clustering].
+
+ex.:
+
+ var_path_db = mysql://user:password@hostname/databasename
+
+
 === tmp_path
 
 Path to a temporary directory. Defaults to /tmp if not set and usually

docs/documentation/faq.asciidoc

@@ -251,14 +251,14 @@ the shipping installer:
 
   #> /usr/share/thruk/script/install_puppeteer.sh
 
-This will install puppeteer into /var/lib/thruk/puppeteer
+This will install puppeteer into /var/lib/thruk/local/puppeteer
 
 It requires node and npm to be installed.
 
 Thruk will use the system chromium if it is installed befor running
 the puppet installer.
 If no chromium is installed, puppeteer will download chromium
-into /var/lib/thruk/puppeteer/chromium.
+into /var/lib/thruk/local/puppeteer/chromium.
 
 You can disable downloading chromium by setting this into the
 environment before running the installer.

lib/Thruk.pm

@@ -702,7 +702,7 @@ sub _create_secret_file {
     return unless (Thruk::Base->mode() eq 'FASTCGI' || Thruk::Base->mode() eq 'DEVSERVER');
     my $var_path   = $config->{'var_path'} || die("no var path!");
     my $secretfile = $var_path.'/secret.key';
-    return if -s $secretfile;
+    return if Thruk::Utils::IO::file_not_empty($secretfile);
     require Thruk::Utils::Crypt;
     my $digest = Thruk::Utils::Crypt::random_uuid([time()]);
     Thruk::Utils::IO::write($secretfile, $digest);

lib/Thruk/Backend/Manager.pm

@@ -3123,10 +3123,10 @@ sub caching_query {
             }
             # simply remove all files older than 24h
             my $yesterday = time() - 86400;
-            for my $file (glob($cache_file."/*.cache")) {
-                my @stat = stat($file);
+            for my $file (@{Thruk::Utils::IO::find_files($cache_file, '\.cache$')}) {
+                my @stat = Thruk::Utils::IO::stat($file);
                 if($stat[9] < $yesterday) {
-                    unlink($file);
+                    Thruk::Utils::IO::unlink($file);
                 }
             }
         } else {

lib/Thruk/Backend/Provider/Mysql.pm

@@ -90,8 +90,35 @@ sub new {
     if(!defined $options->{'peer_key'}) {
         confess('please provide peer_key');
     }
+    my($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock) = _parse_connection_string($options->{'peer'});
+    my $self = {
+        'dbhost'      => $dbhost,
+        'dbport'      => $dbport,
+        'dbname'      => $dbname,
+        'dbuser'      => $dbuser,
+        'dbpass'      => $dbpass,
+        'dbsock'      => $dbsock,
+        'peer_config' => $options,
+        'verbose'     => 0,
+    };
+    bless $self, $class;
+
+    return $self;
+}
+
+##########################################################
+
+=head2 _parse_connection_string
+
+    _parse_connection_string($str)
+
+parse and return connection string
+
+=cut
+sub _parse_connection_string {
+    my($connection_string) = @_;
     my($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock);
-    if($options->{'peer'} =~ m/^mysql:\/\/(.*?)(|:.*?)@([^:]+)(|:.*?)\/([^\/]*?)$/mx) {
+    if($connection_string =~ m/^mysql:\/\/(.*?)(|:.*?)@([^:]+)(|:.*?)\/([^\/]*?)$/mx) {
         $dbuser = $1;
         $dbpass = $2;
         $dbhost = $3;
@@ -106,20 +133,7 @@ sub new {
     } else {
         die('Mysql connection must match this form: mysql://user:password@host:port/dbname');
     }
-
-    my $self = {
-        'dbhost'      => $dbhost,
-        'dbport'      => $dbport,
-        'dbname'      => $dbname,
-        'dbuser'      => $dbuser,
-        'dbpass'      => $dbpass,
-        'dbsock'      => $dbsock,
-        'peer_config' => $options,
-        'verbose'     => 0,
-    };
-    bless $self, $class;
-
-    return $self;
+    return($dbhost, $dbport, $dbuser, $dbpass, $dbname, $dbsock);
 }
 
 ##########################################################
@@ -1514,7 +1528,6 @@ sub _check_lock {
 
     # check if there is already a update / import running
     my $skip          = 0;
-    my $cache_version = 1;
     eval {
         $dbh->do('LOCK TABLES `'.$prefix.'_status` READ') unless $c->config->{'logcache_pxc_strict_mode'};
         my @pids = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 2 LIMIT 1')};
@@ -1524,10 +1537,6 @@ sub _check_lock {
                 $skip = 1;
             }
         }
-        my @versions = @{$dbh->selectcol_arrayref('SELECT value FROM `'.$prefix.'_status` WHERE status_id = 4 LIMIT 1')};
-        if(scalar @versions > 0 and $versions[0]) {
-            $cache_version = $versions[0];
-        }
     };
     $dbh->do('UNLOCK TABLES') unless $c->config->{'logcache_pxc_strict_mode'};
     if($@) {

lib/Thruk/Config.pm

@@ -487,6 +487,7 @@ sub set_default_config {
     # set var dir
     $config->{'var_path'} = $config->{'home'}.'/var' unless defined $config->{'var_path'};
     $config->{'var_path'} =~ s|/$||mx;
+    $Thruk::Utils::IO::var_path = $config->{'var_path'};
 
     if(!defined $config->{'etc_path'}) {
         if($ENV{'THRUK_CONFIG'}) {
@@ -995,9 +996,8 @@ return secret_key
 sub secret_key {
     my $config      = &get_config();
     my $secret_file = $config->{'var_path'}.'/secret.key';
-    return unless -s $secret_file;
-    my $secret_key  = Thruk::Utils::IO::read($secret_file);
-    chomp($secret_key);
+    my $secret_key  = Thruk::Utils::IO::saferead($secret_file);
+    chomp($secret_key) if defined $secret_key;
     return($secret_key);
 }
 

lib/Thruk/Controller/Rest/V1/broadcast.pm

@@ -27,8 +27,8 @@ Thruk::Controller::rest_v1::register_rest_path_v1('GET', qr%^/thruk/broadcasts?$
 sub _rest_get_thruk_broadcast {
     my($c, undef, $file) = @_;
     require Thruk::Utils::Broadcast;
-    $file = '*' unless $file;
-    my @files  = glob($c->config->{'var_path'}.'/broadcast/'.$file.'.json');
+    $file = '.*' unless $file;
+    my @files  = @{Thruk::Utils::IO::find_files($c->config->{'var_path'}.'/broadcast/', $file.'\.json$')};
     my $broadcasts = Thruk::Controller::rest_v1::load_json_files($c, {
             files                  => \@files,
             authorization_callback => $c->user->check_user_roles('authorized_for_broadcasts') ? undef : \&Thruk::Utils::Broadcast::is_authorized_for_broadcast,
@@ -41,7 +41,7 @@ sub _rest_get_thruk_broadcast {
         $b->{'text'} = Thruk::Utils::Filter::replace_macros($b->{'text'}, $b->{'frontmatter'});
     }
 
-    if($file eq '*') {
+    if($file eq '.*') {
         return($broadcasts);
     }
 
@@ -80,7 +80,7 @@ sub _rest_get_thruk_broadcast {
         });
     }
 
-    if($file ne '*') {
+    if($file ne '.*') {
         return($broadcasts->[0]);
     }
 }
@@ -115,7 +115,7 @@ sub _rest_get_thruk_broadcast_new {
     if(!$file) {
         $file = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'.json', localtime);
         my $x  = 1;
-        while(-e $c->config->{'var_path'}.'/broadcast/'.$file) {
+        while(Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/broadcast/'.$file)) {
             $file = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'_'.$x.'.json', localtime);
             $x++;
         }

lib/Thruk/Controller/broadcast.pm

@@ -104,7 +104,7 @@ sub index {
             if($id eq 'new') {
                 $id = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'.json', localtime);
                 my $x  = 1;
-                while(-e $c->config->{'var_path'}.'/broadcast/'.$id) {
+                while(Thruk::Utils::IO::file_exists($c->config->{'var_path'}.'/broadcast/'.$id)) {
                     $id = POSIX::strftime('%Y-%m-%d-'.$c->stash->{'remote_user'}.'_'.$x.'.json', localtime);
                     $x++;
                 }

lib/Thruk/Controller/extinfo.pm

@@ -343,8 +343,8 @@ sub _process_recurring_downtimes_page {
         my $old_file;
         if($nr && !$failed) {
             $old_file  = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk';
-            if(-s $old_file) {
-                my $old_rd = Thruk::Utils::read_data_file($old_file);
+            my $old_rd = Thruk::Utils::read_data_file($old_file);
+            if($old_rd) {
                 if(Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $old_rd) != 2) {
                     $failed = 1;
                 } else {
@@ -381,8 +381,8 @@ sub _process_recurring_downtimes_page {
         }
         for my $nr (@{$numbers}) {
             my $file = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk';
-            if(-s $file) {
-                my $old_rd = Thruk::Utils::read_data_file($file);
+            my $old_rd = Thruk::Utils::read_data_file($file);
+            if($old_rd) {
                 if(Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $old_rd) != 2) {
                     Thruk::Utils::set_message( $c, { style => 'success_message', msg => 'no such downtime!' });
                 } else {
@@ -420,7 +420,8 @@ sub _process_recurring_downtimes_page_edit {
     $c->stash->{can_edit}     = 1;
     if($nr) {
         my $file = $c->config->{'var_path'}.'/downtimes/'.$nr.'.tsk';
-        if(-s $file) {
+        my $exists = Thruk::Utils::IO::saferead($file);
+        if(defined $exists) {
             $c->stash->{rd} = Thruk::Utils::RecurringDowntimes::read_downtime($c, $file, undef, undef, undef, undef, undef, undef, undef, 0);
             my $perms = Thruk::Utils::RecurringDowntimes::check_downtime_permissions($c, $c->stash->{rd});
             # check cmd permission for this downtime

lib/Thruk/Controller/login.pm

@@ -359,6 +359,7 @@ sub _clean_failed_logins {
     }
     my $timeout = time() - (30 * 86400);
     Thruk::Utils::IO::mkdir($dir);
+# TODO: ...
     opendir( my $dh, $dir) or die "can't opendir '$dir': $!";
     for my $entry (readdir($dh)) {
         next if $entry eq '.' or $entry eq '..';

lib/Thruk/Controller/remote.pm

@@ -100,12 +100,10 @@ sub index {
         if($body) {
             if(ref $body eq 'File::Temp') {
                 my $file = $body->filename();
-                if($file and -e $file) {
-                    my $msg = Thruk::Utils::IO::read($file);
-                    unlink($file);
-                    _error($msg);
-                    return $c->render("text" => 'OK');
-                }
+                my $msg = Thruk::Utils::IO::saferead($file);
+                unlink($file);
+                _error($msg) if $msg;
+                return $c->render("text" => 'OK');
             }
             if(ref $body eq 'FileHandle') {
                 while(<$body>) {

lib/Thruk/Controller/rest_v1.pm

@@ -1749,8 +1749,8 @@ sub load_json_files {
     my($c, $options) = @_;
     my $list = [];
     for my $file (@{$options->{'files'}}) {
-        next unless -e $file;
         my $data = Thruk::Utils::IO::json_lock_retrieve($file);
+        next unless $data;
         $data->{'file'} = $file;
         $data->{'file'} =~ s%.*?([^/]+)\.\w+$%$1%mx;
         if($options->{'pre_process_callback'}) {
@@ -1849,8 +1849,15 @@ sub _rest_get_thruk_jobs {
     my($c, undef, $job) = @_;
     require Thruk::Utils::External;
 
+    my $files = Thruk::Utils::IO::find_files($c->config->{'var_path'}."/jobs");
+    my %dirs;
+    for my $f (@{$files}) {
+        my $d = Thruk::Base::dirname($f);
+        $dirs{$d} = 1;
+    }
+
     my $data = [];
-    for my $dir (glob($c->config->{'var_path'}."/jobs/*/.")) {
+    for my $dir (sort keys %dirs) {
         if($dir =~ m%/([^/]+)/\.$%mx) {
             my $id = $1;
             next if $job && $job ne $id;
@@ -1890,7 +1897,7 @@ sub _rest_get_thruk_sessions {
     my $min5 = time() - (5*60);
     my $uniq = {};
     my $uniq5min = {};
-    for my $file (sort glob($c->config->{'var_path'}."/sessions/*")) {
+    for my $file (sort @{Thruk::Utils::IO::find_files($c->config->{'var_path'}."/sessions/")}) {
         $total_number++;
         my $session_data = Thruk::Utils::CookieAuth::retrieve_session(config => $c->config, file => $file);
         next unless $session_data;