[irondb] Change to iRoNDB (logicalclocks#63)

[irondb] Initial benchmark directories [irondb] Add configuration for sysbench [irondb] Change directory name [irondb] Template sysbench multi [irondb] Initialize local_infile mysql configuration property [irondb] Add configuration files for dbt2 [irondb] Change dbt2 data directory [irondb] Rename
AlexHopsworks · Mar 3, 2021 · 9070b2b · 9070b2b
1 parent d79bcba
commit 9070b2b
Show file tree

Hide file tree

Showing 8 changed files with 164 additions and 18 deletions.
diff --git a/attributes/default.rb b/attributes/default.rb
@@ -1,14 +1,14 @@
 include_attribute "kagent"
 
-version                                               ="8"
-default['ndb']['majorVersion']                        ="0"
-default['ndb']['minorVersion']                        ="21"
+version                                               ="21"
+default['ndb']['majorVersion']                        ="04"
+default['ndb']['minorVersion']                        ="0"
 
 default['ndb']['version']                             = "#{version}.#{node['ndb']['majorVersion']}.#{node['ndb']['minorVersion']}"
 default['ndb']['enabled']                             = "true"
-default['ndb']['glib_version']                        = "2.12"
+default['ndb']['glib_version']                        = "2.17"
 
-default['ndb']['url']                                 = node['download_url'] + "/mysql-cluster-#{node['ndb']['version']}-linux-glibc#{node['ndb']['glib_version']}-x86_64.tar.gz"
+default['ndb']['url']                                 = node['download_url'] + "/rondb-#{node['ndb']['version']}-linux-glibc#{node['ndb']['glib_version']}-x86_64.tar.gz"
 # checksum is not a security check - used to improve the speed of downloads by skipping if matched
 # checksum calculated using: shasum -a 256 /var/www/hops/...tgz | cut -c-12
 # checksum calculated using: sha256sum /var/www/hops/...tgz | cut -c-12

diff --git a/files/default/dbt2_run_1.conf.multi b/files/default/dbt2_run_1.conf.multi
@@ -0,0 +1,9 @@
+# NUM_MYSQL_SERVERS  NUM_WAREHOUSES  NUM_TERMINALS
+2                    1               1
+2                    2               1
+2                    4               1
+2                    8               1
+2                    12              1
+2                    16              1
+2                    24              1
+2                    32              1
diff --git a/files/default/dbt2_run_1.conf.single b/files/default/dbt2_run_1.conf.single
@@ -0,0 +1,11 @@
+# NUM_MYSQL_SERVERS  NUM_WAREHOUSES  NUM_TERMINALS
+1                    1               1
+1                    2               1
+1                    4               1
+1                    8               1
+1                    12              1
+1                    16              1
+1                    24              1
+1                    32              1
+1                    48              1
+1                    64              1
diff --git a/recipes/bench.rb b/recipes/bench.rb
@@ -1,24 +1,105 @@
-# Run DB benchmarks 
+benchmarks_dir = "#{node['ndb']['user-home']}/benchmarks"
 
-remote_file "#{node['ndb']['scripts_dir']}/flexAsync" do
+directory benchmarks_dir do
   owner node['ndb']['user']
   group node['ndb']['group']
-  source "http://snurran.sics.se/hops/flexAsync"
-  mode 0755
+  mode "750"
   action :create
 end
 
-template "#{node['ndb']['scripts_dir']}/flexAsync.sh" do
-  source "flexAsync.sh.erb"
+sysbench_single_dir = "#{benchmarks_dir}/sysbench_single"
+directory sysbench_single_dir do
   owner node['ndb']['user']
   group node['ndb']['group']
-  mode 0754
-  variables({ :mgmd_ip => node['ndb']['mgmd']['private_ips'][0] })
+  mode "750"
+  action :create
+end
+
+mysqld_host = ""
+mysqld_hosts = ""
+number_of_mysqld = 0
+if node['ndb'].attribute?('mysqld')
+  number_of_mysqld = node['ndb']['mysqld']['private_ips'].length()
+  mysqld_hosts = node['ndb']['mysqld']['private_ips'].join(',')
+  mysqld_host = node['ndb']['mysqld']['private_ips'][0]
+end
+
+template "#{sysbench_single_dir}/autobench.conf" do
+  source "autobench_sysbench.conf.erb"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
+  variables({
+    :sysbench_instances => "1",
+    :mysqld_hosts => mysqld_host,
+  })
+end
+
+sysbench_multi_dir = "#{benchmarks_dir}/sysbench_multi"
+directory sysbench_multi_dir do
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode "750"
+  action :create
+end
+
+template "#{sysbench_multi_dir}/autobench.conf" do
+  source "autobench_sysbench.conf.erb"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
+  variables({
+    :sysbench_instances => number_of_mysqld,
+    :mysqld_hosts => mysqld_hosts,
+  })
+end
+
+dbt2_single_dir = "#{benchmarks_dir}/dbt2_single"
+directory dbt2_single_dir do
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode "750"
+  action :create
+end
+
+template "#{dbt2_single_dir}/autobench.conf" do
+  source "autobench_dbt2.conf.erb"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
+  variables({
+    :mysqld_hosts => mysqld_host,
+  })
+end
+
+cookbook_file "#{dbt2_single_dir}/dbt2_run_1.conf" do
+  source "dbt2_run_1.conf.single"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
+end
+
+dbt2_multi_dir = "#{benchmarks_dir}/dbt2_multi"
+directory dbt2_multi_dir do
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode "750"
+  action :create
+end
+
+template "#{dbt2_multi_dir}/autobench.conf" do
+  source "autobench_dbt2.conf.erb"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
+  variables({
+    :mysqld_hosts => mysqld_hosts,
+  })
 end
 
-ark "dbt2" do
-  url node['ndb']['dbt2_binaries']
-  home_dir node['ndb']['root_dir']
-  append_env_path true
-  action :install
+cookbook_file "#{dbt2_multi_dir}/dbt2_run_1.conf" do
+  source "dbt2_run_1.conf.multi"
+  owner node['ndb']['user']
+  group node['ndb']['group']
+  mode 0750
 end
diff --git a/templates/default/autobench_dbt2.conf.erb b/templates/default/autobench_dbt2.conf.erb
@@ -0,0 +1,21 @@
+#
+# Software definition
+#
+MYSQL_BIN_INSTALL_DIR="<%= node['mysql']['base_dir'] %>"
+BENCHMARK_TO_RUN="dbt2"
+#
+# Storage definition (empty here)
+#
+#
+# MySQL Server definition
+#
+SERVER_HOST="<%= @mysqld_hosts %>"
+#
+# NDB node definitions (empty here)
+#
+#
+# Benchmark definition
+#
+DBT2_TIME="30"
+DBT2_WAREHOUSES="64"
+DBT2_DATA_DIR="<%= node['ndb']['user-home'] %>/benchmarks/dbt2_data"
diff --git a/templates/default/autobench_sysbench.conf.erb b/templates/default/autobench_sysbench.conf.erb
@@ -0,0 +1,22 @@
+#
+# Software definition
+#
+MYSQL_BIN_INSTALL_DIR="<%= node['mysql']['base_dir'] %>"
+BENCHMARK_TO_RUN="sysbench"
+#
+# Storage definition (empty here)
+#
+#
+# MySQL Server definition
+#
+SERVER_HOST="<%= @mysqld_hosts %>"
+#
+# NDB node definitions (empty here)
+#
+#
+# Benchmark definition
+#
+SYSBENCH_TEST="oltp_rw"
+SYSBENCH_INSTANCES="<%= @sysbench_instances %>"
+THREAD_COUNTS_TO_RUN="1;2;4;8;12;16;24;32;48;64;96;112;128"
+MAX_TIME="30"
diff --git a/templates/default/my-ndb.cnf.erb b/templates/default/my-ndb.cnf.erb
@@ -13,6 +13,7 @@ socket          = <%= node['ndb']['mysql_socket'] %>
 bind-address    = 0.0.0.0
 pid-file        = <%= node['ndb']['log_dir'] %>/mysql_<%= @mysql_id %>.pid
 tmpdir          = /tmp
+local_infile = OFF
 
 # Explicit defaults for TS needed by 'airflow initdb'
 explicit_defaults_for_timestamp = 1

diff --git a/templates/default/ndbmtd.service.erb b/templates/default/ndbmtd.service.erb
@@ -14,6 +14,7 @@ ExecReload = <%= "#{node['ndb']['base_dir']}/scripts/ndbd-restart.sh" %>
 TimeoutSec= <%= node['ndb']['ndbd']['systemctl_timeout_sec'] %>
 Restart=on-failure
 RestartSec=1
+LimitMEMLOCK=infinity
 
 Type = forking