[HOPSWORKS-1063] Cleanup ndb-chef (logicalclocks#30)

README.md

@@ -13,7 +13,7 @@ Platform
 
 
 Tested on:
-* Ubuntu 12.04-14.04
+* Ubuntu 16.04
 * centos 7.0+
 
 

attributes/default.rb

@@ -25,27 +25,27 @@
 default['ndb']['group']                               = node['install']['user'].empty? ? "mysql" : node['install']['user']
 default['ndb']['connectstring']                       = ""
 
-default['ndb']['DataMemory']                          = "50"
-# Calculate IndexMemory size by default, can be overriden by user.
-default['ndb']['IndexMemory']                         = ""
+default['ndb']['DataMemory']                          = "98"
 default['ndb']['NoOfReplicas']                        = "1"
 default['ndb']['TcpBind_INADDR_ANY']                  = "FALSE"
 default['ndb']['NoOfFragmentLogParts']                = "4"
 default['ndb']['NoOfFragmentLogFiles']                = "4"
 default['ndb']['FragmentLogFileSize']                 = "64M"
-default['ndb']['MaxNoOfTables']                       = "3036"
+default['ndb']['MaxNoOfTables']                       = "4096"
 default['ndb']['MaxNoOfOrderedIndexes']               = "2048"
 default['ndb']['MaxNoOfUniqueHashIndexes']            = "512"
-default['ndb']['MaxDMLOperationsPerTransaction']      = "4297295"
+default['ndb']['MaxNoOfTriggers']                     = "768"
+default['ndb']['MaxDMLOperationsPerTransaction']      = "4294967295"
 default['ndb']['TransactionBufferMemory']             = "1M"
 default['ndb']['MaxParallelScansPerFragment']         = "256"
 default['ndb']['MaxDiskWriteSpeed']                   = "20M"
 default['ndb']['MaxDiskWriteSpeedOtherNodeRestart']   = "50M"
 default['ndb']['MaxDiskWriteSpeedOwnRestart']         = "200M"
-default['ndb']['MinDiskWriteSpeed']                   = "5M"
+default['ndb']['MinDiskWriteSpeed']                   = "10M"
 default['ndb']['DiskSyncSize']                        = "4M"
 default['ndb']['RedoBuffer']                          = "32M"
 default['ndb']['LongMessageBuffer']                   = "64M"
+default['ndb']['MaxFKBuildBatchSize']                 = "64"
 default['ndb']['TransactionInactiveTimeout']          = "1500"
 default['ndb']['TransactionDeadlockDetectionTimeout'] = "1500"
 default['ndb']['LockPagesInMainMemory']               = "1"
@@ -57,7 +57,7 @@
 default['ndb']['BackupDataBufferSize']                = "16M"
 default['ndb']['MaxAllocate']                         = "32M"
 default['ndb']['DefaultHashMapSize']                  = "3840"
-default['ndb']['ODirect']                             = "0"
+default['ndb']['ODirect']                             = "1"
 default['ndb']['ExtraSendBufferMemory']               = "0"
 default['ndb']['TotalSendBufferMemory']               = "16M"
 default['ndb']['DiskPageBufferEntries']               = "10"
@@ -83,12 +83,18 @@
 default['ndb']['MaxNoOfConcurrentTransactions']       = "16192"
 default['ndb']['MaxNoOfAttributes']                   = "5000"
 
+default['ndb']['MaxReorgBuildBatchSize']              = "64" 
+default['ndb']['EnablePartialLcp']                    = "1"
+default['ndb']['RecoveryWork']                        = "60"
+default['ndb']['InsertRecoveryWork']                  = "40"
+
+
 #Optimize for throughput: 0 (range 0..10)
 default['ndb']['SchedulerResponsiveness']             = 0
 default['ndb']['SchedulerSpinTimer']                  = 0
 default['ndb']['SchedulerExecutionTimer']             = 75
 
-default['ndb']['BuildIndexThreads']                   = 8
+default['ndb']['BuildIndexThreads']                   = "128"
 default['ndb']['TwoPassInitialNodeRestartCopy']       = "true"
 default['ndb']['Numa']                                = 1
 
@@ -104,7 +110,6 @@
 default['mgm']['scripts']            = %w{ backup-start.sh backup-restore.sh backup-remove.sh enter-singleuser-mode.sh mgm-client.sh mgm-server-start.sh mgm-server-stop.sh mgm-server-restart.sh cluster-shutdown.sh cluster-init.sh cluster-start-with-recovery.sh exit-singleuser-mode.sh }
 default['ndb']['scripts']            = %w{ ndbd-start.sh ndbd-init.sh ndbd-stop.sh ndbd-restart.sh }
 default['mysql']['scripts']          = %w{ get-mysql-socket.sh get-mysql-port.sh mysql-server-start.sh mysql-server-stop.sh mysql-server-restart.sh mysql-client.sh }
-default['memcached']['scripts']      = %w{ memcached-start.sh memcached-stop.sh memcached-restart.sh }
 
 default['ndb']['dir']                                 = node['install']['dir'].empty? ? "/var/lib" : node['install']['dir']
 default['ndb']['root_dir']                            = "#{node['ndb']['dir']}/mysql-cluster"
@@ -142,14 +147,14 @@
 
 default['ndb']['scripts_dir']                         = "#{node['ndb']['root_dir']}/ndb/scripts"
 default['ndb']['mgm_dir']                             = "#{node['ndb']['root_dir']}/mgmd"
-
 # MySQL Server Parameters
 default['ndb']['mysql_server_dir']                    = "#{node['ndb']['root_dir']}/mysql"
 default['ndb']['num_ndb_slots_per_client']            = 1
 default['ndb']['num_ndb_slots_per_mysqld']            = 1
+default['ndb']['num_ndb_open_slots']                  = 10
 
-# Max time that the mysqld and memcached will wait for the MySQL Cluster to be up and running.
-# If the mysqld or memcached starts and the MySQL Cluster isn't running, it will not connect and will
+# Max time that the mysqld will wait for the MySQL Cluster to be up and running.
+# If the mysqld starts and the MySQL Cluster isn't running, it will not connect and will
 # need to be restarted to connect to the cluster.
 # Time in seconds
 default['ndb']['wait_startup']                        = "10800"
@@ -162,9 +167,13 @@
 # Concrete directory with mysql binaries for a specific mysql version
 default['mysql']['version_dir']                       = "#{node['mysql']['base_dir']}-#{node['ndb']['version']}"
 
-default['mysql']['jdbc_url']                          = ""
+# Location for the MySQL socket - needs to be in a directory which is accessible only to the mysql user
+default['ndb']['mysql_socket']                        = "#{node['ndb']['root_dir']}/mysql.sock"
+default['ndb']['mysql_port']                          = "3306"
 
 default['mysql']['localhost']                         = "false"
+default['mysql']['jdbc_url']                          = ""
+
 
 # MySQL Server Master-Slave replication binary log is enabled.
 default['mysql']['replication_enabled']               = "false"
@@ -176,28 +185,14 @@
 
 # Limit the number of mgm_servers to the range 49..51
 default['mgm']['id']                                  = 49
-# All mysqlds, memcacheds, and ndbclients (clusterj) are in the range 52..255
+# All mysqlds, and ndbclients (clusterj) are in the range 52..255
 default['mysql']['id']                                = 52
-# up to 65 memcacheds
-default['memcached']['id']                            = 125
 # up to 65 NameNodes
 default['nn']['id']                                   = 190
 
 # The address of the mysqld that will be used by hop
 default['ndb']['mysql_ip']                            = "10.0.2.15"
 
-# Size in MB of memcached cache
-default['memcached']['mem_size']                      = 64
-# See examples here for configuration: http://dev.mysql.com/doc/ndbapi/en/ndbmemcache-configuration.html
-# options examples: ";dev=role"   or ";dev=role;S:c4,g1,t1" or ";S:c0,g1,t1" ";role=db-only"
-default['memcached']['options']                       = ";role=ndb-caching;usec_rtt=250;max_tps=100000;m=#{default['memcached']['mem_size']}"
-
-#
-# BitTorrent settings for copying NDB binaries
-#
-# default btsync ndb seeder_ip     = "default kagent dashboard_ip "
-default['btsync']['ndb']['leechers']                     = ['10.0.2.15']
-
 # IP addresses of the mgm-server, ndbds must be overridden by role/recipe caller.
 default['ndb']['public_ips']                             = ['']
 default['ndb']['private_ips']                            = ['']
@@ -207,10 +202,6 @@
 default['ndb']['ndbd']['private_ips']                    = ['']
 default['ndb']['mysqld']['public_ips']                   = ['']
 default['ndb']['mysqld']['private_ips']                  = ['']
-default['ndb']['memcached']['public_ips']                = ['']
-default['ndb']['memcached']['private_ips']               = ['']
-
-default['ndb']['ndbapi']['addrs']                        = ['']
 
 #
 # ndbd entries in the config.ini file.
@@ -231,9 +222,6 @@
 default['ndb']['backup_frequency']                    = "daily" # 'daily', 'weekly',
 default['ndb']['backup_time']                         = "03:00"
 
-default['ndb']['systemd']                             = node['systemd']
-
-
 #LocationDomainId
 default['ndb']['mgmd']['private_ips_domainIds']          = {}
 default['ndb']['ndbd']['private_ips_domainIds']          = {}

libraries/default.rb

@@ -0,0 +1,37 @@
+module NDB
+    module Helpers
+        # TODO(Fabio): this method was moved from a recipe. Check if the ID thing actually works.
+        def find_service_id(service, base_id)
+          found_id = -1
+          my_ip = my_private_ip()
+          id = base_id
+
+          if node.attribute?(:ndb) && node['ndb'].attribute?(service) && node['ndb'][service].attribute?(:ips_ids) && !node['ndb'][service]['ips_ids'].empty?
+            for srv in node['ndb'][service]['ips_ids']
+              theNode = srv.split(":")
+              if my_ip.eql? theNode[0]
+                found_id = theNode[1]
+                break
+              end
+            end
+          else
+            for api in node['ndb'][service]['private_ips']
+              if my_ip.eql? api
+                Chef::Log.info "Found matching IP address in the list of #{service} nodes: #{api} . ID= #{id}"
+                found_id = id
+              end
+              id += 1
+            end
+          end
+
+          if found_id == -1
+             Chef::Log.fatal "Could not find matching IP address in list of #{service}."
+          end
+
+          return found_id
+        end
+    end
+end
+
+Chef::Recipe.send(:include, NDB::Helpers)
+Chef::Resource.send(:include, NDB::Helpers)