-
Notifications
You must be signed in to change notification settings - Fork 32
/
Copy pathmetadata.rb
executable file
·176 lines (137 loc) · 6.68 KB
/
metadata.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
name "hadoop_spark"
maintainer "Jim Dowling"
maintainer_email "[email protected]"
license "Apache v2"
description 'Installs/Configures Spark'
long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
version "4.0.0"
source_url "https://github.com/hopshadoop/spark-chef"
depends 'magic_shell', '~> 1.0.0'
depends 'conda'
depends 'kagent'
depends 'ndb'
depends 'hops'
depends 'hive2'
depends 'hopsmonitor'
depends 'java'
recipe "install", "Installs Spark binaries"
#link:<a target='_blank' href='http://%host%:8080/'>Launch the WebUI for the Spark Master</a>
recipe "master", "Starts a Spark master"
#link:<a target='_blank' href='http://%host%:8081/'>Launch the WebUI for the Spark Slave %host%</a>
recipe "worker", "Starts a Spark worker"
recipe "yarn", "Creates directories for yarn. Run on only one machine."
recipe "default", "Install spark binaries only."
recipe "libs", "Install spark jars to nodemanager hosts"
recipe "historyserver", "Installs/starts the Spark historyserver"
recipe "config", "Configuration files for Spark/Yarn"
attribute "hadoop_spark/user",
:description => "Username to run spark master/worker as",
:type => 'string'
attribute "hadoop_spark/user_id",
:description => "Spark user id. Default: 1505",
:type => 'string'
attribute "hadoop_spark/dir",
:description => "Installation dir for spark",
:type => 'string'
attribute "hadoop_spark/executor_memory",
:description => "Executor memory (e.g., 512m)",
:type => 'string'
attribute "hadoop_spark/driver_memory",
:description => "Driver memory (e.g., 1g)",
:type => 'string'
attribute "hadoop_spark/eventlog_enabled",
:description => "Eventlog enabled (true|false)",
:type => 'string'
attribute "hadoop_spark/streaming/stopGracefullyOnShutdown",
:description => "Shut down the StreamingContext gracefully on JVM shutdown rather than immediately (true|false)",
:type => 'string'
attribute "hadoop_spark/worker/cleanup/enabled",
:description => "Spark standalone worker cleanup enabled (true|false)",
:type => 'string'
attribute "hadoop_spark/version",
:description => "Spark version (e.g., 1.6.1 or 2.0.1 or 4.0.0)",
:type => 'string'
attribute "hadoop_spark/history/fs/cleaner/enabled",
:description => "'true' to enable cleanup of the historyservers logs",
:type => 'string'
attribute "hadoop_spark/history/fs/cleaner/interval",
:description => "How often to run the cleanup of the historyservers logs (e.g., '1d' for once per day)",
:type => 'string'
attribute "hadoop_spark/history/fs/cleaner/maxAge",
:description => "Age in days of the historyservers logs before they are removed (e.g., '7d' for 7 days)",
:type => 'string'
attribute "hadoop_spark/history/retainedApplications",
:description => "The number of applications to retain UI data for in the cache. If an application is not in the cache, it will have to be loaded from disk if it is accessed from the UI. (Default 20)",
:type => 'string'
attribute "hadoop_spark/yarn/am/attemptFailuresValidityInterval",
:description => "Defines the validity interval for AM failure tracking. If the AM has been running for at least the defined interval, the AM failure count will be reset.",
:type => 'string'
attribute "hadoop_spark/driver/maxResultSize",
:description => "Default '5g'. Change to '1g', '500m', etc",
:type => 'string'
attribute "hadoop_spark/daemon/memory",
:description => "Heap size for the Spark history server. Default 2G",
:type => 'string'
attribute "hadoop_spark/historyserver/private_ips",
:description => "historyserver ip addr",
:type => 'array'
attribute "install/dir",
:description => "Set to a base directory under which we will install.",
:type => "string"
attribute "install/user",
:description => "User to install the services as",
:type => "string"
attribute "hopslog/default/private_ips",
:description => "elk services ip",
:type => "string"
attribute "hadoop_spark/tf_spark_connector_version",
:description => "the version of the tf-spark-connector .jar",
:type => "string"
attribute "hadoop_spark/spark_tfrecord_version",
:description => "the version of the spark-tfrecord library .jar",
:type => "string"
attribute "hadoop_spark/hopsworks_jobs_py/version",
:description => "the version of the hopsworks-jobs .py",
:type => "string"
attribute "hadoop_spark/hopsworks_jobs_py/url",
:description => "the url for dowloading the hopsworks-jobs .py",
:type => "string"
attribute "hadoop_spark/spark_avro_version",
:description => "the version of the spark-avro jar",
:type => "string"
attribute "hadoop_spark/databricks_delta_version",
:description => "the version of the databricks delta jar",
:type => "string"
attribute "hadoop_spark/url",
:description => "the url for downloading the spark tgz",
:type => "string"
attribute "hadoop_spark/hsfs/rc_version",
:description => "Optionally a Release Candidate version suffixed to the main version ie RC0, Default: ''",
:type => "string"
attribute "hadoop_spark/hsfs/version",
:description => "Version of the HSFS library",
:type => "string"
attribute "hadoop_spark/hsfs/url",
:description => "URL from where to download the HSFS library",
:type => "string"
attribute "hadoop_spark/snowflake-jdbc/version",
:description => "Version of the snowflake jdbc driver",
:type => "string"
attribute "hadoop_spark/snowflake-jdbc/url",
:description => "URL from where to download the snowflake jdbc driver",
:type => "string"
attribute "hadoop_spark/spark-snowflake/artifactID",
:description => "Artifact id of the spark-snowflake connector",
:type => "string"
attribute "hadoop_spark/spark-snowflake/version",
:description => "Version of the spark-snowflake connector",
:type => "string"
attribute "hadoop_spark/spark-snowflake/url",
:description => "URL from where to download the spark-snowflake connector",
:type => "string"
attribute "hadoop_spark/hudi_spark_bundle_url",
:description => "URL for hudi spark jar",
:type => "string"
attribute "hadoop_spark/hudi_util_bundle_url",
:description => "URL for hudi utiljar",
:type => "string"