-
Notifications
You must be signed in to change notification settings - Fork 33
/
Copy pathpnda_env_example.yaml
374 lines (305 loc) · 12.5 KB
/
pnda_env_example.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
infrastructure :
# infrastructure used for pnda deployment
# Valid Values are:
# - aws
# - openstack
# - existing-machines
# - terraform
INFRASTRUCTURE_TYPE: aws
# The user name to use when logging into the instances
# For aws target user-name allowed :
# Target AWS Openstack
#
# Distro Redhat: ec2-user cloud-user
# CentOS: centos cloud-user
OS_USER: centos
# CIDR specifying the address range for the network containing all PNDA instances
networkCidr: 10.0.0.0/16
aws_parameters:
# Settings in this section are passed through as parameters when
# API key for full API access for use creating PNDA
# These keys are only ever stored on the client
AWS_ACCESS_KEY_ID: xxxx
AWS_SECRET_ACCESS_KEY: xxxx
# AWS region to create PNDA ec2 instances in
AWS_REGION: eu-west-1
AWS_AVAILABILITY_ZONE: eu-west-1a
# Base image to use, use the default AWS CentOS/Redhat image for the AWS region
imageId: ami-6e28b517
# CIDR specifying the address range that may access the created PNDA instances
whitelistSshAccess: 0.0.0.0/0
# CIDR specifying the address range that may access the gateway
whitelistUserAccess: 0.0.0.0/0
# CIDR specifying the address range for the public subnet (bastion access)
publicSubnetCidr: 10.0.0.0/24
openstack_parameters:
# KEYSTONE_USER: Username for the openstack clients to use
KEYSTONE_USER: username
# KEYSTONE_PASSWORD: Password for the openstack clients to use
KEYSTONE_PASSWORD: password
# KEYSTONE_TENANT: Name of the tenant / project in the openstack environment. The
# PNDA stack will be created in this project.
KEYSTONE_TENANT: pnda
# KEYSTONE_AUTH_URL: Keystone authentication URL. The Openstack console provides this
# under the Access & Security section.
KEYSTONE_AUTH_URL: 'http://x.x.x.x:5000/v2.0'
# KEYSTONE_AUTH_VERSION: Keystone authentication version. The Openstack console provides this
# under the Access & Security section.
KEYSTONE_AUTH_VERSION: '2'
# KEYSTONE_REGION_NAME: Keystone region. The Openstack console provides this
# under the Access & Security section.
KEYSTONE_REGION_NAME: RegionOne
# imageId: Base image to use for the created instances. It should be created by
# following the guide in https://github.com/pndaproject/pnda-dib-elements
#
imageId: 858abfc7-799e-44fe-8681-1cc38d36fde9
# CIDR specifying the address range that may access the created PNDA instances
whitelistSshAccess: 0.0.0.0/0
# UUID of the public network in openstack to use
externalPublicNetworkId: 5982b761-802a-4af3-9c0b-c3b457559179
# CIDR specifying the address range for the public subnet (bastion access)
publicSubnetCidr: 10.0.0.0/24
# Use an existing Openstack network instead of creating one
useExistingNetwork: false
# Name of existing network in Openstack
existingNetworkId:
# Name of existing subnet in Openstack
existingSubnetId:
terraform_parameters:
# Username to log into vSphere
VS_USER: username
# Password to log into vSphere
VS_PASSWORD: password
# Cluster to create PNDA VMs in
VS_PARENT_CLUSTER: pnda
# Datacenter to create PNDA VMs in
VS_DATACENTER: Datacenter
# Datastore name to use when creating PNDA
VS_DS: pnda_01
# Public network create PNDA VMs on
VS_PUBLIC_NETWORK: abc123
# vSphere server
VS_SERVER: esx.pnda.io:443
# Folder to create VMS in
VS_FOLDER: Virtual Machines
# vSphere guest ID
VS_GUEST_ID: centos7_64Guest
# Image to use for the datanode instances
VS_TEMPLATE_DATANODE: centos_base
# Image to use for the kafka instances
VS_TEMPLATE_KAFKA: centos_base
# Image to use for other instances
VS_TEMPLATE_BASE: centos_base
# Root user and password for the base images
TF_ROOT_USER: root
TF_ROOT_PASSWORD: root
platform_salt:
# Use either PLATFORM_GIT_REPO_URI + PLATFORM_GIT_BRANCH or PLATFORM_SALT_LOCAL
# URI of platform salt git repository
PLATFORM_GIT_REPO_URI: https://github.com/pndaproject/platform-salt.git
# Host name of server for platform salt git repository. Make sure to set this if
# authenticated ssh access (using git.pem) to the git repo is desired.
PLATFORM_GIT_REPO_HOST: github.com
# The branch to select from PLATFORM_GIT_REPO_URI
PLATFORM_GIT_BRANCH: develop
# Local path to folder containing a clone of the platform salt repository
# PLATFORM_SALT_LOCAL: /path/to/platform-salt
pnda_application_repo:
# Type of storage to use for PNDA application packages
# s3 - AWS S3. Also set PNDA_APPS_CONTAINER, PNDA_APPS_FOLDER, PNDA_APPS_REGION, PNDA_APPS_ACCESS_KEY_ID, PNDA_APPS_SECRET_ACCESS_KEY
# sshfs - standard file system. Also set PR_FS_LOCATION_PATH, PR_SSHFS_USER, PR_SSHFS_HOST, PR_SSHFS_PATH and PR_SSHFS_KEY
# local - local filesystem on the package repository service server. Also set PR_FS_LOCATION_PATH.
# swift - Openstack swift. Also set PNDA_APPS_CONTAINER and PNDA_APPS_FOLDER
PR_FS_TYPE: s3
# S3 container to use for PNDA application packages
PNDA_APPS_CONTAINER: pnda-apps
# Name of folder within PNDA_APPS_CONTAINER that contains the PNDA application packages
PNDA_APPS_FOLDER: releases
# AWS region that contains the PNDA_APPS_CONTAINER bucket
PNDA_APPS_REGION: eu-west-1
# API key for s3 access to PNDA_APPS_CONTAINER. These keys are stored on the cloud instances so should be restricted
# only allow access to the PNDA_APPS_CONTAINER bucket
PNDA_APPS_ACCESS_KEY_ID: xxxx
PNDA_APPS_SECRET_ACCESS_KEY: xxxx
# Path on file system if PR_FS_TYPE is 'local' or 'sshfs'
PR_FS_LOCATION_PATH: /opt/pnda/packages
# SSH accessed file system to use for PNDA application packages
PR_SSHFS_USER: centos
PR_SSHFS_HOST: 127.0.0.1
PR_SSHFS_PATH: /mnt/packages
PR_SSHFS_KEY: key.pem
pnda_data_archive:
# S3 container to use for archiving PNDA datasets
PNDA_ARCHIVE_CONTAINER: pnda-archive
# AWS region that contains the PNDA_ARCHIVE_CONTAINER bucket
PNDA_ARCHIVE_REGION: eu-west-1
# API key for s3 access to PNDA_ARCHIVE_CONTAINER. These keys are stored on the cloud instances so should be restricted
# only allow access to the PNDA_ARCHIVE_CONTAINER bucket
PNDA_ARCHIVE_ACCESS_KEY_ID: xxxx
PNDA_ARCHIVE_SECRET_ACCESS_KEY: xxxx
ntp:
# Optional ntp servers. Use this if the standard NTP servers on the Internet cannot be reached
# and a local NTP server has been configured. PNDA will not work without NTP.
# example format: 'xxx.ntp.org'
#For REJECT_OUTBOUND="YES" then NTP server/s must.
NTP_SERVERS:
- ntp.ubuntu.com
# - 91.189.89.198
# - 91.189.89.199
# - 91.189.89.200
# - 91.189.89.201
dns:
# External DNS servers list
nameServers:
- 8.8.8.8
# - 8.8.4.4
mirrors:
# Mirror of resources required for provisioning PNDA, see PNDA guide for instructions on how to set this up
PNDA_MIRROR: http://x.x.x.x
hadoop:
# Hadoop distribution to install
# Valid values are:
# - HDP
# - CDH
HADOOP_DISTRO: HDP
# Spark version to enable for oozie (HDP only)
# Valid values are:
# - 1
# - 2
OOZIE_SPARK_VERSION: 1
connectivity:
# Deploy an iptables ruleset to every node preventing outbound access to all hosts except the PNDA_MIRROR, NTP_SERVER and specified CLIENT_IP. Specify YES to enable.
REJECT_OUTBOUND: "YES"
# If using REJECT_OUTBOUND, the IP address of the client that created PNDA
CLIENT_IP: 1.1.1.1
# Add online repositories for yum, apt-get, pip, etc alongside PNDA mirror
ADD_ONLINE_REPOS: "NO"
# RPM Extras repository to enable when ADD_ONLINE_REPOS=YES
RPM_EXTRAS_REPO_NAME: rhui-REGION-rhel-server-optional
# RPM Optional repository to enable when ADD_ONLINE_REPOS=YES
RPM_OPTIONAL_REPO_NAME: rhui-REGION-rhel-server-extras
#ADDITIONAL_REPOS: |
# [pnda_mirror]
# name=PNDA mirror
# baseurl=XXXX/mirror_rpm
# enabled=1
# priority = 1
# gpgcheck = 1
# keepcache = 0
# [base]
# name=CentOS-$releasever - Base
# mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra
# gpgcheck=1
# gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
#
# #released updates
# [updates]
# name=CentOS-$releasever - Updates
# mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra
# gpgcheck=1
# gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
# #additional packages that may be useful
# [extras]
# name=CentOS-$releasever - Extras
# mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra
# gpgcheck=1
# gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
#
# [epel]
# name=Extra Packages for Enterprise Linux 7 - $basearch
# metalink=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch
# gpgkey=https://archive.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
# failovermethod=priority
# enabled=1
# gpgcheck=1
# #VERSION_NUMBER=2.6.1.0-143
# [ambari-2.6.1.0]
# name=ambari Version - ambari-2.6.1.0
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.1.0
# gpgcheck=1
# gpgkey=http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.1.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
# enabled=1
# priority=1
#
# #VERSION_NUMBER=2.7.0.0-897
# [ambari-2.7.0.0]
# #json.url = http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json
# name=ambari Version - ambari-2.7.0.0
# baseurl=http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.0.0
# gpgcheck=1
# gpgkey=http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.0.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
# enabled=1
# priority=1
#
# [archive.cloudera.com_cm5_redhat_7_x86_64_cm_5.12.1_]
# name=added from: http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.12.1/
# baseurl=http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.12.1/
# gpgkey=https://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
# enabled=1
#
#
# [repo.mysql.com_yum_mysql-5.5-community_el_7_x86_64_]
# name=added from: https://repo.mysql.com/yum/mysql-5.5-community/el/7/x86_64/
# baseurl=https://repo.mysql.com/yum/mysql-5.5-community/el/7/x86_64/
# gpgkey=https://repo.mysql.com/RPM-GPG-KEY-mysql
# enabled=1
#
# [repo.saltstack.com_yum_redhat_7_x86_64_archive_2015.8.11]
# name=added from: https://repo.saltstack.com/yum/redhat/7/x86_64/archive/2015.8.11
# baseurl=https://repo.saltstack.com/yum/redhat/7/x86_64/archive/2015.8.11
# gpgkey=https://repo.saltstack.com/yum/redhat/7/x86_64/archive/2015.8.11/SALTSTACK-GPG-KEY.pub
# enabled=1
network_interfaces:
PNDA_INTERNAL_NETWORK: eth0
PNDA_INGEST_NETWORK: eth0
cli:
# Maximum number of outbound connections that the CLI will attempt to open at once
# Consider increasing this when creating clusters with more than 100 nodes to speed
# up PNDA creation time.
MAX_SIMULTANEOUS_OUTBOUND_CONNECTIONS: 100
security:
# The path were to find the security material (certificate/key).
# The directory should be structured as defined in this' repo's directory structure with the same name.
# The security material should conform to the guidelines defined in the README.md file in
# the containing sub directory.
SECURITY_MATERIAL_PATH: ./platform-certificates/
# Address of LDAP server
# All instances will have PAM configured to authenticate with this LDAP server if set
# Leave blank to disable LDAP-PAM integration
LDAP_SERVER: ''
# Base DN for LDAP server to use when enabling client PAM integration with LDAP
LDAP_BASE_DN: dc=nodomain
features:
# Include experimental features.
# Set to "NO", omit setting or omit features section entirely to turn off experimental features
EXPERIMENTAL_FEATURES: "NO"
domain:
# Top-level domain
TOP_LEVEL_DOMAIN: pnda.local
# Second-level domain
SECOND_LEVEL_DOMAIN: dc1
dataset_compaction:
# Enable/Disable compaction on datasets.
# "YES" to enable.
# "NO" to disable.
COMPACTION: "NO"
# If compaction is enabled, PATTERN sets the frequency of compaction.
# H - hourly compaction.
# d - daily compaction.
# M - monthly compaction.
# Y - yearly compaction.
PATTERN: d
datanode:
# DATANODE_VOLUME_COUNT sets the number of data volumes on each hadoop datanode
DATA_VOLUME_COUNT: 1
# DEVICE_ROOT sets the disk device root name
DEVICE_ROOT: xvdb
kafka:
# DATA_DIRS sets the data dirs on kafka node
KAFKA_DATA_DIRS:
- /var/kafka-logs
# DEVICE_ROOT sets the disk device root name
KAFKA_DEVICE_ROOT: xvdb
generic:
#GENERIC_DEVICE_ROOT sets the disk device root name for generic instances.
GENERIC_DEVICE_ROOT: xvdb