Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-11590. Use JSON to pass config for docker #7672

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ services:
volumes:
- ../..:/opt/ozone
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
- ../_keytabs:/etc/security/keytabs
- ./krb5.conf:/etc/krb5.conf
ports:
@@ -41,6 +42,7 @@ services:
volumes:
- ../..:/opt/ozone
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
- ../_keytabs:/etc/security/keytabs
- ./krb5.conf:/etc/krb5.conf
env_file:
@@ -63,6 +65,7 @@ services:
- ../_keytabs:/etc/security/keytabs
- ./krb5.conf:/etc/krb5.conf
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
ports:
- 8188:8188
env_file:
26 changes: 14 additions & 12 deletions hadoop-ozone/dist/src/main/compose/common/hadoop-security.conf
Original file line number Diff line number Diff line change
@@ -14,15 +14,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.

YARN-SITE.XML_yarn.nodemanager.principal=nm/[email protected]
YARN-SITE.XML_yarn.nodemanager.keytab=/etc/security/keytabs/nm.keytab
YARN-SITE.XML_yarn.resourcemanager.keytab=/etc/security/keytabs/rm.keytab
YARN-SITE.XML_yarn.resourcemanager.principal=rm/[email protected]
YARN-SITE.XML_yarn.timeline-service.principal=jhs/[email protected]
YARN-SITE.XML_yarn.timeline-service.keytab=/etc/security/keytabs/jhs.keytab
HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
HADOOP-POLICY.XML_org.apache.hadoop.yarn.server.api.ResourceTracker.acl=*
OZONE_CONF_CONTAINER_HADOOP_SECURITY='{
"YARN-SITE.XML_yarn.nodemanager.principal": "nm/[email protected]",
"YARN-SITE.XML_yarn.nodemanager.keytab": "/etc/security/keytabs/nm.keytab",
"YARN-SITE.XML_yarn.resourcemanager.keytab": "/etc/security/keytabs/rm.keytab",
"YARN-SITE.XML_yarn.resourcemanager.principal": "rm/[email protected]",
"YARN-SITE.XML_yarn.timeline-service.principal": "jhs/[email protected]",
"YARN-SITE.XML_yarn.timeline-service.keytab": "/etc/security/keytabs/jhs.keytab",
"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*","
"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",
"HADOOP-POLICY.XML_org.apache.hadoop.yarn.server.api.ResourceTracker.acl": "*"
}'
81 changes: 42 additions & 39 deletions hadoop-ozone/dist/src/main/compose/common/hadoop.conf
Original file line number Diff line number Diff line change
@@ -14,48 +14,51 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs

MAPRED-SITE.XML_mapreduce.framework.name=yarn
MAPRED-SITE.XML_mapreduce.map.memory.mb=4096
MAPRED-SITE.XML_mapreduce.reduce.memory.mb=4096
MAPRED-SITE.XML_mapred.child.java.opts=-Xmx2g

YARN-SITE.XML_yarn.app.mapreduce.am.staging-dir=/user
YARN_SITE.XML_yarn.timeline-service.enabled=true
YARN_SITE.XML_yarn.timeline-service.generic.application.history.enabled=true
YARN_SITE.XML_yarn.timeline-service.hostname=jhs
YARN_SITE.XML_yarn.log.server.url=http://jhs:8188/applicationhistory/logs/

YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=6000
YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable=false

YARN-SITE.XML_yarn.resourcemanager.hostname=rm
YARN_SITE_XML_yarn.resourcemanager.system.metrics.publisher.enabled=true

#YARN-SITE.XML_yarn.log-aggregation-enable=true
#YARN-SITE.XML_yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds=3600

#YARN-SITE.yarn.nodemanager.container-executor.class=org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor
#YARN-SITE.XML_yarn.nodemanager.linux-container-executor.path=/opt/hadoop/bin/container-executor
#YARN-SITE.XML_yarn.nodemanager.linux-container-executor.group=hadoop
YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99
YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable=false

CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false

OZONE_CONF_CONTAINER_HADOOP='{
"CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl": "org.apache.hadoop.fs.ozone.OzFs",
"CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl": "org.apache.hadoop.fs.ozone.RootedOzFs",

"MAPRED-SITE.XML_mapreduce.framework.name": "yarn",
"MAPRED-SITE.XML_mapreduce.map.memory.mb": "4096",
"MAPRED-SITE.XML_mapreduce.reduce.memory.mb": "4096",
"MAPRED-SITE.XML_mapred.child.java.opts": "-Xmx2g",

"YARN-SITE.XML_yarn.app.mapreduce.am.staging-dir": "/user",
"YARN_SITE.XML_yarn.timeline-service.enabled": "true",
"YARN_SITE.XML_yarn.timeline-service.generic.application.history.enabled": "true",
"YARN_SITE.XML_yarn.timeline-service.hostname": "jhs",
"YARN_SITE.XML_yarn.log.server.url": "http://jhs:8188/applicationhistory/logs/",

"YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled": "false",
"YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec": "6000",
"YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled": "false",
"YARN-SITE.XML_yarn.nodemanager.aux-services": "mapreduce_shuffle",
"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable": "false",

"YARN-SITE.XML_yarn.resourcemanager.hostname": "rm",
"YARN_SITE_XML_yarn.resourcemanager.system.metrics.publisher.enabled": "true",

"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "99",
"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable": "false",

"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications": "10000",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent": "0.1",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues": "default",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity": "100",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor": "1",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity": "100",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state": "RUNNING",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications": "*",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue": "*",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay": "40",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings": "",
"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable": "false"
}'
2 changes: 2 additions & 0 deletions hadoop-ozone/dist/src/main/compose/common/hadoop.yaml
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ services:
volumes:
- ../..:/opt/ozone
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
ports:
- 8088:8088
env_file:
@@ -36,6 +37,7 @@ services:
volumes:
- ../..:/opt/ozone
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
env_file:
- docker-config
- ../common/hadoop.conf
4 changes: 3 additions & 1 deletion hadoop-ozone/dist/src/main/compose/common/hadoop2.conf
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
OZONE_CONF_CONTAINER_HADOOP2='{
"MAPRED-SITE.XML_mapreduce.application.classpath": "/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar"
}'

HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
4 changes: 3 additions & 1 deletion hadoop-ozone/dist/src/main/compose/common/hadoop3.conf
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
OZONE_CONF_CONTAINER_HADOOP3='{
"MAPRED-SITE.XML_mapreduce.application.classpath": "/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar"
}'

HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar
176 changes: 90 additions & 86 deletions hadoop-ozone/dist/src/main/compose/common/security.conf
Original file line number Diff line number Diff line change
@@ -15,91 +15,95 @@
# limitations under the License.

# For HttpFS service it is required to enable proxying users.
CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*

CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms

OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM

OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM

OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

OZONE-SITE.XML_hdds.block.token.enabled=true
OZONE-SITE.XML_hdds.container.token.enabled=true
OZONE-SITE.XML_hdds.grpc.tls.enabled=true
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"
OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer

OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m
OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m
OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h

OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos

OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/db@EXAMPLE.COM
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab

CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
CORE-SITE.XML_hadoop.http.authentication.type=kerberos
CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

CORE-SITE.XML_hadoop.security.authorization=true
HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*

HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos
HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
OZONE_CONF_CONTAINER_SECURITY='{
"CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts": "*",
"CORE-SITE.XML_hadoop.proxyuser.httpfs.groups": "*",

"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms",

"CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed": "false",
"CORE-SITE.XML_hadoop.http.authentication.signature.secret.file": "/etc/security/http_secret",
"CORE-SITE.XML_hadoop.http.authentication.type": "kerberos",
"CORE-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
"CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",

"CORE-SITE.XML_hadoop.security.authorization": "true",

"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
"OZONE-SITE.XML_ozone.recon.kerberos.keytab.file": "/etc/security/keytabs/recon.keytab",
"OZONE-SITE.XML_ozone.recon.kerberos.principal": "recon/recon@EXAMPLE.COM",

"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",

"OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file": "/etc/security/keytabs/httpfs.keytab",
"OZONE-SITE.XML_ozone.httpfs.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",

"OZONE-SITE.XML_hdds.block.token.enabled": "true",
"OZONE-SITE.XML_hdds.container.token.enabled": "true",
"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",
"OZONE-SITE.XML_ozone.security.enabled": "true",
"OZONE-SITE.XML_ozone.acl.enabled": "true",
"OZONE-SITE.XML_ozone.acl.authorizer.class": "org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer",
"OZONE-SITE.XML_ozone.administrators": "testuser,recon,om",
"OZONE-SITE.XML_ozone.s3.administrators": "testuser,s3g",
"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",
"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.AuthenticationFilterInitializer",

"OZONE-SITE.XML_hdds.secret.key.rotate.duration": "5m",
"OZONE-SITE.XML_hdds.secret.key.rotate.check.duration": "1m",
"OZONE-SITE.XML_hdds.secret.key.expiry.duration": "1h",

"OZONE-SITE.XML_ozone.om.http.auth.type": "kerberos",
"OZONE-SITE.XML_hdds.scm.http.auth.type": "kerberos",
"OZONE-SITE.XML_hdds.datanode.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.s3g.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.s3g.secret.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.httpfs.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.recon.http.auth.type": "kerberos",

"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/db@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal": "HTTP/recon@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",
"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",

"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",

"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",

"HTTPFS-SITE.XML_hadoop.http.authentication.type": "kerberos",
"HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
"HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
"HTTPFS-SITE.XML_httpfs.hadoop.authentication.type": "kerberos",
"HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
"HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",

"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users": "*",
"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups": "*",
"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts": "*",
}'

OZONE_DATANODE_SECURE_USER=root
38 changes: 20 additions & 18 deletions hadoop-ozone/dist/src/main/compose/compatibility/docker-config
Original file line number Diff line number Diff line change
@@ -14,24 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE-SITE.XML_ozone.om.features.disabled=ATOMIC_REWRITE_KEY
OZONE_CONF_CONTAINER_COMPATIBILITY='{
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
"OZONE-SITE.XML_ozone.om.features.disabled": "ATOMIC_REWRITE_KEY"
}'

HADOOP_OPTS="-Dhadoop.opts=test"
HDFS_STORAGECONTAINERMANAGER_OPTS="-Dhdfs.scm.opts=test"
77 changes: 40 additions & 37 deletions hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config
Original file line number Diff line number Diff line change
@@ -15,46 +15,49 @@
# limitations under the License.

# For HttpFS service it is required to enable proxying users.
CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
OZONE_CONF_CONTAINER_BALANCER='{
"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",

CORE-SITE.XML_fs.defaultFS=ofs://om/
CORE-SITE.XML_fs.trash.interval=1
"CORE-SITE.XML_fs.defaultFS": "ofs://om/",
"CORE-SITE.XML_fs.trash.interval": "1",

OZONE-SITE.XML_ozone.om.service.ids=om
OZONE-SITE.XML_ozone.om.nodes.om=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.om.om1=om1
OZONE-SITE.XML_ozone.om.address.om.om2=om2
OZONE-SITE.XML_ozone.om.address.om.om3=om3
"OZONE-SITE.XML_ozone.om.service.ids": "om",
"OZONE-SITE.XML_ozone.om.nodes.om": "om1,om2,om3",
"OZONE-SITE.XML_ozone.om.address.om.om1": "om1",
"OZONE-SITE.XML_ozone.om.address.om.om2": "om2",
"OZONE-SITE.XML_ozone.om.address.om.om3": "om3",

"OZONE-SITE.XML_ozone.scm.service.ids": "scmservice",
"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1": "scm1",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2": "scm2",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3": "scm3",
"OZONE-SITE.XML_ozone.scm.ratis.enable": "true",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.container.size": "100MB",
"OZONE-SITE.XML_ozone.scm.block.size": "20MB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_hdds.node.report.interval": "20s",
"OZONE-SITE.XML_hdds.heartbeat.interval": "20s",
"OZONE-SITE.XML_hdds.datanode.du.refresh.period": "20s",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.container.db.dir": "/data/metadata",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one": "false",
"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.scm.primordial.node.id": "scm1",
"OZONE-SITE.XML_hdds.container.report.interval": "30s",
"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_dfs.container.ratis.datastream.enabled": "true",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
"OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval": "25s",
"OZONE-SITE.XML_hdds.container.balancer.trigger.du.before.move.enable": "false"
}'

OZONE-SITE.XML_ozone.scm.service.ids=scmservice
OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1
OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2
OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3
OZONE-SITE.XML_ozone.scm.ratis.enable=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.container.size=100MB
OZONE-SITE.XML_ozone.scm.block.size=20MB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_hdds.node.report.interval=20s
OZONE-SITE.XML_hdds.heartbeat.interval=20s
OZONE-SITE.XML_hdds.datanode.du.refresh.period=20s
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.container.db.dir=/data/metadata
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one=false
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
OZONE-SITE.XML_hdds.container.report.interval=30s
OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval=25s
OZONE-SITE.XML_hdds.container.balancer.trigger.du.before.move.enable=false
OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop

42 changes: 22 additions & 20 deletions hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
Original file line number Diff line number Diff line change
@@ -14,28 +14,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://om
OZONE_CONF_CONTAINER_BALANCER='{
"CORE-SITE.XML_fs.defaultFS": "ofs://om",

OZONE-SITE.XML_ozone.csi.owner=hadoop
OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
"OZONE-SITE.XML_ozone.csi.owner": "hadoop",
"OZONE-SITE.XML_ozone.csi.socket": "/tmp/csi.sock",

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
}'

OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
71 changes: 39 additions & 32 deletions hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config
Original file line number Diff line number Diff line change
@@ -15,41 +15,48 @@
# limitations under the License.

# For HttpFS service it is required to enable proxying users.
CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*

CORE-SITE.XML_fs.defaultFS=ofs://omservice/
CORE-SITE.XML_fs.trash.interval=1
OZONE_CONF_CONTAINER_HA='{
"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",

OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
"CORE-SITE.XML_fs.defaultFS": "ofs://omservice/",
"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3"
"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1"
"OZONE-SITE.XML_ozone.om.address.omservice.om2=": "om2"
"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3"

OZONE-SITE.XML_ozone.scm.service.ids=scmservice
OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1
OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2
OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3
OZONE-SITE.XML_ozone.scm.ratis.enable=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
OZONE-SITE.XML_hdds.container.report.interval=60s
OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=true
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",
"OZONE-SITE.XML_ozone.om.ratis.enable": "true",

"OZONE-SITE.XML_ozone.scm.service.ids": "scmservice",
"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1": "scm1",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2": "scm2",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3": "scm3",
"OZONE-SITE.XML_ozone.scm.ratis.enable": "true",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.scm.primordial.node.id": "scm1",
"OZONE-SITE.XML_hdds.container.report.interval": "60s",
"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
"OZONE-SITE.XML_hdds.container.ratis.datastream.enabled": "true",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
}'

OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
58 changes: 32 additions & 26 deletions hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config
Original file line number Diff line number Diff line change
@@ -14,33 +14,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://omservice
OZONE_CONF_CONTAINER_OM_HA='{
"CORE-SITE.XML_fs.defaultFS": "ofs://omservice",

"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.handler.type": "distributed",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.server.default.replication": "1",
"OZONE-SITE.XML_ozone.client.failover.max.attempts": "6",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_hdds.profiler.endpoint.enabled": "true",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_hdds.container.report.interval": "60s",
"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
}'

OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.server.default.replication=1
OZONE-SITE.XML_ozone.client.failover.max.attempts=6
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.container.report.interval=60s
OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
ASYNC_PROFILER_HOME=/opt/profiler

LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -96,4 +100,6 @@ OZONE_LOG_DIR=/var/log/hadoop
no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1

# Explicitly enable filesystem snapshot feature for this Docker compose cluster
OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
OZONE_CONF_CONTAINER_OM_HA_SNAPSHOT='{
"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
}'
34 changes: 18 additions & 16 deletions hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config
Original file line number Diff line number Diff line change
@@ -14,23 +14,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://omservice/
OZONE_CONF_CONTAINER_OM_HA='{
"CORE-SITE.XML_fs.defaultFS": "ofs://omservice/",

OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",

OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.client.failover.max.attempts=6
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.client.failover.max.attempts": "6",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
}'

no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1
65 changes: 34 additions & 31 deletions hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
Original file line number Diff line number Diff line change
@@ -14,39 +14,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://om
OZONE_CONF_CONTAINER_TOPOLOGY='{
"CORE-SITE.XML_fs.defaultFS": "ofs://om",

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.container.size=256MB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.ozone.scm.block.size=64MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.server.default.replication=3
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
OZONE-SITE.XML_ozone.network.topology.aware.read=true
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.container.size": "256MB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.ozone.scm.block.size": "64MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.handler.type": "distributed",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.server.default.replication": "3",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
"OZONE-SITE.XML_hdds.profiler.endpoint.enabled": "true",
"OZONE-SITE.XML_ozone.scm.container.placement.impl": "org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware",
"OZONE-SITE.XML_net.topology.node.switch.mapping.impl": "org.apache.hadoop.net.TableMapping",
"OZONE-SITE.XML_net.topology.table.file.name": "/opt/hadoop/compose/ozone-topology/network-config",
"OZONE-SITE.XML_ozone.network.topology.aware.read": "true",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",

"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
}'

HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
ASYNC_PROFILER_HOME=/opt/profiler
OZONE_DATANODE_OPTS=-Dmodule.name=datanode
OZONE_MANAGER_OPTS=-Dmodule.name=om
85 changes: 43 additions & 42 deletions hadoop-ozone/dist/src/main/compose/ozone/docker-config
Original file line number Diff line number Diff line change
@@ -14,53 +14,54 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://om
CORE-SITE.XML_fs.trash.interval=1
# For HttpFS service it is required to enable proxying users.
CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
OZONE_CONF_CONTAINER_BASIC='{
"CORE-SITE.XML_fs.defaultFS": "ofs://om",
"CORE-SITE.XML_fs.trash.interval": "1",

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.block.size=1MB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.container.report.interval=60s
OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
OZONE-SITE.XML_hdds.heartbeat.interval=5s
OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s
OZONE-SITE.XML_hdds.scm.replication.thread.interval=15s
OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval=5s
OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval=5s
OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",

OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=true
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.block.size": "1MB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_hdds.container.report.interval": "60s",
"OZONE-SITE.XML_ozone.scm.stale.node.interval": "30s",
"OZONE-SITE.XML_ozone.scm.dead.node.interval": "45s",
"OZONE-SITE.XML_hdds.heartbeat.interval": "5s",
"OZONE-SITE.XML_ozone.scm.close.container.wait.duration": "5s",
"OZONE-SITE.XML_hdds.scm.replication.thread.interval": "15s",
"OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval": "5s",
"OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval": "5s",
"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",

OZONE-SITE.XML_ozone.fs.hsync.enabled=true
"OZONE-SITE.XML_hdds.container.ratis.datastream.enabled": "true",

"OZONE-SITE.XML_ozone.fs.hsync.enabled": "true",

"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
}'

OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop

no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1

# Explicitly enable filesystem snapshot feature for this Docker compose cluster
OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml
Original file line number Diff line number Diff line change
@@ -23,5 +23,5 @@ services:
- docker-config
- monitoring.conf
environment:
- "OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1}"
- "OZONE_CONF_CONTAINER_FREON_OCKG='{\"OZONE-SITE.XML_ozone.server.default.replication\": \"${OZONE_REPLICATION_FACTOR:-1}\"}'"
command: ["ozone","freon","ockg","-n100000"]
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml
Original file line number Diff line number Diff line change
@@ -23,5 +23,5 @@ services:
- docker-config
- monitoring.conf
environment:
- "OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1}"
- "OZONE_CONF_CONTAINER_FREON_RK='{\"OZONE-SITE.XML_ozone.server.default.replication\": \"${OZONE_REPLICATION_FACTOR:-1}\"}'"
command: ["ozone","freon","rk"]
12 changes: 7 additions & 5 deletions hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf
Original file line number Diff line number Diff line change
@@ -14,11 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
OZONE-SITE.XML_hdds.tracing.enabled=true
OZONE-SITE.XML_ozone.metastore.rocksdb.statistics=ALL
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
OZONE_CONF_CONTAINER_MONITORING='{
"OZONE-SITE.XML_hdds.prometheus.endpoint.enabled": "true",
"OZONE-SITE.XML_hdds.tracing.enabled": "true",
"OZONE-SITE.XML_ozone.metastore.rocksdb.statistics": "ALL",
"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
}'
JAEGER_SAMPLER_PARAM=1
JAEGER_SAMPLER_TYPE=const
JAEGER_AGENT_HOST=jaeger
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ x-profiling-config:
&profiling-config
privileged: true
environment:
- OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
- OZONE_CONF_CONTAINER_PROFILING="{\"OZONE-SITE.XML_hdds.profiler.endpoint.enabled\": true}"
- ASYNC_PROFILER_HOME=/opt/profiler

services:
56 changes: 29 additions & 27 deletions hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
Original file line number Diff line number Diff line change
@@ -14,35 +14,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://om
OZONE_CONF_CONTAINER_BLOCKADE='{
"CORE-SITE.XML_fs.defaultFS": "ofs://om

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.client.max.retries=10
OZONE-SITE.XML_ozone.scm.stale.node.interval=2m
OZONE-SITE.XML_ozone.scm.dead.node.interval=5m
OZONE-SITE.XML_ozone.server.default.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout=15s
OZONE-SITE.XML_hdds.heartbeat.interval=2s
OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s
OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s
OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.handler.type": "distributed",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.client.max.retries": "10",
"OZONE-SITE.XML_ozone.scm.stale.node.interval": "2m",
"OZONE-SITE.XML_ozone.scm.dead.node.interval": "5m",
"OZONE-SITE.XML_ozone.server.default.replication": "1",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout": "15s",
"OZONE-SITE.XML_hdds.heartbeat.interval": "2s",
"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "30s",
"OZONE-SITE.XML_hdds.scm.replication.thread.interval": "6s",
"OZONE-SITE.XML_hdds.scm.replication.event.timeout": "10s",
"OZONE-SITE.XML_dfs.ratis.server.failure.duration": "35s",
"OZONE-SITE.XML_hdds.scm.safemode.min.datanode": "3",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",

HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
}'

#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
30 changes: 16 additions & 14 deletions hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
Original file line number Diff line number Diff line change
@@ -14,20 +14,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.server.default.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE_CONF_CONTAINER_OZONE_SCRIPT='{
"CORE-SITE.XML_fs.defaultFS": "hdfs://namenode:9000",
"OZONE-SITE.XML_ozone.ksm.address": "ksm",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.server.default.replication": "1",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
}'

OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
Original file line number Diff line number Diff line change
@@ -37,6 +37,7 @@ services:
- ../_keytabs:/etc/security/keytabs
- ./krb5.conf:/etc/krb5.conf
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
environment:
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
command: ["hadoop", "kms"]
288 changes: 146 additions & 142 deletions hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -14,7 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.om.ratis.log.purge.gap=50
OZONE-SITE.XML_ozone.om.ratis.segment.size=16KB
OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size=16KB
OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold=500
OZONE_CONF_CONTAINER_RATIS_BOOTSTRAP='{
"OZONE-SITE.XML_ozone.om.ratis.log.purge.gap": "50",
"OZONE-SITE.XML_ozone.om.ratis.segment.size": "16KB",
"OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size": "16KB",
"OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold": "500"
}'
Original file line number Diff line number Diff line change
@@ -14,5 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3,scm4
OZONE-SITE.XML_ozone.scm.address.scmservice.scm4=scm4.org
OZONE_CONF_CONTAINER_SCM4='{
"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3,scm4",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm4": "scm4.org"
}'
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE_CONF_CONTAINER_CA_ROTATION='{
"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
"OZONE-SITE.XML_hdds.x509.max.duration": "PT240S",
"OZONE-SITE.XML_hdds.x509.default.duration": "PT60S",
"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT45S",
"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S",
"OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval": "PT2s",
"OZONE-SITE.XML_hdds.block.token.expiry.time": "15s",
"OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime": "15s",
"OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval": "15s",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "60s",
"OZONE-SITE.XML_hdds.scmclient.failover.retry.interval": "1s",
"OZONE-SITE.XML_hdds.scmclient.failover.max.retry": "60",
"OZONE-SITE.XML_ozone.scm.info.wait.duration": "60s",
"OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout": "2s",
"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.HttpCrossOriginFilterInitializer",
"OZONE-SITE.XML_hdds.x509.ca.rotation.enabled": "true"
}'
Original file line number Diff line number Diff line change
@@ -16,24 +16,8 @@

x-root-cert-rotation-config:
&root-cert-rotation-config
environment:
- OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
- OZONE-SITE.XML_hdds.x509.max.duration=PT240S
- OZONE-SITE.XML_hdds.x509.default.duration=PT60S
- OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S
- OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
- OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
- OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT2s
- OZONE-SITE.XML_hdds.block.token.expiry.time=15s
- OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s
- OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s
- OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=60s
- OZONE-SITE.XML_hdds.scmclient.failover.retry.interval=1s
- OZONE-SITE.XML_hdds.scmclient.failover.max.retry=60
- OZONE-SITE.XML_ozone.scm.info.wait.duration=60s
- OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout=2s
- OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.HttpCrossOriginFilterInitializer
- OZONE-SITE.XML_hdds.x509.ca.rotation.enabled=true
env_file:
- ./root-ca-rotation.conf
services:
datanode1:
<<: *root-cert-rotation-config
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE_CONF_CONTAINER_S3G='{
"OZONE-SITE.XML_ozone.s3g.domain.name": "s3g.internal"
}'
Original file line number Diff line number Diff line change
@@ -16,8 +16,8 @@

x-s3g-virtual-host-config:
&s3g-virtual-host-config
environment:
- OZONE-SITE.XML_ozone.s3g.domain.name=s3g.internal
env_file:
- ./s3g-virtual-host.conf
services:
datanode1:
<<: *s3g-virtual-host-config
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ services:
volumes:
- ./krb5.conf:/etc/krb5.conf
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
command: ["hadoop", "kms"]
datanode:
image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
106 changes: 56 additions & 50 deletions hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
Original file line number Diff line number Diff line change
@@ -14,63 +14,67 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.block.token.enabled=true
OZONE-SITE.XML_hdds.container.token.enabled=true
OZONE-SITE.XML_ozone.server.default.replication=3
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.administrators=*
OZONE-SITE.XML_ozone.s3.administrators="s3g"
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE_CONF_CONTAINER_MR='{
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.handler.type": "distributed",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_hdds.block.token.enabled": "true",
"OZONE-SITE.XML_hdds.container.token.enabled": "true",
"OZONE-SITE.XML_ozone.server.default.replication": "3",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
"OZONE-SITE.XML_ozone.administrators": "*",
"OZONE-SITE.XML_ozone.s3.administrators": "s3g",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",

OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
"OZONE-SITE.XML_ozone.security.enabled": "true",
"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",

OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
OZONE-SITE.XML_hdds.grpc.tls.enabled=true
"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/scm.keytab",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/om.keytab",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/s3g.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",

OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/dn.keytab
"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/dn.keytab",

OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",

CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms"
}'

#temporarily disable authorization as org.apache.hadoop.yarn.server.api.ResourceTrackerPB is not properly annotated to support it
CORE-SITE.XML_hadoop.security.authorization=false
OZONE_CONF_CONTAINER_MR_SECURITY='{
"CORE-SITE.XML_hadoop.security.authorization": "false"
}

#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
@@ -85,4 +89,6 @@ OZONE_LOG_DIR=/var/log/hadoop
no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1

# Explicitly enable filesystem snapshot feature for this Docker compose cluster
OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
OZONE_CONF_CONTAINER_MR_SNAPSHOT='{
"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
}'
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE_CONF_CONTAINER_CERTIFICATE_ROTATION='{
"OZONE-SITE.XML_hdds.x509.default.duration": "PT40s",
"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT30s",
"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S"
}'
Original file line number Diff line number Diff line change
@@ -16,12 +16,8 @@

x-cert-rotation-config:
&cert-rotation-config
environment:
- OZONE-SITE.XML_hdds.x509.default.duration=PT40s
- OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT30s
- OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
- OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
- OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
env_file:
- ./certificate-rotation.conf
services:
datanode:
<<: *cert-rotation-config
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@ services:
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
volumes:
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
command: ["hadoop", "kms"]
datanode:
image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
290 changes: 148 additions & 142 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

Large diffs are not rendered by default.

21 changes: 21 additions & 0 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/fcq.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE_CONF_CONTAINER_FCQ='{
"CORE-SITE.XML_ipc.9862.callqueue.impl": "org.apache.hadoop.ipc.FairCallQueue",
"CORE-SITE.XML_ipc.9862.identity-provider.impl": "org.apache.hadoop.ozone.om.helpers.OzoneIdentityProvider",
"OZONE-SITE.XML_ozone.om.transport.class": "org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory"
}'
7 changes: 2 additions & 5 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/fcq.yaml
Original file line number Diff line number Diff line change
@@ -16,11 +16,8 @@

x-FCQ-config:
&FCQ-config
environment:
- CORE-SITE.XML_ipc.9862.callqueue.impl=org.apache.hadoop.ipc.FairCallQueue
- CORE-SITE.XML_ipc.9862.identity-provider.impl=org.apache.hadoop.ozone.om.helpers.OzoneIdentityProvider
- OZONE-SITE.XML_ozone.om.transport.class=org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory

env_file:
- ./fcq.conf
services:
om:
<<: *FCQ-config
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE_CONF_CONTAINER_ROOT_CA_ROTATION='{
"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
"OZONE-SITE.XML_hdds.x509.max.duration": "PT180S",
"OZONE-SITE.XML_hdds.x509.default.duration": "PT60S",
"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT45S",
"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S",
"OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval": "PT2s",
"OZONE-SITE.XML_hdds.block.token.expiry.time": "15s",
"OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime": "15s",
"OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval": "15s",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "60s",
"OZONE-SITE.XML_hdds.scmclient.failover.retry.interval": "1s",
"OZONE-SITE.XML_hdds.scmclient.failover.max.retry": "60",
"OZONE-SITE.XML_ozone.scm.info.wait.duration": "60s",
"OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout": "2s",
"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.HttpCrossOriginFilterInitializer",
"OZONE-SITE.XML_hdds.x509.ca.rotation.enabled": "true",
"OZONE-SITE.XML_hdds.x509.expired.certificate.check.interval": "PT30s"
}'
Original file line number Diff line number Diff line change
@@ -16,25 +16,8 @@

x-root-cert-rotation-config:
&root-cert-rotation-config
environment:
- OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
- OZONE-SITE.XML_hdds.x509.max.duration=PT180S
- OZONE-SITE.XML_hdds.x509.default.duration=PT60S
- OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S
- OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
- OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
- OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT2s
- OZONE-SITE.XML_hdds.block.token.expiry.time=15s
- OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s
- OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s
- OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=60s
- OZONE-SITE.XML_hdds.scmclient.failover.retry.interval=1s
- OZONE-SITE.XML_hdds.scmclient.failover.max.retry=60
- OZONE-SITE.XML_ozone.scm.info.wait.duration=60s
- OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout=2s
- OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.HttpCrossOriginFilterInitializer
- OZONE-SITE.XML_hdds.x509.ca.rotation.enabled=true
- OZONE-SITE.XML_hdds.x509.expired.certificate.check.interval=PT30s
env_file:
- ./root-ca-rotation.conf
services:
datanode:
<<: *root-cert-rotation-config
19 changes: 12 additions & 7 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/vault.conf
Original file line number Diff line number Diff line change
@@ -14,10 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.secret.s3.store.provider=org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address=http://vault:8200
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace="namespace"
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever=2
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath=secret
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth=TOKEN
OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token=00000000-0000-0000-0000-000000000000
OZONE_CONF_CONTAINER_VAULT='{
"OZONE-SITE.XML_ozone.secret.s3.store.provider": "org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address": "http://vault:8200",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace": "namespace",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever": "2",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath": "secret",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth": "TOKEN",
"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token": "00000000-0000-0000-0000-000000000000"
}'

OZONE_OPTS=-Dcom.sun.net.ssl.checkRevocation=false
OZONE_MANAGER_CLASSPATH=/opt/hadoop/share/ozone/lib/ozone-s3-secret-store-@project.version@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-@vault.driver.version@.jar
3 changes: 0 additions & 3 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/vault.yaml
Original file line number Diff line number Diff line change
@@ -18,9 +18,6 @@ services:
om:
env_file:
- vault.conf
environment:
- OZONE_OPTS=-Dcom.sun.net.ssl.checkRevocation=false
- OZONE_MANAGER_CLASSPATH=/opt/hadoop/share/ozone/lib/ozone-s3-secret-store-@project.version@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-@vault.driver.version@.jar
vault:
image: hashicorp/vault:1.13.2
hostname: vault
Original file line number Diff line number Diff line change
@@ -45,6 +45,7 @@ services:
- ${OZONE_VOLUME}/dn1:/data
- ../..:${OZONE_DIR}
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
dn2:
<<: *datanode
networks:
@@ -54,6 +55,7 @@ services:
- ${OZONE_VOLUME}/dn2:/data
- ../..:${OZONE_DIR}
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
dn3:
<<: *datanode
networks:
@@ -63,6 +65,7 @@ services:
- ${OZONE_VOLUME}/dn3:/data
- ../..:${OZONE_DIR}
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
om:
command: ["ozone","om"]
<<: *common-config
@@ -79,6 +82,7 @@ services:
- ${OZONE_VOLUME}/om:/data
- ../..:${OZONE_DIR}
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
recon:
command: ["ozone","recon"]
<<: *common-config
@@ -92,6 +96,7 @@ services:
volumes:
- ${OZONE_VOLUME}/recon:/data
- ../..:${OZONE_DIR}
- ../../libexec/envtoconf.py:/opt/envtoconf.py
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
s3g:
command: ["ozone","s3g"]
@@ -106,6 +111,7 @@ services:
volumes:
- ${OZONE_VOLUME}/s3g:/data
- ../..:${OZONE_DIR}
- ../../libexec/envtoconf.py:/opt/envtoconf.py
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
scm:
command: ["ozone","scm"]
@@ -122,6 +128,7 @@ services:
volumes:
- ${OZONE_VOLUME}/scm:/data
- ../..:${OZONE_DIR}
- ../../libexec/envtoconf.py:/opt/envtoconf.py
- ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py

networks:
38 changes: 20 additions & 18 deletions hadoop-ozone/dist/src/main/compose/restart/docker-config
Original file line number Diff line number Diff line change
@@ -14,24 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE_CONF_CONTAINER_RESTART='{
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
}'

OZONE_CONF_DIR=/etc/hadoop
OZONE_LOG_DIR=/var/log/hadoop
Original file line number Diff line number Diff line change
@@ -63,6 +63,7 @@ x-volumes:
- &keytabs ../../../_keytabs:/etc/security/keytabs
- &krb5conf ./krb5.conf:/etc/krb5.conf
- &ozone-dir ../../../..:${OZONE_DIR}
- &envtoconf ../../../../libexec/envtoconf.py:/opt/envtoconf.py
- &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py

services:
@@ -95,6 +96,7 @@ services:
- *krb5conf
- ../../../..:/opt/ozone
- *transformation
- *envtoconf
om1:
<<: *om
hostname: om1
@@ -107,6 +109,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
om2:
<<: *om
hostname: om2
@@ -119,6 +122,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
om3:
<<: *om
hostname: om3
@@ -131,6 +135,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf

scm1:
<<: *scm
@@ -147,6 +152,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
scm2:
<<: *scm
environment:
@@ -163,6 +169,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
scm3:
<<: *scm
environment:
@@ -179,7 +186,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation

- *envtoconf
dn1:
<<: *datanode
hostname: dn1
@@ -192,6 +199,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
dn2:
<<: *datanode
hostname: dn2
@@ -204,6 +212,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
dn3:
<<: *datanode
hostname: dn3
@@ -216,6 +225,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
dn4:
<<: *datanode
hostname: dn4
@@ -228,6 +238,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
dn5:
<<: *datanode
hostname: dn5
@@ -240,6 +251,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
recon:
command: ["ozone","recon"]
<<: *common-config
@@ -257,6 +269,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation
- *envtoconf
s3g:
command: ["ozone","s3g"]
<<: *common-config
@@ -274,7 +287,7 @@ services:
- *krb5conf
- *ozone-dir
- *transformation

- *envtoconf
networks:
net:
driver: bridge
66 changes: 37 additions & 29 deletions hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config
Original file line number Diff line number Diff line change
@@ -14,32 +14,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE_CONF_CONTAINER_MR='{
"OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata",

OZONE-SITE.XML_ozone.client.failover.max.attempts=6
"OZONE-SITE.XML_ozone.client.failover.max.attempts=6",

OZONE-SITE.XML_ozone.om.service.ids=omservice
OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
"OZONE-SITE.XML_ozone.om.service.ids=omservice",
"OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3",
"OZONE-SITE.XML_ozone.om.address.omservice.om1=om1",
"OZONE-SITE.XML_ozone.om.address.omservice.om2=om2",
"OZONE-SITE.XML_ozone.om.address.omservice.om3=om3",

OZONE-SITE.XML_ozone.scm.service.ids=scmservice
OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org
OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org
OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org
OZONE-SITE.XML_ozone.scm.ratis.enable=true
OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
"OZONE-SITE.XML_ozone.scm.service.ids=scmservice",
"OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org",
"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org",
"OZONE-SITE.XML_ozone.scm.ratis.enable=true",
"OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1",

OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
OZONE-SITE.XML_ozone.fs.hsync.enabled=true
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata",
"OZONE-SITE.XML_ozone.scm.container.size=1GB",
"OZONE-SITE.XML_hdds.datanode.dir=/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB",
"OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http",
"OZONE-SITE.XML_ozone.fs.hsync.enabled=true"
}'

# If SCM sends container close commands as part of upgrade finalization while
# datanodes are doing a leader election, all 3 replicas may end up in the
@@ -50,20 +52,26 @@ OZONE-SITE.XML_ozone.fs.hsync.enabled=true
# start the replication manager and pipeline scrubber. The default of 5 minutes
# is fine in real clusters to prevent unnecessary over-replication,
# but it is too long for this test.
OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=5s
OZONE_CONF_CONTAINER_MR_SAFEMODE='{
"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "5s"
}'
# If datanodes take too long to close pipelines during finalization, let the
# scrubber force close them to move the test forward.
OZONE-SITE.XML_ozone.scm.pipeline.scrub.interval=1m
OZONE-SITE.XML_ozone.scm.pipeline.allocated.timeout=2m
OZONE_CONF_CONTAINER_MR_PIPELINE='{
"OZONE-SITE.XML_ozone.scm.pipeline.scrub.interval": "1m",
"OZONE-SITE.XML_ozone.scm.pipeline.allocated.timeout": "2m",

OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_ozone.recon.address=recon:9891
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891"
}'

no_proxy=om1,om2,om3,scm1,scm2,scm3,s3g,kdc,localhost,127.0.0.1

OM_SERVICE_ID=omservice

# Explicitly enable filesystem snapshot feature for this Docker compose cluster
# Does not take effect on Ozone versions < 1.4.0
OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
OZONE_CONF_CONTAINER_MR_SNAPSHOT='{
"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
}'
218 changes: 110 additions & 108 deletions hadoop-ozone/dist/src/main/compose/xcompat/docker-config
Original file line number Diff line number Diff line change
@@ -14,114 +14,116 @@
# See the License for the specific language governing permissions and
# limitations under the License.

CORE-SITE.XML_fs.defaultFS=ofs://om
CORE-SITE.XML_fs.trash.interval=1
CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem

OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.http-address=scm:9876
OZONE-SITE.XML_ozone.recon.address=recon:9891
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
OZONE-SITE.XML_ozone.server.default.replication=3
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.scm.container.size=1GB
OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http

OZONE-SITE.XML_hdds.block.token.enabled=true
OZONE-SITE.XML_hdds.container.token.enabled=true
OZONE-SITE.XML_hdds.grpc.tls.enabled=true

OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_ozone.acl.enabled=true
OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om"
OZONE-SITE.XML_ozone.recon.administrators="testuser2"
OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"

HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms

OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM

OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM

OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer

OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos

OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=*
OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab

CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
CORE-SITE.XML_hadoop.http.authentication.type=kerberos
CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

CORE-SITE.XML_hadoop.security.authorization=true
HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=*

KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
OZONE_CONF_CONTAINER_MR='{
"CORE-SITE.XML_fs.defaultFS": "ofs://om",
"CORE-SITE.XML_fs.trash.interval": "1",
"CORE-SITE.XML_fs.ofs.impl": "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem",

"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
"OZONE-SITE.XML_hdds.scm.safemode.min.datanode": "3",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
"OZONE-SITE.XML_ozone.server.default.replication": "3",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
"OZONE-SITE.XML_recon.om.snapshot.task.interval.delay": "1m",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
"OZONE-SITE.XML_ozone.default.bucket.layout": "LEGACY",
"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",

"OZONE-SITE.XML_hdds.block.token.enabled": "true",
"OZONE-SITE.XML_hdds.container.token.enabled": "true",
"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",

"OZONE-SITE.XML_ozone.security.enabled": "true",
"OZONE-SITE.XML_ozone.acl.enabled": "true",
"OZONE-SITE.XML_ozone.acl.authorizer.class": "org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer",
"OZONE-SITE.XML_ozone.administrators": "testuser,recon,om",
"OZONE-SITE.XML_ozone.s3.administrators": "testuser,recon,om",
"OZONE-SITE.XML_ozone.recon.administrators": "testuser2",
"OZONE-SITE.XML_ozone.s3.administrators": "testuser,s3g",

"HDFS-SITE.XML_dfs.datanode.address": "0.0.0.0:1019",
"HDFS-SITE.XML_dfs.datanode.http.address": "0.0.0.0:1012",
"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms",

"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
"OZONE-SITE.XML_ozone.recon.kerberos.keytab.file": "/etc/security/keytabs/recon.keytab",
"OZONE-SITE.XML_ozone.recon.kerberos.principal": "recon/recon@EXAMPLE.COM",

"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",

"OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file": "/etc/security/keytabs/httpfs.keytab",
"OZONE-SITE.XML_ozone.httpfs.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",

"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.datanode.keytab.file": "/etc/security/keytabs/dn.keytab",
"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",

"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",
"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.AuthenticationFilterInitializer",

"OZONE-SITE.XML_ozone.om.http.auth.type": "kerberos",
"OZONE-SITE.XML_hdds.scm.http.auth.type": "kerberos",
"OZONE-SITE.XML_hdds.datanode.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.s3g.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.s3g.secret.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.httpfs.http.auth.type": "kerberos",
"OZONE-SITE.XML_ozone.recon.http.auth.type": "kerberos",

"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/scm.keytab",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/om.keytab",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/dn.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/s3g.keytab",
"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal": "*",
"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",

"CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed": "false",
"CORE-SITE.XML_hadoop.http.authentication.signature.secret.file": "/etc/security/http_secret",
"CORE-SITE.XML_hadoop.http.authentication.type": "kerberos",
"CORE-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
"CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",

"CORE-SITE.XML_hadoop.security.authorization": "true",
"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",
"HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl": "*",

"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users": "*",
"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups": "*",
"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts": "*"
}'

OZONE_DATANODE_SECURE_USER=root
JSVC_HOME=/usr/bin
Original file line number Diff line number Diff line change
@@ -47,6 +47,7 @@ services:
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
volumes:
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
command: [ "hadoop", "kms" ]
datanode:
<<: *new-config
Original file line number Diff line number Diff line change
@@ -47,6 +47,7 @@ services:
HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
volumes:
- ../../libexec/transformation.py:/opt/transformation.py
- ../../libexec/envtoconf.py:/opt/envtoconf.py
command: [ "hadoop", "kms" ]
datanode:
<<: *old-config
73 changes: 43 additions & 30 deletions hadoop-ozone/dist/src/main/dockerlibexec/envtoconf.py
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@
#

"""convert environment variables to config"""

import json
import os
import re

@@ -38,6 +38,7 @@ def __init__(self, args):
self.output_dir = self.args.destination
self.excluded_envs = ['HADOOP_CONF_DIR']
self.configurables = {}
self.conf_container_prefix = 'OZONE_CONF_CONTAINER'

def destination_file_path(self, name, extension):
"""destination file path"""
@@ -51,36 +52,48 @@ def write_env_var(self, name, extension, key, value):
def process_envs(self):
"""Process environment variables"""
for key in os.environ.keys():
if key in self.excluded_envs:
continue
pattern = re.compile("[_\\.]")
parts = pattern.split(key)
extension = None
name = parts[0].lower()
if len(parts) > 1:
extension = parts[1].lower()
config_key = key[len(name) + len(extension) + 2:].strip()
if extension and "!" in extension:
splitted = extension.split("!")
extension = splitted[0]
fmt = splitted[1]
config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
else:
fmt = extension

if extension and extension in self.known_formats:
if name not in self.configurables.keys():
with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
myfile.write("")
self.configurables[name] = (extension, fmt)
self.write_env_var(name, extension, config_key, os.environ[key])
if key.startswith(self.conf_container_prefix):
conf_variables=json.loads(os.environ[key])
for conf_key in conf_variables:
self.process_env_var(conf_key, conf_variables[conf_key])
else:
for configurable_name in self.configurables:
if key.lower().startswith(configurable_name.lower()):
self.write_env_var(configurable_name,
self.configurables[configurable_name],
key[len(configurable_name) + 1:],
os.environ[key])
self.process_env_var(key, os.environ[key])

def process_env_var(self, env_key, env_value):
"""Process environment variable"""
if env_key in self.excluded_envs:
return

pattern = re.compile("[_\\.]")
parts = pattern.split(env_key)
extension = None
name = parts[0].lower()
config_key = ''

if len(parts) > 1:
extension = parts[1].lower()
config_key = env_key[len(name) + len(extension) + 2:].strip()
if extension and "!" in extension:
splitted = extension.split("!")
extension = splitted[0]
fmt = splitted[1]
config_key = env_key[len(name) + len(extension) + len(fmt) + 3:].strip()
else:
fmt = extension

if extension and extension.lower() in self.known_formats:
if name not in self.configurables.keys():
with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
myfile.write("")
self.configurables[name] = (extension, fmt)
self.write_env_var(name, extension, config_key, env_value)
else:
for configurable_name in self.configurables:
if env_key.lower().startswith(configurable_name.lower()):
self.write_env_var(configurable_name,
self.configurables[configurable_name],
env_key[len(configurable_name) + 1:],
env_value)

def transform(self):
"""transform"""
Original file line number Diff line number Diff line change
@@ -14,64 +14,67 @@
# See the License for the specific language governing permissions and
# limitations under the License.

OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_ozone.scm.dead.node.interval=5m
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
OZONE_CONF_CONTAINER_NETWORK_TEST='{
"OZONE-SITE.XML_ozone.om.address": "om",
"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
"OZONE-SITE.XML_ozone.scm.names": "scm",
"OZONE-SITE.XML_ozone.scm.datanode.id": "/data/datanode.id",
"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
"OZONE-SITE.XML_ozone.handler.type": "distributed",
"OZONE-SITE.XML_ozone.scm.client.address": "scm",
"OZONE-SITE.XML_ozone.scm.dead.node.interval": "5m",
"OZONE-SITE.XML_ozone.replication": "1",
"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",

"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300",

"LOG4J.PROPERTIES_log4j.rootLogger": "INFO, stdout",
"LOG4J.PROPERTIES_log4j.appender.stdout": "org.apache.log4j.ConsoleAppender",
"LOG4J.PROPERTIES_log4j.appender.stdout.layout": "org.apache.log4j.PatternLayout",
"LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern": "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n",
"LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader": "ERROR",
"LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils": "WARN",
"LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping": "ERROR",

"LOG4J2.PROPERTIES_monitorInterval": "30",
"LOG4J2.PROPERTIES_filter": "read,write",
"LOG4J2.PROPERTIES_filter.read.type": "MarkerFilter",
"LOG4J2.PROPERTIES_filter.read.marker": "READ",
"LOG4J2.PROPERTIES_filter.read.onMatch": "DENY",
"LOG4J2.PROPERTIES_filter.read.onMismatch": "NEUTRAL",
"LOG4J2.PROPERTIES_filter.write.type": "MarkerFilter",
"LOG4J2.PROPERTIES_filter.write.marker": "WRITE",
"LOG4J2.PROPERTIES_filter.write.onMatch": "NEUTRAL",
"LOG4J2.PROPERTIES_filter.write.onMismatch": "NEUTRAL",
"LOG4J2.PROPERTIES_appenders": "console, rolling",
"LOG4J2.PROPERTIES_appender.console.type": "Console",
"LOG4J2.PROPERTIES_appender.console.name": "STDOUT",
"LOG4J2.PROPERTIES_appender.console.layout.type": "PatternLayout",
"LOG4J2.PROPERTIES_appender.console.layout.pattern": "%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n",
"LOG4J2.PROPERTIES_appender.rolling.type": "RollingFile",
"LOG4J2.PROPERTIES_appender.rolling.name": "RollingFile",
"LOG4J2.PROPERTIES_appender.rolling.fileName": "${sys:hadoop.log.dir}/om-audit-${hostName}.log",
"LOG4J2.PROPERTIES_appender.rolling.filePattern": "${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz",
"LOG4J2.PROPERTIES_appender.rolling.layout.type": "PatternLayout",
"LOG4J2.PROPERTIES_appender.rolling.layout.pattern": "%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n",
"LOG4J2.PROPERTIES_appender.rolling.policies.type": "Policies",
"LOG4J2.PROPERTIES_appender.rolling.policies.time.type": "TimeBasedTriggeringPolicy",
"LOG4J2.PROPERTIES_appender.rolling.policies.time.interval": "86400",
"LOG4J2.PROPERTIES_appender.rolling.policies.size.type": "SizeBasedTriggeringPolicy",
"LOG4J2.PROPERTIES_appender.rolling.policies.size.size": "64MB",
"LOG4J2.PROPERTIES_loggers": "audit",
"LOG4J2.PROPERTIES_logger.audit.type": "AsyncLogger",
"LOG4J2.PROPERTIES_logger.audit.name": "OMAudit",
"LOG4J2.PROPERTIES_logger.audit.level": "INFO",
"LOG4J2.PROPERTIES_logger.audit.appenderRefs": "rolling",
"LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref": "RollingFile",
"LOG4J2.PROPERTIES_rootLogger.level": "INFO",
"LOG4J2.PROPERTIES_rootLogger.appenderRefs": "stdout",
"LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref": "STDOUT"
}'

#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm

#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
LOG4J2.PROPERTIES_monitorInterval=30
LOG4J2.PROPERTIES_filter=read,write
LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
LOG4J2.PROPERTIES_filter.read.marker=READ
LOG4J2.PROPERTIES_filter.read.onMatch=DENY
LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
LOG4J2.PROPERTIES_filter.write.marker=WRITE
LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
LOG4J2.PROPERTIES_appenders=console, rolling
LOG4J2.PROPERTIES_appender.console.type=Console
LOG4J2.PROPERTIES_appender.console.name=STDOUT
LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
LOG4J2.PROPERTIES_loggers=audit
LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
LOG4J2.PROPERTIES_logger.audit.name=OMAudit
LOG4J2.PROPERTIES_logger.audit.level=INFO
LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
LOG4J2.PROPERTIES_rootLogger.level=INFO
LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT