HDDS-11590. Use JSON to pass config for docker

hadoop-ozone/dist/src/main/compose/common/hadoop-secure.yaml

@@ -22,6 +22,7 @@ services:
     volumes:
       - ../..:/opt/ozone
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
       - ../_keytabs:/etc/security/keytabs
       - ./krb5.conf:/etc/krb5.conf
     ports:
@@ -41,6 +42,7 @@ services:
     volumes:
       - ../..:/opt/ozone
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
       - ../_keytabs:/etc/security/keytabs
       - ./krb5.conf:/etc/krb5.conf
     env_file:
@@ -63,6 +65,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
       - ./krb5.conf:/etc/krb5.conf
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     ports:
       - 8188:8188
     env_file:

hadoop-ozone/dist/src/main/compose/common/hadoop-security.conf

@@ -14,15 +14,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-YARN-SITE.XML_yarn.nodemanager.principal=nm/[email protected]
-YARN-SITE.XML_yarn.nodemanager.keytab=/etc/security/keytabs/nm.keytab
-YARN-SITE.XML_yarn.resourcemanager.keytab=/etc/security/keytabs/rm.keytab
-YARN-SITE.XML_yarn.resourcemanager.principal=rm/[email protected]
-YARN-SITE.XML_yarn.timeline-service.principal=jhs/[email protected]
-YARN-SITE.XML_yarn.timeline-service.keytab=/etc/security/keytabs/jhs.keytab
-HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
-HADOOP-POLICY.XML_org.apache.hadoop.yarn.server.api.ResourceTracker.acl=*
+OZONE_CONF_CONTAINER_HADOOP_SECURITY='{
+"YARN-SITE.XML_yarn.nodemanager.principal": "nm/[email protected]",
+"YARN-SITE.XML_yarn.nodemanager.keytab": "/etc/security/keytabs/nm.keytab",
+"YARN-SITE.XML_yarn.resourcemanager.keytab": "/etc/security/keytabs/rm.keytab",
+"YARN-SITE.XML_yarn.resourcemanager.principal": "rm/[email protected]",
+"YARN-SITE.XML_yarn.timeline-service.principal": "jhs/[email protected]",
+"YARN-SITE.XML_yarn.timeline-service.keytab": "/etc/security/keytabs/jhs.keytab",
+"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*","
+"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",
+"HADOOP-POLICY.XML_org.apache.hadoop.yarn.server.api.ResourceTracker.acl": "*"
+}'

hadoop-ozone/dist/src/main/compose/common/hadoop.conf

@@ -14,48 +14,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
-
-MAPRED-SITE.XML_mapreduce.framework.name=yarn
-MAPRED-SITE.XML_mapreduce.map.memory.mb=4096
-MAPRED-SITE.XML_mapreduce.reduce.memory.mb=4096
-MAPRED-SITE.XML_mapred.child.java.opts=-Xmx2g
-
-YARN-SITE.XML_yarn.app.mapreduce.am.staging-dir=/user
-YARN_SITE.XML_yarn.timeline-service.enabled=true
-YARN_SITE.XML_yarn.timeline-service.generic.application.history.enabled=true
-YARN_SITE.XML_yarn.timeline-service.hostname=jhs
-YARN_SITE.XML_yarn.log.server.url=http://jhs:8188/applicationhistory/logs/
-
-YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
-YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=6000
-YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
-YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
-YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable=false
-
-YARN-SITE.XML_yarn.resourcemanager.hostname=rm
-YARN_SITE_XML_yarn.resourcemanager.system.metrics.publisher.enabled=true
-
 #YARN-SITE.XML_yarn.log-aggregation-enable=true
 #YARN-SITE.XML_yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds=3600
 
 #YARN-SITE.yarn.nodemanager.container-executor.class=org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor
 #YARN-SITE.XML_yarn.nodemanager.linux-container-executor.path=/opt/hadoop/bin/container-executor
 #YARN-SITE.XML_yarn.nodemanager.linux-container-executor.group=hadoop
-YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99
-YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable=false
-
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
-CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false
+
+OZONE_CONF_CONTAINER_HADOOP='{
+"CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl": "org.apache.hadoop.fs.ozone.OzFs",
+"CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl": "org.apache.hadoop.fs.ozone.RootedOzFs",
+
+"MAPRED-SITE.XML_mapreduce.framework.name": "yarn",
+"MAPRED-SITE.XML_mapreduce.map.memory.mb": "4096",
+"MAPRED-SITE.XML_mapreduce.reduce.memory.mb": "4096",
+"MAPRED-SITE.XML_mapred.child.java.opts": "-Xmx2g",
+
+"YARN-SITE.XML_yarn.app.mapreduce.am.staging-dir": "/user",
+"YARN_SITE.XML_yarn.timeline-service.enabled": "true",
+"YARN_SITE.XML_yarn.timeline-service.generic.application.history.enabled": "true",
+"YARN_SITE.XML_yarn.timeline-service.hostname": "jhs",
+"YARN_SITE.XML_yarn.log.server.url": "http://jhs:8188/applicationhistory/logs/",
+
+"YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled": "false",
+"YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec": "6000",
+"YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled": "false",
+"YARN-SITE.XML_yarn.nodemanager.aux-services": "mapreduce_shuffle",
+"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable": "false",
+
+"YARN-SITE.XML_yarn.resourcemanager.hostname": "rm",
+"YARN_SITE_XML_yarn.resourcemanager.system.metrics.publisher.enabled": "true",
+
+"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage": "99",
+"YARN-SITE.XML_yarn.nodemanager.disk-health-checker.enable": "false",
+
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications": "10000",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent": "0.1",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues": "default",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity": "100",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor": "1",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity": "100",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state": "RUNNING",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications": "*",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue": "*",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay": "40",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings": "",
+"CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable": "false"
+}'

hadoop-ozone/dist/src/main/compose/common/hadoop.yaml

@@ -21,6 +21,7 @@ services:
     volumes:
       - ../..:/opt/ozone
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     ports:
       - 8088:8088
     env_file:
@@ -36,6 +37,7 @@ services:
     volumes:
       - ../..:/opt/ozone
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     env_file:
       - docker-config
       - ../common/hadoop.conf

hadoop-ozone/dist/src/main/compose/common/hadoop2.conf

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
+OZONE_CONF_CONTAINER_HADOOP2='{
+"MAPRED-SITE.XML_mapreduce.application.classpath": "/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar"
+}'
 
 HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar

hadoop-ozone/dist/src/main/compose/common/hadoop3.conf

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
+OZONE_CONF_CONTAINER_HADOOP3='{
+"MAPRED-SITE.XML_mapreduce.application.classpath": "/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar"
+}'
 
 HADOOP_CLASSPATH=/opt/ozone/share/ozone/lib/[email protected]@.jar

hadoop-ozone/dist/src/main/compose/common/security.conf

@@ -15,91 +15,95 @@
 # limitations under the License.
 
 # For HttpFS service it is required to enable proxying users.
-CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts=*
-CORE-SITE.XML_hadoop.proxyuser.httpfs.groups=*
-
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
-CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
-
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
-
-OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
-
-OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
-
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-OZONE-SITE.XML_hdds.block.token.enabled=true
-OZONE-SITE.XML_hdds.container.token.enabled=true
-OZONE-SITE.XML_hdds.grpc.tls.enabled=true
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_ozone.acl.enabled=true
-OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
-OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"
-OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
-OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
-OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
-
-OZONE-SITE.XML_hdds.secret.key.rotate.duration=5m
-OZONE-SITE.XML_hdds.secret.key.rotate.check.duration=1m
-OZONE-SITE.XML_hdds.secret.key.expiry.duration=1h
-
-OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
-
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/db@EXAMPLE.COM
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=HTTP/recon@EXAMPLE.COM
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
-
-CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
-CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
-CORE-SITE.XML_hadoop.http.authentication.type=kerberos
-CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
-CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-CORE-SITE.XML_hadoop.security.authorization=true
-HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
-
-HTTPFS-SITE.XML_hadoop.http.authentication.type=kerberos
-HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.type=kerberos
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
+OZONE_CONF_CONTAINER_SECURITY='{
+"CORE-SITE.XML_hadoop.proxyuser.httpfs.hosts": "*",
+"CORE-SITE.XML_hadoop.proxyuser.httpfs.groups": "*",
+
+"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
+"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
+"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
+"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms",
+
+"CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed": "false",
+"CORE-SITE.XML_hadoop.http.authentication.signature.secret.file": "/etc/security/http_secret",
+"CORE-SITE.XML_hadoop.http.authentication.type": "kerberos",
+"CORE-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
+"CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+
+"CORE-SITE.XML_hadoop.security.authorization": "true",
+
+"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
+"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
+"OZONE-SITE.XML_ozone.recon.kerberos.keytab.file": "/etc/security/keytabs/recon.keytab",
+"OZONE-SITE.XML_ozone.recon.kerberos.principal": "recon/recon@EXAMPLE.COM",
+
+"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
+"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",
+
+"OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file": "/etc/security/keytabs/httpfs.keytab",
+"OZONE-SITE.XML_ozone.httpfs.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",
+
+"OZONE-SITE.XML_hdds.block.token.enabled": "true",
+"OZONE-SITE.XML_hdds.container.token.enabled": "true",
+"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",
+"OZONE-SITE.XML_ozone.security.enabled": "true",
+"OZONE-SITE.XML_ozone.acl.enabled": "true",
+"OZONE-SITE.XML_ozone.acl.authorizer.class": "org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer",
+"OZONE-SITE.XML_ozone.administrators": "testuser,recon,om",
+"OZONE-SITE.XML_ozone.s3.administrators": "testuser,s3g",
+"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
+"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",
+"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.AuthenticationFilterInitializer",
+
+"OZONE-SITE.XML_hdds.secret.key.rotate.duration": "5m",
+"OZONE-SITE.XML_hdds.secret.key.rotate.check.duration": "1m",
+"OZONE-SITE.XML_hdds.secret.key.expiry.duration": "1h",
+
+"OZONE-SITE.XML_ozone.om.http.auth.type": "kerberos",
+"OZONE-SITE.XML_hdds.scm.http.auth.type": "kerberos",
+"OZONE-SITE.XML_hdds.datanode.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.s3g.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.s3g.secret.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.recon.http.auth.type": "kerberos",
+
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/db@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal": "HTTP/recon@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",
+"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",
+
+"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+
+"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",
+
+"HTTPFS-SITE.XML_hadoop.http.authentication.type": "kerberos",
+"HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
+"HTTPFS-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
+"HTTPFS-SITE.XML_httpfs.hadoop.authentication.type": "kerberos",
+"HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
+"HTTPFS-SITE.XML_httpfs.hadoop.authentication.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",
+
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users": "*",
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups": "*",
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts": "*",
+}'
 
 OZONE_DATANODE_SECURE_USER=root

hadoop-ozone/dist/src/main/compose/compatibility/docker-config

@@ -14,24 +14,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-OZONE-SITE.XML_ozone.om.features.disabled=ATOMIC_REWRITE_KEY
+OZONE_CONF_CONTAINER_COMPATIBILITY='{
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
+"OZONE-SITE.XML_ozone.om.features.disabled": "ATOMIC_REWRITE_KEY"
+}'
 
 HADOOP_OPTS="-Dhadoop.opts=test"
 HDFS_STORAGECONTAINERMANAGER_OPTS="-Dhdfs.scm.opts=test"

hadoop-ozone/dist/src/main/compose/ozone-balancer/docker-config

@@ -15,46 +15,49 @@
 # limitations under the License.
 
 # For HttpFS service it is required to enable proxying users.
-CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
-CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
+OZONE_CONF_CONTAINER_BALANCER='{
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",
 
-CORE-SITE.XML_fs.defaultFS=ofs://om/
-CORE-SITE.XML_fs.trash.interval=1
+"CORE-SITE.XML_fs.defaultFS": "ofs://om/",
+"CORE-SITE.XML_fs.trash.interval": "1",
 
-OZONE-SITE.XML_ozone.om.service.ids=om
-OZONE-SITE.XML_ozone.om.nodes.om=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.om.om1=om1
-OZONE-SITE.XML_ozone.om.address.om.om2=om2
-OZONE-SITE.XML_ozone.om.address.om.om3=om3
+"OZONE-SITE.XML_ozone.om.service.ids": "om",
+"OZONE-SITE.XML_ozone.om.nodes.om": "om1,om2,om3",
+"OZONE-SITE.XML_ozone.om.address.om.om1": "om1",
+"OZONE-SITE.XML_ozone.om.address.om.om2": "om2",
+"OZONE-SITE.XML_ozone.om.address.om.om3": "om3",
+
+"OZONE-SITE.XML_ozone.scm.service.ids": "scmservice",
+"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1": "scm1",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2": "scm2",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3": "scm3",
+"OZONE-SITE.XML_ozone.scm.ratis.enable": "true",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.container.size": "100MB",
+"OZONE-SITE.XML_ozone.scm.block.size": "20MB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_hdds.node.report.interval": "20s",
+"OZONE-SITE.XML_hdds.heartbeat.interval": "20s",
+"OZONE-SITE.XML_hdds.datanode.du.refresh.period": "20s",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.container.db.dir": "/data/metadata",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one": "false",
+"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.scm.primordial.node.id": "scm1",
+"OZONE-SITE.XML_hdds.container.report.interval": "30s",
+"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_dfs.container.ratis.datastream.enabled": "true",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
+"OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval": "25s",
+"OZONE-SITE.XML_hdds.container.balancer.trigger.du.before.move.enable": "false"
+}'
 
-OZONE-SITE.XML_ozone.scm.service.ids=scmservice
-OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3
-OZONE-SITE.XML_ozone.scm.ratis.enable=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.container.size=100MB
-OZONE-SITE.XML_ozone.scm.block.size=20MB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_hdds.node.report.interval=20s
-OZONE-SITE.XML_hdds.heartbeat.interval=20s
-OZONE-SITE.XML_hdds.datanode.du.refresh.period=20s
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.container.db.dir=/data/metadata
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.auto.factor.one=false
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
-OZONE-SITE.XML_hdds.container.report.interval=30s
-OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_dfs.container.ratis.datastream.enabled=true
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-OZONE-SITE.XML_hdds.container.balancer.balancing.iteration.interval=25s
-OZONE-SITE.XML_hdds.container.balancer.trigger.du.before.move.enable=false
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop
 

hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config

@@ -14,28 +14,30 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://om
+OZONE_CONF_CONTAINER_BALANCER='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://om",
 
-OZONE-SITE.XML_ozone.csi.owner=hadoop
-OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
+"OZONE-SITE.XML_ozone.csi.owner": "hadoop",
+"OZONE-SITE.XML_ozone.csi.socket": "/tmp/csi.sock",
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
+}'
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config

@@ -15,41 +15,48 @@
 # limitations under the License.
 
 # For HttpFS service it is required to enable proxying users.
-CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
-CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
 
-CORE-SITE.XML_fs.defaultFS=ofs://omservice/
-CORE-SITE.XML_fs.trash.interval=1
+OZONE_CONF_CONTAINER_HA='{
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",
 
-OZONE-SITE.XML_ozone.om.service.ids=omservice
-OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
-OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
-OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
+"CORE-SITE.XML_fs.defaultFS": "ofs://omservice/",
+"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3"
+"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1"
+"OZONE-SITE.XML_ozone.om.address.omservice.om2=": "om2"
+"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3"
 
-OZONE-SITE.XML_ozone.scm.service.ids=scmservice
-OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3
-OZONE-SITE.XML_ozone.scm.ratis.enable=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
-OZONE-SITE.XML_hdds.container.report.interval=60s
-OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
-OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
-OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=true
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
+"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
+"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
+"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
+"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",
+"OZONE-SITE.XML_ozone.om.ratis.enable": "true",
+
+"OZONE-SITE.XML_ozone.scm.service.ids": "scmservice",
+"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1": "scm1",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2": "scm2",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3": "scm3",
+"OZONE-SITE.XML_ozone.scm.ratis.enable": "true",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.scm.primordial.node.id": "scm1",
+"OZONE-SITE.XML_hdds.container.report.interval": "60s",
+"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
+"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
+"OZONE-SITE.XML_hdds.container.ratis.datastream.enabled": "true",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
+}'
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config

@@ -14,33 +14,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://omservice
+OZONE_CONF_CONTAINER_OM_HA='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://omservice",
+
+"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
+"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
+"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
+"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
+"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.handler.type": "distributed",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.server.default.replication": "1",
+"OZONE-SITE.XML_ozone.client.failover.max.attempts": "6",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_hdds.profiler.endpoint.enabled": "true",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_hdds.container.report.interval": "60s",
+"OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled": "true",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
+"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
+"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
+}'
 
-OZONE-SITE.XML_ozone.om.service.ids=omservice
-OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
-OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
-OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.server.default.replication=1
-OZONE-SITE.XML_ozone.client.failover.max.attempts=6
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_hdds.container.report.interval=60s
-OZONE-SITE.XML_ozone.om.s3.grpc.server_enabled=true
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 ASYNC_PROFILER_HOME=/opt/profiler
+
 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
 LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -96,4 +100,6 @@ OZONE_LOG_DIR=/var/log/hadoop
 no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
 
 # Explicitly enable filesystem snapshot feature for this Docker compose cluster
-OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
+OZONE_CONF_CONTAINER_OM_HA_SNAPSHOT='{
+"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
+}'

hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-config

@@ -14,23 +14,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://omservice/
+OZONE_CONF_CONTAINER_OM_HA='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://omservice/",
 
-OZONE-SITE.XML_ozone.om.service.ids=omservice
-OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
-OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
-OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
+"OZONE-SITE.XML_ozone.om.service.ids": "omservice",
+"OZONE-SITE.XML_ozone.om.nodes.omservice": "om1,om2,om3",
+"OZONE-SITE.XML_ozone.om.address.omservice.om1": "om1",
+"OZONE-SITE.XML_ozone.om.address.omservice.om2": "om2",
+"OZONE-SITE.XML_ozone.om.address.omservice.om3": "om3",
 
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.client.failover.max.attempts=6
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.client.failover.max.attempts": "6",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
+}'
 
 no_proxy=om1,om2,om3,scm,s3g,recon,kdc,localhost,127.0.0.1

hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config

@@ -14,39 +14,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://om
+OZONE_CONF_CONTAINER_TOPOLOGY='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://om",
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.container.size=256MB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.ozone.scm.block.size=64MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.server.default.replication=3
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
-OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
-OZONE-SITE.XML_ozone.scm.container.placement.impl=org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware
-OZONE-SITE.XML_net.topology.node.switch.mapping.impl=org.apache.hadoop.net.TableMapping
-OZONE-SITE.XML_net.topology.table.file.name=/opt/hadoop/compose/ozone-topology/network-config
-OZONE-SITE.XML_ozone.network.topology.aware.read=true
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.container.size": "256MB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.ozone.scm.block.size": "64MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.handler.type": "distributed",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.server.default.replication": "3",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
+"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
+"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
+"OZONE-SITE.XML_hdds.profiler.endpoint.enabled": "true",
+"OZONE-SITE.XML_ozone.scm.container.placement.impl": "org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackAware",
+"OZONE-SITE.XML_net.topology.node.switch.mapping.impl": "org.apache.hadoop.net.TableMapping",
+"OZONE-SITE.XML_net.topology.table.file.name": "/opt/hadoop/compose/ozone-topology/network-config",
+"OZONE-SITE.XML_ozone.network.topology.aware.read": "true",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
+
+"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
+"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
+}'
 
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 ASYNC_PROFILER_HOME=/opt/profiler
 OZONE_DATANODE_OPTS=-Dmodule.name=datanode
 OZONE_MANAGER_OPTS=-Dmodule.name=om

hadoop-ozone/dist/src/main/compose/ozone/docker-config

@@ -14,53 +14,54 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://om
-CORE-SITE.XML_fs.trash.interval=1
-# For HttpFS service it is required to enable proxying users.
-CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts=*
-CORE-SITE.XML_hadoop.proxyuser.hadoop.groups=*
+OZONE_CONF_CONTAINER_BASIC='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://om",
+"CORE-SITE.XML_fs.trash.interval": "1",
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.block.size=1MB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.http-address=0.0.0.0:9888
-OZONE-SITE.XML_ozone.recon.https-address=0.0.0.0:9889
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_hdds.container.report.interval=60s
-OZONE-SITE.XML_ozone.scm.stale.node.interval=30s
-OZONE-SITE.XML_ozone.scm.dead.node.interval=45s
-OZONE-SITE.XML_hdds.heartbeat.interval=5s
-OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s
-OZONE-SITE.XML_hdds.scm.replication.thread.interval=15s
-OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval=5s
-OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval=5s
-OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.hosts": "*",
+"CORE-SITE.XML_hadoop.proxyuser.hadoop.groups": "*",
 
-OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=true
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.block.size": "1MB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_ozone.recon.http-address": "0.0.0.0:9888",
+"OZONE-SITE.XML_ozone.recon.https-address": "0.0.0.0:9889",
+"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
+"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_hdds.container.report.interval": "60s",
+"OZONE-SITE.XML_ozone.scm.stale.node.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.dead.node.interval": "45s",
+"OZONE-SITE.XML_hdds.heartbeat.interval": "5s",
+"OZONE-SITE.XML_ozone.scm.close.container.wait.duration": "5s",
+"OZONE-SITE.XML_hdds.scm.replication.thread.interval": "15s",
+"OZONE-SITE.XML_hdds.scm.replication.under.replicated.interval": "5s",
+"OZONE-SITE.XML_hdds.scm.replication.over.replicated.interval": "5s",
+"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
 
-OZONE-SITE.XML_ozone.fs.hsync.enabled=true
+"OZONE-SITE.XML_hdds.container.ratis.datastream.enabled": "true",
+
+"OZONE-SITE.XML_ozone.fs.hsync.enabled": "true",
+
+"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
+}'
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop
 
 no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
-
-# Explicitly enable filesystem snapshot feature for this Docker compose cluster
-OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true

hadoop-ozone/dist/src/main/compose/ozone/freon-ockg.yaml

@@ -23,5 +23,5 @@ services:
       - docker-config
       - monitoring.conf
     environment:
-      - "OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1}"
+      - "OZONE_CONF_CONTAINER_FREON_OCKG='{\"OZONE-SITE.XML_ozone.server.default.replication\": \"${OZONE_REPLICATION_FACTOR:-1}\"}'"
     command: ["ozone","freon","ockg","-n100000"]

hadoop-ozone/dist/src/main/compose/ozone/freon-rk.yaml

@@ -23,5 +23,5 @@ services:
       - docker-config
       - monitoring.conf
     environment:
-      - "OZONE-SITE.XML_ozone.server.default.replication=${OZONE_REPLICATION_FACTOR:-1}"
+      - "OZONE_CONF_CONTAINER_FREON_RK='{\"OZONE-SITE.XML_ozone.server.default.replication\": \"${OZONE_REPLICATION_FACTOR:-1}\"}'"
     command: ["ozone","freon","rk"]

hadoop-ozone/dist/src/main/compose/ozone/monitoring.conf

@@ -14,11 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
-OZONE-SITE.XML_hdds.tracing.enabled=true
-OZONE-SITE.XML_ozone.metastore.rocksdb.statistics=ALL
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+OZONE_CONF_CONTAINER_MONITORING='{
+"OZONE-SITE.XML_hdds.prometheus.endpoint.enabled": "true",
+"OZONE-SITE.XML_hdds.tracing.enabled": "true",
+"OZONE-SITE.XML_ozone.metastore.rocksdb.statistics": "ALL",
+"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
+"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
+}'
 JAEGER_SAMPLER_PARAM=1
 JAEGER_SAMPLER_TYPE=const
 JAEGER_AGENT_HOST=jaeger

hadoop-ozone/dist/src/main/compose/ozone/profiling.yaml

@@ -18,7 +18,7 @@ x-profiling-config:
   &profiling-config
   privileged: true
   environment:
-    - OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+    - OZONE_CONF_CONTAINER_PROFILING="{\"OZONE-SITE.XML_hdds.profiler.endpoint.enabled\": true}"
     - ASYNC_PROFILER_HOME=/opt/profiler
 
 services:

hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config

@@ -14,35 +14,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://om
+OZONE_CONF_CONTAINER_BLOCKADE='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://om
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.client.max.retries=10
-OZONE-SITE.XML_ozone.scm.stale.node.interval=2m
-OZONE-SITE.XML_ozone.scm.dead.node.interval=5m
-OZONE-SITE.XML_ozone.server.default.replication=1
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout=15s
-OZONE-SITE.XML_hdds.heartbeat.interval=2s
-OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
-OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s
-OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
-OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s
-OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.handler.type": "distributed",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.client.max.retries": "10",
+"OZONE-SITE.XML_ozone.scm.stale.node.interval": "2m",
+"OZONE-SITE.XML_ozone.scm.dead.node.interval": "5m",
+"OZONE-SITE.XML_ozone.server.default.replication": "1",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout": "15s",
+"OZONE-SITE.XML_hdds.heartbeat.interval": "2s",
+"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "30s",
+"OZONE-SITE.XML_hdds.scm.replication.thread.interval": "6s",
+"OZONE-SITE.XML_hdds.scm.replication.event.timeout": "10s",
+"OZONE-SITE.XML_dfs.ratis.server.failure.duration": "35s",
+"OZONE-SITE.XML_hdds.scm.safemode.min.datanode": "3",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
 
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
+"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300"
+}'
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm

hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config

@@ -14,20 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
-OZONE-SITE.XML_ozone.ksm.address=ksm
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.server.default.replication=1
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE_CONF_CONTAINER_OZONE_SCRIPT='{
+"CORE-SITE.XML_fs.defaultFS": "hdfs://namenode:9000",
+"OZONE-SITE.XML_ozone.ksm.address": "ksm",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.server.default.replication": "1",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
+}'
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-compose.yaml

@@ -37,6 +37,7 @@ services:
       - ../_keytabs:/etc/security/keytabs
       - ./krb5.conf:/etc/krb5.conf
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     environment:
       HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
     command: ["hadoop", "kms"]

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-ratis-om-bootstrap

@@ -14,7 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.om.ratis.log.purge.gap=50
-OZONE-SITE.XML_ozone.om.ratis.segment.size=16KB
-OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size=16KB
-OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold=500
+OZONE_CONF_CONTAINER_RATIS_BOOTSTRAP='{
+"OZONE-SITE.XML_ozone.om.ratis.log.purge.gap": "50",
+"OZONE-SITE.XML_ozone.om.ratis.segment.size": "16KB",
+"OZONE-SITE.XML_ozone.om.ratis.segment.preallocated.size": "16KB",
+"OZONE-SITE.XML_ozone.om.ratis.snapshot.auto.trigger.threshold": "500"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config-scm4

@@ -14,5 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3,scm4
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm4=scm4.org
+OZONE_CONF_CONTAINER_SCM4='{
+"OZONE-SITE.XML_ozone.scm.nodes.scmservice": "scm1,scm2,scm3,scm4",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm4": "scm4.org"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/root-ca-rotation.conf

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_CONF_CONTAINER_CA_ROTATION='{
+"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
+"OZONE-SITE.XML_hdds.x509.max.duration": "PT240S",
+"OZONE-SITE.XML_hdds.x509.default.duration": "PT60S",
+"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT45S",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S",
+"OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval": "PT2s",
+"OZONE-SITE.XML_hdds.block.token.expiry.time": "15s",
+"OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime": "15s",
+"OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval": "15s",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "60s",
+"OZONE-SITE.XML_hdds.scmclient.failover.retry.interval": "1s",
+"OZONE-SITE.XML_hdds.scmclient.failover.max.retry": "60",
+"OZONE-SITE.XML_ozone.scm.info.wait.duration": "60s",
+"OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout": "2s",
+"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.HttpCrossOriginFilterInitializer",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.enabled": "true"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/root-ca-rotation.yaml

@@ -16,24 +16,8 @@
 
 x-root-cert-rotation-config:
   &root-cert-rotation-config
-  environment:
-    - OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
-    - OZONE-SITE.XML_hdds.x509.max.duration=PT240S
-    - OZONE-SITE.XML_hdds.x509.default.duration=PT60S
-    - OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
-    - OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT2s
-    - OZONE-SITE.XML_hdds.block.token.expiry.time=15s
-    - OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s
-    - OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s
-    - OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=60s
-    - OZONE-SITE.XML_hdds.scmclient.failover.retry.interval=1s
-    - OZONE-SITE.XML_hdds.scmclient.failover.max.retry=60
-    - OZONE-SITE.XML_ozone.scm.info.wait.duration=60s
-    - OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout=2s
-    - OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.HttpCrossOriginFilterInitializer
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.enabled=true
+  env_file:
+    - ./root-ca-rotation.conf
 services:
   datanode1:
     <<: *root-cert-rotation-config

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.conf

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_CONF_CONTAINER_S3G='{
+"OZONE-SITE.XML_ozone.s3g.domain.name": "s3g.internal"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/s3g-virtual-host.yaml

@@ -16,8 +16,8 @@
 
 x-s3g-virtual-host-config:
   &s3g-virtual-host-config
-  environment:
-    - OZONE-SITE.XML_ozone.s3g.domain.name=s3g.internal
+  env_file:
+    - ./s3g-virtual-host.conf
 services:
   datanode1:
     <<: *s3g-virtual-host-config

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml

@@ -33,6 +33,7 @@ services:
     volumes:
       - ./krb5.conf:/etc/krb5.conf
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     command: ["hadoop", "kms"]
   datanode:
     image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}

hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config

@@ -14,63 +14,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.block.token.enabled=true
-OZONE-SITE.XML_hdds.container.token.enabled=true
-OZONE-SITE.XML_ozone.server.default.replication=3
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.administrators=*
-OZONE-SITE.XML_ozone.s3.administrators="s3g"
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE_CONF_CONTAINER_MR='{
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.handler.type": "distributed",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_hdds.block.token.enabled": "true",
+"OZONE-SITE.XML_hdds.container.token.enabled": "true",
+"OZONE-SITE.XML_ozone.server.default.replication": "3",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
+"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
+"OZONE-SITE.XML_ozone.administrators": "*",
+"OZONE-SITE.XML_ozone.s3.administrators": "s3g",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
 
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
-OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
+"OZONE-SITE.XML_ozone.security.enabled": "true",
+"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
+"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",
 
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
-OZONE-SITE.XML_hdds.grpc.tls.enabled=true
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/scm.keytab",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/om.keytab",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/s3g.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",
 
-OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
+"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
+"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",
 
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/dn@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/dn.keytab
+"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/dn.keytab",
 
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
 
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
-CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
+"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
+"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
+"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
+"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms"
+}'
 
 #temporarily disable authorization as org.apache.hadoop.yarn.server.api.ResourceTrackerPB is not properly annotated to support it
-CORE-SITE.XML_hadoop.security.authorization=false
+OZONE_CONF_CONTAINER_MR_SECURITY='{
+"CORE-SITE.XML_hadoop.security.authorization": "false"
+}
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
@@ -85,4 +89,6 @@ OZONE_LOG_DIR=/var/log/hadoop
 no_proxy=om,scm,s3g,recon,kdc,localhost,127.0.0.1
 
 # Explicitly enable filesystem snapshot feature for this Docker compose cluster
-OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
+OZONE_CONF_CONTAINER_MR_SNAPSHOT='{
+"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure/certificate-rotation.conf

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_CONF_CONTAINER_CERTIFICATE_ROTATION='{
+"OZONE-SITE.XML_hdds.x509.default.duration": "PT40s",
+"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT30s",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
+"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure/certificate-rotation.yaml

@@ -16,12 +16,8 @@
 
 x-cert-rotation-config:
   &cert-rotation-config
-  environment:
-    - OZONE-SITE.XML_hdds.x509.default.duration=PT40s
-    - OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT30s
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
-    - OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
+  env_file:
+    - ./certificate-rotation.conf
 services:
   datanode:
     <<: *cert-rotation-config

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml

@@ -35,6 +35,7 @@ services:
       HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
     volumes:
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     command: ["hadoop", "kms"]
   datanode:
     image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}

hadoop-ozone/dist/src/main/compose/ozonesecure/fcq.conf

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_CONF_CONTAINER_FCQ='{
+"CORE-SITE.XML_ipc.9862.callqueue.impl": "org.apache.hadoop.ipc.FairCallQueue",
+"CORE-SITE.XML_ipc.9862.identity-provider.impl": "org.apache.hadoop.ozone.om.helpers.OzoneIdentityProvider",
+"OZONE-SITE.XML_ozone.om.transport.class": "org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure/fcq.yaml

@@ -16,11 +16,8 @@
 
 x-FCQ-config:
   &FCQ-config
-  environment:
-    - CORE-SITE.XML_ipc.9862.callqueue.impl=org.apache.hadoop.ipc.FairCallQueue
-    - CORE-SITE.XML_ipc.9862.identity-provider.impl=org.apache.hadoop.ozone.om.helpers.OzoneIdentityProvider
-    - OZONE-SITE.XML_ozone.om.transport.class=org.apache.hadoop.ozone.om.protocolPB.Hadoop3OmTransportFactory
-
+  env_file:
+    - ./fcq.conf
 services:
   om:
     <<: *FCQ-config

hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.conf

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_CONF_CONTAINER_ROOT_CA_ROTATION='{
+"OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled": "false",
+"OZONE-SITE.XML_hdds.x509.max.duration": "PT180S",
+"OZONE-SITE.XML_hdds.x509.default.duration": "PT60S",
+"OZONE-SITE.XML_hdds.x509.renew.grace.duration": "PT45S",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval": "PT1S",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout": "PT20S",
+"OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval": "PT2s",
+"OZONE-SITE.XML_hdds.block.token.expiry.time": "15s",
+"OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime": "15s",
+"OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval": "15s",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "60s",
+"OZONE-SITE.XML_hdds.scmclient.failover.retry.interval": "1s",
+"OZONE-SITE.XML_hdds.scmclient.failover.max.retry": "60",
+"OZONE-SITE.XML_ozone.scm.info.wait.duration": "60s",
+"OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout": "2s",
+"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.HttpCrossOriginFilterInitializer",
+"OZONE-SITE.XML_hdds.x509.ca.rotation.enabled": "true",
+"OZONE-SITE.XML_hdds.x509.expired.certificate.check.interval": "PT30s"
+}'

hadoop-ozone/dist/src/main/compose/ozonesecure/root-ca-rotation.yaml

@@ -16,25 +16,8 @@
 
 x-root-cert-rotation-config:
   &root-cert-rotation-config
-  environment:
-    - OZONE-SITE.XML_hdds.x509.grace.duration.token.checks.enabled=false
-    - OZONE-SITE.XML_hdds.x509.max.duration=PT180S
-    - OZONE-SITE.XML_hdds.x509.default.duration=PT60S
-    - OZONE-SITE.XML_hdds.x509.renew.grace.duration=PT45S
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.check.interval=PT1S
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.ack.timeout=PT20S
-    - OZONE-SITE.XML_hdds.x509.rootca.certificate.polling.interval=PT2s
-    - OZONE-SITE.XML_hdds.block.token.expiry.time=15s
-    - OZONE-SITE.XML_ozone.manager.delegation.token.max-lifetime=15s
-    - OZONE-SITE.XML_ozone.manager.delegation.token.renew-interval=15s
-    - OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=60s
-    - OZONE-SITE.XML_hdds.scmclient.failover.retry.interval=1s
-    - OZONE-SITE.XML_hdds.scmclient.failover.max.retry=60
-    - OZONE-SITE.XML_ozone.scm.info.wait.duration=60s
-    - OZONE-SITE.XML_ozone.scm.ha.ratis.request.timeout=2s
-    - OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.HttpCrossOriginFilterInitializer
-    - OZONE-SITE.XML_hdds.x509.ca.rotation.enabled=true
-    - OZONE-SITE.XML_hdds.x509.expired.certificate.check.interval=PT30s
+  env_file:
+    - ./root-ca-rotation.conf
 services:
   datanode:
     <<: *root-cert-rotation-config

hadoop-ozone/dist/src/main/compose/ozonesecure/vault.conf

@@ -14,10 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.secret.s3.store.provider=org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address=http://vault:8200
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace="namespace"
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever=2
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath=secret
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth=TOKEN
-OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token=00000000-0000-0000-0000-000000000000
+OZONE_CONF_CONTAINER_VAULT='{
+"OZONE-SITE.XML_ozone.secret.s3.store.provider": "org.apache.hadoop.ozone.s3.remote.vault.VaultS3SecretStorageProvider",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.address": "http://vault:8200",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.namespace": "namespace",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.enginever": "2",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.secretpath": "secret",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth": "TOKEN",
+"OZONE-SITE.XML_ozone.secret.s3.store.remote.vault.auth.token": "00000000-0000-0000-0000-000000000000"
+}'
+
+OZONE_OPTS=-Dcom.sun.net.ssl.checkRevocation=false
+OZONE_MANAGER_CLASSPATH=/opt/hadoop/share/ozone/lib/ozone-s3-secret-store-@project.version@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-@vault.driver.version@.jar

hadoop-ozone/dist/src/main/compose/ozonesecure/vault.yaml

@@ -18,9 +18,6 @@ services:
   om:
     env_file:
       - vault.conf
-    environment:
-      - OZONE_OPTS=-Dcom.sun.net.ssl.checkRevocation=false
-      - OZONE_MANAGER_CLASSPATH=/opt/hadoop/share/ozone/lib/ozone-s3-secret-store-@project.version@.jar:/opt/hadoop/share/ozone/lib/vault-java-driver-@vault.driver.version@.jar
   vault:
     image: hashicorp/vault:1.13.2
     hostname: vault

hadoop-ozone/dist/src/main/compose/restart/docker-compose.yaml

@@ -45,6 +45,7 @@ services:
       - ${OZONE_VOLUME}/dn1:/data
       - ../..:${OZONE_DIR}
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
   dn2:
     <<: *datanode
     networks:
@@ -54,6 +55,7 @@ services:
       - ${OZONE_VOLUME}/dn2:/data
       - ../..:${OZONE_DIR}
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
   dn3:
     <<: *datanode
     networks:
@@ -63,6 +65,7 @@ services:
       - ${OZONE_VOLUME}/dn3:/data
       - ../..:${OZONE_DIR}
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
   om:
     command: ["ozone","om"]
     <<: *common-config
@@ -79,6 +82,7 @@ services:
       - ${OZONE_VOLUME}/om:/data
       - ../..:${OZONE_DIR}
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
   recon:
     command: ["ozone","recon"]
     <<: *common-config
@@ -92,6 +96,7 @@ services:
     volumes:
       - ${OZONE_VOLUME}/recon:/data
       - ../..:${OZONE_DIR}
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
   s3g:
     command: ["ozone","s3g"]
@@ -106,6 +111,7 @@ services:
     volumes:
       - ${OZONE_VOLUME}/s3g:/data
       - ../..:${OZONE_DIR}
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
   scm:
     command: ["ozone","scm"]
@@ -122,6 +128,7 @@ services:
     volumes:
       - ${OZONE_VOLUME}/scm:/data
       - ../..:${OZONE_DIR}
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
       - ../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
 
 networks:

hadoop-ozone/dist/src/main/compose/restart/docker-config

@@ -14,24 +14,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
+OZONE_CONF_CONTAINER_RESTART='{
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http"
+}'
 
 OZONE_CONF_DIR=/etc/hadoop
 OZONE_LOG_DIR=/var/log/hadoop

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -63,6 +63,7 @@ x-volumes:
     - &keytabs ../../../_keytabs:/etc/security/keytabs
     - &krb5conf ./krb5.conf:/etc/krb5.conf
     - &ozone-dir ../../../..:${OZONE_DIR}
+    - &envtoconf ../../../../libexec/envtoconf.py:/opt/envtoconf.py
     - &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
 
 services:
@@ -95,6 +96,7 @@ services:
       - *krb5conf
       - ../../../..:/opt/ozone
       - *transformation
+      - *envtoconf
   om1:
     <<: *om
     hostname: om1
@@ -107,6 +109,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   om2:
     <<: *om
     hostname: om2
@@ -119,6 +122,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   om3:
     <<: *om
     hostname: om3
@@ -131,6 +135,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
 
   scm1:
     <<: *scm
@@ -147,6 +152,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   scm2:
     <<: *scm
     environment:
@@ -163,6 +169,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   scm3:
     <<: *scm
     environment:
@@ -179,7 +186,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
-
+      - *envtoconf
   dn1:
     <<: *datanode
     hostname: dn1
@@ -192,6 +199,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   dn2:
     <<: *datanode
     hostname: dn2
@@ -204,6 +212,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   dn3:
     <<: *datanode
     hostname: dn3
@@ -216,6 +225,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   dn4:
     <<: *datanode
     hostname: dn4
@@ -228,6 +238,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   dn5:
     <<: *datanode
     hostname: dn5
@@ -240,6 +251,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   recon:
     command: ["ozone","recon"]
     <<: *common-config
@@ -257,6 +269,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+      - *envtoconf
   s3g:
     command: ["ozone","s3g"]
     <<: *common-config
@@ -274,7 +287,7 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
-
+      - *envtoconf
 networks:
   net:
     driver: bridge

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-config

@@ -14,32 +14,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE_CONF_CONTAINER_MR='{
+"OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata",
 
-OZONE-SITE.XML_ozone.client.failover.max.attempts=6
+"OZONE-SITE.XML_ozone.client.failover.max.attempts=6",
 
-OZONE-SITE.XML_ozone.om.service.ids=omservice
-OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
-OZONE-SITE.XML_ozone.om.address.omservice.om1=om1
-OZONE-SITE.XML_ozone.om.address.omservice.om2=om2
-OZONE-SITE.XML_ozone.om.address.omservice.om3=om3
+"OZONE-SITE.XML_ozone.om.service.ids=omservice",
+"OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3",
+"OZONE-SITE.XML_ozone.om.address.omservice.om1=om1",
+"OZONE-SITE.XML_ozone.om.address.omservice.om2=om2",
+"OZONE-SITE.XML_ozone.om.address.omservice.om3=om3",
 
-OZONE-SITE.XML_ozone.scm.service.ids=scmservice
-OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org
-OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org
-OZONE-SITE.XML_ozone.scm.ratis.enable=true
-OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1
+"OZONE-SITE.XML_ozone.scm.service.ids=scmservice",
+"OZONE-SITE.XML_ozone.scm.nodes.scmservice=scm1,scm2,scm3",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm1=scm1.org",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm2=scm2.org",
+"OZONE-SITE.XML_ozone.scm.address.scmservice.scm3=scm3.org",
+"OZONE-SITE.XML_ozone.scm.ratis.enable=true",
+"OZONE-SITE.XML_ozone.scm.primordial.node.id=scm1",
 
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-OZONE-SITE.XML_ozone.fs.hsync.enabled=true
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata",
+"OZONE-SITE.XML_ozone.scm.container.size=1GB",
+"OZONE-SITE.XML_hdds.datanode.dir=/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB",
+"OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http",
+"OZONE-SITE.XML_ozone.fs.hsync.enabled=true"
+}'
 
 # If SCM sends container close commands as part of upgrade finalization while
 # datanodes are doing a leader election, all 3 replicas may end up in the
@@ -50,20 +52,26 @@ OZONE-SITE.XML_ozone.fs.hsync.enabled=true
 # start the replication manager and pipeline scrubber. The default of 5 minutes
 # is fine in real clusters to prevent unnecessary over-replication,
 # but it is too long for this test.
-OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=5s
+OZONE_CONF_CONTAINER_MR_SAFEMODE='{
+"OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit": "5s"
+}'
 # If datanodes take too long to close pipelines during finalization, let the
 # scrubber force close them to move the test forward.
-OZONE-SITE.XML_ozone.scm.pipeline.scrub.interval=1m
-OZONE-SITE.XML_ozone.scm.pipeline.allocated.timeout=2m
+OZONE_CONF_CONTAINER_MR_PIPELINE='{
+"OZONE-SITE.XML_ozone.scm.pipeline.scrub.interval": "1m",
+"OZONE-SITE.XML_ozone.scm.pipeline.allocated.timeout": "2m",
 
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_ozone.recon.address=recon:9891
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay": "1m",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891"
+}'
 
 no_proxy=om1,om2,om3,scm1,scm2,scm3,s3g,kdc,localhost,127.0.0.1
 
 OM_SERVICE_ID=omservice
 
 # Explicitly enable filesystem snapshot feature for this Docker compose cluster
 # Does not take effect on Ozone versions < 1.4.0
-OZONE-SITE.XML_ozone.filesystem.snapshot.enabled=true
+OZONE_CONF_CONTAINER_MR_SNAPSHOT='{
+"OZONE-SITE.XML_ozone.filesystem.snapshot.enabled": "true"
+}'

hadoop-ozone/dist/src/main/compose/xcompat/docker-config

@@ -14,114 +14,116 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.defaultFS=ofs://om
-CORE-SITE.XML_fs.trash.interval=1
-CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
-
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
-OZONE-SITE.XML_hdds.scm.safemode.min.datanode=3
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.http-address=scm:9876
-OZONE-SITE.XML_ozone.recon.address=recon:9891
-OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon
-OZONE-SITE.XML_ozone.server.default.replication=3
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.scm.container.size=1GB
-OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data/metadata
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
-OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
-OZONE-SITE.XML_ozone.datanode.pipeline.limit=1
-OZONE-SITE.XML_recon.om.snapshot.task.interval.delay=1m
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-OZONE-SITE.XML_ozone.default.bucket.layout=LEGACY
-OZONE-SITE.XML_ozone.http.basedir=/tmp/ozone_http
-
-OZONE-SITE.XML_hdds.block.token.enabled=true
-OZONE-SITE.XML_hdds.container.token.enabled=true
-OZONE-SITE.XML_hdds.grpc.tls.enabled=true
-
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_ozone.acl.enabled=true
-OZONE-SITE.XML_ozone.acl.authorizer.class=org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer
-OZONE-SITE.XML_ozone.administrators="testuser,recon,om"
-OZONE-SITE.XML_ozone.s3.administrators="testuser,recon,om"
-OZONE-SITE.XML_ozone.recon.administrators="testuser2"
-OZONE-SITE.XML_ozone.s3.administrators="testuser,s3g"
-
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local="DEFAULT"
-CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
-
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.keytab.file=/etc/security/keytabs/recon.keytab
-OZONE-SITE.XML_ozone.recon.kerberos.principal=recon/recon@EXAMPLE.COM
-
-OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/s3g@EXAMPLE.COM
-
-OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/httpfs@EXAMPLE.COM
-
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/dn@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
-OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
-OZONE-SITE.XML_ozone.http.filter.initializers=org.apache.hadoop.security.AuthenticationFilterInitializer
-
-OZONE-SITE.XML_ozone.om.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.scm.http.auth.type=kerberos
-OZONE-SITE.XML_hdds.datanode.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.s3g.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.s3g.secret.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.httpfs.http.auth.type=kerberos
-OZONE-SITE.XML_ozone.recon.http.auth.type=kerberos
-
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal=HTTP/dn@EXAMPLE.COM
-OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab=/etc/security/keytabs/dn.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab=/etc/security/keytabs/s3g.keytab
-OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal=HTTP/s3g@EXAMPLE.COM
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab=/etc/security/keytabs/httpfs.keytab
-OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal=HTTP/httpfs@EXAMPLE.COM
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal=*
-OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab=/etc/security/keytabs/recon.keytab
-
-CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed=false
-CORE-SITE.XML_hadoop.http.authentication.signature.secret.file=/etc/security/http_secret
-CORE-SITE.XML_hadoop.http.authentication.type=kerberos
-CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/ozone@EXAMPLE.COM
-CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-
-CORE-SITE.XML_hadoop.security.authorization=true
-HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl=*
-HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl=*
-HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl=*
-
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups=*
-KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts=*
+OZONE_CONF_CONTAINER_MR='{
+"CORE-SITE.XML_fs.defaultFS": "ofs://om",
+"CORE-SITE.XML_fs.trash.interval": "1",
+"CORE-SITE.XML_fs.ofs.impl": "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem",
+
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.datanode.volume.min.free.space": "100MB",
+"OZONE-SITE.XML_hdds.scm.safemode.min.datanode": "3",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.http-address": "scm:9876",
+"OZONE-SITE.XML_ozone.recon.address": "recon:9891",
+"OZONE-SITE.XML_ozone.recon.db.dir": "/data/metadata/recon",
+"OZONE-SITE.XML_ozone.server.default.replication": "3",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.scm.container.size": "1GB",
+"OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min": "10MB",
+"OZONE-SITE.XML_ozone.scm.datanode.id.dir": "/data/metadata",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.pipeline.creation.interval": "30s",
+"OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count": "1",
+"OZONE-SITE.XML_ozone.datanode.pipeline.limit": "1",
+"OZONE-SITE.XML_recon.om.snapshot.task.interval.delay": "1m",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+"OZONE-SITE.XML_ozone.default.bucket.layout": "LEGACY",
+"OZONE-SITE.XML_ozone.http.basedir": "/tmp/ozone_http",
+
+"OZONE-SITE.XML_hdds.block.token.enabled": "true",
+"OZONE-SITE.XML_hdds.container.token.enabled": "true",
+"OZONE-SITE.XML_hdds.grpc.tls.enabled": "true",
+
+"OZONE-SITE.XML_ozone.security.enabled": "true",
+"OZONE-SITE.XML_ozone.acl.enabled": "true",
+"OZONE-SITE.XML_ozone.acl.authorizer.class": "org.apache.hadoop.ozone.security.acl.OzoneNativeAuthorizer",
+"OZONE-SITE.XML_ozone.administrators": "testuser,recon,om",
+"OZONE-SITE.XML_ozone.s3.administrators": "testuser,recon,om",
+"OZONE-SITE.XML_ozone.recon.administrators": "testuser2",
+"OZONE-SITE.XML_ozone.s3.administrators": "testuser,s3g",
+
+"HDFS-SITE.XML_dfs.datanode.address": "0.0.0.0:1019",
+"HDFS-SITE.XML_dfs.datanode.http.address": "0.0.0.0:1012",
+"CORE-SITE.XML_dfs.data.transfer.protection": "authentication",
+"CORE-SITE.XML_hadoop.security.authentication": "kerberos",
+"CORE-SITE.XML_hadoop.security.auth_to_local": "DEFAULT",
+"CORE-SITE.XML_hadoop.security.key.provider.path": "kms://http@kms:9600/kms",
+
+"OZONE-SITE.XML_hdds.scm.kerberos.principal": "scm/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.kerberos.keytab.file": "/etc/security/keytabs/scm.keytab",
+"OZONE-SITE.XML_ozone.om.kerberos.principal": "om/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.kerberos.keytab.file": "/etc/security/keytabs/om.keytab",
+"OZONE-SITE.XML_ozone.recon.kerberos.keytab.file": "/etc/security/keytabs/recon.keytab",
+"OZONE-SITE.XML_ozone.recon.kerberos.principal": "recon/recon@EXAMPLE.COM",
+
+"OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file": "/etc/security/keytabs/s3g.keytab",
+"OZONE-SITE.XML_ozone.s3g.kerberos.principal": "s3g/s3g@EXAMPLE.COM",
+
+"OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file": "/etc/security/keytabs/httpfs.keytab",
+"OZONE-SITE.XML_ozone.httpfs.kerberos.principal": "httpfs/httpfs@EXAMPLE.COM",
+
+"HDFS-SITE.XML_dfs.datanode.kerberos.principal": "dn/dn@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.datanode.keytab.file": "/etc/security/keytabs/dn.keytab",
+"HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file": "/etc/security/keytabs/dn.keytab",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
+"HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+
+"OZONE-SITE.XML_ozone.security.http.kerberos.enabled": "true",
+"OZONE-SITE.XML_ozone.s3g.secret.http.enabled": "true",
+"OZONE-SITE.XML_ozone.http.filter.initializers": "org.apache.hadoop.security.AuthenticationFilterInitializer",
+
+"OZONE-SITE.XML_ozone.om.http.auth.type": "kerberos",
+"OZONE-SITE.XML_hdds.scm.http.auth.type": "kerberos",
+"OZONE-SITE.XML_hdds.datanode.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.s3g.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.s3g.secret.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.type": "kerberos",
+"OZONE-SITE.XML_ozone.recon.http.auth.type": "kerberos",
+
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.principal": "HTTP/scm@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.scm.http.auth.kerberos.keytab": "/etc/security/keytabs/scm.keytab",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.principal": "HTTP/om@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.om.http.auth.kerberos.keytab": "/etc/security/keytabs/om.keytab",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.principal": "HTTP/dn@EXAMPLE.COM",
+"OZONE-SITE.XML_hdds.datanode.http.auth.kerberos.keytab": "/etc/security/keytabs/dn.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.keytab": "/etc/security/keytabs/s3g.keytab",
+"OZONE-SITE.XML_ozone.s3g.http.auth.kerberos.principal": "HTTP/s3g@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.keytab": "/etc/security/keytabs/httpfs.keytab",
+"OZONE-SITE.XML_ozone.httpfs.http.auth.kerberos.principal": "HTTP/httpfs@EXAMPLE.COM",
+"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.principal": "*",
+"OZONE-SITE.XML_ozone.recon.http.auth.kerberos.keytab": "/etc/security/keytabs/recon.keytab",
+
+"CORE-SITE.XML_hadoop.http.authentication.simple.anonymous.allowed": "false",
+"CORE-SITE.XML_hadoop.http.authentication.signature.secret.file": "/etc/security/http_secret",
+"CORE-SITE.XML_hadoop.http.authentication.type": "kerberos",
+"CORE-SITE.XML_hadoop.http.authentication.kerberos.principal": "HTTP/ozone@EXAMPLE.COM",
+"CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab": "/etc/security/keytabs/HTTP.keytab",
+
+"CORE-SITE.XML_hadoop.security.authorization": "true",
+"HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.container.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.block.protocol.acl": "*",
+"HADOOP-POLICY.XML_hdds.security.client.scm.certificate.protocol.acl": "*",
+"HADOOP-POLICY.XML_ozone.security.reconfigure.protocol.acl": "*",
+
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.users": "*",
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.groups": "*",
+"KMS-SITE.XML_hadoop.kms.proxyuser.s3g.hosts": "*"
+}'
 
 OZONE_DATANODE_SECURE_USER=root
 JSVC_HOME=/usr/bin

hadoop-ozone/dist/src/main/compose/xcompat/new-cluster.yaml

@@ -47,6 +47,7 @@ services:
       HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
     volumes:
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     command: [ "hadoop", "kms" ]
   datanode:
     <<: *new-config

hadoop-ozone/dist/src/main/compose/xcompat/old-cluster.yaml

@@ -47,6 +47,7 @@ services:
       HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
     volumes:
       - ../../libexec/transformation.py:/opt/transformation.py
+      - ../../libexec/envtoconf.py:/opt/envtoconf.py
     command: [ "hadoop", "kms" ]
   datanode:
     <<: *old-config

hadoop-ozone/dist/src/main/dockerlibexec/envtoconf.py

@@ -17,7 +17,7 @@
 #
 
 """convert environment variables to config"""
-
+import json
 import os
 import re
 
@@ -38,6 +38,7 @@ def __init__(self, args):
     self.output_dir = self.args.destination
     self.excluded_envs = ['HADOOP_CONF_DIR']
     self.configurables = {}
+    self.conf_container_prefix = 'OZONE_CONF_CONTAINER'
 
   def destination_file_path(self, name, extension):
     """destination file path"""
@@ -51,36 +52,48 @@ def write_env_var(self, name, extension, key, value):
   def process_envs(self):
     """Process environment variables"""
     for key in os.environ.keys():
-      if key in self.excluded_envs:
-          continue
-      pattern = re.compile("[_\\.]")
-      parts = pattern.split(key)
-      extension = None
-      name = parts[0].lower()
-      if len(parts) > 1:
-        extension = parts[1].lower()
-        config_key = key[len(name) + len(extension) + 2:].strip()
-      if extension and "!" in extension:
-        splitted = extension.split("!")
-        extension = splitted[0]
-        fmt = splitted[1]
-        config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
-      else:
-        fmt = extension
-
-      if extension and extension in self.known_formats:
-        if name not in self.configurables.keys():
-          with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
-            myfile.write("")
-        self.configurables[name] = (extension, fmt)
-        self.write_env_var(name, extension, config_key, os.environ[key])
+      if key.startswith(self.conf_container_prefix):
+        conf_variables=json.loads(os.environ[key])
+        for conf_key in conf_variables:
+          self.process_env_var(conf_key, conf_variables[conf_key])
       else:
-        for configurable_name in self.configurables:
-          if key.lower().startswith(configurable_name.lower()):
-            self.write_env_var(configurable_name,
-                               self.configurables[configurable_name],
-                               key[len(configurable_name) + 1:],
-                               os.environ[key])
+        self.process_env_var(key, os.environ[key])
+
+  def process_env_var(self, env_key, env_value):
+    """Process environment variable"""
+    if env_key in self.excluded_envs:
+      return
+
+    pattern = re.compile("[_\\.]")
+    parts = pattern.split(env_key)
+    extension = None
+    name = parts[0].lower()
+    config_key = ''
+
+    if len(parts) > 1:
+      extension = parts[1].lower()
+      config_key = env_key[len(name) + len(extension) + 2:].strip()
+    if extension and "!" in extension:
+      splitted = extension.split("!")
+      extension = splitted[0]
+      fmt = splitted[1]
+      config_key = env_key[len(name) + len(extension) + len(fmt) + 3:].strip()
+    else:
+      fmt = extension
+
+    if extension and extension.lower() in self.known_formats:
+      if name not in self.configurables.keys():
+        with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+          myfile.write("")
+      self.configurables[name] = (extension, fmt)
+      self.write_env_var(name, extension, config_key, env_value)
+    else:
+      for configurable_name in self.configurables:
+        if env_key.lower().startswith(configurable_name.lower()):
+          self.write_env_var(configurable_name,
+                             self.configurables[configurable_name],
+                             env_key[len(configurable_name) + 1:],
+                             env_value)
 
   def transform(self):
     """transform"""

hadoop-ozone/fault-injection-test/network-tests/src/test/compose/docker-config

@@ -14,64 +14,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.scm.dead.node.interval=5m
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+OZONE_CONF_CONTAINER_NETWORK_TEST='{
+"OZONE-SITE.XML_ozone.om.address": "om",
+"OZONE-SITE.XML_ozone.om.http-address": "om:9874",
+"OZONE-SITE.XML_ozone.scm.names": "scm",
+"OZONE-SITE.XML_ozone.scm.datanode.id": "/data/datanode.id",
+"OZONE-SITE.XML_ozone.scm.block.client.address": "scm",
+"OZONE-SITE.XML_ozone.metadata.dirs": "/data/metadata",
+"OZONE-SITE.XML_ozone.handler.type": "distributed",
+"OZONE-SITE.XML_ozone.scm.client.address": "scm",
+"OZONE-SITE.XML_ozone.scm.dead.node.interval": "5m",
+"OZONE-SITE.XML_ozone.replication": "1",
+"OZONE-SITE.XML_hdds.datanode.dir": "/data/hdds",
+"OZONE-SITE.XML_hdds.scmclient.max.retry.timeout": "30s",
+
+"HDFS-SITE.XML_rpc.metrics.quantile.enable": "true",
+"HDFS-SITE.XML_rpc.metrics.percentiles.intervals": "60,300",
+
+"LOG4J.PROPERTIES_log4j.rootLogger": "INFO, stdout",
+"LOG4J.PROPERTIES_log4j.appender.stdout": "org.apache.log4j.ConsoleAppender",
+"LOG4J.PROPERTIES_log4j.appender.stdout.layout": "org.apache.log4j.PatternLayout",
+"LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern": "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n",
+"LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader": "ERROR",
+"LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils": "WARN",
+"LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping": "ERROR",
+
+"LOG4J2.PROPERTIES_monitorInterval": "30",
+"LOG4J2.PROPERTIES_filter": "read,write",
+"LOG4J2.PROPERTIES_filter.read.type": "MarkerFilter",
+"LOG4J2.PROPERTIES_filter.read.marker": "READ",
+"LOG4J2.PROPERTIES_filter.read.onMatch": "DENY",
+"LOG4J2.PROPERTIES_filter.read.onMismatch": "NEUTRAL",
+"LOG4J2.PROPERTIES_filter.write.type": "MarkerFilter",
+"LOG4J2.PROPERTIES_filter.write.marker": "WRITE",
+"LOG4J2.PROPERTIES_filter.write.onMatch": "NEUTRAL",
+"LOG4J2.PROPERTIES_filter.write.onMismatch": "NEUTRAL",
+"LOG4J2.PROPERTIES_appenders": "console, rolling",
+"LOG4J2.PROPERTIES_appender.console.type": "Console",
+"LOG4J2.PROPERTIES_appender.console.name": "STDOUT",
+"LOG4J2.PROPERTIES_appender.console.layout.type": "PatternLayout",
+"LOG4J2.PROPERTIES_appender.console.layout.pattern": "%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n",
+"LOG4J2.PROPERTIES_appender.rolling.type": "RollingFile",
+"LOG4J2.PROPERTIES_appender.rolling.name": "RollingFile",
+"LOG4J2.PROPERTIES_appender.rolling.fileName": "${sys:hadoop.log.dir}/om-audit-${hostName}.log",
+"LOG4J2.PROPERTIES_appender.rolling.filePattern": "${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz",
+"LOG4J2.PROPERTIES_appender.rolling.layout.type": "PatternLayout",
+"LOG4J2.PROPERTIES_appender.rolling.layout.pattern": "%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n",
+"LOG4J2.PROPERTIES_appender.rolling.policies.type": "Policies",
+"LOG4J2.PROPERTIES_appender.rolling.policies.time.type": "TimeBasedTriggeringPolicy",
+"LOG4J2.PROPERTIES_appender.rolling.policies.time.interval": "86400",
+"LOG4J2.PROPERTIES_appender.rolling.policies.size.type": "SizeBasedTriggeringPolicy",
+"LOG4J2.PROPERTIES_appender.rolling.policies.size.size": "64MB",
+"LOG4J2.PROPERTIES_loggers": "audit",
+"LOG4J2.PROPERTIES_logger.audit.type": "AsyncLogger",
+"LOG4J2.PROPERTIES_logger.audit.name": "OMAudit",
+"LOG4J2.PROPERTIES_logger.audit.level": "INFO",
+"LOG4J2.PROPERTIES_logger.audit.appenderRefs": "rolling",
+"LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref": "RollingFile",
+"LOG4J2.PROPERTIES_rootLogger.level": "INFO",
+"LOG4J2.PROPERTIES_rootLogger.appenderRefs": "stdout",
+"LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref": "STDOUT"
+}'
 
 #Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
 #BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT