diff --git a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java index 3cab8eb7..e2a5a419 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java +++ b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java @@ -22,6 +22,7 @@ import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -57,6 +58,11 @@ public class ServerConfigDocument { private File configDirectory; private File serverXMLFile; + private static final String WLP_INSTALL_DIR_PROPERTY = "wlp.install.dir"; + private static final String WLP_USER_DIR_PROPERTY = "wlp.user.dir"; + private static final String SERVER_CONFIG_DIR_PROPERTY = "server.config.dir"; + // private static final String CONFIGDROPINS_DEFAULT; + // private static final String CONFIGDROPINS_OVERRIDES; private Set names; private Set namelessLocations; @@ -116,6 +122,17 @@ public File getServerXML() { return serverXMLFile; } + /** + * + * @param log + * @param serverXML + * @param configDir + * @param bootstrapFile + * @param bootstrapProp + * @param serverEnvFile + * @param giveConfigDirPrecedence + * @param libertyDirPropertyFiles - Contains a property to file mapping of direcdty locations + */ public ServerConfigDocument(CommonLoggerI log, File serverXML, File configDir, File bootstrapFile, Map bootstrapProp, File serverEnvFile, boolean giveConfigDirPrecedence, Map libertyDirPropertyFiles) { initializeAppsLocation(log, serverXML, configDir, bootstrapFile, bootstrapProp, serverEnvFile, giveConfigDirPrecedence, libertyDirPropertyFiles); @@ -154,28 +171,26 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf Document doc = parseDocument(serverXMLFile); - // Server variable precedence in ascending order if defined in - // multiple locations. - // - // 1. defaultValue from variables defined in server.xml or defined in files - // e.g. - // 2. variables from 'server.env' - // 3. variables from 'bootstrap.properties' - // 4. variables defined in files - // 5. variables from configDropins/defaults/ - // 6. variables defined in server.xml - // e.g. - // 7. variables from configDropins/overrides/ + // Server variable precedence in ascending order if defined in multiple locations. + // 1. variable default values in the server.xml file + // 2. environment variables + // 3. bootstrap.properties + // 4. Java system properties + // 5. Variables loaded from files in the ${server.config.dir}/variables directory or other directories as specified by the VARIABLE_SOURCE_DIRS environment variable + // 6. variable values declared in the server.xml file + // 7. variables declared on the command line // 1. Need to parse variables in the server.xml for default values before trying to find the include files in case one of the variables is used // in the location. parseVariablesForDefaultValues(doc); // 2. get variables from server.env - processServerEnv(serverEnvFile, giveConfigDirPrecedence); + processServerEnv(); + + // 3. get variables from jvm.options // 3. get variables from bootstrap.properties - processBootstrapProperties(bootstrapFile, bootstrapProp, giveConfigDirPrecedence); + processBootstrapProperties(bootstrapProp, bootstrapFile); // 4. parse variables from include files (both default and non-default values - which we store separately) parseIncludeVariables(doc); @@ -201,16 +216,29 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf } } - public void processServerEnv(File serverEnvFile, boolean giveConfigDirPrecedence) - throws Exception, FileNotFoundException { - File cfgFile = findConfigFile("server.env", serverEnvFile, giveConfigDirPrecedence); - if (cfgFile != null) { - parseProperties(new FileInputStream(cfgFile)); + /** + * server.env file read order + * 1. {wlp.install.dir}/etc/ + * 2. {wlp.user.dir}/shared/ + * 3. {server.config.dir}/ + * @param serverEnvFile + * @throws Exception + * @throws FileNotFoundException + */ + public void processServerEnv() throws Exception, FileNotFoundException { + final String serverEnvString = "server.env"; + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(WLP_INSTALL_DIR_PROPERTY), "etc" + File.separator + serverEnvString)); + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(WLP_USER_DIR_PROPERTY), "shared" + File.separator + serverEnvString)); + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(SERVER_CONFIG_DIR_PROPERTY), serverEnvString)); + } + + public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException { + if (propertiesFile != null && propertiesFile.exists()) { + parseProperties(new FileInputStream(propertiesFile)); } } - public void initializeFields(CommonLoggerI log, File serverXML, File configDir, - Map libertyDirPropertyFiles) { + public void initializeFields(CommonLoggerI log, File serverXML, File configDir, Map libertyDirPropertyFiles) { this.log = log; serverXMLFile = serverXML; configDirectory = configDir; @@ -220,7 +248,6 @@ public void initializeFields(CommonLoggerI log, File serverXML, File configDir, log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); libertyDirectoryPropertyToFile = new HashMap(); } - locations = new HashSet(); names = new HashSet(); namelessLocations = new HashSet(); @@ -229,13 +256,17 @@ public void initializeFields(CommonLoggerI log, File serverXML, File configDir, defaultProps = new Properties(); } - public void processBootstrapProperties(File bootstrapFile, Map bootstrapProp, boolean giveConfigDirPrecedence) - throws Exception, FileNotFoundException { + /** + * Process bootstrap.properties and boostrap.include + * @param bootstrapProp + * @param bootstrapFile - Optional specific file + * @throws Exception + * @throws FileNotFoundException + */ + public void processBootstrapProperties(Map bootstrapProp, File bootstrapFile) throws Exception, FileNotFoundException { File cfgDirFile = getFileFromConfigDirectory("bootstrap.properties"); - - if (giveConfigDirPrecedence && cfgDirFile != null) { - parseProperties(new FileInputStream(cfgDirFile)); - } else if (bootstrapProp != null && !bootstrapProp.isEmpty()) { + // TODO: bootstrap.include + if (bootstrapProp != null && !bootstrapProp.isEmpty()) { for (Map.Entry entry : bootstrapProp.entrySet()) { if (entry.getValue() != null) { props.setProperty(entry.getKey(),entry.getValue()); diff --git a/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java index b59c0923..b97abe27 100644 --- a/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java +++ b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java @@ -2,14 +2,15 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; -import java.util.logging.Logger; +import java.util.Properties; import javax.xml.xpath.XPathExpressionException; @@ -33,18 +34,17 @@ */ public class ServerConfigDocumentTest { - public final static Logger LOGGER = Logger.getLogger(ServerConfigDocumentTest.class.getName()); - private final static String RESOURCES_DIR = "src/test/resources/"; - private final static String WLP_DIR = "serverConfig/liberty/wlp/"; - private final static String WLP_USER_DIR = "serverConfig/liberty/wlp/usr/"; - private final static String DEFAULT_USER_SERVER_DIR = "servers/defaultServer/"; - private final static String DEFAULT_SERVER_DIR = "servers/"; + private final static Path RESOURCES_DIR = Paths.get("src/test/resources/"); + private final static Path WLP_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/"); + private final static Path WLP_USER_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/usr/"); + private final static Path DEFAULTSERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer"); + private final static Path MOCK_SERVER_DIR = RESOURCES_DIR.resolve("servers/"); // 1. variable default values in server.xml file // 6. variable values declared in the server.xml file @Test public void processServerXml() throws FileNotFoundException, IOException, XPathExpressionException, SAXException { - File serversDir = new File(RESOURCES_DIR + DEFAULT_SERVER_DIR); + File serversDir = MOCK_SERVER_DIR.toFile(); Document doc; // no variables defined @@ -83,100 +83,100 @@ public void processServerXml() throws FileNotFoundException, IOException, XPathE // configDropins/overrides } + // when a server.xml references an environment variable that could not be resolved, additionally search for: + // 1. replace all non-alphanumeric characters with underscore char '_' + // 2. change all characters to uppercase + @Test + public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Exception { + File serverXml = DEFAULTSERVER_CONFIG_DIR.resolve("server.xml").toFile(); + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); + configDocument.initializeFields(new TestLogger(), serverXml, DEFAULTSERVER_CONFIG_DIR.toFile(), new HashMap<>()); + Document serverXmlDoc = configDocument.parseDocument(serverXml); + configDocument.parseVariablesForBothValues(serverXmlDoc); + assertEquals("${this.value}", configDocument.getDefaultProperties().getProperty("server.env.defined")); + assertEquals("${this.value}", configDocument.getProperties().getProperty("server.env.defined")); + configDocument.processBootstrapProperties(new HashMap<>(), null); + configDocument.processServerEnv(); + + configDocument.parseVariablesForBothValues(serverXmlDoc); + // TODO: implement feature and uncomment these lines + // assertEquals("DEFINED", configDocument.getProperties().getProperty("server.env.defined")); + // assertEquals("DEFINED", configDocument.getProperties().getProperty("bootstrap.properties.defined")); + } + // server.env files are read in increasing precedence // 1. {wlp.install.dir}/etc // 2. {wlp.user.dir}/shared // 3. {server.config.dir} - // Table of directories: https://openliberty.io/docs/latest/reference/directory-locations-properties.html + // Liberty Directory Properties: https://openliberty.io/docs/latest/reference/directory-locations-properties.html @Test public void processServerEnv() throws FileNotFoundException, Exception { - File serverDir = new File(RESOURCES_DIR + WLP_USER_DIR + DEFAULT_USER_SERVER_DIR); - File specificFile = new File(serverDir, "server2.env"); + File wlpInstallDir = WLP_DIR.toFile(); + File wlpUserDir = WLP_USER_DIR.toFile(); + File serverDir = DEFAULTSERVER_CONFIG_DIR.toFile(); ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), null, serverDir, null); - configDocument.processServerEnv(specificFile, false); - assertEquals("TEST", configDocument.getProperties().getProperty("keystore_password")); + Map libertyDirectoryPropertyToFileMap = new HashMap(); + libertyDirectoryPropertyToFileMap.put("wlp.install.dir", wlpInstallDir); + libertyDirectoryPropertyToFileMap.put("wlp.user.dir", wlpUserDir); + libertyDirectoryPropertyToFileMap.put("server.config.dir", serverDir); - configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), null, serverDir, null); - configDocument.processServerEnv(specificFile, true); - assertNotEquals("TEST", configDocument.getProperties().getProperty("keystore_password")); + configDocument.initializeFields(new TestLogger(), null, serverDir, libertyDirectoryPropertyToFileMap); + configDocument.processServerEnv(); + Properties props = configDocument.getProperties(); - // TODO: multiple file locations - // File wlpDir = new File(RESOURCES_DIR + WLP_DIR); + // {wlp.install.dir}/etc + assertEquals("true", props.get("etc.unique")); + + // {wlp.user.dir}/shared + assertEquals("true", props.get("shared.unique")); + assertEquals("true", props.get("shared.overriden")); + + // {server.config.dir} + assertEquals("old_value", props.get("overriden_value")); + assertEquals("1111", props.get("http.port")); } // 2. environment variables - // TODO: test without using processServerEnv @Test public void environmentVariables() throws FileNotFoundException, Exception { - // TODO: alt var lookups - https://github.com/OpenLiberty/ci.common/issues/126 - File serverConfigDir = new File(RESOURCES_DIR + WLP_USER_DIR + DEFAULT_USER_SERVER_DIR); - ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), new File(serverConfigDir, "server.xml"), serverConfigDir, null); - Document doc = configDocument.parseDocument(new File(serverConfigDir, "server.xml")); - - // env var not set - configDocument.parseVariablesForBothValues(doc); - assertEquals("${my.env.var}", configDocument.getProperties().getProperty("my.env.var.level")); - - // replace non-alphanumeric characters with '_' and to uppercase - configDocument.processServerEnv(new File(serverConfigDir, "server2.env"), false); - assertEquals("3", configDocument.getProperties().getProperty("MY_ENV_VAR")); - configDocument.parseVariablesForBothValues(doc); - // assertEquals("3", configDocument.getProperties().getProperty("my.env.var.level")); - // replace non-alphanumeric characters with '_' - configDocument.processServerEnv(null, true); - assertEquals("2", configDocument.getProperties().getProperty("my_env_var")); - configDocument.parseVariablesForBothValues(doc); - // assertEquals("2", configDocument.getProperties().getProperty("my.env.var.level")); } // 3. bootstrap.properties @Test public void processBootstrapProperties() throws FileNotFoundException, Exception { - File serversDir = new File(RESOURCES_DIR + DEFAULT_SERVER_DIR); - File altBootstrapPropertiesFile = new File(serversDir, "bootstrap2.properties"); - Map bootstrapProp = new HashMap(); - bootstrapProp.put("http.port", "1000"); + File serversDir = MOCK_SERVER_DIR.toFile(); ServerConfigDocument configDocument; - // lowest to highest precedence - // default bootstrap.properties in config dir + // bootstrap.properties in config dir configDocument = new ServerConfigDocument(new TestLogger()); configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(new File("DOES_NOT_EXIST"), new HashMap<>(), false); + configDocument.processBootstrapProperties(new HashMap<>(), new File("DOES_NOT_EXIST")); assertEquals(1, configDocument.getProperties().size()); - // use bootstrapFile + // use bootstrapFile, kept for flexibility configDocument = new ServerConfigDocument(new TestLogger()); configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(altBootstrapPropertiesFile, new HashMap<>(), false); + configDocument.processBootstrapProperties(new HashMap<>(), DEFAULTSERVER_CONFIG_DIR.resolve("bootstrap.properties").toFile()); assertEquals(2, configDocument.getProperties().size()); - assertEquals("9080", configDocument.getProperties().getProperty("http.port")); + assertEquals("DEFINED", configDocument.getProperties().getProperty("THAT_VALUE")); - // test bootstrapProperty overrides + // test bootstrapProperty map overrides + Map bootstrapPropertyMap = new HashMap(); + bootstrapPropertyMap.put("http.port", "1000"); File serverXml = new File(serversDir, "definedVariables.xml"); configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), null, serversDir, null); - Document doc = configDocument.parseDocument(serverXml); - configDocument.parseVariablesForBothValues(doc); + configDocument.initializeFields(new TestLogger(), serverXml, serversDir, null); + configDocument.parseVariablesForBothValues(configDocument.parseDocument(serverXml)); assertEquals("9081", configDocument.getProperties().getProperty("http.port")); - configDocument.processBootstrapProperties(altBootstrapPropertiesFile, bootstrapProp, false); + configDocument.processBootstrapProperties(bootstrapPropertyMap, null); assertEquals("1000", configDocument.getProperties().getProperty("http.port")); - // use giveConfigDirPrecedence - configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(altBootstrapPropertiesFile, new HashMap<>(), true); - assertEquals(1, configDocument.getProperties().size()); - // TODO: bootstraps.include // configDocument = new ServerConfigDocument(new TestLogger()); // configDocument.initializeFields(new TestLogger(), null, serversDir, null); - // configDocument.processBootstrapProperties(new File(serversDir, "bootstrapsInclude.properties"), new HashMap<>(), false); + // configDocument.processBootstrapProperties(new HashMap<>(), MOCK_SERVER_DIR.resolve("bootstrapInclude.properties").toFile()); // assertEquals("9080", configDocument.getProperties().getProperty("http.port")); } @@ -207,8 +207,7 @@ public void CLI() { // server.xml override bootstrap.properties, jvm.options, and server.env @Test public void overrides() { - File serversDir = new File(RESOURCES_DIR + WLP_USER_DIR + DEFAULT_USER_SERVER_DIR); - + File serversDir = DEFAULTSERVER_CONFIG_DIR.toFile(); // server.xml overrides server.env ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), new File(serversDir, "server.xml"), serversDir, new File(serversDir, "bootstrap.properties"), diff --git a/src/test/resources/serverConfig/liberty/wlp/etc/server.env b/src/test/resources/serverConfig/liberty/wlp/etc/server.env index 1eff25ce..437b75ce 100644 --- a/src/test/resources/serverConfig/liberty/wlp/etc/server.env +++ b/src/test/resources/serverConfig/liberty/wlp/etc/server.env @@ -1,2 +1,3 @@ -http.port=9080 -https.port=9081 \ No newline at end of file +etc.unique=true +shared.overriden=false +http.port=9080 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties index 3efd222e..6a28af62 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/bootstrap.properties @@ -1 +1,2 @@ -extras.filename=extraFeatures.xml \ No newline at end of file +extras.filename=extraFeatures.xml +THAT_VALUE=DEFINED \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env index d0a8e5a0..027ead57 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.env @@ -1,3 +1,4 @@ keystore_password=C7ANPlAi0MQD154BJ5ZOURn +http.port=1111 overriden_value=old_value -my_env_var=2 \ No newline at end of file +this_value=DEFINED \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml index fcf023fd..43a7f85e 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server.xml @@ -22,5 +22,7 @@ - + + + \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server2.env b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server2.env deleted file mode 100644 index 6e646b70..00000000 --- a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/server2.env +++ /dev/null @@ -1,2 +0,0 @@ -keystore_password=TEST -MY_ENV_VAR=3 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env b/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env index 6a88f0c8..dcbb2923 100644 --- a/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env +++ b/src/test/resources/serverConfig/liberty/wlp/usr/shared/server.env @@ -1,2 +1,3 @@ -http.port=9082 -https.port=9083 \ No newline at end of file +shared.unique=true +shared.overriden=true +http.port=9081 \ No newline at end of file diff --git a/src/test/resources/servers/bootstrap2.properties b/src/test/resources/servers/bootstrap2.properties deleted file mode 100644 index d2ada960..00000000 --- a/src/test/resources/servers/bootstrap2.properties +++ /dev/null @@ -1,2 +0,0 @@ -extras.filename=extraFeatures.xml -http.port=9080 \ No newline at end of file diff --git a/src/test/resources/servers/bootstrapsInclude.properties b/src/test/resources/servers/bootstrapsInclude.properties index a83d44ec..1f510731 100644 --- a/src/test/resources/servers/bootstrapsInclude.properties +++ b/src/test/resources/servers/bootstrapsInclude.properties @@ -1 +1 @@ -bootstrap.include=bootstrap2.properties \ No newline at end of file +bootstrap.include=bootstrap.properties \ No newline at end of file