diff --git a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java index 67dab44b..7eda33e2 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java +++ b/src/main/java/io/openliberty/tools/common/plugins/config/ServerConfigDocument.java @@ -24,6 +24,7 @@ import java.net.URLConnection; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -143,6 +144,24 @@ public ServerConfigDocument(CommonLoggerI log) { initializeFields(log, null, null, null); } + public void initializeFields(CommonLoggerI log, File serverXML, File configDir, Map libertyDirPropertyFiles) { + this.log = log; + serverXMLFile = serverXML; + configDirectory = configDir; + if (libertyDirPropertyFiles != null) { + libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); + } else { + log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); + libertyDirectoryPropertyToFile = new HashMap(); + } + locations = new HashSet(); + names = new HashSet(); + namelessLocations = new HashSet(); + locationsAndNames = new HashMap(); + props = new Properties(); + defaultProps = new Properties(); + } + private DocumentBuilder getDocumentBuilder() { DocumentBuilder docBuilder; @@ -187,11 +206,25 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf processServerEnv(); // 3. get variables from jvm.options - processJvmOptions(); + // processJvmOptions(); // 3. get variables from bootstrap.properties processBootstrapProperties(bootstrapProp, bootstrapFile); + // 4. Java system properties + // configured in Maven/Gradle + + // 5. Variables loaded from 'variables' directory + processVariablesDirectory(); + + // 6. variable values declared in server.xml + // processVariablesForValues(doc); + + // 7. variables delcared on the command line + // configured in Maven/Gradle + + // TODO: cleanup rest + // 4. parse variables from include files (both default and non-default values - which we store separately) parseIncludeVariables(doc); @@ -216,23 +249,10 @@ private void initializeAppsLocation(CommonLoggerI log, File serverXML, File conf } } - - /** - * jvm.options file read order - * 1. {wlp.user.dir}/ - * 2. {server.config.dir}/configDropins/defaults/ - * 3. {server.config.dir}/ - * 4. {server.config.dir}/configDropins/overrides/ - * TODO: potential processing for system property tags? (-D prefixes?) - * @throws FileNotFoundException - * @throws Exception - */ - public void processJvmOptions() throws FileNotFoundException, Exception { - final String jvmOptionsString = "jvm.options"; - parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), jvmOptionsString)); - parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_DEFAULT + File.separator + jvmOptionsString)); - parsePropertiesFromFile(getFileFromConfigDirectory(jvmOptionsString)); - parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_OVERRIDES + File.separator + jvmOptionsString)); + public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException { + if (propertiesFile != null && propertiesFile.exists()) { + parseProperties(new FileInputStream(propertiesFile)); + } } /** @@ -251,28 +271,17 @@ public void processServerEnv() throws Exception, FileNotFoundException { parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.SERVER_CONFIG_DIR), serverEnvString)); } - public void parsePropertiesFromFile(File propertiesFile) throws Exception, FileNotFoundException { - if (propertiesFile != null && propertiesFile.exists()) { - parseProperties(new FileInputStream(propertiesFile)); - } - } - - public void initializeFields(CommonLoggerI log, File serverXML, File configDir, Map libertyDirPropertyFiles) { - this.log = log; - serverXMLFile = serverXML; - configDirectory = configDir; - if (libertyDirPropertyFiles != null) { - libertyDirectoryPropertyToFile = new HashMap(libertyDirPropertyFiles); - } else { - log.warn("The properties for directories are null and could lead to application locations not being resolved correctly."); - libertyDirectoryPropertyToFile = new HashMap(); - } - locations = new HashSet(); - names = new HashSet(); - namelessLocations = new HashSet(); - locationsAndNames = new HashMap(); - props = new Properties(); - defaultProps = new Properties(); + /** + * Likely not needed to be processed by the LMP/LGP tools. These properties benefit the JVM. + * @throws FileNotFoundException + * @throws Exception + */ + public void processJvmOptions() throws FileNotFoundException, Exception { + final String jvmOptionsString = "jvm.options"; + parsePropertiesFromFile(new File(libertyDirectoryPropertyToFile.get(ServerFeatureUtil.WLP_USER_DIR), jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_DEFAULT + File.separator + jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory(jvmOptionsString)); + parsePropertiesFromFile(getFileFromConfigDirectory(CONFIGDROPINS_OVERRIDES + File.separator + jvmOptionsString)); } /** @@ -326,7 +335,7 @@ private void processBootstrapInclude(String bootstrapIncludeLocation, Set toProcess = new ArrayList(); + if (!props.containsKey(variableDirectoryProperty)) { + toProcess.add(getFileFromConfigDirectory("variables")); + } else { + String delimiter = (File.separator.equals("/")) ? ":" : ";"; // OS heuristic + String[] splitDirectories = props.get(variableDirectoryProperty).toString().split(delimiter); + for (String directory : splitDirectories) { + Path directoryPath = Paths.get(directory); + File directoryFile = directoryPath.toFile(); + if (directoryFile.exists()) { + toProcess.add(directoryFile); + } + } + } + + processVariableSourceDirs(toProcess); + } + + /** + * The file name is the variable and the contents are the values. + * If a directory is within the directory, it is recurisvely processed. + * The parent dir gets prepended to the file name for the property name ("{parent directory}/{file name}") + * If the file name ends with *.properties, then it's processed as a properties file. + * @param directories - The root directories to process + * @throws Exception + * @throws FileNotFoundException + */ + public void processVariableSourceDirs(ArrayList directories) throws FileNotFoundException, Exception { + for (File directory : directories) { + if (!directory.isDirectory()) { + continue; + } + processNestedVariableSourceDirs(directory, ""); + } + } + + /** + * The nested operation + * @param directory - The directory being processed + * @param propertyPrefix - Tracks the nested directories to prepend + * @throws FileNotFoundException + * @throws Exception + */ + public void processNestedVariableSourceDirs(File directory, String propertyPrefix) throws FileNotFoundException, Exception { + for (File child : directory.listFiles()) { + if (child.isDirectory()) { + processNestedVariableSourceDirs(child, child.getName() + File.separator); + continue; + } + + if (child.getName().endsWith(".properties")) { + parsePropertiesFromFile(child); + continue; + } + + String propertyName = propertyPrefix + child.getName(); + String propertyValue = Files.readString(child.toPath()); + props.setProperty(propertyName, propertyValue); + } + } + //Checks for application names in the document. Will add locations without names to a Set private void parseNames(Document doc, String expression) throws XPathExpressionException, IOException, SAXException { // parse input document diff --git a/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java b/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java index 46d5471f..8f5d0091 100644 --- a/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java +++ b/src/main/java/io/openliberty/tools/common/plugins/util/VariableUtility.java @@ -28,6 +28,10 @@ public class VariableUtility { private static final String VARIABLE_NAME_PATTERN = "\\$\\{(.*?)\\}"; private static final Pattern varNamePattern = Pattern.compile(VARIABLE_NAME_PATTERN); + // If a property is not immediately found, replace non-alphanumeric values with '_'. If still not found, search with toUpper + // Integer value properties can be evaluated if 'simple' arithemetic + // A list of ports can be defined using keyword 'list' + /** * Attempts to resolve all variables in the passed in nodeValue. Variable value/defaultValue can reference other variables. * This method is called recursively to resolve the variables. The variableChain collection keeps track of the variable references diff --git a/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java index 8ff5e26e..9d46a6b3 100644 --- a/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java +++ b/src/test/java/io/openliberty/tools/common/config/ServerConfigDocumentTest.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Properties; +import javax.xml.bind.annotation.XmlElement.DEFAULT; import javax.xml.xpath.XPathExpressionException; import org.junit.Test; @@ -37,14 +38,14 @@ public class ServerConfigDocumentTest { private final static Path RESOURCES_DIR = Paths.get("src/test/resources/"); private final static Path WLP_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/"); private final static Path WLP_USER_DIR = RESOURCES_DIR.resolve("serverConfig/liberty/wlp/usr/"); - private final static Path DEFAULTSERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer"); - private final static Path MOCK_SERVER_DIR = RESOURCES_DIR.resolve("servers/"); + private final static Path SERVER_CONFIG_DIR = WLP_USER_DIR.resolve("servers/defaultServer"); + private final static Path SERVERS_RESOURCES_DIR = RESOURCES_DIR.resolve("servers/"); // 1. variable default values in server.xml file // 6. variable values declared in the server.xml file @Test public void processServerXml() throws FileNotFoundException, IOException, XPathExpressionException, SAXException { - File serversDir = MOCK_SERVER_DIR.toFile(); + File serversDir = SERVERS_RESOURCES_DIR.toFile(); Document doc; // no variables defined @@ -88,9 +89,9 @@ public void processServerXml() throws FileNotFoundException, IOException, XPathE // 2. change all characters to uppercase @Test public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Exception { - File serverXml = DEFAULTSERVER_CONFIG_DIR.resolve("server.xml").toFile(); + File serverXml = SERVER_CONFIG_DIR.resolve("server.xml").toFile(); ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); - configDocument.initializeFields(new TestLogger(), serverXml, DEFAULTSERVER_CONFIG_DIR.toFile(), new HashMap<>()); + configDocument.initializeFields(new TestLogger(), serverXml, SERVER_CONFIG_DIR.toFile(), new HashMap<>()); Document serverXmlDoc = configDocument.parseDocument(serverXml); configDocument.parseVariablesForBothValues(serverXmlDoc); assertEquals("${this.value}", configDocument.getDefaultProperties().getProperty("server.env.defined")); @@ -113,7 +114,7 @@ public void serverXmlEnvVarVariationLookup() throws FileNotFoundException, Excep public void processServerEnv() throws FileNotFoundException, Exception { File wlpInstallDir = WLP_DIR.toFile(); File wlpUserDir = WLP_USER_DIR.toFile(); - File serverDir = DEFAULTSERVER_CONFIG_DIR.toFile(); + File serverDir = SERVER_CONFIG_DIR.toFile(); ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); Map libertyDirectoryPropertyToFileMap = new HashMap(); libertyDirectoryPropertyToFileMap.put("wlp.install.dir", wlpInstallDir); @@ -145,7 +146,7 @@ public void environmentVariables() throws FileNotFoundException, Exception { // 3. bootstrap.properties @Test public void processBootstrapProperties() throws FileNotFoundException, Exception { - File serversDir = MOCK_SERVER_DIR.toFile(); + File serversDir = SERVERS_RESOURCES_DIR.toFile(); ServerConfigDocument configDocument; // bootstrap.properties in config dir @@ -157,7 +158,7 @@ public void processBootstrapProperties() throws FileNotFoundException, Exception // use bootstrapFile, kept for flexibility configDocument = new ServerConfigDocument(new TestLogger()); configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(new HashMap<>(), DEFAULTSERVER_CONFIG_DIR.resolve("bootstrap.properties").toFile()); + configDocument.processBootstrapProperties(new HashMap<>(), SERVER_CONFIG_DIR.resolve("bootstrap.properties").toFile()); assertEquals(2, configDocument.getProperties().size()); assertEquals("DEFINED", configDocument.getProperties().getProperty("THAT_VALUE")); @@ -175,13 +176,13 @@ public void processBootstrapProperties() throws FileNotFoundException, Exception // bootstrap.include configDocument = new ServerConfigDocument(new TestLogger()); configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(new HashMap<>(), MOCK_SERVER_DIR.resolve("bootstrapInclude.properties").toFile()); + configDocument.processBootstrapProperties(new HashMap<>(), SERVERS_RESOURCES_DIR.resolve("bootstrapInclude.properties").toFile()); assertEquals("extraFeatures.xml", configDocument.getProperties().getProperty("extras.filename")); // bootstrap.include infinite termination check configDocument = new ServerConfigDocument(new TestLogger()); configDocument.initializeFields(new TestLogger(), null, serversDir, null); - configDocument.processBootstrapProperties(new HashMap<>(), MOCK_SERVER_DIR.resolve("bootstrapOuroboros.properties").toFile()); + configDocument.processBootstrapProperties(new HashMap<>(), SERVERS_RESOURCES_DIR.resolve("bootstrapOuroboros.properties").toFile()); } // 4. Java system properties @@ -193,12 +194,36 @@ public void jvmOptions() { // 5. Variables loaded from files in the ${server.config.dir}/variables directory or other // directories as specified by the VARIABLE_SOURCE_DIRS environment variable @Test - public void variablesDir() { - // TODO: not yet implemented. read copied dir instead of src for now - // see https://github.com/OpenLiberty/ci.common/issues/126 + public void variablesDir() throws FileNotFoundException, Exception { + File serversDir = SERVER_CONFIG_DIR.toFile(); + ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger()); + configDocument.initializeFields(new TestLogger(), null, serversDir, null); + configDocument.processVariablesDirectory(); + + Properties props = configDocument.getProperties(); + assertEquals("9080", props.getProperty("httpPort")); + assertEquals("1000", props.getProperty("nested/httpPort")); + assertEquals("1", props.getProperty("VALUE_1")); + assertEquals("2", props.getProperty("VALUE_2")); + + // process VARIABLE_SOURCE_DIRS + configDocument = new ServerConfigDocument(new TestLogger()); + configDocument.initializeFields(new TestLogger(), null, serversDir, null); + Map bootstrapProp = new HashMap(); + String delimiter = (File.separator.equals("/")) ? ":" : ";"; + String variableSourceDirsTestValue = String.join(delimiter, + SERVERS_RESOURCES_DIR.resolve("variables").toString(), + SERVER_CONFIG_DIR.toString(), + "DOES_NOT_EXIST"); + bootstrapProp.put("VARIABLE_SOURCE_DIRS", variableSourceDirsTestValue); + configDocument.processBootstrapProperties(bootstrapProp, null); + configDocument.processVariablesDirectory(); + + props = configDocument.getProperties(); + assertEquals("outer_space", props.getProperty("outer.source")); + assertEquals("1", props.getProperty("VALUE_1")); } - // 7. variables declared on the command line @Test public void CLI() { @@ -211,7 +236,7 @@ public void CLI() { // server.xml override bootstrap.properties, jvm.options, and server.env @Test public void overrides() { - File serversDir = DEFAULTSERVER_CONFIG_DIR.toFile(); + File serversDir = SERVER_CONFIG_DIR.toFile(); // server.xml overrides server.env ServerConfigDocument configDocument = new ServerConfigDocument(new TestLogger(), new File(serversDir, "server.xml"), serversDir, new File(serversDir, "bootstrap.properties"), diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort new file mode 100644 index 00000000..04a3078b --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/httpPort @@ -0,0 +1 @@ +9080 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort new file mode 100644 index 00000000..e37d32ab --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/httpPort @@ -0,0 +1 @@ +1000 \ No newline at end of file diff --git a/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties new file mode 100644 index 00000000..abccd5b2 --- /dev/null +++ b/src/test/resources/serverConfig/liberty/wlp/usr/servers/defaultServer/variables/nested/nested.properties @@ -0,0 +1,2 @@ +VALUE_1=1 +VALUE_2=2 \ No newline at end of file diff --git a/src/test/resources/servers/variables/outer.source b/src/test/resources/servers/variables/outer.source new file mode 100644 index 00000000..f4ed48a9 --- /dev/null +++ b/src/test/resources/servers/variables/outer.source @@ -0,0 +1 @@ +outer_space \ No newline at end of file