Merge pull request #625 from mendix/develop

Release 2023-04-20

Makefile

@@ -91,13 +91,11 @@ fixup:
 
 .PHONY: test_unit
 test_unit:
-	export PYTHONPATH=.:lib/
-	nosetests --verbosity=3 --nocapture --with-timer --timer-no-color tests/unit/test_*.py
+	pytest -vvv --capture=no --durations=0 --color=no tests/unit/test_*.py
 
 .PHONY: test_integration
 test_integration:
-	export PYTHONPATH=.:lib/
-	nosetests --verbosity=3 --nocapture --processes=${TEST_PROCESSES} --process-timeout=3600 --with-timer --timer-no-color ${TEST_FILES}
+	pytest -vvv --capture=no --timeout=3600 --color=no ${TEST_FILES}
 
 .PHONY: test
 test: test_unit test_integration

README.md

@@ -582,7 +582,10 @@ To enable New Relic, simply bind a New Relic service to this app and settings wi
 
 ### Splunk
 
-To collect Mendix Runtime logs to [Splunk Cloud Platform](https://www.splunk.com/en_us/products/splunk-cloud-platform.html), [Fluent Bit](https://docs.fluentbit.io/manual/) is used.
+#### Set up Splunk integration
+
+To collect Mendix Runtime logs to [Splunk Cloud Platform](https://www.splunk.com/en_us/products/splunk-cloud-platform.html), 
+[Fluent Bit](https://docs.fluentbit.io/manual/) is used.
 
 To enable Splunk integration for a Mendix application, following environment variables should be configured.
 
@@ -601,6 +604,26 @@ button `New Token` in the top-right corner of the page.
 Once the Mendix application is redeployed/restarted, the runtime application logs should appear on the Splunk Cloud under `Search & Reporting`.
 In the search line specify: `source="http:your_token_name"`, click search button.
 
+#### Metadata
+
+In addition to the runtime application logs, the following JSON-formatted metadata is automatically sent to the Splunk Cloud Platform:
+
+* `environment_id` - unique identifier of the environment;
+* `instance_index` - number of the application instance;
+* `hostname` - name of the application host;
+* `application_name` - default application name, retrieved from domain name;
+* `model_version` - model version of the Mendix runtime;
+* `runtime_version` - version of the Mendix runtime.
+
+You can filter the data by these fields on Splunk Cloud Platform web interface.
+
+#### Custom tags
+
+You can also set up custom tags in the following format `key:value`. We recommend that you add the following custom tags:
+
+* `app:{app_name}` – this enables you to identify all logs sent from your app (for example, **app:customermanagement**)
+* `env:{environment_name}` – this enables you to identify logs sent from a particular environment so you can separate out production logs from test logs (for example, **env:accp**)
+
 ### AppDynamics
 
 All the information collected with AppDynamics Java Agent and Machine Agent is pushed to Controller and displayed on the [Controller UI](https://docs.appdynamics.com/21.1/en/appdynamics-essentials/getting-started/controller-ui-overview).

buildpack/core/java.py

@@ -1,24 +1,25 @@
-from email.mime import application
 import json
 import logging
 import os
 import re
 import subprocess
-from distutils.util import strtobool
 
 import certifi
 from buildpack import util
 from lib.m2ee.version import MXVersion
+from lib.m2ee.util import strtobool
 
 
 JAVA_VERSION_OVERRIDE_KEY = "JAVA_VERSION"
 
 
 def get_java_major_version(runtime_version):
     result = 8
+    if os.getenv(JAVA_VERSION_OVERRIDE_KEY):
+        return _get_major_version(os.getenv(JAVA_VERSION_OVERRIDE_KEY))
     if runtime_version >= MXVersion("8.0.0"):
         result = 11
-    return _get_major_version(os.getenv(JAVA_VERSION_OVERRIDE_KEY, result))
+    return _get_major_version(result)
 
 
 def _get_major_version(version):
@@ -28,11 +29,11 @@ def _get_major_version(version):
     ):
         return 8
     # Java >= 11
-    major_version = int(str(version).split(".")[0])
+    major_version = int(str(version).split(".", maxsplit=1)[0])
     if major_version >= 8:
         return major_version
     # Java < 8 is not supported
-    raise ValueError("Cannot determine major version for Java [%s]" % version)
+    raise ValueError(f"Cannot determine major version for Java [{version}]")
 
 
 def _get_security_properties_file(jvm_location, java_major_version):
@@ -43,7 +44,7 @@ def _get_security_properties_file(jvm_location, java_major_version):
         conf_dir = "conf"
     else:
         raise ValueError(
-            "Cannot determine security subdirectory for Java [%s]" % java_major_version
+            f"Cannot determine security subdirectory for Java [{java_major_version}]"
         )
     return os.path.join(
         os.path.abspath(jvm_location), conf_dir, "security", "java.security"
@@ -58,7 +59,8 @@ def _is_outgoing_tls_10_11_enabled():
 
 
 # Configures TLSv1.0 and TLSv1.1 for outgoing connections
-# These two protocols are considered insecure and have been disabled in OpenJDK after March 2021
+# These two protocols are considered insecure
+# and have been disabled in OpenJDK after March 2021
 # Re-enabling them is at your own risk!
 def _configure_outgoing_tls_10_11(jvm_location, java_major_version):
     if _is_outgoing_tls_10_11_enabled():
@@ -67,21 +69,25 @@ def _configure_outgoing_tls_10_11(jvm_location, java_major_version):
             security_properties_file = _get_security_properties_file(
                 jvm_location, java_major_version
             )
-        except ValueError as e:
+        except ValueError as exception:
             logging.error(
-                "Not enabling TLSv1.0 and TLSv1.1 for outgoing connections: " % e
+                "Not enabling TLSv1.0 and TLSv1.1 for outgoing connections: %s",
+                exception,
             )
             return
         if not (
             os.path.exists(security_properties_file)
             and os.access(security_properties_file, os.W_OK)
         ):
             logging.error(
-                "Java security properties file does not exist at expected location or is not writeable, not enabling TLSv1.0 and TLSv1.1 for outgoing connections"
+                "Java security properties file does not exist at expected location"
+                " or is not writeable, not enabling TLSv1.0 and TLSv1.1"
+                " for outgoing connections"
             )
             return
         logging.warning(
-            "Enabling TLSv1.0 and TLSv1.1 for outgoing connections. These protocols are considered insecure and End-Of-Life."
+            "Enabling TLSv1.0 and TLSv1.1 for outgoing connections. "
+            "These protocols are considered insecure and End-Of-Life."
         )
         with open(security_properties_file, "r+") as f:
             lines = f.readlines()
@@ -98,7 +104,8 @@ def _configure_outgoing_tls_10_11(jvm_location, java_major_version):
                     f.write(line)
                     if not line.endswith("\\\n"):
                         # Disable line modification after property has been parsed
-                        # This is required for multi-line properties with one or more line separators (backslash)
+                        # This is required for multi-line properties
+                        # with one or more line separators (backslash)
                         in_property = False
                 else:
                     f.write(line)
@@ -151,19 +158,16 @@ def stage(buildpack_path, cache_path, local_path, java_major_version):
 
 
 def _compose_jvm_target_dir(dependency):
-    return "usr/lib/jvm/%s-%s-%s-%s-x64" % (
-        dependency["vendor"],
-        dependency["type"],
-        dependency["version"],
-        dependency["vendor"],
-    )
+    return f"usr/lib/jvm/{dependency['vendor']}-{dependency['type']}-{dependency['version']}-{dependency['vendor']}-x64"  # noqa: line-too-long
 
 
 def _get_java_dependency(
-    java_major_version, package, buildpack_dir=os.getcwd(), variables={}
+    java_major_version, package, buildpack_dir=os.getcwd(), variables=None
 ):
+    if variables is None:
+        variables = {}
     return util.get_dependency(
-        "java.%s-%s" % (java_major_version, package), variables, buildpack_dir
+        f"java.{java_major_version}-{package}", variables, buildpack_dir
     )
 
 
@@ -178,7 +182,7 @@ def ensure_and_get_jvm(
     override_version = os.getenv(JAVA_VERSION_OVERRIDE_KEY)
     overrides = {}
     if override_version:
-        logging.info("Overriding Java version to [%s]..." % override_version)
+        logging.info("Overriding Java version to [%s]...", override_version)
         if not override_version.isdigit():
             overrides = {
                 "version": override_version,
@@ -191,10 +195,10 @@ def ensure_and_get_jvm(
 
     jdk_dir = _compose_jvm_target_dir(dependency)
 
-    rootfs_java_path = "/{}".format(jdk_dir)
+    rootfs_java_path = f"/{jdk_dir}"
     if not os.path.isdir(rootfs_java_path):
         logging.debug(
-            "Downloading and installing Java {} if required...".format(package.upper())
+            "Downloading and installing Java %s if required...", package.upper()
         )
         util.resolve_dependency(
             dependency,
@@ -204,7 +208,7 @@ def ensure_and_get_jvm(
             unpack_strip_directories=True,
             overrides=overrides,
         )
-        logging.debug("Java {} installed".format(package.upper()))
+        logging.debug("Java %s installed", package.upper())
     else:
         logging.debug("Root FS with Java SDK detected, not installing Java")
 
@@ -222,12 +226,14 @@ def _update_java_cacert(jar, cache_dir, jvm_location):
     cacerts_file = os.path.join(jvm_location, "lib", "security", "cacerts")
     if not os.path.exists(cacerts_file):
         logging.warning(
-            "Cannot locate Java cacerts file %s. Skipping update of JVM CA certificates.",
+            "Cannot locate Java cacerts file %s. "
+            "Skipping update of JVM CA certificates.",
             cacerts_file,
         )
         return
 
-    # Parse the Mozilla CA certificate bundle from certifi and import it into the keystore
+    # Parse the Mozilla CA certificate bundle from certifi
+    # and import it into the keystore
     keyutil_jar = os.path.abspath(os.path.join(cache_dir, jar))
 
     # Import the certificate into the keystore
@@ -249,7 +255,7 @@ def _update_java_cacert(jar, cache_dir, jvm_location):
             stderr=subprocess.STDOUT,
         )
     except subprocess.CalledProcessError as ex:
-        logging.error("Error importing certificates: {}".format(ex.output))
+        logging.error("Error importing certificates: %s", ex.output)
         raise ex
 
     logging.debug("Import of Mozilla certificates finished")
@@ -302,21 +308,22 @@ def _set_jvm_memory(m2ee, vcap):
     env_heap_size = os.environ.get("HEAP_SIZE")
     max_metaspace_size = os.getenv("MAX_METASPACE_SIZE", "256M")
 
-    util.upsert_javaopts(m2ee, "-XX:MaxMetaspaceSize=%s" % max_metaspace_size)
+    util.upsert_javaopts(m2ee, f"-XX:MaxMetaspaceSize={max_metaspace_size}")
 
     if env_heap_size:
         if int(env_heap_size[:-1]) < limit:
             heap_size = env_heap_size
         else:
             logging.warning(
-                "The specified heap size [{}] is larger than the maximum memory of the "
-                "container ([{}]). Falling back to a heap size of [{}]".format(
-                    env_heap_size, str(limit) + "M", heap_size
-                )
+                "The specified heap size [%s] is larger than the maximum memory of the "
+                "container ([%s]). Falling back to a heap size of [%s]",
+                env_heap_size,
+                str(limit) + "M",
+                heap_size,
             )
 
-    util.upsert_javaopts(m2ee, "-Xmx%s" % heap_size)
-    util.upsert_javaopts(m2ee, "-Xms%s" % heap_size)
+    util.upsert_javaopts(m2ee, f"-Xmx{heap_size}")
+    util.upsert_javaopts(m2ee, f"-Xms{heap_size}")
 
     logging.debug("Java heap size set to %s", heap_size)
 
@@ -329,7 +336,7 @@ def _set_jvm_memory(m2ee, vcap):
 
 
 def _set_application_name(m2ee, application_name):
-    util.upsert_javaopts(m2ee, "-DapplicationName=%s" % application_name)
+    util.upsert_javaopts(m2ee, f"-DapplicationName={application_name}")
 
 
 def update_config(m2ee, application_name, vcap_data, runtime_version):

buildpack/core/mono.py

@@ -1,7 +1,7 @@
 import glob
 import logging
 import os
-import platform
+import distro
 
 from buildpack import util
 from buildpack.util import NotFoundException, get_dependency
@@ -17,7 +17,7 @@ def get_env_with_monolib(mono_dir):
         env["LC_ALL"] = "C"
 
     if not os.path.isfile(os.path.join(mono_dir, "lib", "libgdiplus.so")):
-        raise Exception("libgdiplus.so not found in dir %s" % mono_dir)
+        raise Exception(f"libgdiplus.so not found in dir {mono_dir}")
     return env
 
 
@@ -36,30 +36,28 @@ def _detect_mono_version(mx_version):
 def _get_mono_path(directory, mono_version):
     return util.get_existing_directory_or_raise(
         [
-            os.path.join(directory, "mono-%s" % mono_version),
-            "/opt/mono-%s" % mono_version,
-            "/tmp/mono-%s" % mono_version,
+            os.path.join(directory, f"mono-{mono_version}"),
+            f"/opt/mono-{mono_version}",
+            f"/tmp/mono-{mono_version}",
         ],
         "Mono not found",
     )
 
 
 def _compose_mono_dependency_name(mono_version):
-    distrib_id = platform.linux_distribution()[0].lower()
+    distrib_id = distro.id().lower()
     if distrib_id != "ubuntu":
         raise Exception(
-            "Only Ubuntu is supported at present, requested distribution: {}".format(
-                distrib_id
-            )
+            "Only Ubuntu is supported at present, "
+            f"requested distribution: {distrib_id}"
         )
-    distrib_codename = platform.linux_distribution()[2].lower()
-    if distrib_codename not in ["trusty", "bionic"]:
+    distrib_codename = distro.codename().lower()
+    if distrib_codename not in ["trusty", "bionic", "jammy"]:
         raise Exception(
-            "Buildpack supports Trusty and Bionic at the moment, requested version: {}".format(
-                distrib_codename
-            )
+            "Buildpack supports Trusty, Bionic, and Jammy at the moment, "
+            f"requested version: {distrib_codename}"
         )
-    return "mono.{}-{}".format(mono_version, distrib_codename)
+    return f"mono.{mono_version}-{distrib_codename}"
 
 
 def ensure_and_get_mono(mx_version, buildpack_dir, cache_dir):
@@ -68,7 +66,7 @@ def ensure_and_get_mono(mx_version, buildpack_dir, cache_dir):
     dependency_name = _compose_mono_dependency_name(major_version)
     fallback_location = "/tmp/opt"
 
-    if major_version == "3" and platform.linux_distribution()[2].lower() == "bionic":
+    if major_version == "3" and distro.codename().lower() == "bionic":
         dependency = util.resolve_dependency(
             dependency_name,
             os.path.join(fallback_location, "store"),
@@ -77,18 +75,16 @@ def ensure_and_get_mono(mx_version, buildpack_dir, cache_dir):
             unpack_strip_directories=True,
         )
         version = dependency["version"]
-        mono_subpath = glob.glob("/tmp/opt/store/*-mono-env-%s" % version)
-        mono_location = "/tmp/opt/mono-%s" % version
+        mono_subpath = glob.glob(f"/tmp/opt/store/*-mono-env-{version}")
+        mono_location = f"/tmp/opt/mono-{version}"
         os.symlink(mono_subpath[0], mono_location)
-        logging.debug(
-            "Mono available: {mono_location}".format(mono_location=mono_location)
-        )
+        logging.debug("Mono available: %s", mono_location)
         logging.warning(
             "The staging phase is likely going to fail when the default "
-            + "settings are used. As a workaround, more disk space needs to be "
-            + "allocated for the cache. Consult "
-            + "https://docs.cloudfoundry.org/devguide/deploy-apps/large-app-deploy.html "
-            + "for more information."
+            "settings are used. As a workaround, more disk space needs to be "
+            "allocated for the cache. Consult "
+            "https://docs.cloudfoundry.org/devguide/deploy-apps/large-app-deploy.html "
+            "for more information."
         )
         return mono_location
     else:
@@ -99,13 +95,11 @@ def ensure_and_get_mono(mx_version, buildpack_dir, cache_dir):
             logging.debug("Mono not found in default locations")
             util.resolve_dependency(
                 dependency_name,
-                os.path.join(fallback_location, "mono-%s" % version),
+                os.path.join(fallback_location, f"mono-{version}"),
                 buildpack_dir=buildpack_dir,
                 cache_dir=cache_dir,
                 unpack_strip_directories=True,
             )
             mono_location = _get_mono_path(fallback_location, version)
-        logging.debug(
-            "Mono available: {mono_location}".format(mono_location=mono_location)
-        )
+        logging.debug("Mono available: %s", mono_location)
         return mono_location