Use mcr.microsoft.com/windows image as a source for DLLs instead of h…

…ost system (#187)

This commit:
1. Drops Windows 2016 LTSC support.
 Now the oldest supported Windows version is 2019 LTSC a.k.a. 1809
2. Stops copying DLLs from host system and instead uses mcr.microsoft.com/windows.
 This change allows building `ue4-build-prerequisites` even if host system version doesn't match.
 As a consequence of this change, `ue4-docker build` no longer has `-dlldir` command-line argument.
3. Extends list of copied DLLs to allow usage of Windows Advanced Rasterization Platform (WARP)
 in images derived from `ue4-build-prerequisites` image out of the box.

 Resolves #165

ue4docker/build.py

@@ -112,12 +112,12 @@ def build():
 			# Provide the user with feedback so they are aware of the Windows-specific values being used
 			logger.info('WINDOWS CONTAINER SETTINGS', False)
 			logger.info('Isolation mode:               {}'.format(config.isolation), False)
-			logger.info('Base OS image tag:            {}'.format(config.basetag), False)
+			logger.info('Base OS image:                {}'.format(config.baseImage), False)
+			logger.info('Dll source image:             {}'.format(config.dllSrcImage), False)
 			logger.info('Host OS:                      {}'.format(WindowsUtils.systemString()), False)
 			logger.info('Memory limit:                 {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False)
 			logger.info('Detected max image size:      {:.0f}GB'.format(DockerUtils.maxsize()), False)
 			logger.info('Visual Studio:                {}'.format(config.visualStudio), False)
-			logger.info('Directory to copy DLLs from:  {}\n'.format(config.dlldir), False)
 
 			# Verify that the specified base image tag is not a release that has reached End Of Life (EOL)
 			if not config.ignoreEOL and WindowsUtils.isEndOfLifeWindowsVersion(config.basetag):
@@ -144,19 +144,6 @@ def build():
 				logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!')
 				sys.exit(1)
 
-			# Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32
-			differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag
-			if differentKernels == True and config.dlldir == config.defaultDllDir:
-				logger.error('Error: building images with a different kernel version than the host,', False)
-				logger.error('but a custom DLL directory has not specified via the `-dlldir=DIR` arg.', False)
-				logger.error('The DLL files will be the incorrect version and the container OS will', False)
-				logger.error('refuse to load them, preventing the built Engine from running correctly.', False)
-				sys.exit(1)
-
-			# Attempt to copy the required DLL files from the host system
-			for dll in WindowsUtils.requiredHostDlls(config.basetag):
-				shutil.copy2(join(config.dlldir, dll), join(builder.context('ue4-build-prerequisites'), dll))
-
 			# Ensure the Docker daemon is configured correctly
 			requiredLimit = WindowsUtils.requiredSizeLimit()
 			if DockerUtils.maxsize() < requiredLimit:
@@ -245,7 +232,7 @@ def build():
 			prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage]
 			if config.containerPlatform == 'windows':
 				prereqsArgs = prereqsArgs + [
-					'--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild()),
+					'--build-arg', 'DLLSRCIMAGE=' + config.dllSrcImage,
 					'--build-arg', 'VISUAL_STUDIO_BUILD_NUMBER=' + config.visualStudioBuildNumber,
 				]
 

ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile

@@ -1,48 +1,30 @@
 # escape=`
 ARG BASEIMAGE
-FROM ${BASEIMAGE} AS dlls
-SHELL ["cmd", "/S", "/C"]
-
-{% if not disable_labels %}
-# Include our sentinel so `ue4-docker clean` can find this intermediate image
-LABEL com.adamrehn.ue4-docker.sentinel="1"
-{% endif %}
-
-# Create a directory in which to gather the DLL files we need 
-RUN mkdir C:\GatheredDlls
+ARG DLLSRCIMAGE
 
-# Install 7-Zip, curl, and Python using Chocolatey
-# (Note that these need to be separate RUN directives for `choco` to work)
-RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
-RUN choco install -y 7zip curl && choco install -y python --version=3.7.5
-
-# Copy the required DirectSound/DirectDraw and OpenGL DLL files from the host system (since these ship with Windows and don't have installers)
-COPY *.dll C:\GatheredDlls\
-
-# Verify that the DLL files copied from the host can be loaded by the container OS
-ARG HOST_BUILD
-RUN pip install pywin32
-COPY verify-host-dlls.py C:\
-RUN xcopy /y "C:\GatheredDlls\*.dll" C:\Windows\System32\
-RUN python C:\verify-host-dlls.py %HOST_BUILD% C:\GatheredDlls
+FROM ${DLLSRCIMAGE} as dlls
 
-# Gather the required DirectX runtime files, since Windows Server Core does not include them
-RUN curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe
-RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && `
-	expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\GatheredDlls\ && `
-	expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\GatheredDlls\ && `
-	expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\GatheredDlls\ && `
-	expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\GatheredDlls\ && `
-	expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\GatheredDlls\
-
-# Gather the Vulkan runtime library
-RUN curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip
-RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll"
-
-# Copy our gathered DLLs into a clean image to reduce image size
 FROM ${BASEIMAGE} as prerequisites
 SHELL ["cmd", "/S", "/C"]
-COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\
+
+# Gather the system DLLs that we need from the full Windows base image
+COPY --from=dlls `
+    C:\Windows\System32\avicap32.dll `
+    C:\Windows\System32\avrt.dll `
+    C:\Windows\System32\d3d10warp.dll `
+    C:\Windows\System32\D3DSCache.dll `
+    C:\Windows\System32\dsound.dll `
+    C:\Windows\System32\dxva2.dll `
+    C:\Windows\System32\glu32.dll `
+    C:\Windows\System32\mf.dll `
+    C:\Windows\System32\mfplat.dll `
+    C:\Windows\System32\mfplay.dll `
+    C:\Windows\System32\mfreadwrite.dll `
+    C:\Windows\System32\msdmo.dll `
+    C:\Windows\System32\msvfw32.dll `
+    C:\Windows\System32\opengl32.dll `
+    C:\Windows\System32\ResourcePolicyClient.dll `
+    C:\Windows\System32\
 
 {% if not disable_labels %}
 # Add a sentinel label so we can easily identify all derived images, including intermediate images

ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat

@@ -11,6 +11,26 @@ call refreshenv
 @rem Forcibly disable the git credential manager
 git config --system credential.helper "" || goto :error
 
+@rem Gather the required DirectX runtime files, since Windows Server Core does not include them
+curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe && ^
+start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ^
+expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\Windows\System32\ && ^
+expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\Windows\System32\ && ^
+expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\Windows\System32\ && ^
+expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\Windows\System32\ && ^
+expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\Windows\System32\ || goto :error
+
+@rem Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR)
+curl --progress -L "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" --output %TEMP%\dxc.zip && ^
+powershell -Command "Expand-Archive -Path \"$env:TEMP\dxc.zip\" -DestinationPath $env:TEMP" && ^
+xcopy /y %TEMP%\bin\x64\dxcompiler.dll C:\Windows\System32\ && ^
+xcopy /y %TEMP%\bin\x64\dxil.dll C:\Windows\System32\ || goto :error
+
+@rem Gather the Vulkan runtime library
+curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip && ^
+powershell -Command "Expand-Archive -Path \"$env:TEMP\vulkan-runtime-components.zip\" -DestinationPath $env:TEMP" && ^
+powershell -Command "Copy-Item -Path \"*\x64\vulkan-1.dll\" -Destination C:\Windows\System32" || goto :error
+
 set VISUAL_STUDIO_BUILD_NUMBER=%~1
 
 @rem Install the Visual Studio Build Tools workloads and components we need

ue4docker/infrastructure/BuildConfiguration.py

@@ -100,7 +100,6 @@ def addArguments(parser):
 		parser.add_argument('-branch', default=None, help='Set the custom branch/tag to clone when "custom" is specified as the release value')
 		parser.add_argument('-isolation', default=None, help='Set the isolation mode to use for Windows containers (process or hyperv)')
 		parser.add_argument('-basetag', default=None, help='Windows Server Core base image tag to use for Windows containers (default is the host OS version)')
-		parser.add_argument('-dlldir', default=None, help='Set the directory to copy required Windows DLLs from (default is the host System32 directory)')
 		parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images')
 		parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)')
 		parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image')
@@ -235,13 +234,6 @@ def describeExcludedComponents(self):
 		return sorted([ExcludedComponent.description(component) for component in self.excludedComponents])
 
 	def _generateWindowsConfig(self):
-
-		# Store the path to the directory containing our required Windows DLL files
-		hostSysnative = os.path.join(os.environ['SystemRoot'], 'Sysnative')
-		hostSystem32 = os.path.join(os.environ['SystemRoot'], 'System32')
-		self.defaultDllDir = hostSysnative if os.path.exists(hostSysnative) else hostSystem32
-		self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir
-
 		self.visualStudio = self.args.visual_studio
 
 		if not self.custom:
@@ -264,6 +256,7 @@ def _generateWindowsConfig(self):
 		# Store the tag for the base Windows Server Core image
 		self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag
 		self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag
+		self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag)
 		self.prereqsTag = self.basetag + '-vs' + self.visualStudio
 
 		# Verify that any user-specified base tag is valid
@@ -281,9 +274,8 @@ def _generateWindowsConfig(self):
 
 			# If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode
 			differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag
-			hostSupportsProcess = WindowsUtils.supportsProcessIsolation()
 			dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0')
-			if not differentKernels and hostSupportsProcess and dockerSupportsProcess:
+			if not differentKernels and dockerSupportsProcess:
 				self.isolation = 'process'
 			else:
 				self.isolation = DockerUtils.info()['Isolation']

ue4docker/infrastructure/WindowsUtils.py

@@ -7,18 +7,21 @@
 
 class WindowsUtils(object):
 
+	# The oldest Windows build we support
+	_minimumRequiredBuild = 17763
+
 	# The latest Windows build version we recognise as a non-Insider build
 	_latestReleaseBuild = 19042
 
 	# The list of Windows Server Core base image tags that we recognise, in ascending version number order
-	_validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004', '20H2']
+	_validTags = ['ltsc2019', '1903', '1909', '2004', '20H2']
 
 	# The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs
 	# (See: <https://unrealcontainers.com/docs/concepts/windows-containers>)
 	_blacklistedReleases = ['1903', '1909']
 
 	# The list of Windows Server Core container image releases that are unsupported due to having reached EOL
-	_eolReleases = ['1703', '1709', '1803', '1903', '1909']
+	_eolReleases = ['1903', '1909']
 
 	@staticmethod
 	def _getVersionRegKey(subkey : str) -> str:
@@ -32,16 +35,6 @@ def _getVersionRegKey(subkey : str) -> str:
 		winreg.CloseKey(key)
 		return value[0]
 
-	@staticmethod
-	def requiredHostDlls(basetag: str) -> [str]:
-		'''
-		Returns the list of required host DLL files for the specified container image base tag
-		'''
-
-		# `ddraw.dll` is only required under Windows Server 2016 version 1607
-		common = ['dsound.dll', 'opengl32.dll', 'glu32.dll']
-		return ['ddraw.dll'] + common if basetag == 'ltsc2016' else common
-
 	@staticmethod
 	def requiredSizeLimit() -> float:
 		'''
@@ -52,12 +45,9 @@ def requiredSizeLimit() -> float:
 	@staticmethod
 	def minimumRequiredBuild() -> int:
 		'''
-		Returns the minimum required version of Windows 10 / Windows Server, which is release 1607
-
-		(1607 is the first build to support Windows containers, as per:
-		<https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility>)
+		Returns the minimum required version of Windows 10 / Windows Server
 		'''
-		return 14393
+		return WindowsUtils._minimumRequiredBuild
 
 	@staticmethod
 	def systemString() -> str:
@@ -76,7 +66,13 @@ def getWindowsRelease() -> str:
 		'''
 		Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system
 		'''
-		return WindowsUtils._getVersionRegKey('ReleaseId')
+		try:
+			# Starting with Windows 20H2 (also known as 2009), Microsoft stopped updating ReleaseId
+			# and instead updates DisplayVersion
+			return WindowsUtils._getVersionRegKey('DisplayVersion')
+		except FileNotFoundError:
+			# Fallback to ReleaseId for pre-20H2 releases that didn't have DisplayVersion
+			return WindowsUtils._getVersionRegKey('ReleaseId')
 
 	@staticmethod
 	def getWindowsBuild() -> int:
@@ -131,15 +127,19 @@ def getReleaseBaseTag(release: str) -> str:
 
 		# This lookup table is based on the list of valid tags from <https://hub.docker.com/r/microsoft/windowsservercore/>
 		return {
-			'1709': '1709',
-			'1803': '1803',
 			'1809': 'ltsc2019',
-			'1903': '1903',
-			'1909': '1909',
-			'2004': '2004',
-			'2009': '20H2',
-			'20H2': '20H2'
-		}.get(release, 'ltsc2016')
+		}.get(release, release)
+
+	@staticmethod
+	def getDllSrcImage(basetag: str) -> str:
+		'''
+		Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image
+		'''
+		tag = {
+			'ltsc2019': '1809',
+		}.get(basetag, basetag)
+
+		return f'mcr.microsoft.com/windows:{tag}'
 
 	@staticmethod
 	def getValidBaseTags() -> [str]:
@@ -161,12 +161,3 @@ def isNewerBaseTag(older: str, newer: str) -> bool:
 		Determines if the base tag `newer` is chronologically newer than the base tag `older`
 		'''
 		return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older)
-
-	@staticmethod
-	def supportsProcessIsolation() -> bool:
-		'''
-		Determines whether the Windows host system supports process isolation for containers
-
-		@see https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container
-		'''
-		return WindowsUtils.isWindowsServer() or WindowsUtils.getWindowsBuild() >= 17763

ue4docker/setup_cmd.py

@@ -94,49 +94,6 @@ def _setupWindowsServer():
 
 	else:
 		print('Firewall rule for credential endpoint is already configured.')
-
-	# Determine if the host system is Windows Server Core and lacks the required DLL files for building our containers
-	hostRelease = WindowsUtils.getWindowsRelease()
-	requiredDLLs = WindowsUtils.requiredHostDlls(hostRelease)
-	dllDir = os.path.join(os.environ['SystemRoot'], 'System32')
-	existing = [dll for dll in requiredDLLs if os.path.exists(os.path.join(dllDir, dll))]
-	if len(existing) != len(requiredDLLs):
-
-		# Determine if we can extract DLL files from the full Windows base image (version 1809 and newer only)
-		tags = requests.get('https://mcr.microsoft.com/v2/windows/tags/list').json()['tags']
-		if hostRelease in tags:
-
-			# Pull the full Windows base image with the appropriate tag if it does not already exist
-			image = 'mcr.microsoft.com/windows:{}'.format(hostRelease)
-			print('Pulling full Windows base image "{}"...'.format(image))
-			subprocess.run(['docker', 'pull', image], check=True)
-
-			# Start a container from which we will copy the DLL files, bind-mounting our DLL destination directory
-			print('Starting a container to copy DLL files from...')
-			mountPath = 'C:\\dlldir'
-			container = DockerUtils.start(
-				image,
-				['timeout', '/t', '99999', '/nobreak'],
-				mounts = [docker.types.Mount(mountPath, dllDir, 'bind')],
-				stdin_open = True,
-				tty = True,
-				remove = True
-			)
-
-			# Copy the DLL files to the host
-			print('Copying DLL files to the host system...')
-			DockerUtils.execMultiple(container, [['xcopy', '/y', os.path.join(dllDir, dll), mountPath + '\\'] for dll in requiredDLLs])
-
-			# Stop the container
-			print('Stopping the container...')
-			container.stop()
-
-		else:
-			print('The following DLL files will need to be manually copied into {}:'.format(dllDir))
-			print('\n'.join(['- {}'.format(dll) for dll in requiredDLLs if dll not in existing]))
-
-	else:
-		print('All required DLL files are already present on the host system.')
 
 def setup():