Skip to content

Commit

Permalink
[Directories.py] Implement planned changes
Browse files Browse the repository at this point in the history 
- Improve the efficiency of the resolveFilename() method by eliminating non-existent directories from the list.
- Tidy up some code.
- Correct and/or improve some comments.

Developed by @jbleyel with a little help from @IanSav.
  • Loading branch information
@IanSav
IanSav authored Jan 16, 2025
1 parent fa5f092 commit 2c4698d
Showing 1 changed file with 109 additions and 93 deletions.
202 changes: 109 additions & 93 deletions lib/python/Tools/Directories.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
# Planned changes:
# Improve resolveFilename efficiency by not constantly processing directories in resolveLists known not to exist.
# Add callback for skin changes to rebuild the resolveLists.

from errno import ENOENT, EXDEV
from os import F_OK, R_OK, W_OK, access, chmod, link, listdir, makedirs, mkdir, readlink, remove, rename, rmdir, sep, stat, statvfs, symlink, utime, walk
from os.path import basename, dirname, exists, getsize, isdir, isfile, islink, join, normpath, splitext
Expand All @@ -19,7 +15,7 @@
DEFAULT_MODULE_NAME = __name__.split(".")[-1]

forceDebug = eGetEnigmaDebugLvl() > 4
pathExists = exists
pathExists = exists # This is needed for old plugins.

SCOPE_HOME = 0 # DEBUG: Not currently used in Enigma2.
SCOPE_LANGUAGE = 1
Expand Down Expand Up @@ -62,7 +58,7 @@
# ${datadir} = /usr/share
#
defaultPaths = {
SCOPE_HOME: ("", PATH_DONTCREATE), # User home directory
SCOPE_HOME: ("", PATH_DONTCREATE), # User home directory.
SCOPE_LANGUAGE: (eEnv.resolve("${datadir}/enigma2/po/"), PATH_DONTCREATE),
SCOPE_KEYMAPS: (eEnv.resolve("${datadir}/keymaps/"), PATH_CREATE),
SCOPE_METADIR: (eEnv.resolve("${datadir}/meta/"), PATH_CREATE),
Expand Down Expand Up @@ -97,7 +93,38 @@ def InitDefaultPaths():
resolveFilename(SCOPE_CONFIG)


skinResolveList = []
lcdskinResolveList = []
fontsResolveList = []


def clearResolveLists():
global skinResolveList, lcdskinResolveList, fontsResolveList
skinResolveList = []
lcdskinResolveList = []
fontsResolveList = []


def resolveFilename(scope, base="", path_prefix=None):
def addIfExists(paths):
return [path for path in paths if isdir(path)]

def checkPaths(resolveList, base):
# Disable png / svg interchange code for now. SVG files are very CPU intensive.
# baseList = [base]
# if base.endswith(".png"):
# baseList.append(f"{base[:-3]}svg")
# elif base.endswith(".svg"):
# baseList.append(f"{base[:-3]}png")
path = base
for item in resolveList:
# for base in baseList:
file = join(item, base)
if exists(file):
path = file
break
return path

if str(base).startswith(f"~{sep}"): # You can only use the ~/ if we have a prefix directory.
if path_prefix:
base = join(path_prefix, base[2:])
Expand All @@ -109,7 +136,7 @@ def resolveFilename(scope, base="", path_prefix=None):
print(f"[Directories] Error: Invalid scope={scope} provided to resolveFilename!")
return None
path, flag = defaultPaths[scope] # Ensure that the defaultPath directory that should exist for this scope does exist.
if flag == PATH_CREATE and not pathExists(path):
if flag == PATH_CREATE and not exists(path):
try:
makedirs(path)
except OSError as err:
Expand All @@ -121,25 +148,10 @@ def resolveFilename(scope, base="", path_prefix=None):
base = data[0]
suffix = data[1]
path = base

def itemExists(resolveList, base):
# Disable png / svg interchange code for now. SVG files are very CPU intensive.
# baseList = [base]
# if base.endswith(".png"):
# baseList.append(f"{base[:-3]}svg")
# elif base.endswith(".svg"):
# baseList.append(f"{base[:-3]}png")
for item in resolveList:
# for base in baseList:
file = join(item, base)
if pathExists(file):
return file
return base

if base == "": # If base is "" then set path to the scope. Otherwise use the scope to resolve the base filename.
path, flags = defaultPaths.get(scope)
if scope == SCOPE_GUISKIN: # If the scope is SCOPE_GUISKIN append the current skin to the scope path.
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
path = join(path, skin)
elif scope in (SCOPE_PLUGIN_ABSOLUTE, SCOPE_PLUGIN_RELATIVE):
Expand All @@ -151,60 +163,64 @@ def itemExists(resolveList, base):
if len(pluginCode) > 2:
path = join(plugins, pluginCode[0], pluginCode[1])
elif scope == SCOPE_GUISKIN:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.primary_skin.value)
resolveList = [
join(scopeConfig, skin),
join(scopeConfig, "skin_common"),
scopeConfig, # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
join(scopeGUISkin, skin),
join(scopeGUISkin, f"skin_fallback_{getDesktop(0).size().height()}"),
join(scopeGUISkin, "skin_default"),
scopeGUISkin # Can we deprecate top level of SCOPE_GUISKIN directory to allow a clean up?
]
path = itemExists(resolveList, base)
global skinResolveList
if not skinResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
skinResolveList = addIfExists([
join(scopeConfig, skin),
join(scopeConfig, "skin_common"),
join(scopeGUISkin, skin),
join(scopeGUISkin, f"skin_fallback_{getDesktop(0).size().height()}"),
join(scopeGUISkin, "skin_default"),
scopeGUISkin # Deprecate top level of SCOPE_GUISKIN directory to allow a clean up.
])
path = checkPaths(skinResolveList, base)
elif scope == SCOPE_LCDSKIN:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else ""
resolveList = [
join(scopeConfig, "display", skin),
join(scopeConfig, "display", "skin_common"),
scopeConfig, # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
join(scopeLCDSkin, skin),
join(scopeLCDSkin, f"skin_fallback_{getDesktop(1).size().height()}"),
join(scopeLCDSkin, "skin_default"),
scopeLCDSkin # Can we deprecate top level of SCOPE_LCDSKIN directory to allow a clean up?
]
path = itemExists(resolveList, base)
global lcdskinResolveList
if not lcdskinResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else ""
lcdskinResolveList = addIfExists([
join(scopeConfig, "display", skin),
join(scopeConfig, "display", "skin_common"),
join(scopeLCDSkin, skin),
join(scopeLCDSkin, f"skin_fallback_{getDesktop(1).size().height()}"),
join(scopeLCDSkin, "skin_default"),
scopeLCDSkin # Deprecate top level of SCOPE_LCDSKIN directory to allow a clean up.
])
path = checkPaths(lcdskinResolveList, base)
elif scope == SCOPE_FONTS:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.primary_skin.value)
display = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else None
resolveList = [
join(scopeConfig, "fonts"),
join(scopeConfig, skin, "fonts"),
join(scopeConfig, skin)
]
if display:
resolveList.append(join(scopeConfig, "display", display, "fonts"))
resolveList.append(join(scopeConfig, "display", display))
resolveList.append(join(scopeConfig, "skin_common", "fonts"))
resolveList.append(join(scopeConfig, "skin_common"))
resolveList.append(scopeConfig) # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
resolveList.append(join(scopeGUISkin, skin, "fonts"))
resolveList.append(join(scopeGUISkin, skin))
resolveList.append(join(scopeGUISkin, "skin_default", "fonts"))
resolveList.append(join(scopeGUISkin, "skin_default"))
if display:
resolveList.append(join(scopeLCDSkin, display, "fonts"))
resolveList.append(join(scopeLCDSkin, display))
resolveList.append(join(scopeLCDSkin, "skin_default", "fonts"))
resolveList.append(join(scopeLCDSkin, "skin_default"))
resolveList.append(scopeFonts)
path = itemExists(resolveList, base)
global fontsResolveList
if not fontsResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
display = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else None
resolveList = [
join(scopeConfig, "fonts"),
join(scopeConfig, skin, "fonts"),
join(scopeConfig, skin)
]
if display:
resolveList.append(join(scopeConfig, "display", display, "fonts"))
resolveList.append(join(scopeConfig, "display", display))
resolveList.append(join(scopeConfig, "skin_common", "fonts"))
resolveList.append(join(scopeConfig, "skin_common"))
resolveList.append(join(scopeGUISkin, skin, "fonts"))
resolveList.append(join(scopeGUISkin, skin))
resolveList.append(join(scopeGUISkin, "skin_default", "fonts"))
resolveList.append(join(scopeGUISkin, "skin_default"))
if display:
resolveList.append(join(scopeLCDSkin, display, "fonts"))
resolveList.append(join(scopeLCDSkin, display))
resolveList.append(join(scopeLCDSkin, "skin_default", "fonts"))
resolveList.append(join(scopeLCDSkin, "skin_default"))
resolveList.append(scopeFonts)
fontsResolveList = addIfExists(resolveList)
path = checkPaths(fontsResolveList, base)
elif scope == SCOPE_PLUGIN:
file = join(scopePlugins, base)
if pathExists(file):
if exists(file):
path = file
elif scope in (SCOPE_PLUGIN_ABSOLUTE, SCOPE_PLUGIN_RELATIVE):
callingCode = normpath(getframe(1).f_code.co_filename)
Expand Down Expand Up @@ -234,7 +250,7 @@ def fileReadLine(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False
line = fd.read().strip().replace("\0", "")
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
if err.errno != ENOENT: # No such file or directory.
print(f"[{source}] Error {err.errno}: Unable to read a line from file '{filename}'! ({err.strerror})")
line = default
msg = "Default"
Expand Down Expand Up @@ -271,7 +287,7 @@ def fileReadLines(filename, default=None, source=DEFAULT_MODULE_NAME, debug=Fals
lines = fd.read().splitlines()
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
if err.errno != ENOENT: # No such file or directory.
print(f"[{source}] Error {err.errno}: Unable to read lines from file '{filename}'! ({err.strerror})")
lines = default
msg = "Default"
Expand Down Expand Up @@ -317,7 +333,7 @@ def fileReadXML(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False)
except Exception as err:
print(f"[{source}] Error: Unable to parse data in '{filename}' - '{err}'!")
except OSError as err:
if err.errno == ENOENT: # ENOENT - No such file or directory.
if err.errno == ENOENT: # No such file or directory.
print(f"[{source}] Warning: File '{filename}' does not exist!")
else:
print("[%s] Error %d: Opening file '%s'! (%s)" % (source, err.errno, filename, err.strerror))
Expand All @@ -339,13 +355,13 @@ def fileReadXML(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False)


def defaultRecordingLocation(candidate=None):
if candidate and pathExists(candidate):
if candidate and exists(candidate):
return candidate
try:
path = readlink("/hdd") # First, try whatever /hdd points to, or /media/hdd.
except OSError as err:
path = "/media/hdd"
if not pathExists(path): # Find the largest local disk.
if not exists(path): # Find the largest local disk.
from Components import Harddisk
mounts = [mount for mount in Harddisk.getProcMounts() if mount[1].startswith("/media/")]
path = bestRecordingLocation([mount for mount in mounts if mount[0].startswith("/dev/")]) # Search local devices first, use the larger one.
Expand Down Expand Up @@ -383,9 +399,9 @@ def getRecordingFilename(basename, dirname=None):
character = "_"
filename += character
# Max filename length for ext4 is 255 (minus 12 characters for .stream.meta)
# but must not truncate in the middle of a multi-byte utf8 character!
# So convert the truncation to unicode and back, ignoring errors, the
# result will be valid utf8 and so xml parsing will be OK.
# but must not truncate in the middle of a multi-byte UTF8 character!
# So convert the truncation to Unicode and back, ignoring errors, the
# result will be valid UTF8 and so XML parsing will be OK.
filename = filename[:243]
if dirname is None:
dirname = defaultRecordingLocation()
Expand Down Expand Up @@ -479,8 +495,8 @@ def copytree(src, dst, symlinks=False):
return copyTree(src, dst, symlinks=symlinks)


# Renames files or if source and destination are on different devices moves them in background
# input list of (source, destination)
# Renames files or if source and destination are on different devices moves them in the background.
# The input is a list of (source, destination).
#
def moveFiles(fileList):
errorFlag = False
Expand All @@ -490,7 +506,7 @@ def moveFiles(fileList):
rename(item[0], item[1])
movedList.append(item)
except OSError as err:
if err.errno == EXDEV: # EXDEV - Invalid cross-device link.
if err.errno == EXDEV: # Invalid cross-device link.
print("[Directories] Warning: Cannot rename across devices, trying slower move.")
from Tools.CopyFiles import moveFiles as extMoveFiles # OpenViX, OpenATV, Beyonwiz
# from Screens.CopyFiles import moveFiles as extMoveFiles # OpenPLi / OV
Expand Down Expand Up @@ -521,7 +537,7 @@ def comparePaths(leftPath, rightPath):
return True


# Returns a list of tuples containing pathname and filename matching the given pattern
# Returns a list of tuples containing pathname and filename matching the given pattern.
# Example-pattern: match all txt-files: ".*\.txt$"
#
def crawlDirectory(directory, pattern):
Expand Down Expand Up @@ -701,25 +717,25 @@ def sanitizeFilename(filename, maxlen=255): # 255 is max length in ext4 (and mo
"COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5",
"LPT6", "LPT7", "LPT8", "LPT9",
) # Reserved words on Windows
# Remove any blacklisted chars. Remove all charcters below code point 32. Normalize. Strip.
# Remove any blacklisted chars. Remove all characters below code point 32. Normalize. Strip.
filename = normalize("NFKD", "".join(c for c in filename if c not in blacklist and ord(c) > 31)).strip()
if all([x == "." for x in filename]) or filename in reserved: # if filename is a string of dots
filename = f"__{filename}"
# Most Unix file systems typically allow filenames of up to 255 bytes.
# However, the actual number of characters allowed can vary due to the
# representation of Unicode characters. Therefore length checks must
# be done in bytes, not unicode.
# be done in bytes, not Unicode.
#
# Also we cannot leave the byte truncate in the middle of a multi-byte
# utf8 character! So, convert to bytes, truncate then get back to unicode,
# ignoring errors along the way, the result will be valid unicode.
# Prioritise maintaining the complete extension if possible.
# UTF8 character! So, convert to bytes, truncate then get back to Unicode,
# ignoring errors along the way, the result will be valid Unicode.
# Prioritize maintaining the complete extension if possible.
# Any truncation of root or ext will be done at the end of the string
root, ext = splitext(filename.encode(encoding="utf-8", errors="ignore"))
root, ext = splitext(filename.encode(encoding="UTF-8", errors="ignore"))
if len(ext) > maxlen - (1 if root else 0): # leave at least one char for root if root
ext = ext[:maxlen - (1 if root else 0)]
# convert back to unicode, ignoring any incomplete utf8 multibyte chars
filename = root[:maxlen - len(ext)].decode(encoding="utf-8", errors="ignore") + ext.decode(encoding="utf-8", errors="ignore")
# convert back to Unicode, ignoring any incomplete UTF8 multi-byte chars
filename = root[:maxlen - len(ext)].decode(encoding="UTF-8", errors="ignore") + ext.decode(encoding="UTF-8", errors="ignore")
filename = filename.rstrip(". ") # Windows does not allow these at end
if not filename:
filename = "__"
Expand Down

12 comments on commit 2c4698d

@fairbird
Copy link
Contributor

@fairbird fairbird commented on 2c4698d Jan 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@IanSav @jbleyel
You need to apply this commit fairbird/enigma2-dreambox@27cb29f to fix the issue with reading symlink of .xml files that depend on some skin using with plugin
such as

Image

Image

Image

@jbleyel
Copy link
Contributor

@jbleyel jbleyel commented on 2c4698d Jan 28, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No .. we have removed the scan of /etc/enigma2 by purpose because this folder will be scanned for each file.

All skin files must be in

/usr/share/enigma2/<skin>/
/usr/share/enigma2/skin_fallback_<resolution>/
/usr/share/enigma2/skin_default/

or

/etc/enigma2/<skin>/
/etc/enigma2/skin_common/

and not in the /etc/enigma2/ root folder!

This is a very old issue.

@jbleyel
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You may have includes in your xml files and these are the problem.

@jbleyel
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar issue ->

#3508

@fairbird
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay.. Thank you
I will check it again .. But old method (scopeConfig) also need it some skins or the owners of these skins should be forced to change the settings of their skins, which I think is not desirable for them.

@jbleyel
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe .. but the performance impact of scopeConfig will affect all skins and all user.
We may find a compromise to support the old skins without loosing the performance improvements.

@jbleyel
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a compromise.

Directories.py.zip

@fairbird
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes ... After adding (skinResolveListXml) Back again skins/plugin work just fine !

Thank you

@jbleyel
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know ;-)
But this is also only a workaround/compromise.

@jbleyel
Copy link
Contributor

@jbleyel jbleyel commented on 2c4698d Jan 29, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a new update.
Please test.

Directories.py.zip

skin.py.zip

@fairbird
Copy link
Contributor

@fairbird fairbird commented on 2c4698d Jan 29, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please create new branch and add the new codes then I can cherry-pick the codes to test it.

Anyway .. Tested .. Nice codes (works fine) ..

@biko-73
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tested on old plugin method and it work great thanks
i test the new live update too on opatv 7.5.2 and 2 files (Directories.py, skin.py) work without any problem
Thanks openatv team and special thank to you jbleyel for your listening to user problem, make the fix and provide support

Please sign in to comment.