diff --git a/.github/workflows/build_appimage.yml b/.github/workflows/build_appimage.yml new file mode 100644 index 0000000..d550be6 --- /dev/null +++ b/.github/workflows/build_appimage.yml @@ -0,0 +1,26 @@ +name: build_appimage + +on: workflow_dispatch + +jobs: + + build_appimage: + name: build_appimage + + runs-on: 'ubuntu-20.04' + strategy: + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + - name: Build appimage + run: | + bash tools/build_appimage --version=${{ github.ref_name }} --jsonfile=tools/linux_libs.json + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + path: BeeRef*.appimage + retention-days: 5 diff --git a/.gitignore b/.gitignore index 6ff49a6..f0b0c63 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST +*.appimage +squashfs-root/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/beeref/__main__.py b/beeref/__main__.py index 4899c83..333feac 100755 --- a/beeref/__main__.py +++ b/beeref/__main__.py @@ -103,6 +103,7 @@ def main(): logger.info(f'Starting {constants.APPNAME} version {constants.VERSION}') logger.debug('System: %s', ' '.join(platform.uname())) logger.debug('Python: %s', platform.python_version()) + logger.debug('LD_LIBRARY_PATH: %s', os.environ.get('LD_LIBRARY_PATH')) settings = BeeSettings() logger.info(f'Using settings: {settings.fileName()}') logger.info(f'Logging to: {logfile_name()}') diff --git a/setup.cfg b/setup.cfg index e008106..0605e76 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,9 @@ +[flake8] +exclude = squashfs-root + [coverage:run] source = beeref [tool:pytest] +norecursedirs = squashfs-root addopts = --cov-report html --cov-config=setup.cfg \ No newline at end of file diff --git a/tools/build_appimage.py b/tools/build_appimage.py new file mode 100755 index 0000000..68f6cc1 --- /dev/null +++ b/tools/build_appimage.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 + +# Build the BeeRef appimage. Run from the git root directory. +# On github actions: +# ./tools/build_appimage --version=${{ github.ref_name }}\ +# --jsonfile=tools/linux_libs.json +# Locally: +# ./tools/build_appimage --version=0.3.3-dev --jsonfile=tools/linux_libs.json +# --skip-apt + + +import argparse +import json +import logging +import os +import shutil +import subprocess +from urllib.request import urlretrieve + + +parser = argparse.ArgumentParser( + description=('Create an appimage for BeeRef. ' + 'Run from the git root directory.')) +parser.add_argument( + '-v', '--version', + required=True, + help='BeeRef version number/tag for output file') +parser.add_argument( + '-j', '--jsonfile', + required=True, + help='Json with lib files and packages as generated by find_linux_libs') +parser.add_argument( + '--redownload', + default=False, + action='store_true', + help='Re-use downloaded files if present') +parser.add_argument( + '--skip-apt', + default=False, + action='store_true', + help='Skip apt install step') +parser.add_argument( + '-l', '--loglevel', + default='INFO', + choices=list(logging._nameToLevel.keys()), + help='log level for console output') + +args = parser.parse_args() + + +BEEVERSION = args.version.removeprefix('v') +APPIMAGE = 'python3.11.8-cp311-cp311-manylinux_2_28_x86_64.AppImage' +PYVER = '3.11' +logger = logging.getLogger(__name__) +logging.basicConfig(level=getattr(logging, args.loglevel)) + + +def run_command(*args, capture_output=False): + logger.info(f'Running command: {args}') + result = subprocess.run(args, capture_output=capture_output) + assert result.returncode == 0, f'Failed with exit code {result.returncode}' + + +def download_file(url, filename): + if not args.redownload and os.path.exists(filename): + logger.info(f'Found file: {filename}') + else: + logger.info(f'Downloading: {url}') + logger.info(f'Saving as: {filename}') + urlretrieve(url, filename=filename) + os.chmod(filename, 0o755) + + +url = ('https://github.com/niess/python-appimage/releases/download/' + f'python{PYVER}/{APPIMAGE}') +download_file(url, filename='python.appimage') + + +try: + shutil.rmtree('squashfs-root') +except FileNotFoundError: + pass +run_command('./python.appimage', '--appimage-extract', + capture_output=True) + +run_command('squashfs-root/usr/bin/pip', + 'install', + '.', + f'--target=squashfs-root/opt/python{PYVER}/lib/python{PYVER}/') + +logger.info(f'Reading from: {args.jsonfile}') +with open(args.jsonfile, 'r') as f: + data = json.loads(f.read()) +libs = data['libs'] +packages = data['packages'] +excludes = data['excludes'] +paths = set() + +if not args.skip_apt: + run_command('sudo', 'apt', 'install', *packages) + +logger.info('Copying .so files to appimage...') + +existing_files = [] +for root, subdirs, files in os.walk('squashfs-root'): + existing_files.extend(files) + +for lib in libs: + if os.path.basename(lib) in existing_files: + logger.debug(f'Skipping {lib} (already in appimage)') + continue + if os.path.basename(lib) in excludes: + logger.debug(f'Skipping {lib} (excluded)') + continue + paths.add(os.path.dirname(lib)) + if os.path.exists(lib): + filename = lib + else: + filename, _ = os.path.splitext(lib) + dest = f'squashfs-root{filename}' + os.makedirs(os.path.dirname(dest), exist_ok=True) + logger.debug(f'Copying {filename} to {dest}') + shutil.copyfile(filename, f'squashfs-root{filename}') + + +logger.info('Writing run script...') +# Adapted from usr/bin/python3.x in the python appimage +os.remove('squashfs-root/AppRun') +# ^ This is only a symlink to usr/bin/python3.x + +paths = [ + '/usr/lib', # The libs that come with the python appimage ar in /usr/lib +] + list(paths) +ld_paths = ['${APPDIR}' + p for p in paths] + ['${LD_LIBRARY_PATH}'] +ld_paths = ':'.join(ld_paths) +logger.debug(f'LD_LIBRARY_PATH: {ld_paths}') + +content = """#! /bin/bash + +# If running from an extracted image, then export ARGV0 and APPDIR +if [ -z "${APPIMAGE}" ]; then + export ARGV0="$0" + + self=$(readlink -f -- "$0") # Protect spaces (issue 55) + here="${self%/*}" + tmp="${here%/*}" + export APPDIR="${tmp%/*}" +fi + +# Resolve the calling command (preserving symbolic links). +export APPIMAGE_COMMAND=$(command -v -- "$ARGV0") + +# Export SSL certificate +export SSL_CERT_FILE="${APPDIR}/opt/_internal/certs.pem" +""" +content += f'export LD_LIBRARY_PATH="{ld_paths}"\n' +content += f'"$APPDIR/opt/python{PYVER}/bin/python{PYVER}" -I -m beeref "$@"\n' + +with open('squashfs-root/AppRun', 'w') as f: + f.write(content) +os.chmod('squashfs-root/AppRun', 0o755) + +url = ('https://github.com/AppImage/AppImageKit/releases/download/' + 'continuous/appimagetool-x86_64.AppImage') + +download_file(url, filename='appimagetool.appimage') +run_command('./appimagetool.appimage', + 'squashfs-root', + f'BeeRef-{BEEVERSION}.appimage', + '--no-appstream') diff --git a/tools/find_linux_libs.py b/tools/find_linux_libs.py new file mode 100755 index 0000000..8de0eac --- /dev/null +++ b/tools/find_linux_libs.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +# Create JSON with Linux libs needed for BeeRef appimage + +import argparse +import json +import logging +import os +import pathlib +import re +import subprocess +import sys +from urllib import request + + +parser = argparse.ArgumentParser( + description=('Create JSON with Linux libs needed for BeeRef appimage')) +parser.add_argument( + 'pid', + nargs=1, + default=None, + help='PID of running BeeRef process') +parser.add_argument( + '-l', '--loglevel', + default='INFO', + choices=list(logging._nameToLevel.keys()), + help='log level for console output') +parser.add_argument( + '--jsonfile', + default='linux_libs.json', + help='JSON input/output file') +parser.add_argument( + '--check-appimage', + default=False, + action='store_true', + help='Check a running appimage process for missing libraries') + + +args = parser.parse_args() + + +def strip_minor_versions(path): + # foo2.so.2.1.1 -> foo2.so.2 + return re.sub('(.so.[0-9]*)[.0-9]*$', r'\1', path) + + +def what_links_to(path): + links = set() + dirname = os.path.dirname(path) + for filename in os.listdir(dirname): + filename = os.path.join(dirname, filename) + if (os.path.islink(filename) + and str(pathlib.Path(filename).resolve()) == path): + links.add(filename) + return sorted(links, key=len) + + +def is_lib(path): + return ('.so' in path + and os.path.expanduser('~') not in path + and 'python3' not in path + and 'mesa-diverted' not in path) + + +def iter_lsofoutput(output): + for line in output.splitlines(): + line = line.split() + if line[3] == 'mem': + path = line[-1] + if is_lib(path): + yield path + + +PID = args.pid[0] +logger = logging.getLogger(__name__) +logging.basicConfig(level=getattr(logging, args.loglevel)) + + +result = subprocess.run(('lsof', '-p', PID), capture_output=True) +assert result.returncode == 0, result.stderr +output = result.stdout.decode('utf-8') + + +if args.check_appimage: + logger.info('Checking appimage...') + errors = False + for lib in iter_lsofoutput(output): + if 'mount_BeeRef' not in lib: + print(f'Not in appimage: {lib}') + errors = True + if not errors: + print('No missing libs found.') + sys.exit() + + +libs = [] + +if os.path.exists(args.jsonfile): + logger.info(f'Reading from: {args.jsonfile}') + with open(args.jsonfile, 'r') as f: + data = json.loads(f.read()) + known_libs = data['libs'] + packages = set(data['packages']) +else: + logger.info(f'No file {args.jsonfile}; starting from scratch') + known_libs = [] + packages = set() + + +for lib in iter_lsofoutput(output): + links = what_links_to(lib) + if len(links) == 1: + lib = links[0] + else: + logger.warning(f'Double check: {lib} {links}') + lib = links[0] + if lib in known_libs: + logger.debug(f'Found known lib: {lib}') + else: + logger.debug(f'Found unknown lib: {lib}') + libs.append(lib) + + +for lib in libs: + result = subprocess.run(('apt-file', 'search', lib), capture_output=True) + if result.returncode != 0: + logger.warning(f'Fix manually: {lib}') + continue + output = result.stdout.decode('utf-8') + pkgs = set() + for line in output.splitlines(): + pkg = line.split(': ')[0] + if not (pkg.endswith('-dev') or pkg.endswith('-dbg')): + pkgs.add(pkg) + if len(pkgs) == 1: + pkg = pkgs.pop() + logger.debug(f'Found package: {pkg}') + packages.add(pkg) + else: + logger.warning(f'Fix manually: {lib}') + + +# Find the libs we shouldn't include in the appimage +with request.urlopen( + 'https://raw.githubusercontent.com/AppImageCommunity/pkg2appimage/' + 'master/excludelist') as f: + response = f.read().decode() + + +exclude_masterlist = set() +for line in response.splitlines(): + if not line or line.startswith('#'): + continue + line = line.split()[0] + line = strip_minor_versions(line) + exclude_masterlist.add(line) + +excludes = [] +for ex in exclude_masterlist: + for lib in (libs + known_libs): + if lib.endswith(ex): + excludes.append(ex) + continue + + +logger.info(f'Writing to: {args.jsonfile}') +with open(args.jsonfile, 'w') as f: + data = {'libs': sorted(libs + known_libs), + 'packages': sorted(packages), + 'excludes': sorted(excludes)} + f.write(json.dumps(data, indent=4)) diff --git a/tools/linux_libs.json b/tools/linux_libs.json new file mode 100644 index 0000000..492a5df --- /dev/null +++ b/tools/linux_libs.json @@ -0,0 +1,238 @@ +{ + "libs": [ + "/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + "/lib/x86_64-linux-gnu/libbz2.so.1", + "/lib/x86_64-linux-gnu/libc.so.6", + "/lib/x86_64-linux-gnu/libcom_err.so.2", + "/lib/x86_64-linux-gnu/libdbus-1.so.3", + "/lib/x86_64-linux-gnu/libdl.so.2", + "/lib/x86_64-linux-gnu/libexpat.so.1", + "/lib/x86_64-linux-gnu/libgcc_s.so.1", + "/lib/x86_64-linux-gnu/libgpg-error.so.0", + "/lib/x86_64-linux-gnu/libkeyutils.so.1", + "/lib/x86_64-linux-gnu/liblzma.so.5", + "/lib/x86_64-linux-gnu/libm.so.6", + "/lib/x86_64-linux-gnu/libpcre.so.3", + "/lib/x86_64-linux-gnu/libpthread.so.0", + "/lib/x86_64-linux-gnu/libresolv.so.2", + "/lib/x86_64-linux-gnu/librt.so.1", + "/lib/x86_64-linux-gnu/libselinux.so.1", + "/lib/x86_64-linux-gnu/libutil.so.1", + "/lib/x86_64-linux-gnu/libz.so.1", + "/usr/lib/x86_64-linux-gnu/gio/modules/libgvfsdbus.so", + "/usr/lib/x86_64-linux-gnu/gtk-3.0/modules/libcanberra-gtk-module.so", + "/usr/lib/x86_64-linux-gnu/gvfs/libgvfscommon.so", + "/usr/lib/x86_64-linux-gnu/libGLX.so.0", + "/usr/lib/x86_64-linux-gnu/libGLdispatch.so.0", + "/usr/lib/x86_64-linux-gnu/libX11-xcb.so.1", + "/usr/lib/x86_64-linux-gnu/libX11.so.6", + "/usr/lib/x86_64-linux-gnu/libXau.so.6", + "/usr/lib/x86_64-linux-gnu/libXcomposite.so.1", + "/usr/lib/x86_64-linux-gnu/libXcursor.so.1", + "/usr/lib/x86_64-linux-gnu/libXdamage.so.1", + "/usr/lib/x86_64-linux-gnu/libXdmcp.so.6", + "/usr/lib/x86_64-linux-gnu/libXext.so.6", + "/usr/lib/x86_64-linux-gnu/libXfixes.so.3", + "/usr/lib/x86_64-linux-gnu/libXi.so.6", + "/usr/lib/x86_64-linux-gnu/libXinerama.so.1", + "/usr/lib/x86_64-linux-gnu/libXrandr.so.2", + "/usr/lib/x86_64-linux-gnu/libXrender.so.1", + "/usr/lib/x86_64-linux-gnu/libatk-1.0.so.0", + "/usr/lib/x86_64-linux-gnu/libatk-bridge-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libatspi.so.0", + "/usr/lib/x86_64-linux-gnu/libblkid.so.1", + "/usr/lib/x86_64-linux-gnu/libbrotlicommon.so.1", + "/usr/lib/x86_64-linux-gnu/libbrotlidec.so.1", + "/usr/lib/x86_64-linux-gnu/libbsd.so.0", + "/usr/lib/x86_64-linux-gnu/libcairo-gobject.so.2", + "/usr/lib/x86_64-linux-gnu/libcairo.so.2", + "/usr/lib/x86_64-linux-gnu/libcanberra-gtk3.so.0", + "/usr/lib/x86_64-linux-gnu/libcanberra.so.0", + "/usr/lib/x86_64-linux-gnu/libcrypto.so", + "/usr/lib/x86_64-linux-gnu/libdatrie.so.1", + "/usr/lib/x86_64-linux-gnu/libepoxy.so.0", + "/usr/lib/x86_64-linux-gnu/libffi.so.7", + "/usr/lib/x86_64-linux-gnu/libfontconfig.so.1", + "/usr/lib/x86_64-linux-gnu/libfreetype.so.6", + "/usr/lib/x86_64-linux-gnu/libfribidi.so.0", + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20", + "/usr/lib/x86_64-linux-gnu/libgdk-3.so.0", + "/usr/lib/x86_64-linux-gnu/libgdk_pixbuf-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libgio-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libglib-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libgmodule-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libgobject-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libgraphite2.so.3", + "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", + "/usr/lib/x86_64-linux-gnu/libgthread-2.0.so.0", + "/usr/lib/x86_64-linux-gnu/libgtk-3.so.0", + "/usr/lib/x86_64-linux-gnu/libharfbuzz.so.0", + "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", + "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", + "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", + "/usr/lib/x86_64-linux-gnu/libltdl.so.7", + "/usr/lib/x86_64-linux-gnu/liblz4.so.1", + "/usr/lib/x86_64-linux-gnu/libmd.so.0", + "/usr/lib/x86_64-linux-gnu/libmount.so.1", + "/usr/lib/x86_64-linux-gnu/libogg.so.0", + "/usr/lib/x86_64-linux-gnu/libpango-1.0.so.0", + "/usr/lib/x86_64-linux-gnu/libpangocairo-1.0.so.0", + "/usr/lib/x86_64-linux-gnu/libpangoft2-1.0.so.0", + "/usr/lib/x86_64-linux-gnu/libpcre2-8.so.0", + "/usr/lib/x86_64-linux-gnu/libpixman-1.so.0", + "/usr/lib/x86_64-linux-gnu/libpng16.so.16", + "/usr/lib/x86_64-linux-gnu/libsqlite3.so", + "/usr/lib/x86_64-linux-gnu/libssl.so", + "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0", + "/usr/lib/x86_64-linux-gnu/libtdb.so.1", + "/usr/lib/x86_64-linux-gnu/libthai.so.0", + "/usr/lib/x86_64-linux-gnu/libuuid.so.1", + "/usr/lib/x86_64-linux-gnu/libvorbis.so.0", + "/usr/lib/x86_64-linux-gnu/libvorbisfile.so.3", + "/usr/lib/x86_64-linux-gnu/libwayland-client.so.0", + "/usr/lib/x86_64-linux-gnu/libwayland-cursor.so.0", + "/usr/lib/x86_64-linux-gnu/libwayland-egl.so.1", + "/usr/lib/x86_64-linux-gnu/libxcb-cursor.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-icccm.so.4", + "/usr/lib/x86_64-linux-gnu/libxcb-image.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-keysyms.so.1", + "/usr/lib/x86_64-linux-gnu/libxcb-randr.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-render-util.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-render.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-shape.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-shm.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-sync.so.1", + "/usr/lib/x86_64-linux-gnu/libxcb-util.so.1", + "/usr/lib/x86_64-linux-gnu/libxcb-xfixes.so.0", + "/usr/lib/x86_64-linux-gnu/libxcb-xkb.so.1", + "/usr/lib/x86_64-linux-gnu/libxcb.so.1", + "/usr/lib/x86_64-linux-gnu/libxkbcommon-x11.so.0", + "/usr/lib/x86_64-linux-gnu/libxkbcommon.so.0", + "/usr/lib/x86_64-linux-gnu/libzstd.so.1" + ], + "packages": [ + "gvfs", + "gvfs-libs", + "libatk-bridge2.0-0", + "libatk1.0-0", + "libatspi2.0-0", + "libblkid1", + "libbrotli1", + "libbsd0", + "libbz2-1.0", + "libc6", + "libcairo-gobject2", + "libcairo2", + "libcanberra-gtk3-0", + "libcanberra-gtk3-module", + "libcanberra0", + "libcom-err2", + "libdatrie1", + "libdbus-1-3", + "libepoxy0", + "libexpat1", + "libffi7", + "libfontconfig1", + "libfreetype6", + "libfribidi0", + "libgcc-s1", + "libgcrypt20", + "libgdk-pixbuf-2.0-0", + "libglib2.0-0", + "libglvnd0", + "libglx0", + "libgpg-error0", + "libgraphite2-3", + "libgssapi-krb5-2", + "libgtk-3-0", + "libharfbuzz0b", + "libk5crypto3", + "libkeyutils1", + "libkrb5-3", + "libkrb5support0", + "libltdl7", + "liblz4-1", + "liblzma5", + "libmd0", + "libmount1", + "libogg0", + "libpango-1.0-0", + "libpangocairo-1.0-0", + "libpangoft2-1.0-0", + "libpcre2-8-0", + "libpcre3", + "libpixman-1-0", + "libpng16-16", + "libselinux1", + "libsqlite3-0", + "libssl1.1", + "libstdc++6", + "libsystemd0", + "libtdb1", + "libthai0", + "libuuid1", + "libvorbis0a", + "libvorbisfile3", + "libwayland-client0", + "libwayland-cursor0", + "libwayland-egl1", + "libx11-6", + "libx11-xcb1", + "libxau6", + "libxcb-cursor0", + "libxcb-icccm4", + "libxcb-image0", + "libxcb-keysyms1", + "libxcb-randr0", + "libxcb-render-util0", + "libxcb-render0", + "libxcb-shape0", + "libxcb-shm0", + "libxcb-sync1", + "libxcb-util1", + "libxcb-xfixes0", + "libxcb-xkb1", + "libxcb1", + "libxcomposite1", + "libxcursor1", + "libxdamage1", + "libxdmcp6", + "libxext6", + "libxfixes3", + "libxi6", + "libxinerama1", + "libxkbcommon-x11-0", + "libxkbcommon0", + "libxrandr2", + "libxrender1", + "libzstd1", + "zlib1g" + ], + "excludes": [ + "ld-linux-x86-64.so.2", + "libGLX.so.0", + "libGLdispatch.so.0", + "libX11-xcb.so.1", + "libX11.so.6", + "libc.so.6", + "libcom_err.so.2", + "libdl.so.2", + "libexpat.so.1", + "libfontconfig.so.1", + "libfreetype.so.6", + "libfribidi.so.0", + "libgcc_s.so.1", + "libgpg-error.so.0", + "libharfbuzz.so.0", + "libm.so.6", + "libpthread.so.0", + "libresolv.so.2", + "librt.so.1", + "libstdc++.so.6", + "libthai.so.0", + "libutil.so.1", + "libxcb.so.1", + "libz.so.1" + ] +}