Add appimage build script and github action

.github/workflows/build_appimage.yml

@@ -0,0 +1,26 @@
+name: build_appimage
+
+on: workflow_dispatch
+
+jobs:
+
+  build_appimage:
+    name: build_appimage
+
+    runs-on: 'ubuntu-20.04'
+    strategy:
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Build appimage
+        run: |
+          bash tools/build_appimage --version=${{ github.ref_name }} --jsonfile=tools/linux_libs.json
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          path: BeeRef*.appimage
+          retention-days: 5

.gitignore

@@ -28,6 +28,8 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+*.appimage
+squashfs-root/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

beeref/__main__.py

@@ -103,6 +103,7 @@ def main():
     logger.info(f'Starting {constants.APPNAME} version {constants.VERSION}')
     logger.debug('System: %s', ' '.join(platform.uname()))
     logger.debug('Python: %s', platform.python_version())
+    logger.debug('LD_LIBRARY_PATH: %s', os.environ.get('LD_LIBRARY_PATH'))
     settings = BeeSettings()
     logger.info(f'Using settings: {settings.fileName()}')
     logger.info(f'Logging to: {logfile_name()}')

setup.cfg

@@ -1,5 +1,9 @@
+[flake8]
+exclude = squashfs-root
+
 [coverage:run]
 source = beeref
 
 [tool:pytest]
+norecursedirs = squashfs-root
 addopts = --cov-report html --cov-config=setup.cfg

tools/build_appimage.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+
+# Build the BeeRef appimage. Run from the git root directory.
+# On github actions:
+#   ./tools/build_appimage --version=${{ github.ref_name }}\
+#      --jsonfile=tools/linux_libs.json
+# Locally:
+#   ./tools/build_appimage --version=0.3.3-dev --jsonfile=tools/linux_libs.json
+#      --skip-apt
+
+
+import argparse
+import json
+import logging
+import os
+import shutil
+import subprocess
+from urllib.request import urlretrieve
+
+
+parser = argparse.ArgumentParser(
+    description=('Create an appimage for BeeRef. '
+                 'Run from the git root directory.'))
+parser.add_argument(
+    '-v', '--version',
+    required=True,
+    help='BeeRef version number/tag for output file')
+parser.add_argument(
+    '-j', '--jsonfile',
+    required=True,
+    help='Json with lib files and packages as generated by find_linux_libs')
+parser.add_argument(
+    '--redownload',
+    default=False,
+    action='store_true',
+    help='Re-use downloaded files if present')
+parser.add_argument(
+    '--skip-apt',
+    default=False,
+    action='store_true',
+    help='Skip apt install step')
+parser.add_argument(
+    '-l', '--loglevel',
+    default='INFO',
+    choices=list(logging._nameToLevel.keys()),
+    help='log level for console output')
+
+args = parser.parse_args()
+
+
+BEEVERSION = args.version.removeprefix('v')
+APPIMAGE = 'python3.11.8-cp311-cp311-manylinux_2_28_x86_64.AppImage'
+PYVER = '3.11'
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=getattr(logging, args.loglevel))
+
+
+def run_command(*args, capture_output=False):
+    logger.info(f'Running command: {args}')
+    result = subprocess.run(args, capture_output=capture_output)
+    assert result.returncode == 0, f'Failed with exit code {result.returncode}'
+
+
+def download_file(url, filename):
+    if not args.redownload and os.path.exists(filename):
+        logger.info(f'Found file: {filename}')
+    else:
+        logger.info(f'Downloading: {url}')
+        logger.info(f'Saving as: {filename}')
+        urlretrieve(url, filename=filename)
+    os.chmod(filename, 0o755)
+
+
+url = ('https://github.com/niess/python-appimage/releases/download/'
+       f'python{PYVER}/{APPIMAGE}')
+download_file(url, filename='python.appimage')
+
+
+try:
+    shutil.rmtree('squashfs-root')
+except FileNotFoundError:
+    pass
+run_command('./python.appimage', '--appimage-extract',
+            capture_output=True)
+
+run_command('squashfs-root/usr/bin/pip',
+            'install',
+            '.',
+            f'--target=squashfs-root/opt/python{PYVER}/lib/python{PYVER}/')
+
+logger.info(f'Reading from: {args.jsonfile}')
+with open(args.jsonfile, 'r') as f:
+    data = json.loads(f.read())
+libs = data['libs']
+packages = data['packages']
+excludes = data['excludes']
+paths = set()
+
+if not args.skip_apt:
+    run_command('sudo', 'apt', 'install', *packages)
+
+logger.info('Copying .so files to appimage...')
+
+existing_files = []
+for root, subdirs, files in os.walk('squashfs-root'):
+    existing_files.extend(files)
+
+for lib in libs:
+    if os.path.basename(lib) in existing_files:
+        logger.debug(f'Skipping {lib} (already in appimage)')
+        continue
+    if os.path.basename(lib) in excludes:
+        logger.debug(f'Skipping {lib} (excluded)')
+        continue
+    paths.add(os.path.dirname(lib))
+    if os.path.exists(lib):
+        filename = lib
+    else:
+        filename, _ = os.path.splitext(lib)
+    dest = f'squashfs-root{filename}'
+    os.makedirs(os.path.dirname(dest), exist_ok=True)
+    logger.debug(f'Copying {filename} to {dest}')
+    shutil.copyfile(filename, f'squashfs-root{filename}')
+
+
+logger.info('Writing run script...')
+# Adapted from usr/bin/python3.x in the python appimage
+os.remove('squashfs-root/AppRun')
+# ^ This is only a symlink to usr/bin/python3.x
+
+paths = [
+    '/usr/lib',   # The libs that come with the python appimage ar in /usr/lib
+] + list(paths)
+ld_paths = ['${APPDIR}' + p for p in paths] + ['${LD_LIBRARY_PATH}']
+ld_paths = ':'.join(ld_paths)
+logger.debug(f'LD_LIBRARY_PATH: {ld_paths}')
+
+content = """#! /bin/bash
+
+# If running from an extracted image, then export ARGV0 and APPDIR
+if [ -z "${APPIMAGE}" ]; then
+    export ARGV0="$0"
+
+    self=$(readlink -f -- "$0") # Protect spaces (issue 55)
+    here="${self%/*}"
+    tmp="${here%/*}"
+    export APPDIR="${tmp%/*}"
+fi
+
+# Resolve the calling command (preserving symbolic links).
+export APPIMAGE_COMMAND=$(command -v -- "$ARGV0")
+
+# Export SSL certificate
+export SSL_CERT_FILE="${APPDIR}/opt/_internal/certs.pem"
+"""
+content += f'export LD_LIBRARY_PATH="{ld_paths}"\n'
+content += f'"$APPDIR/opt/python{PYVER}/bin/python{PYVER}" -I -m beeref "$@"\n'
+
+with open('squashfs-root/AppRun', 'w') as f:
+    f.write(content)
+os.chmod('squashfs-root/AppRun', 0o755)
+
+url = ('https://github.com/AppImage/AppImageKit/releases/download/'
+       'continuous/appimagetool-x86_64.AppImage')
+
+download_file(url, filename='appimagetool.appimage')
+run_command('./appimagetool.appimage',
+            'squashfs-root',
+            f'BeeRef-{BEEVERSION}.appimage',
+            '--no-appstream')

tools/find_linux_libs.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+
+# Create JSON with Linux libs needed for BeeRef appimage
+
+import argparse
+import json
+import logging
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from urllib import request
+
+
+parser = argparse.ArgumentParser(
+    description=('Create JSON with Linux libs needed for BeeRef appimage'))
+parser.add_argument(
+    'pid',
+    nargs=1,
+    default=None,
+    help='PID of running BeeRef process')
+parser.add_argument(
+    '-l', '--loglevel',
+    default='INFO',
+    choices=list(logging._nameToLevel.keys()),
+    help='log level for console output')
+parser.add_argument(
+    '--jsonfile',
+    default='linux_libs.json',
+    help='JSON input/output file')
+parser.add_argument(
+    '--check-appimage',
+    default=False,
+    action='store_true',
+    help='Check a running appimage process for missing libraries')
+
+
+args = parser.parse_args()
+
+
+def strip_minor_versions(path):
+    # foo2.so.2.1.1 -> foo2.so.2
+    return re.sub('(.so.[0-9]*)[.0-9]*$', r'\1', path)
+
+
+def what_links_to(path):
+    links = set()
+    dirname = os.path.dirname(path)
+    for filename in os.listdir(dirname):
+        filename = os.path.join(dirname, filename)
+        if (os.path.islink(filename)
+                and str(pathlib.Path(filename).resolve()) == path):
+            links.add(filename)
+    return sorted(links, key=len)
+
+
+def is_lib(path):
+    return ('.so' in path
+            and os.path.expanduser('~') not in path
+            and 'python3' not in path
+            and 'mesa-diverted' not in path)
+
+
+def iter_lsofoutput(output):
+    for line in output.splitlines():
+        line = line.split()
+        if line[3] == 'mem':
+            path = line[-1]
+            if is_lib(path):
+                yield path
+
+
+PID = args.pid[0]
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=getattr(logging, args.loglevel))
+
+
+result = subprocess.run(('lsof', '-p', PID), capture_output=True)
+assert result.returncode == 0, result.stderr
+output = result.stdout.decode('utf-8')
+
+
+if args.check_appimage:
+    logger.info('Checking appimage...')
+    errors = False
+    for lib in iter_lsofoutput(output):
+        if 'mount_BeeRef' not in lib:
+            print(f'Not in appimage: {lib}')
+            errors = True
+    if not errors:
+        print('No missing libs found.')
+    sys.exit()
+
+
+libs = []
+
+if os.path.exists(args.jsonfile):
+    logger.info(f'Reading from: {args.jsonfile}')
+    with open(args.jsonfile, 'r') as f:
+        data = json.loads(f.read())
+    known_libs = data['libs']
+    packages = set(data['packages'])
+else:
+    logger.info(f'No file {args.jsonfile}; starting from scratch')
+    known_libs = []
+    packages = set()
+
+
+for lib in iter_lsofoutput(output):
+    links = what_links_to(lib)
+    if len(links) == 1:
+        lib = links[0]
+    else:
+        logger.warning(f'Double check: {lib} {links}')
+        lib = links[0]
+    if lib in known_libs:
+        logger.debug(f'Found known lib: {lib}')
+    else:
+        logger.debug(f'Found unknown lib: {lib}')
+        libs.append(lib)
+
+
+for lib in libs:
+    result = subprocess.run(('apt-file', 'search', lib), capture_output=True)
+    if result.returncode != 0:
+        logger.warning(f'Fix manually: {lib}')
+        continue
+    output = result.stdout.decode('utf-8')
+    pkgs = set()
+    for line in output.splitlines():
+        pkg = line.split(': ')[0]
+        if not (pkg.endswith('-dev') or pkg.endswith('-dbg')):
+            pkgs.add(pkg)
+    if len(pkgs) == 1:
+        pkg = pkgs.pop()
+        logger.debug(f'Found package: {pkg}')
+        packages.add(pkg)
+    else:
+        logger.warning(f'Fix manually: {lib}')
+
+
+# Find the libs we shouldn't include in the appimage
+with request.urlopen(
+        'https://raw.githubusercontent.com/AppImageCommunity/pkg2appimage/'
+        'master/excludelist') as f:
+    response = f.read().decode()
+
+
+exclude_masterlist = set()
+for line in response.splitlines():
+    if not line or line.startswith('#'):
+        continue
+    line = line.split()[0]
+    line = strip_minor_versions(line)
+    exclude_masterlist.add(line)
+
+excludes = []
+for ex in exclude_masterlist:
+    for lib in (libs + known_libs):
+        if lib.endswith(ex):
+            excludes.append(ex)
+            continue
+
+
+logger.info(f'Writing to: {args.jsonfile}')
+with open(args.jsonfile, 'w') as f:
+    data = {'libs': sorted(libs + known_libs),
+            'packages': sorted(packages),
+            'excludes': sorted(excludes)}
+    f.write(json.dumps(data, indent=4))