From 0d0e1e6fbd6d3a295f4313438ea29fdd353ee79b Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Sat, 2 Nov 2024 18:52:34 +0000 Subject: [PATCH] feat(deps): add support for Nix package manager This serves as an alternative to Homebrew. It should be much more stable and cause less headaches over time for automated builds. There should be no change to the end user experience of using the build script, as it should still work with and use Homebrew by default. Additionally, Nix provides older Apple SDK's, so builds run against macOS 11.x SDKs via Nix. This should in theory allow the resulting Emacs.app builds should be compatible with older macOS versions. Exactly how well that holds up in practice remains to be seen. Nix does support customizing the Apple SDK version, but I have not yet figured out how to do so via a flake, and it's not a priority at the moment, as the default v11 SDK is sufficient. --- .github/workflows/ci.yml | 16 +- .gitignore | 1 - .golangci.yml | 9 +- Gemfile | 7 - Gemfile.lock | 15 ++ Makefile | 4 +- README.md | 67 ++++- build-emacs-for-macos | 560 +++++++++++++++++++++++++++++---------- flake.lock | 61 +++++ flake.nix | 77 ++++++ go.mod | 2 +- go.sum | 1 + pkg/dmgbuild/license.go | 1 - pkg/osinfo/osinfo.go | 48 +++- pkg/plan/create.go | 9 +- pkg/sign/emacs.go | 2 +- 16 files changed, 692 insertions(+), 188 deletions(-) create mode 100644 Gemfile.lock create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f565f7..6955e01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,13 +8,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: - version: v1.55 + version: v1.61 env: VERBOSE: "true" @@ -23,9 +23,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: Check if mods are tidy run: make check-tidy @@ -34,9 +34,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: - go-version: "1.20" + go-version: "1.23" - name: Run tests run: make test env: diff --git a/.gitignore b/.gitignore index 1c59c21..1149c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ .DS_Store .envrc Formula/* -Gemfile.lock bin builds sources diff --git a/.golangci.yml b/.golangci.yml index 1e1671d..5081642 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,6 @@ linters-settings: gocyclo: min-complexity: 20 govet: - check-shadowing: true enable-all: true disable: - fieldalignment @@ -23,9 +22,9 @@ linters: disable-all: true enable: - bodyclose + - copyloopvar - dupl - errcheck - - exportloopref - funlen - gochecknoinits - goconst @@ -72,12 +71,12 @@ issues: - source: "`yaml:" linters: - lll - -run: - skip-dirs: + exclude-dirs: - builds - sources - tarballs + +run: timeout: 2m allow-parallel-runners: true modules-download-mode: readonly diff --git a/Gemfile b/Gemfile index b81a189..ac49709 100644 --- a/Gemfile +++ b/Gemfile @@ -3,10 +3,3 @@ source 'http://rubygems.org/' gem 'ruby-macho' - -group :development do - gem 'byebug' - gem 'rubocop' - gem 'rubocop-daemon' - gem 'solargraph', '~> 0.39.17' -end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..fdd9017 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,15 @@ +GEM + remote: http://rubygems.org/ + specs: + ruby-macho (4.1.0) + +PLATFORMS + arm64-darwin + ruby + x86_64-darwin + +DEPENDENCIES + ruby-macho + +BUNDLED WITH + 2.5.23 diff --git a/Makefile b/Makefile index 3c4b4bb..4a1d803 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,9 @@ bootstrap-ruby: bundle install bootstrap-brew: +ifndef IN_NIX_SHELL brew bundle --verbose +endif bootstrap-pip: $(PIP) install -r requirements-ci.txt @@ -69,7 +71,7 @@ $(TOOLDIR)/$(1): Makefile endef $(eval $(call tool,gofumpt,mvdan.cc/gofumpt@latest)) -$(eval $(call tool,golangci-lint,github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55)) +$(eval $(call tool,golangci-lint,github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61)) $(eval $(call tool,gomod,github.com/Helcaraxan/gomod@latest)) .PHONY: tools diff --git a/README.md b/README.md index 9be1227..e245639 100644 --- a/README.md +++ b/README.md @@ -36,18 +36,40 @@ The build produced does have some limitations: ## Requirements +Required with both Nix and Homebrew approaches: + - [Xcode](https://apps.apple.com/gb/app/xcode/id497799835?mt=12) -- [Homebrew](https://brew.sh/) -- Ruby 2.3.0 or later is needed to execute the build script itself. macOS comes - with Ruby, check your version with `ruby --version`. If it's too old, you can - install a newer version with: - ``` - brew install ruby - ``` -- All dependencies can all easily be installed by running: - ``` - make bootstrap - ``` + +### Nix + +The [Nix](https://nixos.org/) package manager is the preferred and most reliable +way to install all dependencies required to build Emacs, by way of a Nix flake +included in the project root. + +To install all required dependencies within the nix shell, run: + +``` +nix develop --command make bootstrap +``` + +### Homebrew + +If you do not have Nix installed, then the alternative way to manage and install +build-time dependencies is via [Homebrew](https://brew.sh/). + +Ruby 3.3.x or later is also needed to execute the build script. Earlier versions +may work, but are untested. Simplest way to install a recent Ruby version is via +Homebrew: + +``` +brew install ruby +``` + +And finally, to install all built-time dependencies, run: + +``` +make bootstrap +``` ## Status @@ -71,6 +93,24 @@ Nightly builds are built with GitHub Actions on GitHub-hosted runners, using ## Usage +### Nix + +Ensure [Flakes](https://nixos.wiki/wiki/Flakes) are enabled, and enter the flake +development environment with `nix develop`. Within this environment, you can +execute the `./build-emacs-for-macos --help` to get started. + +Or you can run the build script via `nix develop`: + +``` +nix develop --command ./build-emacs-for-macos --help +``` + +### Homebrew + +Run `make boostrap` to ensure all Ruby and Homebrew dependencies are installed. + +### Build Script + ``` Usage: ./build-emacs-for-macos [options] @@ -78,10 +118,13 @@ Branch, tag, and SHA are from the emacs-mirror/emacs/emacs Github repo, available here: https://github.com/emacs-mirror/emacs Options: + --info Print environment info and detected library paths, then exit + --preview Print preview details about build and exit. -j, --parallel COUNT Compile using COUNT parallel processes (detected: 16) --git-sha SHA Override detected git SHA of specified branch allowing builds of old commits + --[no-]use-nix Use Nix instead of Homebrew to find dependencies (default: enabled if EMACS_BUILD_USE_NIX is set to truthy value) --[no-]xwidgets Enable/disable XWidgets if supported (default: enabled) - --[no-]tree-sitter Enable/disable tree-sitter if supported (default: enabled) + --[no-]tree-sitter Enable/disable tree-sitter if supported(default: enabled) --[no-]native-comp Enable/disable native-comp (default: enabled if supported) --[no-]native-march Enable/disable -march=native CFLAG(default: disabled) --[no-]native-full-aot Enable/disable NATIVE_FULL_AOT / Ahead of Time compilation (default: disabled) diff --git a/build-emacs-for-macos b/build-emacs-for-macos index 8b3a7b3..c695c8e 100755 --- a/build-emacs-for-macos +++ b/build-emacs-for-macos @@ -96,17 +96,26 @@ class OS @version ||= OSVersion.new end + def self.sdk_version + @sdk_version ||= SDKVersion.new + end + def self.arch @arch ||= `uname -m`.strip end end -class OSVersion +class AbstractVersion + attr_reader :version + def initialize - @version = - `sw_vers -productVersion`.match( - /(?\d+)(?:\.(?\d+)(?:\.(?\d+))?)?/ - ) + @version = load_version.match( + /(?\d+)(?:\.(?\d+)(?:\.(?\d+))?)?/ + ) + end + + def load_version + raise NotImplementedError end def to_s @@ -126,6 +135,21 @@ class OSVersion end end +class OSVersion < AbstractVersion + def load_version + `sw_vers -productVersion`.strip + end +end + +class SDKVersion < AbstractVersion + def load_version + ENV.fetch( + 'MACOSX_DEPLOYMENT_TARGET', + `xcrun --show-sdk-version 2>/dev/null`.strip + ).strip + end +end + class Build include Output include System @@ -142,12 +166,12 @@ class Build @root_dir = root_dir @ref = ref || 'master' @options = options - @gcc_info = GccInfo.new - end + @gcc_info = GccInfo.new(use_nix: options[:use_nix]) - def build load_plan(options[:plan]) if options[:plan] + end + def build unless meta[:sha] && meta[:date] fatal 'Failed to get commit info from GitHub.' end @@ -167,7 +191,7 @@ class Build CSourcesEmbedder.new(app, @source_dir).embed LibEmbedder.new( app, - brew_dir, + [brew_dir, '/nix/store'], extra_libs, relink_eln_files: options[:relink_eln] ).embed @@ -177,6 +201,60 @@ class Build archive_build(build_dir) if options[:archive] end + def print_info + # Force-enable native-comp to ensure all env vars are setup. + options[:native_comp] = true + + puts YAML.dump( + { + 'os' => OS.version.to_s, + 'sdk' => OS.sdk_version.to_s, + 'arch' => OS.arch, + 'gcc' => { + 'root' => gcc_info.root_dir, + 'lib' => gcc_info.lib_dir, + 'darwin_lib' => gcc_info.darwin_lib_dir, + 'target_lib' => gcc_info.target_lib_dir, + 'target_darwin_lib' => gcc_info.target_darwin_lib_dir, + 'sanitized_target_darwin_lib_dir' => gcc_info.sanitized_target_darwin_lib_dir, + 'version' => gcc_info.major_version + }, + 'libgccjit' => { + 'root' => gcc_info.libgccjit_root_dir, + 'lib' => gcc_info.libgccjit_lib_dir, + 'version' => gcc_info.libgccjit_major_version + }, + 'env' => { + 'CC' => compile_env['CC'], + 'CFLAGS' => compile_env['CFLAGS']&.split, + 'LDFLAGS' => compile_env['LDFLAGS']&.split, + 'LIBRARY_PATH' => compile_env['LIBRARY_PATH']&.split(':'), + 'PKG_CONFIG_PATH' => compile_env['PKG_CONFIG_PATH']&.split(':'), + 'PATH' => compile_env['PATH']&.split(':') + } + } + ) + end + + def print_preview + puts YAML.dump( + { + 'build_name' => build_name, + 'emacs' => { + 'ref' => meta[:ref], + 'sha' => meta[:sha], + 'date' => meta[:date] + }, + 'os_version' => OS.version.to_s, + 'sdk_version' => OS.sdk_version.to_s, + 'arch' => OS.arch, + 'native_comp' => options[:native_comp], + 'gcc_version' => gcc_info.major_version, + 'libgccjit_version' => gcc_info.libgccjit_major_version + } + ) + end + private def load_plan(filename) @@ -217,11 +295,16 @@ class Build @github_src_repo ||= options[:github_src_repo] || DEFAULT_GITHUB_REPO end + def use_nix? + !!options[:use_nix] + end + def brew_dir @brew_dir ||= `brew --prefix`.chomp end def extra_libs + return [] if use_nix? return @extra_libs if @extra_libs libs = [ @@ -359,6 +442,11 @@ class Build def autogen FileUtils.cd(source_dir) do + if File.exist?('configure') + info 'configure script exists, skipping autogen.' + return + end + if File.exist?('autogen/copy_autogen') run_cmd 'autogen/copy_autogen' elsif File.exist?('autogen.sh') @@ -367,6 +455,135 @@ class Build end end + # rubocop:disable Naming/MethodName,Naming/VariableName + def env_CFLAGS + return @env_CFLAGS if @env_CFLAGS + + env = [] + + env << '-O2' + + if options[:native_comp] + env += [ + "-I#{File.join(gcc_info.root_dir, 'include')}", + "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}" + ] + end + + env << '-march=native' if options[:native_march] + + if options[:fd_setsize].respond_to?(:>=) && options[:fd_setsize] >= 1024 + env += [ + "-DFD_SETSIZE=#{options[:fd_setsize]}", + '-DDARWIN_UNLIMITED_SELECT' + ] + end + + if use_nix? && ENV['NIX_CFLAGS_COMPILE'] + env += ENV['NIX_CFLAGS_COMPILE'].split + end + + @env_CFLAGS = env + end + + def env_LDFLAGS + return @env_LDFLAGS if @env_LDFLAGS + + env = [] + + # Ensure library re-linking and code signing will work after building. + env << '-Wl,-headerpad_max_install_names' + + if options[:native_comp] + env += [ + "-L#{gcc_info.lib_dir}", + "-L#{gcc_info.darwin_lib_dir}", + "-L#{gcc_info.libgccjit_lib_dir}", + "-I#{File.join(gcc_info.root_dir, 'include')}", + "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}" + ] + end + + env += ENV['NIX_LDFLAGS'].split if use_nix? && ENV['NIX_LDFLAGS'] + + @env_LDFLAGS = env + end + + def env_LIBRARY_PATH + return @env_LIBRARY_PATH if @env_LIBRARY_PATH + + env = [] + + if options[:native_comp] + env += [ + gcc_info.lib_dir, + gcc_info.darwin_lib_dir, + gcc_info.libgccjit_lib_dir + ] + end + + env << '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib' + + @env_LIBRARY_PATH = env + end + + def env_PKG_CONFIG_PATH + return [] if use_nix? + + @env_PKG_CONFIG_PATH ||= [ + File.join(brew_dir, 'lib/pkgconfig'), + File.join(brew_dir, 'share/pkgconfig'), + File.join(brew_dir, 'opt/expat/lib/pkgconfig'), + File.join(brew_dir, 'opt/libxml2/lib/pkgconfig'), + File.join(brew_dir, 'opt/ncurses/lib/pkgconfig'), + File.join(brew_dir, 'opt/zlib/lib/pkgconfig'), + File.join( + brew_dir, + 'Homebrew/Library/Homebrew/os/mac/pkgconfig', + OS.version.to_s + ) + ] + end + + def env_PATH + return [] if use_nix? + + @env_PATH ||= [ + File.join(brew_dir, 'opt/make/libexec/gnubin'), + File.join(brew_dir, 'opt/coreutils/libexec/gnubin'), + File.join(brew_dir, 'opt/gnu-sed/libexec/gnubin'), + File.join(brew_dir, 'bin'), + File.join(brew_dir, 'opt/texinfo/bin') + ] + end + # rubocop:enable Naming/MethodName,Naming/VariableName + + def compile_env + return @compile_env if @compile_env + + env = { + 'CC' => use_nix? ? 'clang' : '/usr/bin/clang', + 'PATH' => [ + env_PATH, ENV.fetch('PATH', nil) + ].flatten.compact.reject(&:empty?).join(':'), + 'PKG_CONFIG_PATH' => [ + env_PKG_CONFIG_PATH, + ENV.fetch('PKG_CONFIG_PATH', nil) + ].flatten.compact.reject(&:empty?).join(':') + } + + if options[:native_comp] + env['CFLAGS'] = [env_CFLAGS, ENV.fetch('CFLAGS', nil)] + .flatten.compact.reject(&:empty?).join(' ') + env['LDFLAGS'] = [env_LDFLAGS, ENV.fetch('LDFLAGS', nil)] + .flatten.compact.reject(&:empty?).join(' ') + env['LIBRARY_PATH'] = [env_LIBRARY_PATH, ENV.fetch('LIBRARY_PATH', nil)] + .flatten.compact.reject(&:empty?).join(':') + end + + @compile_env = env + end + def compile_source(source) target = File.join(source, 'nextstep') emacs_app = File.join(target, 'Emacs.app') @@ -384,71 +601,9 @@ class Build info 'Compiling with native-comp enabled' verify_native_comp gcc_info.verify_libgccjit - - ENV['CFLAGS'] = [ - "-I#{File.join(gcc_info.root_dir, 'include')}", - "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}", - '-O2', - (options[:native_march] ? '-march=native' : nil), - ENV.fetch('CFLAGS', nil) - ].compact.join(' ') - - ENV['LDFLAGS'] = [ - "-L#{gcc_info.lib_dir}", - "-L#{gcc_info.darwin_lib_dir}", - "-L#{gcc_info.libgccjit_lib_dir}", - "-I#{File.join(gcc_info.root_dir, 'include')}", - "-I#{File.join(gcc_info.libgccjit_root_dir, 'include')}", - # Ensure library re-linking and code signing will work after building. - '-Wl,-headerpad_max_install_names', - ENV.fetch('LDFLAGS', nil) - ].compact.join(' ') - - ENV['LIBRARY_PATH'] = [ - gcc_info.lib_dir, - gcc_info.darwin_lib_dir, - gcc_info.libgccjit_lib_dir, - ENV.fetch('LIBRARY_PATH', nil) - ].compact.join(':') end - if options[:fd_setsize].respond_to?(:>=) && options[:fd_setsize] >= 1024 - ENV['CFLAGS'] = [ - "-DFD_SETSIZE=#{options[:fd_setsize]}", - '-DDARWIN_UNLIMITED_SELECT', - ENV.fetch('CFLAGS', nil) - ].compact.join(' ') - end - - ENV['CC'] = 'clang' - ENV['PKG_CONFIG_PATH'] = [ - File.join(brew_dir, 'lib/pkgconfig'), - File.join(brew_dir, 'share/pkgconfig'), - File.join(brew_dir, 'opt/expat/lib/pkgconfig'), - File.join(brew_dir, 'opt/libxml2/lib/pkgconfig'), - File.join(brew_dir, 'opt/ncurses/lib/pkgconfig'), - File.join(brew_dir, 'opt/zlib/lib/pkgconfig'), - File.join( - brew_dir, - 'Homebrew/Library/Homebrew/os/mac/pkgconfig', - OS.version.to_s - ), - ENV.fetch('PKG_CONFIG_PATH', nil) - ].compact.join(':') - - ENV['PATH'] = [ - File.join(brew_dir, 'opt/make/libexec/gnubin'), - File.join(brew_dir, 'opt/coreutils/libexec/gnubin'), - File.join(brew_dir, 'opt/gnu-sed/libexec/gnubin'), - File.join(brew_dir, 'bin'), - File.join(brew_dir, 'opt/texinfo/bin'), - ENV.fetch('PATH', nil) - ].compact.join(':') - - ENV['LIBRARY_PATH'] = [ - ENV.fetch('LIBRARY_PATH', nil), - '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib' - ].compact.join(':') + compile_env.each { |k, v| ENV[k] = v } local_lisp_path = [ ENV.fetch('EMACS_LOCAL_LISP_PATH', '').split(':'), @@ -477,7 +632,7 @@ class Build # Disable aligned_alloc on Mojave and below. See issue: # https://github.com/daviderestivo/homebrew-emacs-head/issues/15 - if OS.version.major <= 10 && OS.version.minor <= 14 + if OS.sdk_version.major <= 10 && OS.sdk_version.minor <= 14 info 'Force disabling of aligned_alloc on macOS Mojave (10.14.x) ' \ 'and earlier' disable_alligned_alloc @@ -649,7 +804,7 @@ class Build meta[:date]&.strftime('%Y-%m-%d'), meta[:sha][0..6], meta[:ref], - "macOS-#{OS.version}", + "macOS-#{OS.sdk_version}", OS.arch ].compact.map { |v| v.gsub(/[^\w_-]+/, '-') } @@ -972,6 +1127,16 @@ class AbstractEmbedder def resources_dir @resources_dir ||= File.join(app, 'Contents', 'Resources') end + + private + + def while_writable(file) + mode = File.stat(file).mode + File.chmod(0o775, file) + yield + ensure + File.chmod(mode, file) if File.exist?(file) + end end class CLIHelperEmbedder < AbstractEmbedder @@ -1036,14 +1201,14 @@ class CSourcesEmbedder < AbstractEmbedder end class LibEmbedder < AbstractEmbedder - attr_reader :lib_source + attr_reader :lib_sources attr_reader :extra_libs attr_reader :relink_eln_files - def initialize(app, lib_source, extra_libs = [], relink_eln_files: true) + def initialize(app, sources = [], extra_libs = [], relink_eln_files: true) super(app) - @lib_source = lib_source + @lib_sources = sources @extra_libs = extra_libs @relink_eln_files = relink_eln_files end @@ -1167,9 +1332,9 @@ class LibEmbedder < AbstractEmbedder debug "-- -- Resolved to: #{lib_filepath}" if linked_dylib != lib_filepath - # Only bundle libraries from lib_source. - unless lib_filepath.start_with?(lib_source) - debug "-- -- Skipping, not from lib_source: #{lib_source}" + # Only bundle libraries from lib_sources. + unless lib_sources.any? { |p| lib_filepath.start_with?(p) } + debug "-- -- Skipping, not from lib_sources: #{lib_sources.join(', ')}" next end @@ -1219,7 +1384,15 @@ class LibEmbedder < AbstractEmbedder next if dylib_id.nil? || dylib_id == '' while_writable(target) do - MachO::Tools.change_dylib_id(target, dylib_id) + file = MachO.open(target) + file.change_dylib_id(dylib_id) + + # Remove all rpaths except for @loader_path. Any other rpaths present in + # embedded libraries will potentially cause issues. + rpaths = file.rpaths.reject { |r| r == '@loader_path' } + rpaths.each { |r| file.delete_rpath(r) } + + file.write! end end @@ -1245,14 +1418,6 @@ class LibEmbedder < AbstractEmbedder while_writable(target_file) { mf.write! } if changed end end - - def while_writable(file) - mode = File.stat(file).mode - File.chmod(0o775, file) - yield - ensure - File.chmod(mode, file) if File.exist?(file) - end end class GccLibEmbedder < AbstractEmbedder @@ -1275,12 +1440,31 @@ class GccLibEmbedder < AbstractEmbedder fatal "No suitable GCC lib dir found in #{gcc_info.root_dir}" end - FileUtils.mkdir_p(File.dirname(target_dir)) - run_cmd('cp', '-pRL', source_dir, target_dir) - FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true) + FileUtils.mkdir_p(target_dir) + run_cmd( + 'rsync', '-rlptD', + # Exclude lib symlink which points at itself when using nix. + '--exclude', 'lib', + # Exclude gcc directory which holds apple-darwin libs, we copy those + # separately. + '--exclude', 'gcc', + File.join(source_dir, ''), target_dir + ) run_cmd('chmod', '-R', 'u+w', target_dir) - if source_darwin_dir != target_darwin_dir - run_cmd('mv', source_darwin_dir, target_darwin_dir) + tidy_lib_rpaths(target_dir) + + FileUtils.mkdir_p(target_darwin_dir) + run_cmd( + 'rsync', '-rlptD', + File.join(source_darwin_dir, ''), target_darwin_dir + ) + run_cmd('chmod', '-R', 'u+w', target_darwin_dir) + tidy_lib_rpaths(target_darwin_dir) + + FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true) + + if target_darwin_dir != sanitized_target_darwin_dir + run_cmd('mv', target_darwin_dir, sanitized_target_darwin_dir) end env_setup = ERB.new(NATIVE_COMP_ENV_VAR_TPL).result(gcc_info.get_binding) @@ -1305,10 +1489,10 @@ class GccLibEmbedder < AbstractEmbedder (devtools-dir "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib") (gcc-dir (expand-file-name - "<%= app_bundle_relative_lib_dir %>" + "<%= app_bundle_target_lib_dir %>" invocation-directory)) (darwin-dir (expand-file-name - "<%= app_bundle_relative_darwin_lib_dir %>" + "<%= app_bundle_target_darwin_lib_dir %>" invocation-directory)) (lib-paths (list))) @@ -1322,26 +1506,50 @@ class GccLibEmbedder < AbstractEmbedder (setenv "LIBRARY_PATH" (mapconcat 'identity lib-paths ":")))) ELISP + # Remove all rpaths from Mach-O library files except for @loader_path. + def tidy_lib_rpaths(directory) + Dir[File.join(directory, '**', '*.{dylib,so}')].each do |file_path| + next if File.symlink?(file_path) + + begin + mf = MachO.open(file_path) + rescue MachO::NotAMachOError + next + end + + rpaths = mf.rpaths.reject { |r| r == '@loader_path' } + next if rpaths.none? + + debug "Tidying up rpaths from: #{relative_path(file_path)}" + rpaths.each { |r| mf.delete_rpath(r) } + mf.write! + end + end + def embedded? Dir[File.join(target_dir, 'libgcc*')].any? end def target_dir - File.join(lib_dir, gcc_info.relative_lib_dir) + File.join(lib_dir, gcc_info.target_lib_dir) end - def source_darwin_dir - File.join(lib_dir, gcc_info.relative_darwin_lib_dir) + def target_darwin_dir + File.join(lib_dir, gcc_info.target_darwin_lib_dir) end - def target_darwin_dir - File.join(lib_dir, gcc_info.sanitized_relative_darwin_lib_dir) + def sanitized_target_darwin_dir + File.join(lib_dir, gcc_info.sanitized_target_darwin_lib_dir) end def source_dir gcc_info.lib_dir end + def source_darwin_dir + gcc_info.darwin_lib_dir + end + def relative_dir(path, root) Pathname.new(path).relative_path_from(Pathname.new(root)).to_s end @@ -1354,86 +1562,130 @@ end class GccInfo include Output + def initialize(use_nix: false) + @use_nix = use_nix + end + + def use_nix? + @use_nix + end + def root_dir - @root_dir ||= `brew --prefix gcc`.chomp + @root_dir ||= + if use_nix? + libgccjit_root_dir + else + `brew --prefix gcc`.chomp + end end def major_version - @major_version ||= File.basename(lib_dir) + @major_version ||= + if use_nix? + libgccjit_major_version + else + File.basename(lib_dir) + end end def lib_dir @lib_dir ||= - Dir[File.join(root_dir, 'lib/gcc/*/libgcc*')] - .map { |path| File.dirname(path) } - .select { |path| File.basename(path).match(/^\d+$/) } - .max_by { |path| File.basename(path).to_i } + if use_nix? + File.join(root_dir, 'lib') + else + Dir[File.join(root_dir, 'lib/gcc/*/libgcc*')] + .map { |path| File.dirname(path) } + .select { |path| File.basename(path).match(/^\d+$/) } + .max_by { |path| File.basename(path).to_i } + end end - def relative_lib_dir - @relative_lib_dir ||= relative_dir(lib_dir, File.join(root_dir, 'lib')) + def target_lib_dir + File.join('gcc', 'lib') end def darwin_lib_dir - @darwin_lib_dir ||= - Dir[File.join(lib_dir, 'gcc/*apple-darwin*/*')].max_by do |path| - [ - File.basename(File.dirname(path)).match(/darwin(\d+)$/)[1].to_i, - File.basename(path).split('.').map(&:to_i) - ] + return @darwin_lib_dir if @darwin_lib_dir + + search_path = File.join(lib_dir, 'gcc/*apple-darwin*/*') + + @darwin_lib_dir ||= Dir[search_path].max_by do |path| + vers = [] + + unless use_nix? + matches = File.basename(File.dirname(path)).match(/darwin(\d+)$/) + vers << matches[1].to_i if matches end + + vers << File.basename(path).split('.').map(&:to_i) + vers.flatten + end end - def relative_darwin_lib_dir - @relative_darwin_lib_dir ||= - relative_dir(darwin_lib_dir, File.join(root_dir, 'lib')) + def target_darwin_lib_dir + File.join('gcc', 'lib', 'apple-darwin') end # Sanitize folder name with full "MAJOR.MINOR.PATCH" version number to just # the MAJOR version. Apple's codesign CLI tool throws a "bundle format # unrecognized" error if there are any folders with two dots in their name # within the Emacs.app application bundle. - def sanitized_relative_darwin_lib_dir - @sanitized_relative_darwin_lib_dir ||= + def sanitized_target_darwin_lib_dir + @sanitized_target_darwin_lib_dir ||= File.join( - File.dirname(relative_darwin_lib_dir), - File.basename(relative_darwin_lib_dir).gsub('.', '_') + File.dirname(target_darwin_lib_dir), + File.basename(target_darwin_lib_dir).gsub('.', '_') ) end - def app_bundle_relative_lib_dir - @app_bundle_relative_lib_dir ||= + def app_bundle_target_lib_dir + @app_bundle_target_lib_dir ||= relative_dir( - File.join(embedder.lib_dir, relative_lib_dir), + File.join(embedder.lib_dir, target_lib_dir), embedder.invocation_dir ) end - def app_bundle_relative_darwin_lib_dir - @app_bundle_relative_darwin_lib_dir ||= + def app_bundle_target_darwin_lib_dir + @app_bundle_target_darwin_lib_dir ||= relative_dir( - File.join(embedder.lib_dir, sanitized_relative_darwin_lib_dir), + File.join(embedder.lib_dir, sanitized_target_darwin_lib_dir), embedder.invocation_dir ) end def libgccjit_root_dir - @libgccjit_root_dir ||= `brew --prefix libgccjit`.chomp + @libgccjit_root_dir ||= + if use_nix? + ENV['NIX_LIBGCCJIT_ROOT']&.strip + else + `brew --prefix libgccjit`.chomp + end end def libgccjit_major_version - @libgccjit_major_version ||= File.basename(libgccjit_lib_dir.to_s) + @libgccjit_major_version ||= + if use_nix? + ENV['NIX_LIBGCCJIT_VERSION']&.strip&.split('.')&.first + else + File.basename(libgccjit_lib_dir.to_s) + end end def libgccjit_lib_dir @libgccjit_lib_dir ||= - Dir[ - File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit*.dylib'), - File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit.so*') - ] - .map { |path| File.dirname(path) } - .select { |path| File.basename(path).match(/^\d+$/) } - .max_by { |path| File.basename(path).to_i } + if use_nix? + Dir[File.join(libgccjit_root_dir, 'lib/libgccjit*.dylib')] + .map { |path| File.dirname(path) }.first + else + Dir[ + File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit*.dylib'), + File.join(libgccjit_root_dir, 'lib/gcc/*/libgccjit.so*'), + ] + .map { |path| File.dirname(path) } + .select { |path| File.basename(path).match(/^\d+$/) } + .max_by { |path| File.basename(path).to_i } + end end def verify_libgccjit @@ -1446,6 +1698,11 @@ class GccInfo 'brew reinstall libgccjit' end + # No need to verify gcc vs libgccjit for Nix, as we can pull everything we + # need from the libgccjit package. On homebrew we need to pull parts from + # gcc and parts from libgccjit, hence we need to ensure versions match. + return if use_nix? + return if major_version == libgccjit_major_version fatal <<~TEXT @@ -1472,6 +1729,8 @@ class GccInfo end if __FILE__ == $PROGRAM_NAME + use_nix_default = !ENV.fetch('IN_NIX_SHELL', '').empty? + cli_options = { work_dir: File.expand_path(__dir__), native_full_aot: false, @@ -1480,6 +1739,7 @@ if __FILE__ == $PROGRAM_NAME parallel: Etc.nprocessors, rsvg: true, dbus: true, + use_nix: use_nix_default, xwidgets: true, tree_sitter: true, fd_setsize: 10_000, @@ -1502,6 +1762,16 @@ if __FILE__ == $PROGRAM_NAME Options: DOC + opts.on( + '--info', + 'Print environment info and detected library paths, then exit' + ) { |v| cli_options[:info] = v } + + opts.on( + '--preview', + 'Print preview details about build and exit.' + ) { |v| cli_options[:preview] = v } + opts.on( '-j', '--parallel COUNT', @@ -1515,6 +1785,12 @@ if __FILE__ == $PROGRAM_NAME 'branch allowing builds of old commits' ) { |v| cli_options[:git_sha] = v } + opts.on( + '--[no-]use-nix', + 'Use Nix instead of Homebrew to find dependencies ' \ + '(default: enabled if EMACS_BUILD_USE_NIX is set to truthy value)' + ) { |v| cli_options[:use_nix] = v } + opts.on( '--[no-]xwidgets', 'Enable/disable XWidgets if supported ' \ @@ -1653,7 +1929,15 @@ if __FILE__ == $PROGRAM_NAME Output.log_level = cli_options[:log_level] work_dir = cli_options.delete(:work_dir) - Build.new(work_dir, ARGV.shift, cli_options).build + build = Build.new(work_dir, ARGV.shift, cli_options) + + if cli_options[:info] + build.print_info + elsif cli_options[:preview] + build.print_preview + else + build.build + end rescue Error => e warn "ERROR: #{e.message}" exit 1 diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..18a86a5 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1726560853, + "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1731139594, + "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..84bc423 --- /dev/null +++ b/flake.nix @@ -0,0 +1,77 @@ +{ + description = "Development environment flake"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + devShells.default = pkgs.mkShell { + packages = with pkgs; [ + # Package list specifically excludes ncurses, so that we link + # against the system version of ncurses. This ensures emacs' TUI + # works out of the box without the user having to manually set + # TERMINFO in the shell before launching emacs. + apple-sdk + autoconf + cairo + clang + coreutils + curl + darwin.DarwinTools # sw_vers + dbus + expat + findutils + gcc + gettext + giflib + git + gmp + gnumake + gnupatch + gnused + gnutar + gnutls + harfbuzz + jansson + lcms2 + libffi + libgccjit + libiconv + libjpeg + libpng + librsvg + libtasn1 + libunistring + libwebp + libxml2 + mailutils + nettle + pkg-config + python3 + rsync + ruby_3_3 + sqlite + texinfo + time + tree-sitter + which + xcbuild + zlib + ]; + + shellHook = '' + export CC=clang + export NIX_LIBGCCJIT_VERSION="${pkgs.libgccjit.version}" + export NIX_LIBGCCJIT_ROOT="${pkgs.libgccjit.outPath}" + ''; + }; + } + ); +} diff --git a/go.mod b/go.mod index 3122b4a..cc240ae 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jimeh/build-emacs-for-macos -go 1.20 +go 1.23 require ( github.com/bearer/gon v0.0.36 diff --git a/go.sum b/go.sum index 773cb52..d4b4505 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v35 v35.3.0 h1:fU+WBzuukn0VssbayTT+Zo3/ESKX9JYWjbZTLOTEyho= github.com/google/go-github/v35 v35.3.0/go.mod h1:yWB7uCcVWaUbUP74Aq3whuMySRMatyRmq5U9FTNlbio= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= diff --git a/pkg/dmgbuild/license.go b/pkg/dmgbuild/license.go index 996283e..92f0499 100644 --- a/pkg/dmgbuild/license.go +++ b/pkg/dmgbuild/license.go @@ -116,7 +116,6 @@ func NewLicense() License { return License{} } -//nolint:goconst func (s *License) Render() []string { var l []string diff --git a/pkg/osinfo/osinfo.go b/pkg/osinfo/osinfo.go index 31d35b8..823db27 100644 --- a/pkg/osinfo/osinfo.go +++ b/pkg/osinfo/osinfo.go @@ -1,41 +1,65 @@ package osinfo import ( + "os" "os/exec" "strconv" "strings" ) type OSInfo struct { - Name string `yaml:"name" json:"name"` - Version string `yaml:"version" json:"version"` - Arch string `yaml:"arch" json:"arch"` + Name string `yaml:"name" json:"name"` + Version string `yaml:"version" json:"version"` + SDKVersion string `yaml:"sdk_version" json:"sdk_version"` + Arch string `yaml:"arch" json:"arch"` } func New() (*OSInfo, error) { - version, err := exec.Command("sw_vers", "-productVersion").CombinedOutput() + version, err := exec.Command("sw_vers", "-productVersion").Output() if err != nil { return nil, err } + sdkVersion := os.Getenv("MACOSX_DEPLOYMENT_TARGET") + if sdkVersion == "" { + var ver []byte + ver, err = exec.Command("xcrun", "--show-sdk-version").Output() + if err != nil { + return nil, err + } + + sdkVersion = string(ver) + } + arch, err := exec.Command("uname", "-m").CombinedOutput() if err != nil { return nil, err } return &OSInfo{ - Name: "macOS", - Version: strings.TrimSpace(string(version)), - Arch: strings.TrimSpace(string(arch)), + Name: "macOS", + Version: strings.TrimSpace(string(version)), + SDKVersion: strings.TrimSpace(sdkVersion), + Arch: strings.TrimSpace(string(arch)), }, nil } -// DistinctVersion returns macOS version down to a distinct "major" -// version. For macOS 10.x, this will include the first two numeric parts of the -// version (10.15), while for 11.x and later, the first numeric part is enough -// (11). +// DistinctVersion returns macOS version down to a distinct "major" version. For +// macOS 10.x, this will include the first two numeric parts of the version +// (10.15), while for 11.x and later, the first numeric part is enough (11). func (s *OSInfo) DistinctVersion() string { - parts := strings.Split(s.Version, ".") + return s.distinctVersion(s.Version) +} + +// DistinctSDKVersion returns macOS version down to a distinct "major" version. +// For macOS 10.x, this will include the first two numeric parts of the version +// (10.15), while for 11.x and later, the first numeric part is enough (11). +func (s *OSInfo) DistinctSDKVersion() string { + return s.distinctVersion(s.SDKVersion) +} + +func (s *OSInfo) distinctVersion(version string) string { + parts := strings.Split(version, ".") if n, _ := strconv.Atoi(parts[0]); n >= 11 { return parts[0] diff --git a/pkg/plan/create.go b/pkg/plan/create.go index 50d59c0..277fab1 100644 --- a/pkg/plan/create.go +++ b/pkg/plan/create.go @@ -95,10 +95,17 @@ func Create(ctx context.Context, opts *Options) (*Plan, error) { //nolint:funlen releaseName = "Emacs." + version } + // Attempt to get the macOS SDK version from the environment, if it's not + // available, use the version from the system. + targetMacOSVersion := osInfo.DistinctSDKVersion() + if targetMacOSVersion == "" { + targetMacOSVersion = osInfo.DistinctVersion() + } + buildName := fmt.Sprintf( "Emacs.%s.%s.%s", absoluteVersion, - sanitize.String(osInfo.Name+"-"+osInfo.DistinctVersion()), + sanitize.String(osInfo.Name+"-"+targetMacOSVersion), sanitize.String(osInfo.Arch), ) diskImage := buildName + ".dmg" diff --git a/pkg/sign/emacs.go b/pkg/sign/emacs.go index c59e790..62c343d 100644 --- a/pkg/sign/emacs.go +++ b/pkg/sign/emacs.go @@ -123,7 +123,7 @@ func signCLIHelper(ctx context.Context, appBundle string, opts *Options) error { // app bundle itself. func elnFiles(emacsApp string) ([]string, error) { var files []string - walkDirFunc := func(path string, d fs.DirEntry, _err error) error { + walkDirFunc := func(path string, d fs.DirEntry, _ error) error { if d.Type().IsRegular() && strings.HasSuffix(path, ".eln") && !strings.Contains(path, ".app/Contents/Frameworks/") { files = append(files, path)