From 5a078c2b0f6904060e25621a263fd2a14c3c04e0 Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Mon, 22 Apr 2024 13:04:39 +0200 Subject: [PATCH 1/6] Fix "can't use string as a HASH ref" error This could happen for some dependency cycles. --- PBuild/Checker.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PBuild/Checker.pm b/PBuild/Checker.pm index e316cc458..397b50370 100644 --- a/PBuild/Checker.pm +++ b/PBuild/Checker.pm @@ -728,7 +728,7 @@ sub handlecycle { my $notready = $ctx->{'notready'}; my $pkgsrc = $ctx->{'pkgsrc'}; if (grep {$notready->{$_->{'name'} || $_->{'pkg'}}} map {$pkgsrc->{$_}} @cycp) { - $notready->{$_->{'name'} || $_->{'pkg'}} ||= 1 for @cycp; + $notready->{$_->{'name'} || $_->{'pkg'}} ||= 1 for map {$pkgsrc->{$_}} @cycp; } return (undef, 3); } From 57b93bf6a69e5872ce8ec08bb29f11f25f6e25ad Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Mon, 22 Apr 2024 14:19:12 +0200 Subject: [PATCH 2/6] Rework mkosi handling - use PBuild::Mkosi instead of PBuild::Container - be more carful in the binlink generation code - we do not need the query function in Build::Mkosi. --- Build.pm | 2 - Build/Mkosi.pm | 61 --------------------------- PBuild/BuildResult.pm | 5 ++- PBuild/Container.pm | 62 ---------------------------- PBuild/Mkosi.pm | 96 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 127 deletions(-) create mode 100644 PBuild/Mkosi.pm diff --git a/Build.pm b/Build.pm index 72c4fe211..3d4262c4a 100644 --- a/Build.pm +++ b/Build.pm @@ -1367,7 +1367,6 @@ sub query { return Build::Kiwi::queryiso($handle, %opts) if $do_kiwi && $binname =~ /\.iso$/; return Build::Arch::query($handle, %opts) if $do_arch && $binname =~ /\.pkg\.tar(?:\.gz|\.xz|\.zst)?$/; return Build::Arch::query($handle, %opts) if $do_arch && $binname =~ /\.arch$/; - return Build::Mkosi::queryiso($handle, %opts) if $do_mkosi && $binname =~ /\.(raw|tar|cpio|qcow2)(?:\.gz|\.xz|\.zstd)?$/; return undef; } @@ -1393,7 +1392,6 @@ sub queryhdrmd5 { my ($binname) = @_; return Build::Rpm::queryhdrmd5(@_) if $do_rpm && $binname =~ /\.d?rpm$/; return Build::Deb::queryhdrmd5(@_) if $do_deb && $binname =~ /\.deb$/; - return Build::Mkosi::queryhdrmd5(@_) if $do_mkosi && $binname =~ /\.(raw|tar|cpio|qcow2)(?:\.gz|\.xz|\.zstd)?$/; return Build::Kiwi::queryhdrmd5(@_) if $do_kiwi && $binname =~ /\.iso$/; return Build::Kiwi::queryhdrmd5(@_) if $do_kiwi && $binname =~ /\.raw$/; return Build::Kiwi::queryhdrmd5(@_) if $do_kiwi && $binname =~ /\.raw.install$/; diff --git a/Build/Mkosi.pm b/Build/Mkosi.pm index cfdc38fe9..ef65cceaa 100644 --- a/Build/Mkosi.pm +++ b/Build/Mkosi.pm @@ -78,65 +78,4 @@ sub parse { return $ret; } -sub queryiso { - my ($file, %opts) = @_; - my $json_fh; - my $md5 = Digest::MD5->new; - - open(my $fh, '<', $file) or die("Error opening $file: $!\n"); - $md5->addfile($fh); - close($fh); - # If we also have split verity artifacts, the manifest file is the same as the main image, - # so remove the suffixes to find it - $file =~ s/(\.root|\.usr)//g; - $file = $file . ".manifest.gz"; - - eval { require JSON; }; - *JSON::decode_json = sub {die("JSON::decode_json is not available\n")} unless defined &JSON::decode_json; - - eval { require IO::Uncompress::Gunzip; }; - *IO::Uncompress::Gunzip::new = sub {die("IO::Uncompress::Gunzip is not available\n")} unless defined &IO::Uncompress::Gunzip::new; - - my $json_text = do { - open($json_fh, "<", $file) or die("Error opening $file: $!\n"); - $json_fh = IO::Uncompress::Gunzip->new($json_fh) or die("Error opening $file: $IO::Uncompress::Gunzip::GunzipError\n"); - local $/; - <$json_fh> - }; - - my $metadata = JSON::decode_json($json_text); - close $json_fh; - - if (!$metadata || !$metadata->{'config'}) { - return {}; - } - - my $distribution = $metadata->{'config'}->{'distribution'}; - my $release = $metadata->{'config'}->{'release'}; - my $architecture = $metadata->{'config'}->{'architecture'}; - my $name = $metadata->{'config'}->{'name'}; - my $version = $metadata->{'config'}->{'version'}; - my @provides = ("$distribution:$release"); - - return { - 'provides' => \@provides, - 'version' => $version, - 'arch' => $architecture, - 'name' => $name, - 'source' => $name, - 'hdrmd5' => $md5->hexdigest(), - }; -} - -sub queryhdrmd5 { - my ($bin) = @_; - - open(my $fh, '<', $bin) or croak("could not open $bin"); - my $md5 = Digest::MD5->new; - $md5->addfile($fh); - close($fh); - - return $md5->hexdigest(); -} - 1; diff --git a/PBuild/BuildResult.pm b/PBuild/BuildResult.pm index 92f1e6797..b161d43b1 100644 --- a/PBuild/BuildResult.pm +++ b/PBuild/BuildResult.pm @@ -28,6 +28,7 @@ use Build::SimpleXML; use PBuild::Util; use PBuild::Verify; use PBuild::Container; +use PBuild::Mkosi; my @binsufs = qw{rpm deb pkg.tar.gz pkg.tar.xz pkg.tar.zst}; my $binsufsre = join('|', map {"\Q$_\E"} @binsufs); @@ -130,11 +131,11 @@ sub integrate_build_result { PBuild::Util::cp_a($result->{$file}, "$dst/$file"); next; } - if ($file =~ /(.*)\.manifest(?:\.(?:gz|bz2|xz|zst|zstd))?$/) { + if ($p->{'buildtype'} eq 'mkosi' && $file =~ /(.*)\.manifest(?:\.(?:gz|bz2|xz|zst|zstd))?$/) { # create an obsbinlnk file from the mkosi manifest my $prefix = $1; die unless $result->{$file} =~ /^(.*)\/([^\/]+)$/; - my $obsbinlnk = PBuild::Container::manifest2obsbinlnk($1, $2, $prefix, $p->{'pkg'}); + my $obsbinlnk = PBuild::Mkosi::manifest2obsbinlnk($1, $2, $prefix, $p->{'pkg'}); PBuild::Util::store("$dst/$prefix.obsbinlnk", undef, $obsbinlnk) if $obsbinlnk; } PBuild::Util::cp($result->{$file}, "$dst/$file"); diff --git a/PBuild/Container.pm b/PBuild/Container.pm index 9598deff1..796e89212 100644 --- a/PBuild/Container.pm +++ b/PBuild/Container.pm @@ -28,70 +28,8 @@ use PBuild::Verify; eval { require JSON::XS }; *JSON::XS::decode_json = sub {die("JSON::XS is not available\n")} unless defined &JSON::XS::decode_json; -eval { require IO::Uncompress::Gunzip; }; -*IO::Uncompress::Gunzip::new = sub {die("IO::Uncompress::Gunzip is not available\n")} unless defined &IO::Uncompress::Gunzip::new; - use strict; -sub manifest2obsbinlnk { - my ($dir, $file, $prefix, $packid) = @_; - my $json_fh; - my $md5 = Digest::MD5->new; - my $image; - my $json_text = do { - unless (open($json_fh, "<", "$dir/$file")) { - warn("Error opening $dir/$file: $!\n"); - return {}; - } - if ($file =~ /\.gz$/) { - $json_fh = IO::Uncompress::Gunzip->new($json_fh) or die("Error opening $dir/$file: $IO::Uncompress::Gunzip::GunzipError\n"); - } - local $/; - <$json_fh> - }; - - my $metadata = JSON::XS::decode_json($json_text); - if (!$metadata || !$metadata->{'config'}) { - return {}; - } - - for my $ext ("", ".raw", ".gz", ".xz", ".zst", ".zstd") { - my $fn = "$dir/$prefix$ext"; - if (-e $fn) { - if (-l $fn) { - $prefix = readlink($fn); - } - open(my $fh, '<', "$dir/$prefix$ext") or die("Error opening $dir/$prefix$ext: $!\n"); - $md5->addfile($fh); - close($fh); - $image = $prefix . $ext; - last; - } - } - if (!$image) { - return {}; - } - - my $distribution = $metadata->{'config'}->{'distribution'}; - my $release = $metadata->{'config'}->{'release'}; - my $architecture = $metadata->{'config'}->{'architecture'}; - my $name = $metadata->{'config'}->{'name'}; - my $version = $metadata->{'config'}->{'version'} || '0'; - # Note: release here is not the RPM release, but the distribution release (eg: Debian 10) - my @provides = ("$distribution:$release", "mkosi:$name = $version", "mkosi:$packid = $version"); - - return { - 'provides' => \@provides, - 'source' => $packid, - 'name' => "mkosi:$name", - 'version' => $version, - 'release' => '0', - 'arch' => $architecture, - 'hdrmd5' => $md5->hexdigest(), - 'lnk' => $image, - }; -} - sub containerinfo2nevra { my ($d) = @_; my $lnk = {}; diff --git a/PBuild/Mkosi.pm b/PBuild/Mkosi.pm new file mode 100644 index 000000000..18617aed7 --- /dev/null +++ b/PBuild/Mkosi.pm @@ -0,0 +1,96 @@ +################################################################ # +# Copyright (c) 2021 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program (see the file COPYING); if not, write to the +# Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +# +################################################################ + +package PBuild::Mkosi; + +use Digest::MD5 (); +use IO::Uncompress::Gunzip (); + +use PBuild::Verify; + +eval { require JSON::XS }; +*JSON::XS::decode_json = sub {die("JSON::XS is not available\n")} unless defined &JSON::XS::decode_json; + +use strict; + +sub manifest2obsbinlnk { + my ($dir, $file, $prefix, $packid) = @_; + my $json_fh; + my $image; + my $json_text = do { + unless (open($json_fh, "<", "$dir/$file")) { + warn("Error opening $dir/$file: $!\n"); + return {}; + } + if ($file =~ /\.gz$/) { + $json_fh = IO::Uncompress::Gunzip->new($json_fh) or die("Error opening $dir/$file: $IO::Uncompress::Gunzip::GunzipError\n"); + } + local $/; + <$json_fh> + }; + + my $metadata = eval { JSON::XS::decode_json($json_text) }; + return unless $metadata && $metadata->{'config'} && ref($metadata->{'config'}) eq 'HASH'; + + for my $ext ("", ".raw", ".gz", ".xz", ".zst", ".zstd") { + my $fn = "$dir/$prefix$ext"; + if (-e $fn) { + if (-l $fn) { + $prefix = readlink($fn); + $prefix =~ s/.*\///; + } + $image = $prefix . $ext; + last + } + } + return unless $image; + eval { PBuild::Verify::verify_filename($image) }; + return undef if $@; + + open(my $fh, '<', "$dir/$image") or die("Error opening $dir/$image: $!\n"); + my $md5 = Digest::MD5->new; + $md5->addfile($fh); + close($fh); + + my $config = $metadata->{'config'}; + my $distribution = $config->{'distribution'}; + my $distrelease = $config->{'release'}; # distribution release (eg: Debian 10) + my $architecture = $config->{'architecture'}; + my $name = $config->{'name'}; + my $version = $config->{'version'} || '0'; + my $release = '0'; + my @provides = ("mkosi:$distribution:$distrelease", "mkosi:$name = $version-$release"); + + my $lnk = { + 'name' => "mkosi:$name", + 'version' => $version, + 'release' => $release, + 'arch' => 'noarch', + 'provides' => \@provides, + 'source' => $packid, + }; + eval { PBuild::Verify::verify_nevraquery($lnk) }; + return undef if $@; + + $lnk->{'hdrmd5'} = $md5->hexdigest(); + $lnk->{'lnk'} = $image; + return $lnk; +} + +1; From e05006583bceedb5d6d2b44a7cea71767020851b Mon Sep 17 00:00:00 2001 From: Michael Schroeder Date: Fri, 26 Apr 2024 10:35:56 +0200 Subject: [PATCH 3/6] Obey the 'debuginfo' option in the Checker This makes it possible to turn debuginfo generation off with a '--debuginfo=no' option. --- PBuild/Checker.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PBuild/Checker.pm b/PBuild/Checker.pm index 397b50370..fc8cc4a6f 100644 --- a/PBuild/Checker.pm +++ b/PBuild/Checker.pm @@ -48,7 +48,7 @@ sub create { 'builddir' => $builddir, 'block' => undef, # block strategy (all,never) 'rebuild' => undef, # rebuild strategy (transitive,direct,local) - 'debuginfo' => 1, # create debug packages? + 'debuginfo' => defined($opts->{'debuginfo'}) ? $opts->{'debuginfo'} : 1, # create debug packages? 'genmetaalgo' => $genmetaalgo, 'lastcheck' => {}, 'metacache' => {}, From fd43ddef7453cf19b006b794230a66ed2b4ef4f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Fri, 26 Apr 2024 13:21:40 +0200 Subject: [PATCH 4/6] add missing repo definition for 15.6 --- configs/sl15.6.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/sl15.6.conf b/configs/sl15.6.conf index 51eee9f11..de58344a2 100644 --- a/configs/sl15.6.conf +++ b/configs/sl15.6.conf @@ -1,5 +1,8 @@ %define gcc_version 7 +RepoURL: https://download.opensuse.org/distribution/leap/15.6/repo/oss/ +RegistryURL: https://registry.opensuse.org + BuildFlags: kvm-max-cpu-compat-power8 # create conflicts for packages installed into the target image From 347cf3461257351c1e7c4db9ebd4201c2c88168e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Thu, 2 May 2024 11:20:14 +0200 Subject: [PATCH 5/6] vm-build: ext3 & ext4: fix disk space allocation We did fallocate the root device explicit to ensure available disk space according to the config. However, the mkfs is operation with discard and frees the disk space again. This leads to over-commited when having multiple worker instances. So we explicit disable the punch hole/fallocate call inside of mkfs.ext3 and mkfs.ext4 by setting nodiscard. An obs worker instance will disable itself when running into this situation. --- build-vm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-vm b/build-vm index 5237bb4e8..9611a79b2 100644 --- a/build-vm +++ b/build-vm @@ -433,10 +433,10 @@ vm_img_mkfs() { esac # defaults for creating the filesystem - vm_img_mkfs_ext4_options='-O ^has_journal,^huge_file,^resize_inode,sparse_super,^metadata_csum' - vm_img_mkfs_ext4_extra='-E lazy_itable_init,discard' + vm_img_mkfs_ext4_options='-O ^has_journal,^huge_file,^resize_inode,sparse_super,^metadata_csum -E nodiscard' + vm_img_mkfs_ext4_extra='-E lazy_itable_init,nodiscard' # overwrites -E from _options by default vm_img_mkfs_ext4="mkfs.ext4 -m 0 -q -F $vm_img_mkfs_ext4_options" - vm_img_mkfs_ext3='mkfs.ext3 -m 0 -q -F' + vm_img_mkfs_ext3='mkfs.ext3 -m 0 -q -F -E nodiscard' vm_img_mkfs_ext2='mkfs.ext2 -m 0 -q -F' vm_img_mkfs_reiserfs='mkreiserfs -q -f' vm_img_mkfs_btrfs='mkfs.btrfs' From 5c4ac4e479fc64406204fab88c01112958f25b13 Mon Sep 17 00:00:00 2001 From: zhangwenlong Date: Thu, 2 May 2024 20:50:31 +0800 Subject: [PATCH 6/6] Add syscall for personality switch on loongarch64 kernel (#998) --- build-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-vm b/build-vm index 9611a79b2..6a29e52bc 100644 --- a/build-vm +++ b/build-vm @@ -569,7 +569,7 @@ vm_set_personality_syscall() { alpha*) PERSONALITY_SYSCALL=324 ;; sparc*) PERSONALITY_SYSCALL=191 ;; ia64*) PERSONALITY_SYSCALL=1140 ;; - aarch64*) PERSONALITY_SYSCALL=92 ;; + aarch64*|loongarch*) PERSONALITY_SYSCALL=92 ;; i?86*|ppc*|arm*|sh4|cris|m68k*|s390*|unicore32|microblaze|riscv*) PERSONALITY_SYSCALL=136 ;; *) cleanup_and_exit 1 "Unknown architecture personality: '$archname'" ;; esac