Merge pull request #45 from DataDog/isabelle.sauve/reduce-allocations

Reduce allocations

apt/apt.go

@@ -11,11 +11,11 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
-	"github.com/aptly-dev/aptly/database/goleveldb"
-	"github.com/aptly-dev/aptly/deb"
-	"github.com/aptly-dev/aptly/http"
+	"github.com/DataDog/aptly/aptly"
+	"github.com/DataDog/aptly/database"
+	"github.com/DataDog/aptly/database/goleveldb"
+	"github.com/DataDog/aptly/deb"
+	"github.com/DataDog/aptly/http"
 	"github.com/arduino/go-apt-client"
 	"github.com/xor-gate/ar"
 
@@ -65,14 +65,18 @@ func (b *Backend) downloadPackage(downloader aptly.Downloader, factory *deb.Coll
 	var packageURL *url.URL
 	var packageDeps *deb.PackageDependencies
 
+	stanza := make(deb.Stanza, 32)
+
 	err := b.repoCollection.ForEach(func(repo *deb.RemoteRepo) error {
 		if packageURL != nil {
 			return nil
 		}
 
 		b.logger.Debugf("Fetching repository: name=%s, distribution=%s, components=%v, arch=%v", repo.Name, repo.Distribution, repo.Components, repo.Architectures)
 		repo.SkipComponentCheck = true
-		if err := repo.Fetch(downloader, nil); err != nil {
+
+		stanza.Clear()
+		if err := repo.FetchBuffered(stanza, downloader, nil); err != nil {
 			b.logger.Debugf("Error fetching repo: %s", err)
 			return err
 		}

extract/tarball.go

@@ -15,6 +15,10 @@ import (
 	"github.com/xi2/xz"
 )
 
+type onlyWriter struct {
+	io.Writer
+}
+
 func ExtractTarball(reader io.Reader, filename, directory string, logger types.Logger) error {
 	var compressedTarReader io.Reader
 	var err error
@@ -37,6 +41,7 @@ func ExtractTarball(reader io.Reader, filename, directory string, logger types.L
 		return fmt.Errorf("failed to read %s: %w", filename, err)
 	}
 
+	buf := make([]byte, 50)
 	tarReader := tar.NewReader(compressedTarReader)
 	for {
 		hdr, err := tarReader.Next()
@@ -67,7 +72,13 @@ func ExtractTarball(reader io.Reader, filename, directory string, logger types.L
 				return fmt.Errorf("failed to create output file '%s': %w", path, err)
 			}
 
-			if _, err := io.Copy(output, tarReader); err != nil {
+			// By default, an os.File implements the io.ReaderFrom interface.
+			// As a result, CopyBuffer will attempt to use the output.ReadFrom method to perform
+			// the requested copy, which ends up calling the unbuffered io.Copy function & performing
+			// a large number of allocations.
+			// In order to force CopyBuffer to actually utilize the given buffer, we have to ensure
+			// output does not implement the io.ReaderFrom interface.
+			if _, err := io.CopyBuffer(onlyWriter{output}, tarReader, buf); err != nil {
 				return fmt.Errorf("failed to uncompress file %s: %w", hdr.Name, err)
 
 			}
@@ -78,4 +89,4 @@ func ExtractTarball(reader io.Reader, filename, directory string, logger types.L
 	}
 
 	return nil
-}
+}

go.mod

@@ -5,30 +5,27 @@ go 1.14
 require (
 	cloud.google.com/go/storage v1.12.0
 	github.com/AlekSi/pointer v1.1.0 // indirect
+	github.com/DataDog/aptly v1.5.0
 	github.com/DataDog/gopsutil v1.1.0
 	github.com/DataDog/zstd v1.5.0
 	github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/acobaugh/osrelease v0.1.0
-	github.com/aptly-dev/aptly v1.4.1-0.20211102140819-ab2f5420c617
 	github.com/arduino/go-apt-client v0.0.0-20190812130613-5613f843fdc8
 	github.com/awalterschulze/gographviz v2.0.1+incompatible // indirect
 	github.com/cheggaaa/pb v1.0.29 // indirect
 	github.com/go-ini/ini v1.63.2
 	github.com/h2non/filetype v1.1.0 // indirect
 	github.com/jlaffaye/ftp v0.0.0-20200812143550-39e3779af0db // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/pborman/uuid v1.2.1 // indirect
 	github.com/sassoftware/go-rpmutils v0.2.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.0.0
-	github.com/ugorji/go v1.1.7 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8
 	github.com/xor-gate/ar v0.0.0-20170530204233-5c72ae81e2b7
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
+	golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64
 	google.golang.org/api v0.32.0
-	gopkg.in/yaml.v2 v2.2.8 // indirect
 )
 
 replace github.com/DataDog/gopsutil => github.com/DataDog/gopsutil v0.0.0-20211112231309-1e2204a98d89