Identify some of dual licenses and skip built-in packages' licenses (#18

)

go.sum

@@ -16,8 +16,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/apache/skywalking-eyes v0.0.0-20201226021955-d798d4f56aa5 h1:cMn7kz937Lz92NBpMXbawQlizM4qe3LYMil+DWQxznE=
-github.com/apache/skywalking-eyes v0.0.0-20201227090648-038d724e60b2 h1:ZcHWJoGStYz0e8hSzRoyc9sKWvDuBWfvh5CPY2W9zic=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -64,7 +62,6 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM=
 github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -290,8 +287,6 @@ golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktyp
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
-golang.org/x/tools v0.0.0-20201226215659-b1c90890d22a h1:pdfjQ7VswBeGam3EpuEJ4e8EAb7JgaubV570LO/SIQM=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/deps/golang.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 
 	"golang.org/x/mod/modfile"
 	"golang.org/x/tools/go/packages"
@@ -33,18 +34,18 @@ import (
 	"github.com/apache/skywalking-eyes/license-eye/pkg/license"
 )
 
-type GoModeResolver struct {
+type GoModResolver struct {
 	Resolver
 }
 
-func (resolver *GoModeResolver) CanResolve(file string) bool {
+func (resolver *GoModResolver) CanResolve(file string) bool {
 	base := filepath.Base(file)
 	logger.Log.Debugln("Base name:", base)
 	return base == "go.mod"
 }
 
 // Resolve resolves licenses of all dependencies declared in the go.mod file.
-func (resolver *GoModeResolver) Resolve(goModFile string, report *Report) error {
+func (resolver *GoModResolver) Resolve(goModFile string, report *Report) error {
 	content, err := ioutil.ReadFile(goModFile)
 	if err != nil {
 		return err
@@ -76,7 +77,7 @@ func (resolver *GoModeResolver) Resolve(goModFile string, report *Report) error
 }
 
 // ResolvePackages resolves the licenses of the given packages.
-func (resolver *GoModeResolver) ResolvePackages(pkgNames []string, report *Report) error {
+func (resolver *GoModResolver) ResolvePackages(pkgNames []string, report *Report) error {
 	requiredPkgs, err := packages.Load(&packages.Config{
 		Context: context.Background(),
 		Mode:    packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedDeps,
@@ -87,12 +88,25 @@ func (resolver *GoModeResolver) ResolvePackages(pkgNames []string, report *Repor
 	}
 
 	packages.Visit(requiredPkgs, func(p *packages.Package) bool {
+		if isBuiltIn(p) {
+			logger.Log.Debugln("Built-in package doesn't require license check:", p.PkgPath)
+			return false
+		}
+
+		if len(p.Errors) > 0 {
+			logger.Log.Warnln("Failed to visit package:", p.PkgPath, p.Errors)
+			report.Skip(&Result{
+				Dependency:    p.PkgPath,
+				LicenseSpdxID: Unknown,
+			})
+			return true
+		}
 		err := resolver.ResolvePackageLicense(p, report)
 		if err != nil {
 			logger.Log.Warnln("Failed to resolve the license of dependency:", p.PkgPath, err)
 			report.Skip(&Result{
 				Dependency:    p.PkgPath,
-				LicenseSpdxID: []string{Unknown},
+				LicenseSpdxID: Unknown,
 			})
 		}
 		return true
@@ -103,7 +117,7 @@ func (resolver *GoModeResolver) ResolvePackages(pkgNames []string, report *Repor
 
 var possibleLicenseFileName = regexp.MustCompile(`(?i)^LICENSE|LICENCE(\.txt)?$`)
 
-func (resolver *GoModeResolver) ResolvePackageLicense(p *packages.Package, report *Report) error {
+func (resolver *GoModResolver) ResolvePackageLicense(p *packages.Package, report *Report) error {
 	var filesInPkg []string
 	if len(p.GoFiles) > 0 {
 		filesInPkg = p.GoFiles
@@ -137,15 +151,15 @@ func (resolver *GoModeResolver) ResolvePackageLicense(p *packages.Package, repor
 			if err != nil {
 				return err
 			}
-			identifier, err := license.Identify(string(content))
+			identifier, err := license.Identify(p.PkgPath, string(content))
 			if err != nil {
 				return err
 			}
 			report.Resolve(&Result{
 				Dependency:      p.PkgPath,
 				LicenseFilePath: licenseFilePath,
 				LicenseContent:  string(content),
-				LicenseSpdxID:   []string{identifier},
+				LicenseSpdxID:   identifier,
 			})
 			return nil
 		}
@@ -157,11 +171,15 @@ func (resolver *GoModeResolver) ResolvePackageLicense(p *packages.Package, repor
 	return nil
 }
 
-func (resolver *GoModeResolver) shouldStopAt(dir string) bool {
+func (resolver *GoModResolver) shouldStopAt(dir string) bool {
 	for _, srcDir := range build.Default.SrcDirs() {
 		if srcDir == dir {
 			return true
 		}
 	}
 	return false
 }
+
+func isBuiltIn(pkg *packages.Package) bool {
+	return len(pkg.GoFiles) > 0 && strings.HasPrefix(pkg.GoFiles[0], build.Default.GOROOT)
+}

pkg/deps/resolve.go

@@ -27,7 +27,7 @@ type Resolver interface {
 }
 
 var Resolvers = []Resolver{
-	new(GoModeResolver),
+	new(GoModResolver),
 }
 
 func Resolve(config *ConfigDeps, report *Report) error {

pkg/deps/result.go

@@ -34,7 +34,7 @@ type Result struct {
 	Dependency      string
 	LicenseFilePath string
 	LicenseContent  string
-	LicenseSpdxID   []string
+	LicenseSpdxID   string
 }
 
 // Report is a collection of resolved Result.
@@ -57,22 +57,18 @@ func (report *Report) String() string {
 	dWidth, lWidth := .0, .0
 	for _, r := range report.Skipped {
 		dWidth = math.Max(float64(len(r.Dependency)), dWidth)
-		for _, s := range r.LicenseSpdxID {
-			lWidth = math.Max(float64(len(s)), lWidth)
-		}
+		lWidth = math.Max(float64(len(r.LicenseSpdxID)), lWidth)
 	}
 	for _, r := range report.Resolved {
 		dWidth = math.Max(float64(len(r.Dependency)), dWidth)
-		for _, s := range r.LicenseSpdxID {
-			lWidth = math.Max(float64(len(s)), lWidth)
-		}
+		lWidth = math.Max(float64(len(r.LicenseSpdxID)), lWidth)
 	}
 
 	rowTemplate := fmt.Sprintf("%%-%dv | %%%dv\n", int(dWidth), int(lWidth))
 	s := fmt.Sprintf(rowTemplate, "Dependency", "License")
 	s += fmt.Sprintf(rowTemplate, strings.Repeat("-", int(dWidth)), strings.Repeat("-", int(lWidth)))
 	for _, r := range report.Resolved {
-		s += fmt.Sprintf(rowTemplate, r.Dependency, strings.Join(r.LicenseSpdxID, ","))
+		s += fmt.Sprintf(rowTemplate, r.Dependency, r.LicenseSpdxID)
 	}
 	for _, r := range report.Skipped {
 		s += fmt.Sprintf(rowTemplate, r.Dependency, Unknown)

pkg/license/identifier.go

@@ -20,15 +20,29 @@ package license
 import (
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/apache/skywalking-eyes/license-eye/assets"
 )
 
 const templatesDir = "assets/lcs-templates"
 
+var dualLicensePatterns = []*regexp.Regexp{
+	regexp.MustCompile(`(?i)This project is covered by two different licenses: (?P<license>[^.]+)`),
+}
+
 // Identify identifies the Spdx ID of the given license content
-func Identify(content string) (string, error) {
+func Identify(pkgPath, content string) (string, error) {
+	for _, pattern := range dualLicensePatterns {
+		matches := pattern.FindStringSubmatch(content)
+		for i, name := range pattern.SubexpNames() {
+			if name == "license" && len(matches) >= i {
+				return matches[i], nil
+			}
+		}
+	}
+
 	content = Normalize(content)
 
 	templates, err := assets.AssetDir(templatesDir)