Skip to content

Commit

Permalink
Merge branch 'buildout' into 'v3'
Browse files Browse the repository at this point in the history 
Buildout

See merge request develop/client!13
  • Loading branch information
chenhaoxuan committed Dec 25, 2024
2 parents 048dd69 + 3649e78 commit 81c260c
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 61 deletions.
64 changes: 23 additions & 41 deletions module/python/buildout/buildout.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import (
"github.com/murphysecurity/murphysec/model"
"github.com/murphysecurity/murphysec/scanerr"
"github.com/murphysecurity/murphysec/utils"
"github.com/repeale/fp-go"
"golang.org/x/exp/maps"
)

func doBuildout(ctx context.Context, dir string) (errorText string, e error) {
Expand Down Expand Up @@ -104,9 +102,9 @@ func DirHasBuildout(dir string) bool {
return utils.IsFile(filepath.Join(dir, "buildout.cfg"))
}

func InspectProject(ctx context.Context, dir string) (*model.Module, error) {
func InspectProject(ctx context.Context, dir string) error {
var log = logctx.Use(ctx).Sugar()

var task = model.UseInspectionTask(ctx)
var errText, e = doBuildout(ctx, dir)
if e != nil {
log.Warnf("failed to run buildout: %s", e.Error())
Expand All @@ -117,9 +115,6 @@ func InspectProject(ctx context.Context, dir string) (*model.Module, error) {
})
}
}
var comps = make(map[[2]string]struct{})
MetadataComps := make(map[string]string)
BuildoutCfgComps := make(map[string]string)
_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, e error) error {
if ctx.Err() != nil {
return ctx.Err()
Expand All @@ -136,48 +131,35 @@ func InspectProject(ctx context.Context, dir string) (*model.Module, error) {
if e != nil || n == "" {
return nil
}
comps[[2]string{n, v}] = struct{}{}
MetadataComps[n] = v
task.AddModule(model.Module{
ModuleName: filepath.Base(path),
ModulePath: filepath.Join(dir, "METADATA"),
PackageManager: "Buildout",
Dependencies: []model.DependencyItem{
{
Component: model.Component{
CompName: n,
CompVersion: v,
EcoRepo: model.EcoRepo{
Ecosystem: "pypi",
Repository: "",
},
},
IsOnline: model.IsOnlineTrue(),
},
},
ScanStrategy: model.ScanStrategyNormal,
})
}
if d.Name() == "buildout.cfg" {
if err := base(ctx, path, BuildoutCfgComps); err != nil {
if err := base(ctx, path); err != nil {
return err
}
}
return nil
})

for k, v := range BuildoutCfgComps {
if METADATAv, ok := MetadataComps[k]; !ok || METADATAv == "" {
comps[[2]string{k, v}] = struct{}{}
MetadataComps[k] = v
}
}
var compList = maps.Keys(comps)
if len(compList) == 0 {
return nil, nil
}
var module = model.Module{
ModuleName: filepath.Dir(dir),
ModulePath: filepath.Join(dir, "buildout.cfg"),
PackageManager: "Buildout",
Dependencies: fp.Map(func(it [2]string) model.DependencyItem {
return model.DependencyItem{
Component: model.Component{
CompName: it[0],
CompVersion: it[1],
EcoRepo: model.EcoRepo{
Ecosystem: "pypi",
Repository: "",
},
},
IsOnline: model.IsOnlineTrue(),
}
})(compList),
ScanStrategy: model.ScanStrategyNormal,
}

return &module, nil
return nil
}

func parseMetadataFile(ctx context.Context, path string) (name, version string, e error) {
Expand Down
85 changes: 69 additions & 16 deletions module/python/buildout/parsingFiles.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package buildout

import (
"context"
"github.com/murphysecurity/murphysec/model"
"io"
"net/http"
"os"
Expand All @@ -13,9 +14,9 @@ import (
"gopkg.in/ini.v1"
)

func base(ctx context.Context, path string, result map[string]string) error {
func base(ctx context.Context, path string) error {
var log = logctx.Use(ctx).Sugar()
e := findVersionsFile(ctx, path, result)
e := findVersionsFile(ctx, path)
if e != nil {
return e
}
Expand All @@ -29,13 +30,13 @@ func base(ctx context.Context, path string, result map[string]string) error {
if j == "buildout.cfg" {
continue
}
if err := NoCurrentDirectoryCfg(ctx, filepath.Dir(j), j, result); err != nil {
if err := NoCurrentDirectoryCfg(ctx, filepath.Dir(j), j); err != nil {
return err
}
}
return nil
}
func NoCurrentDirectoryCfg(ctx context.Context, NowPath string, path string, result map[string]string) error {
func NoCurrentDirectoryCfg(ctx context.Context, NowPath string, path string) error {
var log = logctx.Use(ctx).Sugar()
var e error
var extends []string
Expand All @@ -57,25 +58,32 @@ func NoCurrentDirectoryCfg(ctx context.Context, NowPath string, path string, res
log.Error("read body failed", zap.Error(err))
return err
}
extends, e = parseBuildoutBytes(ctx, by, result)
var urlPath = ""
if strings.Contains(path, "http://") {
urlPath = strings.ReplaceAll(path, "http://", "")
}
if strings.Contains(path, "https://") {
urlPath = strings.ReplaceAll(path, "https://", "")
}
extends, e = parseBuildoutBytes(ctx, by, urlPath)
if e != nil {
return e
}
} else {
extends, e = parseBuildoutCfgFile(ctx, path, result)
extends, e = parseBuildoutCfgFile(ctx, path)
if e != nil {
return e
}
}
for _, j := range extends {
if j != "" {
log.Debug("find extends", zap.String("path", j))
e = findVersionsFile(ctx, j, result)
e = findVersionsFile(ctx, j)
if e != nil {
return e
}
} else {
e = NoCurrentDirectoryCfg(ctx, NowPath, j, result)
e = NoCurrentDirectoryCfg(ctx, NowPath, j)
if e != nil {
return e
}
Expand All @@ -84,7 +92,7 @@ func NoCurrentDirectoryCfg(ctx context.Context, NowPath string, path string, res

return nil
}
func findVersionsFile(ctx context.Context, path string, result map[string]string) error {
func findVersionsFile(ctx context.Context, path string) error {
var log = logctx.Use(ctx).Sugar()
var extends []string
var e error
Expand All @@ -101,13 +109,20 @@ func findVersionsFile(ctx context.Context, path string, result map[string]string
log.Error("read body failed", zap.Error(err))
return err
}
extends, e = parseBuildoutBytes(ctx, by, result)
var urlPath = ""
if strings.Contains(path, "http://") {
urlPath = strings.ReplaceAll(path, "http://", "")
}
if strings.Contains(path, "https://") {
urlPath = strings.ReplaceAll(path, "https://", "")
}
extends, e = parseBuildoutBytes(ctx, by, "[Remote]/"+urlPath)
if e != nil {
return e
}
} else {
// 如果不是远程链接 则尝试打开读取
extends, e = parseBuildoutCfgFile(ctx, path, result)
extends, e = parseBuildoutCfgFile(ctx, path)
if e != nil {
return e
}
Expand All @@ -116,7 +131,7 @@ func findVersionsFile(ctx context.Context, path string, result map[string]string
for _, j := range extends {
if j != "" {
log.Debug("find extends", zap.String("path", j))
e = findVersionsFile(ctx, j, result)
e = findVersionsFile(ctx, j)
if e != nil {
log.Error("find file error:", zap.Error(e))
continue
Expand All @@ -126,8 +141,10 @@ func findVersionsFile(ctx context.Context, path string, result map[string]string
}
return nil
}
func parseBuildoutBytes(ctx context.Context, by []byte, result map[string]string) ([]string, error) {
func parseBuildoutBytes(ctx context.Context, by []byte, path string) ([]string, error) {
var log = logctx.Use(ctx).Sugar()
var task = model.UseInspectionTask(ctx)
var dep []model.DependencyItem
cfg, err := ini.LoadSources(ini.LoadOptions{
AllowPythonMultilineValues: true,
}, by)
Expand All @@ -140,11 +157,28 @@ func parseBuildoutBytes(ctx context.Context, by []byte, result map[string]string
for _, key := range section.Keys() {
if key.Name() != "" && key.Value() != "" {
log.Debug("buildout bytes :", zap.String(key.Name(), key.Value()))
result[key.Name()] = key.Value()
dep = append(dep, model.DependencyItem{
Component: model.Component{
CompName: key.Name(),
CompVersion: key.Value(),
EcoRepo: model.EcoRepo{
Ecosystem: "pypi",
Repository: "",
},
},
IsOnline: model.IsOnlineTrue(),
})
}
}
}
}
task.AddModule(model.Module{
ModuleName: filepath.Base(task.Dir()),
ModulePath: path,
PackageManager: "Buildout",
Dependencies: dep,
ScanStrategy: model.ScanStrategyNormal,
})
var resultStrings []string
extends := cfg.Section("buildout").Key("extends").Strings("\n")
if len(extends) == 0 {
Expand All @@ -157,8 +191,10 @@ func parseBuildoutBytes(ctx context.Context, by []byte, result map[string]string
}
return resultStrings, nil
}
func parseBuildoutCfgFile(ctx context.Context, path string, result map[string]string) ([]string, error) {
func parseBuildoutCfgFile(ctx context.Context, path string) ([]string, error) {
var log = logctx.Use(ctx).Sugar()
var dep []model.DependencyItem
var task = model.UseInspectionTask(ctx)
by, err := os.ReadFile(path)
if err != nil {
log.Error("read file failed", zap.Error(err))
Expand All @@ -177,11 +213,28 @@ func parseBuildoutCfgFile(ctx context.Context, path string, result map[string]st
for _, key := range section.Keys() {
if key.Name() != "" && key.Value() != "" {
log.Debug("from path:", zap.String(path, key.Name()))
result[key.Name()] = key.Value()
dep = append(dep, model.DependencyItem{
Component: model.Component{
CompName: key.Name(),
CompVersion: key.Value(),
EcoRepo: model.EcoRepo{
Ecosystem: "pypi",
Repository: "",
},
},
IsOnline: model.IsOnlineTrue(),
})
}
}
}
}
task.AddModule(model.Module{
ModuleName: filepath.Base(task.Dir()),
ModulePath: path,
PackageManager: "Buildout",
Dependencies: dep,
ScanStrategy: model.ScanStrategyNormal,
})
var resultStrings []string
extends := cfg.Section("buildout").Key("extends").Strings("\n")
if len(extends) == 0 {
Expand Down
5 changes: 1 addition & 4 deletions module/python/python.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,7 @@ func (i Inspector) InspectProject(ctx context.Context) error {
task := model.UseInspectionTask(ctx)
dir := task.Dir()
if !task.IsNoBuild() && buildout.DirHasBuildout(dir) {
module, _ := buildout.InspectProject(ctx, dir)
if module != nil && len(module.Dependencies) > 0 {
task.AddModule(*module)
}
buildout.InspectProject(ctx, dir)

Check failure on line 59 in module/python/python.go

View workflow job for this annotation

GitHub Actions / lint 

Error return value of `buildout.InspectProject` is not checked (errcheck)

Check failure on line 59 in module/python/python.go

View workflow job for this annotation

GitHub Actions / lint 

Error return value of `buildout.InspectProject` is not checked (errcheck)
}
info, e := collectDepsInfo(ctx, dir)
if e != nil {
Expand Down

0 comments on commit 81c260c

Please sign in to comment.