diff --git a/module/python/python.go b/module/python/python.go index d7504a1..c7c1bf3 100644 --- a/module/python/python.go +++ b/module/python/python.go @@ -55,6 +55,7 @@ func (i Inspector) InspectProject(ctx context.Context) error { logger := logctx.Use(ctx).Sugar() task := model.UseInspectionTask(ctx) dir := task.Dir() + var nvMp = make(map[string]string) if !task.IsNoBuild() && buildout.DirHasBuildout(dir) { if err := buildout.InspectProject(ctx, dir); err != nil { logger.Warnf("buildout inspect project fail: %s", err.Error()) @@ -85,6 +86,16 @@ func (i Inspector) InspectProject(ctx context.Context) error { di.EcoRepo = EcoRepo m.Dependencies = append(m.Dependencies, di) } + if !task.IsNoBuild() { + deps, err := Run(ctx, task.Dir(), logger, nvMp) + if err != nil { + logger.Warn("construction failed, enable basic scanning") + model.UseInspectionTask(ctx).AddModule(m) + return err + } else { + m.Dependencies = append(m.Dependencies, deps...) + } + } model.UseInspectionTask(ctx).AddModule(m) return nil } diff --git a/module/python/venv.go b/module/python/venv.go new file mode 100644 index 0000000..3aaec39 --- /dev/null +++ b/module/python/venv.go @@ -0,0 +1,279 @@ +package python + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "github.com/murphysecurity/murphysec/model" + "go.uber.org/zap" + "golang.org/x/net/context" + "os" + "os/exec" + "path/filepath" + "runtime" +) + +const pipConf = `[global] +index-url = https://%s/simple/ +trusted-host = %s` + +type PipdeptreeStruct struct { + Key string `json:"key"` + PackageName string `json:"package_name"` + InstalledVersion string `json:"installed_version"` + RequiredVersion string `json:"required_version"` + Dependencies []PipdeptreeStruct `json:"dependencies"` +} + +func getVenvPath(basePath string) string { + goos := runtime.GOOS + switch goos { + case "darwin": + path := filepath.Join(basePath, "/virtual_venv/bin") + return path + case "linux": + path := filepath.Join(basePath, "/virtual_venv/bin") + return path + case "windows": + path := filepath.Join(basePath, "/virtual_venv/Scripts") + return path + } + + return "" +} + +func newVenv(dir string, logger *zap.SugaredLogger) error { + var out bytes.Buffer + cmd := exec.Command("python", "-m", "venv", "virtual_venv") + cmd.Dir = dir + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + logger.Error("new venv error :", zap.Error(err)) + return err + } + logger.Debug("new venv success ") + return nil +} +func newPipConf(basePath string, privateAddr string) error { + goos := runtime.GOOS + switch goos { + case "darwin": + path := filepath.Join(basePath, "/virtual_venv/pip.conf") + if err := os.WriteFile(path, []byte(fmt.Sprintf(pipConf, privateAddr, privateAddr)), 0777); err != nil { + return err + } + return nil + case "linux": + path := filepath.Join(basePath, "/virtual_venv/pip.conf") + if err := os.WriteFile(path, []byte(fmt.Sprintf(pipConf, privateAddr, privateAddr)), 0777); err != nil { + return err + } + return nil + case "windows": + path := filepath.Join(basePath, "/virtual_venv/pip.ini") + if err := os.WriteFile(path, []byte(fmt.Sprintf(pipConf, privateAddr, privateAddr)), 0777); err != nil { + return err + } + return nil + } + return nil +} +func pipreqs(dir string, projectPath, savePath string, logger *zap.SugaredLogger) error { + + savePath = filepath.Join(savePath, "requirements.txt") + cmd := exec.Command("./pipreqs", projectPath, "--savepath", savePath, "--encoding=utf-8") + cmd.Dir = dir + stdout, err := cmd.StdoutPipe() + if err != nil { + err = fmt.Errorf("create stdout pipe failed: %w", err) + logger.Error(err.Error()) + return err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + err = fmt.Errorf("create stderr pipe failed: %w", err) + logger.Error(err.Error()) + return err + } + + go func() { + defer stderr.Close() + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + logger.Warn("go: " + scanner.Text()) + } + }() + + if err := cmd.Start(); err != nil { + err = fmt.Errorf("start command failed: %w", err) + logger.Error(err.Error()) + return err + } + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + logger.Debug("pipreqs info: " + scanner.Text()) + } + return nil +} +func installpipreqs(dir string, logger *zap.SugaredLogger) error { + var out bytes.Buffer + var stderr bytes.Buffer + cmd := exec.Command("./pip", "install", "pipreqs") + cmd.Dir = dir + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + logger.Error(zap.String("installpipreqs error :", stderr.String())) + return err + } + logger.Debug("install pipreqs success ") + return nil +} +func installRequirements(dir string, textDir string, logger *zap.SugaredLogger) error { + var out bytes.Buffer + var stderr bytes.Buffer + by, err := os.ReadFile(textDir) + if err != nil { + logger.Error("read requirements.txt error :", zap.Error(err)) + return err + } + nvmp := parseRequirements(string(by)) + for k, v := range nvmp { + var cmd *exec.Cmd + if v != "" { + cmd = exec.Command("./pip", "install", k+"=="+v) + } else { + cmd = exec.Command("./pip", "install", k) + } + cmd.Dir = dir + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + logger.Error("install requirements error :", zap.String("stderr", stderr.String())) + } + logger.Debug("install requirements success ") + } + return nil +} +func installpipdeptree(dir string, logger *zap.SugaredLogger) error { + var out bytes.Buffer + cmd := exec.Command("./pip", "install", "pipdeptree") + cmd.Dir = dir + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + logger.Error("install pipdeptree error :", zap.Error(err)) + return err + } + logger.Debug("install pipdeptree success ") + return nil +} +func pipdeptree(dir string, logger *zap.SugaredLogger) ([]PipdeptreeStruct, error) { + var out bytes.Buffer + var result []PipdeptreeStruct + cmd := exec.Command("./pipdeptree", "--json-tree") + cmd.Stdout = &out + cmd.Dir = dir + if err := cmd.Run(); err != nil { + logger.Error("pipdeptree error :", zap.Error(err)) + return nil, err + } + if err := json.Unmarshal(out.Bytes(), &result); err != nil { + logger.Error("pipdeptree json unmarshal error :", zap.Error(err)) + return nil, err + } + logger.Debug("pipdeptree success ", zap.String("exec:", out.String())) + return result, nil +} +func updatePackage(dir string, logger *zap.SugaredLogger, k, v string) { + var out bytes.Buffer + cmd := exec.Command("./pip", "install", k+"=="+v) + cmd.Stdout = &out + cmd.Dir = dir + if err := cmd.Run(); err != nil { + logger.Error("update pip install error :", zap.Error(err)) + } + logger.Debug("update pip install success :" + k + "==" + v) +} +func buildTree(pipdeptree PipdeptreeStruct, level int) model.DependencyItem { + directDependency := false + if level == 0 { + directDependency = true + } + var mod = model.DependencyItem{ + Component: model.Component{ + CompName: pipdeptree.Key, + CompVersion: pipdeptree.InstalledVersion, + EcoRepo: model.EcoRepo{ + Ecosystem: "pip", + Repository: "", + }, + }, + IsDirectDependency: directDependency, + } + for _, i := range pipdeptree.Dependencies { + mod.Dependencies = append(mod.Dependencies, buildTree(i, level+1)) + } + return mod +} +func delVenv(dir string, logger *zap.SugaredLogger) { + if err := os.RemoveAll(dir); err != nil { + logger.Error("delete venv error :", zap.Error(err)) + return + } + logger.Debug("delete venv success ") +} +func Run(ctx context.Context, dir string, logger *zap.SugaredLogger, nvMp map[string]string) ([]model.DependencyItem, error) { + var mod []model.DependencyItem + var venvDir = filepath.Join(dir, "virtual_venv") + venvPath := getVenvPath(dir) + requirementsPath := filepath.Join(dir, "requirements.txt") + venvRequirementsPath := filepath.Join(venvPath, "requirements.txt") + if err := newVenv(dir, logger); err != nil { + return nil, err + } + if privatePath, ok := ctx.Value("privateSourceAddr").(string); ok { + logger.Debug("Use private path", zap.String("path", privatePath)) + if err := newPipConf(dir, privatePath); err != nil { + return nil, err + } + } + + if err := installpipreqs(venvPath, logger); err != nil { + return nil, err + } + if err := pipreqs(venvPath, dir, venvPath, logger); err != nil { + return nil, err + } + if err := installRequirements(venvPath, venvRequirementsPath, logger); err != nil { + return nil, err + } + if err := installpipdeptree(venvPath, logger); err != nil { + return nil, err + } + // 读取新创建的 requirements.txt + data, err := readTextFile(requirementsPath, 64*1024) + if err != nil { + logger.Warnf("read requirement: %s %v", requirementsPath, err) + return nil, err + } + // 对比原本的 requirements.txt 拿到原本包的版本 + newRequirements := parseRequirements(string(data)) + for k, v := range nvMp { + if newV, ok := newRequirements[k]; ok && newV != v { + updatePackage(venvPath, logger, k, v) + } + } + result, err := pipdeptree(venvPath, logger) + if err != nil { + return nil, err + } + for _, j := range result { + if j.Key != "" && j.Key != "pipdeptree" && j.Key != "pipreqs" && j.Key != "pip" { + mod = append(mod, buildTree(j, 0)) + } + } + defer delVenv(venvDir, logger) + return mod, err +}