diff --git a/module/python/python.go b/module/python/python.go index 4ac3c802..d3bdeb99 100644 --- a/module/python/python.go +++ b/module/python/python.go @@ -122,7 +122,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) { return nil } if isDockerfile(filename) { - data, e := readFile(path, 64*1024) // Dockerfile max 64K + data, e := readTextFile(path, 64*1024) // Dockerfile max 64K if e != nil { logger.Warnf("read dockerfile: %s %v", path, e) return nil @@ -133,7 +133,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) { return nil } if isRequirementsFile(filename) { - data, e := readFile(path, 64*1024) + data, e := readTextFile(path, 64*1024) if e != nil { logger.Warnf("read requirement: %s %v", path, e) return nil @@ -148,7 +148,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) { return nil } if filepath.Ext(filename) == ".py" { - data, e := readFile(path, 256*1024) + data, e := readTextFile(path, 256*1024) if e != nil { logger.Warnf("read py: %s %v", path, e) return nil diff --git a/module/python/utils.go b/module/python/utils.go index 50e10789..dd346331 100644 --- a/module/python/utils.go +++ b/module/python/utils.go @@ -1,15 +1,25 @@ package python import ( + "golang.org/x/net/html/charset" "io" "os" ) -func readFile(path string, maxLength int) ([]byte, error) { +func readTextFile(path string, maxLength int) ([]byte, error) { f, e := os.Open(path) if e != nil { return nil, e } defer func() { _ = f.Close() }() - return io.ReadAll(io.LimitReader(f, int64(maxLength))) + r, e := charset.NewReader(f, "") + if e != nil { + _ = f.Close() + f, e = os.Open(path) + if e != nil { + return nil, e + } + r = f + } + return io.ReadAll(io.LimitReader(r, int64(maxLength))) }