fix(python): handling non-utf8 encoding

module/python/python.go

@@ -122,7 +122,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) {
 			return nil
 		}
 		if isDockerfile(filename) {
-			data, e := readFile(path, 64*1024) // Dockerfile max 64K
+			data, e := readTextFile(path, 64*1024) // Dockerfile max 64K
 			if e != nil {
 				logger.Warnf("read dockerfile: %s %v", path, e)
 				return nil
@@ -133,7 +133,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) {
 			return nil
 		}
 		if isRequirementsFile(filename) {
-			data, e := readFile(path, 64*1024)
+			data, e := readTextFile(path, 64*1024)
 			if e != nil {
 				logger.Warnf("read requirement: %s %v", path, e)
 				return nil
@@ -148,7 +148,7 @@ func collectDepsInfo(ctx context.Context, dir string) ([][2]string, error) {
 			return nil
 		}
 		if filepath.Ext(filename) == ".py" {
-			data, e := readFile(path, 256*1024)
+			data, e := readTextFile(path, 256*1024)
 			if e != nil {
 				logger.Warnf("read py: %s %v", path, e)
 				return nil

module/python/utils.go

@@ -1,15 +1,25 @@
 package python
 
 import (
+	"golang.org/x/net/html/charset"
 	"io"
 	"os"
 )
 
-func readFile(path string, maxLength int) ([]byte, error) {
+func readTextFile(path string, maxLength int) ([]byte, error) {
 	f, e := os.Open(path)
 	if e != nil {
 		return nil, e
 	}
 	defer func() { _ = f.Close() }()
-	return io.ReadAll(io.LimitReader(f, int64(maxLength)))
+	r, e := charset.NewReader(f, "")
+	if e != nil {
+		_ = f.Close()
+		f, e = os.Open(path)
+		if e != nil {
+			return nil, e
+		}
+		r = f
+	}
+	return io.ReadAll(io.LimitReader(r, int64(maxLength)))
 }