Merge pull request #145 from DanielIntruder/chrome-ws-url

Allow running chrome instance to be used
Qianlitp · Jul 3, 2023 · 89c0209 · 89c0209
2 parents d31a1b4 + ec2eeec
commit 89c0209
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 1 deletion.
diff --git a/cmd/crawlergo/flag.go b/cmd/crawlergo/flag.go
@@ -9,6 +9,7 @@ import (
 
 var cliFlags = []cli.Flag{
 	SetChromePath(),
+	SetChromeWSUrl(),
 	SetCustomHeaders(),
 	SetPostData(),
 	SetMaxCrawledCount(),
@@ -47,6 +48,16 @@ func SetChromePath() *cli.PathFlag {
 	}
 }
 
+func SetChromeWSUrl() *cli.StringFlag {
+	return &cli.StringFlag{
+		Name:        "chrome-ws-url",
+		Aliases:     []string{"w"},
+		Usage:       "`URL` of chromium websockets debugger. Please note that any chromium flags specified will not apply when using this option. Such as \"ws://127.0.0.1:9222/devtools/browser/94b947d7-2a08-4cee-8e21-d77055e8c465\"",
+		Destination: &taskConfig.ChromiumWSUrl,
+		EnvVars:     []string{"CRAWLERGO_CHROME_WS_URL"},
+	}
+}
+
 func SetCustomHeaders() *cli.StringFlag {
 	return &cli.StringFlag{
 		Name:        "custom-headers",

diff --git a/pkg/engine/browser.go b/pkg/engine/browser.go
@@ -80,6 +80,25 @@ func InitBrowser(chromiumPath string, extraHeaders map[string]interface{}, proxy
 	return &bro
 }
 
+func ConnectBrowser(wsUrl string, extraHeaders map[string]interface{}) *Browser {
+	var bro Browser
+	allocCtx, cancel := chromedp.NewRemoteAllocator(context.Background(), wsUrl)
+	bctx, _ := chromedp.NewContext(allocCtx,
+		chromedp.WithLogf(log.Printf),
+	)
+
+	err := chromedp.Run(bctx)
+	if err != nil {
+		// couldn't connect to the remote browser, need to exit
+		logger.Logger.Fatal("chromedp run error: ", err.Error())
+	}
+	bro.Cancel = &cancel
+	bro.Ctx = &bctx
+	bro.ExtraHeaders = extraHeaders
+
+	return &bro
+}
+
 func (bro *Browser) NewTab(timeout time.Duration) (*context.Context, context.CancelFunc) {
 	bro.lock.Lock()
 	ctx, cancel := chromedp.NewContext(*bro.Ctx)

diff --git a/pkg/task_main.go b/pkg/task_main.go
@@ -108,7 +108,11 @@ func NewCrawlerTask(targets []*model.Request, taskConf TaskConfig) (*CrawlerTask
 		}
 	}
 
-	crawlerTask.Browser = engine2.InitBrowser(taskConf.ChromiumPath, taskConf.ExtraHeaders, taskConf.Proxy, taskConf.NoHeadless)
+	if len(taskConf.ChromiumWSUrl) > 0 {
+		crawlerTask.Browser = engine2.ConnectBrowser(taskConf.ChromiumWSUrl, taskConf.ExtraHeaders)
+	} else {
+		crawlerTask.Browser = engine2.InitBrowser(taskConf.ChromiumPath, taskConf.ExtraHeaders, taskConf.Proxy, taskConf.NoHeadless)
+	}
 	crawlerTask.RootDomain = targets[0].URL.RootDomain()
 
 	// 创建协程池

diff --git a/pkg/taskconfig.go b/pkg/taskconfig.go
@@ -17,6 +17,7 @@ type TaskConfig struct {
 	PathFromRobots          bool              // 解析Robots文件找出路径
 	MaxTabsCount            int               // 允许开启的最大标签页数量 即同时爬取的数量
 	ChromiumPath            string            // Chromium的程序路径  `/home/zhusiyu1/chrome-linux/chrome`
+	ChromiumWSUrl           string            // Websocket debugging URL for a running chrome session
 	EventTriggerMode        string            // 事件触发的调用方式： 异步 或 顺序
 	EventTriggerInterval    time.Duration     // 事件触发的间隔
 	BeforeExitDelay         time.Duration     // 退出前的等待时间，等待DOM渲染，等待XHR发出捕获