diff --git a/pkg/task_main.go b/pkg/task_main.go index 275249e..78e5815 100755 --- a/pkg/task_main.go +++ b/pkg/task_main.go @@ -73,7 +73,8 @@ func NewCrawlerTask(targets []*model.Request, taskConf TaskConfig) (*CrawlerTask } // 业务代码与数据代码分离, 初始化一些默认配置 - taskConf.SetConf( + // 使用 funtion option 和一个代理来初始化 taskConf 的配置 + for _, fn := range []TaskConfigOptFunc{ WithTabRunTimeout(config.TabRunTimeout), WithMaxTabsCount(config.MaxTabsCount), WithMaxCrawlCount(config.MaxCrawlCount), @@ -82,7 +83,9 @@ func NewCrawlerTask(targets []*model.Request, taskConf TaskConfig) (*CrawlerTask WithBeforeExitDelay(config.BeforeExitDelay), WithEventTriggerMode(config.DefaultEventTriggerMode), WithIgnoreKeywords(config.DefaultIgnoreKeywords), - ) + } { + fn(&taskConf) + } if taskConf.ExtraHeadersString != "" { err := json.Unmarshal([]byte(taskConf.ExtraHeadersString), &taskConf.ExtraHeaders) diff --git a/pkg/taskconfig.go b/pkg/taskconfig.go index 6f8ec1b..9391c65 100644 --- a/pkg/taskconfig.go +++ b/pkg/taskconfig.go @@ -38,12 +38,6 @@ func NewTaskConfig(optFuncs ...TaskConfigOptFunc) *TaskConfig { return conf } -func (t *TaskConfig) SetConf(optFuncs ...TaskConfigOptFunc) { - for _, fn := range optFuncs { - fn(t) - } -} - func WithMaxCrawlCount(maxCrawlCount int) TaskConfigOptFunc { return func(tc *TaskConfig) { if tc.MaxCrawlCount == 0 { diff --git a/pkg/taskconfig_test.go b/pkg/taskconfig_test.go new file mode 100644 index 0000000..7d96fc4 --- /dev/null +++ b/pkg/taskconfig_test.go @@ -0,0 +1,45 @@ +package pkg_test + +import ( + "testing" + "time" + + "github.com/Qianlitp/crawlergo/pkg" + "github.com/Qianlitp/crawlergo/pkg/config" + "github.com/stretchr/testify/assert" +) + +func TestTaskConfigOptFunc(t *testing.T) { + // 测试 https://github.com/Qianlitp/crawlergo/pull/101 修改的代码 + var taskConf pkg.TaskConfig + for _, fn := range []pkg.TaskConfigOptFunc{ + pkg.WithTabRunTimeout(config.TabRunTimeout), + pkg.WithMaxTabsCount(config.MaxTabsCount), + pkg.WithMaxCrawlCount(config.MaxCrawlCount), + pkg.WithDomContentLoadedTimeout(config.DomContentLoadedTimeout), + pkg.WithEventTriggerInterval(config.EventTriggerInterval), + pkg.WithBeforeExitDelay(config.BeforeExitDelay), + pkg.WithEventTriggerMode(config.DefaultEventTriggerMode), + pkg.WithIgnoreKeywords(config.DefaultIgnoreKeywords), + } { + fn(&taskConf) + } + + // 应该都要等于默认配置 + assert.Equal(t, taskConf.TabRunTimeout, config.TabRunTimeout) + assert.Equal(t, taskConf.MaxTabsCount, config.MaxTabsCount) + assert.Equal(t, taskConf.MaxCrawlCount, config.MaxCrawlCount) + assert.Equal(t, taskConf.DomContentLoadedTimeout, config.DomContentLoadedTimeout) + assert.Equal(t, taskConf.EventTriggerInterval, config.EventTriggerInterval) + assert.Equal(t, taskConf.BeforeExitDelay, config.BeforeExitDelay) + assert.Equal(t, taskConf.EventTriggerMode, config.DefaultEventTriggerMode) + assert.Equal(t, taskConf.IgnoreKeywords, config.DefaultIgnoreKeywords) + + // 重设超时时间 + taskConf.TabRunTimeout = time.Minute * 5 + + // 企图覆盖自定义的时间, 不应该允许, 程序初始化时只能配置一次, 先由用户配置 + pkg.WithTabRunTimeout(time.Second * 5)(&taskConf) + assert.NotEqual(t, taskConf.TabRunTimeout, time.Second*5) + assert.NotEqual(t, taskConf.TabRunTimeout, config.TabRunTimeout) +}