Merge pull request #366 from Kostov6/enh/remove-document-and-resource…

…-jobs

Remove document and resource jobs

cmd/app/exec.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gardener/docforge/pkg/registry/repositoryhost"
 	"github.com/gardener/docforge/pkg/workers/document"
 	"github.com/gardener/docforge/pkg/workers/githubinfo"
+	"github.com/gardener/docforge/pkg/workers/linkresolver"
 	"github.com/gardener/docforge/pkg/workers/linkvalidator"
 	"github.com/gardener/docforge/pkg/workers/resourcedownloader"
 	"github.com/gardener/docforge/pkg/workers/taskqueue"
@@ -59,20 +60,31 @@ func exec(ctx context.Context) error {
 		fmt.Println(documentNodes[0])
 	}
 
-	dScheduler, downloadTasks, err := resourcedownloader.New(config.ResourceDownloadWorkersCount, config.FailFast, reactorWG, rhRegistry, config.ResourceDownloadWriter)
+	downloader, err := resourcedownloader.NewDownloader(rhRegistry, config.ResourceDownloadWriter)
 	if err != nil {
 		return err
 	}
 	v, validatorTasks, err := linkvalidator.New(config.ValidationWorkersCount, config.FailFast, reactorWG, rhRegistry, config.HostsToReport)
 	if err != nil {
 		return err
 	}
-	docProcessor, docTasks, err := document.New(config.DocumentWorkersCount, config.FailFast, reactorWG, documentNodes, config.ResourcesWebsitePath, dScheduler, v, rhRegistry, config.Hugo, config.Writer, config.SkipLinkValidation)
-	if err != nil {
-		return err
+	lr := &linkresolver.LinkResolver{
+		Repositoryhosts: rhRegistry,
+		Hugo:            config.Hugo,
+		SourceToNode:    make(map[string][]*manifest.Node),
+	}
+	for _, node := range documentNodes {
+		if node.Source != "" {
+			lr.SourceToNode[node.Source] = append(lr.SourceToNode[node.Source], node)
+		} else if len(node.MultiSource) > 0 {
+			for _, s := range node.MultiSource {
+				lr.SourceToNode[s] = append(lr.SourceToNode[s], node)
+			}
+		}
 	}
+	worker := document.NewDocumentWorker(config.ResourcesWebsitePath, downloader, v, lr, rhRegistry, config.Hugo, config.Writer, config.SkipLinkValidation)
 
-	qcc := taskqueue.NewQueueControllerCollection(reactorWG, downloadTasks, validatorTasks, docTasks)
+	qcc := taskqueue.NewQueueControllerCollection(reactorWG, validatorTasks)
 
 	if config.GitInfoWriter != nil {
 		ghInfo, ghInfoTasks, err = githubinfo.New(config.ResourceDownloadWorkersCount, config.FailFast, reactorWG, rhRegistry, config.GitInfoWriter)
@@ -86,7 +98,7 @@ func exec(ctx context.Context) error {
 	}
 
 	for _, node := range documentNodes {
-		docProcessor.ProcessNode(node)
+		worker.ProcessNode(ctx, node)
 	}
 
 	qcc.Start(ctx)

cmd/app/flags.go

@@ -44,10 +44,6 @@ func configureFlags(command *cobra.Command) {
 		"Runs the command end-to-end but instead of writing files, it will output the projected file/folder hierarchy to the standard output and statistics for the processing of each file.")
 	_ = vip.BindPFlag("dry-run", command.Flags().Lookup("dry-run"))
 
-	command.Flags().Int("document-workers", 25,
-		"Number of parallel workers for document processing.")
-	_ = vip.BindPFlag("document-workers", command.Flags().Lookup("document-workers"))
-
 	command.Flags().Int("validation-workers", 10,
 		"Number of parallel workers to validate the markdown links")
 	_ = vip.BindPFlag("validation-workers", command.Flags().Lookup("validation-workers"))

cmd/app/types.go

@@ -13,7 +13,6 @@ import (
 // Options encapsulates the parameters for creating
 // new Reactor objects
 type Options struct {
-	DocumentWorkersCount         int      `mapstructure:"document-workers"`
 	ValidationWorkersCount       int      `mapstructure:"validation-workers"`
 	FailFast                     bool     `mapstructure:"fail-fast"`
 	DestinationPath              string   `mapstructure:"destination"`

pkg/workers/document/document_worker.go

@@ -137,6 +137,7 @@ func (d *Worker) process(ctx context.Context, b *bytes.Buffer, n *manifest.Node)
 			*d,
 			n,
 			cnt.docURI,
+			ctx,
 		}
 		if strings.HasSuffix(cnt.docURI, ".md") {
 			rnd := markdown.NewLinkModifierRenderer(markdown.WithLinkResolver(lrt.resolveLink))
@@ -170,6 +171,7 @@ type linkResolverTask struct {
 	Worker
 	node   *manifest.Node
 	source string
+	ctx    context.Context
 }
 
 // DownloadURLName create resource name that will be dowloaded from a resource link
@@ -230,7 +232,7 @@ func (d *linkResolverTask) resolveEmbededLink(link string, source string) (strin
 	}
 	// download urls from referenced repositories
 	downloadResourceName := DownloadURLName(*resourceURL)
-	if err = d.downloader.Schedule(link, downloadResourceName, source); err != nil {
+	if err = d.downloader.Download(d.ctx, link, downloadResourceName, source); err != nil {
 		return link, err
 	}
 	return "/" + path.Join(d.hugo.BaseURL, d.resourcesRoot, downloadResourceName), nil

pkg/workers/document/document_worker_test.go

@@ -19,7 +19,7 @@ import (
 	"github.com/gardener/docforge/pkg/workers/document"
 	"github.com/gardener/docforge/pkg/workers/linkresolver/linkresolverfakes"
 	"github.com/gardener/docforge/pkg/workers/linkvalidator/linkvalidatorfakes"
-	"github.com/gardener/docforge/pkg/workers/resourcedownloader/downloaderfakes"
+	"github.com/gardener/docforge/pkg/workers/resourcedownloader/resourcedownloaderfakes"
 	"github.com/gardener/docforge/pkg/writers/writersfakes"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -46,7 +46,7 @@ var _ = Describe("Document resolving", func() {
 			BaseURL:        "baseURL",
 			IndexFileNames: []string{"readme.md", "readme", "read.me", "index.md", "index"},
 		}
-		df := &downloaderfakes.FakeInterface{}
+		df := &resourcedownloaderfakes.FakeInterface{}
 		vf := &linkvalidatorfakes.FakeInterface{}
 		lrf := &linkresolverfakes.FakeInterface{}
 		lrf.ResolveResourceLinkCalls(func(s1 string, n *manifest.Node, s2 string) (string, error) {