Merge pull request #4983 from projectdiscovery/dev

v3.2.3

.gitignore

@@ -35,4 +35,6 @@ pkg/protocols/headless/engine/.cache
 **/*-cache
 /fuzzplayground
 integration_tests/fuzzplayground
+/dsl.md
+
 

Makefile

@@ -47,4 +47,7 @@ fuzzplayground:
 memogen:
 	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go
 	./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl
-
+dsl-docs:
+	rm -f dsl.md scrapefuncs 2>/dev/null
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go
+	./scrapefuncs -out dsl.md

README.md

@@ -221,7 +221,8 @@ INTERACTSH:
 FUZZING:
    -ft, -fuzzing-type string  overrides fuzzing type set in template (replace, prefix, postfix, infix)
    -fm, -fuzzing-mode string  overrides fuzzing mode set in template (multiple, single)
-   -fuzz                      enable loading fuzzing templates
+   -fuzz                      enable loading fuzzing templates (Deprecated: use -dast instead)
+   -dast                      only run DAST templates
 
 UNCOVER:
    -uc, -uncover                  enable uncover engine

README_CN.md

@@ -57,7 +57,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主
 
 # 安装Nuclei
 
-Nuclei需要**go1**才能安装成功。执行下列命令安装最新版本的Nuclei
+Nuclei需要 **go1.21** 才能安装成功。执行下列命令安装最新版本的Nuclei
 
 ```sh
 go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
@@ -396,8 +396,8 @@ Nuclei通过增加手动、自动的过程，极大地改变了安全评估的
 
 Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei可以随时扫描来了解安全威胁。Nuclei通常用来用于复测，以确定漏洞是否被修复。
 
-- **CI/CD：**工程师已经支持了CI/CD，可以通过Nuclei使用定制模板来监控模拟环境和生产环境
-- **周期性扫描：**使用Nuclei创建新发现的漏洞模板，通过Nuclei可以周期性扫描消除漏洞
+- **CI/CD：** 工程师已经支持了CI/CD，可以通过Nuclei使用定制模板来监控模拟环境和生产环境
+- **周期性扫描：** 使用Nuclei创建新发现的漏洞模板，通过Nuclei可以周期性扫描消除漏洞
 
 我们有个[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)，黑客提交自己的模板后可以获得赏金，这可以减少资产的漏洞，并且减少重复。如果你想实行该计划，可以[联系我](mailto:[email protected])。我们非常乐意提供帮助，或者在[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)中发布相关信息。
 

SYNTAX-REFERENCE.md

@@ -379,6 +379,7 @@ Stop execution once first match is found
 <div class="dt">
 
 Signature is the request signature method
+WARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks
 
 
 Valid values:
@@ -1613,25 +1614,25 @@ DisablePathAutomerge disables merging target url path with raw request path
 
 <div class="dd">
 
-<code>filters</code>  <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
+<code>pre-condition</code>  <i>[]<a href="#matchersmatcher">matchers.Matcher</a></i>
 
 </div>
 <div class="dt">
 
-Filter is matcher-like field to check if fuzzing should be performed on this request or not
+Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not
 
 </div>
 
 <hr />
 
 <div class="dd">
 
-<code>filters-condition</code>  <i>string</i>
+<code>pre-condition-operator</code>  <i>string</i>
 
 </div>
 <div class="dt">
 
-Filter condition is the condition to apply on the filter (AND/OR). Default is OR
+FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR
 
 </div>
 
@@ -1981,7 +1982,7 @@ Matcher is used to match a part in the output from a protocol.
 Appears in:
 
 
-- <code><a href="#httprequest">http.Request</a>.filters</code>
+- <code><a href="#httprequest">http.Request</a>.pre-condition</code>
 
 
 
@@ -4156,6 +4157,19 @@ Engine type
 
 <div class="dd">
 
+<code>pre-condition</code>  <i>string</i>
+
+</div>
+<div class="dt">
+
+PreCondition is a condition which is evaluated before sending the request.
+
+</div>
+
+<hr />
+
+<div class="dd">
+
 <code>args</code>  <i>[]string</i>
 
 </div>

cmd/docgen/docgen.go

@@ -8,7 +8,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/alecthomas/jsonschema"
+	"github.com/invopop/jsonschema"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 )
@@ -32,11 +32,7 @@ func main() {
 	}
 
 	// Generate jsonschema
-	r := &jsonschema.Reflector{
-		PreferYAMLSchema:      true,
-		YAMLEmbeddedStructs:   true,
-		FullyQualifyTypeNames: true,
-	}
+	r := &jsonschema.Reflector{}
 	jsonschemaData := r.Reflect(&templates.Template{})
 
 	var buf bytes.Buffer

cmd/integration-test/code.go

@@ -23,6 +23,7 @@ var codeTestCases = []TestCaseInfo{
 	{Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled},
 	{Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled},
 	{Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }},
+	{Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled},
 }
 
 const (
@@ -94,6 +95,22 @@ func (h *codeSnippet) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type codePreCondition struct{}
+
+// Execute executes a test case and returns an error if occurred
+func (h *codePreCondition) Execute(filePath string) error {
+	results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code")
+	if err != nil {
+		return err
+	}
+	if osutils.IsLinux() {
+		return expectResultsCount(results, 1)
+	} else {
+		return expectResultsCount(results, 0)
+
+	}
+}
+
 type codeFile struct{}
 
 // Execute executes a test case and returns an error if occurred

cmd/integration-test/fuzz.go

@@ -21,8 +21,6 @@ var fuzzingTestCases = []TestCaseInfo{
 	{Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}},
 	{Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}},
 	{Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}},
-	{Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}},
-	{Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}},
 	// for fuzzing we should prioritize adding test case related backend
 	// logic in fuzz playground server instead of adding them here
 	{Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}},
@@ -176,52 +174,3 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error {
 	}
 	return expectResultsCount(got, 2)
 }
-
-type FuzzHeaderBasic struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *FuzzHeaderBasic) Execute(filePath string) error {
-	router := httprouter.New()
-	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		host := r.Header.Get("Origin")
-		// redirect to different domain
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprintf(w, "<html><body><a href="+host+">Click Here</a></body></html>")
-	})
-	ts := httptest.NewTLSServer(router)
-	defer ts.Close()
-
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(got, 1)
-}
-
-type FuzzHeaderMultiple struct{}
-
-// Execute executes a test case and returns an error if occurred
-func (h *FuzzHeaderMultiple) Execute(filePath string) error {
-	router := httprouter.New()
-	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
-		host1 := r.Header.Get("Origin")
-		host2 := r.Header.Get("X-Forwared-For")
-
-		fmt.Printf("host1: %s, host2: %s\n", host1, host2)
-		if host1 == host2 && host2 == "secret.local" {
-			w.WriteHeader(http.StatusOK)
-			fmt.Fprintf(w, "welcome! to secret admin panel")
-			return
-		}
-		// redirect to different domain
-		w.WriteHeader(http.StatusForbidden)
-	})
-	ts := httptest.NewTLSServer(router)
-	defer ts.Close()
-
-	got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz")
-	if err != nil {
-		return err
-	}
-	return expectResultsCount(got, 1)
-}

cmd/integration-test/http.go

@@ -82,6 +82,8 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}},
 	{Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}},
 	{Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}},
+	{Path: "protocols/http/raw-path-single-slash.yaml", TestCase: &httpRawPathSingleSlash{}},
+	{Path: "protocols/http/raw-unsafe-path-single-slash.yaml", TestCase: &httpRawUnsafePathSingleSlash{}},
 }
 
 type httpMultiVarSharing struct{}
@@ -1560,3 +1562,53 @@ func (h *httpMultiRequest) Execute(filePath string) error {
 
 	return expectResultsCount(results, 1)
 }
+
+type httpRawPathSingleSlash struct{}
+
+func (h *httpRawPathSingleSlash) Execute(filepath string) error {
+	expectedPath := "/index.php"
+	results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh/index.php", "-debug-req"})
+	if err != nil {
+		return err
+	}
+
+	var actual string
+	for _, v := range strings.Split(results, "\n") {
+		if strings.Contains(v, "GET") {
+			parts := strings.Fields(v)
+			if len(parts) == 3 {
+				actual = parts[1]
+			}
+		}
+	}
+
+	if actual != expectedPath {
+		return fmt.Errorf("expected: %v\n\nactual: %v", expectedPath, actual)
+	}
+	return nil
+}
+
+type httpRawUnsafePathSingleSlash struct{}
+
+func (h *httpRawUnsafePathSingleSlash) Execute(filepath string) error {
+	expectedPath := "/index.php"
+	results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh/index.php", "-debug-req"})
+	if err != nil {
+		return err
+	}
+
+	var actual string
+	for _, v := range strings.Split(results, "\n") {
+		if strings.Contains(v, "GET") {
+			parts := strings.Fields(v)
+			if len(parts) == 3 {
+				actual = parts[1]
+			}
+		}
+	}
+
+	if actual != expectedPath {
+		return fmt.Errorf("expected: %v\n\nactual: %v", expectedPath, actual)
+	}
+	return nil
+}

cmd/nuclei/main.go

@@ -186,6 +186,7 @@ func readConfig() *goflags.FlagSet {
 	// when true updates nuclei binary to latest version
 	var updateNucleiBinary bool
 	var pdcpauth string
+	var fuzzFlag bool
 
 	flagSet := goflags.NewFlagSet()
 	flagSet.CaseSensitive = true
@@ -313,7 +314,8 @@ on extensive configurability, massive extensibility and ease of use.`)
 	flagSet.CreateGroup("fuzzing", "Fuzzing",
 		flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"),
 		flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"),
-		flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable loading fuzzing templates"),
+		flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
+		flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
 	)
 
 	flagSet.CreateGroup("uncover", "Uncover",
@@ -436,6 +438,12 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started
 	goflags.DisableAutoConfigMigration = true
 	_ = flagSet.Parse()
 
+	// when fuzz flag is enabled, set the dast flag to true
+	if fuzzFlag {
+		// backwards compatibility for fuzz flag
+		options.DAST = true
+	}
+
 	// api key hierarchy: cli flag > env var > .pdcp/credential file
 	if pdcpauth == "true" {
 		runner.AuthWithPDCP()