Add --include-path flag (#155)

spec/unit_test/models/endpoint_spec.cr

@@ -24,4 +24,41 @@ describe "Initialize 3 arguments" do
   it "detect_params" do
     endpoint.params.should eq([Param.new("a", "b", "query")])
   end
+
+  path = "path/a/b/c"
+  line = 123
+  path_info = PathInfo.new(path, line)
+  endpoint2 = Endpoint.new("/abcd", "GET", Details.new(path_info))
+  it "detect_url" do
+    endpoint2.url.should eq("/abcd")
+  end
+  it "detect_method" do
+    endpoint2.method.should eq("GET")
+  end
+  it "detect_details" do
+    endpoint2.details.should eq(Details.new(path_info))
+    endpoint2.details.code_paths[0].path.should eq(path)
+    endpoint2.details.code_paths[0].line.should eq(line)
+  end
+end
+
+describe "Initialize 4 arguments" do
+  path = "path/a/b/c"
+  line = 123
+  path_info = PathInfo.new(path, line)
+  endpoint = Endpoint.new("/abcd", "GET", [Param.new("a", "b", "query")], Details.new(path_info))
+  it "detect_url" do
+    endpoint.url.should eq("/abcd")
+  end
+  it "detect_method" do
+    endpoint.method.should eq("GET")
+  end
+  it "detect_params" do
+    endpoint.params.should eq([Param.new("a", "b", "query")])
+  end
+  it "detect_details" do
+    endpoint.details.should eq(Details.new(path_info))
+    endpoint.details.code_paths[0].path.should eq(path)
+    endpoint.details.code_paths[0].line.should eq(line)
+  end
 end

src/analyzer/analyzers/analyzer_armeria.cr

@@ -12,6 +12,8 @@ class AnalyzerArmeria < Analyzer
         next if File.directory?(path)
 
         if File.exists?(path) && (path.ends_with?(".java") || path.ends_with?(".kt"))
+          details = Details.new(PathInfo.new(path))
+
           content = File.read(path, encoding: "utf-8", invalid: :skip)
           content.scan(REGEX_SERVER_CODE_BLOCK) do |server_codeblcok_match|
             server_codeblock = server_codeblcok_match[0]
@@ -30,7 +32,7 @@ class AnalyzerArmeria < Analyzer
               endpoint = split_params[endpoint_param_index].strip
 
               endpoint = endpoint[1..-2]
-              @result << Endpoint.new("#{endpoint}", "GET")
+              @result << Endpoint.new("#{endpoint}", "GET", details)
             end
 
             server_codeblock.scan(REGEX_ROUTE_CODE) do |route_code_match|
@@ -47,7 +49,7 @@ class AnalyzerArmeria < Analyzer
               next if endpoint[0] != '"'
 
               endpoint = endpoint[1..-2]
-              @result << Endpoint.new("#{endpoint}", method)
+              @result << Endpoint.new("#{endpoint}", method, details)
             end
           end
         end

src/analyzer/analyzers/analyzer_crystal_kemal.cr

@@ -9,9 +9,11 @@ class AnalyzerCrystalKemal < Analyzer
         if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(PathInfo.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

src/analyzer/analyzers/analyzer_crystal_lucky.cr

@@ -21,9 +21,11 @@ class AnalyzerCrystalLucky < Analyzer
         if File.exists?(path) && File.extname(path) == ".cr" && !path.includes?("lib")
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(PathInfo.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

src/analyzer/analyzers/analyzer_cs_aspnet_mvc.cr

@@ -11,7 +11,7 @@ class AnalyzerCsAspNetMvc < Analyzer
         maproute_check = false
         maproute_buffer = ""
 
-        file.each_line do |line|
+        file.each_line.with_index do |line, index|
           if line.includes? ".MapRoute("
             maproute_check = true
             maproute_buffer = line
@@ -25,7 +25,8 @@ class AnalyzerCsAspNetMvc < Analyzer
               buffer.split(",").each do |item|
                 if item.includes? "url:"
                   url = item.gsub(/url:/, "").gsub(/"/, "")
-                  @result << Endpoint.new("/#{url}", "GET")
+                  details = Details.new(PathInfo.new(route_config_file, index + 1))
+                  @result << Endpoint.new("/#{url}", "GET", details)
                 end
               end
 

src/analyzer/analyzers/analyzer_django.cr

@@ -112,7 +112,6 @@ class AnalyzerDjango < AnalyzerPython
       route = route.gsub(/^\^/, "").gsub(/\$$/, "")
       view = route_match[2].split(",")[0]
       url = "/#{django_urls.prefix}/#{route}".gsub(/\/+/, "/")
-
       new_django_urls = nil
       view.scan(REGEX_INCLUDE_URLS) do |include_pattern_match|
         # Detect new url configs
@@ -121,15 +120,18 @@ class AnalyzerDjango < AnalyzerPython
 
         if File.exists?(new_route_path)
           new_django_urls = DjangoUrls.new("#{django_urls.prefix}#{route}", new_route_path, django_urls.basepath)
+          details = Details.new(PathInfo.new(new_route_path))
           get_endpoints(new_django_urls).each do |endpoint|
+            endpoint.set_details(details)
             endpoints << endpoint
           end
         end
       end
       next if new_django_urls != nil
 
+      details = Details.new(PathInfo.new(django_urls.filepath))
       if view == ""
-        endpoints << Endpoint.new(url, "GET")
+        endpoints << Endpoint.new(url, "GET", details)
       else
         dotted_as_names_split = view.split(".")
 
@@ -149,12 +151,13 @@ class AnalyzerDjango < AnalyzerPython
 
         if filepath != ""
           get_endpoint_from_files(url, filepath, function_or_class_name).each do |endpoint|
+            endpoint.set_details(details)
             endpoints << endpoint
           end
         else
           # By default, Django allows requests with methods other than GET as well
           # Prevent this flow, we need to improve trace code of 'get_endpoint_from_files()
-          endpoints << Endpoint.new(url, "GET")
+          endpoints << Endpoint.new(url, "GET", details)
         end
       end
     end

src/analyzer/analyzers/analyzer_elixir_phoenix.cr

@@ -8,13 +8,14 @@ class AnalyzerElixirPhoenix < Analyzer
         next if File.directory?(path)
         if File.exists?(path) && File.extname(path) == ".ex"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
-            last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
-              endpoint = line_to_endpoint(line)
-              if endpoint.method != ""
-                @result << endpoint
-                last_endpoint = endpoint
-                _ = last_endpoint
+            file.each_line.with_index do |line, index|
+              endpoints = line_to_endpoint(line)
+              endpoints.each do |endpoint|
+                if endpoint.method != ""
+                  details = Details.new(PathInfo.new(path, index + 1))
+                  endpoint.set_details(details)
+                  @result << endpoint
+                end
               end
             end
           end
@@ -27,34 +28,36 @@ class AnalyzerElixirPhoenix < Analyzer
     @result
   end
 
-  def line_to_endpoint(line : String) : Endpoint
+  def line_to_endpoint(line : String) : Array(Endpoint)
+    endpoints = Array(Endpoint).new
+
     line.scan(/get\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "GET")
+      endpoints << Endpoint.new("#{match[1]}", "GET")
     end
 
     line.scan(/post\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "POST")
+      endpoints << Endpoint.new("#{match[1]}", "POST")
     end
 
     line.scan(/patch\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "PATCH")
+      endpoints << Endpoint.new("#{match[1]}", "PATCH")
     end
 
     line.scan(/put\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "PUT")
+      endpoints << Endpoint.new("#{match[1]}", "PUT")
     end
 
     line.scan(/delete\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
-      @result << Endpoint.new("#{match[1]}", "DELETE")
+      endpoints << Endpoint.new("#{match[1]}", "DELETE")
     end
 
     line.scan(/socket\s+['"](.+?)['"]\s*,\s*(.+?)\s*/) do |match|
       tmp = Endpoint.new("#{match[1]}", "GET")
       tmp.set_protocol("ws")
-      @result << tmp
+      endpoints << tmp
     end
 
-    Endpoint.new("", "")
+    endpoints
   end
 end
 

src/analyzer/analyzers/analyzer_express.cr

@@ -9,9 +9,11 @@ class AnalyzerExpress < Analyzer
         if File.exists?(path)
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
               endpoint = line_to_endpoint(line)
               if endpoint.method != ""
+                details = Details.new(PathInfo.new(path, index + 1))
+                endpoint.set_details(details)
                 result << endpoint
                 last_endpoint = endpoint
               end

src/analyzer/analyzers/analyzer_fastapi.cr

@@ -167,7 +167,8 @@ class AnalyzerFastAPI < AnalyzerPython
                   end
                 end
 
-                result << Endpoint.new(router_class.join(http_route_path), http_method_name, params)
+                details = Details.new(PathInfo.new(path, index + 1))
+                result << Endpoint.new(router_class.join(http_route_path), http_method_name, params, details)
               end
             end
           end

src/analyzer/analyzers/analyzer_flask.cr

@@ -30,7 +30,7 @@ class AnalyzerFlask < AnalyzerPython
       Dir.glob("#{base_path}/**/*.py") do |path|
         next if File.directory?(path)
         File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
-          file.each_line do |line|
+          file.each_line.with_index do |line, index|
             # [TODO] We should be cautious about instance replace with other variable
             match = line.match /(#{REGEX_PYTHON_VARIABLE_NAME})\s*=\s*Flask\s*\(/
             if !match.nil?
@@ -77,7 +77,8 @@ class AnalyzerFlask < AnalyzerPython
                     if !route_url.starts_with? "/"
                       route_url = "/#{route_url}"
                     end
-                    result << Endpoint.new(route_url, "GET")
+                    details = Details.new(PathInfo.new(path, index + 1))
+                    result << Endpoint.new(route_url, "GET", details)
                   end
                 end
               end
@@ -123,6 +124,8 @@ class AnalyzerFlask < AnalyzerPython
                 codeblock_lines = codeblock_lines[1..]
 
                 get_endpoints(route_path, extra_params, codeblock_lines, prefix).each do |endpoint|
+                  details = Details.new(PathInfo.new(path, line_index + 1))
+                  endpoint.set_details(details)
                   result << endpoint
                 end
               end

src/analyzer/analyzers/analyzer_go_echo.cr

@@ -10,11 +10,12 @@ class AnalyzerGoEcho < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(PathInfo.new(path, index + 1))
               if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0])
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details)
                     result << new_endpoint
                     last_endpoint = new_endpoint
                   end
@@ -69,7 +70,8 @@ class AnalyzerGoEcho < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(PathInfo.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

src/analyzer/analyzers/analyzer_go_fiber.cr

@@ -10,11 +10,12 @@ class AnalyzerGoFiber < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(PathInfo.new(path, index + 1))
               if line.includes?(".Get(") || line.includes?(".Post(") || line.includes?(".Put(") || line.includes?(".Delete(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0].upcase)
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0].upcase, details)
                     if line.includes?("websocket.New(")
                       new_endpoint.set_protocol("ws")
                     end
@@ -80,7 +81,8 @@ class AnalyzerGoFiber < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(PathInfo.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

src/analyzer/analyzers/analyzer_go_gin.cr

@@ -10,11 +10,12 @@ class AnalyzerGoGin < Analyzer
         if File.exists?(path) && File.extname(path) == ".go"
           File.open(path, "r", encoding: "utf-8", invalid: :skip) do |file|
             last_endpoint = Endpoint.new("", "")
-            file.each_line do |line|
+            file.each_line.with_index do |line, index|
+              details = Details.new(PathInfo.new(path, index + 1))
               if line.includes?(".GET(") || line.includes?(".POST(") || line.includes?(".PUT(") || line.includes?(".DELETE(")
                 get_route_path(line).tap do |route_path|
                   if route_path.size > 0
-                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0])
+                    new_endpoint = Endpoint.new("#{route_path}", line.split(".")[1].split("(")[0], details)
                     result << new_endpoint
                     last_endpoint = new_endpoint
                   end
@@ -63,7 +64,8 @@ class AnalyzerGoGin < Analyzer
             p_dir["static_path"] = p_dir["static_path"][0..-2]
           end
 
-          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET")
+          details = Details.new(PathInfo.new(path))
+          result << Endpoint.new("#{p_dir["static_path"]}#{path.gsub(full_path, "")}", "GET", details)
         end
       end
     end

src/analyzer/analyzers/analyzer_jsp.cr

@@ -33,7 +33,8 @@ class AnalyzerJsp < Analyzer
             rescue
               next
             end
-            result << Endpoint.new("/#{relative_path}", "GET", params_query)
+            details = Details.new(PathInfo.new(path))
+            result << Endpoint.new("/#{relative_path}", "GET", params_query, details)
           end
         end
       end

src/analyzer/analyzers/analyzer_oas2.cr

@@ -9,6 +9,7 @@ class AnalyzerOAS2 < Analyzer
     if swagger_jsons.is_a?(Array(String))
       swagger_jsons.each do |swagger_json|
         if File.exists?(swagger_json)
+          details = Details.new(PathInfo.new(swagger_json))
           content = File.read(swagger_json, encoding: "utf-8", invalid: :skip)
           json_obj = JSON.parse(content)
           base_path = ""
@@ -44,9 +45,9 @@ class AnalyzerOAS2 < Analyzer
                       params << param
                     end
                   end
-                  @result << Endpoint.new(base_path + path, method.upcase, params)
+                  @result << Endpoint.new(base_path + path, method.upcase, params, details)
                 else
-                  @result << Endpoint.new(base_path + path, method.upcase)
+                  @result << Endpoint.new(base_path + path, method.upcase, details)
                 end
               rescue e
                 @logger.debug "Exception of #{swagger_json}/paths/path/method"
@@ -67,6 +68,7 @@ class AnalyzerOAS2 < Analyzer
     if swagger_yamls.is_a?(Array(String))
       swagger_yamls.each do |swagger_yaml|
         if File.exists?(swagger_yaml)
+          details = Details.new(PathInfo.new(swagger_yaml))
           content = File.read(swagger_yaml, encoding: "utf-8", invalid: :skip)
           yaml_obj = YAML.parse(content)
           base_path = ""
@@ -102,9 +104,9 @@ class AnalyzerOAS2 < Analyzer
                       params << param
                     end
                   end
-                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params)
+                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, params, details)
                 else
-                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase)
+                  @result << Endpoint.new(base_path + path.to_s, method.to_s.upcase, details)
                 end
               rescue e
                 @logger.debug "Exception of #{swagger_yaml}/paths/path/method"