Misc bugfixes

bbot/core/helpers/validators.py

@@ -176,6 +176,7 @@ def clean_url(url: str, url_querystring_remove=True):
         scheme = parsed.scheme
     except ValueError:
         scheme = "https"
+    port = None
     with suppress(Exception):
         port = parsed.port
     if port is None:

bbot/modules/internetdb.py → bbot/modules/shodan_idb.py

@@ -1,9 +1,12 @@
 from bbot.modules.base import BaseModule
 
 
-class internetdb(BaseModule):
+class shodan_idb(BaseModule):
     """
     Query IP in Shodan InternetDB, returning open ports, discovered technologies, and findings/vulnerabilities
+
+    InternetDB is especially nice because it doesn't require an API key
+
     API reference: https://internetdb.shodan.io/docs
 
     Example API response:
@@ -43,22 +46,15 @@ class internetdb(BaseModule):
         "created_date": "2023-12-22",
         "author": "@TheTechromancer",
     }
-    options = {"show_open_ports": False}
-    options_desc = {
-        "show_open_ports": "Display OPEN_TCP_PORT events in output, even if they didn't lead to an interesting discovery"
-    }
 
     # we get lots of 404s, that's normal
     _api_failure_abort_threshold = 9999999999
 
+    # there aren't any rate limits to speak of, so our outgoing queue can be pretty big
     _qsize = 500
 
     base_url = "https://internetdb.shodan.io"
 
-    async def setup(self):
-        self.show_open_ports = self.config.get("show_open_ports", False)
-        return True
-
     def _incoming_dedup_hash(self, event):
         return hash(self.get_ip(event))
 
@@ -115,7 +111,6 @@ async def _parse_response(self, data: dict, event, ip):
                 self.helpers.make_netloc(event.data, port),
                 "OPEN_TCP_PORT",
                 parent=event,
-                internal=(not self.show_open_ports),
                 context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}',
             )
         vulns = data.get("vulns", [])

bbot/test/test_step_2/module_tests/test_module_internetdb.py → bbot/test/test_step_2/module_tests/test_module_shodan_idb.py

@@ -1,7 +1,7 @@
 from .base import ModuleTestBase
 
 
-class TestInternetDB(ModuleTestBase):
+class TestShodan_IDB(ModuleTestBase):
     config_overrides = {"dns": {"minimal": False}}
 
     async def setup_before_prep(self, module_test):
@@ -36,15 +36,17 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-        assert 5 == len([e for e in events if str(e.module) == "internetdb"])
+        assert 8 == len([e for e in events if str(e.module) == "shodan_idb"])
         assert 1 == len(
             [e for e in events if e.type == "DNS_NAME" and e.data == "autodiscover.blacklanternsecurity.com"]
         )
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "mail.blacklanternsecurity.com"])
-        assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT"])
-        assert 1 == len([e for e in events if e.type == "FINDING" and str(e.module) == "internetdb"])
+        assert 3 == len(
+            [e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "blacklanternsecurity.com" and str(e.module) == "shodan_idb"]
+        )
+        assert 1 == len([e for e in events if e.type == "FINDING" and str(e.module) == "shodan_idb"])
         assert 1 == len([e for e in events if e.type == "FINDING" and "CVE-2021-26857" in e.data["description"]])
-        assert 2 == len([e for e in events if e.type == "TECHNOLOGY" and str(e.module) == "internetdb"])
+        assert 2 == len([e for e in events if e.type == "TECHNOLOGY" and str(e.module) == "shodan_idb"])
         assert 1 == len(
             [
                 e

bbot/test/test_step_2/module_tests/test_module_speculate.py

@@ -24,7 +24,7 @@ def check(self, module_test, events):
 
 class TestSpeculate_OpenPorts(ModuleTestBase):
     targets = ["evilcorp.com"]
-    modules_overrides = ["speculate", "certspotter", "internetdb"]
+    modules_overrides = ["speculate", "certspotter", "shodan_idb"]
     config_overrides = {"speculate": True}
 
     async def setup_before_prep(self, module_test):

docs/index.md

@@ -10,7 +10,7 @@ _A BBOT scan in real-time - visualization with [VivaGraphJS](https://github.com/
 
     Only **Linux** is supported at this time. **Windows** and **macOS** are *not* supported. If you use one of these platforms, consider using [Docker](#Docker).
 
-BBOT offers multiple methods of installation, including **pipx** and **Docker**. If you plan to dev on BBOT, see [Installation (Poetry)](./contribution/#installation-poetry).
+BBOT offers multiple methods of installation, including **pipx** and **Docker**. If you're looking to tinker or write your own module, see [Setting up a Dev Environment](./dev/dev_environment.md).
 
 ### [Python (pip / pipx)](https://pypi.org/project/bbot/)