Initial commit

NotNite · Dec 23, 2023 · 692fc10 · 692fc10
commit 692fc10
Show file tree

Hide file tree

Showing 8 changed files with 7,970 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+/node_modules
+/dist
+/.parcel-cache
+graph.json
+graph.bak.json
diff --git a/.prettierrc b/.prettierrc
@@ -0,0 +1,6 @@
+{
+  "printWidth": 80,
+  "trailingComma": "none",
+  "tabWidth": 2,
+  "singleQuote": false
+}
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 NotNite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
@@ -0,0 +1,3 @@
+# eightyeightthirtyone
+
+Building a graph of the Internet, one button at a time. This program uses [Puppeteer](https://github.com/puppeteer/puppeteer) to find 88x31s on a website, then visit those websites, and rinse and repeat.
diff --git a/index.html b/index.html
@@ -0,0 +1,109 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>eightyeightthirtyone</title>
+    <style>
+      body,
+      html,
+      #container {
+        margin: 0;
+        padding: 0;
+        width: 100vw;
+        height: 100vh;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="container"></div>
+    <script async defer type="module">
+      import Sigma from "sigma";
+      import Graph from "graphology";
+      import forceAtlas2 from "graphology-layout-forceatlas2";
+
+      // parcel knows how to handle this
+      import fs from "fs";
+      const graphText = fs.readFileSync(__dirname + "/graph.json", "utf8");
+
+      async function main() {
+        const graph = JSON.parse(graphText);
+
+        const container = document.querySelector("#container");
+        const sigmaGraph = new Graph();
+
+        const domains = Array.from(
+          new Set(
+            Object.entries(graph)
+              .map((x) => x[1].concat([x[0]]))
+              .flat()
+          )
+        );
+
+        let important = [];
+        for (const domain of ["notnite.com"]) {
+          important.push(domain);
+          important = [...important, ...graph[domain]];
+        }
+        important = Array.from(new Set(important));
+
+        for (const domain of domains) {
+          sigmaGraph.addNode(domain, {
+            label: domain,
+            size: 5,
+            color: important.includes(domain) ? "#cb2027" : undefined
+          });
+        }
+
+        const twoWay = [];
+        for (const [domain, connections] of Object.entries(graph)) {
+          for (const connection of connections) {
+            let label = `${domain} -> ${connection}`;
+            if (
+              twoWay.find(
+                (x) =>
+                  (x[0] === domain && x[1] === connection) ||
+                  (x[1] === domain && x[0] === connection)
+              )
+            ) {
+              continue;
+            }
+
+            if (
+              graph[connection] != undefined &&
+              graph[connection].includes(domain)
+            ) {
+              twoWay.push([domain, connection]);
+              label = `${domain} <-> ${connection}`;
+            }
+
+            sigmaGraph.addEdge(domain, connection, {
+              type: "arrow",
+              label
+            });
+          }
+        }
+
+        sigmaGraph.nodes().forEach((node, i) => {
+          const angle = (i * 2 * Math.PI) / sigmaGraph.order;
+          sigmaGraph.setNodeAttribute(node, "x", 100 * Math.cos(angle));
+          sigmaGraph.setNodeAttribute(node, "y", 100 * Math.sin(angle));
+        });
+        const positions = forceAtlas2(sigmaGraph, { iterations: 50 });
+        sigmaGraph.nodes().forEach((node, i) => {
+          const name = sigmaGraph.getNodeAttribute(node, "label");
+          const { x, y } = positions[name];
+          sigmaGraph.setNodeAttribute(node, "x", x);
+          sigmaGraph.setNodeAttribute(node, "y", y);
+        });
+
+        const renderer = new Sigma(sigmaGraph, container, {
+          renderEdgeLabels: true,
+          color: "#303030"
+        });
+      }
+
+      main();
+    </script>
+  </body>
+</html>
diff --git a/index.js b/index.js
@@ -0,0 +1,124 @@
+const puppeteer = require("puppeteer");
+const fs = require("fs");
+
+const startUrl = new URL("https://notnite.com/");
+
+function cleanGraph(graph) {
+  const newGraph = {};
+  for (const [key, value] of Object.entries(graph)) {
+    newGraph[key] = Array.from(new Set(value)).filter(
+      (x) => x != null && x.trim() !== ""
+    );
+  }
+  return newGraph;
+}
+
+function readGraph() {
+  if (!fs.existsSync("./graph.json")) return {};
+  return JSON.parse(fs.readFileSync("./graph.json", "utf8"));
+}
+
+function write(data) {
+  // We don't need SQLite where we're going
+  if (fs.existsSync("graph.bak.json")) {
+    fs.rmSync("graph.bak.json");
+  }
+
+  if (fs.existsSync("graph.json")) {
+    fs.renameSync("graph.json", "graph.bak.json");
+  }
+
+  fs.writeFileSync("graph.json", JSON.stringify(data, null, 2));
+}
+
+function alreadyVisited(graph, host) {
+  return Object.keys(graph).find((x) => x === host) != null;
+}
+
+async function main() {
+  let graph = readGraph();
+  graph = cleanGraph(graph);
+  const queue = [];
+
+  if (!alreadyVisited(graph, startUrl.host)) queue.push(startUrl.toString());
+  const notVisited = Object.values(graph)
+    .flat()
+    .filter((x) => x != null && x.trim() !== "" && !alreadyVisited(graph, x))
+    .map((x) => `https://${x}/`);
+  console.log("Not visited", notVisited.length);
+  queue.push(...notVisited);
+
+  const browser = await puppeteer.launch({ headless: false });
+  const page = await browser.newPage();
+
+  const ua = await browser.userAgent();
+  page.setUserAgent(
+    `${ua} eightyeightthirtyone/1.0.0 (https://github.com/NotNite/eightyeightthirtyone)`
+  );
+  while (queue.length > 0) {
+    const url = queue.shift();
+    const uri = new URL(url);
+    if (alreadyVisited(graph, uri.host)) continue;
+    console.log("Visiting", uri.host);
+
+    try {
+      await page.goto(url, { waitUntil: "networkidle2", timeout: 10000 });
+      const links = await page.evaluate(() => {
+        function recursiveChildren(el) {
+          const children = Array.from(el.childNodes);
+          if (children.length === 0) return [el];
+          return children.map((x) => recursiveChildren(x)).flat();
+        }
+
+        const links = Array.from(document.querySelectorAll("a"));
+        const linksChildren = links.map((x) => [x, recursiveChildren(x)]);
+        const buttons = [];
+
+        const variance = 2;
+        const width = 88;
+        const height = 31;
+        for (const [link, children] of linksChildren) {
+          const images = children.filter(
+            (x) =>
+              x.tagName === "IMG" &&
+              x.width <= width + variance &&
+              x.width >= width - variance &&
+              x.height <= height + variance &&
+              x.height >= height - variance
+          );
+          if (images.length > 0) {
+            buttons.push(link.href);
+          }
+        }
+
+        return buttons.filter((x) => x != null && x.trim() !== "");
+      });
+
+      const oldLinks = graph[url] || [];
+      const newLinks = [...oldLinks, ...links].map((x) => new URL(x).host);
+      graph[uri.host] = Array.from(new Set(newLinks));
+      write(graph);
+
+      for (const link of links) {
+        const url = new URL(link);
+        if (url.host.trim() === "") continue;
+        if (!alreadyVisited(graph, url.host)) {
+          queue.push(`https://` + url.host + url.pathname);
+        }
+      }
+    } catch (e) {
+      console.error(e);
+      // Remove the host from the graph
+      delete graph[uri.host];
+      graph = Object.fromEntries(
+        Object.entries(graph).map(([key, value]) => [
+          key,
+          value.filter((x) => x !== uri.host)
+        ])
+      );
+      write(graph);
+    }
+  }
+}
+
+main();
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# eightyeightthirtyone

		Building a graph of the Internet, one button at a time. This program uses [Puppeteer](https://github.com/puppeteer/puppeteer) to find 88x31s on a website, then visit those websites, and rinse and repeat.