From 256017444f380d3c0b420241d76241a7488d3c3f Mon Sep 17 00:00:00 2001 From: mondaychen Date: Wed, 14 Feb 2024 14:13:29 -0500 Subject: [PATCH] feat: extract web-wand-lib --- .gitignore | 1 + package.json | 5 +- package.lib.json | 9 ++ pnpm-lock.yaml | 74 ++++++++++-- rollup.lib.config.js | 44 +++++++ src/common/CopyButton.tsx | 2 +- src/helpers/index.ts | 6 + src/helpers/{ => rpc}/domActions.ts | 4 +- src/helpers/{ => rpc}/pageRPC.ts | 4 +- src/helpers/{ => rpc}/performAction.ts | 2 +- .../{ => rpc}/runtimeFunctionStrings.ts | 4 +- src/helpers/simplifyDom.ts | 44 +++---- src/pages/Content/domOperations.ts | 51 ++++++++ src/pages/Content/injected.ts | 52 +------- src/pages/Content/mainWorld.ts | 27 ++--- src/state/currentTask.ts | 113 +++++++++--------- 16 files changed, 280 insertions(+), 162 deletions(-) create mode 100644 package.lib.json create mode 100644 rollup.lib.config.js create mode 100644 src/helpers/index.ts rename src/helpers/{ => rpc}/domActions.ts (98%) rename src/helpers/{ => rpc}/pageRPC.ts (95%) rename src/helpers/{ => rpc}/performAction.ts (99%) rename src/helpers/{ => rpc}/runtimeFunctionStrings.ts (87%) create mode 100644 src/pages/Content/domOperations.ts diff --git a/.gitignore b/.gitignore index 20d4c37..f01e1e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # build /dist +/dist-lib # etc .DS_Store diff --git a/package.json b/package.json index 113d833..1711a03 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,11 @@ "description": "chrome extension", "repository": { "type": "git", - "url": "https://github.com/normal-computing/wingmate" + "url": "https://github.com/normal-computing/web-wand" }, "scripts": { "build": "tsc --noEmit && vite build", + "build:lib": "rollup --config rollup.lib.config.js && cp package.lib.json dist-lib/package.json", "build:firefox": "tsc --noEmit && cross-env __FIREFOX__=true vite build", "build:watch": "cross-env __DEV__=true vite build -w --mode development", "build:firefox:watch": "cross-env __DEV__=true __FIREFOX__=true vite build -w --mode development", @@ -78,6 +79,8 @@ "npm-run-all": "4.1.5", "prettier": "3.1.0", "rollup": "4.3.0", + "rollup-plugin-dts": "^6.1.0", + "rollup-plugin-esbuild": "^6.1.1", "sass": "1.69.5", "ts-jest": "29.1.1", "ts-loader": "9.5.0", diff --git a/package.lib.json b/package.lib.json new file mode 100644 index 0000000..0a8be7d --- /dev/null +++ b/package.lib.json @@ -0,0 +1,9 @@ +{ + "name": "web-wand-lib", + "version": "2.0.0-beta-1", + "description": "Helper library for Web Wand", + "repository": { + "type": "git", + "url": "https://github.com/normal-computing/web-wand" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 925a7fc..7916fb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,12 +163,18 @@ devDependencies: rollup: specifier: 4.3.0 version: 4.3.0 + rollup-plugin-dts: + specifier: ^6.1.0 + version: 6.1.0(rollup@4.3.0)(typescript@5.2.2) + rollup-plugin-esbuild: + specifier: ^6.1.1 + version: 6.1.1(esbuild@0.19.12)(rollup@4.3.0) sass: specifier: 1.69.5 version: 1.69.5 ts-jest: specifier: 29.1.1 - version: 29.1.1(@babel/core@7.23.9)(jest@29.7.0)(typescript@5.2.2) + version: 29.1.1(@babel/core@7.23.9)(esbuild@0.19.12)(jest@29.7.0)(typescript@5.2.2) ts-loader: specifier: 9.5.0 version: 9.5.0(typescript@5.2.2)(webpack@5.90.1) @@ -347,6 +353,7 @@ packages: /@babel/highlight@7.23.4: resolution: {integrity: sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==} engines: {node: '>=6.9.0'} + requiresBuild: true dependencies: '@babel/helper-validator-identifier': 7.22.20 chalk: 2.4.2 @@ -4565,6 +4572,7 @@ packages: /escape-string-regexp@1.0.5: resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} engines: {node: '>=0.8.0'} + requiresBuild: true /escape-string-regexp@2.0.0: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} @@ -5217,6 +5225,12 @@ packages: get-intrinsic: 1.2.4 dev: true + /get-tsconfig@4.7.2: + resolution: {integrity: sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==} + dependencies: + resolve-pkg-maps: 1.0.0 + dev: true + /git-raw-commits@2.0.11: resolution: {integrity: sha512-VnctFhw+xfj8Va1xtfEqCUD2XDrbAPSJx+hSrE5K7fGdjZruW7XV+QOrN7LF/RJyvspRiD2I0asWsxFp0ya26A==} engines: {node: '>=10'} @@ -5332,6 +5346,7 @@ packages: /has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} + requiresBuild: true /has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} @@ -6642,6 +6657,13 @@ packages: hasBin: true dev: true + /magic-string@0.30.7: + resolution: {integrity: sha512-8vBuFF/I/+OSLRmdf2wwFCJCz+nSn0m6DPvGH1fS/KiQoSaR+sETbov0eIk9KhEKy8CYqIkIAnbohxT/4H0kuA==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: true + /make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} @@ -7686,6 +7708,10 @@ packages: global-dirs: 0.1.1 dev: true + /resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + dev: true + /resolve.exports@2.0.2: resolution: {integrity: sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg==} engines: {node: '>=10'} @@ -7731,6 +7757,37 @@ packages: glob: 7.2.3 dev: true + /rollup-plugin-dts@6.1.0(rollup@4.3.0)(typescript@5.2.2): + resolution: {integrity: sha512-ijSCPICkRMDKDLBK9torss07+8dl9UpY9z1N/zTeA1cIqdzMlpkV3MOOC7zukyvQfDyxa1s3Dl2+DeiP/G6DOw==} + engines: {node: '>=16'} + peerDependencies: + rollup: ^3.29.4 || ^4 + typescript: ^4.5 || ^5.0 + dependencies: + magic-string: 0.30.7 + rollup: 4.3.0 + typescript: 5.2.2 + optionalDependencies: + '@babel/code-frame': 7.23.5 + dev: true + + /rollup-plugin-esbuild@6.1.1(esbuild@0.19.12)(rollup@4.3.0): + resolution: {integrity: sha512-CehMY9FAqJD5OUaE/Mi1r5z0kNeYxItmRO2zG4Qnv2qWKF09J2lTy5GUzjJR354ZPrLkCj4fiBN41lo8PzBUhw==} + engines: {node: '>=14.18.0'} + peerDependencies: + esbuild: '>=0.18.0' + rollup: ^1.20.0 || ^2.0.0 || ^3.0.0 || ^4.0.0 + dependencies: + '@rollup/pluginutils': 5.1.0(rollup@4.3.0) + debug: 4.3.4 + es-module-lexer: 1.4.1 + esbuild: 0.19.12 + get-tsconfig: 4.7.2 + rollup: 4.3.0 + transitivePeerDependencies: + - supports-color + dev: true + /rollup@4.3.0: resolution: {integrity: sha512-scIi1NrKLDIYSPK66jjECtII7vIgdAMFmFo8h6qm++I6nN9qDSV35Ku6erzGVqYjx+lj+j5wkusRMr++8SyDZg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -8213,6 +8270,7 @@ packages: /supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} engines: {node: '>=4'} + requiresBuild: true dependencies: has-flag: 3.0.0 @@ -8282,7 +8340,7 @@ packages: engines: {node: '>=6'} dev: true - /terser-webpack-plugin@5.3.10(webpack@5.90.1): + /terser-webpack-plugin@5.3.10(esbuild@0.19.12)(webpack@5.90.1): resolution: {integrity: sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==} engines: {node: '>= 10.13.0'} peerDependencies: @@ -8299,11 +8357,12 @@ packages: optional: true dependencies: '@jridgewell/trace-mapping': 0.3.22 + esbuild: 0.19.12 jest-worker: 27.5.1 schema-utils: 3.3.0 serialize-javascript: 6.0.2 terser: 5.27.0 - webpack: 5.90.1 + webpack: 5.90.1(esbuild@0.19.12) dev: true /terser@5.27.0: @@ -8428,7 +8487,7 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} dev: false - /ts-jest@29.1.1(@babel/core@7.23.9)(jest@29.7.0)(typescript@5.2.2): + /ts-jest@29.1.1(@babel/core@7.23.9)(esbuild@0.19.12)(jest@29.7.0)(typescript@5.2.2): resolution: {integrity: sha512-D6xjnnbP17cC85nliwGiL+tpoKN0StpgE0TeOjXQTU6MVCfsB4v7aW05CgQ/1OywGb0x/oy9hHFnN+sczTiRaA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -8451,6 +8510,7 @@ packages: dependencies: '@babel/core': 7.23.9 bs-logger: 0.2.6 + esbuild: 0.19.12 fast-json-stable-stringify: 2.1.0 jest: 29.7.0(@types/node@20.8.10) jest-util: 29.7.0 @@ -8475,7 +8535,7 @@ packages: semver: 7.6.0 source-map: 0.7.4 typescript: 5.2.2 - webpack: 5.90.1 + webpack: 5.90.1(esbuild@0.19.12) dev: true /tsconfig-paths@3.15.0: @@ -8805,7 +8865,7 @@ packages: engines: {node: '>=10.13.0'} dev: true - /webpack@5.90.1: + /webpack@5.90.1(esbuild@0.19.12): resolution: {integrity: sha512-SstPdlAC5IvgFnhiRok8hqJo/+ArAbNv7rhU4fnWGHNVfN59HSQFaxZDSAL3IFG2YmqxuRs+IU33milSxbPlog==} engines: {node: '>=10.13.0'} hasBin: true @@ -8836,7 +8896,7 @@ packages: neo-async: 2.6.2 schema-utils: 3.3.0 tapable: 2.2.1 - terser-webpack-plugin: 5.3.10(webpack@5.90.1) + terser-webpack-plugin: 5.3.10(esbuild@0.19.12)(webpack@5.90.1) watchpack: 2.4.0 webpack-sources: 3.2.3 transitivePeerDependencies: diff --git a/rollup.lib.config.js b/rollup.lib.config.js new file mode 100644 index 0000000..b7fe847 --- /dev/null +++ b/rollup.lib.config.js @@ -0,0 +1,44 @@ +import dts from "rollup-plugin-dts"; +import esbuild from "rollup-plugin-esbuild"; +import { resolve, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +const OUT_DIR = "dist-lib"; +const rootDir = resolve(__dirname); +const srcDir = resolve(rootDir, "src"); +const pagesDir = resolve(srcDir, "pages"); + +function createConfigPair(name, path) { + return [ + { + input: path, + plugins: [esbuild()], + output: [ + { + file: `${OUT_DIR}/${name}.js`, + format: "cjs", + sourcemap: true, + exports: "auto", + }, + ], + }, + { + input: path, + plugins: [dts()], + output: { + file: `${OUT_DIR}/${name}.d.ts`, + format: "es", + }, + }, + ]; +} + +export default [ + ...createConfigPair( + "domOperations", + resolve(pagesDir, "content", "domOperations.ts"), + ), + ...createConfigPair("helpers", resolve(srcDir, "helpers", "index.ts")), +]; diff --git a/src/common/CopyButton.tsx b/src/common/CopyButton.tsx index c74f5ae..9021a72 100644 --- a/src/common/CopyButton.tsx +++ b/src/common/CopyButton.tsx @@ -1,7 +1,7 @@ import React from "react"; import { CopyIcon } from "@chakra-ui/icons"; import { useToast } from "@chakra-ui/react"; -import { callRPC } from "../helpers/pageRPC"; +import { callRPC } from "../helpers/rpc/pageRPC"; export default function CopyButton(props: { text: string }) { const toast = useToast(); diff --git a/src/helpers/index.ts b/src/helpers/index.ts new file mode 100644 index 0000000..e12187b --- /dev/null +++ b/src/helpers/index.ts @@ -0,0 +1,6 @@ +export { DomActions } from "./rpc/domActions"; +export { callRPC, callRPCWithTab } from "./rpc/pageRPC"; +export { attachDebugger, detachDebugger } from "./chromeDebugger"; +import performAction from "./rpc/performAction"; +export { performAction }; +export type { Action } from "./rpc/performAction"; diff --git a/src/helpers/domActions.ts b/src/helpers/rpc/domActions.ts similarity index 98% rename from src/helpers/domActions.ts rename to src/helpers/rpc/domActions.ts index 6d687f8..02089c2 100644 --- a/src/helpers/domActions.ts +++ b/src/helpers/rpc/domActions.ts @@ -1,7 +1,7 @@ -import { TAXY_ELEMENT_SELECTOR } from "../constants"; +import { TAXY_ELEMENT_SELECTOR } from "../../constants"; import { callRPCWithTab } from "./pageRPC"; import { scrollScriptString } from "./runtimeFunctionStrings"; -import { sleep, waitFor, waitTillStable } from "./utils"; +import { sleep, waitFor, waitTillStable } from "../utils"; const DEFAULT_INTERVAL = 500; const DEFAULT_TIMEOUT = 0; diff --git a/src/helpers/pageRPC.ts b/src/helpers/rpc/pageRPC.ts similarity index 95% rename from src/helpers/pageRPC.ts rename to src/helpers/rpc/pageRPC.ts index 5cbb7cd..6a98710 100644 --- a/src/helpers/pageRPC.ts +++ b/src/helpers/rpc/pageRPC.ts @@ -1,5 +1,5 @@ -import { sleep } from "./utils"; -import type { RPCMethods } from "../pages/content/injected"; +import { sleep } from "../utils"; +import type { RPCMethods } from "../../pages/content/domOperations"; // Call these functions to execute code in the content script diff --git a/src/helpers/performAction.ts b/src/helpers/rpc/performAction.ts similarity index 99% rename from src/helpers/performAction.ts rename to src/helpers/rpc/performAction.ts index 4309c56..e316ea4 100644 --- a/src/helpers/performAction.ts +++ b/src/helpers/rpc/performAction.ts @@ -2,7 +2,7 @@ import { DomActions } from "./domActions"; import { WEB_WAND_LABEL_ATTRIBUTE_NAME, VISIBLE_TEXT_ATTRIBUTE_NAME, -} from "../constants"; +} from "../../constants"; function getSelector(selectorName: string): string { return `[${WEB_WAND_LABEL_ATTRIBUTE_NAME}="${selectorName}"]`; diff --git a/src/helpers/runtimeFunctionStrings.ts b/src/helpers/rpc/runtimeFunctionStrings.ts similarity index 87% rename from src/helpers/runtimeFunctionStrings.ts rename to src/helpers/rpc/runtimeFunctionStrings.ts index ca51d24..a1004d2 100644 --- a/src/helpers/runtimeFunctionStrings.ts +++ b/src/helpers/rpc/runtimeFunctionStrings.ts @@ -2,8 +2,8 @@ function scrollIntoViewFunction() { // @ts-expect-error this is run in the browser context this.scrollIntoView({ - block: 'center', - inline: 'center', + block: "center", + inline: "center", // behavior: 'smooth', }); } diff --git a/src/helpers/simplifyDom.ts b/src/helpers/simplifyDom.ts index 87697e4..23dae70 100644 --- a/src/helpers/simplifyDom.ts +++ b/src/helpers/simplifyDom.ts @@ -1,11 +1,11 @@ -import { callRPC } from './pageRPC'; -import { truthyFilter } from './utils'; +import { callRPC } from "./rpc/pageRPC"; +import { truthyFilter } from "./utils"; export async function getSimplifiedDom() { - const fullDom = await callRPC('getAnnotatedDOM', [], 3); - if (!fullDom || typeof fullDom !== 'string') return null; + const fullDom = await callRPC("getAnnotatedDOM", [], 3); + if (!fullDom || typeof fullDom !== "string") return null; - const dom = new DOMParser().parseFromString(fullDom, 'text/html'); + const dom = new DOMParser().parseFromString(fullDom, "text/html"); // Mount the DOM to the document in an iframe so we can use getComputedStyle @@ -13,7 +13,7 @@ export async function getSimplifiedDom() { const simplifiedDom = generateSimplifiedDom( dom.documentElement, - interactiveElements + interactiveElements, ) as HTMLElement; return simplifiedDom; @@ -21,16 +21,16 @@ export async function getSimplifiedDom() { export function generateSimplifiedDom( element: ChildNode, - interactiveElements: HTMLElement[] + interactiveElements: HTMLElement[], ): ChildNode | null { if (element.nodeType === Node.TEXT_NODE && element.textContent?.trim()) { - return document.createTextNode(element.textContent + ' '); + return document.createTextNode(element.textContent + " "); } if (!(element instanceof HTMLElement || element instanceof SVGElement)) return null; - const isVisible = element.getAttribute('data-visible') === 'true'; + const isVisible = element.getAttribute("data-visible") === "true"; if (!isVisible) return null; let children = Array.from(element.childNodes) @@ -38,14 +38,14 @@ export function generateSimplifiedDom( .filter(truthyFilter); // Don't bother with text that is the direct child of the body - if (element.tagName === 'BODY') + if (element.tagName === "BODY") children = children.filter((c) => c.nodeType !== Node.TEXT_NODE); const interactive = - element.getAttribute('data-interactive') === 'true' || - element.hasAttribute('role'); + element.getAttribute("data-interactive") === "true" || + element.hasAttribute("role"); const hasLabel = - element.hasAttribute('aria-label') || element.hasAttribute('name'); + element.hasAttribute("aria-label") || element.hasAttribute("name"); const includeNode = interactive || hasLabel; if (!includeNode && children.length === 0) return null; @@ -56,14 +56,14 @@ export function generateSimplifiedDom( const container = document.createElement(element.tagName); const allowedAttributes = [ - 'aria-label', - 'data-name', - 'name', - 'type', - 'placeholder', - 'value', - 'role', - 'title', + "aria-label", + "data-name", + "name", + "type", + "placeholder", + "value", + "role", + "title", ]; for (const attr of allowedAttributes) { @@ -73,7 +73,7 @@ export function generateSimplifiedDom( } if (interactive) { interactiveElements.push(element as HTMLElement); - container.setAttribute('id', element.getAttribute('data-id') as string); + container.setAttribute("id", element.getAttribute("data-id") as string); } children.forEach((child) => container.appendChild(child)); diff --git a/src/pages/Content/domOperations.ts b/src/pages/Content/domOperations.ts new file mode 100644 index 0000000..5b03d76 --- /dev/null +++ b/src/pages/Content/domOperations.ts @@ -0,0 +1,51 @@ +// The content script runs inside each page this extension is enabled on +// Do NOT import from here from outside of content script (other than types). + +import getAnnotatedDOM, { getUniqueElementSelectorId } from "./getAnnotatedDOM"; +import { copyToClipboard } from "./copyToClipboard"; +import attachFile from "./attachFile"; +import { drawLabels, removeLabels } from "./drawLabels"; +import ripple from "./ripple"; +import { getDataFromRenderedMarkdown } from "./reverseMarkdown"; + +export const rpcMethods = { + getAnnotatedDOM, + getUniqueElementSelectorId, + ripple, + copyToClipboard, + attachFile, + drawLabels, + removeLabels, + getDataFromRenderedMarkdown, +} as const; + +export type RPCMethods = typeof rpcMethods; +type MethodName = keyof RPCMethods; + +export type RPCMessage = { + [K in MethodName]: { + method: K; + payload: Parameters; + }; +}[MethodName]; + +// This function should run in the content script +export const initializeRPC = () => { + chrome.runtime.onMessage.addListener( + (message: RPCMessage, sender, sendResponse): true | undefined => { + const { method, payload } = message; + if (method in rpcMethods) { + // @ts-expect-error - we know this is valid (see pageRPC) + const resp = rpcMethods[method as keyof RPCMethods](...payload); + if (resp instanceof Promise) { + resp.then((resolvedResp) => { + sendResponse(resolvedResp); + }); + return true; + } else { + sendResponse(resp); + } + } + }, + ); +}; diff --git a/src/pages/Content/injected.ts b/src/pages/Content/injected.ts index b0fb35a..d585e04 100644 --- a/src/pages/Content/injected.ts +++ b/src/pages/Content/injected.ts @@ -1,53 +1,5 @@ // The content script runs inside each page this extension is enabled on -// Do NOT export anything other than types from this file. -import getAnnotatedDOM, { getUniqueElementSelectorId } from "./getAnnotatedDOM"; -import { copyToClipboard } from "./copyToClipboard"; -import attachFile from "./attachFile"; -import { drawLabels, removeLabels } from "./drawLabels"; -import ripple from "./ripple"; -import { getDataFromRenderedMarkdown } from "./reverseMarkdown"; +import { initializeRPC } from "./domOperations"; -export const rpcMethods = { - getAnnotatedDOM, - getUniqueElementSelectorId, - ripple, - copyToClipboard, - attachFile, - drawLabels, - removeLabels, - getDataFromRenderedMarkdown, -} as const; - -export type RPCMethods = typeof rpcMethods; -type MethodName = keyof RPCMethods; - -type RPCMessage = { - [K in MethodName]: { - method: K; - payload: Parameters; - }; -}[MethodName]; - -// This function should run in the content script -const watchForRPCRequests = () => { - chrome.runtime.onMessage.addListener( - (message: RPCMessage, sender, sendResponse): true | undefined => { - const { method, payload } = message; - if (method in rpcMethods) { - // @ts-expect-error - we know this is valid (see pageRPC) - const resp = rpcMethods[method as keyof RPCMethods](...payload); - if (resp instanceof Promise) { - resp.then((resolvedResp) => { - sendResponse(resolvedResp); - }); - return true; - } else { - sendResponse(resp); - } - } - }, - ); -}; - -watchForRPCRequests(); +initializeRPC(); diff --git a/src/pages/Content/mainWorld.ts b/src/pages/Content/mainWorld.ts index 005e622..7e46614 100644 --- a/src/pages/Content/mainWorld.ts +++ b/src/pages/Content/mainWorld.ts @@ -1,19 +1,16 @@ // This file will be inject dynamically into the page as a content script running in the context of the page // see Background/index.ts for how this is done -import { debugMode } from '../../constants'; -import { generateSimplifiedDom } from '../../helpers/simplifyDom'; -import getAnnotatedDOM, { getUniqueElementSelectorId } from './getAnnotatedDOM'; -import { copyToClipboard } from './copyToClipboard'; -import attachFile from './attachFile'; -import { drawLabels, removeLabels } from './drawLabels'; -import ripple from './ripple'; +import { debugMode } from "../../constants"; +import { generateSimplifiedDom } from "../../helpers/simplifyDom"; +import getAnnotatedDOM from "./getAnnotatedDOM"; +import { rpcMethods } from "./domOperations"; async function getSimplifiedDomFromPage() { const fullDom = getAnnotatedDOM(); - if (!fullDom || typeof fullDom !== 'string') return null; + if (!fullDom || typeof fullDom !== "string") return null; - const dom = new DOMParser().parseFromString(fullDom, 'text/html'); + const dom = new DOMParser().parseFromString(fullDom, "text/html"); // Mount the DOM to the document in an iframe so we can use getComputedStyle @@ -21,7 +18,7 @@ async function getSimplifiedDomFromPage() { const simplifiedDom = generateSimplifiedDom( dom.documentElement, - interactiveElements + interactiveElements, ) as HTMLElement; if (!simplifiedDom) { @@ -31,16 +28,10 @@ async function getSimplifiedDomFromPage() { } if (debugMode) { - console.log('debug mode enabled'); + console.log("debug mode enabled"); // @ts-expect-error - this is for debugging only window.WW_RPC_METHODS = { - getAnnotatedDOM, - getUniqueElementSelectorId, - ripple, - copyToClipboard, - attachFile, - drawLabels, - removeLabels, getSimplifiedDomFromPage, + ...rpcMethods, }; } diff --git a/src/state/currentTask.ts b/src/state/currentTask.ts index 1652718..faab542 100644 --- a/src/state/currentTask.ts +++ b/src/state/currentTask.ts @@ -1,25 +1,25 @@ -import OpenAI from 'openai'; -import { attachDebugger, detachDebugger } from '../helpers/chromeDebugger'; +import OpenAI from "openai"; +import { attachDebugger, detachDebugger } from "../helpers/chromeDebugger"; import { disableIncompatibleExtensions, reenableExtensions, -} from '../helpers/disableExtensions'; -import { DomActions } from '../helpers/domActions'; +} from "../helpers/disableExtensions"; +// import { DomActions } from '../helpers/rpc/domActions'; import { ParsedResponse, ParsedResponseSuccess, parseResponse, -} from '../helpers/parseResponse'; +} from "../helpers/parseResponse"; import { determineNextAction, determineNextActionWithVision, type NextAction, -} from '../helpers/determineNextAction'; -import { callRPCWithTab } from '../helpers/pageRPC'; -import { getSimplifiedDom } from '../helpers/simplifyDom'; -import { sleep, truthyFilter } from '../helpers/utils'; -import performAction from '../helpers/performAction'; -import { MyStateCreator, useAppState } from './store'; +} from "../helpers/determineNextAction"; +import { callRPCWithTab } from "../helpers/rpc/pageRPC"; +import { getSimplifiedDom } from "../helpers/simplifyDom"; +import { sleep, truthyFilter } from "../helpers/utils"; +import performAction from "../helpers/rpc/performAction"; +import { MyStateCreator, useAppState } from "./store"; async function findActiveTab() { const inspectedTabId = chrome?.devtools?.inspectedWindow?.tabId; @@ -28,7 +28,7 @@ async function findActiveTab() { } const currentWindow = await chrome.windows.getCurrent(); if (!currentWindow || !currentWindow.id) { - throw new Error('Could not find window'); + throw new Error("Could not find window"); } const tabs = await chrome.tabs.query({ active: true, @@ -52,15 +52,15 @@ export type CurrentTaskSlice = { tabId: number; instructions: string | null; history: TaskHistoryEntry[]; - status: 'idle' | 'running' | 'success' | 'error' | 'interrupted'; + status: "idle" | "running" | "success" | "error" | "interrupted"; actionStatus: - | 'idle' - | 'attaching-debugger' - | 'pulling-dom' - | 'transforming-dom' - | 'performing-query' - | 'performing-action' - | 'waiting'; + | "idle" + | "attaching-debugger" + | "pulling-dom" + | "transforming-dom" + | "performing-query" + | "performing-action" + | "waiting"; actions: { runTask: (onError: (error: string) => void) => Promise; interrupt: () => void; @@ -72,17 +72,17 @@ export type CurrentTaskSlice = { }; export const createCurrentTaskSlice: MyStateCreator = ( set, - get + get, ) => ({ tabId: -1, instructions: null, history: [], - status: 'idle', - actionStatus: 'idle', + status: "idle", + actionStatus: "idle", actions: { runTask: async (onError) => { - const wasStopped = () => get().currentTask.status !== 'running'; - const setActionStatus = (status: CurrentTaskSlice['actionStatus']) => { + const wasStopped = () => get().currentTask.status !== "running"; + const setActionStatus = (status: CurrentTaskSlice["actionStatus"]) => { set((state) => { state.currentTask.actionStatus = status; }); @@ -90,19 +90,19 @@ export const createCurrentTaskSlice: MyStateCreator = ( const instructions = get().ui.instructions; - if (!instructions || get().currentTask.status === 'running') return; + if (!instructions || get().currentTask.status === "running") return; set((state) => { state.currentTask.instructions = instructions; state.currentTask.history = []; - state.currentTask.status = 'running'; - state.currentTask.actionStatus = 'attaching-debugger'; + state.currentTask.status = "running"; + state.currentTask.actionStatus = "attaching-debugger"; }); try { const activeTab = await findActiveTab(); - if (!activeTab?.id) throw new Error('No active tab found'); + if (!activeTab?.id) throw new Error("No active tab found"); const tabId = activeTab.id; set((state) => { state.currentTask.tabId = tabId; @@ -110,7 +110,7 @@ export const createCurrentTaskSlice: MyStateCreator = ( await attachDebugger(tabId); await disableIncompatibleExtensions(); - const domActions = new DomActions(tabId); + // const domActions = new DomActions(tabId); // eslint-disable-next-line no-constant-condition while (true) { @@ -120,36 +120,36 @@ export const createCurrentTaskSlice: MyStateCreator = ( .currentTask.history.map((entry) => entry.action) .filter(truthyFilter); - setActionStatus('performing-query'); + setActionStatus("performing-query"); let query: NextAction | null = null; if ( useAppState.getState().settings.selectedModel === - 'gpt-4-vision-preview' + "gpt-4-vision-preview" ) { - await callRPCWithTab(tabId, 'drawLabels', []); + await callRPCWithTab(tabId, "drawLabels", []); const imgData = await chrome.tabs.captureVisibleTab({ - format: 'jpeg', + format: "jpeg", quality: 85, }); if (wasStopped()) break; - await callRPCWithTab(tabId, 'removeLabels', []); + await callRPCWithTab(tabId, "removeLabels", []); query = await determineNextActionWithVision( instructions, previousActions.filter( - (pa) => !('error' in pa) + (pa) => !("error" in pa), ) as ParsedResponseSuccess[], imgData, 3, - onError + onError, ); } else { - setActionStatus('pulling-dom'); + setActionStatus("pulling-dom"); const pageDOM = await getSimplifiedDom(); if (!pageDOM) { set((state) => { - state.currentTask.status = 'error'; + state.currentTask.status = "error"; }); break; } @@ -158,24 +158,24 @@ export const createCurrentTaskSlice: MyStateCreator = ( query = await determineNextAction( instructions, previousActions.filter( - (pa) => !('error' in pa) + (pa) => !("error" in pa), ) as ParsedResponseSuccess[], pageDOM.outerHTML, 3, - onError + onError, ); } if (query == null) { set((state) => { - state.currentTask.status = 'error'; + state.currentTask.status = "error"; }); break; } if (wasStopped()) break; - setActionStatus('performing-action'); + setActionStatus("performing-action"); const action = parseResponse(query.response); set((state) => { @@ -187,14 +187,14 @@ export const createCurrentTaskSlice: MyStateCreator = ( usage: query.usage, }); }); - if ('error' in action) { + if ("error" in action) { onError(action.error); break; } if ( action === null || - action.parsedAction.name === 'finish' || - action.parsedAction.name === 'fail' + action.parsedAction.name === "finish" || + action.parsedAction.name === "fail" ) { break; } @@ -214,17 +214,18 @@ export const createCurrentTaskSlice: MyStateCreator = ( break; } - setActionStatus('waiting'); + setActionStatus("waiting"); // sleep 2 seconds. This is pretty arbitrary; we should figure out a better way to determine when the page has settled. await sleep(2000); } set((state) => { - state.currentTask.status = 'success'; + state.currentTask.status = "success"; }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (e: any) { onError(e.message); set((state) => { - state.currentTask.status = 'error'; + state.currentTask.status = "error"; }); } finally { await detachDebugger(get().currentTask.tabId); @@ -233,13 +234,13 @@ export const createCurrentTaskSlice: MyStateCreator = ( }, interrupt: () => { set((state) => { - state.currentTask.status = 'interrupted'; + state.currentTask.status = "interrupted"; }); }, // for debugging attachDebugger: async () => { const activeTab = await findActiveTab(); - if (!activeTab?.id) throw new Error('No active tab found'); + if (!activeTab?.id) throw new Error("No active tab found"); const tabId = activeTab.id; set((state) => { state.currentTask.tabId = tabId; @@ -251,19 +252,19 @@ export const createCurrentTaskSlice: MyStateCreator = ( }, prepareLabels: async () => { const tabId = get().currentTask.tabId; - await callRPCWithTab(tabId, 'drawLabels', []); + await callRPCWithTab(tabId, "drawLabels", []); await sleep(800); - await callRPCWithTab(tabId, 'removeLabels', []); + await callRPCWithTab(tabId, "removeLabels", []); }, performActionString: async (actionString: string) => { const action = parseResponse(actionString); - if ('error' in action) { + if ("error" in action) { throw action.error; } if ( action === null || - action.parsedAction.name === 'finish' || - action.parsedAction.name === 'fail' + action.parsedAction.name === "finish" || + action.parsedAction.name === "fail" ) { return; }