diff --git a/astro.config.ts b/astro.config.ts index 58d1e64..2df1dc4 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from 'astro/config'; import tailwind from "@astrojs/tailwind"; import remarkLesetid from "remark-lesetid/astro"; -import { rehypeTitles, rehypeCodeCopy, rehypePreClass } from './src/rehype'; +import { rehypeTitles, rehypeCodeCopy, rehypePreClass, rehypeHighlight } from './src/rehype'; import { rehypeHeadingIds } from '@astrojs/markdown-remark'; import { remarkAlert } from 'remark-github-blockquote-alert'; import react from "@astrojs/react"; @@ -38,13 +38,14 @@ export default defineConfig({ rehypeHeadingIds, rehypeTitles, rehypeCodeCopy, - [rehypeShiki, { - themes: { - dark: "vesper", - light: 'github-light', - } - }], + // [rehypeShiki, { + // themes: { + // dark: "vesper", + // light: 'github-light', + // } + // }], rehypePreClass, + rehypeHighlight, ], }, build: { diff --git a/flake.lock b/flake.lock index d4f86fa..15c09e1 100644 --- a/flake.lock +++ b/flake.lock @@ -18,7 +18,28 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "tree-sitter": "tree-sitter" + } + }, + "tree-sitter": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1722771651, + "narHash": "sha256-OqK+V68vT/LTI4+YGim8uMhFrAapW8lFp7++Aw59T/c=", + "owner": "viperML", + "repo": "tree-sitter", + "rev": "dc02125ff4e4059c8a583ac297c1656c7f14399d", + "type": "github" + }, + "original": { + "owner": "viperML", + "repo": "tree-sitter", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 82e28d7..280a690 100644 --- a/flake.nix +++ b/flake.nix @@ -1,11 +1,16 @@ { inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + tree-sitter = { + url = "github:viperML/tree-sitter"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = { self, nixpkgs, + tree-sitter, }: let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; @@ -64,6 +69,7 @@ env = { ASTRO_TELEMETRY_DISABLED = true; + TS_GRAMMAR_PATH = tree-sitter.packages.${system}.bundle; } // (lib.optionalAttrs (builtins.hasAttr "rev" self) { GIT_COMMIT = self.rev; @@ -89,7 +95,8 @@ ltex-ls ]; - env.RUST_SRC_PATH = "${rustPlatform.rustLibSrc}"; + env.RUST_SRC_PATH = "${rustPlatform.rustLibSrc}"; + env.TS_GRAMMAR_PATH = tree-sitter.packages.${system}.bundle; }; }; } diff --git a/neohome-rs/Cargo.lock b/neohome-rs/Cargo.lock index 7b263ae..0567d54 100644 --- a/neohome-rs/Cargo.lock +++ b/neohome-rs/Cargo.lock @@ -17,6 +17,12 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "cc" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" + [[package]] name = "cfg-if" version = "1.0.0" @@ -42,6 +48,34 @@ dependencies = [ "syn", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + [[package]] name = "libloading" version = "0.8.5" @@ -122,6 +156,8 @@ dependencies = [ "napi", "napi-build", "napi-derive", + "tree-sitter-dynamic", + "tree-sitter-highlight", ] [[package]] @@ -194,6 +230,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tree-sitter" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-dynamic" +version = "0.1.0" +source = "git+https://github.com/viperML/tree-sitter#dc02125ff4e4059c8a583ac297c1656c7f14399d" +dependencies = [ + "cc", + "eyre", + "libc", + "libloading", + "tree-sitter", + "tree-sitter-highlight", +] + +[[package]] +name = "tree-sitter-highlight" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaca0fe34fa96eec6aaa8e63308dbe1bafe65a6317487c287f93938959b21907" +dependencies = [ + "lazy_static", + "regex", + "thiserror", + "tree-sitter", +] + [[package]] name = "unicode-ident" version = "1.0.12" diff --git a/neohome-rs/Cargo.toml b/neohome-rs/Cargo.toml index ecb826b..a5e1576 100644 --- a/neohome-rs/Cargo.toml +++ b/neohome-rs/Cargo.toml @@ -10,6 +10,8 @@ crate-type = ["cdylib"] # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.12.2", default-features = false, features = ["napi4"] } napi-derive = "2.12.2" +tree-sitter-dynamic = { git = "https://github.com/viperML/tree-sitter", version = "0.1.0" } +tree-sitter-highlight = "0.22.6" [build-dependencies] napi-build = "2.0.1" diff --git a/neohome-rs/src/lib.rs b/neohome-rs/src/lib.rs index b479ed7..8593614 100644 --- a/neohome-rs/src/lib.rs +++ b/neohome-rs/src/lib.rs @@ -3,7 +3,39 @@ #[macro_use] extern crate napi_derive; +use tree_sitter_dynamic::{DynTS, STANDARD_CAPTURE_NAMES}; +use tree_sitter_highlight::{Highlight, HighlightEvent}; + #[napi] pub fn sum(a: i32, b: i32) -> i32 { - a + b + a + b +} + +#[napi] +pub fn highlight(text: String, lang: String) -> Option { + let mut lang = DynTS::new(lang, STANDARD_CAPTURE_NAMES).ok()?; + + let mut res: Vec = Vec::new(); + + let bytes = text.as_bytes(); + + for event in lang.highlight(bytes) { + match event.ok()? { + HighlightEvent::Source { start, end } => res.extend(&bytes[start..end]), + HighlightEvent::HighlightStart(Highlight(n)) => { + let capture = STANDARD_CAPTURE_NAMES[n]; + let classes = capture.replace(".", " "); + let text = format!(r#""#, classes); + res.extend(text.as_bytes()); + } + HighlightEvent::HighlightEnd => { + let text = format!(""); + res.extend(text.as_bytes()); + } + }; + } + + let res_text = String::from_utf8(res).ok()?; + + Some(res_text) } diff --git a/src/pages/experiments/index.astro b/src/pages/experiments/index.astro index bff75f5..0034681 100644 --- a/src/pages/experiments/index.astro +++ b/src/pages/experiments/index.astro @@ -1,6 +1,8 @@ --- import Base from "../../layouts/Base.astro"; import Centered from "../../components/Centered.astro"; + +import { highlight } from "neohome-rs"; ---

Coming soon...

+ + {highlight("let x = 1", "javascript")} diff --git a/src/rehype.ts b/src/rehype.ts index 840a6a7..747f03a 100644 --- a/src/rehype.ts +++ b/src/rehype.ts @@ -8,6 +8,8 @@ import { fromHtml } from 'hast-util-from-html' import IconCopy from "@tabler/icons/outline/copy.svg?raw"; import IconCopyCheckFilled from "@tabler/icons/filled/copy-check.svg?raw"; +import { highlight } from "neohome-rs"; + function icon2node(raw: string): Element { return fromHtml(raw, { fragment: true, @@ -28,6 +30,18 @@ export function rehypePreClass(): (tree: Root) => void { } } +export function rehypeHighlight(): (tree: Root) => void { + return function (tree: Root) { + visit(tree, "element", node => { + if (node.tagName === "pre") { + console.log(node); + + } + }); + } +} + + export function rehypeTitles(): (tree: Root) => void { return function (tree: Root) { visit(tree, 'element', function (node) {