Improve error handling for Pagefind HTML parsing

CHANGELOG.md

@@ -9,6 +9,8 @@
 
 ## Unreleased
 
+* Pagefind now gracefully skips pages that fail HTML parsing, and provides more context when these errors are hit.
+
 ## v0.8.0 (August 23, 2022)
 
 ### Important Changes

pagefind/features/errors.feature

@@ -0,0 +1,38 @@
+Feature: Graceful Pagefind Errors
+    Background:
+        Given I have the environment variables:
+            | PAGEFIND_SOURCE | public |
+        Given I have a "public/index.html" file with the body:
+            """
+            <p data-url>Nothing</p>
+            """
+
+    Scenario: Pagefind gracefully skips pages with parsing ambiguities
+        Given I have a "public/cat/index.html" file with the body:
+            """
+            <h1>hello world</h1>
+            """
+        Given I have a "public/dog/index.html" file with the body:
+            """
+            <h1>hello world</h1>
+            <select><xmp><script>"use strict";</script></select>
+            """
+        When I run my program
+        Then I should see "Running Pagefind" in stdout
+        Then I should see "Failed to parse file public/dog/index.html" in stdout
+        Then I should see the file "public/_pagefind/pagefind.js"
+        When I serve the "public" directory
+        When I load "/"
+        When I evaluate:
+            """
+            async function() {
+                let pagefind = await import("/_pagefind/pagefind.js");
+
+                let search = await pagefind.search("world");
+                let results = await Promise.all(search.results.map(r => r.data()));
+
+                document.querySelector('[data-url]').innerText = results.map(r => r.url).sort().join(', ');
+            }
+            """
+        Then There should be no logs
+        Then The selector "[data-url]" should contain "/cat/"

pagefind/src/fossick/mod.rs

@@ -59,7 +59,11 @@ impl Fossicker {
                 break;
             }
             if let Err(error) = rewriter.write(&buf[..read]) {
-                panic!("HTML parse encountered an error: {:#?}", error);
+                println!(
+                    "Failed to parse file {} — skipping this file. Error:\n{error}",
+                    self.file_path.to_str().unwrap_or("[unknown file]")
+                );
+                return Ok(());
             }
         }
 
@@ -75,6 +79,10 @@ impl Fossicker {
 
     fn parse_digest(&mut self) -> (String, HashMap<String, Vec<u32>>) {
         let mut map: HashMap<String, Vec<u32>> = HashMap::new();
+        // TODO: push this error handling up a level and return an Err from parse_digest
+        if self.data.as_ref().is_none() {
+            return ("".into(), map); // empty page result, will be dropped from search
+        }
         let data = self.data.as_ref().unwrap();
         let stemmer = get_stemmer(&data.language);
 
@@ -144,6 +152,10 @@ impl Fossicker {
 
         let (content, word_data) = self.parse_digest();
 
+        if self.data.is_none() {
+            return Err(());
+        }
+
         let data = self.data.unwrap();
         let url = build_url(&self.file_path, options);