feat: customize search_data for an extra

This allows configuring the "search data" for any given extra.
When you do this, you take over the way that the extra appears
both in autocomplete *and* in search_data. The current system
assumes that you give this information in markdown, which I
think is a safe assumption, but we could do something like
allow for an `html_body` or `markdown_body` to be given,
etc.

This is likely not going to be useful for 99% of people, and
as such I think its okay that its not super ergonomic.
Specifically, you have to completely manage the search content
etc. The way this will be used, however, is in `Spark` where
we generate markdown files for DSL documentation. With this change,
we can add a function called `Spark.Dsl.search_data_for(Your.Dsl)` which
will improve the autocomplete experience and the search experience
in various ways (by adding more things to autocomplete than just
headers, and by customizing the `type` of the search data nodes

assets/js/autocomplete/suggestions.js

@@ -18,7 +18,8 @@ const SUGGESTION_CATEGORY = {
   moduleChild: 'module-child',
   mixTask: 'mix-task',
   extra: 'extra',
-  section: 'section'
+  section: 'section',
+  custom: 'custom'
 }
 
 /**
@@ -42,7 +43,8 @@ export function getSuggestions (query, limit = 8) {
     ...findSuggestionsInTopLevelNodes(nodes.extras, query, SUGGESTION_CATEGORY.extra, 'page'),
     ...findSuggestionsInSectionsOfNodes(nodes.modules, query, SUGGESTION_CATEGORY.section, 'module'),
     ...findSuggestionsInSectionsOfNodes(nodes.tasks, query, SUGGESTION_CATEGORY.section, 'mix task'),
-    ...findSuggestionsInSectionsOfNodes(nodes.extras, query, SUGGESTION_CATEGORY.section, 'page')
+    ...findSuggestionsInSectionsOfNodes(nodes.extras, query, SUGGESTION_CATEGORY.section, 'page'),
+    ...findSuggestionsInCustomSidebarNodes(nodes.custom || [], query, SUGGESTION_CATEGORY.custom, 'custom')
   ].filter(suggestion => suggestion !== null)
 
   return sort(suggestions).slice(0, limit)
@@ -55,6 +57,13 @@ function findSuggestionsInTopLevelNodes (nodes, query, category, label) {
   return nodes.map(node => nodeSuggestion(node, query, category, label))
 }
 
+/**
+ * Finds suggestions in custom sidebar nodes.
+ */
+function findSuggestionsInCustomSidebarNodes (nodes, query, category, label) {
+  return nodes.map(node => customNodeSuggestion(node, query, category, label))
+}
+
 /**
  * Finds suggestions in node groups of the given parent nodes.
  */
@@ -96,6 +105,7 @@ function nodeSections (node) {
  * Returns null if the node doesn't match the query.
  */
 function nodeSuggestion (node, query, category, label) {
+  if (node.hide_in_autocomplete) { return null }
   if (!matchesAll(node.title, query)) { return null }
 
   return {
@@ -104,7 +114,25 @@ function nodeSuggestion (node, query, category, label) {
     description: null,
     matchQuality: matchQuality(node.title, query),
     deprecated: node.deprecated,
-    labels: [label],
+    labels: node.labels || [label],
+    category
+  }
+}
+
+/**
+ * Builds a suggestion for a custom top level node.
+ * Returns null if the node doesn't match the query.
+ */
+function customNodeSuggestion (node, query, category, label) {
+  if (!matchesAll(node.title, query)) { return null }
+
+  return {
+    link: node.link,
+    title: highlightMatches(node.title, query),
+    description: node.description,
+    matchQuality: matchQuality(node.title, query),
+    deprecated: node.deprecated,
+    labels: node.labels || [label],
     category
   }
 }
@@ -131,6 +159,7 @@ function childNodeSuggestion (childNode, parentId, query, category, label) {
  * Builds a suggestion for a node section.
  */
 function nodeSectionSuggestion (node, section, query, category, label) {
+  if (node.hide_in_autocomplete) { return null }
   if (!matchesAny(section.id, query)) { return null }
 
   return {
@@ -211,7 +240,8 @@ function categoryPriority (category) {
     case SUGGESTION_CATEGORY.module: return 1
     case SUGGESTION_CATEGORY.moduleChild: return 2
     case SUGGESTION_CATEGORY.mixTask: return 3
-    default: return 4
+    case SUGGESTION_CATEGORY.custom: return 4
+    default: return 5
   }
 }
 

lib/ex_doc/formatter/html.ex

@@ -6,6 +6,7 @@ defmodule ExDoc.Formatter.HTML do
 
   @main "api-reference"
   @assets_dir "assets"
+  @search_data_keys [:anchor, :body, :description, :title, :type]
 
   @doc """
   Generates HTML documentation for the given modules.
@@ -183,7 +184,9 @@ defmodule ExDoc.Formatter.HTML do
   end
 
   defp generate_sidebar_items(nodes_map, extras, config) do
-    content = Templates.create_sidebar_items(nodes_map, extras)
+    content =
+      Templates.create_sidebar_items(nodes_map, extras)
+
     path = "dist/sidebar_items-#{digest(content)}.js"
     File.write!(Path.join(config.output, path), content)
     [path]
@@ -429,13 +432,16 @@ defmodule ExDoc.Formatter.HTML do
     source_path = source_file |> Path.relative_to(File.cwd!()) |> String.replace_leading("./", "")
     source_url = Utils.source_url_pattern(source_url_pattern, source_path, 1)
 
+    search_data = normalize_search_data!(id, input_options[:search_data])
+
     %{
       source: source,
       content: content_html,
       group: group,
       id: id,
       source_path: source_path,
       source_url: source_url,
+      search_data: search_data,
       title: title,
       title_content: title_html || title
     }
@@ -445,6 +451,32 @@ defmodule ExDoc.Formatter.HTML do
     build_extra({input, []}, groups, language, autolink_opts, source_url_pattern)
   end
 
+  defp normalize_search_data!(_id, nil), do: nil
+
+  defp normalize_search_data!(id, search_data) when is_list(search_data) do
+    Enum.map(search_data, fn search_data ->
+      has_keys = Map.keys(search_data)
+
+      if Enum.sort(has_keys) != @search_data_keys do
+        raise ArgumentError,
+              "Expected search data to be a list of maps with the keys: #{inspect(@search_data_keys)}, found keys: #{inspect(has_keys)}"
+      end
+
+      %{
+        link: "#{id}.html##{search_data.anchor}",
+        title: search_data.title,
+        body: search_data.body,
+        description: search_data.description,
+        type: search_data.type
+      }
+    end)
+  end
+
+  defp normalize_search_data!(_id, search_data) do
+    raise ArgumentError,
+          "Expected search data to be a list of maps with the keys: #{inspect(@search_data_keys)}, found: #{inspect(search_data)}"
+  end
+
   defp extension_name(input) do
     input
     |> Path.extname()

lib/ex_doc/formatter/html/search_data.ex

@@ -19,27 +19,37 @@ defmodule ExDoc.Formatter.HTML.SearchData do
   end
 
   defp extra(map) do
-    {intro, sections} = extract_sections_from_markdown(map.source)
+    if custom_search_data = map[:search_data] do
+      extra_search_data(map, custom_search_data)
+    else
+      {intro, sections} = extract_sections_from_markdown(map.source)
 
-    intro_json_item =
-      encode(
-        "#{map.id}.html",
-        map.title,
-        :extras,
-        intro
-      )
-
-    section_json_items =
-      for {header, body} <- sections do
+      intro_json_item =
         encode(
-          "#{map.id}.html##{Utils.text_to_id(header)}",
-          header <> " - #{map.title}",
+          "#{map.id}.html",
+          map.title,
           :extras,
-          body
+          intro
         )
-      end
 
-    [intro_json_item | section_json_items]
+      section_json_items =
+        for {header, body} <- sections do
+          encode(
+            "#{map.id}.html##{Utils.text_to_id(header)}",
+            header <> " - #{map.title}",
+            :extras,
+            body
+          )
+        end
+
+      [intro_json_item | section_json_items]
+    end
+  end
+
+  defp extra_search_data(map, custom_search_data) do
+    Enum.map(custom_search_data, fn item ->
+      encode(item.link, item.title <> " - #{map.id}", item.type, clean_markdown(item.body))
+    end)
   end
 
   defp module(%ExDoc.ModuleNode{} = node) do

lib/ex_doc/formatter/html/templates.ex

@@ -76,10 +76,20 @@ defmodule ExDoc.Formatter.HTML.Templates do
       |> Enum.map(&sidebar_module/1)
       |> Map.new()
       |> Map.put(:extras, sidebar_extras(extras))
+      |> Map.put(:custom, custom_autocompletions_from_extras(extras))
 
     ["sidebarNodes=" | ExDoc.Utils.to_json(nodes)]
   end
 
+  defp custom_autocompletions_from_extras(extras) do
+    for %{search_data: search_data} when is_list(search_data) <- extras,
+        search_item <- search_data do
+      search_item
+      |> Map.drop([:type, :body])
+      |> Map.put(:labels, [search_item.type])
+    end
+  end
+
   defp sidebar_extras(extras) do
     for extra <- extras do
       %{id: id, title: title, group: group, content: content} = extra
@@ -90,6 +100,13 @@ defmodule ExDoc.Formatter.HTML.Templates do
         group: to_string(group),
         headers: extract_headers(content)
       }
+      |> then(fn item ->
+        if is_nil(extra[:search_data]) do
+          item
+        else
+          Map.put(item, :hide_in_autocomplete, true)
+        end
+      end)
     end
   end
 

lib/mix/tasks/docs.ex

@@ -153,7 +153,8 @@ defmodule Mix.Tasks.Docs do
       May be overridden by command line argument.
 
     * `:redirects` - A map or list of tuples, where the key is the path to redirect from and the
-       value is the path to redirect to. The extension is omitted in both cases, i.e `%{"old-readme" => "readme"}`
+       value is the path to redirect to. The extension is omitted in both cases, i.e `%{"old-readme" => "readme"}`.
+       See the "Changing documentation over time" section below for more.
 
     * `:skip_undefined_reference_warnings_on` - ExDoc warns when it can't create a `Mod.fun/arity`
       reference in the current project docs e.g. because of a typo. This list controls where to
@@ -339,14 +340,12 @@ defmodule Mix.Tasks.Docs do
 
   ## Changing documentation over time
 
-  As your project grows, your documentation may very likely change, even structurally.
-  There are a few important things to consider in this regard:
+  As your project grows, your documentation may very likely change, even structurally. There are a few important things to consider in this regard:
 
-    * Links to your *extras* will break if you change or move file names.
-    * Links to your *modules, and mix tasks* will change if you change their name.
-    * Links to *functions* are actually links to modules with anchor links.
-      If you change the function name, the link does not break but will leave users
-      at the top of the module's documentation.
+  - Links to your *extras* will break if you change or move file names.
+  - Links to your *modules, and mix tasks* will change if you change their name.
+  - Links to *functions* are actually links to modules with anchor links. If you change the function name, the link does
+  not break but will leave users at the top of the module's documentation.
 
   Because these docs are static files, the behavior of a missing page will depend on where they are hosted.
   In particular, [hexdocs.pm](https://hexdocs.pm) will show a 404 page.

test/ex_doc/formatter/html_test.exs

@@ -592,6 +592,41 @@ defmodule ExDoc.Formatter.HTMLTest do
              ] = Jason.decode!(content)["extras"]
     end
 
+    test "custom search data is added to the sidebar", %{tmp_dir: tmp_dir} = context do
+      generate_docs(
+        doc_config(context,
+          source_beam: "unknown",
+          extras: [
+            {"test/fixtures/README.md",
+             search_data: [
+               %{
+                 anchor: "heading-without-content",
+                 title: "custom-text",
+                 description: "Some Custom Text",
+                 type: "custom",
+                 body: """
+                 Some longer text!
+
+                 Here it is :)
+                 """
+               }
+             ]}
+          ]
+        )
+      )
+
+      "sidebarNodes=" <> content = read_wildcard!(tmp_dir <> "/html/dist/sidebar_items-*.js")
+
+      assert [
+               %{
+                 "link" => "readme.html#heading-without-content",
+                 "title" => "custom-text",
+                 "description" => "Some Custom Text",
+                 "labels" => ["custom"]
+               }
+             ] = Jason.decode!(content)["custom"]
+    end
+
     test "containing settext headers while discarding links on header",
          %{tmp_dir: tmp_dir} = context do
       generate_docs(