Add support for hidden files in documentation TOC (#381)

Hidden files can now be excluded from navigation and indexing but still linked directly if needed. Adjustments include handling hidden attributes in TOC parsing and ensuring related navigation logic respects these exclusions.

docs/configure/content-set/navigation.md

@@ -73,16 +73,19 @@ toc:
   - file: index.md
 ```
 
-The table of contents can be created independent of the directory structure of the files it defines. You can use directories to define nesting in the TOC, but you don't have to. For example, both of the following create the same nav structure:
+The TOC in principle follows the directory structure on disk.
+
+#### `folder:`
 
 ```yaml
   ...
-  - file: subsection/index.md
-    children:
-      - file: subsection/page-one.md
-      - file: subsection/page-two.md
+  - folder: subsection
 ```
 
+If a folder does not explicitly define `children` all markdown files within that folder are included automatically 
+
+If a folder does define `children` all markdown files within that folder have to be included. `docs-builder` will error if it detects dangling documentation files.
+
 ```yaml
   ...
   - folder: subsection
@@ -92,7 +95,33 @@ The table of contents can be created independent of the directory structure of t
       - file: page-two.md
 ```
 
-#### Nest `toc`
+#### Virtual grouping
+
+A `file` element may include children to create a virtual grouping that 
+does not match the directory structure. 
+
+```yaml
+  ...
+  - file: subsection/index.md
+    children:
+      - file: subsection/page-one.md
+      - file: subsection/page-two.md
+```
+
+A `file` may only select siblings and more deeply nested files as its children. It may not select files outside its own subtree on disk.
+
+#### Hidden files
+
+A hidden file can be declared in the TOC. 
+```yaml
+  - hidden: developer-pages.md
+```
+
+It may not have any children and won't show up in the navigation.
+
+It [may be linked to locally however](../../developer-notes.md)
+
+#### Nesting `toc`
 
 The `toc` key can include nested `toc.yml` files.
 
@@ -122,4 +151,6 @@ See [Attributes](./attributes.md) to learn more.
 
 As a rule, each `docset.yml` file can only be included once in the assembler. This prevents us from accidentally duplicating pages in the docs. However, there are times when you want to split content sets and include them partially in different areas of the TOC. That's what `toc.yml` files are for. These files split one documentation set into multiple “sub-TOCs,” each mapped to a different navigation node.
 
-All configuration options that `docset.yml` supports are supported by `toc.yml`.
+A `toc.yml` file may only declare a nested [TOC](#toc), other options are ignored. 
+
+A `toc.yml` may not link to further nested `toc.yml` files. Doing so will result in an error

docs/developer-notes.md

@@ -0,0 +1,18 @@
+# These are developer notes
+
+Because this page is included as hidden:
+
+
+```yaml
+toc:
+  - file: index.md
+  - hidden: developer-notes.md
+```
+
+This page 
+
+- will not appear in the TOC navigation
+- will always include a `noindexed` meta header.
+
+
+This page **can** be linked to using [a regular link](developer-notes.md)

docs/docset.yml

@@ -24,6 +24,7 @@ subs:
   a-global-variable: "This was defined in docset.yml"
 toc:
   - file: index.md
+  - hidden: developer-notes.md
   - folder: contribute
     children:
       - file: index.md

src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs

@@ -39,7 +39,6 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 		if (!sourceFile.Exists)
 		{
 			Project = "unknown";
-			TableOfContents = [];
 			context.EmitWarning(sourceFile, "No configuration file found");
 			return;
 		}
@@ -138,6 +137,7 @@ private List<ITocItem> ReadChildren(KeyValuePair<YamlNode, YamlNode> entry, stri
 		ConfigurationFile? toc = null;
 		var fileFound = false;
 		var folderFound = false;
+		var hiddenFile = false;
 		IReadOnlyCollection<ITocItem>? children = null;
 		foreach (var entry in tocEntry.Children)
 		{
@@ -147,8 +147,10 @@ private List<ITocItem> ReadChildren(KeyValuePair<YamlNode, YamlNode> entry, stri
 				case "toc":
 					toc = ReadNestedToc(entry, parentPath, out fileFound);
 					break;
+				case "hidden":
 				case "file":
-					file = ReadFile(entry, parentPath, out fileFound);
+					hiddenFile = key == "hidden";
+					file = ReadFile(entry, parentPath, key, out fileFound);
 					break;
 				case "folder":
 					folder = ReadFolder(entry, parentPath, out folderFound);
@@ -169,7 +171,7 @@ private List<ITocItem> ReadChildren(KeyValuePair<YamlNode, YamlNode> entry, stri
 		}
 
 		if (file is not null)
-			return [new FileReference($"{parentPath}/{file}".TrimStart('/'), fileFound, children ?? [])];
+			return [new FileReference($"{parentPath}/{file}".TrimStart('/'), fileFound, hiddenFile, children ?? [])];
 
 		if (folder is not null)
 		{
@@ -227,7 +229,7 @@ private Dictionary<string, string> ReadDictionary(KeyValuePair<YamlNode, YamlNod
 		return folder;
 	}
 
-	private string? ReadFile(KeyValuePair<YamlNode, YamlNode> entry, string parentPath, out bool found)
+	private string? ReadFile(KeyValuePair<YamlNode, YamlNode> entry, string parentPath, string key, out bool found)
 	{
 		found = false;
 		var file = ReadString(entry);

src/Elastic.Markdown/IO/Configuration/ITocItem.cs

@@ -6,6 +6,6 @@ namespace Elastic.Markdown.IO.Configuration;
 
 public interface ITocItem;
 
-public record FileReference(string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+public record FileReference(string Path, bool Found, bool Hidden, IReadOnlyCollection<ITocItem> Children) : ITocItem;
 
 public record FolderReference(string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -92,6 +92,39 @@ public DocumentationSet(BuildContext context)
 		return null;
 	}
 
+	public MarkdownFile? GetPrevious(MarkdownFile current)
+	{
+		var index = current.NavigationIndex;
+		do
+		{
+			var previous = MarkdownFiles.GetValueOrDefault(index - 1);
+			if (previous is null)
+				return null;
+			if (!previous.Hidden)
+				return previous;
+			index--;
+		} while (index > 0);
+
+		return null;
+	}
+
+	public MarkdownFile? GetNext(MarkdownFile current)
+	{
+		var index = current.NavigationIndex;
+		do
+		{
+			var previous = MarkdownFiles.GetValueOrDefault(index + 1);
+			if (previous is null)
+				return null;
+			if (!previous.Hidden)
+				return previous;
+			index++;
+		} while (index <= MarkdownFiles.Count - 1);
+
+		return null;
+	}
+
+
 	public async Task ResolveDirectoryTree(Cancel ctx) =>
 		await Tree.Resolve(ctx);
 

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -39,6 +39,7 @@ public DocumentationGroup? Parent
 		set => _parent = value;
 	}
 
+	public bool Hidden { get; internal set; }
 	public string? UrlPathPrefix { get; }
 	private MarkdownParser MarkdownParser { get; }
 	public YamlFrontMatter? YamlFrontMatter { get; private set; }

src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs

@@ -89,11 +89,15 @@ public DocumentationGroup(
 					continue;
 
 				md.Parent = this;
+				md.Hidden = file.Hidden;
 				var navigationIndex = Interlocked.Increment(ref fileIndex);
 				md.NavigationIndex = navigationIndex;
 
 				if (file.Children.Count > 0 && d is MarkdownFile virtualIndex)
 				{
+					if (file.Hidden)
+						context.EmitError(context.ConfigurationPath, $"The following file is hidden but has children: {file.Path}");
+
 					var group = new DocumentationGroup(context, file.Children, lookup, folderLookup, ref fileIndex, depth + 1, virtualIndex)
 					{
 						Parent = this
@@ -110,7 +114,8 @@ public DocumentationGroup(
 				// add the page to navigation items unless it's the index file
 				// the index file can either be the discovered `index.md` or the parent group's
 				// explicit index page. E.g. when grouping related files together.
-				if (indexFile != md)
+				// if the page is referenced as hidden in the TOC do not include it in the navigation
+				if (indexFile != md && !md.Hidden)
 					navigationItems.Add(new FileNavigation(index, depth, md));
 			}
 			else if (tocItem is FolderReference folder)
@@ -120,7 +125,7 @@ public DocumentationGroup(
 					&& folderLookup.TryGetValue(folder.Path, out var documentationFiles))
 				{
 					children = documentationFiles
-						.Select(d => new FileReference(d.RelativePath, true, []))
+						.Select(d => new FileReference(d.RelativePath, true, false, []))
 						.ToArray();
 				}
 

src/Elastic.Markdown/IO/State/LinkReference.cs

@@ -12,6 +12,10 @@ public record LinkMetadata
 	[JsonPropertyName("anchors")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public required string[]? Anchors { get; init; } = [];
+
+	[JsonPropertyName("hidden")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
+	public required bool Hidden { get; init; }
 }
 
 public record LinkReference
@@ -33,11 +37,11 @@ public static LinkReference Create(DocumentationSet set)
 	{
 		var crossLinks = set.Context.Collector.CrossLinks.ToHashSet().ToArray();
 		var links = set.MarkdownFiles.Values
-			.Select(m => (m.RelativePath, m.Anchors))
+			.Select(m => (m.RelativePath, File: m))
 			.ToDictionary(k => k.RelativePath, v =>
 			{
-				var anchors = v.Anchors.Count == 0 ? null : v.Anchors.ToArray();
-				return new LinkMetadata { Anchors = anchors };
+				var anchors = v.File.Anchors.Count == 0 ? null : v.File.Anchors.ToArray();
+				return new LinkMetadata { Anchors = anchors, Hidden = v.File.Hidden };
 			});
 		return new LinkReference
 		{

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -45,8 +45,8 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 		await DocumentationSet.Tree.Resolve(ctx);
 		var navigationHtml = await RenderNavigation(markdown, ctx);
 
-		var previous = DocumentationSet.MarkdownFiles.GetValueOrDefault(markdown.NavigationIndex - 1);
-		var next = DocumentationSet.MarkdownFiles.GetValueOrDefault(markdown.NavigationIndex + 1);
+		var previous = DocumentationSet.GetPrevious(markdown);
+		var next = DocumentationSet.GetNext(markdown);
 
 		var remote = DocumentationSet.Context.Git.RepositoryName;
 		var branch = DocumentationSet.Context.Git.Branch;
@@ -67,7 +67,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 			UrlPathPrefix = markdown.UrlPathPrefix,
 			Applies = markdown.YamlFrontMatter?.AppliesTo,
 			GithubEditUrl = editUrl,
-			AllowIndexing = DocumentationSet.Context.AllowIndexing
+			AllowIndexing = DocumentationSet.Context.AllowIndexing && !markdown.Hidden
 		});
 		return await slice.RenderAsync(cancellationToken: ctx);
 	}

src/Elastic.Markdown/Slices/Layout/_TocTree.cshtml

@@ -6,7 +6,7 @@
 				<p class="caption" role="heading" aria-level="3">
 					<span class="caption-text">Elastic Docs Guide</span>
 				</p>
-				<ul class="current">@await RenderPartialAsync(Elastic.Markdown.Slices.Layout._TocTreeNav.Create(new NavigationTreeItem
+				<ul class="current">@await RenderPartialAsync(_TocTreeNav.Create(new NavigationTreeItem
 					{
 						Level = Model.Tree.Depth,
 						SubTree = Model.Tree,