fix(mdream): cleaner llms-full.txt output

Author: harlan-zw

packages/mdream/src/llms-txt.ts

@@ -202,6 +202,48 @@ function generateLlmsTxtContent(files: ProcessedFile[], options: Pick<LlmsTxtArt
   return content
 }
 
+/**
+ * Parse frontmatter from markdown content
+ */
+function parseFrontmatter(content: string): { frontmatter: Record<string, any> | null, body: string } {
+  const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/
+  const match = content.match(frontmatterRegex)
+
+  if (!match) {
+    return { frontmatter: null, body: content }
+  }
+
+  const frontmatterContent = match[1]
+  const body = match[2]
+
+  const frontmatter: Record<string, any> = {}
+  const lines = frontmatterContent.split('\n')
+
+  for (const line of lines) {
+    const colonIndex = line.indexOf(':')
+    if (colonIndex > 0) {
+      const key = line.substring(0, colonIndex).trim()
+      const value = line.substring(colonIndex + 1).trim()
+      frontmatter[key] = value
+    }
+  }
+
+  return { frontmatter, body }
+}
+
+/**
+ * Serialize frontmatter object to YAML-like format
+ */
+function serializeFrontmatter(data: Record<string, any>): string {
+  const lines: string[] = []
+  for (const [key, value] of Object.entries(data)) {
+    if (value !== undefined && value !== null) {
+      lines.push(`${key}: ${String(value)}`)
+    }
+  }
+  return lines.join('\n')
+}
+
 /**
  * Generate llms-full.txt content with complete page content
  */
@@ -229,16 +271,46 @@ function generateLlmsFullTxtContent(files: ProcessedFile[], options: Pick<LlmsTx
       const url = file.url.startsWith('http://') || file.url.startsWith('https://')
         ? file.url
         : (origin ? origin + file.url : file.url)
-      content += `## ${file.title}\n\n`
-      content += `**URL:** ${url}\n`
+
+      // Parse existing frontmatter from content
+      const { frontmatter, body } = parseFrontmatter(file.content)
+
+      // Prepare metadata to add
+      const metadata: Record<string, any> = {
+        title: file.title,
+        url,
+      }
+
       if (file.filePath && options.outputDir) {
-        const relativePath = relative(options.outputDir, file.filePath)
-        content += `**File:** ${relativePath}\n`
+        metadata.file = relative(options.outputDir, file.filePath)
       }
       else if (file.filePath) {
-        content += `**File:** ${file.filePath}\n`
+        metadata.file = file.filePath
+      }
+
+      // Add any additional metadata from the file
+      if (file.metadata) {
+        if (file.metadata.description)
+          metadata.description = file.metadata.description
+        if (file.metadata.keywords)
+          metadata.keywords = file.metadata.keywords
+        if (file.metadata.author)
+          metadata.author = file.metadata.author
       }
-      content += `\n${file.content}\n\n---\n\n`
+
+      // Always include frontmatter for uniform formatting
+      const mergedFrontmatter = frontmatter ? { ...frontmatter, ...metadata } : metadata
+      const frontmatterString = serializeFrontmatter(mergedFrontmatter)
+      let contentBody = frontmatter ? body : file.content
+
+      // Remove duplicate title from the beginning of content if it exists
+      const titleLine = contentBody.trim().split('\n')[0]
+      if (titleLine === file.title || titleLine === `# ${file.title}`) {
+        // Remove the first line (title) and any following empty lines
+        contentBody = contentBody.trim().split('\n').slice(1).join('\n').trimStart()
+      }
+
+      content += `---\n${frontmatterString}\n---\n\n${contentBody}\n\n---\n\n`
     }
   }
 

packages/mdream/test/unit/cli-llms.test.ts

@@ -66,13 +66,15 @@ it('should generate llms-full.txt with complete content', async () => {
   expect(fullContent).toContain('- [About Us - Test Site](#about-us---test-site)')
 
   // Check full content is included
-  expect(fullContent).toContain('## Test Site - Welcome to Our Homepage')
-  expect(fullContent).toContain('**URL:** /')
-  expect(fullContent).toContain('Welcome to Test Site')
+  // Now using frontmatter format with title only in frontmatter
+  expect(fullContent).toContain('---')
+  expect(fullContent).toContain('title: Test Site - Welcome to Our Homepage')
+  expect(fullContent).toContain('url: /')
+  expect(fullContent).toContain('# Welcome to Test Site')
   expect(fullContent).toContain('Homepage Content')
 
   // Check blog post content
-  expect(fullContent).toContain('## First Blog Post - Test Site Blog')
+  expect(fullContent).toContain('title: First Blog Post - Test Site Blog')
   expect(fullContent).toContain('First Blog Post: Testing HTML to Markdown')
   expect(fullContent).toContain('function convertHtmlToMarkdown')
 })

packages/mdream/test/unit/llms-txt.test.ts

@@ -1,7 +1,8 @@
+import type { ProcessedFile } from '../../src/llms-txt.ts'
 import { mkdir, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { expect, it } from 'vitest'
+import { describe, expect, it } from 'vitest'
 import { generateLlmsTxtArtifacts } from '../../src/llms-txt.ts'
 
 const testDir = join(tmpdir(), 'mdream-llms-txt-test')
@@ -108,9 +109,9 @@ it('should generate llms-full.txt when requested', async () => {
   expect(result.llmsFullTxt).toContain('# Test Site')
   expect(result.llmsFullTxt).toContain('> A test site for mdream')
   expect(result.llmsFullTxt).toContain('## Table of Contents')
-  expect(result.llmsFullTxt).toContain('## Test Site - Home')
-  expect(result.llmsFullTxt).toContain('**URL:** https://example.com/')
-  expect(result.llmsFullTxt).toContain('Welcome to Test Site')
+  expect(result.llmsFullTxt).toContain('title: Test Site - Home')
+  expect(result.llmsFullTxt).toContain('url: https://example.com/')
+  expect(result.llmsFullTxt).toContain('# Welcome to Test Site')
 
   await cleanup()
 })
@@ -222,3 +223,126 @@ it('should work with pre-processed files', async () => {
   expect(result.llmsFullTxt).toContain('# Welcome')
   expect(result.llmsFullTxt).toContain('# About')
 })
+
+describe('llms-txt frontmatter handling', () => {
+  it('should prepend metadata to existing frontmatter', async () => {
+    const filesWithFrontmatter: ProcessedFile[] = [{
+      title: 'Test Page',
+      content: `---
+existingKey: existingValue
+tags: test, sample
+---
+
+# Content
+
+This is the main content of the page.`,
+      url: '/test-page',
+      metadata: {
+        description: 'A test page with frontmatter',
+        author: 'Test Author',
+      },
+    }]
+
+    const result = await generateLlmsTxtArtifacts({
+      files: filesWithFrontmatter,
+      siteName: 'Test Site',
+      origin: 'https://example.com',
+      generateFull: true,
+    })
+
+    expect(result.llmsFullTxt).toBeDefined()
+    expect(result.llmsFullTxt).toContain('---')
+    expect(result.llmsFullTxt).toContain('title: Test Page')
+    expect(result.llmsFullTxt).toContain('url: https://example.com/test-page')
+    expect(result.llmsFullTxt).toContain('existingKey: existingValue')
+    expect(result.llmsFullTxt).toContain('tags: test, sample')
+    expect(result.llmsFullTxt).toContain('description: A test page with frontmatter')
+    expect(result.llmsFullTxt).toContain('author: Test Author')
+    expect(result.llmsFullTxt).toContain('# Content')
+  })
+
+  it('should add frontmatter to content without existing frontmatter', async () => {
+    const filesWithoutFrontmatter: ProcessedFile[] = [{
+      title: 'Simple Page',
+      content: `# Simple Content
+
+This page has no frontmatter.`,
+      url: '/simple-page',
+    }]
+
+    const result = await generateLlmsTxtArtifacts({
+      files: filesWithoutFrontmatter,
+      siteName: 'Test Site',
+      origin: 'https://example.com',
+      generateFull: true,
+    })
+
+    expect(result.llmsFullTxt).toBeDefined()
+    // Should now always have frontmatter with title and url
+    expect(result.llmsFullTxt).toContain('---')
+    expect(result.llmsFullTxt).toContain('title: Simple Page')
+    expect(result.llmsFullTxt).toContain('url: https://example.com/simple-page')
+    expect(result.llmsFullTxt).toContain('# Simple Content')
+    // Should not have section header
+    expect(result.llmsFullTxt).not.toContain('## Simple Page')
+  })
+
+  it('should handle files with file paths and outputDir', async () => {
+    const filesWithPaths: ProcessedFile[] = [{
+      title: 'File Page',
+      filePath: '/home/user/output/md/page.md',
+      content: `---
+category: documentation
+---
+
+# Documentation Page`,
+      url: '/docs/page',
+    }]
+
+    const result = await generateLlmsTxtArtifacts({
+      files: filesWithPaths,
+      siteName: 'Test Site',
+      origin: 'https://example.com',
+      outputDir: '/home/user/output',
+      generateFull: true,
+    })
+
+    expect(result.llmsFullTxt).toBeDefined()
+    expect(result.llmsFullTxt).toContain('file: md/page.md')
+    expect(result.llmsFullTxt).toContain('category: documentation')
+  })
+
+  it('should merge metadata correctly with existing frontmatter', async () => {
+    const filesWithConflict: ProcessedFile[] = [{
+      title: 'Conflict Test',
+      content: `---
+title: Original Title
+url: /old-url
+custom: value
+---
+
+Content here`,
+      url: '/new-url',
+      metadata: {
+        description: 'New description',
+      },
+    }]
+
+    const result = await generateLlmsTxtArtifacts({
+      files: filesWithConflict,
+      siteName: 'Test Site',
+      origin: 'https://example.com',
+      generateFull: true,
+    })
+
+    expect(result.llmsFullTxt).toBeDefined()
+    // New metadata should be prepended, existing should be preserved
+    expect(result.llmsFullTxt).toContain('title: Conflict Test')
+    expect(result.llmsFullTxt).toContain('url: https://example.com/new-url')
+    expect(result.llmsFullTxt).toContain('description: New description')
+    expect(result.llmsFullTxt).toContain('custom: value')
+    // Original values that conflict should be overwritten
+    expect(result.llmsFullTxt).not.toContain('title: Original Title')
+    expect(result.llmsFullTxt).not.toContain('url: /old-url')
+  })
+})