feat: ignore frontmatter, support mdx

Author: palkan

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["3.1", "3.2", "3.3"]
+        ruby: ["3.1", "3.2", "3.3", "3.4"]
     steps:
     - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## master
 
+## 0.2.0 (2025-08-05)
+
+- Ignore frontmatter. ([@palkan][])
+
+- Support `.mdx` ([@palkan][])
+
 ## 0.1.2 (2025-01-03)
 
 - Fix the bug with links including PWD. ([@palkan][])

etc/list_chunks.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# A script to remove broken chunks from the dataset, i.e., chunks matching the given link pattern
+
+require "uptriever"
+require "ruby-progressbar"
+
+client = Uptriever::Client.new
+
+chunks = {}
+
+usage = client.usage
+
+puts "Total chunks: #{usage["chunk_count"]}"
+
+progressbar = ProgressBar.create(title: "Scroll chunks", total: usage["chunk_count"])
+
+client.scroll_chunks do |chunk|
+  progressbar.increment
+  chunks[chunk["id"]] = chunk
+end
+
+chunks.each_value do |chunk|
+  puts "Chunk [#{chunk["id"]}]: #{chunk["link"]}"
+end

lib/uptriever/chunker.rb

@@ -41,6 +41,6 @@ def chunks
 
     private
 
-    def chunk_dup = chunk.dup.tap { _1[:chunk_html] = +"" }
+    def chunk_dup = chunk.dup
   end
 end

lib/uptriever/client.rb

@@ -26,7 +26,7 @@ def push_group(group, upsert: true)
 
     def push_chunk(chunk, upsert: true)
       chunk[:upsert_by_tracking_id] = upsert
-      perform_request("/chunk", chunk.to_json)
+      perform_request("/chunk", chunk.to_json).inspect
     end
 
     def scroll_chunks(per_page: 100)

lib/uptriever/document.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require "redcarpet"
+require "front_matter_parser"
+require "front_matter_parser/syntax_parser"
+FrontMatterParser::SyntaxParser::Mdx = FrontMatterParser::SyntaxParser::Md
 
 module Uptriever
   class Document
@@ -17,9 +20,10 @@ def initialize(id, path, link, tags: nil, groups: nil, weight: 1.0)
 
     def to_html
       case File.extname(path)
-      when ".md"
+      when ".md", ".mdx"
+        parsed = FrontMatterParser::Parser.parse_file(path)
         markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML, autolink: true, tables: true)
-        markdown.render(File.read(path))
+        markdown.render(parsed.content)
       when ".html"
         File.read(path)
       else
@@ -29,7 +33,7 @@ def to_html
 
     def to_chunk_json
       {
-        chunk_html: to_html,
+        chunk_html: +to_html,
         link:,
         tracking_id: id,
         weight:

lib/uptriever/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Uptriever # :nodoc:
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

test/chunker_test.rb

@@ -5,7 +5,7 @@
 module Uptriever
   class ChunkerTest < TestCase
     def setup
-      @chunk = {chunk_html: "<h1>Title</h1>", link: "http://example.com", tracking_id: "root"}
+      @chunk = {chunk_html: +"<h1>Title</h1>", link: "http://example.com", tracking_id: "root"}
       @chunker = Chunker.new(@chunk)
     end
 
@@ -21,7 +21,7 @@ def test_chunks_with_single_h1_html
     end
 
     def test_chunks_with_h2_splitting
-      @chunker.chunk[:chunk_html] = "<h1>Title</h1><h2>Section 1</h2><p>Content 1</p><h2>Section 2</h2><p>Content 2</p>"
+      @chunker.chunk[:chunk_html] = +"<h1>Title</h1><h2>Section 1</h2><p>Content 1</p><h2>Section 2</h2><p>Content 2</p>"
       chunks = @chunker.chunks
       assert_equal 3, chunks.size
       assert chunks[1][:link].include?("?id=section-1")
@@ -31,7 +31,7 @@ def test_chunks_with_h2_splitting
     end
 
     def test_chunks_with_non_h2_content
-      @chunker.chunk[:chunk_html] = "<h1>Title</h1><p>Intro</p>"
+      @chunker.chunk[:chunk_html] = +"<h1>Title</h1><p>Intro</p>"
       chunks = @chunker.chunks
       assert_equal 1, chunks.size
       assert chunks.first[:chunk_html].include?("Intro")

uptriever.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
   s.add_dependency "json"
   s.add_dependency "optparse"
   s.add_dependency "erb"
+  s.add_dependency "front_matter_parser"
 
   s.add_development_dependency "bundler", ">= 1.15"
   s.add_development_dependency "rake", ">= 13.0"