feat(crawl): --skip-sitemap

Fixes #10

Author: harlan-zw

packages/crawl/src/cli.ts

@@ -154,6 +154,10 @@ async function interactiveCrawl(): Promise<CrawlOptions | null> {
         ],
         initialValues: ['llms.txt', 'llms-full.txt', 'markdown'],
       }),
+      skipSitemap: () => p.confirm({
+        message: 'Skip sitemap.xml and robots.txt discovery?',
+        initialValue: false,
+      }),
     },
     {
       onCancel: () => {
@@ -192,12 +196,16 @@ async function interactiveCrawl(): Promise<CrawlOptions | null> {
     `Max pages: Unlimited`,
     `Follow links: Yes (depth 3)`,
     `Output formats: ${outputFormats.join(', ')}`,
-    `Sitemap discovery: Automatic`,
+    `Sitemap discovery: ${advancedOptions.skipSitemap ? 'Skipped' : 'Automatic'}`,
     inferredOrigin && `Origin: ${inferredOrigin}`,
   ].filter(Boolean)
 
   p.note(summary.join('\n'), 'Crawl Configuration')
 
+  // Warn if using skip-sitemap with wildcard URLs in interactive mode
+  if (advancedOptions.skipSitemap && globPatterns.some(p => p.isGlob)) {
+    p.log.warn('Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.')  }
+
   return {
     urls,
     outputDir: resolve(outputDir),
@@ -211,6 +219,7 @@ async function interactiveCrawl(): Promise<CrawlOptions | null> {
     globPatterns,
     verbose: false,
     maxDepth: 3,
+    skipSitemap: advancedOptions.skipSitemap,
   }
 }
 
@@ -254,17 +263,19 @@ Options:
   --max-pages <number>        Maximum pages to crawl (default: unlimited)
   --crawl-delay <seconds>     Crawl delay in seconds
   --exclude <pattern>         Exclude URLs matching glob patterns (can be used multiple times)
+  --skip-sitemap              Skip sitemap.xml and robots.txt discovery
   -v, --verbose               Enable verbose logging
   -h, --help                  Show this help message
   --version                   Show version number
 
-Note: Sitemap discovery and robots.txt checking are automatic
+Note: Sitemap discovery and robots.txt checking are automatic unless --skip-sitemap is used.
 
 Examples:
   @mdream/crawl -u harlanzw.com --artifacts "llms.txt,markdown"
   @mdream/crawl --url https://docs.example.com --depth 2 --artifacts "llms-full.txt"
   @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
   @mdream/crawl -u example.com --verbose
+  @mdream/crawl -u example.com --skip-sitemap
 `)
     process.exit(0)
   }
@@ -417,6 +428,14 @@ Examples:
   // Check for verbose flag
   const verbose = args.includes('--verbose') || args.includes('-v')
 
+  // Check for skip-sitemap flag
+  const skipSitemap = args.includes('--skip-sitemap')
+
+  // Warn if using skip-sitemap with wildcard URLs
+  if (skipSitemap && parsed.isGlob) {
+    p.log.warn('Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.')
+  }
+
   return {
     urls: [url],
     outputDir: resolve(getArgValue('--output') || getArgValue('-o') || 'output'),
@@ -434,6 +453,7 @@ Examples:
     crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : undefined,
     exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
     verbose,
+    skipSitemap,
   }
 }
 
@@ -465,6 +485,7 @@ async function main() {
       `Depth: ${options.maxDepth}`,
       `Formats: ${formats.join(', ')}`,
       options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(', ')}`,
+      options.skipSitemap && `Skip sitemap: Yes`,
       options.verbose && `Verbose: Enabled`,
     ].filter(Boolean)
 

packages/crawl/src/crawl.ts

@@ -72,6 +72,7 @@ export async function crawlAndGenerate(options: CrawlOptions, onProgress?: (prog
     siteNameOverride,
     descriptionOverride,
     verbose = false,
+    skipSitemap = false,
   } = options
 
   // Normalize and resolve the output directory
@@ -105,7 +106,7 @@ export async function crawlAndGenerate(options: CrawlOptions, onProgress?: (prog
   // Track sitemap discovery attempts
   const sitemapAttempts: { url: string, success: boolean, error?: string }[] = []
 
-  if (startingUrls.length > 0) {
+  if (startingUrls.length > 0 && !skipSitemap) {
     const baseUrl = new URL(startingUrls[0]).origin
     const homePageUrl = baseUrl
 
@@ -290,6 +291,15 @@ export async function crawlAndGenerate(options: CrawlOptions, onProgress?: (prog
     progress.crawling.total = startingUrls.length
     onProgress?.(progress)
   }
+  else if (skipSitemap && startingUrls.length > 0) {
+    // When skipping sitemap discovery, immediately mark as completed
+    progress.sitemap.status = 'completed'
+    progress.sitemap.found = 0
+    progress.sitemap.processed = 0
+    progress.crawling.total = startingUrls.length
+    onProgress?.(progress)
+    // Don't show any sitemap discovery box when skipping
+  }
 
   // Ensure output directory exists
   if (!existsSync(outputDir)) {
@@ -460,7 +470,7 @@ export async function crawlAndGenerate(options: CrawlOptions, onProgress?: (prog
       }
     },
     maxRequestsPerCrawl,
-    respectRobotsTxtFile: true,
+    respectRobotsTxtFile: !skipSitemap,
   }
 
   // Add crawl delay if specified

packages/crawl/src/types.ts

@@ -17,6 +17,7 @@ export interface CrawlOptions {
   siteNameOverride?: string
   descriptionOverride?: string
   verbose?: boolean
+  skipSitemap?: boolean
 }
 
 export interface ParsedUrlPattern {