feat(www): add sitemap and robots.txt (#5)

Author: nchopra

apps/www/.gitignore

@@ -25,4 +25,7 @@ yarn-error.log*
 # others
 .env*.local
 .vercel
-next-env.d.ts
+next-env.d.ts
+
+# sitemap
+/public/sitemap.xml

apps/www/app/robots.txt

@@ -0,0 +1,5 @@
+User-Agent: *
+Allow: /
+Disallow: /private/
+
+Sitemap: https://amical.ai/sitemap.xml

apps/www/package.json

@@ -3,7 +3,8 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "build": "next build",
+    "build": "pnpm build:sitemap && next build",
+    "build:sitemap": "pnpm exec tsx ./scripts/generate-sitemap.mts",
     "dev": "next dev --turbo",
     "start": "next start",
     "serve": "pnpm dlx serve out -p 3000",
@@ -36,9 +37,11 @@
     "@types/node": "22.15.12",
     "@types/react": "^19.1.3",
     "@types/react-dom": "^19.1.3",
+    "globby": "^14.1.0",
     "postcss": "^8.5.3",
     "server": "^1.0.41",
     "tailwindcss": "^4.1.5",
+    "tsx": "^4.19.4",
     "tw-animate-css": "^1.2.9",
     "typescript": "^5.8.3"
   }

apps/www/scripts/generate-sitemap.mts

@@ -0,0 +1,129 @@
+import fs from 'fs';
+import { globby } from 'globby';
+import prettier from 'prettier';
+import path from 'path';
+
+async function generate() {
+  const prettierConfig = await prettier.resolveConfig('./.prettierrc');
+
+  // Ensure directories exist
+  fs.mkdirSync('public', { recursive: true });
+  fs.mkdirSync('out', { recursive: true });
+
+  // Get static pages
+  const pages = await globby([
+    'app/**/page.tsx',
+    '!app/**/_*/**',
+    '!app/**/api/**',
+    '!app/docs/**', // Exclude docs directory as we'll handle it separately
+  ]);
+
+  // Get doc pages from the build output
+  const docPages = await globby(['out/docs/**/*.html'])
+    .then(pages => pages
+      .map(page => page
+        .replace('out', '')
+        .replace('/index.html', '')
+        .replace('.html', '')
+      )
+      .filter(page => !page.includes('/_'))
+    );
+
+  // Get blog pages from the build output
+  const blogPages = await globby(['out/blog/**/*.html'])
+    .then(pages => pages
+      .map(page => page
+        .replace('out', '')
+        .replace('/index.html', '')
+        .replace('.html', '')
+      )
+      .filter(page => !page.includes('/_') && !page.includes('/blog/index'))
+    );
+
+  const baseUrl = 'https://amical.ai';
+
+  const sitemap = `
+    <?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      ${[
+        // Add static pages
+        ...pages.map((page) => {
+          const path = page
+            .replace('app', '')
+            .replace('/page.tsx', '')
+            .replace('/(home)', '')
+            .replace(/\[\[\.\.\..*?\]\]/, '');
+
+          // Skip dynamic routes with parameters
+          if (path.includes('[') || path.includes(']')) {
+            return '';
+          }
+
+          const route = path === '' ? '' : path;
+
+          return `
+            <url>
+              <loc>${baseUrl}${route}</loc>
+              <lastmod>${new Date().toISOString()}</lastmod>
+              <changefreq>daily</changefreq>
+              <priority>0.7</priority>
+            </url>
+          `;
+        }),
+        // Add docs index page
+        `
+          <url>
+            <loc>${baseUrl}/docs</loc>
+            <lastmod>${new Date().toISOString()}</lastmod>
+            <changefreq>daily</changefreq>
+            <priority>0.7</priority>
+          </url>
+        `,
+        // Add doc pages
+        ...docPages.map((path) => `
+          <url>
+            <loc>${baseUrl}${path}</loc>
+            <lastmod>${new Date().toISOString()}</lastmod>
+            <changefreq>daily</changefreq>
+            <priority>0.7</priority>
+          </url>
+        `),
+        // Add blog index page
+        `
+          <url>
+            <loc>${baseUrl}/blog</loc>
+            <lastmod>${new Date().toISOString()}</lastmod>
+            <changefreq>weekly</changefreq>
+            <priority>0.8</priority>
+          </url>
+        `,
+        // Add blog pages
+        ...blogPages.map((path) => `
+          <url>
+            <loc>${baseUrl}${path}</loc>
+            <lastmod>${new Date().toISOString()}</lastmod>
+            <changefreq>weekly</changefreq>
+            <priority>0.7</priority>
+          </url>
+        `),
+      ]
+        .filter(Boolean)
+        .join('')}
+    </urlset>
+  `;
+
+  const formatted = await prettier.format(sitemap, {
+    ...prettierConfig,
+    parser: 'html',
+  });
+
+  fs.writeFileSync('public/sitemap.xml', formatted);
+  fs.writeFileSync('out/sitemap.xml', formatted);
+
+  console.log('✅ Generated sitemap.xml');
+}
+
+generate().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});