Merge pull request #1613 from ksylvan/0714-fixes-for-custom-directory-unique-patterns-list-changelog-cache

Improve AI Summarization for Consistent Professional Changelog Entries

Author: ksylvan

.vscode/settings.json

@@ -99,6 +99,7 @@
 		"seaborn",
 		"semgrep",
 		"sess",
+		"storer",
 		"Streamlit",
 		"stretchr",
 		"talkpanel",

CHANGELOG.md

@@ -82,8 +82,8 @@ func (g *Generator) collectData() error {
 		if cachedTag != "" {
 			// Get the current latest tag from git
 			currentTag, err := g.gitWalker.GetLatestTag()
-			if err == nil && currentTag == cachedTag {
-				// Same tag - load cached data and walk commits since tag for "Unreleased"
+			if err == nil {
+				// Load cached data - we can use it even if there are new tags
 				cachedVersions, err := g.cache.GetVersions()
 				if err == nil && len(cachedVersions) > 0 {
 					g.versions = cachedVersions
@@ -97,7 +97,25 @@ func (g *Generator) collectData() error {
 						}
 					}
 
-					// Walk commits since the latest tag to get new unreleased commits
+					// If we have new tags since cache, process the new versions only
+					if currentTag != cachedTag {
+						fmt.Fprintf(os.Stderr, "Processing new versions since %s...\n", cachedTag)
+						newVersions, err := g.gitWalker.WalkHistorySinceTag(cachedTag)
+						if err != nil {
+							fmt.Fprintf(os.Stderr, "Warning: Failed to walk history since tag %s: %v\n", cachedTag, err)
+						} else {
+							// Merge new versions into cached versions (only add if not already cached)
+							for name, version := range newVersions {
+								if name != "Unreleased" { // Handle Unreleased separately
+									if _, exists := g.versions[name]; !exists {
+										g.versions[name] = version
+									}
+								}
+							}
+						}
+					}
+
+					// Always update Unreleased section with latest commits
 					unreleasedVersion, err := g.gitWalker.WalkCommitsSinceTag(currentTag)
 					if err != nil {
 						fmt.Fprintf(os.Stderr, "Warning: Failed to walk commits since tag %s: %v\n", currentTag, err)
@@ -110,6 +128,29 @@ func (g *Generator) collectData() error {
 						g.versions["Unreleased"] = unreleasedVersion
 					}
 
+					// Save any new versions to cache (after potential AI processing)
+					if currentTag != cachedTag {
+						for _, version := range g.versions {
+							// Skip versions that were already cached and Unreleased
+							if version.Name != "Unreleased" {
+								if err := g.cache.SaveVersion(version); err != nil {
+									fmt.Fprintf(os.Stderr, "Warning: Failed to save version to cache: %v\n", err)
+								}
+
+								for _, commit := range version.Commits {
+									if err := g.cache.SaveCommit(commit, version.Name); err != nil {
+										fmt.Fprintf(os.Stderr, "Warning: Failed to save commit to cache: %v\n", err)
+									}
+								}
+							}
+						}
+
+						// Update the last processed tag
+						if err := g.cache.SetLastProcessedTag(currentTag); err != nil {
+							fmt.Fprintf(os.Stderr, "Warning: Failed to update last processed tag: %v\n", err)
+						}
+					}
+
 					return nil
 				}
 			}
@@ -298,6 +339,7 @@ func (g *Generator) formatVersion(version *git.Version) string {
 			}
 		}
 
+		// For released versions, if we have cached AI summary, use it!
 		if version.Name != "Unreleased" && version.AISummary != "" {
 			fmt.Fprintf(os.Stderr, "✅ %s already summarized (skipping)\n", version.Name)
 			sb.WriteString(version.AISummary)
@@ -529,8 +571,6 @@ func normalizeLineEndings(content string) string {
 }
 
 func (g *Generator) formatCommitMessage(message string) string {
-	prefixes := []string{"fix:", "feat:", "docs:", "style:", "refactor:",
-		"test:", "chore:", "perf:", "ci:", "build:", "revert:", "# docs:"}
 	strings_to_remove := []string{
 		"### CHANGES\n", "## CHANGES\n", "# CHANGES\n",
 		"...\n", "---\n", "## Changes\n", "## Change",
@@ -543,13 +583,6 @@ func (g *Generator) formatCommitMessage(message string) string {
 	// No hard tabs
 	message = strings.ReplaceAll(message, "\t", " ")
 
-	for _, prefix := range prefixes {
-		if strings.HasPrefix(strings.ToLower(message), prefix) {
-			message = strings.TrimSpace(message[len(prefix):])
-			break
-		}
-	}
-
 	if len(message) > 0 {
 		message = strings.ToUpper(message[:1]) + message[1:]
 	}

cmd/generate_changelog/changelog.db

@@ -10,6 +10,38 @@ import (
 const DefaultSummarizeModel = "claude-sonnet-4-20250514"
 const MinContentLength = 256 // Minimum content length to consider for summarization
 
+const prompt = `# ROLE
+You are an expert Technical Writer specializing in creating clear, concise,
+and professional release notes from raw Git commit logs.
+
+# TASK
+Your goal is to transform a provided block of Git commit logs into a clean,
+human-readable changelog summary. You will identify the most important changes,
+format them as a bulleted list, and preserve the associated Pull Request (PR)
+information.
+
+# INSTRUCTIONS:
+Follow these steps in order:
+1. Deeply analyze the input. You will be given a block of text containing PR
+   information and commit log messages. Carefully read through the logs
+   to identify individual commits and their descriptions.
+2. Identify Key Changes: Focus on commits that represent significant changes,
+   such as new features ("feat"), bug fixes ("fix"), performance improvements ("perf"),
+   or breaking changes ("BREAKING CHANGE").
+3. Select the Top 5: From the identified key changes, select a maximum of the five
+   (5) most impactful ones to include in the summary.
+   If there are five or fewer total changes, include all of them.
+4. Format the Output:
+    - Where you see a PR header, include the PR header verbatim. NO CHANGES.
+	  **This is a critical rule: Do not modify the PR header, as it contains
+	  important links.** What follow the PR header are the related changes.
+	- Do not add any additional text or preamble. Begin directly with the output.
+	- Use bullet points for each key change. Starting each point with a hyphen ("-").
+	- Ensure that the summary is concise and focused on the main changes.
+	- The summary should be in American English (en-US), using proper grammar and punctuation.
+5. If the content is too brief or you do not see any PR headers, return the content as is.
+`
+
 // getSummarizeModel returns the model to use for AI summarization
 func getSummarizeModel() string {
 	if model := os.Getenv("FABRIC_CHANGELOG_SUMMARIZE_MODEL"); model != "" {
@@ -30,17 +62,6 @@ func SummarizeVersionContent(content string) (string, error) {
 
 	model := getSummarizeModel()
 
-	prompt := `Summarize the changes extracted from Git commit logs in a concise, professional way.
-Pay particular attention to the following rules:
-- Preserve the PR headers verbatim to your summary.
-- I REPEAT: Do not change the PR headers in any way. They contain links to the PRs and Author Profiles.
-- Use bullet points for lists and key changes (rendered using "-")
-- Focus on the main changes and improvements.
-- Avoid unnecessary details or preamble.
-- Keep it under 800 characters.
-- Be brief. List only the 5 most important changes along with the PR information which should be kept intact.
-- If the content is too brief or you do not see any PR headers, return the content as is.`
-
 	cmd := exec.Command("fabric", "-m", model, prompt)
 	cmd.Stdin = strings.NewReader(content)
 

cmd/generate_changelog/internal/changelog/generator.go

@@ -5,10 +5,12 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/storer"
 )
 
 var (
@@ -280,6 +282,102 @@ func parseGitHubURL(url string) (owner, repo string) {
 	return "", ""
 }
 
+// WalkHistorySinceTag walks git history from HEAD down to (but not including) the specified tag
+// and returns any version commits found along the way
+func (w *Walker) WalkHistorySinceTag(sinceTag string) (map[string]*Version, error) {
+	// Get the commit SHA for the sinceTag
+	tagRef, err := w.repo.Tag(sinceTag)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tag %s: %w", sinceTag, err)
+	}
+
+	tagCommit, err := w.repo.CommitObject(tagRef.Hash())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get commit for tag %s: %w", sinceTag, err)
+	}
+
+	// Get HEAD reference
+	ref, err := w.repo.Head()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get HEAD: %w", err)
+	}
+
+	// Walk from HEAD down to the tag commit (excluding it)
+	commitIter, err := w.repo.Log(&git.LogOptions{
+		From:  ref.Hash(),
+		Order: git.LogOrderCommitterTime,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create commit iterator: %w", err)
+	}
+	defer commitIter.Close()
+
+	versions := make(map[string]*Version)
+	currentVersion := "Unreleased"
+
+	err = commitIter.ForEach(func(c *object.Commit) error {
+		// Stop iteration when the hash of the current commit matches the hash of the specified sinceTag commit
+		if c.Hash == tagCommit.Hash {
+			return storer.ErrStop
+		}
+
+		commit := &Commit{
+			SHA:     c.Hash.String(),
+			Message: strings.TrimSpace(c.Message),
+			Author:  c.Author.Name,
+			Email:   c.Author.Email,
+			Date:    c.Author.When,
+			IsMerge: len(c.ParentHashes) > 1,
+		}
+
+		// Check for version pattern
+		if matches := versionPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
+			commit.IsVersion = true
+			commit.Version = matches[1]
+			currentVersion = commit.Version
+
+			if _, exists := versions[currentVersion]; !exists {
+				versions[currentVersion] = &Version{
+					Name:      currentVersion,
+					Date:      commit.Date,
+					CommitSHA: commit.SHA,
+					Commits:   []*Commit{},
+				}
+			}
+			return nil
+		}
+
+		// Check for PR merge pattern
+		if commit.IsMerge {
+			if matches := prPattern.FindStringSubmatch(commit.Message); len(matches) > 1 {
+				if prNumber, err := strconv.Atoi(matches[1]); err == nil {
+					commit.PRNumber = prNumber
+				}
+			}
+		}
+
+		// Add commit to current version
+		if _, exists := versions[currentVersion]; !exists {
+			versions[currentVersion] = &Version{
+				Name:      currentVersion,
+				Date:      time.Time{}, // Zero value, will be set by version commit
+				CommitSHA: "",
+				Commits:   []*Commit{},
+			}
+		}
+
+		versions[currentVersion].Commits = append(versions[currentVersion].Commits, commit)
+		return nil
+	})
+
+	// Handle the stop condition - storer.ErrStop is expected
+	if err == storer.ErrStop {
+		err = nil
+	}
+
+	return versions, err
+}
+
 func dedupInts(ints []int) []int {
 	seen := make(map[int]bool)
 	result := []int{}

cmd/generate_changelog/internal/changelog/summarize.go

@@ -259,8 +259,24 @@ func (o *PluginRegistry) GetModels() (ret *ai.VendorsModels, err error) {
 func (o *PluginRegistry) Configure() (err error) {
 	o.ConfigureVendors()
 	_ = o.Defaults.Configure()
+	if err := o.CustomPatterns.Configure(); err != nil {
+		return fmt.Errorf("error configuring CustomPatterns: %w", err)
+	}
 	_ = o.PatternsLoader.Configure()
 
+	// Refresh the database custom patterns directory after custom patterns plugin is configured
+	customPatternsDir := os.Getenv("CUSTOM_PATTERNS_DIRECTORY")
+	if customPatternsDir != "" {
+		// Expand home directory if needed
+		if strings.HasPrefix(customPatternsDir, "~/") {
+			if homeDir, err := os.UserHomeDir(); err == nil {
+				customPatternsDir = filepath.Join(homeDir, customPatternsDir[2:])
+			}
+		}
+		o.Db.Patterns.CustomPatternsDir = customPatternsDir
+		o.PatternsLoader.Patterns.CustomPatternsDir = customPatternsDir
+	}
+
 	//YouTube and Jina are not mandatory, so ignore not configured error
 	_ = o.YouTube.Configure()
 	_ = o.Jina.Configure()

cmd/generate_changelog/internal/git/walker.go

@@ -86,9 +86,10 @@ func (o *Session) String() (ret string) {
 		ret += fmt.Sprintf("\n--- \n[%v]\n%v", message.Role, message.Content)
 		if message.MultiContent != nil {
 			for _, part := range message.MultiContent {
-				if part.Type == chat.ChatMessagePartTypeImageURL {
+				switch part.Type {
+				case chat.ChatMessagePartTypeImageURL:
 					ret += fmt.Sprintf("\n%v: %v", part.Type, *part.ImageURL)
-				} else if part.Type == chat.ChatMessagePartTypeText {
+				case chat.ChatMessagePartTypeText:
 					ret += fmt.Sprintf("\n%v: %v", part.Type, part.Text)
 				}
 			}