Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func Main() {
slack.OptionAppLevelToken(cfg.SlackAppToken),
)

nudge.StartNudgeScheduler(cfg, api)
nudge.StartNudgeScheduler(cfg, db, api)
fetch.StartAutoFetchScheduler(cfg, db, api)

log.Println("Starting Engineering Report Bot...")
Expand Down
101 changes: 99 additions & 2 deletions internal/integrations/llm/llm.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"sync"

Expand Down Expand Up @@ -65,6 +67,8 @@ const defaultOpenAIModel = "gpt-4o-mini"
const maxTemplateGuidanceChars = 8000
const defaultLLMMaxConcurrentBatches = 4

var confidenceFieldRe = regexp.MustCompile(`("confidence"\s*:\s*)([^,}\]]+)`)

func llmBatchConcurrencyLimit(totalBatches int) int {
if totalBatches <= 0 {
return 1
Expand Down Expand Up @@ -295,7 +299,7 @@ Also:
- choose normalized_status from: done, in testing, in progress, other
- extract ticket IDs if present (e.g. [1247202] or bare ticket numbers)
- if this item is the same underlying work as an existing item, set duplicate_of to that existing key (Kxx); otherwise empty string
- set confidence between 0 and 1.
- set confidence between 0 and 1, using digits only (example: 0.91). Never spell out numbers.
%s%s

Respond with JSON only (no markdown):
Expand Down Expand Up @@ -362,7 +366,14 @@ func parseSectionClassifiedResponse(responseText string) (map[int64]LLMSectionDe

var classified []sectionClassifiedItem
if err := json.Unmarshal([]byte(responseText), &classified); err != nil {
return nil, fmt.Errorf("parsing LLM section response: %w (response: %s)", err, responseText)
repaired := repairSectionClassifiedResponse(responseText)
if repaired == responseText {
return nil, fmt.Errorf("parsing LLM section response: %w (response: %s)", err, responseText)
}
if repairErr := json.Unmarshal([]byte(repaired), &classified); repairErr != nil {
return nil, fmt.Errorf("parsing LLM section response: %w (response: %s)", err, responseText)
Comment thread
WZ marked this conversation as resolved.
Outdated
}
log.Printf("llm section response repaired before parse")
}

decisions := make(map[int64]LLMSectionDecision)
Expand Down Expand Up @@ -424,6 +435,92 @@ func parseTicketIDsField(raw json.RawMessage) string {
return ""
}

func repairSectionClassifiedResponse(responseText string) string {
return confidenceFieldRe.ReplaceAllStringFunc(responseText, func(match string) string {
parts := confidenceFieldRe.FindStringSubmatch(match)
if len(parts) != 3 {
return match
}
return parts[1] + normalizeConfidenceLiteral(parts[2])
})
}

func normalizeConfidenceLiteral(raw string) string {
text := strings.TrimSpace(strings.Trim(raw, `"`))
if text == "" {
return "0"
}
if parsed, ok := parseLooseConfidence(text); ok {
return strconv.FormatFloat(parsed, 'f', -1, 64)
}
return "0"
}

func parseLooseConfidence(raw string) (float64, bool) {
text := strings.TrimSpace(raw)
if text == "" {
return 0, false
}
if parsed, err := strconv.ParseFloat(text, 64); err == nil {
return clampConfidence(parsed), true
}

compact := strings.ReplaceAll(strings.ToLower(text), " ", "")
if parsed, err := strconv.ParseFloat(compact, 64); err == nil {
return clampConfidence(parsed), true
}

if strings.HasPrefix(compact, "0.") {
if digit, ok := digitWord(compact[2:]); ok {
return float64(digit) / 10, true
}
}
if compact == "zero" {
return 0, true
}
if compact == "one" {
return 1, true
}
return 0, false
}

func digitWord(s string) (int, bool) {
switch s {
case "zero":
return 0, true
case "one":
return 1, true
case "two":
return 2, true
case "three":
return 3, true
case "four":
return 4, true
case "five":
return 5, true
case "six":
return 6, true
case "seven":
return 7, true
case "eight":
return 8, true
case "nine":
return 9, true
default:
return 0, false
}
}

func clampConfidence(v float64) float64 {
if v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}

func loadGlossaryIfConfigured(cfg Config) (*LLMGlossary, error) {
if strings.TrimSpace(cfg.LLMGlossaryPath) == "" {
return nil, nil
Expand Down
41 changes: 41 additions & 0 deletions internal/integrations/llm/llm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,47 @@ func TestParseSectionClassifiedResponse_AcceptsArrayTicketIDs(t *testing.T) {
}
}

func TestParseSectionClassifiedResponse_RepairsMalformedConfidence(t *testing.T) {
response := `[
{"id": 1, "section_id": "S0_0", "normalized_status": "done", "ticket_ids": "", "duplicate_of": "", "confidence": 0. Nine},
{"id": 2, "section_id": "S0_1", "normalized_status": "in progress", "ticket_ids": "", "duplicate_of": "", "confidence": 0. seven},
{"id": 3, "section_id": "S0_2", "normalized_status": "done", "ticket_ids": "", "duplicate_of": "", "confidence": nope}
]`

got, err := parseSectionClassifiedResponse(response)
if err != nil {
t.Fatalf("parseSectionClassifiedResponse should repair malformed confidence: %v", err)
}
if got[1].Confidence != 0.9 {
t.Fatalf("expected repaired confidence 0.9, got %v", got[1].Confidence)
}
if got[2].Confidence != 0.7 {
t.Fatalf("expected repaired confidence 0.7, got %v", got[2].Confidence)
}
if got[3].Confidence != 0 {
t.Fatalf("expected fallback confidence 0, got %v", got[3].Confidence)
}
}

func TestNormalizeConfidenceLiteral(t *testing.T) {
tests := []struct {
raw string
want string
}{
{raw: "0.91", want: "0.91"},
{raw: `"0.82"`, want: "0.82"},
{raw: "0. Nine", want: "0.9"},
{raw: "0. seven", want: "0.7"},
{raw: "garbage", want: "0"},
}

for _, tt := range tests {
if got := normalizeConfidenceLiteral(tt.raw); got != tt.want {
t.Fatalf("normalizeConfidenceLiteral(%q) = %q, want %q", tt.raw, got, tt.want)
}
}
}

func TestParseTicketIDsField_MixedArray(t *testing.T) {
raw := json.RawMessage(`[ "123", 456, "", " 789 " ]`)
got := parseTicketIDsField(raw)
Expand Down
20 changes: 18 additions & 2 deletions internal/integrations/slack/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type ClassificationStats = domain.ClassificationStats
type sectionOption = llm.SectionOption
type BuildResult = report.BuildResult
type LLMSectionDecision = llm.LLMSectionDecision
type RenderedNudge = nudge.RenderedNudge

type loadStatus int

Expand All @@ -32,6 +33,13 @@ const (
templateFirstEver
)

const (
actionNudgeDone = nudge.ActionDone
actionNudgeMore = nudge.ActionMore
actionNudgePagePrev = nudge.ActionPagePrev
actionNudgePageNext = nudge.ActionPageNext
)

func ReportWeekRange(cfg Config, now time.Time) (time.Time, time.Time) {
return domain.ReportWeekRange(cfg, now)
}
Expand Down Expand Up @@ -136,6 +144,10 @@ func UpdateWorkItemTextAndStatus(db *sql.DB, id int64, description, status strin
return sqlite.UpdateWorkItemTextAndStatus(db, id, description, status)
}

func UpdateWorkItemStatus(db *sql.DB, id int64, status string) error {
return sqlite.UpdateWorkItemStatus(db, id, status)
}

func UpdateWorkItemCategory(db *sql.DB, id int64, category string) error {
return sqlite.UpdateWorkItemCategory(db, id, category)
}
Expand Down Expand Up @@ -180,8 +192,12 @@ func extractGlossaryPhrase(description string) string {
return llm.ExtractGlossaryPhrase(description)
}

func sendNudges(api *slack.Client, cfg Config, memberIDs []string, reportChannelID string) {
nudge.SendNudges(api, cfg, memberIDs, reportChannelID)
func sendNudges(api *slack.Client, db *sql.DB, cfg Config, memberIDs []string, reportChannelID string) {
nudge.SendNudges(api, db, cfg, memberIDs, reportChannelID)
}

func RenderNudgeForUser(api *slack.Client, db *sql.DB, cfg Config, userID, reportChannelID string, now time.Time, page int, updated bool) (RenderedNudge, error) {
return nudge.RenderNudgeForUser(api, db, cfg, userID, reportChannelID, now, page, updated)
}

func normalizeTextToken(s string) string {
Expand Down
Loading