dag-feedback-synthesizer

1. Feedback Aggregation

• Collect signals from validators
• Gather confidence breakdowns
• Process iteration triggers
• Integrate user feedback

2. Prioritization

• Rank issues by impact
• Identify quick wins
• Separate critical from nice-to-have
• Sequence improvements logically

3. Actionable Guidance

• Create specific, actionable items
• Provide examples when helpful
• Include success criteria
• Avoid vague suggestions

4. Context Preservation

• Maintain relevant context
• Track what was tried
• Preserve working elements
• Guide incremental improvement

```typescript

interface SynthesizedFeedback {

taskId: string;

iterationNumber: number;

synthesizedAt: Date;

summary: FeedbackSummary;

improvements: Improvement[];

context: FeedbackContext;

guidance: ExecutionGuidance;

}

interface FeedbackSummary {

overallAssessment: 'poor' | 'needs_work' | 'close' | 'acceptable';

mainIssues: string[];

strengths: string[];

estimatedEffort: 'minor' | 'moderate' | 'significant';

}

interface Improvement {

id: string;

priority: 'critical' | 'high' | 'medium' | 'low';

category: ImprovementCategory;

issue: string;

suggestion: string;

example?: string;

successCriteria: string;

estimatedImpact: number; // 0-1

}

type ImprovementCategory =

| 'missing_content'

| 'incorrect_content'

| 'structural'

| 'quality'

| 'formatting'

| 'completeness'

| 'accuracy'

| 'clarity';

```

```typescript

interface QualitySignals {

validation: ValidationResult;

confidence: ConfidenceScore;

hallucination: HallucinationReport;

iteration: IterationDecision;

userFeedback?: UserFeedback;

}

function collectSignals(

taskId: string,

sources: SignalSources

): QualitySignals {

return {

validation: sources.validator.getResult(taskId),

confidence: sources.confidenceScorer.getScore(taskId),

hallucination: sources.hallucinationDetector.getReport(taskId),

iteration: sources.iterationDetector.getDecision(taskId),

userFeedback: sources.userFeedback?.get(taskId),

};

}

```

```typescript

function extractImprovements(signals: QualitySignals): Improvement[] {

const improvements: Improvement[] = [];

// From validation errors

for (const error of signals.validation.errors) {

improvements.push({

id: val-${error.code},

priority: error.severity === 'critical' ? 'critical' : 'high',

category: categorizeValidationError(error),

issue: error.message,

suggestion: generateValidationFix(error),

example: generateValidationExample(error),

successCriteria: Validation passes for ${error.path},

estimatedImpact: error.severity === 'critical' ? 0.9 : 0.6,

});

}

// From confidence breakdown

const weakFactors = Object.entries(signals.confidence.factors)

.filter(([_, score]) => score < 0.6)

.sort((a, b) => a[1] - b[1]);

for (const [factor, score] of weakFactors) {

improvements.push({

id: conf-${factor},

priority: score < 0.4 ? 'high' : 'medium',

category: mapConfidenceToCategory(factor),

issue: Low ${factor} score: ${(score * 100).toFixed(0)}%,

suggestion: getConfidenceImprovement(factor as keyof ConfidenceFactors),

successCriteria: ${factor} score above 70%,

estimatedImpact: 0.5,

});

}

// From hallucination findings

for (const finding of signals.hallucination.findings) {

if (finding.severity !== 'warning') {

improvements.push({

id: hall-${finding.type},

priority: finding.severity === 'confirmed' ? 'critical' : 'high',

category: 'accuracy',

issue: ${finding.type}: "${finding.claim}",

suggestion: Remove or verify: ${finding.suggestedAction},

successCriteria: 'No hallucinations detected in this area',

estimatedImpact: 0.8,

});

}

}

// From iteration triggers

for (const trigger of signals.iteration.triggers) {

if (!isDuplicateImprovement(improvements, trigger)) {

improvements.push({

id: iter-${trigger.type},

priority: trigger.severity > 0.8 ? 'high' : 'medium',

category: mapTriggerToCategory(trigger.type),

issue: trigger.details,

suggestion: generateTriggerFix(trigger),

successCriteria: ${trigger.type} trigger resolved,

estimatedImpact: trigger.severity,

});

}

}

// From user feedback

if (signals.userFeedback) {

improvements.push({

id: 'user-feedback',

priority: 'high',

category: 'quality',

issue: signals.userFeedback.message,

suggestion: parseUserFeedbackToAction(signals.userFeedback),

successCriteria: 'User feedback addressed',

estimatedImpact: 0.9,

});

}

return improvements;

}

function getConfidenceImprovement(factor: keyof ConfidenceFactors): string {

const suggestions: Record = {

reasoning: 'Add step-by-step reasoning, explain the logic, consider alternatives',

sources: 'Add citations, reference documentation, link to trusted sources',

consistency: 'Check for contradictions, use consistent terminology throughout',

completeness: 'Cover all required topics, add conclusion, meet word count',

uncertainty: 'Add confidence qualifiers, acknowledge limitations, note edge cases',

};

return suggestions[factor];

}

```

```typescript

function prioritizeImprovements(

improvements: Improvement[],

budget: IterationBudget

): Improvement[] {

// Score each improvement

const scored = improvements.map(imp => ({

...imp,

priorityScore: calculatePriorityScore(imp),

}));

// Sort by priority score

scored.sort((a, b) => b.priorityScore - a.priorityScore);

// Apply budget constraints

const budgeted = applyBudgetConstraints(scored, budget);

// Ensure dependencies are respected

return orderByDependencies(budgeted);

}

function calculatePriorityScore(improvement: Improvement): number {

const priorityWeights: Record = {

critical: 1.0,

high: 0.75,

medium: 0.5,

low: 0.25,

};

const categoryWeights: Record = {

incorrect_content: 0.95, // Wrong is worse than missing

missing_content: 0.9,

accuracy: 0.85,

structural: 0.7,

completeness: 0.65,

quality: 0.5,

clarity: 0.4,

formatting: 0.3,

};

return (

priorityWeights[improvement.priority] * 0.4 +

categoryWeights[improvement.category] * 0.3 +

improvement.estimatedImpact * 0.3

);

}

function applyBudgetConstraints(

improvements: Array,

budget: IterationBudget

): Improvement[] {

// If budget is low, focus on critical only

if (budget.remainingIterations <= 1) {

return improvements.filter(i => i.priority === 'critical');

}

// If budget is moderate, include high priority

if (budget.remainingIterations <= 2) {

return improvements.filter(i =>

i.priority === 'critical' || i.priority === 'high'

);

}

// Otherwise, include based on estimated effort

let tokenBudget = budget.remainingTokens * 0.5; // Reserve half for execution

const selected: Improvement[] = [];

for (const imp of improvements) {

const estimatedTokens = estimateImprovementTokens(imp);

if (tokenBudget >= estimatedTokens) {

selected.push(imp);

tokenBudget -= estimatedTokens;

}

}

return selected;

}

```

```typescript

interface FeedbackContext {

preserveElements: string[]; // What worked well

avoidElements: string[]; // What failed

previousAttempts: AttemptSummary[];

relevantExamples: string[];

}

function buildFeedbackContext(

output: TaskOutput,

signals: QualitySignals,

history: IterationHistory

): FeedbackContext {

return {

preserveElements: identifyStrengths(output, signals),

avoidElements: identifyFailures(output, signals),

previousAttempts: summarizeHistory(history),

relevantExamples: findRelevantExamples(signals),

};

}

function identifyStrengths(

output: TaskOutput,

signals: QualitySignals

): string[] {

const strengths: string[] = [];

// High-scoring confidence factors

for (const [factor, score] of Object.entries(signals.confidence.factors)) {

if (score >= 0.8) {

strengths.push(Strong ${factor} (${(score * 100).toFixed(0)}%));

}

}

// Passed validations

if (signals.validation.valid) {

strengths.push('Schema validation passed');

}

// Specific positive aspects

if (signals.hallucination.overallRisk === 'low') {

strengths.push('Content appears factually grounded');

}

return strengths;

}

function identifyFailures(

output: TaskOutput,

signals: QualitySignals

): string[] {

const failures: string[] = [];

// Validation failures

for (const error of signals.validation.errors) {

failures.push(Failed: ${error.path} - ${error.code});

}

// Hallucinations

for (const finding of signals.hallucination.findings) {

if (finding.severity === 'confirmed') {

failures.push(Hallucination: ${finding.claim});

}

}

return failures;

}

function summarizeHistory(history: IterationHistory): AttemptSummary[] {

return history.iterations.map(iter => ({

iteration: iter.number,

approach: iter.strategyUsed,

outcome: iter.succeeded ? 'improved' : 'no_improvement',

qualityScore: iter.qualityScore,

keyChanges: iter.changesApplied,

}));

}

```

```typescript

interface ExecutionGuidance {

systemPromptAdditions: string[];

focusAreas: string[];

avoidPatterns: string[];

exampleOutputs?: string[];

successMetrics: SuccessMetric[];

}

function generateExecutionGuidance(

improvements: Improvement[],

context: FeedbackContext

): ExecutionGuidance {

return {

systemPromptAdditions: generatePromptAdditions(improvements),

focusAreas: extractFocusAreas(improvements),

avoidPatterns: [...context.avoidElements, ...extractAntiPatterns(improvements)],

exampleOutputs: context.relevantExamples,

successMetrics: improvements.map(i => ({

metric: i.successCriteria,

weight: i.estimatedImpact,

})),

};

}

function generatePromptAdditions(improvements: Improvement[]): string[] {

const additions: string[] = [];

// Group by category

const byCategory = groupBy(improvements, 'category');

for (const [category, items] of Object.entries(byCategory)) {

const categoryGuidance = generateCategoryGuidance(category, items);

additions.push(categoryGuidance);

}

return additions;

}

function generateCategoryGuidance(

category: ImprovementCategory,

improvements: Improvement[]

): string {

const templates: Record string> = {

missing_content: (items) =>

MUST INCLUDE: ${items.map(i => i.suggestion).join(', ')},

incorrect_content: (items) =>

FIX THESE ERRORS: ${items.map(i => ${i.issue} → ${i.suggestion}).join('; ')},

structural: (items) =>

STRUCTURE REQUIREMENTS: ${items.map(i => i.suggestion).join(', ')},

quality: (items) =>

QUALITY IMPROVEMENTS: ${items.map(i => i.suggestion).join(', ')},

formatting: (items) =>

FORMATTING: ${items.map(i => i.suggestion).join(', ')},

completeness: (items) =>

COMPLETE THESE: ${items.map(i => i.suggestion).join(', ')},

accuracy: (items) =>

VERIFY ACCURACY: ${items.map(i => i.suggestion).join(', ')},

clarity: (items) =>

CLARIFY: ${items.map(i => i.suggestion).join(', ')},

};

return templates[category](improvements);

}

```

```yaml

feedbackReport:

taskId: code-review-task

iterationNumber: 2

synthesizedAt: "2024-01-15T10:30:00Z"

summary:

overallAssessment: needs_work

mainIssues:

- "Missing security analysis section"

- "Low source citation score"

- "Incomplete performance coverage"

strengths:

- "Good reasoning structure"

- "Consistent terminology"

estimatedEffort: moderate

improvements:

- id: val-REQUIRED_FIELD_MISSING

priority: critical

category: missing_content

issue: "Required field 'security' is missing"

suggestion: "Add a security analysis section covering authentication, authorization, and data validation"

example: |

## Security Analysis

- Authentication: JWT-based, properly validated

- Authorization: Role-based access control

- Data Validation: Input sanitization on all endpoints

successCriteria: "Validation passes for $.analysis.security"

estimatedImpact: 0.9

- id: conf-sources

priority: high

category: accuracy

issue: "Low sources score: 45%"

suggestion: "Add citations, reference documentation, link to trusted sources"

successCriteria: "Sources score above 70%"

estimatedImpact: 0.5

- id: iter-requirement_unmet

priority: high

category: completeness

issue: "Requirement not met: Must include performance analysis"

suggestion: "Add performance metrics including time complexity and space complexity"

successCriteria: "Performance analysis requirement satisfied"

estimatedImpact: 0.6

context:

preserveElements:

- "Strong reasoning (78%)"

- "Good consistency (85%)"

avoidElements:

- "Generic security advice without specifics"

- "Performance claims without metrics"

previousAttempts:

- iteration: 1

approach: retry

outcome: no_improvement

qualityScore: 0.58

guidance:

systemPromptAdditions:

- "MUST INCLUDE: security analysis section, performance metrics"

- "VERIFY ACCURACY: All claims should have supporting evidence"

focusAreas:

- "Security analysis with specific findings"

- "Performance metrics with complexity analysis"

- "Citation of sources for all claims"

avoidPatterns:

- "Generic security advice without specifics"

- "Unsupported performance claims"

successMetrics:

- metric: "Validation passes for $.analysis.security"

weight: 0.9

- metric: "Sources score above 70%"

weight: 0.5

- metric: "Performance analysis requirement satisfied"

weight: 0.6

```

• Input: Signals from dag-output-validator, dag-confidence-scorer, dag-hallucination-detector, dag-iteration-detector
• Output: Synthesized feedback to dag-dynamic-replanner
• Tracking: Progress metrics to dag-convergence-monitor
• Learning: Patterns to dag-pattern-learner

1. Be Specific: Vague feedback doesn't help
2. Prioritize Ruthlessly: Focus on high-impact fixes
3. Preserve Success: Don't break what's working
4. Learn from History: Avoid repeating failed approaches
5. Set Clear Criteria: Define what success looks like

---

Actionable feedback. Prioritized improvements. Clear path forward.