Skip to content

Commit d43801e

Browse files
author
Tom Brandenburg
committed
fix: template validation critical fixes
- Fix CLI stdout pollution by moving validation warnings to stderr - Fix double-escaping in answer-node.ts with smart escaping logic - Update Makefile validation to properly detect shell script errors - Add templates-syntax-strict target for zero-tolerance validation - All 21/21 templates now have valid shell syntax (was 0/21) - All 40/40 templates pass comprehensive validation - Maintain 419/419 unit tests passing Resolves template system critical validation issues and establishes professional quality standards for generated shell scripts.
1 parent fc4f18f commit d43801e

6 files changed

Lines changed: 757 additions & 5 deletions

File tree

Makefile

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,12 @@ templates-all: build
123123
success=$$((success + 1)); \
124124
else \
125125
# Check if this is expected behavior due to missing API keys, env vars, or template variables \
126-
if grep -q "✅.*succeeded\|✅.*completed\|Workflow completed successfully\|unbound variable\|not set\|Missing.*key\|requires.*variable\|Failed to resolve template content\|Telegram chat_id is required\|invalid variable name" "$$result_file"; then \
126+
# NOTE: We no longer accept "invalid variable name" or shell script errors as "expected behavior" \
127+
if grep -q "✅.*succeeded\|✅.*completed\|Workflow completed successfully" "$$result_file"; then \
128+
echo " ✅ Executed successfully"; \
129+
success=$$((success + 1)); \
130+
elif grep -q "unbound variable\|not set\|Missing.*key\|requires.*variable\|Failed to resolve template content\|Telegram chat_id is required" "$$result_file" && \
131+
! grep -q "bash:.*invalid variable name\|syntax error\|command not found.*get_var" "$$result_file"; then \
127132
echo " ✅ Expected behavior - template works (requires environment/template variables)"; \
128133
success=$$((success + 1)); \
129134
else \
@@ -201,6 +206,74 @@ templates-syntax: build
201206
exit 1; \
202207
fi
203208

209+
# Enhanced shell syntax validation with zero tolerance for shell script errors
210+
templates-syntax-strict: build
211+
@echo "🔒 STRICT Shell Syntax Validation - Zero Tolerance for Shell Script Errors"
212+
@echo "=========================================================================="
213+
@success=0; total=0; actual_failures=0; \
214+
mkdir -p dev/test-outputs dev/execution-results/syntax-strict; \
215+
for template in templates/enhanced/*.yaml templates/advanced/*/*.yaml; do \
216+
if [ -f "$$template" ]; then \
217+
total=$$((total + 1)); \
218+
template_name=$$(basename "$$template"); \
219+
echo "🧪 Testing: $$template_name"; \
220+
script_file="dev/test-outputs/syntax-strict-$$template_name.sh"; \
221+
result_file="dev/execution-results/syntax-strict/$$template_name.log"; \
222+
if node dist/cli/index.js compile "$$template" > "$$script_file" 2>/dev/null; then \
223+
if bash -n "$$script_file" 2>/dev/null; then \
224+
echo " ✅ Shell syntax valid"; \
225+
chmod +x "$$script_file"; \
226+
if timeout 30 "$$script_file" > "$$result_file" 2>&1; then \
227+
echo " ✅ Execution completed"; \
228+
success=$$((success + 1)); \
229+
else \
230+
if grep -q "bash:.*invalid variable name\|bash:.*unbound variable\|syntax error\|command not found.*get_var" "$$result_file"; then \
231+
echo " ❌ SHELL SCRIPT ERRORS DETECTED"; \
232+
echo " Errors:"; \
233+
grep "bash:.*invalid variable name\|bash:.*unbound variable\|syntax error" "$$result_file" | head -3 | sed 's/^/ /'; \
234+
actual_failures=$$((actual_failures + 1)); \
235+
elif grep -q "Workflow completed successfully\|✅.*succeeded\|✅.*completed" "$$result_file"; then \
236+
echo " ✅ Execution successful"; \
237+
success=$$((success + 1)); \
238+
elif grep -q "OPENAI_API_KEY\|Telegram chat_id\|Missing.*API.*key\|requires.*variable" "$$result_file"; then \
239+
echo " ⚠️ Acceptable failure (missing env vars)"; \
240+
success=$$((success + 1)); \
241+
else \
242+
echo " ❌ Other execution failure - see $$result_file"; \
243+
tail -3 "$$result_file" | sed 's/^/ /'; \
244+
fi; \
245+
fi; \
246+
else \
247+
echo " ❌ INVALID SHELL SYNTAX"; \
248+
bash -n "$$script_file" 2>&1 | head -3 | sed 's/^/ /'; \
249+
actual_failures=$$((actual_failures + 1)); \
250+
fi; \
251+
else \
252+
echo " ❌ COMPILATION FAILED"; \
253+
actual_failures=$$((actual_failures + 1)); \
254+
fi; \
255+
fi; \
256+
done; \
257+
echo ""; \
258+
echo "📊 STRICT VALIDATION RESULTS"; \
259+
echo "============================="; \
260+
echo "Total Templates: $$total"; \
261+
echo "Syntax & Logic Valid: $$success"; \
262+
echo "Actual Failures: $$actual_failures"; \
263+
if [ $$actual_failures -eq 0 ]; then \
264+
echo ""; \
265+
echo "🎉 PERFECT! All templates pass strict validation!"; \
266+
echo " ✅ Zero shell script errors"; \
267+
echo " ✅ All syntax valid"; \
268+
echo " ✅ Professional quality achieved"; \
269+
else \
270+
echo ""; \
271+
echo "❌ STRICT VALIDATION FAILED"; \
272+
echo " Templates with actual shell script errors: $$actual_failures"; \
273+
echo " This blocks deployment until fixed."; \
274+
exit 1; \
275+
fi
276+
204277
# Comprehensive quality gates (all validations) - Critical PRP requirement
205278
templates-quality-gates: templates-validate templates-syntax templates-all
206279
@echo "🎉 All template quality gates passed!"

0 commit comments

Comments
 (0)