@@ -98,66 +98,7 @@ templates-all: build
9898 @echo " Generating and executing shell scripts from all production templates..."
9999 @mkdir -p dev/generated-outputs/templates/
100100 @mkdir -p dev/execution-results/templates/
101- @success=0; total=0; \
102- for template in templates/basic/* -template.yaml templates/enhanced/* -simple.yaml templates/enhanced/* -template.yaml templates/advanced/* /* .yaml; do \
103- if [ -f " $$ template" ]; then \
104- total=$$((total + 1 ) ); \
105- echo " Processing: $$ template" ; \
106- basename=$$(basename "$$template" .yaml) ; \
107- script_file=" dev/generated-outputs/templates/$$ basename.sh" ; \
108- result_file=" dev/execution-results/templates/$$ basename.result" ; \
109- if node dist/cli/index.js compile " $$ template" > " $$ script_file" 2> /dev/null; then \
110- echo " ✅ Generated: $$ script_file" ; \
111- if bash -n " $$ script_file" 2> /dev/null; then \
112- echo " ✅ Shell syntax valid" ; \
113- else \
114- echo " ❌ Invalid shell syntax in $$ script_file" ; \
115- bash -n " $$ script_file" 2>&1 | head -3 | sed ' s/^/ /' ; \
116- continue ; \
117- fi ; \
118- chmod +x " $$ script_file" ; \
119- echo " 🚀 Executing: $$ basename (production template - may require env vars)" ; \
120- # Note: Templates may require API keys and environment variables for execution \
121- if timeout 120 " $$ script_file" > " $$ result_file" 2>&1 ; then \
122- echo " ✅ Executed successfully" ; \
123- success=$$((success + 1 ) ); \
124- else \
125- # Check if this is expected behavior due to missing API keys, env vars, or template variables \
126- # NOTE: We no longer accept "invalid variable name" or shell script errors as "expected behavior" \
127- if grep -q " ✅.*succeeded\|✅.*completed\|Workflow completed successfully" " $$ result_file" ; then \
128- echo " ✅ Executed successfully" ; \
129- success=$$((success + 1 ) ); \
130- # STRICT ERROR CLASSIFICATION - Zero tolerance for critical failures
131- elif grep -q " bash:.*unbound variable\|syntax error\|command not found.*get_var\|Failed to resolve template content" " $$ result_file" ; then \
132- echo " ❌ CRITICAL FAILURE - Shell script error detected" ; \
133- echo " Critical errors:" ; \
134- grep " bash:.*unbound variable\|syntax error\|command not found.*get_var\|Failed to resolve template content" " $$ result_file" | head -3 | sed ' s/^/ /' ; \
135- elif grep -q " Missing.*API.*key\|OPENAI_API_KEY.*required\|Telegram chat_id is required\|requires.*API.*key" " $$ result_file" && \
136- ! grep -q " bash:.*unbound variable\|syntax error\|Failed to resolve template content" " $$ result_file" ; then \
137- echo " ⚠️ ACCEPTABLE - Missing environment variables (template functional)" ; \
138- success=$$((success + 1 ) ); \
139- elif grep -q " Mock circuit breaker.*operation failed\|Circuit breaker operation failed\|Mock circuit breaker: operation failed" " $$ result_file" && \
140- ! grep -q " bash:.*unbound variable\|syntax error\|command not found.*get_var\|Failed to resolve template content" " $$ result_file" ; then \
141- echo " ✅ Expected behavior - circuit breaker demonstrating failure handling" ; \
142- success=$$((success + 1 ) ); \
143- else \
144- echo " ❌ Execution failed - see $$ result_file" ; \
145- tail -3 " $$ result_file" | sed ' s/^/ /' ; \
146- fi ; \
147- fi ; \
148- else \
149- echo " ❌ Failed to compile $$ template" ; \
150- fi ; \
151- fi ; \
152- done ; \
153- echo " " ; \
154- echo " 📊 Results: $$ success/$$ total templates executed successfully" ; \
155- if [ $$ success -eq $$ total ]; then \
156- echo " 🎉 All templates passed!" ; \
157- else \
158- echo " ⚠️ Some templates failed - check dev/execution-results/templates/ for details" ; \
159- exit 1; \
160- fi
101+ @./dev/template-validation/run-all-templates.sh
161102
162103# Validate all production templates
163104templates-validate : build
0 commit comments