Skip to content

Commit e3d11c4

Browse files
author
Tom Brandenburg
committed
feat: add parallel step type for concurrent workflow execution (#25)
Workflows previously executed steps strictly sequentially. This adds a declarative 'parallel' step type that runs child steps concurrently via a fork-join bash pattern. Changes: - Add ParallelStep model to models.py with min_length=1 validation - Add model_rebuild() call after Step union definition - Add ParallelStep branch in render_step() using fork-join bash pattern - Add ParallelStep title to default_step_title() - Import ParallelStep in render.py - Add 8 tests: parse, validation rejection, render assertions, execution, failure propagation, and sequential coexistence Fixes #25
1 parent a71eabb commit e3d11c4

3 files changed

Lines changed: 313 additions & 2 deletions

File tree

src/flowsh_cli/models.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,10 +168,19 @@ def validate_var_name(cls, value: str) -> str:
168168
return value
169169

170170

171-
Step = Annotated[VarsStep | BashStep | AgentStep | ForStep, Field(discriminator="type")]
171+
class ParallelStep(BaseStep):
172+
type: Literal["parallel"]
173+
steps: list[Step] = Field(min_length=1)
174+
175+
176+
Step = Annotated[
177+
VarsStep | BashStep | AgentStep | ForStep | ParallelStep,
178+
Field(discriminator="type"),
179+
]
172180

173181

174182
ForStep.model_rebuild()
183+
ParallelStep.model_rebuild()
175184

176185

177186
class WorkflowParam(StrictModel):

src/flowsh_cli/render.py

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,16 @@
33
import re
44
from pathlib import Path
55

6-
from flowsh_cli.models import AgentStep, BashStep, ForStep, Step, VarsStep, Workflow, WorkflowParam
6+
from flowsh_cli.models import (
7+
AgentStep,
8+
BashStep,
9+
ForStep,
10+
ParallelStep,
11+
Step,
12+
VarsStep,
13+
Workflow,
14+
WorkflowParam,
15+
)
716

817

918
def harness_path(workflow: Workflow) -> Path:
@@ -242,6 +251,25 @@ def render_step(index: int, step: Step, used_function_names: set[str] | None = N
242251
*[f" run_step {fn}" for fn in inner_fns],
243252
f' done <<< "${{{step.in_}}}"',
244253
]
254+
elif isinstance(step, ParallelStep):
255+
child_fns: list[str] = []
256+
for i, child in enumerate(step.steps, start=1):
257+
child_title = child.name or default_step_title(i, child)
258+
child_fn = step_function_name(i, child.name, used_function_names)
259+
child_fns.append(child_fn)
260+
prefix_lines.append(section(f"Parallel child {i} ({child.type}): {child_title}"))
261+
prefix_lines.append(f"{child_fn}() {{")
262+
prefix_lines.extend(_render_step_body(child))
263+
prefix_lines.append("}")
264+
prefix_lines.append("")
265+
266+
body_lines = [" local status=0"]
267+
for child_fn in child_fns:
268+
body_lines.append(f' "{child_fn}" &')
269+
body_lines.append(f" local pid_{child_fn}=$!")
270+
for child_fn in child_fns:
271+
body_lines.append(f' wait "$pid_{child_fn}" || status=$?')
272+
body_lines.append(' return "$status"')
245273
else:
246274
body_lines = _render_step_body(step)
247275

@@ -351,6 +379,8 @@ def default_step_title(index: int, step: Step) -> str:
351379
return truncate_one_line(step.prompt)
352380
if isinstance(step, ForStep):
353381
return f"for {step.item} in {step.in_}"
382+
if isinstance(step, ParallelStep):
383+
return f"parallel ({len(step.steps)} steps)"
354384
return f"step {index}"
355385

356386

tests/test_workflow_to_harness.py

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
MAX_WORKFLOW_YAML_BYTES,
1313
AgentStep,
1414
BashStep,
15+
ParallelStep,
16+
VarsStep,
1517
Workflow,
1618
WorkflowFile,
1719
WorkflowParam,
@@ -2343,3 +2345,273 @@ def test_generated_harness_for_step_dry_run_skips_inner_steps(tmp_path: Path) ->
23432345
assert executed.returncode == 0, executed.stderr
23442346
assert "should-not-appear" not in executed.stdout
23452347
assert "[DRY-RUN]" in executed.stderr
2348+
2349+
2350+
# ---------------------------------------------------------------------------
2351+
# ParallelStep tests (issue #25)
2352+
# ---------------------------------------------------------------------------
2353+
2354+
2355+
def test_parse_workflows_accepts_parallel_step(tmp_path: Path) -> None:
2356+
workflow_file = tmp_path / "workflows.yml"
2357+
workflow_file.write_text(
2358+
"""\
2359+
workflows:
2360+
- id: wf_parallel
2361+
name: Parallel Workflow
2362+
steps:
2363+
- type: parallel
2364+
name: Fan out
2365+
steps:
2366+
- type: bash
2367+
name: Build
2368+
run: printf 'build\\n'
2369+
- type: bash
2370+
name: Test
2371+
run: printf 'test\\n'
2372+
""",
2373+
encoding="utf-8",
2374+
)
2375+
2376+
workflows = parse_workflows(workflow_file)
2377+
2378+
assert len(workflows) == 1
2379+
assert len(workflows[0].steps) == 1
2380+
step = workflows[0].steps[0]
2381+
assert isinstance(step, ParallelStep)
2382+
assert step.name == "Fan out"
2383+
assert len(step.steps) == 2
2384+
assert isinstance(step.steps[0], BashStep)
2385+
assert isinstance(step.steps[1], BashStep)
2386+
2387+
2388+
def test_parse_workflows_rejects_empty_parallel_steps(tmp_path: Path) -> None:
2389+
workflow_file = tmp_path / "workflows.yml"
2390+
workflow_file.write_text(
2391+
"""\
2392+
workflows:
2393+
- id: wf_empty_parallel
2394+
name: Empty Parallel
2395+
steps:
2396+
- type: parallel
2397+
name: Empty
2398+
steps: []
2399+
""",
2400+
encoding="utf-8",
2401+
)
2402+
2403+
with pytest.raises(WorkflowParseError):
2404+
parse_workflows(workflow_file)
2405+
2406+
2407+
def test_parse_workflows_accepts_parallel_step_with_vars_and_agent(tmp_path: Path) -> None:
2408+
workflow_file = tmp_path / "workflows.yml"
2409+
workflow_file.write_text(
2410+
"""\
2411+
workflows:
2412+
- id: wf_parallel_mixed
2413+
name: Parallel Mixed
2414+
steps:
2415+
- type: parallel
2416+
name: Mixed
2417+
steps:
2418+
- type: vars
2419+
name: Capture value
2420+
values:
2421+
VALUE: printf 'hello'
2422+
- type: agent
2423+
name: Ask
2424+
prompt: Say hello.
2425+
""",
2426+
encoding="utf-8",
2427+
)
2428+
2429+
workflows = parse_workflows(workflow_file)
2430+
step = workflows[0].steps[0]
2431+
assert isinstance(step, ParallelStep)
2432+
assert isinstance(step.steps[0], VarsStep)
2433+
assert isinstance(step.steps[1], AgentStep)
2434+
2435+
2436+
def test_render_harness_parallel_step_generates_fork_join_bash() -> None:
2437+
workflow = Workflow(
2438+
id="wf_parallel_render",
2439+
name="Parallel Render",
2440+
steps=[
2441+
ParallelStep(
2442+
type="parallel",
2443+
name="Fan out",
2444+
steps=[
2445+
BashStep(type="bash", name="Build", run="printf 'build\\n'"),
2446+
BashStep(type="bash", name="Test", run="printf 'test\\n'"),
2447+
],
2448+
)
2449+
],
2450+
)
2451+
2452+
script = render_harness(workflow)
2453+
2454+
# Child functions are emitted
2455+
assert "step_build()" in script
2456+
assert "step_test()" in script
2457+
# Fork-join pattern present
2458+
assert '"step_build" &' in script
2459+
assert '"step_test" &' in script
2460+
assert "pid_step_build=$!" in script
2461+
assert "pid_step_test=$!" in script
2462+
assert 'wait "$pid_step_build"' in script
2463+
assert 'wait "$pid_step_test"' in script
2464+
# Wrapper function emitted
2465+
assert "step_fan_out()" in script
2466+
# Wrapper is called via run_step
2467+
assert "run_step step_fan_out" in script
2468+
# Children are NOT directly called via run_step at top level
2469+
assert "run_step step_build" not in script
2470+
assert "run_step step_test" not in script
2471+
2472+
2473+
def test_render_harness_parallel_step_section_comment() -> None:
2474+
workflow = Workflow(
2475+
id="wf_par_comment",
2476+
name="Par Comment",
2477+
steps=[
2478+
ParallelStep(
2479+
type="parallel",
2480+
steps=[
2481+
BashStep(type="bash", run="echo a"),
2482+
BashStep(type="bash", run="echo b"),
2483+
],
2484+
)
2485+
],
2486+
)
2487+
2488+
script = render_harness(workflow)
2489+
2490+
assert "Parallel child 1" in script
2491+
assert "Parallel child 2" in script
2492+
assert "parallel (2 steps)" in script
2493+
2494+
2495+
def test_generated_harness_runs_parallel_steps(tmp_path: Path) -> None:
2496+
workflow_file = tmp_path / "workflows.yml"
2497+
workflow_file.write_text(
2498+
"""\
2499+
workflows:
2500+
- id: wf_parallel_exec
2501+
name: Parallel Exec
2502+
steps:
2503+
- type: parallel
2504+
name: Fan out
2505+
steps:
2506+
- type: bash
2507+
name: Write A
2508+
run: printf 'output_a\\n'
2509+
- type: bash
2510+
name: Write B
2511+
run: printf 'output_b\\n'
2512+
""",
2513+
encoding="utf-8",
2514+
)
2515+
2516+
generated = subprocess.run(
2517+
[sys.executable, "-m", "flowsh_cli", str(workflow_file)],
2518+
check=False,
2519+
capture_output=True,
2520+
text=True,
2521+
cwd=tmp_path,
2522+
)
2523+
assert generated.returncode == 0, generated.stderr
2524+
2525+
harness = tmp_path / ".harness" / "parallel_exec.sh"
2526+
assert harness.exists()
2527+
2528+
syntax = subprocess.run(
2529+
["bash", "-n", str(harness)], check=False, capture_output=True, text=True
2530+
)
2531+
assert syntax.returncode == 0, syntax.stderr
2532+
2533+
executed = subprocess.run(
2534+
["bash", str(harness)],
2535+
check=False,
2536+
capture_output=True,
2537+
text=True,
2538+
cwd=tmp_path,
2539+
env={**os.environ, "FLOWSH_LOG_DIR": "logs"},
2540+
)
2541+
assert executed.returncode == 0, executed.stderr
2542+
assert "output_a" in executed.stdout
2543+
assert "output_b" in executed.stdout
2544+
2545+
2546+
def test_generated_harness_parallel_step_propagates_child_failure(tmp_path: Path) -> None:
2547+
workflow_file = tmp_path / "workflows.yml"
2548+
workflow_file.write_text(
2549+
"""\
2550+
workflows:
2551+
- id: wf_parallel_fail
2552+
name: Parallel Fail
2553+
steps:
2554+
- type: parallel
2555+
name: Fan out
2556+
steps:
2557+
- type: bash
2558+
name: Success
2559+
run: printf 'ok\\n'
2560+
- type: bash
2561+
name: Failure
2562+
run: "false"
2563+
""",
2564+
encoding="utf-8",
2565+
)
2566+
2567+
generated = subprocess.run(
2568+
[sys.executable, "-m", "flowsh_cli", str(workflow_file)],
2569+
check=False,
2570+
capture_output=True,
2571+
text=True,
2572+
cwd=tmp_path,
2573+
)
2574+
assert generated.returncode == 0, generated.stderr
2575+
2576+
harness = tmp_path / ".harness" / "parallel_fail.sh"
2577+
2578+
executed = subprocess.run(
2579+
["bash", str(harness)],
2580+
check=False,
2581+
capture_output=True,
2582+
text=True,
2583+
cwd=tmp_path,
2584+
env={**os.environ, "FLOWSH_LOG_DIR": "logs"},
2585+
)
2586+
assert executed.returncode != 0
2587+
assert "Step failed: step_fan_out" in executed.stderr
2588+
2589+
2590+
def test_render_harness_parallel_coexists_with_sequential_steps() -> None:
2591+
workflow = Workflow(
2592+
id="wf_mixed_seq_par",
2593+
name="Mixed Seq Par",
2594+
steps=[
2595+
BashStep(type="bash", name="Setup", run="echo setup"),
2596+
ParallelStep(
2597+
type="parallel",
2598+
steps=[
2599+
BashStep(type="bash", name="Build", run="echo build"),
2600+
BashStep(type="bash", name="Test", run="echo test"),
2601+
],
2602+
),
2603+
BashStep(type="bash", name="Teardown", run="echo done"),
2604+
],
2605+
)
2606+
2607+
script = render_harness(workflow)
2608+
2609+
# Sequential steps called at top level
2610+
assert "run_step step_setup" in script
2611+
assert "run_step step_teardown" in script
2612+
# Children backgrounded inside wrapper, not at top level
2613+
assert '"step_build" &' in script
2614+
assert '"step_test" &' in script
2615+
# Children not individually called at top level
2616+
assert "run_step step_build" not in script
2617+
assert "run_step step_test" not in script

0 commit comments

Comments
 (0)