Skip to content

Multi-Node Pipeline with DAG and Streaming¤

This guide shows how to build a directed acyclic graph (DAG) of operations where each node consumes typed outputs from upstream nodes and produces a typed artifact of its own. The pipeline is executed with execute_stream(), which yields RuntimeStreamChunk events via Server-Sent Events (SSE) — ideal for real-time dashboards and progress indicators.

Concepts covered

What you'll build¤

A brand identity pipeline with three nodes connected in a DAG:

graph LR A[BrandSeed] --> B[TypographyDesigner] B --> C[Colorist] B --> D[Designer] C --> D A --> C A --> D B -->|TypographyDesign| C B -->|TypographyDesign| D C -->|ColorPalette| D D -->|VisualIdentity| OUT
  • TypographyDesigner selects fonts from a brand seed
  • Colorist creates a palette harmonizing with the typography
  • Designer synthesizes both into a final visual identity

Each node uses its own agent, tool scope, and task blocks. The pipeline streams progress via SSE as each node completes.


1 Define domain models¤

Start with Pydantic models for every artifact that flows through the DAG:

from pydantic import BaseModel, ConfigDict, Field


class BrandSeed(BaseModel):
    """The seed brand identity provided by the user."""
    model_config = ConfigDict(frozen=True)

    name: str = Field(description="Brand name")
    industry: str = Field(description="Industry sector")
    values: list[str] = Field(description="Core brand values")


class TypographyDesign(BaseModel):
    """Typography design output from the TypographyDesigner agent."""
    model_config = ConfigDict(frozen=True)

    primary_font: str = Field(description="Primary typeface name")
    secondary_font: str = Field(description="Secondary typeface name")
    font_weights: list[str] = Field(description="Font weights used")
    rationale: str = Field(description="Design rationale")


class ColorPalette(BaseModel):
    """Color palette output from the Colorist agent."""
    model_config = ConfigDict(frozen=True)

    primary_color: str = Field(description="Primary brand color (hex)")
    secondary_color: str = Field(description="Secondary brand color (hex)")
    accent_color: str = Field(description="Accent color (hex)")
    background_color: str = Field(description="Background color (hex)")
    rationale: str = Field(description="Color selection rationale")


class VisualIdentity(BaseModel):
    """Final visual identity output from the Designer agent."""
    model_config = ConfigDict(frozen=True)

    brand_name: str = Field(description="Brand name")
    typography: TypographyDesign = Field(description="Typography choices")
    colors: ColorPalette = Field(description="Color palette")
    identity_summary: str = Field(description="Synthesis of visual identity")

2 Create a context projector¤

A ContextProjector converts domain objects into template-friendly dicts. Register its factory (from_domain) in the ContextStore so the runtime can project BrandSeed data into every node's prompt:

from typing import Any

from crewmaster.agents.context.projector import ContextProjector


class BrandSeedProjector(ContextProjector):
    """Projects BrandSeed into template context."""

    def __init__(self, brand_name: str, brand_industry: str,
                 brand_values: list[str]) -> None:
        self._brand_name = brand_name
        self._brand_industry = brand_industry
        self._brand_values = brand_values

    @classmethod
    def from_domain(cls, data: BrandSeed) -> "BrandSeedProjector":
        return cls(
            brand_name=data.name,
            brand_industry=data.industry,
            brand_values=list(data.values),
        )

    def to_context(self) -> dict[str, Any]:
        return {
            "brand_name": self._brand_name,
            "brand_industry": self._brand_industry,
            "brand_values": ", ".join(self._brand_values),
        }

3 Define tools¤

Three tools, each scoped to a specific agent so they only see what they need:

from crewmaster.tools.models import Capability, ToolSchema
from crewmaster.tools.registry import ToolRegistry

FONT_DATABASE_TOOL = ToolSchema(
    name="font_database",
    description="Query the font database for typeface recommendations",
    input_schema={
        "type": "object",
        "properties": {
            "industry": {"type": "string"},
            "style": {"type": "string"},
        },
        "required": ["industry"],
    },
)

COLOR_HARMONY_TOOL = ToolSchema(
    name="color_harmony",
    description="Generate a harmonious color palette based on constraints",
    input_schema={
        "type": "object",
        "properties": {
            "base_hue": {"type": "string"},
            "mood": {"type": "string"},
        },
    },
)

DESIGN_VALIDATOR_TOOL = ToolSchema(
    name="design_validator",
    description="Validate visual identity for accessibility and contrast",
    input_schema={
        "type": "object",
        "properties": {
            "primary_color": {"type": "string"},
            "background_color": {"type": "string"},
            "primary_font": {"type": "string"},
        },
    },
)

tool_registry = ToolRegistry()
tool_registry.register("typography", FONT_DATABASE_TOOL)
tool_registry.register("color", COLOR_HARMONY_TOOL)
tool_registry.register("design", DESIGN_VALIDATOR_TOOL)

4 Configure agents¤

Each agent gets a dedicated identity block and tool scope:

from crewmaster.agents.agent import AgentConfig

typographer = AgentConfig(
    name="typographer",
    identity_blocks=["blocks://pipeline/identity/typographer"],
    default_tool_scope="typography",
)

colorist = AgentConfig(
    name="colorist",
    identity_blocks=["blocks://pipeline/identity/colorist"],
    default_tool_scope="color",
)

designer = AgentConfig(
    name="designer",
    identity_blocks=["blocks://pipeline/identity/designer"],
    default_tool_scope="design",
)

5 Create the prompt block templates¤

Use a LocalDiskStore to host BlockStore templates on disk. The keys {{ ctx.brand_name }}, {{ ctx.brand_industry }}, and {{ ctx.brand_values }} are resolved from the BrandSeedProjector at runtime:

import tempfile
from pathlib import Path

from crewmaster.agents.prompts.local_disk_store import LocalDiskStore

blocks_dir = Path(tempfile.mkdtemp())


def _write(path_parts, content):
    p = blocks_dir.joinpath(*path_parts)
    p.parent.mkdir(parents=True, exist_ok=True)
    p.write_text(content)


_write(("pipeline", "identity", "typographer.j2"), """\
---
id: blocks://pipeline/identity/typographer
kind: identity
provides: typographer_identity
---
You are a typography expert. Your role is to select fonts that reflect
the brand's personality.
""")

_write(("pipeline", "identity", "colorist.j2"), """\
---
id: blocks://pipeline/identity/colorist
kind: identity
provides: colorist_identity
---
You are a color expert. Create color palettes that evoke the right mood
and align with brand values.
""")

_write(("pipeline", "identity", "designer.j2"), """\
---
id: blocks://pipeline/identity/designer
kind: identity
provides: designer_identity
---
You are a visual identity designer. Synthesize typography and color into
a cohesive brand identity.
""")

_write(("pipeline", "task", "typographer.j2"), """\
---
id: blocks://pipeline/task/typographer
kind: task
provides: typography_design_output
---
Select a font pairing for brand **{{ ctx.brand_name }}** in the
**{{ ctx.brand_industry }}** sector. The brand values are:
**{{ ctx.brand_values }}**.

Output a TypographyDesign with primary_font, secondary_font,
font_weights, and rationale.
""")

_write(("pipeline", "task", "colorist.j2"), """\
---
id: blocks://pipeline/task/colorist
kind: task
provides: color_palette_output
---
Create a color palette for **{{ ctx.brand_name }}** that complements
the typography from the upstream TypographyDesigner.

Output a ColorPalette with primary_color, secondary_color, accent_color,
background_color (all hex), and rationale.
""")

_write(("pipeline", "task", "designer.j2"), """\
---
id: blocks://pipeline/task/designer
kind: task
provides: visual_identity_output
---
Synthesize the typography and color palette from upstream nodes into a
cohesive VisualIdentity for **{{ ctx.brand_name }}**.

Output a VisualIdentity with brand_name, typography (the TypographyDesign),
colors (the ColorPalette), and identity_summary.
""")

block_store = LocalDiskStore(root=str(blocks_dir))

6 Build the operation tree¤

The DAG is declared via sub_operations with typed consumes. CrewMaster infers execution order from the dependency graph — the Designer runs after both TypographyDesigner and Colorist complete:

from crewmaster.operations.operation import Operation

pipeline = Operation(
    name="brand_identity_pipeline",
    produces=VisualIdentity,
    kind="artifact",
    sub_operations=[
        Operation(
            name="TypographyDesigner",
            produces=TypographyDesign,
            kind="artifact",
            agent=typographer,
            task_blocks=["blocks://pipeline/task/typographer"],
            consumes=[BrandSeedProjector],
        ),
        Operation(
            name="Colorist",
            produces=ColorPalette,
            kind="artifact",
            agent=colorist,
            task_blocks=["blocks://pipeline/task/colorist"],
            consumes=[BrandSeedProjector, TypographyDesign],
        ),
        Operation(
            name="Designer",
            produces=VisualIdentity,
            kind="artifact",
            agent=designer,
            task_blocks=["blocks://pipeline/task/designer"],
            consumes=[BrandSeedProjector, TypographyDesign, ColorPalette],
        ),
    ],
)
How the DAG is resolved

consumes=[BrandSeedProjector, TypographyDesign] on the Colorist node tells CrewMaster that this node depends on the output of the node that produces TypographyDesign. The execution plan is automatically topologically sorted so each node runs only after its dependencies complete.


7 Execute with streaming¤

Stream the execution with execute_stream(). Each node emits text_delta chunks as it generates, then a node_complete chunk with the typed output:

import asyncio

from crewmaster import execute_stream
from crewmaster.agents.context.store import ContextStore

# Register the BrandSeed domain object
brand = BrandSeed(
    name="NovaTech",
    industry="AI & Developer Tools",
    values=["innovation", "clarity", "empowerment"],
)
store = ContextStore()
store.register(BrandSeed, brand)


async def main():
    from crewmaster.execution.drivers.pydantic_ai import PydanticAIDriver

    driver = PydanticAIDriver(model_name="openai:gpt-4o-mini")

    async for chunk in execute_stream(
        operation=pipeline,
        context_store=store,
        default_runtime=driver,
        block_store=block_store,
        tool_registry=tool_registry,
    ):
        if chunk.kind == "node_start":
            print(f"\n▶ Starting: {chunk.node_name}")
        elif chunk.kind == "text_delta":
            print(chunk.content, end="", flush=True)
        elif chunk.kind == "node_complete":
            print(f"\n✔ Completed: {chunk.node_name}")
            final = chunk.output
        elif chunk.kind == "final":
            print(f"\n🏁 Pipeline complete")
            break

    # final holds the VisualIdentity
    print(f"\nBrand: {final.brand_name}")
    print(f"Font:  {final.typography.primary_font} + {final.typography.secondary_font}")
    print(f"Colors: {final.colors.primary_color} / {final.colors.accent_color}")
    print(f"Summary: {final.identity_summary}")


asyncio.run(main())

Expected output (SSE stream):

▶ Starting: TypographyDesigner
Selecting fonts for NovaTech...
✔ Completed: TypographyDesigner

▶ Starting: Colorist
Generating color palette...
✔ Completed: Colorist

▶ Starting: Designer
Synthesizing visual identity...
✔ Completed: Designer
🏁 Pipeline complete

Brand: NovaTech
Font:  Inter + Source Serif
Colors: #2563EB / #F59E0B
Summary: NovaTech's visual identity combines...

8 Streaming over HTTP (SSE)¤

Expose the pipeline as a Server-Sent Events endpoint with FastAPI:

from fastapi import FastAPI
from fastapi.responses import StreamingResponse


app = FastAPI()


@app.get("/pipeline/stream")
async def stream_pipeline():
    async def event_generator():
        async for chunk in execute_stream(
            operation=pipeline,
            context_store=store,
            default_runtime=driver,
            block_store=block_store,
            tool_registry=tool_registry,
        ):
            yield f"data: {chunk.model_dump_json()}\n\n"

    return StreamingResponse(
        event_generator(),
        media_type="text/event-stream",
    )

The SSE stream delivers these event types:

Kind Description
node_start A sub-operation has started execution
text_delta Incremental text output from the LLM
node_complete A sub-operation finished (with typed output)
final The entire pipeline completed

9 Complete script¤

pipeline.py
"""Multi-node pipeline with DAG and SSE streaming."""
import asyncio
import tempfile
from pathlib import Path
from typing import Any

from pydantic import BaseModel, ConfigDict, Field

from crewmaster import execute_stream
from crewmaster.agents.agent import AgentConfig
from crewmaster.agents.context.projector import ContextProjector
from crewmaster.agents.context.store import ContextStore
from crewmaster.agents.prompts.local_disk_store import LocalDiskStore
from crewmaster.execution.drivers.pydantic_ai import PydanticAIDriver
from crewmaster.operations.operation import Operation
from crewmaster.tools.models import ToolSchema
from crewmaster.tools.registry import ToolRegistry


# ── Domain models ─────────────────────────────────────────────────
class BrandSeed(BaseModel):
    model_config = ConfigDict(frozen=True)
    name: str
    industry: str
    values: list[str]


class TypographyDesign(BaseModel):
    model_config = ConfigDict(frozen=True)
    primary_font: str
    secondary_font: str
    font_weights: list[str]
    rationale: str


class ColorPalette(BaseModel):
    model_config = ConfigDict(frozen=True)
    primary_color: str
    secondary_color: str
    accent_color: str
    background_color: str
    rationale: str


class VisualIdentity(BaseModel):
    model_config = ConfigDict(frozen=True)
    brand_name: str
    typography: TypographyDesign
    colors: ColorPalette
    identity_summary: str


# ── Context projector ─────────────────────────────────────────────
class BrandSeedProjector(ContextProjector):
    def __init__(self, brand_name, brand_industry, brand_values):
        self._brand_name = brand_name
        self._brand_industry = brand_industry
        self._brand_values = brand_values

    @classmethod
    def from_domain(cls, data: BrandSeed) -> "BrandSeedProjector":
        return cls(
            brand_name=data.name,
            brand_industry=data.industry,
            brand_values=list(data.values),
        )

    def to_context(self) -> dict[str, Any]:
        return {
            "brand_name": self._brand_name,
            "brand_industry": self._brand_industry,
            "brand_values": ", ".join(self._brand_values),
        }


# ── Tools ─────────────────────────────────────────────────────────
FONT_DB = ToolSchema(
    name="font_database",
    description="Query font database for typeface recommendations",
    input_schema={
        "type": "object",
        "properties": {
            "industry": {"type": "string"},
            "style": {"type": "string"},
        },
        "required": ["industry"],
    },
)

COLOR_TOOL = ToolSchema(
    name="color_harmony",
    description="Generate a harmonious color palette",
    input_schema={
        "type": "object",
        "properties": {
            "base_hue": {"type": "string"},
            "mood": {"type": "string"},
        },
    },
)

VALIDATOR = ToolSchema(
    name="design_validator",
    description="Validate visual identity for accessibility",
    input_schema={
        "type": "object",
        "properties": {
            "primary_color": {"type": "string"},
            "background_color": {"type": "string"},
            "primary_font": {"type": "string"},
        },
    },
)

tool_registry = ToolRegistry()
tool_registry.register("typography", FONT_DB)
tool_registry.register("color", COLOR_TOOL)
tool_registry.register("design", VALIDATOR)


# ── Agents ────────────────────────────────────────────────────────
typographer = AgentConfig(
    name="typographer",
    identity_blocks=["blocks://pipeline/identity/typographer"],
    default_tool_scope="typography",
)
colorist = AgentConfig(
    name="colorist",
    identity_blocks=["blocks://pipeline/identity/colorist"],
    default_tool_scope="color",
)
designer = AgentConfig(
    name="designer",
    identity_blocks=["blocks://pipeline/identity/designer"],
    default_tool_scope="design",
)


# ── Prompt blocks ─────────────────────────────────────────────────
def _write(path_parts, content):
    p = BLOCKS_DIR.joinpath(*path_parts)
    p.parent.mkdir(parents=True, exist_ok=True)
    p.write_text(content)

BLOCKS_DIR = Path(tempfile.mkdtemp())
_write(("pipeline", "identity", "typographer.j2"), """\
---
id: blocks://pipeline/identity/typographer
kind: identity
provides: typographer_identity
---
You are a typography expert.
""")
_write(("pipeline", "identity", "colorist.j2"), """\
---
id: blocks://pipeline/identity/colorist
kind: identity
provides: colorist_identity
---
You are a color expert.
""")
_write(("pipeline", "identity", "designer.j2"), """\
---
id: blocks://pipeline/identity/designer
kind: identity
provides: designer_identity
---
You are a visual identity designer.
""")
_write(("pipeline", "task", "typographer.j2"), """\
---
id: blocks://pipeline/task/typographer
kind: task
provides: typography_design_output
---
Select fonts for **{{ ctx.brand_name }}**.
Output TypographyDesign with primary_font, secondary_font,
font_weights, rationale.
""")
_write(("pipeline", "task", "colorist.j2"), """\
---
id: blocks://pipeline/task/colorist
kind: task
provides: color_palette_output
---
Create palette for **{{ ctx.brand_name }}**.
Output ColorPalette with primary_color, secondary_color,
accent_color, background_color, rationale.
""")
_write(("pipeline", "task", "designer.j2"), """\
---
id: blocks://pipeline/task/designer
kind: task
provides: visual_identity_output
---
Synthesize visual identity for **{{ ctx.brand_name }}**.
Output VisualIdentity with brand_name, typography, colors,
identity_summary.
""")

block_store = LocalDiskStore(root=str(BLOCKS_DIR))


# ── Operation tree ────────────────────────────────────────────────
pipeline = Operation(
    name="brand_identity_pipeline",
    produces=VisualIdentity,
    kind="artifact",
    sub_operations=[
        Operation(
            name="TypographyDesigner",
            produces=TypographyDesign,
            kind="artifact",
            agent=typographer,
            task_blocks=["blocks://pipeline/task/typographer"],
            consumes=[BrandSeedProjector],
        ),
        Operation(
            name="Colorist",
            produces=ColorPalette,
            kind="artifact",
            agent=colorist,
            task_blocks=["blocks://pipeline/task/colorist"],
            consumes=[BrandSeedProjector, TypographyDesign],
        ),
        Operation(
            name="Designer",
            produces=VisualIdentity,
            kind="artifact",
            agent=designer,
            task_blocks=["blocks://pipeline/task/designer"],
            consumes=[BrandSeedProjector, TypographyDesign, ColorPalette],
        ),
    ],
)


# ── Execute ───────────────────────────────────────────────────────
async def main():
    brand = BrandSeed(
        name="NovaTech",
        industry="AI & Developer Tools",
        values=["innovation", "clarity", "empowerment"],
    )
    store = ContextStore()
    store.register(BrandSeed, brand)

    driver = PydanticAIDriver(model_name="openai:gpt-4o-mini")

    async for chunk in execute_stream(
        operation=pipeline,
        context_store=store,
        default_runtime=driver,
        block_store=block_store,
        tool_registry=tool_registry,
    ):
        if chunk.kind == "node_start":
            print(f"\n{chunk.node_name}")
        elif chunk.kind == "text_delta":
            print(chunk.content, end="", flush=True)
        elif chunk.kind == "node_complete":
            print(f"\n{chunk.node_name}")
        elif chunk.kind == "final":
            final = chunk.output
            print(f"\n🏁 Pipeline complete\n")
            print(f"Brand:  {final.brand_name}")
            print(f"Font:   {final.typography.primary_font}")
            print(f"Colors: {final.colors.primary_color} / {final.colors.accent_color}")
            print(f"Summary: {final.identity_summary}")


asyncio.run(main())

Next steps¤