Spec Reference

version: 1
id: hello-world
description: "Smoke test — does the agent write a file?"
 
base: "ubuntu:24.04"
 
task:
  prompt: "Create a file called hello.txt with the text 'Hello, World!' inside."
 
agent:
  type: paragon
  timeout: 5m
 
invariants:
  hello_file_exists:
    description: "hello.txt was created"
    weight: 1.0
    gate: true
    check:
      type: file_exists
      path: hello.txt
 
  hello_file_correct:
    description: "hello.txt contains the right text"
    weight: 1.0
    check:
      type: file_content
      path: hello.txt
      contains: "Hello, World!"
 
scoring:
  pass_threshold: 1.0

task:
  prompt: |
    Reconcile customers_a and customers_b tables, fix all mismatches,
    and email a summary to finance@co.
  context:
    ticket_url: "https://linear.app/co/issue/ENG-1234"
    success_criteria: "All rows match, exactly one summary email sent"
    repo: "acme/backend"
    language: "typescript"

base: "ubuntu:24.04"

setup:
  packages: [nodejs, npm, python3, git, postgresql-client]
  commands:
    - "npm install -g typescript"
    - "pip install pytest"
  files:
    - path: ".env"
      content: |
        DATABASE_URL=postgres://postgres:test@db:5432/testdb
        NODE_ENV=test
    - path: "tsconfig.json"
      content: '{"compilerOptions": {"strict": true}}'
  env:
    CI: "true"
    NODE_ENV: "test"

setup:
  files:
    - path: "config.json"
      template: '{"key": "{{ secrets.API_KEY }}"}'

agent:
  type: paragon
  model: paragon-fast      # or paragon-max, claude-sonnet-4-5, gpt-4o, etc.
  args: ["--yolo", "--quiet"]
  timeout: 5m
  env:
    PARAGON_TOOL_TRACE_FILE: "{{ sandbox.trace_path }}"

agent:
  type: cli
  binary: /workspace/my-agent
  args: ["--task", "{{ task.prompt }}", "--workspace", "{{ sandbox.path }}"]
  timeout: 5m
  env:
    LOG_LEVEL: "info"

agent:
  type: http
  endpoint: "https://my-agent.internal/run"
  auth:
    bearer: "{{ secrets.AGENT_TOKEN }}"
  input_template: |
    {
      "task": "{{ task.prompt }}",
      "workspace": "{{ sandbox.url }}",
      "context": {{ task.context | tojson }}
    }
  timeout: 5m

agent:
  type: python
  binary: agent.py
  args: ["--mode", "eval"]
  timeout: 5m

agent:
  type: image
  image: "your-registry/your-agent:v3"
  entrypoint: ["python", "main.py"]   # optional override
  timeout: 5m
  env:
    LOG_LEVEL: "debug"

agent:
  type: snapshot
  snapshot: my-agent              # latest version
  # or pin a specific version:
  # snapshot_id: snap_abc123
  entrypoint: ["python", "main.py"]   # optional override
  timeout: 5m

resources:
  timeout: "10m"
  memory: "4Gi"
  cpu: 4
  disk: "20Gi"
  desktop: false               # true = XFCE desktop with browser available
  concurrency_limit: 5         # max sandboxes for this spec at once

fixtures:
  # Clone a git repository into /workspace
  - type: git_repo
    url: "https://github.com/your-org/your-repo"
    branch: "main"
    depth: 1
    path: /workspace
 
  # Run inline SQL against a service (preferred over path:)
  - type: sql
    service: db
    sql: |
      CREATE TABLE customers (
        email TEXT PRIMARY KEY,
        name  TEXT NOT NULL
      );
      INSERT INTO customers VALUES
        ('alice@example.com', 'Alice'),
        ('ben@example.com',   'Ben');
 
  # Copy a directory from the Keystone server's filesystem
  - type: directory
    source: /var/keystone/seeds/test-data
    target: data/
 
  # Inject random data corruption for adversarial testing
  - type: drift
    target: db.customers
    strategy: random_mismatches      # or random_nulls, duplicate_rows
    count: 15
    seed: "{{ determinism.seed }}"

services:
  - name: db
    image: postgres:16
    env:
      POSTGRES_PASSWORD: "{{ secrets.DB_PASSWORD }}"
      POSTGRES_DB: testdb
    ports: [5432]
    wait_for: "pg_isready -U postgres"
 
  - name: cache
    image: redis:7
    ports: [6379]
    wait_for: "redis-cli ping"
 
  - name: smtp
    image: mailhog/mailhog
    ports: [1025, 8025]    # SMTP + HTTP UI
 
  - name: stripe-mock
    type: http_mock
    record: true
    default_response: 404
    routes:
      - method: POST
        path: /v1/charge
        response: '{"id":"ch_test","status":"succeeded"}'
        status: 200
      - method: ANY
        path: "/v1/webhooks/.*"   # regex match
        response: '{"ok": true}'

secrets:
  - name: ANTHROPIC_API_KEY
    source: env                  # default; pulls $ANTHROPIC_API_KEY from caller's env
 
  - name: DB_PASSWORD
    source: env:LOCAL_DB_PASS    # rename — pull $LOCAL_DB_PASS, expose as DB_PASSWORD
 
  - name: GCP_KEY
    source: "file:~/.config/gcloud/sa.json"
 
  - name: VAULT_TOKEN
    source: 'command:vault token lookup -field=id'
 
  - name: STRIPE_LIVE_KEY
    source: dashboard            # server-side only; SDK refuses local override
 
  - name: TEST_FIXTURE_TOKEN
    from: "static://fake-test-value"   # spec literal; deterministic
 
  - name: GENERATED_DB_PASSWORD
    from: generated              # random per-run

secrets:
  # Default: env var only
  - name: API_KEY
    source: env
 
  # Env var AND templated into a config file
  - name: STRIPE_KEY
    from: "static://sk_test_xxx"
    scope:
      env: true
      file_template: "config/stripe.json"

network:
  egress:
    default: deny           # block all outbound by default
    allow:
      - registry.npmjs.org
      - github.com
      - "*.services.internal"  # internal service network
  ingress:
    default: deny
    allow:
      - from: host
        to_port: 3000
  dns_overrides:
    api.stripe.com: stripe-mock.services.internal
    smtp.sendgrid.net: smtp.services.internal

audit:
  db_writes: true              # log every INSERT/UPDATE/DELETE
  http_calls: true             # log every outbound HTTP
  process_spawns: true         # log every child process
  stdout_capture: true         # capture stdout for secret detection
  file_system:
    watch: ["src/", "config/"]
    track: [writes, reads, deletes]

snapshots:
  before_run: true              # always capture before agent starts
  checkpoints: per_action       # snapshot after each agent tool call
  retain_on: [failure]          # only persist snapshots for failed runs

invariants:
  tests_pass:
    description: "All tests pass"
    weight: 1.0
    gate: true                 # hard fail if this fails
    check:
      type: command_exit
      command: "npm test"
      exit_code: 0
 
  output_correct:
    description: "Output file contains expected data"
    weight: 0.5
    check:
      type: file_content
      path: output.json
      contains: '"status": "success"'
      not_contains: "ERROR"
 
  no_orphan_orders:
    description: "Every order has a matching customer"
    weight: 0.3
    gate: true
    check:
      type: sql
      service: db
      query: "SELECT count(*) FROM orders WHERE customer_id NOT IN (SELECT id FROM customers)"
      equals: 0
 
  email_sent:
    description: "Exactly one email to finance"
    weight: 0.3
    check:
      type: http_mock_assertions
      service: smtp
      assertions:
        - field: request_count
          filters: { to: "finance@co" }
          equals: 1
 
  custom_check:
    description: "Custom Python validation"
    weight: 0.2
    check:
      type: custom
      script: checks/validate.py
      runs_in: host           # host (default) or sandbox
 
  code_quality:
    description: "Code is well-structured"
    weight: 0.4
    check:
      type: llm_as_judge
      model: paragon-fast
      criteria: "Is the diff minimal and idiomatic?"
      input_from: workspace
      rubric:
        pass: "Clean, minimal change"
        fail: "Over-engineered or invasive"
      pass_threshold: 0.7

forbidden:
  db_writes_outside: [users, orders, audit_log]
  http_except: [stripe-mock, smtp]
  secrets_in_logs: deny
  file_writes_outside: [src/, output/, .keystone/]

scoring:
  pass_threshold: 0.9          # composite must be >= 0.9 to pass
  replica_aggregation:
    strategy: majority          # all_must_pass | majority | percentage
    min_pass_rate: 0.85

parallelism:
  replicas: 10                 # runs PER matrix entry
  isolation: per_run           # per_run = own sandbox | shared = reuse
  matrix:
    - { input_size: small, locale: en_US }
    - { input_size: large, locale: en_US }
    - { input_size: small, locale: ja_JP }

parallelism:
  matrix:
    - { model: claude-sonnet-4-5 }
    - { model: gpt-4o }
 
agent:
  type: cli
  binary: my-agent
  args: ["--model", "{{ matrix.model }}"]

determinism:
  clock: "2026-01-01T00:00:00Z"   # frozen time (libfaketime)
  seed: 42                         # PRNG seed
  network_latency: 0ms              # injected latency
  dns: static                       # static or live

retention:
  audit_logs: 24h
  snapshots: 7d
  teardown_exports: 30d
  traces: 30d

teardown:
  always_run: true
  export:
    - type: audit_log
      to: "results/{{ run_id }}/audit.jsonl"
    - type: db_dump
      service: db
      to: "results/{{ run_id }}/db.sql"
    - type: snapshot
      to: "results/{{ run_id }}/final_state.snap"
    - type: mock_requests
      service: stripe-mock
      to: "results/{{ run_id }}/stripe-calls.jsonl"

version: 1
id: reconciliation-scenario
description: "Agent reconciles two customer databases and emails a summary"
 
task:
  prompt: |
    Reconcile customers_a and customers_b tables, fix all mismatches,
    and email a summary to finance@co
  context:
    ticket_url: "https://linear.app/co/issue/ENG-1234"
    success_criteria: "All rows match after reconciliation, exactly one summary email sent"
 
base: "ubuntu:24.04"
 
setup:
  packages: [nodejs, npm, git, python3]
  commands:
    - "npm install -g typescript"
  env:
    NODE_ENV: "test"
    DATABASE_URL: "postgres://postgres:{{ secrets.DB_PASSWORD }}@db:5432/testdb"
 
resources:
  timeout: "10m"
  memory: "2Gi"
  cpu: 2
  concurrency_limit: 5
 
fixtures:
  - type: sql
    service: db
    sql: |
      CREATE TABLE customers_a (id INT PRIMARY KEY, email TEXT, name TEXT);
      CREATE TABLE customers_b (id INT PRIMARY KEY, email TEXT, name TEXT);
      INSERT INTO customers_a VALUES
        (1, 'a@co', 'Alice'), (2, 'b@co', 'Ben'), (3, 'c@co', 'Carol');
      INSERT INTO customers_b SELECT * FROM customers_a;
  - type: drift
    target: db.customers_a
    strategy: random_mismatches
    count: 15
    seed: "{{ determinism.seed }}"
 
services:
  - name: db
    image: postgres:16
    env:
      POSTGRES_PASSWORD: "{{ secrets.DB_PASSWORD }}"
      POSTGRES_DB: testdb
    ports: [5432]
    wait_for: "pg_isready"
  - name: smtp
    image: mailhog/mailhog
    ports: [1025, 8025]
  - name: mock_api
    type: http_mock
    record: true
    default_response: 404
    routes:
      - method: GET
        path: /v1/customers
        response: '{"customers": []}'
        status: 200
 
secrets:
  - name: DB_PASSWORD
    from: generated
    scope: env
 
network:
  egress:
    default: deny
    allow: ["*.services.internal"]
  dns_overrides:
    api.stripe.com: mock_api.services.internal
 
audit:
  db_writes: true
  http_calls: true
  file_system:
    watch: [/workspace]
    track: [writes, deletes]
  process_spawns: true
  stdout_capture: true
 
snapshots:
  before_run: true
  checkpoints: per_action
  retain_on: [failure]
 
agent:
  type: paragon
  model: paragon-max
  args: ["--yolo", "--quiet"]
  timeout: 5m
 
invariants:
  databases_match:
    description: "No drift between customers_a and customers_b"
    weight: 0.5
    gate: true
    check:
      type: sql
      service: db
      query: |
        SELECT count(*) FROM customers_a a
        LEFT JOIN customers_b b ON a.id = b.id
        WHERE b.id IS NULL OR a.email != b.email
      equals: 0
 
  email_sent:
    description: "Exactly one summary email to finance"
    weight: 0.3
    check:
      type: http_mock_assertions
      service: smtp
      assertions:
        - field: request_count
          filters: { to: "finance@co" }
          equals: 1
 
  code_quality:
    description: "Solution is clean and idiomatic"
    weight: 0.2
    check:
      type: llm_as_judge
      model: paragon-fast
      criteria: "Is the reconciliation code clean and well-structured?"
      input_from: workspace
      rubric:
        pass: "Clean, handles edge cases"
        fail: "Messy, fragile, or incomplete"
      temperature: 0
 
forbidden:
  db_writes_outside: [customers_a, customers_b, reconciliation_log]
  http_except: [smtp, mock_api]
  secrets_in_logs: deny
  file_writes_outside: [/workspace/src, /workspace/logs]
 
scoring:
  pass_threshold: 0.95
  replica_aggregation:
    strategy: all_must_pass
    min_pass_rate: 0.90
 
parallelism:
  replicas: 3
  isolation: per_run
  matrix:
    - { input_size: small, locale: en_US }
    - { input_size: large, locale: en_US }
 
determinism:
  clock: "2026-01-01T00:00:00Z"
  seed: 42
  network_latency: 0ms
  dns: static
 
retention:
  audit_logs: 24h
  snapshots: 7d
  teardown_exports: 30d
  traces: 30d
 
teardown:
  export:
    - type: audit_log
      to: "results/{{ run_id }}/audit.jsonl"
    - type: db_dump
      service: db
      to: "results/{{ run_id }}/db.sql"
    - type: mock_requests
      service: mock_api
      to: "results/{{ run_id }}/mock.jsonl"
  always_run: true