"expectedItems": [
  { "id": "brute-force", "description": "An alert fired for repeated failed SSH logins from a single source IP" },
  { "id": "priv-esc",    "description": "An alert fired for a sudo invocation by a non-admin service account" },
  { "id": "exfil",       "description": "An alert fired for an outbound transfer larger than 100 MB to an unknown host" }
]

curl -X POST https://api.logpulse.io/api/v1/evaluations/run-adhoc \
  -H "Authorization: Bearer $LOGPULSE_PAT" \
  -H "Content-Type: application/json" \
  -d '{
    "mode": "test_classification",
    "criteria": "These are SIEM logs from a purple-team exercise. For each expected item, determine whether the corresponding detection alert fired.",
    "expectedItems": [
      { "id": "brute-force", "description": "An alert fired for repeated failed SSH logins from a single source IP" },
      { "id": "priv-esc", "description": "An alert fired for a sudo invocation by a non-admin service account" }
    ],
    "rows": [
      {"timestamp": "2026-06-12T09:13:55Z", "source": "sshd", "event": "Failed password for root from 203.0.113.7", "count": 23},
      {"timestamp": "2026-06-12T09:14:02Z", "source": "siem", "alert": "SSH brute force detected", "src_ip": "203.0.113.7"},
      {"timestamp": "2026-06-12T09:20:11Z", "source": "auditd", "event": "sudo invoked", "user": "svc-backup"}
    ]
  }'

curl -X POST https://api.logpulse.io/api/v1/evaluations/run-adhoc \
  -H "Authorization: Bearer $LOGPULSE_PAT" \
  -H "Content-Type: application/json" \
  -d '{
    "mode": "generic_verdict",
    "criteria": "Does this log entry indicate a security-relevant event? If so, classify the severity and name the affected account.",
    "log": {
      "timestamp": "2026-06-12T03:12:44Z",
      "source": "auth-service",
      "event": "Password changed",
      "user": "[email protected]",
      "ip": "198.51.100.23",
      "geo": "country=BR",
      "mfa": false
    }
  }'

import os
import requests

PAT = os.environ["LOGPULSE_PAT"]
BASE_URL = "https://api.logpulse.io"

def run_evaluation(rows, criteria, expected_items):
    response = requests.post(
        f"{BASE_URL}/api/v1/evaluations/run-adhoc",
        headers={"Authorization": f"Bearer {PAT}"},
        json={
            "mode": "test_classification",
            "rows": rows,
            "criteria": criteria,
            "expectedItems": expected_items,
        },
        timeout=120,  # synchronous AI call; allow time for large inputs
    )
    response.raise_for_status()
    return response.json()["data"]

result = run_evaluation(
    rows=load_exercise_logs(),  # your own loader
    criteria="For each expected item, determine whether the detection alert fired.",
    expected_items=[
        {"id": "brute-force", "description": "Alert for repeated failed SSH logins from one IP"},
        {"id": "priv-esc", "description": "Alert for sudo by a non-admin service account"},
    ],
)

if result["truncated"]:
    print(f"warning: input clipped, {result['inputRowCount']} rows evaluated")

for finding in result["verdict"]:
    print(f"{finding['id']}: {finding.get('status')} — {finding['explanation']}")

failed = [f for f in result["verdict"] if f.get("status") != "passed"]
if failed:
    raise SystemExit(f"{len(failed)} expected detection(s) did not pass")

const PAT = process.env.LOGPULSE_PAT;

async function runEvaluation({ rows, criteria, expectedItems }) {
  const response = await fetch(
    'https://api.logpulse.io/api/v1/evaluations/run-adhoc',
    {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${PAT}`,
      },
      body: JSON.stringify({
        mode: 'test_classification',
        rows,
        criteria,
        expectedItems,
      }),
    },
  );

  if (response.status === 429) {
    const body = await response.json();
    if (body.code === 'AI_CREDITS_EXHAUSTED') {
      throw new Error(`AI credits exhausted, resets at ${body.resetsAt}`);
    }
    throw new Error('Rate limited — retry with backoff');
  }
  if (!response.ok) {
    throw new Error(`Evaluation failed: ${response.status}`);
  }

  const { data } = await response.json();
  return data;
}

const data = await runEvaluation({
  rows: exerciseLogs,
  criteria: 'For each expected item, determine whether the detection alert fired.',
  expectedItems: [
    { id: 'brute-force', description: 'Alert for repeated failed SSH logins from one IP' },
    { id: 'priv-esc', description: 'Alert for sudo by a non-admin service account' },
  ],
});

const notPassed = data.verdict.filter((f) => f.status !== 'passed');
console.log(`${data.verdict.length - notPassed.length}/${data.verdict.length} passed`);
if (notPassed.length > 0) process.exit(1);

jobs:
  detection-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Run attack simulation
        run: ./scripts/run-simulation.sh > exercise-logs.json

      - name: Evaluate detections
        env:
          LOGPULSE_PAT: ${{ secrets.LOGPULSE_PAT }}
        run: |
          RESULT=$(curl -sf -X POST \
            https://api.logpulse.io/api/v1/evaluations/run-adhoc \
            -H "Authorization: Bearer $LOGPULSE_PAT" \
            -H "Content-Type: application/json" \
            -d "$(jq -n --slurpfile rows exercise-logs.json \
              --slurpfile expected detection-expectations.json '{
                mode: "test_classification",
                criteria: "For each expected item, determine whether the detection alert fired.",
                rows: $rows[0],
                expectedItems: $expected[0]
              }')")
          echo "$RESULT" | jq '.data.verdict'
          echo "$RESULT" | jq -e \
            '[.data.verdict[] | select(.status != "passed")] | length == 0'