POST
/api/v1/prompts/ab-experimentsCreate a prompt A/B experiment
Creates an experiment + 2-8 variants in one call. The worker runs each variant against the dataset with the configured scorers and declares a winner via significance testing. requiredRole: editor.
Authentication
Send Authorization: Bearer YOUR_API_KEY on every request. Generate API keys at /dashboard/settings/api-keys.
Request body required
Example
{
"projectId": "00000000-0000-0000-0000-000000000000",
"name": "string",
"description": "string",
"prompt_name": "string",
"model": "gpt-4o-mini",
"dataset_id": "00000000-0000-0000-0000-000000000000",
"scorer_ids": [
"string"
],
"variants": [
{
"label": "string",
"content": "string",
"weight": 1
}
],
"winner_config": {
"metric": "score",
"direction": "higher_is_better",
"confidence": 0.95,
"min_samples": 30
}
}Schema
{
"application/json": {
"schema": {
"type": "object",
"properties": {
"projectId": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$"
},
"name": {
"type": "string",
"minLength": 1,
"maxLength": 200
},
"description": {
"type": "string",
"maxLength": 2000
},
"prompt_name": {
"type": "string",
"minLength": 1,
"maxLength": 200
},
"model": {
"default": "gpt-4o-mini",
"type": "string",
"minLength": 1,
"maxLength": 64
},
"dataset_id": {
"type": "string",
"format": "uuid",
"pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-8][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}|00000000-0000-0000-0000-000000000000|ffffffff-ffff-ffff-ffff-ffffffffffff)$"
},
"scorer_ids": {
"minItems": 1,
"maxItems": 20,
"type": "array",
"items": {
"type": "string"
}
},
"variants": {
"minItems": 2,
"maxItems": 8,
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string",
"minLength": 1,
"maxLength": 64
},
"content": {
"type": "string",
"minLength": 1,
"maxLength": 50000
},
"weight": {
"default": 1,
"type": "integer",
"minimum": 1,
"maximum": 100
}
},
"required": [
"label",
"content",
"weight"
],
"additionalProperties": false
}
},
"winner_config": {
"type": "object",
"properties": {
"metric": {
"default": "score",
"type": "string",
"enum": [
"score",
"pass_rate",
"latency",
"cost"
]
},
"direction": {
"default": "higher_is_better",
"type": "string",
"enum": [
"higher_is_better",
"lower_is_better"
]
},
"confidence": {
"default": 0.95,
"type": "number",
"minimum": 0.5,
"maximum": 0.999
},
"min_samples": {
"default": 30,
"type": "integer",
"minimum": 10,
"maximum": 10000
}
},
"required": [
"metric",
"direction",
"confidence",
"min_samples"
],
"additionalProperties": false
}
},
"required": [
"projectId",
"name",
"prompt_name",
"model",
"scorer_ids",
"variants"
],
"additionalProperties": false
}
}
}Response
201 example
{
"success": true
}All status codes
201Experiment created ({ id, status:'draft', variants }).
400(no description)
401(no description)
403Forbidden — editor role required.
404Project not found.
409Conflict — CONFLICT.
429(no description)
500Internal Server Error — DB_ERROR.
Code samples
cURL
curl -X POST \
https://evalguard.ai/api/v1/prompts/ab-experiments \
-H "Authorization: Bearer $EVALGUARD_API_KEY" \
-H "Content-Type: application/json" \
-d '{ "projectId": "00000000-0000-0000-0000-000000000000", "name": "string", "description": "string", "prompt_name": "string", "model": "gpt-4o-mini", "dataset_id": "00000000-0000-0000-0000-000000000000", "scorer_ids": [ "string" ], "variants": [ { "label": "string", "content": "string", "weight": 1 } ], "winner_config": { "metric": "score", "direction": "higher_is_better", "confidence": 0.95, "min_samples": 30 } }'TypeScript
// The TypeScript SDK (@evalguard/sdk) exposes TYPED methods — runEval,
// getEval, runSecurityScan, checkFirewall, … — not a generic request().
// For an arbitrary endpoint, call it directly:
const res = await fetch("https://evalguard.ai/api/v1/prompts/ab-experiments", {
method: "POST",
headers: {
Authorization: `Bearer ${process.env.EVALGUARD_API_KEY}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
"projectId": "00000000-0000-0000-0000-000000000000",
"name": "string",
"description": "string",
"prompt_name": "string",
"model": "gpt-4o-mini",
"dataset_id": "00000000-0000-0000-0000-000000000000",
"scorer_ids": [
"string"
],
"variants": [
{
"label": "string",
"content": "string",
"weight": 1
}
],
"winner_config": {
"metric": "score",
"direction": "higher_is_better",
"confidence": 0.95,
"min_samples": 30
}
}),
});
console.log(res.status, await res.json());Python
# The Python SDK (pip install evalguardai) exposes TYPED methods on
# EvalGuardClient — run_eval, get_eval, … — not a generic request().
# For an arbitrary endpoint, call it directly:
import os
import requests
headers = {"Authorization": f"Bearer {os.environ['EVALGUARD_API_KEY']}"}
headers["Content-Type"] = "application/json"
response = requests.request(
"POST",
"https://evalguard.ai/api/v1/prompts/ab-experiments",
headers=headers,
json={
"projectId": "00000000-0000-0000-0000-000000000000",
"name": "string",
"description": "string",
"prompt_name": "string",
"model": "gpt-4o-mini",
"dataset_id": "00000000-0000-0000-0000-000000000000",
"scorer_ids": [
"string"
],
"variants": [
{
"label": "string",
"content": "string",
"weight": 1
}
],
"winner_config": {
"metric": "score",
"direction": "higher_is_better",
"confidence": 0.95,
"min_samples": 30
}
},
)
print(response.status_code, response.json())Go
package main
import (
"context"
"fmt"
"net/http"
"os"
"strings"
)
func main() {
body := strings.NewReader(`{"projectId":"00000000-0000-0000-0000-000000000000","name":"string","description":"string","prompt_name":"string","model":"gpt-4o-mini","dataset_id":"00000000-0000-0000-0000-000000000000","scorer_ids":["string"],"variants":[{"label":"string","content":"string","weight":1}],"winner_config":{"metric":"score","direction":"higher_is_better","confidence":0.95,"min_samples":30}}`)
req, _ := http.NewRequestWithContext(context.Background(), "POST", "https://evalguard.ai/api/v1/prompts/ab-experiments", body)
req.Header.Set("Authorization", "Bearer "+os.Getenv("EVALGUARD_API_KEY"))
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil { panic(err) }
defer resp.Body.Close()
fmt.Println(resp.Status)
}Errors
400401403404409429500