Run experiment

curl --request POST \
  --url https://api.sidenet.ai/v1/experiments \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'X-User-Id: <x-user-id>' \
  --data '
{
  "agentId": "6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10",
  "datasetId": "ds_orders_v3",
  "async": false,
  "name": "Refund tone — August",
  "description": "Checks the refund answers after the new brand-voice block",
  "metadata": {
    "ticket": "ENG-412"
  },
  "version": 3,
  "maxConcurrency": 5,
  "itemTimeout": 60000,
  "maxRetries": 2,
  "variables": {
    "locale": "English",
    "instance": {
      "store_count": 12
    }
  },
  "strictVariables": true,
  "unmockedToolPolicy": "deny"
}
'

import requests

url = "https://api.sidenet.ai/v1/experiments"

payload = {
    "agentId": "6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10",
    "datasetId": "ds_orders_v3",
    "async": False,
    "name": "Refund tone — August",
    "description": "Checks the refund answers after the new brand-voice block",
    "metadata": { "ticket": "ENG-412" },
    "version": 3,
    "maxConcurrency": 5,
    "itemTimeout": 60000,
    "maxRetries": 2,
    "variables": {
        "locale": "English",
        "instance": { "store_count": 12 }
    },
    "strictVariables": True,
    "unmockedToolPolicy": "deny"
}
headers = {
    "X-User-Id": "<x-user-id>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'X-User-Id': '<x-user-id>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    agentId: '6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10',
    datasetId: 'ds_orders_v3',
    async: false,
    name: 'Refund tone — August',
    description: 'Checks the refund answers after the new brand-voice block',
    metadata: {ticket: 'ENG-412'},
    version: 3,
    maxConcurrency: 5,
    itemTimeout: 60000,
    maxRetries: 2,
    variables: {locale: 'English', instance: {store_count: 12}},
    strictVariables: true,
    unmockedToolPolicy: 'deny'
  })
};

fetch('https://api.sidenet.ai/v1/experiments', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.sidenet.ai/v1/experiments",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'agentId' => '6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10',
    'datasetId' => 'ds_orders_v3',
    'async' => false,
    'name' => 'Refund tone — August',
    'description' => 'Checks the refund answers after the new brand-voice block',
    'metadata' => [
        'ticket' => 'ENG-412'
    ],
    'version' => 3,
    'maxConcurrency' => 5,
    'itemTimeout' => 60000,
    'maxRetries' => 2,
    'variables' => [
        'locale' => 'English',
        'instance' => [
                'store_count' => 12
        ]
    ],
    'strictVariables' => true,
    'unmockedToolPolicy' => 'deny'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "X-User-Id: <x-user-id>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.sidenet.ai/v1/experiments"

	payload := strings.NewReader("{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-User-Id", "<x-user-id>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.sidenet.ai/v1/experiments")
  .header("X-User-Id", "<x-user-id>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.sidenet.ai/v1/experiments")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-User-Id"] = '<x-user-id>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}"

response = http.request(request)
puts response.read_body

{
  "status": "started",
  "experimentId": "exp_7f31a9c2",
  "scorers": [
    {
      "name": "tone",
      "scorerType": "tone",
      "model": null
    },
    {
      "name": "faithfulness",
      "scorerType": "faithfulness",
      "model": "openai/gpt-4o-mini"
    }
  ],
  "variables": {
    "supplied": [
      "locale"
    ],
    "required": [
      "locale",
      "instance.store_count"
    ],
    "missing": [
      "instance.store_count"
    ],
    "dropped": [],
    "unused": []
  }
}

Scoring & Experiments

Run experiment

Runs an agent — or a workflow — over every item of a dataset and scores each answer. Send agentId (agent id or agent version id) and datasetId; or workflowId instead of agentId to run a workflow, with workflowVersion (active default, draft, or a number), scorers (catalogue types, since a workflow has none attached; each { scorerType, model?, options? } or just the type) and runtimeAuth for its tool steps. For a workflow each item’s input is the run’s input (an object shaped by the version’s input_schema; items are checked against it before anything runs, 400 unless strictInput: false), and the scorers grade the version’s output_text rendered for that run — what a scheduled run would post. The scorers are the ones attached to the agent — the same scorers, judge models and options that score its live traffic — applied to every item rather than sampled; the response lists which ran. An agent with no scorers still runs and records its answers. Set async: true for fire-and-forget (returns an experimentId to poll with GET /v1/experiments/); omit it to block and receive the full summary. Requires X-User-Id. Each dataset item runs in its own conversation thread by default, so items never share memory state and can run concurrently. To pin an item to a specific thread (e.g. several items forming one conversation), set requestContext.threadId on that dataset item — and run with maxConcurrency: 1 if multiple items share a thread. Send variables to fill the in the agent’s prompt blocks for every item; a dataset item’s own requestContext.variables are merged over them leaf by leaf, so a row overrides only what it varies. By default the run is REFUSED (400) when a placeholder with no || 'default' has no value — pass strictVariables: false to run anyway. The response always carries a variables report (supplied, required, missing, dropped, unused). Tool calls run for real unless the dataset item declares toolMocks for them; send unmockedToolPolicy: "deny" to fail any item that calls a tool it has no mock for, so the run never touches the outside world. Each result carries a toolMockReport.

POST

experiments

Run experiment

curl --request POST \
  --url https://api.sidenet.ai/v1/experiments \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --header 'X-User-Id: <x-user-id>' \
  --data '
{
  "agentId": "6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10",
  "datasetId": "ds_orders_v3",
  "async": false,
  "name": "Refund tone — August",
  "description": "Checks the refund answers after the new brand-voice block",
  "metadata": {
    "ticket": "ENG-412"
  },
  "version": 3,
  "maxConcurrency": 5,
  "itemTimeout": 60000,
  "maxRetries": 2,
  "variables": {
    "locale": "English",
    "instance": {
      "store_count": 12
    }
  },
  "strictVariables": true,
  "unmockedToolPolicy": "deny"
}
'

import requests

url = "https://api.sidenet.ai/v1/experiments"

payload = {
    "agentId": "6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10",
    "datasetId": "ds_orders_v3",
    "async": False,
    "name": "Refund tone — August",
    "description": "Checks the refund answers after the new brand-voice block",
    "metadata": { "ticket": "ENG-412" },
    "version": 3,
    "maxConcurrency": 5,
    "itemTimeout": 60000,
    "maxRetries": 2,
    "variables": {
        "locale": "English",
        "instance": { "store_count": 12 }
    },
    "strictVariables": True,
    "unmockedToolPolicy": "deny"
}
headers = {
    "X-User-Id": "<x-user-id>",
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {
    'X-User-Id': '<x-user-id>',
    Authorization: 'Bearer <token>',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    agentId: '6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10',
    datasetId: 'ds_orders_v3',
    async: false,
    name: 'Refund tone — August',
    description: 'Checks the refund answers after the new brand-voice block',
    metadata: {ticket: 'ENG-412'},
    version: 3,
    maxConcurrency: 5,
    itemTimeout: 60000,
    maxRetries: 2,
    variables: {locale: 'English', instance: {store_count: 12}},
    strictVariables: true,
    unmockedToolPolicy: 'deny'
  })
};

fetch('https://api.sidenet.ai/v1/experiments', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.sidenet.ai/v1/experiments",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'agentId' => '6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10',
    'datasetId' => 'ds_orders_v3',
    'async' => false,
    'name' => 'Refund tone — August',
    'description' => 'Checks the refund answers after the new brand-voice block',
    'metadata' => [
        'ticket' => 'ENG-412'
    ],
    'version' => 3,
    'maxConcurrency' => 5,
    'itemTimeout' => 60000,
    'maxRetries' => 2,
    'variables' => [
        'locale' => 'English',
        'instance' => [
                'store_count' => 12
        ]
    ],
    'strictVariables' => true,
    'unmockedToolPolicy' => 'deny'
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json",
    "X-User-Id: <x-user-id>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.sidenet.ai/v1/experiments"

	payload := strings.NewReader("{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-User-Id", "<x-user-id>")
	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.sidenet.ai/v1/experiments")
  .header("X-User-Id", "<x-user-id>")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.sidenet.ai/v1/experiments")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-User-Id"] = '<x-user-id>'
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"agentId\": \"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10\",\n  \"datasetId\": \"ds_orders_v3\",\n  \"async\": false,\n  \"name\": \"Refund tone — August\",\n  \"description\": \"Checks the refund answers after the new brand-voice block\",\n  \"metadata\": {\n    \"ticket\": \"ENG-412\"\n  },\n  \"version\": 3,\n  \"maxConcurrency\": 5,\n  \"itemTimeout\": 60000,\n  \"maxRetries\": 2,\n  \"variables\": {\n    \"locale\": \"English\",\n    \"instance\": {\n      \"store_count\": 12\n    }\n  },\n  \"strictVariables\": true,\n  \"unmockedToolPolicy\": \"deny\"\n}"

response = http.request(request)
puts response.read_body

{
  "status": "started",
  "experimentId": "exp_7f31a9c2",
  "scorers": [
    {
      "name": "tone",
      "scorerType": "tone",
      "model": null
    },
    {
      "name": "faithfulness",
      "scorerType": "faithfulness",
      "model": "openai/gpt-4o-mini"
    }
  ],
  "variables": {
    "supplied": [
      "locale"
    ],
    "required": [
      "locale",
      "instance.store_count"
    ],
    "missing": [
      "instance.store_count"
    ],
    "dropped": [],
    "unused": []
  }
}

Authorizations

Authorization

string

header

required

Organization API key, generated in studio.sidenet.ai. Backend only — never in a browser.

Headers

X-User-Id

string

required

User id. May also be sent in the request body or as a userId query param.

Body

application/json

agentId

string

required

Example:

"6f1c0f4e-2b7a-4a51-9a2f-0c9d1b3e5a10"

datasetId

string

required

Example:

"ds_orders_v3"

async

boolean

default:false

name

string

Example:

"Refund tone — August"

description

string

Example:

"Checks the refund answers after the new brand-voice block"

metadata

object

Example:

{ "ticket": "ENG-412" }

version

integer

Dataset version to run against. Defaults to the newest.

Example:

3

maxConcurrency

integer

Example:

5

itemTimeout

integer

Milliseconds one item may take before it is failed.

Example:

60000

maxRetries

integer

default:2

Retries per dataset item before it is recorded as failed. Defaults to 2 so a transient provider fault does not fail an item; pass 0 to fail on the first error.

variables

object

Values for the {{variable}} placeholders in the agent's prompt blocks, applied to every item. Scalars or arrays of them; a dot path up to 3 segments may be sent nested ({ "user": { "language": "FR" } }) or dotted ({ "user.language": "FR" }). Reserved: now.*. A dataset item's own requestContext.variables are merged over these leaf by leaf. Display-only — never used for authorization or identity.

Example:

{
  "locale": "English",
  "instance": { "store_count": 12 }
}

strictVariables

boolean

default:true

Refuse to start (400) when a placeholder with no default has no value for some item, or a supplied value is rejected by the limits. Pass false to run anyway and read the variables report on the response.

unmockedToolPolicy

enum<string>

default:allow

What happens when an item calls a tool it declares no mock for (see toolMocks on dataset items). allow runs the real tool; deny fails the item with error.code TOOL_MOCK_NOT_DECLARED before the tool runs. An item's own unmockedToolPolicy overrides this for that item.

Available options:

allow,

deny

Example:

"deny"

Response

The finished run, or — with async: true — the id of the run that was started

status

enum<string>

started on an async run; the scores are not ready yet.

Available options:

completed,

started

experimentId

string

Async runs only — poll the experiment by this id.

summary

object

Sync runs only: every item with its answer and its scores, plus the run totals. The same data GET /v1/experiments/{experimentId} returns.

scorers

object[]

The agent's attached scorers, each applied to every item. Empty when the agent has none — the run then records answers only.

Show child attributes

skippedScorers

object[]

Scorers configured on the agent that could not be built (an LLM judge with no judge key, an unknown type). Present only when non-empty.

Show child attributes

variables

object

Show child attributes

Authentication

Users & Groups

Chat & Messaging

Agents

Prompt Blocks

Skills

Workflows

Workflow Schedules

Dashboards

Networks

Tools

Tool Providers

Datasets

Scoring & Experiments

System

Run experiment

Authorizations

Headers

Body

Response