[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-\u002Fen\u002Fdocs\u002Fsupervision\u002Fbenchmarks":3,"docs-nav-en":475},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"body":10,"_type":469,"_id":470,"_source":471,"_file":472,"_stem":473,"_extension":474},"\u002Fen\u002Fdocs\u002Fsupervision\u002Fbenchmarks","supervision",false,"","Benchmarks","Measure pipeline quality with reproducible scenarios. Detect regressions before production.",{"type":11,"children":12,"toc":460},"root",[13,36,43,48,68,73,79,84,108,114,273,278,284,310,339,349,355,374,379,411,417],{"type":14,"tag":15,"props":16,"children":17},"element","p",{},[18,21,27,29,34],{"type":19,"value":20},"text","A ",{"type":14,"tag":22,"props":23,"children":24},"strong",{},[25],{"type":19,"value":26},"benchmark",{"type":19,"value":28}," is a set of evaluation scenarios replayed automatically against a pipeline. It is the tool for ",{"type":14,"tag":22,"props":30,"children":31},{},[32],{"type":19,"value":33},"objectively",{"type":19,"value":35}," measuring agent quality and catching regressions before they reach production.",{"type":14,"tag":37,"props":38,"children":40},"h2",{"id":39},"why",[41],{"type":19,"value":42},"Why",{"type":14,"tag":15,"props":44,"children":45},{},[46],{"type":19,"value":47},"An agent that \"seemed to work\" can silently regress after:",{"type":14,"tag":49,"props":50,"children":51},"ul",{},[52,58,63],{"type":14,"tag":53,"props":54,"children":55},"li",{},[56],{"type":19,"value":57},"a mission change;",{"type":14,"tag":53,"props":59,"children":60},{},[61],{"type":19,"value":62},"a LLM model change;",{"type":14,"tag":53,"props":64,"children":65},{},[66],{"type":19,"value":67},"a new tool being added.",{"type":14,"tag":15,"props":69,"children":70},{},[71],{"type":19,"value":72},"Without a benchmark, you discover this in production. With a benchmark, you see it on the next run — before your users do.",{"type":14,"tag":37,"props":74,"children":76},{"id":75},"anatomy-of-a-scenario",[77],{"type":19,"value":78},"Anatomy of a scenario",{"type":14,"tag":15,"props":80,"children":81},{},[82],{"type":19,"value":83},"A scenario contains:",{"type":14,"tag":49,"props":85,"children":86},{},[87,98],{"type":14,"tag":53,"props":88,"children":89},{},[90,91,96],{"type":19,"value":20},{"type":14,"tag":22,"props":92,"children":93},{},[94],{"type":19,"value":95},"stimulus",{"type":19,"value":97}," — the input the pipeline receives (a message or a sequence of turns, data fixtures).",{"type":14,"tag":53,"props":99,"children":100},{},[101,106],{"type":14,"tag":22,"props":102,"children":103},{},[104],{"type":19,"value":105},"Checks",{"type":19,"value":107}," — explicit assertions evaluated on each turn: the output produced, the shape of the response, the tool-call chain, the cost of actions.",{"type":14,"tag":37,"props":109,"children":111},{"id":110},"check-catalogue",[112],{"type":19,"value":113},"Check catalogue",{"type":14,"tag":115,"props":116,"children":117},"table",{},[118,137],{"type":14,"tag":119,"props":120,"children":121},"thead",{},[122],{"type":14,"tag":123,"props":124,"children":125},"tr",{},[126,132],{"type":14,"tag":127,"props":128,"children":129},"th",{},[130],{"type":19,"value":131},"Check",{"type":14,"tag":127,"props":133,"children":134},{},[135],{"type":19,"value":136},"Verifies",{"type":14,"tag":138,"props":139,"children":140},"tbody",{},[141,160,184,201,224,241],{"type":14,"tag":123,"props":142,"children":143},{},[144,155],{"type":14,"tag":145,"props":146,"children":147},"td",{},[148],{"type":14,"tag":149,"props":150,"children":152},"code",{"className":151},[],[153],{"type":19,"value":154},"pipeline_output_match",{"type":14,"tag":145,"props":156,"children":157},{},[158],{"type":19,"value":159},"A key of the output flow matches the expected value (path + assertion)",{"type":14,"tag":123,"props":161,"children":162},{},[163,172],{"type":14,"tag":145,"props":164,"children":165},{},[166],{"type":14,"tag":149,"props":167,"children":169},{"className":168},[],[170],{"type":19,"value":171},"response_shape",{"type":14,"tag":145,"props":173,"children":174},{},[175,177,182],{"type":19,"value":176},"The ",{"type":14,"tag":22,"props":178,"children":179},{},[180],{"type":19,"value":181},"shape",{"type":19,"value":183}," of the agent's response (length, structure), measured without a judge",{"type":14,"tag":123,"props":185,"children":186},{},[187,196],{"type":14,"tag":145,"props":188,"children":189},{},[190],{"type":14,"tag":149,"props":191,"children":193},{"className":192},[],[194],{"type":19,"value":195},"llm_rubric",{"type":14,"tag":145,"props":197,"children":198},{},[199],{"type":19,"value":200},"The substance of the response, scored by an LLM judge on a rubric of verifiable criteria",{"type":14,"tag":123,"props":202,"children":203},{},[204,213],{"type":14,"tag":145,"props":205,"children":206},{},[207],{"type":14,"tag":149,"props":208,"children":210},{"className":209},[],[211],{"type":19,"value":212},"tool_chain_analysis",{"type":14,"tag":145,"props":214,"children":215},{},[216,217,222],{"type":19,"value":176},{"type":14,"tag":22,"props":218,"children":219},{},[220],{"type":19,"value":221},"sequence",{"type":19,"value":223}," of tool calls follows the expected strategy",{"type":14,"tag":123,"props":225,"children":226},{},[227,236],{"type":14,"tag":145,"props":228,"children":229},{},[230],{"type":14,"tag":149,"props":231,"children":233},{"className":232},[],[234],{"type":19,"value":235},"tool_cost",{"type":14,"tag":145,"props":237,"children":238},{},[239],{"type":19,"value":240},"The action budget: each call has a cost, reaching the target earns a bonus",{"type":14,"tag":123,"props":242,"children":243},{},[244,268],{"type":14,"tag":145,"props":245,"children":246},{},[247,253,255,261,262],{"type":14,"tag":149,"props":248,"children":250},{"className":249},[],[251],{"type":19,"value":252},"static",{"type":19,"value":254}," \u002F ",{"type":14,"tag":149,"props":256,"children":258},{"className":257},[],[259],{"type":19,"value":260},"tests",{"type":19,"value":254},{"type":14,"tag":149,"props":263,"children":265},{"className":264},[],[266],{"type":19,"value":267},"diff_quality",{"type":14,"tag":145,"props":269,"children":270},{},[271],{"type":19,"value":272},"File- and code-oriented checks (output of a development pipeline)",{"type":14,"tag":15,"props":274,"children":275},{},[276],{"type":19,"value":277},"The catalogue is extensible platform-side: each check is a plugin registered on the server.",{"type":14,"tag":37,"props":279,"children":281},{"id":280},"creating-a-benchmark",[282],{"type":19,"value":283},"Creating a benchmark",{"type":14,"tag":285,"props":286,"children":287},"ol",{},[288,298],{"type":14,"tag":53,"props":289,"children":290},{},[291,296],{"type":14,"tag":22,"props":292,"children":293},{},[294],{"type":19,"value":295},"Applications → Evaluation → Scenarios",{"type":19,"value":297},".",{"type":14,"tag":53,"props":299,"children":300},{},[301,303,308],{"type":19,"value":302},"Create the scenarios (stimulus, checks, data fixtures); each scenario is dedicated to one model (the target pipeline). Fixtures are managed in the ",{"type":14,"tag":22,"props":304,"children":305},{},[306],{"type":19,"value":307},"Fixtures",{"type":19,"value":309}," tab.",{"type":14,"tag":15,"props":311,"children":312},{},[313,315,322,324,330,331,337],{"type":19,"value":314},"A scenario can also be created or modified by proposal from the ",{"type":14,"tag":316,"props":317,"children":319},"a",{"href":318},"\u002Fen\u002Fdocs\u002Fdeveloppeurs\u002Fmcp",[320],{"type":19,"value":321},"MCP server",{"type":19,"value":323}," (",{"type":14,"tag":149,"props":325,"children":327},{"className":326},[],[328],{"type":19,"value":329},"benchmark_create",{"type":19,"value":254},{"type":14,"tag":149,"props":332,"children":334},{"className":333},[],[335],{"type":19,"value":336},"benchmark_update",{"type":19,"value":338},"), which stays pending until it is accepted.",{"type":14,"tag":340,"props":341,"children":343},"alert",{"type":342},"info",[344],{"type":14,"tag":15,"props":345,"children":346},{},[347],{"type":19,"value":348},"A good benchmark includes a mix: nominal case, edge cases, adversarial cases (manipulation attempts, ambiguity, missing data).",{"type":14,"tag":37,"props":350,"children":352},{"id":351},"running-a-benchmark",[353],{"type":19,"value":354},"Running a benchmark",{"type":14,"tag":15,"props":356,"children":357},{},[358,360,365,367,372],{"type":19,"value":359},"From the admin interface, launch a ",{"type":14,"tag":22,"props":361,"children":362},{},[363],{"type":19,"value":364},"run",{"type":19,"value":366},": it is placed in a durable queue then executed by a dedicated worker, outside the web server. If this worker stops mid-run, the run is marked ",{"type":14,"tag":22,"props":368,"children":369},{},[370],{"type":19,"value":371},"failed",{"type":19,"value":373}," on restart (it is not resumed automatically): relaunch it. A drawer gives the per-scenario detail.",{"type":14,"tag":15,"props":375,"children":376},{},[377],{"type":19,"value":378},"The report shows:",{"type":14,"tag":49,"props":380,"children":381},{},[382,387,406],{"type":14,"tag":53,"props":383,"children":384},{},[385],{"type":19,"value":386},"Pass rate per check",{"type":14,"tag":53,"props":388,"children":389},{},[390,392,397,399,404],{"type":19,"value":391},"Score evolution across runs (",{"type":14,"tag":22,"props":393,"children":394},{},[395],{"type":19,"value":396},"History",{"type":19,"value":398}," and ",{"type":14,"tag":22,"props":400,"children":401},{},[402],{"type":19,"value":403},"Evolution",{"type":19,"value":405}," tabs)",{"type":14,"tag":53,"props":407,"children":408},{},[409],{"type":19,"value":410},"List of failed scenarios, with the detail of each check",{"type":14,"tag":37,"props":412,"children":414},{"id":413},"best-practices",[415],{"type":19,"value":416},"Best practices",{"type":14,"tag":49,"props":418,"children":419},{},[420,430,440,450],{"type":14,"tag":53,"props":421,"children":422},{},[423,428],{"type":14,"tag":22,"props":424,"children":425},{},[426],{"type":19,"value":427},"Start small",{"type":19,"value":429}," — 5 well-targeted scenarios are worth more than 50 approximate ones.",{"type":14,"tag":53,"props":431,"children":432},{},[433,438],{"type":14,"tag":22,"props":434,"children":435},{},[436],{"type":19,"value":437},"Benchmark before every publish",{"type":19,"value":439},", and compare with the previous run before pushing.",{"type":14,"tag":53,"props":441,"children":442},{},[443,448],{"type":14,"tag":22,"props":444,"children":445},{},[446],{"type":19,"value":447},"Version your missions.",{"type":19,"value":449}," Not just pipelines: missions evolve too.",{"type":14,"tag":53,"props":451,"children":452},{},[453,458],{"type":14,"tag":22,"props":454,"children":455},{},[456],{"type":19,"value":457},"Keep checks readable.",{"type":19,"value":459}," A check that requires 20 lines of explanation to understand is a check that needs to be broken down.",{"title":7,"searchDepth":461,"depth":461,"links":462},2,[463,464,465,466,467,468],{"id":39,"depth":461,"text":42},{"id":75,"depth":461,"text":78},{"id":110,"depth":461,"text":113},{"id":280,"depth":461,"text":283},{"id":351,"depth":461,"text":354},{"id":413,"depth":461,"text":416},"markdown","content:en:docs:4.supervision:3.benchmarks.md","content","en\u002Fdocs\u002F4.supervision\u002F3.benchmarks.md","en\u002Fdocs\u002F4.supervision\u002F3.benchmarks","md",[476],{"title":477,"_path":478,"children":479},"En","\u002Fen",[480],{"title":481,"_path":482,"children":483},"Docs","\u002Fen\u002Fdocs",[484,497,525,553,564,580,592,653],{"title":485,"_path":486,"children":487},"Getting started","\u002Fen\u002Fdocs\u002Fdemarrer",[488,491,494],{"title":489,"_path":490},"Welcome","\u002Fen\u002Fdocs\u002Fdemarrer\u002Fbienvenue",{"title":492,"_path":493},"Create an account","\u002Fen\u002Fdocs\u002Fdemarrer\u002Fcreer-un-compte",{"title":495,"_path":496},"Connect an LLM key","\u002Fen\u002Fdocs\u002Fdemarrer\u002Fbrancher-llm",{"title":498,"_path":499,"children":500},"Channels","\u002Fen\u002Fdocs\u002Fcanaux",[501,504,507,510,513,516,519,522],{"title":502,"_path":503},"Channels overview","\u002Fen\u002Fdocs\u002Fcanaux\u002Fvue-densemble",{"title":505,"_path":506},"Real-time telephony","\u002Fen\u002Fdocs\u002Fcanaux\u002Ftelephone",{"title":508,"_path":509},"Email","\u002Fen\u002Fdocs\u002Fcanaux\u002Femail",{"title":511,"_path":512},"Web chat widget","\u002Fen\u002Fdocs\u002Fcanaux\u002Fchat-widget",{"title":514,"_path":515},"HTTP Webhook","\u002Fen\u002Fdocs\u002Fcanaux\u002Fwebhook",{"title":517,"_path":518},"Messaging (WhatsApp, Messenger, Telegram, Mattermost)","\u002Fen\u002Fdocs\u002Fcanaux\u002Fmessageries",{"title":520,"_path":521},"Meta Lead Ads","\u002Fen\u002Fdocs\u002Fcanaux\u002Ffacebook-leads",{"title":523,"_path":524},"Schedule (cron)","\u002Fen\u002Fdocs\u002Fcanaux\u002Fschedule",{"title":526,"_path":527,"children":528},"Pipelines","\u002Fen\u002Fdocs\u002Fpipelines",[529,532,535,538,541,544,547,550],{"title":530,"_path":531},"The Visual Editor","\u002Fen\u002Fdocs\u002Fpipelines\u002Fediteur-visuel",{"title":533,"_path":534},"AI Agents","\u002Fen\u002Fdocs\u002Fpipelines\u002Fagents",{"title":536,"_path":537},"Deterministic Operators","\u002Fen\u002Fdocs\u002Fpipelines\u002Foperateurs",{"title":539,"_path":540},"Flow Control","\u002Fen\u002Fdocs\u002Fpipelines\u002Fcontrole-flux",{"title":542,"_path":543},"Code Steps","\u002Fen\u002Fdocs\u002Fpipelines\u002Fcode-step",{"title":545,"_path":546},"Sub-pipelines","\u002Fen\u002Fdocs\u002Fpipelines\u002Fsous-pipelines",{"title":548,"_path":549},"No-code actions","\u002Fen\u002Fdocs\u002Fpipelines\u002Factions-no-code",{"title":551,"_path":552},"Knowledge base","\u002Fen\u002Fdocs\u002Fpipelines\u002Fbase-de-connaissances",{"title":554,"_path":555,"children":556},"Supervision","\u002Fen\u002Fdocs\u002Fsupervision",[557,560,563],{"title":558,"_path":559},"Execution History","\u002Fen\u002Fdocs\u002Fsupervision\u002Fexecutions",{"title":561,"_path":562},"Credits & Metering","\u002Fen\u002Fdocs\u002Fsupervision\u002Fcredits-metering",{"title":8,"_path":4},{"title":565,"_path":566,"children":567},"Security","\u002Fen\u002Fdocs\u002Fsecurite",[568,571,574,577],{"title":569,"_path":570},"Security principles","\u002Fen\u002Fdocs\u002Fsecurite\u002Fprincipes",{"title":572,"_path":573},"Strict multi-tenancy","\u002Fen\u002Fdocs\u002Fsecurite\u002Fmulti-tenant",{"title":575,"_path":576},"GDPR audit & access log","\u002Fen\u002Fdocs\u002Fsecurite\u002Faudit-rgpd",{"title":578,"_path":579},"Sovereign deployment","\u002Fen\u002Fdocs\u002Fsecurite\u002Fdeploiement-souverain",{"title":581,"_path":582,"children":583},"Developers","\u002Fen\u002Fdocs\u002Fdeveloppeurs",[584,587,590],{"title":585,"_path":586},"Admin API","\u002Fen\u002Fdocs\u002Fdeveloppeurs\u002Fapi-admin",{"title":588,"_path":589},"Outbound webhooks","\u002Fen\u002Fdocs\u002Fdeveloppeurs\u002Fwebhooks-sortants",{"title":591,"_path":318},"Drive betool from Claude (MCP)",{"title":593,"_path":594,"children":595},"Integrations","\u002Fen\u002Fdocs\u002Fintegrations",[596,599,602,605,608,611,614,617,620,623,626,629,632,635,638,641,644,647,650],{"title":597,"_path":598},"Integrations overview","\u002Fen\u002Fdocs\u002Fintegrations\u002Fvue-densemble",{"title":600,"_path":601},"Stripe","\u002Fen\u002Fdocs\u002Fintegrations\u002Fstripe",{"title":603,"_path":604},"GitHub","\u002Fen\u002Fdocs\u002Fintegrations\u002Fgithub",{"title":606,"_path":607},"Google","\u002Fen\u002Fdocs\u002Fintegrations\u002Fgoogle",{"title":609,"_path":610},"LinkedIn Ads","\u002Fen\u002Fdocs\u002Fintegrations\u002Flinkedin-ads",{"title":612,"_path":613},"Kavkom","\u002Fen\u002Fdocs\u002Fintegrations\u002Fkavkom",{"title":615,"_path":616},"Jira","\u002Fen\u002Fdocs\u002Fintegrations\u002Fjira",{"title":618,"_path":619},"Slack","\u002Fen\u002Fdocs\u002Fintegrations\u002Fslack",{"title":621,"_path":622},"SMS","\u002Fen\u002Fdocs\u002Fintegrations\u002Fsms",{"title":624,"_path":625},"Enable Banking (open banking)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fenable-banking",{"title":627,"_path":628},"CRM, marketing & productivity","\u002Fen\u002Fdocs\u002Fintegrations\u002Fcrm-marketing",{"title":630,"_path":631},"HeyGen (video)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fheygen",{"title":633,"_path":634},"La Poste","\u002Fen\u002Fdocs\u002Fintegrations\u002Fla-poste",{"title":636,"_path":637},"Industry CRM (beTool Application, Qhare)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fcrm-metier",{"title":639,"_path":640},"Mitti (field inspections)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fmitti",{"title":642,"_path":643},"Meta (Messenger, Pages, Meta Ads)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fmeta",{"title":645,"_path":646},"Onoff Business (Beta)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fonoff",{"title":648,"_path":649},"Smart Life \u002F Tuya (connected devices)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fsmart-life",{"title":651,"_path":652},"beTool products (Desktop, Kiosk, Terminal)","\u002Fen\u002Fdocs\u002Fintegrations\u002Fproduits-betool",{"title":654,"_path":655,"children":656},"Use cases","\u002Fen\u002Fdocs\u002Fcas-usage",[657,660,663,666,669,672],{"title":658,"_path":659},"Typical business agents","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fvue-densemble",{"title":661,"_path":662},"CFO IA","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fcfo-ia",{"title":664,"_path":665},"COO IA","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fcoo-ia",{"title":667,"_path":668},"CTO IA","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fcto-ia",{"title":670,"_path":671},"Product Manager IA","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fproduct-manager-ia",{"title":673,"_path":674},"Personal assistant","\u002Fen\u002Fdocs\u002Fcas-usage\u002Fassistant-personnel"]