[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-\u002Fde\u002Fdocs\u002Fsupervision\u002Fbenchmarks":3,"docs-nav-de":476},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"body":10,"_type":470,"_id":471,"_source":472,"_file":473,"_stem":474,"_extension":475},"\u002Fde\u002Fdocs\u002Fsupervision\u002Fbenchmarks","supervision",false,"","Benchmarks","Die Qualität einer Pipeline mit reproduzierbaren Szenarien messen. Regressionen vor der Produktion erkennen.",{"type":11,"children":12,"toc":461},"root",[13,36,43,48,68,73,79,84,109,115,274,279,285,311,340,350,356,375,380,412,418],{"type":14,"tag":15,"props":16,"children":17},"element","p",{},[18,21,27,29,34],{"type":19,"value":20},"text","Ein ",{"type":14,"tag":22,"props":23,"children":24},"strong",{},[25],{"type":19,"value":26},"Benchmark",{"type":19,"value":28}," ist ein Satz von Evaluierungsszenarien, die automatisch gegen eine Pipeline abgespielt werden. Es ist das Werkzeug, um die Qualität eines Agents ",{"type":14,"tag":22,"props":30,"children":31},{},[32],{"type":19,"value":33},"objektiv",{"type":19,"value":35}," zu messen und Regressionen zu erkennen, bevor sie die Produktion erreichen.",{"type":14,"tag":37,"props":38,"children":40},"h2",{"id":39},"warum",[41],{"type":19,"value":42},"Warum",{"type":14,"tag":15,"props":44,"children":45},{},[46],{"type":19,"value":47},"Ein Agent, der „zu funktionieren schien“, kann still regressieren nach:",{"type":14,"tag":49,"props":50,"children":51},"ul",{},[52,58,63],{"type":14,"tag":53,"props":54,"children":55},"li",{},[56],{"type":19,"value":57},"einer Missionsänderung;",{"type":14,"tag":53,"props":59,"children":60},{},[61],{"type":19,"value":62},"einem Wechsel des LLM-Modells;",{"type":14,"tag":53,"props":64,"children":65},{},[66],{"type":19,"value":67},"dem Hinzufügen eines neuen Tools.",{"type":14,"tag":15,"props":69,"children":70},{},[71],{"type":19,"value":72},"Ohne Benchmark entdecken Sie es in der Produktion. Mit einem Benchmark sehen Sie es beim nächsten Lauf — vor Ihren Benutzern.",{"type":14,"tag":37,"props":74,"children":76},{"id":75},"anatomie-eines-szenarios",[77],{"type":19,"value":78},"Anatomie eines Szenarios",{"type":14,"tag":15,"props":80,"children":81},{},[82],{"type":19,"value":83},"Ein Szenario enthält:",{"type":14,"tag":49,"props":85,"children":86},{},[87,99],{"type":14,"tag":53,"props":88,"children":89},{},[90,92,97],{"type":19,"value":91},"Einen ",{"type":14,"tag":22,"props":93,"children":94},{},[95],{"type":19,"value":96},"Stimulus",{"type":19,"value":98}," — die Eingabe, die die Pipeline erhält (Nachricht oder Folge von Schritten, Daten-Fixtures).",{"type":14,"tag":53,"props":100,"children":101},{},[102,107],{"type":14,"tag":22,"props":103,"children":104},{},[105],{"type":19,"value":106},"Checks",{"type":19,"value":108}," — explizite Zusicherungen, die bei jedem Schritt ausgewertet werden: die erzeugte Ausgabe, die Form der Antwort, die Kette der Tool-Aufrufe, die Kosten der Aktionen.",{"type":14,"tag":37,"props":110,"children":112},{"id":111},"katalog-der-checks",[113],{"type":19,"value":114},"Katalog der Checks",{"type":14,"tag":116,"props":117,"children":118},"table",{},[119,138],{"type":14,"tag":120,"props":121,"children":122},"thead",{},[123],{"type":14,"tag":124,"props":125,"children":126},"tr",{},[127,133],{"type":14,"tag":128,"props":129,"children":130},"th",{},[131],{"type":19,"value":132},"Check",{"type":14,"tag":128,"props":134,"children":135},{},[136],{"type":19,"value":137},"Prüft",{"type":14,"tag":139,"props":140,"children":141},"tbody",{},[142,161,185,202,225,242],{"type":14,"tag":124,"props":143,"children":144},{},[145,156],{"type":14,"tag":146,"props":147,"children":148},"td",{},[149],{"type":14,"tag":150,"props":151,"children":153},"code",{"className":152},[],[154],{"type":19,"value":155},"pipeline_output_match",{"type":14,"tag":146,"props":157,"children":158},{},[159],{"type":19,"value":160},"Ein Schlüssel des Ausgabeflusses entspricht dem erwarteten Wert (Pfad + Zusicherung)",{"type":14,"tag":124,"props":162,"children":163},{},[164,173],{"type":14,"tag":146,"props":165,"children":166},{},[167],{"type":14,"tag":150,"props":168,"children":170},{"className":169},[],[171],{"type":19,"value":172},"response_shape",{"type":14,"tag":146,"props":174,"children":175},{},[176,178,183],{"type":19,"value":177},"Die ",{"type":14,"tag":22,"props":179,"children":180},{},[181],{"type":19,"value":182},"Form",{"type":19,"value":184}," der Agent-Antwort (Länge, Struktur), ohne Richter gemessen",{"type":14,"tag":124,"props":186,"children":187},{},[188,197],{"type":14,"tag":146,"props":189,"children":190},{},[191],{"type":14,"tag":150,"props":192,"children":194},{"className":193},[],[195],{"type":19,"value":196},"llm_rubric",{"type":14,"tag":146,"props":198,"children":199},{},[200],{"type":19,"value":201},"Den Inhalt der Antwort, bewertet von einem LLM-Richter anhand einer Rubrik überprüfbarer Kriterien",{"type":14,"tag":124,"props":203,"children":204},{},[205,214],{"type":14,"tag":146,"props":206,"children":207},{},[208],{"type":14,"tag":150,"props":209,"children":211},{"className":210},[],[212],{"type":19,"value":213},"tool_chain_analysis",{"type":14,"tag":146,"props":215,"children":216},{},[217,218,223],{"type":19,"value":177},{"type":14,"tag":22,"props":219,"children":220},{},[221],{"type":19,"value":222},"Abfolge",{"type":19,"value":224}," der Tool-Aufrufe folgt der erwarteten Strategie",{"type":14,"tag":124,"props":226,"children":227},{},[228,237],{"type":14,"tag":146,"props":229,"children":230},{},[231],{"type":14,"tag":150,"props":232,"children":234},{"className":233},[],[235],{"type":19,"value":236},"tool_cost",{"type":14,"tag":146,"props":238,"children":239},{},[240],{"type":19,"value":241},"Das Aktionsbudget: Jeder Aufruf hat Kosten, das Erreichen des Ziels bringt einen Bonus",{"type":14,"tag":124,"props":243,"children":244},{},[245,269],{"type":14,"tag":146,"props":246,"children":247},{},[248,254,256,262,263],{"type":14,"tag":150,"props":249,"children":251},{"className":250},[],[252],{"type":19,"value":253},"static",{"type":19,"value":255}," \u002F ",{"type":14,"tag":150,"props":257,"children":259},{"className":258},[],[260],{"type":19,"value":261},"tests",{"type":19,"value":255},{"type":14,"tag":150,"props":264,"children":266},{"className":265},[],[267],{"type":19,"value":268},"diff_quality",{"type":14,"tag":146,"props":270,"children":271},{},[272],{"type":19,"value":273},"Datei- und codeorientierte Checks (Ausgabe einer Entwicklungs-Pipeline)",{"type":14,"tag":15,"props":275,"children":276},{},[277],{"type":19,"value":278},"Der Katalog ist plattformseitig erweiterbar: Jeder Check ist ein serverseitig registriertes Plugin.",{"type":14,"tag":37,"props":280,"children":282},{"id":281},"einen-benchmark-erstellen",[283],{"type":19,"value":284},"Einen Benchmark erstellen",{"type":14,"tag":286,"props":287,"children":288},"ol",{},[289,299],{"type":14,"tag":53,"props":290,"children":291},{},[292,297],{"type":14,"tag":22,"props":293,"children":294},{},[295],{"type":19,"value":296},"Anwendungen → Evaluierung → Szenarien",{"type":19,"value":298},".",{"type":14,"tag":53,"props":300,"children":301},{},[302,304,309],{"type":19,"value":303},"Erstellen Sie die Szenarien (Stimulus, Checks, Daten-Fixtures); jedes Szenario ist einem Modell (der Ziel-Pipeline) zugeordnet. Die Fixtures werden im Reiter ",{"type":14,"tag":22,"props":305,"children":306},{},[307],{"type":19,"value":308},"Fixtures",{"type":19,"value":310}," verwaltet.",{"type":14,"tag":15,"props":312,"children":313},{},[314,316,323,325,331,332,338],{"type":19,"value":315},"Ein Szenario kann auch per Vorschlag über den ",{"type":14,"tag":317,"props":318,"children":320},"a",{"href":319},"\u002Fde\u002Fdocs\u002Fdeveloppeurs\u002Fmcp",[321],{"type":19,"value":322},"MCP-Server",{"type":19,"value":324}," erstellt oder geändert werden (",{"type":14,"tag":150,"props":326,"children":328},{"className":327},[],[329],{"type":19,"value":330},"benchmark_create",{"type":19,"value":255},{"type":14,"tag":150,"props":333,"children":335},{"className":334},[],[336],{"type":19,"value":337},"benchmark_update",{"type":19,"value":339},"); der Vorschlag bleibt ausstehend, bis er angenommen wird.",{"type":14,"tag":341,"props":342,"children":344},"alert",{"type":343},"info",[345],{"type":14,"tag":15,"props":346,"children":347},{},[348],{"type":19,"value":349},"Ein guter Benchmark enthält eine Mischung: Normalfall, Randfälle, adversariale Fälle (Manipulationsversuche, Mehrdeutigkeit, fehlende Daten).",{"type":14,"tag":37,"props":351,"children":353},{"id":352},"einen-benchmark-ausführen",[354],{"type":19,"value":355},"Einen Benchmark ausführen",{"type":14,"tag":15,"props":357,"children":358},{},[359,361,366,368,373],{"type":19,"value":360},"Starten Sie im Adminbereich einen ",{"type":14,"tag":22,"props":362,"children":363},{},[364],{"type":19,"value":365},"Lauf",{"type":19,"value":367}," (Run): Er wird in eine dauerhafte Warteschlange gestellt und dann von einem dedizierten Worker außerhalb des Webservers ausgeführt. Stoppt dieser Worker mitten im Run, wird der Run beim Neustart als ",{"type":14,"tag":22,"props":369,"children":370},{},[371],{"type":19,"value":372},"fehlgeschlagen",{"type":19,"value":374}," markiert (er wird nicht automatisch fortgesetzt): Starten Sie ihn erneut. Ein Drawer zeigt das Detail pro Szenario.",{"type":14,"tag":15,"props":376,"children":377},{},[378],{"type":19,"value":379},"Der Bericht zeigt:",{"type":14,"tag":49,"props":381,"children":382},{},[383,388,407],{"type":14,"tag":53,"props":384,"children":385},{},[386],{"type":19,"value":387},"Erfolgsquote pro Check",{"type":14,"tag":53,"props":389,"children":390},{},[391,393,398,400,405],{"type":19,"value":392},"Entwicklung der Scores zwischen den Läufen (Reiter ",{"type":14,"tag":22,"props":394,"children":395},{},[396],{"type":19,"value":397},"Verlauf",{"type":19,"value":399}," und ",{"type":14,"tag":22,"props":401,"children":402},{},[403],{"type":19,"value":404},"Entwicklung",{"type":19,"value":406},")",{"type":14,"tag":53,"props":408,"children":409},{},[410],{"type":19,"value":411},"Liste der fehlgeschlagenen Szenarien, mit dem Detail jedes Checks",{"type":14,"tag":37,"props":413,"children":415},{"id":414},"bewährte-praktiken",[416],{"type":19,"value":417},"Bewährte Praktiken",{"type":14,"tag":49,"props":419,"children":420},{},[421,431,441,451],{"type":14,"tag":53,"props":422,"children":423},{},[424,429],{"type":14,"tag":22,"props":425,"children":426},{},[427],{"type":19,"value":428},"Klein anfangen",{"type":19,"value":430}," — 5 gut gezielte Szenarien sind mehr wert als 50 ungefähre.",{"type":14,"tag":53,"props":432,"children":433},{},[434,439],{"type":14,"tag":22,"props":435,"children":436},{},[437],{"type":19,"value":438},"Vor jeder Veröffentlichung benchmarken",{"type":19,"value":440}," und vor dem Ausrollen mit dem vorherigen Lauf vergleichen.",{"type":14,"tag":53,"props":442,"children":443},{},[444,449],{"type":14,"tag":22,"props":445,"children":446},{},[447],{"type":19,"value":448},"Missionen versionieren.",{"type":19,"value":450}," Nicht nur Pipelines: Auch Missionen entwickeln sich weiter.",{"type":14,"tag":53,"props":452,"children":453},{},[454,459],{"type":14,"tag":22,"props":455,"children":456},{},[457],{"type":19,"value":458},"Checks lesbar halten.",{"type":19,"value":460}," Ein Check, der 20 Zeilen Erklärung braucht, um verstanden zu werden, muss aufgeteilt werden.",{"title":7,"searchDepth":462,"depth":462,"links":463},2,[464,465,466,467,468,469],{"id":39,"depth":462,"text":42},{"id":75,"depth":462,"text":78},{"id":111,"depth":462,"text":114},{"id":281,"depth":462,"text":284},{"id":352,"depth":462,"text":355},{"id":414,"depth":462,"text":417},"markdown","content:de:docs:4.supervision:3.benchmarks.md","content","de\u002Fdocs\u002F4.supervision\u002F3.benchmarks.md","de\u002Fdocs\u002F4.supervision\u002F3.benchmarks","md",[477],{"title":478,"_path":479,"children":480},"De","\u002Fde",[481],{"title":482,"_path":483,"children":484},"Docs","\u002Fde\u002Fdocs",[485,498,526,554,565,581,593,654],{"title":486,"_path":487,"children":488},"Erste Schritte","\u002Fde\u002Fdocs\u002Fdemarrer",[489,492,495],{"title":490,"_path":491},"Willkommen","\u002Fde\u002Fdocs\u002Fdemarrer\u002Fbienvenue",{"title":493,"_path":494},"Ein Konto erstellen","\u002Fde\u002Fdocs\u002Fdemarrer\u002Fcreer-un-compte",{"title":496,"_path":497},"Einen LLM-Schlüssel anbinden","\u002Fde\u002Fdocs\u002Fdemarrer\u002Fbrancher-llm",{"title":499,"_path":500,"children":501},"Kanäle","\u002Fde\u002Fdocs\u002Fcanaux",[502,505,508,511,514,517,520,523],{"title":503,"_path":504},"Überblick über die Kanäle","\u002Fde\u002Fdocs\u002Fcanaux\u002Fvue-densemble",{"title":506,"_path":507},"Echtzeit-Telefonie","\u002Fde\u002Fdocs\u002Fcanaux\u002Ftelephone",{"title":509,"_path":510},"E-Mail","\u002Fde\u002Fdocs\u002Fcanaux\u002Femail",{"title":512,"_path":513},"Web-Chat-Widget","\u002Fde\u002Fdocs\u002Fcanaux\u002Fchat-widget",{"title":515,"_path":516},"HTTP-Webhook","\u002Fde\u002Fdocs\u002Fcanaux\u002Fwebhook",{"title":518,"_path":519},"Messaging (WhatsApp, Messenger, Telegram, Mattermost)","\u002Fde\u002Fdocs\u002Fcanaux\u002Fmessageries",{"title":521,"_path":522},"Meta Lead Ads","\u002Fde\u002Fdocs\u002Fcanaux\u002Ffacebook-leads",{"title":524,"_path":525},"Schedule (Cron)","\u002Fde\u002Fdocs\u002Fcanaux\u002Fschedule",{"title":527,"_path":528,"children":529},"Pipelines","\u002Fde\u002Fdocs\u002Fpipelines",[530,533,536,539,542,545,548,551],{"title":531,"_path":532},"Der visuelle Editor","\u002Fde\u002Fdocs\u002Fpipelines\u002Fediteur-visuel",{"title":534,"_path":535},"KI-Agents","\u002Fde\u002Fdocs\u002Fpipelines\u002Fagents",{"title":537,"_path":538},"Deterministische Operatoren","\u002Fde\u002Fdocs\u002Fpipelines\u002Foperateurs",{"title":540,"_path":541},"Flusssteuerung","\u002Fde\u002Fdocs\u002Fpipelines\u002Fcontrole-flux",{"title":543,"_path":544},"Code-Steps","\u002Fde\u002Fdocs\u002Fpipelines\u002Fcode-step",{"title":546,"_path":547},"Sub-Pipelines","\u002Fde\u002Fdocs\u002Fpipelines\u002Fsous-pipelines",{"title":549,"_path":550},"No-Code-Aktionen","\u002Fde\u002Fdocs\u002Fpipelines\u002Factions-no-code",{"title":552,"_path":553},"Wissensdatenbank","\u002Fde\u002Fdocs\u002Fpipelines\u002Fbase-de-connaissances",{"title":555,"_path":556,"children":557},"Überwachung","\u002Fde\u002Fdocs\u002Fsupervision",[558,561,564],{"title":559,"_path":560},"Ausführungsverlauf","\u002Fde\u002Fdocs\u002Fsupervision\u002Fexecutions",{"title":562,"_path":563},"Credits & Metering","\u002Fde\u002Fdocs\u002Fsupervision\u002Fcredits-metering",{"title":8,"_path":4},{"title":566,"_path":567,"children":568},"Sicherheit","\u002Fde\u002Fdocs\u002Fsecurite",[569,572,575,578],{"title":570,"_path":571},"Sicherheitsprinzipien","\u002Fde\u002Fdocs\u002Fsecurite\u002Fprincipes",{"title":573,"_path":574},"Strikte Mandantentrennung","\u002Fde\u002Fdocs\u002Fsecurite\u002Fmulti-tenant",{"title":576,"_path":577},"DSGVO-Audit & Zugriffsprotokoll","\u002Fde\u002Fdocs\u002Fsecurite\u002Faudit-rgpd",{"title":579,"_path":580},"Souveräne Bereitstellung","\u002Fde\u002Fdocs\u002Fsecurite\u002Fdeploiement-souverain",{"title":582,"_path":583,"children":584},"Entwickler","\u002Fde\u002Fdocs\u002Fdeveloppeurs",[585,588,591],{"title":586,"_path":587},"Admin-API","\u002Fde\u002Fdocs\u002Fdeveloppeurs\u002Fapi-admin",{"title":589,"_path":590},"Ausgehende Webhooks","\u002Fde\u002Fdocs\u002Fdeveloppeurs\u002Fwebhooks-sortants",{"title":592,"_path":319},"betool von Claude aus steuern (MCP)",{"title":594,"_path":595,"children":596},"Integrationen","\u002Fde\u002Fdocs\u002Fintegrations",[597,600,603,606,609,612,615,618,621,624,627,630,633,636,639,642,645,648,651],{"title":598,"_path":599},"Integrationen im Überblick","\u002Fde\u002Fdocs\u002Fintegrations\u002Fvue-densemble",{"title":601,"_path":602},"Stripe","\u002Fde\u002Fdocs\u002Fintegrations\u002Fstripe",{"title":604,"_path":605},"GitHub","\u002Fde\u002Fdocs\u002Fintegrations\u002Fgithub",{"title":607,"_path":608},"Google","\u002Fde\u002Fdocs\u002Fintegrations\u002Fgoogle",{"title":610,"_path":611},"LinkedIn Ads","\u002Fde\u002Fdocs\u002Fintegrations\u002Flinkedin-ads",{"title":613,"_path":614},"Kavkom","\u002Fde\u002Fdocs\u002Fintegrations\u002Fkavkom",{"title":616,"_path":617},"Jira","\u002Fde\u002Fdocs\u002Fintegrations\u002Fjira",{"title":619,"_path":620},"Slack","\u002Fde\u002Fdocs\u002Fintegrations\u002Fslack",{"title":622,"_path":623},"SMS","\u002Fde\u002Fdocs\u002Fintegrations\u002Fsms",{"title":625,"_path":626},"Enable Banking (Open Banking)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fenable-banking",{"title":628,"_path":629},"CRM, Marketing & Produktivität","\u002Fde\u002Fdocs\u002Fintegrations\u002Fcrm-marketing",{"title":631,"_path":632},"HeyGen (Video)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fheygen",{"title":634,"_path":635},"La Poste","\u002Fde\u002Fdocs\u002Fintegrations\u002Fla-poste",{"title":637,"_path":638},"Branchen-CRM (beTool Application, Qhare)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fcrm-metier",{"title":640,"_path":641},"Mitti (Vor-Ort-Inspektionen)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fmitti",{"title":643,"_path":644},"Meta (Messenger, Seiten, Meta Ads)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fmeta",{"title":646,"_path":647},"Onoff Business (Beta)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fonoff",{"title":649,"_path":650},"Smart Life \u002F Tuya (vernetzte Geräte)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fsmart-life",{"title":652,"_path":653},"beTool-Produkte (Desktop, Kiosk, Terminal)","\u002Fde\u002Fdocs\u002Fintegrations\u002Fproduits-betool",{"title":655,"_path":656,"children":657},"Anwendungsfälle","\u002Fde\u002Fdocs\u002Fcas-usage",[658,661,664,667,670,673],{"title":659,"_path":660},"Typische Fachagenten","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fvue-densemble",{"title":662,"_path":663},"CFO IA","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fcfo-ia",{"title":665,"_path":666},"COO IA","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fcoo-ia",{"title":668,"_path":669},"CTO IA","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fcto-ia",{"title":671,"_path":672},"Product Manager IA","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fproduct-manager-ia",{"title":674,"_path":675},"Persönlicher Assistent","\u002Fde\u002Fdocs\u002Fcas-usage\u002Fassistant-personnel"]