[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"doc-\u002Fru\u002Fdocs\u002Fsupervision\u002Fbenchmarks":3,"docs-nav-ru":468},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"body":10,"_type":462,"_id":463,"_source":464,"_file":465,"_stem":466,"_extension":467},"\u002Fru\u002Fdocs\u002Fsupervision\u002Fbenchmarks","supervision",false,"","Бенчмарки","Измеряйте качество pipeline с помощью воспроизводимых сценариев. Выявляйте регрессии до продакшена.",{"type":11,"children":12,"toc":453},"root",[13,34,41,46,66,71,77,82,105,111,267,272,278,303,332,342,348,367,372,404,410],{"type":14,"tag":15,"props":16,"children":17},"element","p",{},[18,25,27,32],{"type":14,"tag":19,"props":20,"children":21},"strong",{},[22],{"type":23,"value":24},"text","Бенчмарк",{"type":23,"value":26}," — это набор сценариев оценки, автоматически воспроизводимых на pipeline. Это инструмент для ",{"type":14,"tag":19,"props":28,"children":29},{},[30],{"type":23,"value":31},"объективного",{"type":23,"value":33}," измерения качества агента и выявления регрессий до того, как они попадут в продакшен.",{"type":14,"tag":35,"props":36,"children":38},"h2",{"id":37},"зачем",[39],{"type":23,"value":40},"Зачем",{"type":14,"tag":15,"props":42,"children":43},{},[44],{"type":23,"value":45},"Агент, который «вроде бы работал», может незаметно деградировать после:",{"type":14,"tag":47,"props":48,"children":49},"ul",{},[50,56,61],{"type":14,"tag":51,"props":52,"children":53},"li",{},[54],{"type":23,"value":55},"изменения миссии;",{"type":14,"tag":51,"props":57,"children":58},{},[59],{"type":23,"value":60},"смены модели LLM;",{"type":14,"tag":51,"props":62,"children":63},{},[64],{"type":23,"value":65},"добавления нового tool.",{"type":14,"tag":15,"props":67,"children":68},{},[69],{"type":23,"value":70},"Без бенчмарка вы обнаружите это в продакшене. С бенчмарком вы увидите это при следующем запуске — раньше ваших пользователей.",{"type":14,"tag":35,"props":72,"children":74},{"id":73},"анатомия-сценария",[75],{"type":23,"value":76},"Анатомия сценария",{"type":14,"tag":15,"props":78,"children":79},{},[80],{"type":23,"value":81},"Сценарий содержит:",{"type":14,"tag":47,"props":83,"children":84},{},[85,95],{"type":14,"tag":51,"props":86,"children":87},{},[88,93],{"type":14,"tag":19,"props":89,"children":90},{},[91],{"type":23,"value":92},"Стимул",{"type":23,"value":94}," — входные данные, которые получает pipeline (сообщение или последовательность реплик, фикстуры данных).",{"type":14,"tag":51,"props":96,"children":97},{},[98,103],{"type":14,"tag":19,"props":99,"children":100},{},[101],{"type":23,"value":102},"Проверки",{"type":23,"value":104}," — явные утверждения, оцениваемые на каждой реплике: произведённый вывод, форма ответа, цепочка вызовов tools, стоимость действий.",{"type":14,"tag":35,"props":106,"children":108},{"id":107},"каталог-проверок",[109],{"type":23,"value":110},"Каталог проверок",{"type":14,"tag":112,"props":113,"children":114},"table",{},[115,134],{"type":14,"tag":116,"props":117,"children":118},"thead",{},[119],{"type":14,"tag":120,"props":121,"children":122},"tr",{},[123,129],{"type":14,"tag":124,"props":125,"children":126},"th",{},[127],{"type":23,"value":128},"Проверка",{"type":14,"tag":124,"props":130,"children":131},{},[132],{"type":23,"value":133},"Проверяет",{"type":14,"tag":135,"props":136,"children":137},"tbody",{},[138,157,179,196,218,235],{"type":14,"tag":120,"props":139,"children":140},{},[141,152],{"type":14,"tag":142,"props":143,"children":144},"td",{},[145],{"type":14,"tag":146,"props":147,"children":149},"code",{"className":148},[],[150],{"type":23,"value":151},"pipeline_output_match",{"type":14,"tag":142,"props":153,"children":154},{},[155],{"type":23,"value":156},"Ключ выходного потока соответствует ожидаемому значению (путь + утверждение)",{"type":14,"tag":120,"props":158,"children":159},{},[160,169],{"type":14,"tag":142,"props":161,"children":162},{},[163],{"type":14,"tag":146,"props":164,"children":166},{"className":165},[],[167],{"type":23,"value":168},"response_shape",{"type":14,"tag":142,"props":170,"children":171},{},[172,177],{"type":14,"tag":19,"props":173,"children":174},{},[175],{"type":23,"value":176},"Форму",{"type":23,"value":178}," ответа агента (длина, структура), измеряемую без судьи",{"type":14,"tag":120,"props":180,"children":181},{},[182,191],{"type":14,"tag":142,"props":183,"children":184},{},[185],{"type":14,"tag":146,"props":186,"children":188},{"className":187},[],[189],{"type":23,"value":190},"llm_rubric",{"type":14,"tag":142,"props":192,"children":193},{},[194],{"type":23,"value":195},"Содержание ответа, оцениваемое LLM-судьёй по рубрике проверяемых критериев",{"type":14,"tag":120,"props":197,"children":198},{},[199,208],{"type":14,"tag":142,"props":200,"children":201},{},[202],{"type":14,"tag":146,"props":203,"children":205},{"className":204},[],[206],{"type":23,"value":207},"tool_chain_analysis",{"type":14,"tag":142,"props":209,"children":210},{},[211,216],{"type":14,"tag":19,"props":212,"children":213},{},[214],{"type":23,"value":215},"Последовательность",{"type":23,"value":217}," вызовов tools следует ожидаемой стратегии",{"type":14,"tag":120,"props":219,"children":220},{},[221,230],{"type":14,"tag":142,"props":222,"children":223},{},[224],{"type":14,"tag":146,"props":225,"children":227},{"className":226},[],[228],{"type":23,"value":229},"tool_cost",{"type":14,"tag":142,"props":231,"children":232},{},[233],{"type":23,"value":234},"Бюджет действий: каждый вызов имеет стоимость, достижение цели приносит бонус",{"type":14,"tag":120,"props":236,"children":237},{},[238,262],{"type":14,"tag":142,"props":239,"children":240},{},[241,247,249,255,256],{"type":14,"tag":146,"props":242,"children":244},{"className":243},[],[245],{"type":23,"value":246},"static",{"type":23,"value":248}," \u002F ",{"type":14,"tag":146,"props":250,"children":252},{"className":251},[],[253],{"type":23,"value":254},"tests",{"type":23,"value":248},{"type":14,"tag":146,"props":257,"children":259},{"className":258},[],[260],{"type":23,"value":261},"diff_quality",{"type":14,"tag":142,"props":263,"children":264},{},[265],{"type":23,"value":266},"Проверки, ориентированные на файлы и код (вывод pipeline разработки)",{"type":14,"tag":15,"props":268,"children":269},{},[270],{"type":23,"value":271},"Каталог расширяем на стороне платформы: каждая проверка — это плагин, зарегистрированный на стороне сервера.",{"type":14,"tag":35,"props":273,"children":275},{"id":274},"создание-бенчмарка",[276],{"type":23,"value":277},"Создание бенчмарка",{"type":14,"tag":279,"props":280,"children":281},"ol",{},[282,292],{"type":14,"tag":51,"props":283,"children":284},{},[285,290],{"type":14,"tag":19,"props":286,"children":287},{},[288],{"type":23,"value":289},"Приложения → Оценка → Сценарии",{"type":23,"value":291},".",{"type":14,"tag":51,"props":293,"children":294},{},[295,297,302],{"type":23,"value":296},"Создайте сценарии (стимул, проверки, фикстуры данных); каждый сценарий привязан к одной модели (целевому pipeline). Фикстуры управляются во вкладке ",{"type":14,"tag":19,"props":298,"children":299},{},[300],{"type":23,"value":301},"Фиксчеры",{"type":23,"value":291},{"type":14,"tag":15,"props":304,"children":305},{},[306,308,315,317,323,324,330],{"type":23,"value":307},"Сценарий также можно создать или изменить через предложение с ",{"type":14,"tag":309,"props":310,"children":312},"a",{"href":311},"\u002Fru\u002Fdocs\u002Fdeveloppeurs\u002Fmcp",[313],{"type":23,"value":314},"MCP-сервера",{"type":23,"value":316}," (",{"type":14,"tag":146,"props":318,"children":320},{"className":319},[],[321],{"type":23,"value":322},"benchmark_create",{"type":23,"value":248},{"type":14,"tag":146,"props":325,"children":327},{"className":326},[],[328],{"type":23,"value":329},"benchmark_update",{"type":23,"value":331},"); оно остаётся в ожидании до принятия.",{"type":14,"tag":333,"props":334,"children":336},"alert",{"type":335},"info",[337],{"type":14,"tag":15,"props":338,"children":339},{},[340],{"type":23,"value":341},"Хороший бенчмарк включает разнообразие: штатный случай, граничные случаи, состязательные случаи (попытки манипуляции, неоднозначность, отсутствующие данные).",{"type":14,"tag":35,"props":343,"children":345},{"id":344},"запуск-бенчмарка",[346],{"type":23,"value":347},"Запуск бенчмарка",{"type":14,"tag":15,"props":349,"children":350},{},[351,353,358,360,365],{"type":23,"value":352},"Из панели администрирования запустите ",{"type":14,"tag":19,"props":354,"children":355},{},[356],{"type":23,"value":357},"прогон",{"type":23,"value":359},": он помещается в надёжную очередь, а затем выполняется выделенным worker, вне веб-сервера. Если этот worker останавливается посреди прогона, прогон помечается как ",{"type":14,"tag":19,"props":361,"children":362},{},[363],{"type":23,"value":364},"неудачный",{"type":23,"value":366}," при перезапуске (он не возобновляется автоматически): запустите его заново. Панель показывает детализацию по каждому сценарию.",{"type":14,"tag":15,"props":368,"children":369},{},[370],{"type":23,"value":371},"Отчёт показывает:",{"type":14,"tag":47,"props":373,"children":374},{},[375,380,399],{"type":14,"tag":51,"props":376,"children":377},{},[378],{"type":23,"value":379},"Долю успешных результатов по каждой проверке",{"type":14,"tag":51,"props":381,"children":382},{},[383,385,390,392,397],{"type":23,"value":384},"Динамику оценок между прогонами (вкладки ",{"type":14,"tag":19,"props":386,"children":387},{},[388],{"type":23,"value":389},"История",{"type":23,"value":391}," и ",{"type":14,"tag":19,"props":393,"children":394},{},[395],{"type":23,"value":396},"Динамика",{"type":23,"value":398},")",{"type":14,"tag":51,"props":400,"children":401},{},[402],{"type":23,"value":403},"Список проваленных сценариев с детализацией по каждой проверке",{"type":14,"tag":35,"props":405,"children":407},{"id":406},"лучшие-практики",[408],{"type":23,"value":409},"Лучшие практики",{"type":14,"tag":47,"props":411,"children":412},{},[413,423,433,443],{"type":14,"tag":51,"props":414,"children":415},{},[416,421],{"type":14,"tag":19,"props":417,"children":418},{},[419],{"type":23,"value":420},"Начинайте с малого",{"type":23,"value":422}," — 5 точно нацеленных сценариев ценнее, чем 50 приблизительных.",{"type":14,"tag":51,"props":424,"children":425},{},[426,431],{"type":14,"tag":19,"props":427,"children":428},{},[429],{"type":23,"value":430},"Прогоняйте бенчмарк перед каждой публикацией",{"type":23,"value":432}," и сравнивайте с предыдущим прогоном перед выкладкой.",{"type":14,"tag":51,"props":434,"children":435},{},[436,441],{"type":14,"tag":19,"props":437,"children":438},{},[439],{"type":23,"value":440},"Версионируйте свои миссии.",{"type":23,"value":442}," Не только pipelines: миссии тоже эволюционируют.",{"type":14,"tag":51,"props":444,"children":445},{},[446,451],{"type":14,"tag":19,"props":447,"children":448},{},[449],{"type":23,"value":450},"Держите проверки читаемыми.",{"type":23,"value":452}," Проверка, для понимания которой нужно 20 строк объяснений, — это проверка, которую нужно разбить.",{"title":7,"searchDepth":454,"depth":454,"links":455},2,[456,457,458,459,460,461],{"id":37,"depth":454,"text":40},{"id":73,"depth":454,"text":76},{"id":107,"depth":454,"text":110},{"id":274,"depth":454,"text":277},{"id":344,"depth":454,"text":347},{"id":406,"depth":454,"text":409},"markdown","content:ru:docs:4.supervision:3.benchmarks.md","content","ru\u002Fdocs\u002F4.supervision\u002F3.benchmarks.md","ru\u002Fdocs\u002F4.supervision\u002F3.benchmarks","md",[469],{"title":470,"_path":471,"children":472},"Ru","\u002Fru",[473],{"title":474,"_path":475,"children":476},"Docs","\u002Fru\u002Fdocs",[477,490,518,546,557,573,585,646],{"title":478,"_path":479,"children":480},"Начало работы","\u002Fru\u002Fdocs\u002Fdemarrer",[481,484,487],{"title":482,"_path":483},"Добро пожаловать","\u002Fru\u002Fdocs\u002Fdemarrer\u002Fbienvenue",{"title":485,"_path":486},"Создание учётной записи","\u002Fru\u002Fdocs\u002Fdemarrer\u002Fcreer-un-compte",{"title":488,"_path":489},"Подключение ключа LLM","\u002Fru\u002Fdocs\u002Fdemarrer\u002Fbrancher-llm",{"title":491,"_path":492,"children":493},"Каналы","\u002Fru\u002Fdocs\u002Fcanaux",[494,497,500,503,506,509,512,515],{"title":495,"_path":496},"Обзор каналов","\u002Fru\u002Fdocs\u002Fcanaux\u002Fvue-densemble",{"title":498,"_path":499},"Телефония в реальном времени","\u002Fru\u002Fdocs\u002Fcanaux\u002Ftelephone",{"title":501,"_path":502},"Электронная почта","\u002Fru\u002Fdocs\u002Fcanaux\u002Femail",{"title":504,"_path":505},"Веб-чат-виджет","\u002Fru\u002Fdocs\u002Fcanaux\u002Fchat-widget",{"title":507,"_path":508},"HTTP Webhook","\u002Fru\u002Fdocs\u002Fcanaux\u002Fwebhook",{"title":510,"_path":511},"Мессенджеры (WhatsApp, Messenger, Telegram, Mattermost)","\u002Fru\u002Fdocs\u002Fcanaux\u002Fmessageries",{"title":513,"_path":514},"Meta Lead Ads","\u002Fru\u002Fdocs\u002Fcanaux\u002Ffacebook-leads",{"title":516,"_path":517},"Расписание (cron)","\u002Fru\u002Fdocs\u002Fcanaux\u002Fschedule",{"title":519,"_path":520,"children":521},"Пайплайны","\u002Fru\u002Fdocs\u002Fpipelines",[522,525,528,531,534,537,540,543],{"title":523,"_path":524},"Визуальный редактор","\u002Fru\u002Fdocs\u002Fpipelines\u002Fediteur-visuel",{"title":526,"_path":527},"AI-агенты","\u002Fru\u002Fdocs\u002Fpipelines\u002Fagents",{"title":529,"_path":530},"Детерминированные операторы","\u002Fru\u002Fdocs\u002Fpipelines\u002Foperateurs",{"title":532,"_path":533},"Управление потоком","\u002Fru\u002Fdocs\u002Fpipelines\u002Fcontrole-flux",{"title":535,"_path":536},"Шаги с кодом","\u002Fru\u002Fdocs\u002Fpipelines\u002Fcode-step",{"title":538,"_path":539},"Суб-пайплайны","\u002Fru\u002Fdocs\u002Fpipelines\u002Fsous-pipelines",{"title":541,"_path":542},"No-code действия","\u002Fru\u002Fdocs\u002Fpipelines\u002Factions-no-code",{"title":544,"_path":545},"База знаний","\u002Fru\u002Fdocs\u002Fpipelines\u002Fbase-de-connaissances",{"title":547,"_path":548,"children":549},"Супервизия","\u002Fru\u002Fdocs\u002Fsupervision",[550,553,556],{"title":551,"_path":552},"История выполнений","\u002Fru\u002Fdocs\u002Fsupervision\u002Fexecutions",{"title":554,"_path":555},"Кредиты и тарификация","\u002Fru\u002Fdocs\u002Fsupervision\u002Fcredits-metering",{"title":8,"_path":4},{"title":558,"_path":559,"children":560},"Безопасность","\u002Fru\u002Fdocs\u002Fsecurite",[561,564,567,570],{"title":562,"_path":563},"Принципы безопасности","\u002Fru\u002Fdocs\u002Fsecurite\u002Fprincipes",{"title":565,"_path":566},"Строгая мультитенантность","\u002Fru\u002Fdocs\u002Fsecurite\u002Fmulti-tenant",{"title":568,"_path":569},"Аудит и журнал доступа GDPR","\u002Fru\u002Fdocs\u002Fsecurite\u002Faudit-rgpd",{"title":571,"_path":572},"Суверенное развёртывание","\u002Fru\u002Fdocs\u002Fsecurite\u002Fdeploiement-souverain",{"title":574,"_path":575,"children":576},"Разработчикам","\u002Fru\u002Fdocs\u002Fdeveloppeurs",[577,580,583],{"title":578,"_path":579},"Административный API","\u002Fru\u002Fdocs\u002Fdeveloppeurs\u002Fapi-admin",{"title":581,"_path":582},"Исходящие webhook","\u002Fru\u002Fdocs\u002Fdeveloppeurs\u002Fwebhooks-sortants",{"title":584,"_path":311},"Управление betool из Claude (MCP)",{"title":586,"_path":587,"children":588},"Интеграции","\u002Fru\u002Fdocs\u002Fintegrations",[589,592,595,598,601,604,607,610,613,616,619,622,625,628,631,634,637,640,643],{"title":590,"_path":591},"Обзор интеграций","\u002Fru\u002Fdocs\u002Fintegrations\u002Fvue-densemble",{"title":593,"_path":594},"Stripe","\u002Fru\u002Fdocs\u002Fintegrations\u002Fstripe",{"title":596,"_path":597},"GitHub","\u002Fru\u002Fdocs\u002Fintegrations\u002Fgithub",{"title":599,"_path":600},"Google","\u002Fru\u002Fdocs\u002Fintegrations\u002Fgoogle",{"title":602,"_path":603},"LinkedIn Ads","\u002Fru\u002Fdocs\u002Fintegrations\u002Flinkedin-ads",{"title":605,"_path":606},"Kavkom","\u002Fru\u002Fdocs\u002Fintegrations\u002Fkavkom",{"title":608,"_path":609},"Jira","\u002Fru\u002Fdocs\u002Fintegrations\u002Fjira",{"title":611,"_path":612},"Slack","\u002Fru\u002Fdocs\u002Fintegrations\u002Fslack",{"title":614,"_path":615},"SMS","\u002Fru\u002Fdocs\u002Fintegrations\u002Fsms",{"title":617,"_path":618},"Enable Banking (open banking)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fenable-banking",{"title":620,"_path":621},"CRM, маркетинг и продуктивность","\u002Fru\u002Fdocs\u002Fintegrations\u002Fcrm-marketing",{"title":623,"_path":624},"HeyGen (видео)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fheygen",{"title":626,"_path":627},"La Poste","\u002Fru\u002Fdocs\u002Fintegrations\u002Fla-poste",{"title":629,"_path":630},"Отраслевые CRM (beTool Application, Qhare)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fcrm-metier",{"title":632,"_path":633},"Mitti (полевые инспекции)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fmitti",{"title":635,"_path":636},"Meta (Messenger, Pages, Meta Ads)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fmeta",{"title":638,"_path":639},"Onoff Business (Beta)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fonoff",{"title":641,"_path":642},"Smart Life \u002F Tuya (умные устройства)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fsmart-life",{"title":644,"_path":645},"Продукты beTool (Desktop, Kiosk, Terminal)","\u002Fru\u002Fdocs\u002Fintegrations\u002Fproduits-betool",{"title":647,"_path":648,"children":649},"Сценарии использования","\u002Fru\u002Fdocs\u002Fcas-usage",[650,653,656,659,662,665],{"title":651,"_path":652},"Типовые бизнес-агенты","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fvue-densemble",{"title":654,"_path":655},"CFO ИИ","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fcfo-ia",{"title":657,"_path":658},"COO ИИ","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fcoo-ia",{"title":660,"_path":661},"CTO ИИ","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fcto-ia",{"title":663,"_path":664},"Product Manager ИИ","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fproduct-manager-ia",{"title":666,"_path":667},"Персональный ассистент","\u002Fru\u002Fdocs\u002Fcas-usage\u002Fassistant-personnel"]