{"actions":{"bootstrapAndRun":{"effect":"installs rote if missing, inspects, prepares, and asks before running","href":"https://play.modiqo.ai/install?play=modiqo/evaluate-agentic-web-search@0.3.1","method":"GET","rel":"https://rote.dev/rels/bootstrap-and-run","requiresConsent":true,"responseMediaType":"text/x-shellscript"},"inspect":{"command":"rote play inspect https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1","effect":"read-only"},"installCliOnly":{"effect":"installs the rote CLI, nothing else","href":"https://play.modiqo.ai/install","method":"GET","rel":"https://rote.dev/rels/install-cli","requiresConsent":true,"responseMediaType":"text/x-shellscript"},"run":{"command":"rote play run https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1","effect":"executes the play locally after consent","headless":{"approvalAssertion":"--yes","approvalRequiredBeforeInvocation":true,"commandTemplate":"rote play run https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1 <name=value...> --yes","stdinPolicy":"never pipe input to automate the interactive Ready selector"},"requiresConsent":true}},"description":"Evaluates final agent answers across isolated web-search arms with strict pilot and publication gates, traceable grading, human audits, confidence intervals, statistical ties, and Pareto reports.","distribution":{"digest":"sha256:b8109412c81f7c6cb63f2ef1054941a778e5f0d312730bb2bdd82c10fb3c4a56","mediaType":"application/vnd.modiqo.rote-flow","size":54256,"verifiedBy":"rote verifies the downloaded archive against this digest before it runs"},"effects":{"credentialsProvidedBy":"runner","credentialsRemainLocal":true,"declaredWrites":[],"publisherReceivesCredentials":false},"id":"https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1","inputPolicy":{"optionalWithDefault":"show_default_and_accept_override","optionalWithoutDefault":"omit_unless_supplied","required":"ask","secrets":"collect_locally_outside_conversation"},"links":{"docs":"https://rote.dev","page":"https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1","protocol":"https://play.modiqo.ai/.well-known/rote","self":"https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1.json"},"name":"evaluate-agentic-web-search","owner":{"kind":"org","slug":"modiqo"},"parameters":[{"default":"run","description":"init creates a guided project, validate checks every input without provider calls, and run executes the evaluation","input":{"allowCustom":true,"choices":[],"label":"Action"},"name":"action","required":false,"type":"string"},{"description":"Guided project directory containing config.json, cases.jsonl, schemas, documentation, and dated run outputs","input":{"allowCustom":true,"choices":[],"label":"Project dir"},"name":"project_dir","required":false,"type":"string"},{"description":"Advanced override for config.json; project_dir supplies this path for the guided workflow","input":{"allowCustom":true,"choices":[],"label":"Config path"},"name":"config_path","required":false,"type":"string"},{"description":"Advanced override for cases.jsonl; each line follows the bundled case schema and project_dir supplies the default path","input":{"allowCustom":true,"choices":[],"label":"Gold set"},"name":"gold_set","required":false,"type":"string"},{"description":"Advanced output override for immutable artifacts; action=run otherwise creates a dated directory under project_dir/runs","input":{"allowCustom":true,"choices":[],"label":"Output dir"},"name":"output_dir","required":false,"type":"string"},{"default":"pilot","description":"pilot permits small wiring checks and labels them not for ranking; publication enforces at least 100 reviewed cases, 3 repeats, a 10 percent audit, and live frozen methodology","input":{"allowCustom":true,"choices":[],"label":"Run purpose"},"name":"run_purpose","required":false,"type":"string"},{"default":3,"description":"Independent runs per case and arm; accepted range 1 through 20, with at least 3 required for publication","input":{"allowCustom":true,"choices":[],"label":"Repeats"},"name":"repeats","required":false,"type":"integer"},{"default":3,"description":"Varied cases per arm used to verify wiring before the benchmark; accepted range 1 through 20","input":{"allowCustom":true,"choices":[],"label":"Pilot count"},"name":"pilot_count","required":false,"type":"integer"},{"default":"0.10","description":"Fraction of graded runs selected for human review; accepted range greater than 0 through 1, with at least 0.10 required for publication","input":{"allowCustom":true,"choices":[],"label":"Audit fraction"},"name":"audit_fraction","required":false,"type":"string"},{"default":2000,"description":"Bootstrap samples used for confidence intervals; accepted range 100 through 100000","input":{"allowCustom":true,"choices":[],"label":"Bootstrap iterations"},"name":"bootstrap_iterations","required":false,"type":"integer"},{"default":42,"description":"Deterministic seed for case selection, audit sampling, and confidence intervals","input":{"allowCustom":true,"choices":[],"label":"Seed"},"name":"seed","required":false,"type":"integer"},{"description":"Optional JSONL human-review decisions produced from the generated audit queue","input":{"allowCustom":true,"choices":[],"label":"Audit results"},"name":"audit_results","required":false,"type":"string"}],"preparation":[{"action":{"command":"rote play inspect https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1 --json","effect":"read-only"},"step":1,"type":"inspect_local_readiness"},{"references":["/parameters"],"step":2,"type":"collect_parameters"},{"references":["/parameters","/requirements","/effects"],"step":3,"type":"review"},{"consentBoundary":"the user approves the exact play and parameter values","references":["/parameters","/requirements","/effects"],"step":4,"type":"obtain_run_consent"},{"action":{"command":"rote play run https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1","headlessCommandTemplate":"rote play run https://play.modiqo.ai/modiqo/evaluate-agentic-web-search@0.3.1 <name=value...> --yes"},"preservesAcquisitionBoundaries":["adapter_selection","oauth_dcr","google_discovery","static_token_setup","runtime_security_checks"],"requiresConsent":true,"step":5,"type":"run"}],"producedBy":{"roteVersion":"0.77.0"},"publishedAt":"2026-09-01T06:01:32.835568+00:00","reference":"modiqo/evaluate-agentic-web-search@0.3.1","requirements":{"adapters":[],"browser":{"dependencies":[],"runtime":false,"signIn":false},"localTools":["python3","parallel-cli","codex"],"roteCli":{"minimumVersion":"0.62.0"},"sessions":false},"resolution":"pinned","schema":"rote.play.v1","stats":{"downloads":3,"installs":0},"steps":{"count":14,"names":["build_audit_queue","build_run_matrix","calculate_metrics","grade_answer","ingest_human_audit","inspect_config","inspect_gold_set","prepare_project","read_parallel_auth","read_parallel_version","render_report","run_benchmark_case","run_pilot_case","verify_pilot"]},"title":"evaluate-agentic-web-search","type":"play","version":"0.3.1","visibility":"public"}