Product Kit · 1.0.0
Evaluation Dataset QA Kit
Preflight one bounded evaluation dataset and compare submitted candidate and baseline outputs with deterministic exact-match evidence.
AI builders preparing agent or evaluation datasets before model runs without asking Utilito to grade semantic quality.
QA envelope
{
"ok": true,
"schema_version": "utilito.product-kit.v1",
"kit": {
"id": "evaluation-dataset-qa-kit",
"version": "1.0.0"
},
"result": {
"status": "review_required",
"normalized_jsonl": "{\"expected\":{\"answer\":4},\"id\":\"math-1\",\"input\":\"2+2?\",\"source\":{\"index\":0,\"line\":1}}\n{\"expected\":\"Paris\",\"id\":\"capital-1\",\"input\":\"Capital of France?\",\"source\":{\"index\":1,\"line\":2}}",
"records": [
{
"id": "math-1",
"input": "2+2?",
"expected": {
"answer": 4
},
"source": {
"index": 0,
"line": 1
}
},
{
"id": "capital-1",
"input": "Capital of France?",
"expected": "Paris",
"source": {
"index": 1,
"line": 2
}
}
],
"schema_diagnostics": [],
"duplicate_diagnostics": {
"policy": "remove",
"duplicate_occurrences": 0,
"removed": 0,
"retained": 0,
"groups": []
},
"leakage_diagnostics": [],
"comparison": {
"candidate": {
"summary": {
"exact_matches": 1,
"mismatches": 1,
"missing": 0,
"extras": 0,
"denominator": 2
},
"items": [
{
"id": "math-1",
"status": "exact_match",
"exact_match": true
},
{
"id": "capital-1",
"status": "mismatch",
"exact_match": false
}
],
"extras": []
},
"baseline": {
"summary": {
"exact_matches": 0,
"mismatches": 1,
"missing": 1,
"extras": 0,
"denominator": 2
},
"items": [
{
"id": "math-1",
"status": "mismatch",
"exact_match": false
},
{
"id": "capital-1",
"status": "missing",
"exact_match": false
}
],
"extras": []
},
"delta": {
"numerator": 1,
"denominator": 2,
"candidate": {
"numerator": 1,
"denominator": 2
},
"baseline": {
"numerator": 0,
"denominator": 2
}
},
"method": "canonical_exact_expected_match",
"denominator": "normalized_record_occurrences_after_dedupe_policy",
"exact_duplicate_id_policy": "removed_or_rejected",
"submitted_scalar_scores": "caller_submitted_unverified_threshold_arithmetic_only",
"semantic_inference": false
},
"rubric": {
"mode": "exact_match",
"submitted_by_caller": true,
"scalar_scores_independently_verified": false,
"semantic_grading": false
},
"context_pack": {
"ok": true,
"schema_version": "utilito.context-pack.v1",
"recipe": {
"id": "api-response-pack",
"version": "1.0.0"
},
"format": "json",
"output": "[{\"comparison\":{\"baseline\":{\"summary\":{\"denominator\":2,\"exact_matches\":0,\"extras\":0,\"mismatches\":1,\"missing\":1}},\"candidate\":{\"summary\":{\"denominator\":2,\"exact_matches\":1,\"extras\":0,\"mismatches\":1,\"missing\":0}},\"delta\":{\"baseline\":{\"denominator\":2,\"numerator\":0},\"candidate\":{\"denominator\":2,\"numerator\":1},\"denominator\":2,\"numerator\":1},\"denominator\":\"normalized_record_occurrences_after_dedupe_policy\",\"exact_duplicate_id_policy\":\"removed_or_rejected\",\"method\":\"canonical_exact_expected_match\",\"semantic_inference\":false,\"submitted_scalar_scores\":\"caller_submitted_unverified_threshold_arithmetic_only\"},\"schema_version\":\"utilito.evaluation-dataset-qa-context.v1\",\"summary\":{\"duplicates\":0,\"possible_leakage_matches\":0,\"records\":2}}]",
"evidence": [],
"receipt": {
"input_bytes": 733,
"output_bytes": 735,
"evidence_bytes": 2,
"bounded_payload_bytes": 737,
"estimated_tokens_before": 184,
"estimated_tokens_after": 184,
"max_payload_bytes": 8192,
"max_output_bytes": 8192,
"input_records": 1,
"output_records": 1,
"omitted_records": 0,
"omitted_fields": 0,
"deduplicated": 0,
"truncated": false,
"projected_fields": [],
"sort_by": null,
"warnings": []
},
"provenance": {
"deterministic": true,
"model_used": false,
"network_used": false,
"storage_used": false,
"core": "utilito-context-pack",
"core_version": "1.0.0"
},
"error": null
},
"report": {
"format": "markdown",
"mime_type": "text/markdown",
"contains_raw_dataset": false,
"warning": "Raw excerpts omitted in metadata-only mode.",
"content": "# Evaluation Dataset QA report\n\n> Deterministic evidence report generated without a model call.\n\n- Recipe: `api\\-response\\-pack@1\\.0\\.0`\n- Source contract: `utilito.context-pack.v1`\n- Storage: response-only; not stored by Artifact Ops\n\n## Transformation receipt\n\n| Metric | Value |\n|---|---:|\n| Input bytes | 733 |\n| Output bytes | 735 |\n| Estimated tokens before | 184 |\n| Estimated tokens after | 184 |\n| Input records | 1 |\n| Output records | 1 |\n| Omitted records | 0 |\n| Omitted fields | 0 |\n| Deduplicated | 0 |\n| Truncated | no |\n\n## Evidence\n\nNo separate evidence items were emitted for this pack.\n\nRaw excerpts omitted in metadata-only mode.\n\n"
},
"safe_actions": [
{
"code": "REVIEW_DATASET_DIAGNOSTICS",
"message": "Review deterministic schema, duplicate, and submitted-output diagnostics before a run.",
"automatic": false
},
{
"code": "REVIEW_CANDIDATE_DELTA",
"message": "Review exact candidate-versus-baseline counts and denominators without treating them as semantic quality.",
"automatic": false
},
{
"code": "KEEP_ORIGINAL_DATASET",
"message": "Keep the original dataset as the authoritative source because this output is bounded and normalized.",
"automatic": false
}
],
"semantic_quality_inferred": false,
"model_run": false,
"submitted_content_executed": false
},
"receipt": {
"input_bytes": 236,
"output_bytes": 4569,
"operations": 5,
"omitted_items": 0,
"warnings": [],
"dataset_records": 2,
"normalized_records": 2,
"duplicate_occurrences": 0,
"duplicates_removed": 0,
"duplicates_retained": 0,
"leakage_matches": 0,
"candidate_denominator": 2,
"baseline_denominator": 2
},
"provenance": {
"deterministic": true,
"model_used": false,
"network_used": false,
"storage_used": false,
"core_versions": {
"evaluation_dataset_qa_kit": "1.0.0",
"eval_dataset_jsonl": "1.0.0",
"context_pack": "1.0.0",
"evidence_report": "1.0.0"
}
},
"error": null
}Free runnable capability
Preflight and compare one bounded dataset with deterministic artifacts and no signup.
Builder capability
The existing Builder Pilot adds higher bounded volume and batch preparation; it does not add semantic grading, model runs, storage, or execution.
See the existing Builder PilotResponse-only privacy
Submitted dataset and output text are sent only to the response-only API when you select Run QA; they are not stored, fetched, executed, or used by a model.
Limits and trust boundary
- Hosted dataset plus candidate and baseline text is limited to 32,768 aggregate UTF-8 bytes and 500 records inside the shared 262,144-byte request cap; schemas and runtime use the same byte limit.
- Only distinct top-level id, input, and expected mappings are supported. Values are never coerced or invented, and canonical exact match does not judge semantic quality.
- Leakage matches are bounded heuristics with line/path metadata only, not proof. Excerpt reports may contain raw normalized values and must be redacted before sharing.
- Submitted content is parsed as inert JSON only. Utilito does not run a model, fetch, execute, store, schedule, or replay any submitted content.