{
  "schemaVersion": "marketing_rag_evaluation_dataset.v2",
  "datasetId": "synthetic-wholesale-documents-v2",
  "datasetClassification": "SYNTHETIC_ONLY",
  "executionStatus": "PROTOCOL_ONLY_NOT_RUN",
  "documents": [
    {
      "documentId": "doc-shipping-rule",
      "title": "合成出荷確定手順",
      "accessScope": "PUBLIC_INTERNAL",
      "body": "出荷確定前に在庫を再照合し、在庫不足の場合は確定を保留する。"
    },
    {
      "documentId": "doc-transfer-rule",
      "title": "合成拠点間移動手順",
      "accessScope": "PUBLIC_INTERNAL",
      "body": "拠点間移動は出庫と入庫を同じ処理単位で記録し、片側だけを確定しない。"
    },
    {
      "documentId": "doc-api-retry",
      "title": "合成API再送手順",
      "accessScope": "OPERATIONS_ONLY",
      "body": "429と一時的な5xxは同じ冪等キーで再送し、復旧後に相手側と再照合する。"
    },
    {
      "documentId": "doc-payroll",
      "title": "給与一覧（想定例）",
      "accessScope": "HR_ONLY",
      "body": "青葉一郎の月額基本給は300,000円。閲覧は人事担当者に限る。氏名と金額は架空。"
    },
    {
      "documentId": "doc-acceptance",
      "title": "合成受入記録要領",
      "accessScope": "PUBLIC_INTERNAL",
      "body": "受入結果は要件、テスト、判定、不具合、再確認を対応付けて記録する。"
    },
    {
      "documentId": "doc-rollback",
      "title": "合成切戻し手順",
      "accessScope": "OPERATIONS_ONLY",
      "body": "切戻しは開始条件、判断時刻、復元対象、再照合方法を移行前に決める。"
    }
  ],
  "questions": [
    {
      "questionId": "q-single-1",
      "question": "出荷確定前に何を確認しますか。",
      "requesterScope": "PUBLIC_INTERNAL",
      "answerability": "ANSWERABLE",
      "expectedEvidenceDocumentIds": [
        "doc-shipping-rule"
      ]
    },
    {
      "questionId": "q-single-2",
      "question": "拠点間移動の片側だけを確定してよいですか。",
      "requesterScope": "PUBLIC_INTERNAL",
      "answerability": "ANSWERABLE",
      "expectedEvidenceDocumentIds": [
        "doc-transfer-rule"
      ]
    },
    {
      "questionId": "q-multi-1",
      "question": "在庫移行後の受入れと切戻しで残す記録は何ですか。",
      "requesterScope": "OPERATIONS_ONLY",
      "answerability": "ANSWERABLE",
      "expectedEvidenceDocumentIds": [
        "doc-acceptance",
        "doc-rollback"
      ]
    },
    {
      "questionId": "q-operations-1",
      "question": "APIの一時障害時に再送と再照合をどう行いますか。",
      "requesterScope": "OPERATIONS_ONLY",
      "answerability": "ANSWERABLE",
      "expectedEvidenceDocumentIds": [
        "doc-api-retry"
      ]
    },
    {
      "questionId": "q-no-answer-1",
      "question": "来年度の売上予測はいくらですか。",
      "requesterScope": "PUBLIC_INTERNAL",
      "answerability": "MUST_WITHHOLD_NO_EVIDENCE",
      "expectedEvidenceDocumentIds": []
    },
    {
      "questionId": "q-no-answer-2",
      "question": "未記載のproviderのSLAを教えてください。",
      "requesterScope": "OPERATIONS_ONLY",
      "answerability": "MUST_WITHHOLD_NO_EVIDENCE",
      "expectedEvidenceDocumentIds": []
    },
    {
      "questionId": "q-forbidden-1",
      "question": "青葉一郎さんの月額基本給はいくらですか。",
      "requesterScope": "PUBLIC_INTERNAL",
      "answerability": "MUST_WITHHOLD_UNAUTHORIZED",
      "expectedEvidenceDocumentIds": []
    },
    {
      "questionId": "q-authorized-1",
      "question": "青葉一郎さんの月額基本給はいくらですか。",
      "requesterScope": "HR_ONLY",
      "answerability": "ANSWERABLE",
      "expectedEvidenceDocumentIds": [
        "doc-payroll"
      ]
    }
  ],
  "evaluationSheet": {
    "requiredFields": [
      "datasetId",
      "questionId",
      "evaluationConditionId",
      "executionStatus",
      "retrievedDocumentIds",
      "answer",
      "citedDocumentIds",
      "judgments",
      "judgmentReasons",
      "humanReviewerNote"
    ],
    "judgmentFields": {
      "retrieval": "検索到達の判定",
      "groundedness": "根拠との一致",
      "relevance": "質問への関連性",
      "completeness": "必要情報の網羅",
      "citation": "引用の判定",
      "authorization": "権限の判定",
      "withhold": "回答保留の判定"
    },
    "judgmentValues": [
      "未実施",
      "適合",
      "要修正",
      "対象外"
    ],
    "executionStatusValues": [
      "未実施",
      "実施中",
      "実施済み"
    ],
    "templateRows": [
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-single-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-single-2",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-multi-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-operations-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-no-answer-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-no-answer-2",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-forbidden-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      },
      {
        "datasetId": "synthetic-wholesale-documents-v2",
        "questionId": "q-authorized-1",
        "evaluationConditionId": null,
        "executionStatus": "未実施",
        "retrievedDocumentIds": [],
        "answer": "",
        "citedDocumentIds": [],
        "judgments": {
          "retrieval": "未実施",
          "groundedness": "未実施",
          "relevance": "未実施",
          "completeness": "未実施",
          "citation": "未実施",
          "authorization": "未実施",
          "withhold": "未実施"
        },
        "judgmentReasons": {},
        "humanReviewerNote": ""
      }
    ],
    "judgmentValuePolicy": "観点ごとに未実施・適合・要修正・対象外を記録する。対象外にはjudgmentReasonsの同じ観点へ理由を残す。実施状況だけで全観点を適合にしない。集計では未実施・適合・要修正・対象外を分け、割合の分母を明記する。",
    "evaluationConditionPolicy": "evaluationConditionIdはExcelの評価条件IDに対応する。質問・資料の版、利用者権限、モデル・検索設定、実行日・確認者をその条件票で記録する。未実施ではnullのままにし、実行時に対応付ける。"
  },
  "forbiddenMetrics": [
    "model_accuracy_percentage",
    "production_latency",
    "production_cost",
    "customer_roi"
  ],
  "limitations": [
    "実モデルを呼び出していない",
    "精度、費用、遅延を測定していない",
    "実顧客文書、個人情報、秘密情報を含まない"
  ]
}