{ "title": "AI Detector Accuracy and False Positive Claims", "version": "1.0.0", "published": "2026-09-16", "publisher": { "name": "ForgeRank AI", "url": "https://forgerankai.com", "about": "https://forgerankai.com/about" }, "scope": "结构化汇编：主流 AI 检测器**自己公布**的准确率与误报率，逐条附上该数字成立的条件、计量单位与原文引句；另收同行评议的误报发现，以及我们对自家评分器误报类的实测记录。目的不是给检测器排名，而是让每个头条数字与它附带的条件一起被引用——二手转述最常丢掉的就是条件那一栏。", "not_included": "本文件**不含**任何第一方检测器实测：我们没有 Turnitin / GPTZero / ZeroGPT / Copyleaks 的 API 或机构权限，无法对同一批文本复跑这些工具。任何声称横向对比检测器准确率的表格，只要作者没有这些访问权限，都是二手转述。", "citation": "ForgeRank AI. (2026). AI Detector Accuracy and False Positive Claims, v1.0.0. https://forgerankai.com/blog/ai-detector-accuracy-benchmark", "units_glossary": { "document_fpr": "文档级误报率：被判为 AI 的**整份文档**中，实际由人类完整撰写的占比。", "sentence_fpr": "句级误报率：被判为 AI 的**句子**中，实际由人类撰写的占比。与文档级不是同一个量。", "tpr_at_fpr_1pct": "TPR@FPR=1%：在把 1% 人类文本误判为 AI 的前提下，能检出多少比例的 AI 文本。" }, "vendor_claims": [ { "tool": "Turnitin", "claim": "文档级误报率", "value": "<1%", "unit": "document_fpr", "condition": "仅适用于「已被判定含 20% 或以上 AI 写作」的文档", "source_title": "Understanding AI writing detection: False positive rates (Annie Chechitelli, Chief Product Officer)", "source_url": "https://www.turnitin.com/blog/understanding-the-false-positive-rate-for-sentences-of-our-ai-writing-detection-capability", "source_quote": "Our document false positive rate - incorrectly identifying fully human-written text as AI-generated within a document- is less than 1% for documents with 20% or more AI writing.", "url_verified": true }, { "tool": "Turnitin", "claim": "句级误报率", "value": "约 4%", "unit": "sentence_fpr", "condition": "无附加条件；与上一条不是同一个量", "source_title": "Understanding AI writing detection: False positive rates", "source_url": "https://www.turnitin.com/blog/understanding-the-false-positive-rate-for-sentences-of-our-ai-writing-detection-capability", "source_quote": "Our sentence-level false positive rate is around 4%.", "url_verified": true }, { "tool": "Turnitin", "claim": "验证样本量", "value": "800,000", "unit": "documents", "condition": "样本全部写成于 ChatGPT 发布之前，用作文档级误报率的护栏", "source_title": "New research: Turnitin's AI detector shows no statistically significant bias against English Language Learners", "source_url": "https://www.turnitin.com/blog/new-research-turnitin-s-ai-detector-shows-no-statistically-significant-bias-against-english-language-learners", "source_quote": "To further ensure that we maintain a less than 1% false positive rate, we tested the detector on 800,000 academic writing samples that were written before the release of ChatGPT", "url_verified": true }, { "tool": "Turnitin", "claim": "短文误报偏高 → 最小提交长度改为 300 词", "value": "300", "unit": "words (minimum submission length)", "condition": "厂商自述：少于 300 词的提交误报率「高于舒适区间」，因此拒绝评分", "source_title": "New research: Turnitin's AI detector shows no statistically significant bias against English Language Learners", "source_url": "https://www.turnitin.com/blog/new-research-turnitin-s-ai-detector-shows-no-statistically-significant-bias-against-english-language-learners", "source_quote": "Our evaluation of 800,000 documents—mentioned earlier in this blog—shows a slightly higher-than-comfortable false positive rate on short submissions (fewer than 300 words). In order to ensure we keep our false positive rate below 1%, we moved quickly to update the minimum submission length to 300 words for our AI writing detection capability to process a submission.", "url_verified": true }, { "tool": "Turnitin", "claim": "对英语非母语写作者的偏差：厂商称无统计显著偏差", "value": "no statistically significant bias", "unit": "vendor research claim", "condition": "厂商自己的研究，直接回应 Liang 等（arXiv 2304.02819）的 61% 发现", "source_title": "New research: Turnitin's AI detector shows no statistically significant bias against English Language Learners", "source_url": "https://www.turnitin.com/blog/new-research-turnitin-s-ai-detector-shows-no-statistically-significant-bias-against-english-language-learners", "source_quote": "In addition to concerns around false positives, more recently, there have been a number of papers and articles claiming that AI writing detection tools are biased against writers for whom English is not their first language", "url_verified": true }, { "tool": "GPTZero", "claim": "RAID 基准上的检出率", "value": "95.7%", "unit": "tpr_at_fpr_1pct", "condition": "RAID 基准；同一句自述在排除已停用模型（如 GPT-3.5）后升至 99% 以上", "source_title": "GPTZero: Officially The Most Accurate Commercial AI Detector", "source_url": "https://gptzero.me/news/gptzero-accuracy-stats/", "source_quote": "Our AI checker is able to detect 95.7% of AI texts while only incorrectly predicting 1% of human texts as AI, an accuracy that jumps to over 99% when filtering out discontinued LLMs like GPT3.5.", "url_verified": true }, { "tool": "GPTZero", "claim": "自述基准局限（厂商主动声明）", "value": "only fully human or fully AI texts", "unit": "benchmark property", "condition": "厂商原话：RAID 及多数检测基准都不含「人机混合 / 轻度编辑」文本", "source_title": "GPTZero: Officially The Most Accurate Commercial AI Detector", "source_url": "https://gptzero.me/news/gptzero-accuracy-stats/", "source_quote": "Additional nuance not captured by RAID, or many other AI detection benchmarks, is that it considers only fully human or fully AI texts.", "url_verified": true }, { "tool": "GPTZero", "claim": "对抗性改写会让所有方法的检出率下降", "value": "every method suffers a decrease in TPR@FPR=1%", "unit": "benchmark property", "condition": "同义词替换、拼写错误、改写等对抗攻击下", "source_title": "GPTZero: Officially The Most Accurate Commercial AI Detector", "source_url": "https://gptzero.me/news/gptzero-accuracy-stats/", "source_quote": "With adversarial attacks, every method suffers a decrease in TPR@FPR=1%, but GPTZero maintains its rank", "url_verified": true }, { "tool": "GPTZero", "claim": "芝加哥 Booth 基准上的准确率与相对误差", "value": ">99% accuracy；比 Pangram 少 40% 错误、比 Originality 少 95% 错误", "unit": "vendor-reported relative error", "condition": "基准由 University of Chicago Booth School of Business 研究者于 2025-08-26 发布；数字为 GPTZero 自行复算并自述", "source_title": "GPTZero Tops Accuracy on Chicago Booth Benchmark in 2026", "source_url": "https://gptzero.me/news/chicago-booth-2026/", "source_quote": "This is 40% fewer errors than Pangram and 95% fewer errors than Originality.", "url_verified": true }, { "tool": "OpenAI", "claim": "自有关停的分类器：检出率与人类误判率", "value": "26% detection；9% human false positive", "unit": "vendor-reported", "condition": "2023 年 7 月停用该分类器；厂商自述信号太弱不足以支撑工具所暗示的置信度", "source_title": "OpenAI, AI classifier for indicating AI-written text（含 2023 年 7 月停用通知）", "source_url": "https://openai.com/index/new-ai-classifier-for-indicating-ai-written-text/", "source_quote": "（我们无法抓到原始页面：openai.com 对本环境的抓取返回 HTTP 403。该组数字由多家独立媒体于 2023 年 7 月报道，本文件按「已由第三方佐证、原始页面未由本机构直取」收录。）", "url_verified": false, "corroborating_urls": [ "https://www.techweb.com.cn/world/2023-07-26/2930949.shtml", "https://www.thepaper.cn/newsDetail_forward_23997902", "https://news.pconline.com.cn/broadcasting/2307/16398644.html" ] } ], "independent_findings": [ { "finding": "人类撰写的 TOEFL 论文被大量误判为 AI 生成", "value": "约 61%", "population": "非英语母语写作者的人类撰写论文", "unit": "document-level flag rate on human-written text", "source_title": "Liang et al., GPT detectors are biased against non-native English writers", "source_url": "https://arxiv.org/abs/2304.02819", "note": "发表于 Patterns；该结果直接说明「误报率」高度依赖被测人群。" }, { "finding": "检测器的机制：机器文本落在模型自身概率分布的局部最优附近", "value": "perturbation test", "population": "不适用（机制研究）", "unit": "method", "source_title": "Mitchell et al., DetectGPT: Zero-Shot Machine-Generated Text Detection using Probability Curvature", "source_url": "https://arxiv.org/abs/2301.11305", "note": "ICML 2023。遮蔽少量 token 后，机器文本的似然下降，人类文本基本持平。" }, { "finding": "改写攻击确实显著降低检测器准确率；恢复准确率靠的是检索式防御", "value": "paraphrase attack lowers accuracy", "population": "不适用（攻击/防御研究）", "unit": "method", "source_title": "Krishna et al., Paraphrasing evades detectors of AI-generated text, but retrieval is an effective defense", "source_url": "https://arxiv.org/abs/2303.13408", "note": "这条同时是「改写能骗过检测器」与「骗过检测器 ≠ 稿子变好」两个论点的依据。" } ], "first_party": { "what_this_is": "我们对自家评分器（ForgeRank AI 四维诊断 + AI 腔 lint）的误报类实测记录。公布这一块的理由：检测类工具极少公开自己的误报类与缓解措施，而这恰好是使用者最需要知道的部分。", "scorer_false_positive_classes": [ { "class": "aphorism 规则把列表序号读成格言", "example": "\"not structural. 5.\" / \"not copying. 8.\"", "mitigation": "收窄正则：句末单词改为必须以字母开头，数字序号不再能补完该模式", "status": "已修" }, { "class": "标题型模式误报正常小标题，系统性高估长文家族数", "example": "单个 \"the first sentence\" / \"the first paragraph\" 是正常英语", "mitigation": "改为仅全大写命中；实测自家方法论页不再误报，语料 32 篇中 0 篇受影响", "status": "已修" }, { "class": "真实步骤列表被读成模板化序号", "example": "Step 1 / Step 2 … 的教程式编号", "mitigation": "_mask_real_step_lists：判定前抹掉真步骤列表；未被豁免的长段落式编号仍会命中", "status": "已缓解" }, { "class": "表格单元格里的数字区间被读成 em dash", "example": "表格内 0–10 这类区间写法", "mitigation": "em_dash_count_numeric_exempt：数字区间豁免", "status": "已修" }, { "class": "纯正则判「对仗」误报率高", "example": "\"The rest is just details\" 会命中", "mitigation": "只作低权重记录信号，不据此单独改写", "status": "受限于日志" }, { "class": "not-X-but-Y 结构正则误报", "example": "真人叙事中的正当对比句", "mitigation": "NOT_XY_ENABLE_SCORE = False：先只挂家族不扣分，配 FALSE_POSITIVE_PAIRS 后置过滤，观察样本后再决定是否计分", "status": "保守：只记日志" }, { "class": "特定短语的叙事性误报", "example": "\"The one thing that…\" 存在少量真人叙事误报", "mitigation": "先上线跑日志，若误报占比高再迭代", "status": "保守：只记日志" } ], "conservative_policy": "对风险最高的规则（not-X-but-Y）采取「先记录、不扣分」策略：NOT_XY_ENABLE_SCORE = False。理由是正则判对仗的误报率未经样本验证前直接计分，会把真人叙事判成 AI 腔。", "regression_suite": { "script": "scripts/test_ai_tone_lint_regression.py", "result": "61 passed, 0 failed", "measured_on": "2026-09-16" }, "reproducibility": { "finding": "同一段文本两次评分可以相差 ±0.3–0.5 分", "evidence": "本项目 QC 表在同一文本上的重复实测记录（_measured_20260912 / _measured_20260916）；另有一篇措辞清洗前后的对照实测为 7.8 与 8.0，差值落在该区间内", "implication": "任何单次评分的「改前 vs 改后」对比，只要差值小于约 0.5 分，都无法归因于内容变动。声称「清洗 AI 味后分数提升 0.2」的结论在这套工具上是不可测的。" } }, "comparability_warnings": [ "单位不同：文档级误报率、句级误报率、TPR@FPR=1% 不是同一个量，横排在一起会暗示它们可比。", "人群不同：TOEFL 论文、RAID 的新闻/评测/社媒/书籍、80 万份学术写作，抽样人群互不重叠。", "阈值不同且多数未公布：同一个工具在不同运行点上会给出不同数字，两次研究报出不同结果并不必然矛盾。", "厂商数字为自报：上表中 Turnitin / GPTZero / OpenAI 的数字均为厂商自己公布，本文件原样转录并保留其条件，未做独立复现。", "基准的端点问题：GPTZero 自己指出 RAID 只含「纯人类或纯 AI」文本；由干净端点组成的基准无法估计混合文本上的误报率，而混合文本正是误报发生的场景。", "对抗性改写会降低所有方法的表现：这不是某一家的缺陷，是该类方法的共同属性。" ] }