{"data":{"id":"2160081b-c4e2-4207-b4b2-7b464aff0653","title":"Beyond AUROC: Deployment-Aware Evaluation of Model-Internal Jailbreak Detectors","summary":"Researchers evaluated four model-internal jailbreak detectors across four LLMs and twelve jailbreak families. Thresholds calibrated on curated benign prompts frequently failed to hold their false-positive rate budgets on real-user traffic, and recalibration exposed family-level coverage gaps, including high-AUROC cases with low true-positive rates. Changing the reference set could substantially shift which attack families were detected.","solution":"The authors recommend evaluating model-internal jailbreak defenses using realized FPR on deployment-like traffic and per-family TPR at explicit low-FPR operating points, with AUROC as a complementary ranking metric rather than the primary security measure.","labels":["security","research"],"sourceUrl":"https://arxiv.org/abs/2611.11067v1","publishedAt":"2026-11-08T01:54:01.000Z","cveId":null,"cweIds":null,"cvssScore":null,"cvssSeverity":null,"severity":"info","attackType":["jailbreak"],"issueType":"research","affectedPackages":null,"affectedPackageNames":null,"affectedPackageRefs":null,"affectedVendors":[],"affectedVendorsRaw":[],"classifierModel":"claude-haiku-5-5","classifierPromptVersion":"v4","summaryPromptVersion":"v2","headline":null,"headlinePromptVersion":null,"cvssVector":null,"attackVector":null,"attackComplexity":null,"privilegesRequired":null,"userInteraction":null,"exploitMaturity":null,"epssScore":null,"epssCheckedAt":null,"kevDateAdded":null,"advisoryAliases":null,"affectedPackagesSource":null,"affectedPackagesCheckedAt":null,"patchAvailable":null,"disclosureDate":"2026-11-08T01:54:01.000Z","fixReleasedAt":null,"daysToFix":null,"fixCheckedAt":null,"capecIds":null,"crossRefCount":0,"attackSophistication":"advanced","impactType":["safety","integrity"],"aiComponentTargeted":"model","llmSpecific":true,"classifierConfidence":0.95,"researchCategory":"preprint","atlasIds":["AML.T0054"]}}