{"data":{"id":"710a9f26-5c10-43cf-991a-b62c48e99f13","title":"Concerns in Benchmarking Jailbreaks: An Empirical Study of Attack Success Rates","summary":"This paper examines how Attack Success Rate (ASR) for eight jailbreak attacks against three open-weight language models depends on the evaluation procedure. Low-resource language attacks score a mean ASR of 96.0% under keyword matching but 2.7% under a Qwen3 evaluator following StrongREJECT scoring instructions. Repeated sampling at Ollama's default temperature of 0.8 yields 100% ASR under keyword matching versus 50% under the Qwen3 evaluator, suggesting residual unsafe behavior can persist despite an initial refusal.","solution":"The paper recommends explicit success definitions, matched response budgets, validated evaluators, and reporting cumulative failure under repeated interaction rather than relying on single-generation tests.","labels":["research","safety"],"sourceUrl":"https://arxiv.org/abs/2610.26595v1","publishedAt":"2026-10-19T15:39:16.000Z","cveId":null,"cweIds":null,"cvssScore":null,"cvssSeverity":null,"severity":"info","attackType":["jailbreak"],"issueType":"research","affectedPackages":null,"affectedPackageNames":null,"affectedPackageRefs":null,"affectedVendors":[],"affectedVendorsRaw":["Ollama","Qwen3","DeepInception"],"classifierModel":"claude-haiku-5-5","classifierPromptVersion":"v4","summaryPromptVersion":"v2","headline":null,"headlinePromptVersion":null,"cvssVector":null,"attackVector":null,"attackComplexity":null,"privilegesRequired":null,"userInteraction":null,"exploitMaturity":null,"epssScore":null,"epssCheckedAt":null,"kevDateAdded":null,"advisoryAliases":null,"affectedPackagesSource":null,"affectedPackagesCheckedAt":null,"patchAvailable":null,"disclosureDate":"2026-10-19T15:39:16.000Z","capecIds":null,"crossRefCount":0,"attackSophistication":"moderate","impactType":["safety"],"aiComponentTargeted":"model","llmSpecific":true,"classifierConfidence":0.95,"researchCategory":"preprint","atlasIds":["AML.T0054"]}}