{"data":{"id":"34226bc2-d79a-4a47-91a6-7e746b397f5e","title":"On the Reliability of CyberGym Evaluation: Task Material Consistency and Reference-Target Identity","summary":"This research audits the reliability of CyberGym, a benchmark that scores vulnerability reproduction by checking whether a proof-of-concept input crashes a vulnerable program version but not its fixed version. Auditing all 1,507 tasks, the authors flag 338 (22.4%) as materially inconsistent, mostly through coverage gaps and descriptions and patches pointing to a different defect. Of 1,286 differential passes from one Level-1 GLM-5.3 run, 89 were judged non-target and 10 had insufficient evidence, lowering the pass rate from 85.3% to 78.8% when a target hit is required.","solution":"N/A -- no mitigation discussed in source.","labels":["research","security"],"sourceUrl":"https://arxiv.org/abs/2611.25790v1","publishedAt":"2026-11-19T09:20:55.000Z","cveId":null,"cweIds":null,"cvssScore":null,"cvssSeverity":null,"severity":"info","attackType":[],"issueType":"research","affectedPackages":null,"affectedPackageNames":null,"affectedPackageRefs":null,"affectedVendors":[],"affectedVendorsRaw":["GLM-5.3"],"classifierModel":"claude-haiku-5-5","classifierPromptVersion":"v4","summaryPromptVersion":"v2","headline":null,"headlinePromptVersion":null,"cvssVector":null,"attackVector":null,"attackComplexity":null,"privilegesRequired":null,"userInteraction":null,"exploitMaturity":null,"epssScore":null,"epssCheckedAt":null,"kevDateAdded":null,"advisoryAliases":null,"affectedPackagesSource":null,"affectedPackagesCheckedAt":null,"patchAvailable":null,"disclosureDate":"2026-11-19T09:20:55.000Z","fixReleasedAt":null,"daysToFix":null,"fixCheckedAt":null,"capecIds":null,"crossRefCount":0,"attackSophistication":"moderate","impactType":null,"aiComponentTargeted":"framework","llmSpecific":true,"classifierConfidence":0.8,"researchCategory":"preprint","atlasIds":null}}