{"data":{"id":"7eb32746-d75e-49f6-bd62-48aa3efbd9cc","title":"Separating signal from noise in coding evaluations","summary":"OpenAI audited SWE-Bench Pro, the successor to SWE-bench Verified, using an automated datapoint analysis pipeline plus multi-pass agent review and human annotation by five software engineers. The audit flagged 200 (27.4%) tasks automatically and 249 (34.1%) through human annotation, with issues in four categories: overly strict tests, underspecified prompts, low-coverage tests, and misleading prompts. OpenAI estimates that about 30% of SWE-Bench Pro tasks are broken and advises model developers to examine results carefully.","solution":"N/A -- no mitigation discussed in source.","labels":["research","industry"],"sourceUrl":"https://openai.com/index/separating-signal-from-noise-coding-evaluations","publishedAt":"2026-07-08T13:00:00.000Z","cveId":null,"cweIds":null,"cvssScore":null,"cvssSeverity":null,"severity":"info","attackType":[],"issueType":"research","affectedPackages":null,"affectedPackageNames":null,"affectedPackageRefs":null,"affectedVendors":["OpenAI"],"affectedVendorsRaw":["OpenAI","SWE-bench Verified","SWE-Bench Pro","Codex"],"classifierModel":"claude-haiku-5-5","classifierPromptVersion":"v4","summaryPromptVersion":"v2","headline":null,"headlinePromptVersion":null,"cvssVector":null,"attackVector":null,"attackComplexity":null,"privilegesRequired":null,"userInteraction":null,"exploitMaturity":null,"epssScore":null,"epssCheckedAt":null,"kevDateAdded":null,"advisoryAliases":null,"affectedPackagesSource":null,"affectedPackagesCheckedAt":null,"patchAvailable":null,"disclosureDate":"2026-07-08T13:00:00.000Z","capecIds":null,"crossRefCount":0,"attackSophistication":"moderate","impactType":null,"aiComponentTargeted":null,"llmSpecific":true,"classifierConfidence":0.85,"researchCategory":"blog","atlasIds":null}}