| 1 | // SPDX-License-Identifier: Apache-2.0 | |
| 2 | // Copyright 2026 Egothor | |
| 3 | // Copyright 2026 Accenture | |
| 4 | package org.egothor.methodatlas.eval; | |
| 5 | ||
| 6 | import java.io.IOException; | |
| 7 | import java.nio.file.Path; | |
| 8 | import java.util.ArrayList; | |
| 9 | import java.util.HashMap; | |
| 10 | import java.util.HashSet; | |
| 11 | import java.util.List; | |
| 12 | import java.util.Map; | |
| 13 | import java.util.Set; | |
| 14 | ||
| 15 | import org.egothor.methodatlas.api.ScanRecord; | |
| 16 | import org.egothor.methodatlas.emit.ClassificationOverride; | |
| 17 | import org.egothor.methodatlas.emit.DeltaReport; | |
| 18 | ||
| 19 | /** | |
| 20 | * Joins AI predictions with human-reviewed labels and scores them with | |
| 21 | * {@link AccuracyMetrics}. | |
| 22 | * | |
| 23 | * <h2>Ground truth</h2> | |
| 24 | * | |
| 25 | * <p> | |
| 26 | * The labels are a {@link ClassificationOverride} document: the same | |
| 27 | * human-reviewed YAML that {@code -override-file} applies to scans. Resolution | |
| 28 | * follows the override rules: a method-level entry wins over a class-level entry | |
| 29 | * for the same class, and the last duplicate wins. An entry yields: | |
| 30 | * </p> | |
| 31 | * | |
| 32 | * <ul> | |
| 33 | * <li>a <b>relevance label</b> when {@code securityRelevant} is present;</li> | |
| 34 | * <li>a <b>tag label</b> when {@code tags} is present, or — as a deliberate | |
| 35 | * convention — an empty tag set when {@code securityRelevant: false} is | |
| 36 | * present without tags (a reviewer who says "not security-relevant" asserts | |
| 37 | * that no security tag applies). {@code securityRelevant: true} without | |
| 38 | * {@code tags} labels relevance only.</li> | |
| 39 | * </ul> | |
| 40 | * | |
| 41 | * <p> | |
| 42 | * An entry that carries neither (for example a {@code note}-only entry) is not a | |
| 43 | * label and is ignored. | |
| 44 | * </p> | |
| 45 | * | |
| 46 | * <h2>Predictions</h2> | |
| 47 | * | |
| 48 | * <p> | |
| 49 | * Predictions are the {@code ai_security_relevant} and {@code ai_tags} columns of | |
| 50 | * a MethodAtlas scan CSV produced with {@code -ai}. The scan used to produce them | |
| 51 | * must <em>not</em> have been run with {@code -override-file}: overrides replace | |
| 52 | * the AI values and would make the evaluation score the human's labels against | |
| 53 | * themselves. | |
| 54 | * </p> | |
| 55 | * | |
| 56 | * <h2>Coverage bookkeeping</h2> | |
| 57 | * | |
| 58 | * <p> | |
| 59 | * Missing and extra entries never fail the evaluation; they are counted and | |
| 60 | * reported so that a reader can judge how representative the numbers are: | |
| 61 | * predictions without a label ({@link Evaluation#unlabelledPredictions()}), | |
| 62 | * labelled methods without an AI prediction | |
| 63 | * ({@link Evaluation#unpredictedLabels()}), and label entries that match no | |
| 64 | * scanned method or class ({@link Evaluation#staleLabelEntries()}). | |
| 65 | * </p> | |
| 66 | * | |
| 67 | * <p> | |
| 68 | * This class is stateless and thread-safe. | |
| 69 | * </p> | |
| 70 | * | |
| 71 | * @see AccuracyMetrics | |
| 72 | * @see AccuracyReportRenderer | |
| 73 | * @since 6.1.0 | |
| 74 | */ | |
| 75 | public final class AccuracyEvaluator { | |
| 76 | ||
| 77 | private AccuracyEvaluator() { | |
| 78 | // static utility; no instances | |
| 79 | } | |
| 80 | ||
| 81 | /** | |
| 82 | * Metrics together with the coverage bookkeeping of the join. | |
| 83 | * | |
| 84 | * @param metrics the computed quality metrics | |
| 85 | * @param predictionRows number of rows read from the predictions CSV | |
| 86 | * @param evaluated number of methods that were both labelled and | |
| 87 | * predicted and therefore took part in scoring | |
| 88 | * @param unlabelledPredictions predicted methods without a usable label | |
| 89 | * @param unpredictedLabels labelled methods whose row carries no AI | |
| 90 | * prediction | |
| 91 | * @param staleLabelEntries label entries matching no row of the | |
| 92 | * predictions CSV (renamed or deleted methods, | |
| 93 | * or a different scan scope) | |
| 94 | */ | |
| 95 | public record Evaluation(AccuracyMetrics.Result metrics, int predictionRows, int evaluated, | |
| 96 | int unlabelledPredictions, int unpredictedLabels, int staleLabelEntries) { | |
| 97 | } | |
| 98 | ||
| 99 | /** A resolved ground-truth label; either field may be {@code null} (undecided). */ | |
| 100 | private record Label(Boolean relevant, Set<String> tags) { | |
| 101 | } | |
| 102 | ||
| 103 | /** | |
| 104 | * Loads the label file and the predictions CSV and evaluates them. | |
| 105 | * | |
| 106 | * @param labelsFile override-format YAML holding the human-reviewed | |
| 107 | * labels; must exist | |
| 108 | * @param predictionsCsv MethodAtlas scan CSV produced with {@code -ai} | |
| 109 | * @return the evaluation; never {@code null} | |
| 110 | * @throws IOException if either file cannot be read or parsed | |
| 111 | * @throws IllegalArgumentException if the CSV lacks the required | |
| 112 | * {@code fqcn} or {@code method} columns | |
| 113 | */ | |
| 114 | public static Evaluation evaluate(Path labelsFile, Path predictionsCsv) throws IOException { | |
| 115 | ClassificationOverride labels = ClassificationOverride.load(labelsFile); | |
| 116 | List<ScanRecord> predictions = DeltaReport.loadRecords(predictionsCsv); | |
| 117 |
1
1. evaluate : replaced return value with null for org/egothor/methodatlas/eval/AccuracyEvaluator::evaluate → KILLED |
return evaluate(labels, predictions); |
| 118 | } | |
| 119 | ||
| 120 | /** | |
| 121 | * Evaluates already-loaded labels against already-loaded predictions. | |
| 122 | * | |
| 123 | * @param labels human-reviewed labels; must not be {@code null} | |
| 124 | * @param predictions scan records carrying the AI columns; must not be | |
| 125 | * {@code null} | |
| 126 | * @return the evaluation; never {@code null} | |
| 127 | */ | |
| 128 | public static Evaluation evaluate(ClassificationOverride labels, List<ScanRecord> predictions) { | |
| 129 | List<ClassificationOverride.Entry> entries = labels.entries(); | |
| 130 | Map<String, Label> classLevel = new HashMap<>(); | |
| 131 | Map<String, Label> methodLevel = new HashMap<>(); | |
| 132 | for (ClassificationOverride.Entry e : entries) { | |
| 133 | Label label = toLabel(e); | |
| 134 |
2
1. evaluate : removed conditional - replaced equality check with false → KILLED 2. evaluate : removed conditional - replaced equality check with true → KILLED |
if (e.method() == null) { |
| 135 | classLevel.put(e.fqcn(), label); | |
| 136 | } else { | |
| 137 | methodLevel.put(key(e.fqcn(), e.method()), label); | |
| 138 | } | |
| 139 | } | |
| 140 | ||
| 141 | List<AccuracyMetrics.Sample> samples = new ArrayList<>(); | |
| 142 | Set<String> seenMethods = new HashSet<>(); | |
| 143 | Set<String> seenClasses = new HashSet<>(); | |
| 144 | int unlabelled = 0; | |
| 145 | int unpredicted = 0; | |
| 146 | for (ScanRecord r : predictions) { | |
| 147 | seenMethods.add(key(r.fqcn(), r.method())); | |
| 148 | seenClasses.add(r.fqcn()); | |
| 149 | Label label = methodLevel.getOrDefault(key(r.fqcn(), r.method()), classLevel.get(r.fqcn())); | |
| 150 |
6
1. evaluate : removed conditional - replaced equality check with true → SURVIVED 2. evaluate : removed conditional - replaced equality check with false → KILLED 3. evaluate : removed conditional - replaced equality check with true → KILLED 4. evaluate : removed conditional - replaced equality check with false → KILLED 5. evaluate : removed conditional - replaced equality check with false → KILLED 6. evaluate : removed conditional - replaced equality check with true → KILLED |
if (label == null || (label.relevant() == null && label.tags() == null)) { |
| 151 |
2
1. evaluate : removed conditional - replaced equality check with true → SURVIVED 2. evaluate : removed conditional - replaced equality check with false → KILLED |
if (r.aiSecurityRelevant() != null) { |
| 152 |
1
1. evaluate : Changed increment from 1 to -1 → KILLED |
unlabelled++; |
| 153 | } | |
| 154 |
2
1. evaluate : removed conditional - replaced equality check with false → KILLED 2. evaluate : removed conditional - replaced equality check with true → KILLED |
} else if (r.aiSecurityRelevant() == null) { |
| 155 |
1
1. evaluate : Changed increment from 1 to -1 → KILLED |
unpredicted++; |
| 156 | } else { | |
| 157 | samples.add(new AccuracyMetrics.Sample(key(r.fqcn(), r.method()), | |
| 158 | r.aiSecurityRelevant(), label.relevant(), r.aiTags(), label.tags())); | |
| 159 | } | |
| 160 | } | |
| 161 | ||
| 162 | int stale = 0; | |
| 163 | for (ClassificationOverride.Entry e : entries) { | |
| 164 |
2
1. evaluate : removed conditional - replaced equality check with true → KILLED 2. evaluate : removed conditional - replaced equality check with false → KILLED |
boolean matched = e.method() == null |
| 165 | ? seenClasses.contains(e.fqcn()) | |
| 166 | : seenMethods.contains(key(e.fqcn(), e.method())); | |
| 167 |
2
1. evaluate : removed conditional - replaced equality check with false → KILLED 2. evaluate : removed conditional - replaced equality check with true → KILLED |
if (!matched) { |
| 168 |
1
1. evaluate : Changed increment from 1 to -1 → KILLED |
stale++; |
| 169 | } | |
| 170 | } | |
| 171 | ||
| 172 |
1
1. evaluate : replaced return value with null for org/egothor/methodatlas/eval/AccuracyEvaluator::evaluate → KILLED |
return new Evaluation(AccuracyMetrics.compute(samples), predictions.size(), samples.size(), |
| 173 | unlabelled, unpredicted, stale); | |
| 174 | } | |
| 175 | ||
| 176 | private static Label toLabel(ClassificationOverride.Entry e) { | |
| 177 | Set<String> tags = null; | |
| 178 |
2
1. toLabel : removed conditional - replaced equality check with false → KILLED 2. toLabel : removed conditional - replaced equality check with true → KILLED |
if (e.tags() != null) { |
| 179 | tags = AccuracyMetrics.normalise(e.tags()); | |
| 180 |
2
1. toLabel : removed conditional - replaced equality check with true → KILLED 2. toLabel : removed conditional - replaced equality check with false → KILLED |
} else if (Boolean.FALSE.equals(e.securityRelevant())) { |
| 181 | tags = Set.of(); | |
| 182 | } | |
| 183 |
1
1. toLabel : replaced return value with null for org/egothor/methodatlas/eval/AccuracyEvaluator::toLabel → KILLED |
return new Label(e.securityRelevant(), tags); |
| 184 | } | |
| 185 | ||
| 186 | private static String key(String fqcn, String method) { | |
| 187 |
1
1. key : replaced return value with "" for org/egothor/methodatlas/eval/AccuracyEvaluator::key → KILLED |
return fqcn + "::" + method; |
| 188 | } | |
| 189 | } | |
Mutations | ||
| 117 |
1.1 |
|
| 134 |
1.1 2.2 |
|
| 150 |
1.1 2.2 3.3 4.4 5.5 6.6 |
|
| 151 |
1.1 2.2 |
|
| 152 |
1.1 |
|
| 154 |
1.1 2.2 |
|
| 155 |
1.1 |
|
| 164 |
1.1 2.2 |
|
| 167 |
1.1 2.2 |
|
| 168 |
1.1 |
|
| 172 |
1.1 |
|
| 178 |
1.1 2.2 |
|
| 180 |
1.1 2.2 |
|
| 183 |
1.1 |
|
| 187 |
1.1 |