| 1 | // SPDX-License-Identifier: Apache-2.0 | |
| 2 | // Copyright 2026 Egothor | |
| 3 | // Copyright 2026 Accenture | |
| 4 | package org.egothor.methodatlas.eval; | |
| 5 | ||
| 6 | import java.util.ArrayList; | |
| 7 | import java.util.Collection; | |
| 8 | import java.util.List; | |
| 9 | import java.util.Locale; | |
| 10 | import java.util.Map; | |
| 11 | import java.util.OptionalDouble; | |
| 12 | import java.util.Set; | |
| 13 | import java.util.TreeMap; | |
| 14 | import java.util.TreeSet; | |
| 15 | ||
| 16 | /** | |
| 17 | * Pure, deterministic computation of AI-classification quality metrics from | |
| 18 | * (prediction, truth) pairs. | |
| 19 | * | |
| 20 | * <p> | |
| 21 | * This class performs no I/O and calls no AI provider: given the same | |
| 22 | * {@link Sample} list it always returns the same {@link Result}, so the metric | |
| 23 | * math is unit-testable offline. Reading labels and predictions from files is | |
| 24 | * the job of {@link AccuracyEvaluator}. | |
| 25 | * </p> | |
| 26 | * | |
| 27 | * <h2>Two evaluated dimensions</h2> | |
| 28 | * | |
| 29 | * <ul> | |
| 30 | * <li><b>{@code securityRelevant}</b> — a binary decision per method, scored as a | |
| 31 | * confusion matrix (TP/FP/FN/TN) where "positive" means security-relevant. | |
| 32 | * Only samples whose truth label carries a relevance decision take part.</li> | |
| 33 | * <li><b>taxonomy tags</b> — a multi-label decision per method, scored | |
| 34 | * one-vs-rest per tag (a tag is a TP when predicted and labelled, an FP when | |
| 35 | * predicted only, an FN when labelled only) plus micro- and macro-averages. | |
| 36 | * Only samples whose truth label carries a tag set take part.</li> | |
| 37 | * </ul> | |
| 38 | * | |
| 39 | * <h2>Undefined values</h2> | |
| 40 | * | |
| 41 | * <p> | |
| 42 | * Precision is {@code TP/(TP+FP)}, recall is {@code TP/(TP+FN)}, F1 is | |
| 43 | * {@code 2TP/(2TP+FP+FN)} and accuracy is {@code (TP+TN)/total}. When a | |
| 44 | * denominator is zero the metric is mathematically undefined, and it is reported | |
| 45 | * as {@link OptionalDouble#empty()} rather than silently as {@code 0} or | |
| 46 | * {@code 1}: "no evidence" must not read as "perfect" or "terrible" in an audit | |
| 47 | * report. | |
| 48 | * </p> | |
| 49 | * | |
| 50 | * <h2>Tag normalisation</h2> | |
| 51 | * | |
| 52 | * <p> | |
| 53 | * Tags are trimmed, lower-cased with {@link Locale#ROOT} and de-duplicated, and | |
| 54 | * blank tags are dropped, so {@code "Crypto "} and {@code "crypto"} are the same | |
| 55 | * taxonomy tag. The taxonomy is a closed lower-case vocabulary, so this never | |
| 56 | * merges distinct tags. | |
| 57 | * </p> | |
| 58 | * | |
| 59 | * <p> | |
| 60 | * This class is stateless and thread-safe. | |
| 61 | * </p> | |
| 62 | * | |
| 63 | * @see AccuracyEvaluator | |
| 64 | * @since 6.1.0 | |
| 65 | */ | |
| 66 | public final class AccuracyMetrics { | |
| 67 | ||
| 68 | private static final double TWO = 2.0; | |
| 69 | ||
| 70 | private AccuracyMetrics() { | |
| 71 | // static utility; no instances | |
| 72 | } | |
| 73 | ||
| 74 | /** | |
| 75 | * One method for which a prediction is compared with a human label. | |
| 76 | * | |
| 77 | * @param id stable identifier, used only for diagnostics | |
| 78 | * @param predictedRelevant the AI's {@code securityRelevant} decision | |
| 79 | * @param truthRelevant the human {@code securityRelevant} label, or | |
| 80 | * {@code null} when the label does not decide | |
| 81 | * relevance (the sample is then excluded from the | |
| 82 | * relevance confusion matrix) | |
| 83 | * @param predictedTags the AI's tags; {@code null} is treated as empty | |
| 84 | * @param truthTags the human tag set, or {@code null} when the label | |
| 85 | * does not decide tags (the sample is then excluded | |
| 86 | * from the tag metrics); an empty set is a real | |
| 87 | * label meaning "no tags" | |
| 88 | */ | |
| 89 | public record Sample(String id, boolean predictedRelevant, Boolean truthRelevant, | |
| 90 | Collection<String> predictedTags, Collection<String> truthTags) { | |
| 91 | } | |
| 92 | ||
| 93 | /** | |
| 94 | * Confusion counts for one binary decision. | |
| 95 | * | |
| 96 | * @param tp true positives | |
| 97 | * @param fp false positives | |
| 98 | * @param fn false negatives | |
| 99 | * @param tn true negatives | |
| 100 | */ | |
| 101 | public record Counts(int tp, int fp, int fn, int tn) { | |
| 102 | ||
| 103 | /** | |
| 104 | * Returns the number of truly positive samples. | |
| 105 | * | |
| 106 | * @return {@code tp + fn} | |
| 107 | */ | |
| 108 | public int support() { | |
| 109 |
2
1. support : replaced int return with 0 for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::support → KILLED 2. support : Replaced integer addition with subtraction → KILLED |
return tp + fn; |
| 110 | } | |
| 111 | ||
| 112 | /** | |
| 113 | * Returns the total number of samples counted. | |
| 114 | * | |
| 115 | * @return {@code tp + fp + fn + tn} | |
| 116 | */ | |
| 117 | public int total() { | |
| 118 |
4
1. total : Replaced integer addition with subtraction → KILLED 2. total : Replaced integer addition with subtraction → KILLED 3. total : replaced int return with 0 for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::total → KILLED 4. total : Replaced integer addition with subtraction → KILLED |
return tp + fp + fn + tn; |
| 119 | } | |
| 120 | ||
| 121 | /** | |
| 122 | * Returns the precision. | |
| 123 | * | |
| 124 | * @return {@code tp/(tp+fp)}, or empty when nothing was predicted positive | |
| 125 | */ | |
| 126 | public OptionalDouble precision() { | |
| 127 |
2
1. precision : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::precision → KILLED 2. precision : Replaced integer addition with subtraction → KILLED |
return ratio(tp, tp + fp); |
| 128 | } | |
| 129 | ||
| 130 | /** | |
| 131 | * Returns the recall. | |
| 132 | * | |
| 133 | * @return {@code tp/(tp+fn)}, or empty when there are no true positives | |
| 134 | * in the truth (zero support) | |
| 135 | */ | |
| 136 | public OptionalDouble recall() { | |
| 137 |
2
1. recall : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::recall → KILLED 2. recall : Replaced integer addition with subtraction → KILLED |
return ratio(tp, tp + fn); |
| 138 | } | |
| 139 | ||
| 140 | /** | |
| 141 | * Returns the F1 score, the harmonic mean of precision and recall. | |
| 142 | * | |
| 143 | * @return {@code 2tp/(2tp+fp+fn)}, or empty when there is neither a | |
| 144 | * predicted nor a true positive | |
| 145 | */ | |
| 146 | public OptionalDouble f1() { | |
| 147 |
5
1. f1 : Replaced double addition with subtraction → KILLED 2. f1 : Replaced double multiplication with division → KILLED 3. f1 : Replaced double addition with subtraction → KILLED 4. f1 : Replaced double multiplication with division → KILLED 5. f1 : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::f1 → KILLED |
return ratio(TWO * tp, TWO * tp + fp + fn); |
| 148 | } | |
| 149 | ||
| 150 | /** | |
| 151 | * Returns the accuracy. | |
| 152 | * | |
| 153 | * @return {@code (tp+tn)/total}, or empty when no samples were counted | |
| 154 | */ | |
| 155 | public OptionalDouble accuracy() { | |
| 156 |
2
1. accuracy : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::accuracy → KILLED 2. accuracy : Replaced integer addition with subtraction → KILLED |
return ratio(tp + tn, total()); |
| 157 | } | |
| 158 | ||
| 159 | private static OptionalDouble ratio(double numerator, double denominator) { | |
| 160 |
4
1. ratio : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics$Counts::ratio → KILLED 2. ratio : Replaced double division with multiplication → KILLED 3. ratio : removed conditional - replaced equality check with false → KILLED 4. ratio : removed conditional - replaced equality check with true → KILLED |
return denominator == 0.0 ? OptionalDouble.empty() : OptionalDouble.of(numerator / denominator); |
| 161 | } | |
| 162 | } | |
| 163 | ||
| 164 | /** | |
| 165 | * Metrics of a single taxonomy tag. | |
| 166 | * | |
| 167 | * @param tag normalised tag | |
| 168 | * @param counts one-vs-rest counts; {@link Counts#support()} is the number of | |
| 169 | * labelled methods carrying the tag | |
| 170 | */ | |
| 171 | public record TagResult(String tag, Counts counts) { | |
| 172 | } | |
| 173 | ||
| 174 | /** | |
| 175 | * The complete metric set. | |
| 176 | * | |
| 177 | * @param relevance securityRelevant confusion counts | |
| 178 | * @param tags per-tag results sorted by tag name; includes every | |
| 179 | * tag that appears in any prediction or label | |
| 180 | * @param tagsMicro tag counts summed over all tags (micro-average basis) | |
| 181 | * @param tagsMacroF1 unweighted mean of the per-tag F1 over <em>every</em> tag | |
| 182 | * present in any prediction or label (including tags with | |
| 183 | * zero support, i.e. ones the AI invented, which score F1 | |
| 184 | * 0); empty when no tag occurs at all | |
| 185 | * @param relevanceSamples number of samples that took part in | |
| 186 | * {@code relevance} | |
| 187 | * @param tagSamples number of samples that took part in the tag metrics | |
| 188 | */ | |
| 189 | public record Result(Counts relevance, List<TagResult> tags, Counts tagsMicro, | |
| 190 | OptionalDouble tagsMacroF1, int relevanceSamples, int tagSamples) { | |
| 191 | } | |
| 192 | ||
| 193 | /** | |
| 194 | * Computes all metrics for the given samples. | |
| 195 | * | |
| 196 | * @param samples prediction/label pairs; must not be {@code null}; may be | |
| 197 | * empty, which yields all-zero counts and undefined ratios | |
| 198 | * @return the metrics; never {@code null}; per-tag list sorted by tag | |
| 199 | */ | |
| 200 | public static Result compute(Collection<Sample> samples) { | |
| 201 | Tally relevance = new Tally(); | |
| 202 | Map<String, Tally> perTag = new TreeMap<>(); | |
| 203 | int relevanceSamples = 0; | |
| 204 | int tagSamples = 0; | |
| 205 | ||
| 206 | for (Sample s : samples) { | |
| 207 |
2
1. compute : removed conditional - replaced equality check with true → KILLED 2. compute : removed conditional - replaced equality check with false → KILLED |
if (s.truthRelevant() != null) { |
| 208 |
1
1. compute : Changed increment from 1 to -1 → KILLED |
relevanceSamples++; |
| 209 | relevance.add(s.predictedRelevant(), s.truthRelevant()); | |
| 210 | } | |
| 211 |
2
1. compute : removed conditional - replaced equality check with true → KILLED 2. compute : removed conditional - replaced equality check with false → KILLED |
if (s.truthTags() != null) { |
| 212 |
1
1. compute : Changed increment from 1 to -1 → KILLED |
tagSamples++; |
| 213 | accumulateTags(perTag, normalise(s.predictedTags()), normalise(s.truthTags())); | |
| 214 | } | |
| 215 | } | |
| 216 |
1
1. compute : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics::compute → KILLED |
return summarise(relevance, relevanceSamples, perTag, tagSamples); |
| 217 | } | |
| 218 | ||
| 219 | @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") | |
| 220 | private static Result summarise(Tally relevance, int relevanceSamples, Map<String, Tally> perTag, | |
| 221 | int tagSamples) { | |
| 222 | List<TagResult> tags = new ArrayList<>(perTag.size()); | |
| 223 | int microTp = 0; | |
| 224 | int microFp = 0; | |
| 225 | int microFn = 0; | |
| 226 | double f1Sum = 0.0; | |
| 227 | int f1Count = 0; | |
| 228 | for (Map.Entry<String, Tally> e : perTag.entrySet()) { | |
| 229 | Tally c = e.getValue(); | |
| 230 |
3
1. summarise : Replaced integer subtraction with addition → KILLED 2. summarise : Replaced integer subtraction with addition → KILLED 3. summarise : Replaced integer subtraction with addition → KILLED |
Counts counts = new Counts(c.tp, c.fp, c.fn, tagSamples - c.tp - c.fp - c.fn); |
| 231 | tags.add(new TagResult(e.getKey(), counts)); | |
| 232 |
1
1. summarise : Replaced integer addition with subtraction → KILLED |
microTp += counts.tp(); |
| 233 |
1
1. summarise : Replaced integer addition with subtraction → KILLED |
microFp += counts.fp(); |
| 234 |
1
1. summarise : Replaced integer addition with subtraction → KILLED |
microFn += counts.fn(); |
| 235 | // Every tag seen in a prediction or a label counts (scikit-learn convention): a tag the AI | |
| 236 | // invented has zero support and F1 0, so hallucinated tags pull the macro average down. | |
| 237 |
1
1. summarise : Replaced double addition with subtraction → KILLED |
f1Sum += counts.f1().orElse(0.0); |
| 238 |
1
1. summarise : Changed increment from 1 to -1 → KILLED |
f1Count++; |
| 239 | } | |
| 240 |
3
1. summarise : Replaced double division with multiplication → KILLED 2. summarise : removed conditional - replaced equality check with false → KILLED 3. summarise : removed conditional - replaced equality check with true → KILLED |
OptionalDouble macro = f1Count == 0 ? OptionalDouble.empty() : OptionalDouble.of(f1Sum / f1Count); |
| 241 | // Micro tn is not meaningful across tags (it would sum unrelated one-vs-rest | |
| 242 | // negatives), so the micro row carries tn = 0 and its accuracy is not reported. | |
| 243 | Counts micro = new Counts(microTp, microFp, microFn, 0); | |
| 244 | Counts relevanceCounts = new Counts(relevance.tp, relevance.fp, relevance.fn, relevance.tn); | |
| 245 |
1
1. summarise : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics::summarise → KILLED |
return new Result(relevanceCounts, List.copyOf(tags), micro, macro, relevanceSamples, tagSamples); |
| 246 | } | |
| 247 | ||
| 248 | @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") | |
| 249 | private static void accumulateTags(Map<String, Tally> perTag, Set<String> predicted, Set<String> truth) { | |
| 250 | for (String t : predicted) { | |
| 251 |
1
1. lambda$accumulateTags$0 : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics::lambda$accumulateTags$0 → KILLED |
perTag.computeIfAbsent(t, k -> new Tally()).add(true, truth.contains(t)); |
| 252 | } | |
| 253 | for (String t : truth) { | |
| 254 |
2
1. accumulateTags : removed conditional - replaced equality check with true → KILLED 2. accumulateTags : removed conditional - replaced equality check with false → KILLED |
if (!predicted.contains(t)) { |
| 255 |
1
1. lambda$accumulateTags$1 : replaced return value with null for org/egothor/methodatlas/eval/AccuracyMetrics::lambda$accumulateTags$1 → KILLED |
perTag.computeIfAbsent(t, k -> new Tally()).add(false, true); |
| 256 | } | |
| 257 | } | |
| 258 | } | |
| 259 | ||
| 260 | /** Mutable confusion tally used while accumulating; never escapes this class. */ | |
| 261 | private static final class Tally { | |
| 262 | /* default */ int tp; | |
| 263 | /* default */ int fp; | |
| 264 | /* default */ int fn; | |
| 265 | /* default */ int tn; | |
| 266 | ||
| 267 | /* default */ void add(boolean predicted, boolean truth) { | |
| 268 |
4
1. add : removed conditional - replaced equality check with true → KILLED 2. add : removed conditional - replaced equality check with false → KILLED 3. add : removed conditional - replaced equality check with true → KILLED 4. add : removed conditional - replaced equality check with false → KILLED |
if (predicted && truth) { |
| 269 |
1
1. add : Replaced integer addition with subtraction → KILLED |
tp++; |
| 270 |
2
1. add : removed conditional - replaced equality check with true → KILLED 2. add : removed conditional - replaced equality check with false → KILLED |
} else if (predicted) { |
| 271 |
1
1. add : Replaced integer addition with subtraction → KILLED |
fp++; |
| 272 |
2
1. add : removed conditional - replaced equality check with true → KILLED 2. add : removed conditional - replaced equality check with false → KILLED |
} else if (truth) { |
| 273 |
1
1. add : Replaced integer addition with subtraction → KILLED |
fn++; |
| 274 | } else { | |
| 275 |
1
1. add : Replaced integer addition with subtraction → KILLED |
tn++; |
| 276 | } | |
| 277 | } | |
| 278 | } | |
| 279 | ||
| 280 | /** | |
| 281 | * Normalises a tag collection (trim, lower-case with {@link Locale#ROOT}, | |
| 282 | * drop blanks, de-duplicate). | |
| 283 | * | |
| 284 | * @param tags tags to normalise; {@code null} is treated as empty | |
| 285 | * @return sorted, normalised tag set; never {@code null} | |
| 286 | */ | |
| 287 | public static Set<String> normalise(Collection<String> tags) { | |
| 288 | Set<String> out = new TreeSet<>(); | |
| 289 |
2
1. normalise : removed conditional - replaced equality check with false → KILLED 2. normalise : removed conditional - replaced equality check with true → KILLED |
if (tags != null) { |
| 290 | for (String t : tags) { | |
| 291 |
4
1. normalise : removed conditional - replaced equality check with true → KILLED 2. normalise : removed conditional - replaced equality check with false → KILLED 3. normalise : removed conditional - replaced equality check with true → KILLED 4. normalise : removed conditional - replaced equality check with false → KILLED |
if (t != null && !t.isBlank()) { |
| 292 | out.add(t.trim().toLowerCase(Locale.ROOT)); | |
| 293 | } | |
| 294 | } | |
| 295 | } | |
| 296 |
1
1. normalise : replaced return value with Collections.emptySet for org/egothor/methodatlas/eval/AccuracyMetrics::normalise → KILLED |
return out; |
| 297 | } | |
| 298 | } | |
Mutations | ||
| 109 |
1.1 2.2 |
|
| 118 |
1.1 2.2 3.3 4.4 |
|
| 127 |
1.1 2.2 |
|
| 137 |
1.1 2.2 |
|
| 147 |
1.1 2.2 3.3 4.4 5.5 |
|
| 156 |
1.1 2.2 |
|
| 160 |
1.1 2.2 3.3 4.4 |
|
| 207 |
1.1 2.2 |
|
| 208 |
1.1 |
|
| 211 |
1.1 2.2 |
|
| 212 |
1.1 |
|
| 216 |
1.1 |
|
| 230 |
1.1 2.2 3.3 |
|
| 232 |
1.1 |
|
| 233 |
1.1 |
|
| 234 |
1.1 |
|
| 237 |
1.1 |
|
| 238 |
1.1 |
|
| 240 |
1.1 2.2 3.3 |
|
| 245 |
1.1 |
|
| 251 |
1.1 |
|
| 254 |
1.1 2.2 |
|
| 255 |
1.1 |
|
| 268 |
1.1 2.2 3.3 4.4 |
|
| 269 |
1.1 |
|
| 270 |
1.1 2.2 |
|
| 271 |
1.1 |
|
| 272 |
1.1 2.2 |
|
| 273 |
1.1 |
|
| 275 |
1.1 |
|
| 289 |
1.1 2.2 |
|
| 291 |
1.1 2.2 3.3 4.4 |
|
| 296 |
1.1 |