From 57c19ca45f08d25f0c72bc621a2d7c1c0a34918a Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Wed, 18 Nov 2026 16:56:25 +0100 Subject: [PATCH] Use the full hash when deduplicating and filtering bench cases hash_from_json_repr keeps only 5 hex digits (20 bits). remove_duplicated_bench_cases keys cases by that short hash, so two different cases with the same prefix silently replace each other. Running configs/regular, configs/weekly and configs/experiments together has 1679 distinct cases but kept 1678: ExtraTreesClassifier/mnist and a raft NearestNeighbors/fashion_mnist case share a9f34. Use the full digest for deduplication and for bench_case_filter. The short hash is still used where it is only part of a display name. Fixes #228 Signed-off-by: Dezhen Lu --- sklbench/utils/common.py | 2 +- sklbench/utils/config.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sklbench/utils/common.py b/sklbench/utils/common.py index ae666007..cee60100 100755 --- a/sklbench/utils/common.py +++ b/sklbench/utils/common.py @@ -74,7 +74,7 @@ def read_output_from_command(command: str) -> Tuple[int, str, str]: ) -def hash_from_json_repr(x: JsonTypesUnion, hash_limit: int = 5) -> str: +def hash_from_json_repr(x: JsonTypesUnion, hash_limit: Union[int, None] = 5) -> str: h = hashlib.sha256() h.update(bytes(json.dumps(x), encoding="utf-8")) return h.hexdigest()[:hash_limit] diff --git a/sklbench/utils/config.py b/sklbench/utils/config.py index 1010b830..45106fb9 100644 --- a/sklbench/utils/config.py +++ b/sklbench/utils/config.py @@ -206,9 +206,11 @@ def expand_template( def remove_duplicated_bench_cases(bench_cases: List[BenchCase]) -> List[BenchCase]: + # full hash: the default 5 hex digits collide for different cases + # in runs with ~1000+ cases, which silently dropped one of them hash_map = dict() for bench_case in bench_cases: - hash_map[hash_from_json_repr(bench_case)] = bench_case + hash_map[hash_from_json_repr(bench_case, hash_limit=None)] = bench_case return list(hash_map.values()) @@ -216,9 +218,9 @@ def bench_case_filter(bench_case: BenchCase, filters: List[BenchCase]): # filtering is implemented by comparison of # benchmark case and case merged with filters: # filtering is passed if one of merged cases has same hash as original - original_hash = hash_from_json_repr(bench_case) + original_hash = hash_from_json_repr(bench_case, hash_limit=None) filtered_hashes = [ - hash_from_json_repr(merge_dicts(bench_case, bench_filter)) + hash_from_json_repr(merge_dicts(bench_case, bench_filter), hash_limit=None) for bench_filter in filters ] return original_hash in filtered_hashes or len(filtered_hashes) == 0