From d5f8c60884885a56df39de647513e62aab0ad38a Mon Sep 17 00:00:00 2001 From: Jaimin2687 Date: Wed, 2 Sep 2026 08:33:28 +0530 Subject: [PATCH 1/2] Fix Nuclei deduplication issue across multiple endpoints (#12397) This commit addresses the issue where Nuclei findings from different hosts were incorrectly deduplicated into a single finding. Two main fixes were implemented: 1. dojo/tools/nuclei/parser.py: Fixed dupe_host extraction for protocol-less URLs by parsing them similarly to how LocationData operates, ensuring different hosts generate distinct dupe_key values. 2. dojo/importers/default_reimporter.py & dojo/finding/deduplication.py: Added endpoint differentiation via are_locations_duplicates during re-import matching to prevent incorrect merging of findings with identical hash codes but distinct endpoints. Extended finding_locations to correctly parse LocationData DTOs. --- dojo/finding/deduplication.py | 79 +++++++++++++++++++++++++++- dojo/importers/default_reimporter.py | 9 +++- dojo/tools/nuclei/parser.py | 5 +- 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 1e8c7c0a615..bc45d986712 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -255,8 +255,83 @@ def are_urls_equal(url1, url2, fields): def finding_locations(location_refs): - """Extract URLs from a list of location references.""" - return [ref.location.url for ref in location_refs] + """ + Extract URL-like objects from a list of location references. + + Handles both saved Location_Reference instances (via .location.url) and + unsaved LocationData DTOs (via parsing .data["url"]). The returned + objects expose the same attribute names (.protocol, .host, .port, .path, + .query, .fragment, .user_info) that are_location_urls_equal relies on. + """ + from dojo.tools.locations import LocationData # noqa: PLC0415 -- lazy import, avoids circular dependency + + urls = [] + for ref in location_refs: + if isinstance(ref, LocationData): + if ref.type != "url": + continue + raw = ref.data.get("url", "") + if not raw: + continue + # Parse exactly the way the location handler will when persisting. + parseable = raw if "://" in raw else "//" + raw + try: + parsed = hyperlink.parse(parseable) + except Exception: + deduplicationLogger.debug("Failed to parse location URL %r, skipping", raw) + continue + urls.append(_ParsedLocationProxy(parsed)) + else: + # Standard path: Location_Reference -> AbstractLocation (URL model) + urls.append(ref.location.url) + return urls + + +class _ParsedLocationProxy: + + """ + Thin adapter that maps hyperlink attribute names to URL-model names. + + are_location_urls_equal accesses .protocol, .host, .port, .path, .query, + .fragment, .user_info — a hyperlink.DecodedURL uses .scheme, .host, .port, + .path, .query, .fragment, .userinfo instead. + """ + + __slots__ = ("_url",) + + def __init__(self, parsed_url): + self._url = parsed_url + + @property + def protocol(self): + return self._url.scheme or "" + + @property + def host(self): + return self._url.host or "" + + @property + def port(self): + return self._url.port + + @property + def path(self): + return "/".join(self._url.path) if self._url.path else "" + + @property + def query(self): + return self._url.query or "" + + @property + def fragment(self): + return self._url.fragment or "" + + @property + def user_info(self): + return self._url.userinfo or "" + + def __repr__(self): + return f"_ParsedLocationProxy({self._url!r})" def are_location_urls_equal(url1, url2, fields): diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index ff8a0c6af50..f1e456459a4 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -10,6 +10,7 @@ import dojo.finding.helper as finding_helper from dojo.celery_dispatch import dojo_dispatch_task from dojo.finding.deduplication import ( + are_locations_duplicates, deduplication_ordering_key, find_candidates_for_deduplication_hash, find_candidates_for_deduplication_uid_or_hash, @@ -731,6 +732,11 @@ def match_finding_to_candidate_reimport( if candidates_by_hash is None or unsaved_finding.hash_code is None: return [] matches = candidates_by_hash.get(unsaved_finding.hash_code, []) + # Findings that share a hash code but differ by endpoint should not + # be merged during reimport. This mirrors the endpoint check that + # the import-time deduplication logic already performs via + # get_matches_from_hash_candidates / are_locations_duplicates. + matches = [m for m in matches if are_locations_duplicates(unsaved_finding, m)] return sorted(matches, key=lambda f: f.id) if self.deduplication_algorithm == "unique_id_from_tool": @@ -752,7 +758,8 @@ def match_finding_to_candidate_reimport( if unsaved_finding.hash_code is not None: hash_matches = candidates_by_hash.get(unsaved_finding.hash_code, []) for match in hash_matches: - matches_by_id[match.id] = match + if are_locations_duplicates(unsaved_finding, match): + matches_by_id[match.id] = match if unsaved_finding.unique_id_from_tool is not None: uid_matches = candidates_by_uid.get(unsaved_finding.unique_id_from_tool, []) diff --git a/dojo/tools/nuclei/parser.py b/dojo/tools/nuclei/parser.py index 039dd4a6ab7..7ef468c519c 100644 --- a/dojo/tools/nuclei/parser.py +++ b/dojo/tools/nuclei/parser.py @@ -155,7 +155,10 @@ def get_findings(self, filename, test): ) if locations_enabled(): - dupe_host = (urlparse(matched).hostname or "") if matched else "" + # Prepend "//" for protocol-less URLs so urlparse extracts + # the hostname correctly (mirrors LocationData construction). + parseable = matched if "://" in matched else "//" + matched + dupe_host = (urlparse(parseable).hostname or "") if matched else "" else: # TODO: Delete this after the move to Locations dupe_host = str(location.host) if location else "" From b5f37d2f574a6d65a70d17acbc2d2b04a7e4ec5d Mon Sep 17 00:00:00 2001 From: Jaimin2687 Date: Wed, 2 Sep 2026 14:56:06 +0530 Subject: [PATCH 2/2] Revert reimporter endpoint filtering that broke same-batch matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The are_locations_duplicates check in match_finding_to_candidate_reimport incorrectly rejected hash_code matches between findings in the same report that have different endpoints but should merge (endpoint accumulation). This broke test_reimport_prefetch tests that rely on same-hash findings within one report being matched so their endpoints accumulate on a single finding. The parser fix (dupe_host extraction for protocol-less URLs) alone is sufficient to resolve #12397 — it ensures different hosts produce distinct hash codes, preventing incorrect deduplication at the source. --- dojo/finding/deduplication.py | 79 +--------------------------- dojo/importers/default_reimporter.py | 9 +--- 2 files changed, 3 insertions(+), 85 deletions(-) diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index bc45d986712..1e8c7c0a615 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -255,83 +255,8 @@ def are_urls_equal(url1, url2, fields): def finding_locations(location_refs): - """ - Extract URL-like objects from a list of location references. - - Handles both saved Location_Reference instances (via .location.url) and - unsaved LocationData DTOs (via parsing .data["url"]). The returned - objects expose the same attribute names (.protocol, .host, .port, .path, - .query, .fragment, .user_info) that are_location_urls_equal relies on. - """ - from dojo.tools.locations import LocationData # noqa: PLC0415 -- lazy import, avoids circular dependency - - urls = [] - for ref in location_refs: - if isinstance(ref, LocationData): - if ref.type != "url": - continue - raw = ref.data.get("url", "") - if not raw: - continue - # Parse exactly the way the location handler will when persisting. - parseable = raw if "://" in raw else "//" + raw - try: - parsed = hyperlink.parse(parseable) - except Exception: - deduplicationLogger.debug("Failed to parse location URL %r, skipping", raw) - continue - urls.append(_ParsedLocationProxy(parsed)) - else: - # Standard path: Location_Reference -> AbstractLocation (URL model) - urls.append(ref.location.url) - return urls - - -class _ParsedLocationProxy: - - """ - Thin adapter that maps hyperlink attribute names to URL-model names. - - are_location_urls_equal accesses .protocol, .host, .port, .path, .query, - .fragment, .user_info — a hyperlink.DecodedURL uses .scheme, .host, .port, - .path, .query, .fragment, .userinfo instead. - """ - - __slots__ = ("_url",) - - def __init__(self, parsed_url): - self._url = parsed_url - - @property - def protocol(self): - return self._url.scheme or "" - - @property - def host(self): - return self._url.host or "" - - @property - def port(self): - return self._url.port - - @property - def path(self): - return "/".join(self._url.path) if self._url.path else "" - - @property - def query(self): - return self._url.query or "" - - @property - def fragment(self): - return self._url.fragment or "" - - @property - def user_info(self): - return self._url.userinfo or "" - - def __repr__(self): - return f"_ParsedLocationProxy({self._url!r})" + """Extract URLs from a list of location references.""" + return [ref.location.url for ref in location_refs] def are_location_urls_equal(url1, url2, fields): diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index f1e456459a4..ff8a0c6af50 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -10,7 +10,6 @@ import dojo.finding.helper as finding_helper from dojo.celery_dispatch import dojo_dispatch_task from dojo.finding.deduplication import ( - are_locations_duplicates, deduplication_ordering_key, find_candidates_for_deduplication_hash, find_candidates_for_deduplication_uid_or_hash, @@ -732,11 +731,6 @@ def match_finding_to_candidate_reimport( if candidates_by_hash is None or unsaved_finding.hash_code is None: return [] matches = candidates_by_hash.get(unsaved_finding.hash_code, []) - # Findings that share a hash code but differ by endpoint should not - # be merged during reimport. This mirrors the endpoint check that - # the import-time deduplication logic already performs via - # get_matches_from_hash_candidates / are_locations_duplicates. - matches = [m for m in matches if are_locations_duplicates(unsaved_finding, m)] return sorted(matches, key=lambda f: f.id) if self.deduplication_algorithm == "unique_id_from_tool": @@ -758,8 +752,7 @@ def match_finding_to_candidate_reimport( if unsaved_finding.hash_code is not None: hash_matches = candidates_by_hash.get(unsaved_finding.hash_code, []) for match in hash_matches: - if are_locations_duplicates(unsaved_finding, match): - matches_by_id[match.id] = match + matches_by_id[match.id] = match if unsaved_finding.unique_id_from_tool is not None: uid_matches = candidates_by_uid.get(unsaved_finding.unique_id_from_tool, [])