diff --git a/core/scanner.py b/core/scanner.py index f6e7a84..f033629 100644 --- a/core/scanner.py +++ b/core/scanner.py @@ -550,7 +550,22 @@ class Scanner: self._collect_node_attributes(source, provider_name, rel_type, rel_target, raw_data, node_attributes[source]) - if _is_valid_ip(rel_target): + if isinstance(rel_target, list): + # If the target is a list, iterate and process each item + for single_target in rel_target: + if _is_valid_ip(single_target): + self.graph.add_node(single_target, NodeType.IP) + if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data): + print(f"Added IP relationship: {source} -> {single_target} ({rel_type})") + discovered_targets.add(single_target) + elif _is_valid_domain(single_target): + self.graph.add_node(single_target, NodeType.DOMAIN) + if self.graph.add_edge(source, single_target, rel_type, confidence, provider_name, raw_data): + print(f"Added domain relationship: {source} -> {single_target} ({rel_type})") + discovered_targets.add(single_target) + self._collect_node_attributes(single_target, provider_name, rel_type, source, raw_data, node_attributes[single_target]) + + elif _is_valid_ip(rel_target): self.graph.add_node(rel_target, NodeType.IP) if self.graph.add_edge(source, rel_target, rel_type, confidence, provider_name, raw_data): print(f"Added IP relationship: {source} -> {rel_target} ({rel_type})") @@ -573,6 +588,7 @@ class Scanner: return discovered_targets, False + def _create_large_entity(self, source: str, provider_name: str, results: List, current_depth: int) -> Set[str]: """Create a large entity node and returns the members for DNS processing.""" entity_id = f"large_entity_{provider_name}_{hash(source) & 0x7FFFFFFF}" diff --git a/dump.rdb b/dump.rdb index 2fd4d59..b85d9b8 100644 Binary files a/dump.rdb and b/dump.rdb differ diff --git a/providers/crtsh_provider.py b/providers/crtsh_provider.py index 58425f0..e24edce 100644 --- a/providers/crtsh_provider.py +++ b/providers/crtsh_provider.py @@ -213,17 +213,11 @@ class CrtShProvider(BaseProvider): cert_domains = self._extract_domains_from_certificate(cert_data) # Add all domains from this certificate to our tracking + all_discovered_domains.update(cert_domains) for cert_domain in cert_domains: - # Additional stop check during domain processing - if i % 20 == 0 and self._stop_event and self._stop_event.is_set(): - print(f"CrtSh domain processing cancelled for domain: {domain}") - break - if not _is_valid_domain(cert_domain): continue - all_discovered_domains.add(cert_domain) - # Initialize domain certificate list if needed if cert_domain not in domain_certificates: domain_certificates[cert_domain] = [] diff --git a/providers/dns_provider.py b/providers/dns_provider.py index 93240a5..d73ef6c 100644 --- a/providers/dns_provider.py +++ b/providers/dns_provider.py @@ -121,6 +121,9 @@ class DNSProvider(BaseProvider): discovery_method="reverse_dns_lookup" ) + except resolver.NXDOMAIN: + self.failed_requests += 1 + self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: NXDOMAIN") except Exception as e: self.failed_requests += 1 self.logger.logger.debug(f"Reverse DNS lookup failed for {ip}: {e}")