Merge branch 'main' of https://git.cc24.dev/mstoeck3/dnsrecon

.env.example

@@ -29,6 +29,6 @@ MAX_CONCURRENT_REQUESTS=5
 # The number of results from a provider that triggers the "large entity" grouping.
 LARGE_ENTITY_THRESHOLD=100
 # The number of times to retry a target if a provider fails.
-MAX_RETRIES_PER_TARGET=3
+MAX_RETRIES_PER_TARGET=8
 # How long cached provider responses are stored (in hours).
 CACHE_EXPIRY_HOURS=12

cache/crtsh/coturn_ms-it-services_de.json

@@ -0,0 +1 @@
+{"domain":"coturn.ms-it-services.de","first_cached":"2025-09-14T21:03:44.169328+00:00","last_upstream_query":"2025-09-14T21:03:44.169332+00:00","upstream_query_count":1,"certificates":[{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"coturn.ms-it-services.de","name_value":"coturn.ms-it-services.de","id":14781803935,"entry_timestamp":"2024-10-03T09:53:12.473","not_before":"2024-10-03T08:54:42","not_after":"2025-01-01T08:54:41","serial_number":"0395c04e522a2715eebcb7fc4ffb3da1fdba","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"coturn.ms-it-services.de","name_value":"coturn.ms-it-services.de","id":14781794097,"entry_timestamp":"2024-10-03T09:53:12.142","not_before":"2024-10-03T08:54:42","not_after":"2025-01-01T08:54:41","serial_number":"0395c04e522a2715eebcb7fc4ffb3da1fdba","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"coturn.ms-it-services.de","name_value":"coturn.ms-it-services.de","id":14773518990,"entry_timestamp":"2024-10-02T19:20:49.687","not_before":"2024-10-02T18:22:19","not_after":"2024-12-31T18:22:18","serial_number":"04f26242ac1b2ac659ac2e19ae2522ce3274","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"coturn.ms-it-services.de","name_value":"coturn.ms-it-services.de","id":14773501988,"entry_timestamp":"2024-10-02T19:20:49.356","not_before":"2024-10-02T18:22:19","not_after":"2024-12-31T18:22:18","serial_number":"04f26242ac1b2ac659ac2e19ae2522ce3274","result_count":2}]}

cache/crtsh/mx00_ionos_de.json

@@ -0,0 +1 @@
+{"domain":"mx00.ionos.de","first_cached":"2025-09-14T21:05:21.043082+00:00","last_upstream_query":"2025-09-14T21:05:21.043085+00:00","upstream_query_count":1,"certificates":[{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":17921174855,"entry_timestamp":"2025-04-18T11:32:56.685","not_before":"2024-05-14T10:13:42","not_after":"2025-05-18T23:59:59","serial_number":"01f21195d95cb3f63712c59f40b2f75c","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":17755974719,"entry_timestamp":"2025-04-10T06:20:35.546","not_before":"2025-04-10T06:20:33","not_after":"2026-04-14T23:59:59","serial_number":"27efd5b7b17610e4ae86d40dea979ad7","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":13038507917,"entry_timestamp":"2024-05-14T10:13:43.64","not_before":"2024-05-14T10:13:42","not_after":"2025-05-18T23:59:59","serial_number":"01f21195d95cb3f63712c59f40b2f75c","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":9700366741,"entry_timestamp":"2023-06-20T11:08:14.981","not_before":"2023-06-20T11:08:11","not_after":"2024-06-24T23:59:59","serial_number":"153f3cd769500d3eebec07c90476d817","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":7107715703,"entry_timestamp":"2022-07-12T10:00:03.026","not_before":"2022-07-12T10:00:01","not_after":"2023-07-16T23:59:59","serial_number":"210a3739eb290b28d1199f6a9c04d294","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":4981516894,"entry_timestamp":"2021-08-04T08:35:13.1","not_before":"2021-08-04T08:35:11","not_after":"2022-08-08T23:59:59","serial_number":"11ae5449f5d5cc2a5ec198105748f927","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":2864694015,"entry_timestamp":"2020-05-28T07:54:38.37","not_before":"2020-05-28T07:54:37","not_after":"2022-06-02T23:59:59","serial_number":"325b678601aae53e99926c1834988786","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":2857249553,"entry_timestamp":"2020-05-26T09:01:33.944","not_before":"2020-05-26T09:01:33","not_after":"2022-05-31T23:59:59","serial_number":"0ceee698ba17a744881fbb3998c05748","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":2857204010,"entry_timestamp":"2020-05-26T08:46:52.8","not_before":"2020-05-26T08:46:52","not_after":"2022-05-31T23:59:59","serial_number":"0ac520de92fde640943c1d2c0eb2f6e8","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx00.ionos.de","id":2856876813,"entry_timestamp":"2020-05-26T06:58:49.881","not_before":"2020-05-26T06:58:49","not_after":"2022-05-31T23:59:59","serial_number":"2f9201af62a3408df1579b8a39c0f7b6","result_count":1}]}

cache/crtsh/mx01_ionos_de.json

@@ -0,0 +1 @@
+{"domain":"mx01.ionos.de","first_cached":"2025-09-14T21:05:58.567956+00:00","last_upstream_query":"2025-09-14T21:05:58.567958+00:00","upstream_query_count":1,"certificates":[{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":17921174855,"entry_timestamp":"2025-04-18T11:32:56.685","not_before":"2024-05-14T10:13:42","not_after":"2025-05-18T23:59:59","serial_number":"01f21195d95cb3f63712c59f40b2f75c","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":17755974719,"entry_timestamp":"2025-04-10T06:20:35.546","not_before":"2025-04-10T06:20:33","not_after":"2026-04-14T23:59:59","serial_number":"27efd5b7b17610e4ae86d40dea979ad7","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":13038507917,"entry_timestamp":"2024-05-14T10:13:43.64","not_before":"2024-05-14T10:13:42","not_after":"2025-05-18T23:59:59","serial_number":"01f21195d95cb3f63712c59f40b2f75c","result_count":1},{"issuer_ca_id":245439,"issuer_name":"C=DE, O=Deutsche Telekom Security GmbH, CN=Telekom Security ServerID OV Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":9700366741,"entry_timestamp":"2023-06-20T11:08:14.981","not_before":"2023-06-20T11:08:11","not_after":"2024-06-24T23:59:59","serial_number":"153f3cd769500d3eebec07c90476d817","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":7107715703,"entry_timestamp":"2022-07-12T10:00:03.026","not_before":"2022-07-12T10:00:01","not_after":"2023-07-16T23:59:59","serial_number":"210a3739eb290b28d1199f6a9c04d294","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":4981516894,"entry_timestamp":"2021-08-04T08:35:13.1","not_before":"2021-08-04T08:35:11","not_after":"2022-08-08T23:59:59","serial_number":"11ae5449f5d5cc2a5ec198105748f927","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":2864694015,"entry_timestamp":"2020-05-28T07:54:38.37","not_before":"2020-05-28T07:54:37","not_after":"2022-06-02T23:59:59","serial_number":"325b678601aae53e99926c1834988786","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":2857249553,"entry_timestamp":"2020-05-26T09:01:33.944","not_before":"2020-05-26T09:01:33","not_after":"2022-05-31T23:59:59","serial_number":"0ceee698ba17a744881fbb3998c05748","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":2857204010,"entry_timestamp":"2020-05-26T08:46:52.8","not_before":"2020-05-26T08:46:52","not_after":"2022-05-31T23:59:59","serial_number":"0ac520de92fde640943c1d2c0eb2f6e8","result_count":1},{"issuer_ca_id":6069,"issuer_name":"C=DE, O=T-Systems International GmbH, OU=T-Systems Trust Center, ST=Nordrhein Westfalen, postalCode=57250, L=Netphen, street=Untere Industriestr. 20, CN=TeleSec ServerPass Class 2 CA","common_name":"mx.kundenserver.de","name_value":"mx01.ionos.de","id":2856876813,"entry_timestamp":"2020-05-26T06:58:49.881","not_before":"2020-05-26T06:58:49","not_after":"2022-05-31T23:59:59","serial_number":"2f9201af62a3408df1579b8a39c0f7b6","result_count":1}]}

cache/crtsh/signaling_mikoshi_de.json

@@ -0,0 +1 @@
+{"domain":"signaling.mikoshi.de","first_cached":"2025-09-14T21:05:14.189157+00:00","last_upstream_query":"2025-09-14T21:05:14.189159+00:00","upstream_query_count":1,"certificates":[{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":19208276669,"entry_timestamp":"2025-06-23T18:21:11.885","not_before":"2025-06-23T17:22:40","not_after":"2025-09-21T17:22:39","serial_number":"0542fce9cb99bb1c1d18e5e452c90d850936","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":19208276557,"entry_timestamp":"2025-06-23T18:21:11.215","not_before":"2025-06-23T17:22:40","not_after":"2025-09-21T17:22:39","serial_number":"0542fce9cb99bb1c1d18e5e452c90d850936","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":19208272013,"entry_timestamp":"2025-06-23T18:20:29.619","not_before":"2025-06-23T17:21:56","not_after":"2025-09-21T17:21:55","serial_number":"0540be6c8cb99dcaa5492af7b934f40466f9","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":19208263539,"entry_timestamp":"2025-06-23T18:20:26.82","not_before":"2025-06-23T17:21:56","not_after":"2025-09-21T17:21:55","serial_number":"0540be6c8cb99dcaa5492af7b934f40466f9","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":17428224909,"entry_timestamp":"2025-03-23T15:21:03.771","not_before":"2025-03-23T14:22:33","not_after":"2025-06-21T14:22:32","serial_number":"0537513d69487cead5f018b5322aa54fb52e","result_count":2},{"issuer_ca_id":295819,"issuer_name":"C=US, O=Let's Encrypt, CN=E6","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":17346408492,"entry_timestamp":"2025-03-23T15:21:03.617","not_before":"2025-03-23T14:22:33","not_after":"2025-06-21T14:22:32","serial_number":"0537513d69487cead5f018b5322aa54fb52e","result_count":2},{"issuer_ca_id":295815,"issuer_name":"C=US, O=Let's Encrypt, CN=R11","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":17170806575,"entry_timestamp":"2025-02-13T00:01:39.507","not_before":"2025-02-12T23:03:07","not_after":"2025-05-13T23:03:06","serial_number":"04a3141d80551d2287647211445f32131b19","result_count":2},{"issuer_ca_id":295815,"issuer_name":"C=US, O=Let's Encrypt, CN=R11","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":16704105641,"entry_timestamp":"2025-02-13T00:01:37.463","not_before":"2025-02-12T23:03:07","not_after":"2025-05-13T23:03:06","serial_number":"04a3141d80551d2287647211445f32131b19","result_count":2},{"issuer_ca_id":295815,"issuer_name":"C=US, O=Let's Encrypt, CN=R11","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":17122350772,"entry_timestamp":"2025-02-09T15:19:01.645","not_before":"2025-02-09T14:20:31","not_after":"2025-05-10T14:20:30","serial_number":"0456c53707eac4c8e79109b09eb9257674ef","result_count":2},{"issuer_ca_id":295815,"issuer_name":"C=US, O=Let's Encrypt, CN=R11","common_name":"signaling.mikoshi.de","name_value":"signaling.mikoshi.de","id":16635514012,"entry_timestamp":"2025-02-09T15:19:01.438","not_before":"2025-02-09T14:20:31","not_after":"2025-05-10T14:20:30","serial_number":"0456c53707eac4c8e79109b09eb9257674ef","result_count":2}]}

config.py

@@ -19,10 +19,10 @@ class Config:
         
         # --- General Settings ---
         self.default_recursion_depth = 2
-        self.default_timeout = 15
+        self.default_timeout = 20
         self.max_concurrent_requests = 5
         self.large_entity_threshold = 100
-        self.max_retries_per_target = 3
+        self.max_retries_per_target = 8
         self.cache_expiry_hours = 12
         
         # --- Provider Caching Settings ---

providers/crtsh_provider.py

@@ -5,46 +5,32 @@ import re
 import os
 from pathlib import Path
 from typing import List, Dict, Any, Tuple, Set
+from urllib.parse import quote
 from datetime import datetime, timezone
-
-# New dependency required for this provider
-try:
-    import psycopg2
-    import psycopg2.extras
-    PSYCOPG2_AVAILABLE = True
-except ImportError:
-    PSYCOPG2_AVAILABLE = False
+import requests
 
 from .base_provider import BaseProvider
 from utils.helpers import _is_valid_domain
 
-# We use requests only to raise the same exception type for compatibility with core retry logic
-import requests
-
 
 class CrtShProvider(BaseProvider):
     """
-    Provider for querying crt.sh certificate transparency database via its public PostgreSQL endpoint.
-    This version is designed to be a drop-in, high-performance replacement for the API-based provider.
-    It preserves the same caching and data processing logic.
+    Provider for querying crt.sh certificate transparency database.
+    Now uses session-specific configuration and caching with accumulative behavior.
     """
     
     def __init__(self, name=None, session_config=None):
-        """Initialize CrtShDB provider with session-specific configuration."""
+        """Initialize CrtSh provider with session-specific configuration."""
         super().__init__(
             name="crtsh",
-            rate_limit=0,  # No rate limit for direct DB access
-            timeout=60,    # Increased timeout for potentially long DB queries
+            rate_limit=60,
+            timeout=15,
             session_config=session_config
         )
-        # Database connection details
-        self.db_host = "crt.sh"
-        self.db_port = 5432
-        self.db_name = "certwatch"
-        self.db_user = "guest"
+        self.base_url = "https://crt.sh/"
         self._stop_event = None
         
-        # Initialize cache directory (same as original provider)
+        # Initialize cache directory
         self.cache_dir = Path('cache') / 'crtsh'
         self.cache_dir.mkdir(parents=True, exist_ok=True)
     
@@ -54,7 +40,7 @@ class CrtShProvider(BaseProvider):
     
     def get_display_name(self) -> str:
         """Return the provider display name for the UI."""
-        return "crt.sh (DB)"
+        return "crt.sh"
 
     def requires_api_key(self) -> bool:
         """Return True if the provider requires an API key."""
@@ -66,161 +52,23 @@ class CrtShProvider(BaseProvider):
 
     def is_available(self) -> bool:
         """
-        Check if the provider can be used. Requires the psycopg2 library.
+        Check if the provider is configured to be used.
+        This method is intentionally simple and does not perform a network request
+        to avoid blocking application startup.
         """
-        if not PSYCOPG2_AVAILABLE:
-            self.logger.logger.warning("psycopg2 library not found. CrtShDBProvider is unavailable. "
-                                      "Please run 'pip install psycopg2-binary'.")
-            return False
         return True
     
-    def _query_crtsh(self, domain: str) -> List[Dict[str, Any]]:
-        """
-        Query the crt.sh PostgreSQL database for raw certificate data.
-        Raises exceptions for DB/network errors to allow core logic to retry.
-        """
-        conn = None
-        certificates = []
-        
-        # SQL Query to find all certificate IDs related to the domain (including subdomains),
-        # then retrieve comprehensive details for each certificate, mimicking the JSON API structure.
-        sql_query = """
-        WITH certificates_of_interest AS (
-            SELECT DISTINCT ci.certificate_id
-            FROM certificate_identity ci
-            WHERE ci.name_value ILIKE %(domain_wildcard)s OR ci.name_value = %(domain)s
-        )
-        SELECT
-            c.id,
-            c.serial_number,
-            c.not_before,
-            c.not_after,
-            (SELECT min(entry_timestamp) FROM ct_log_entry cle WHERE cle.certificate_id = c.id) as entry_timestamp,
-            ca.id as issuer_ca_id,
-            ca.name as issuer_name,
-            (SELECT array_to_string(array_agg(DISTINCT ci.name_value), E'\n') FROM certificate_identity ci WHERE ci.certificate_id = c.id) as name_value,
-            (SELECT name_value FROM certificate_identity ci WHERE ci.certificate_id = c.id AND ci.name_type = 'commonName' LIMIT 1) as common_name
-        FROM
-            certificate c
-        JOIN ca ON c.issuer_ca_id = ca.id
-        WHERE c.id IN (SELECT certificate_id FROM certificates_of_interest);
-        """
-        
-        try:
-            conn = psycopg2.connect(
-                dbname=self.db_name,
-                user=self.db_user,
-                host=self.db_host,
-                port=self.db_port,
-                connect_timeout=self.timeout
-            )
-            
-            with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cursor:
-                cursor.execute(sql_query, {'domain': domain, 'domain_wildcard': f'%.{domain}'})
-                results = cursor.fetchall()
-                certificates = [dict(row) for row in results]
-                
-            self.logger.logger.info(f"crt.sh DB query for '{domain}' returned {len(certificates)} certificates.")
-
-        except psycopg2.Error as e:
-            self.logger.logger.error(f"PostgreSQL query failed for {domain}: {e}")
-            # Raise a RequestException to be compatible with the existing retry logic in the core application
-            raise requests.exceptions.RequestException(f"PostgreSQL query failed: {e}") from e
-        finally:
-            if conn:
-                conn.close()
-        
-        return certificates
-
-    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
-        """
-        Query crt.sh for certificates containing the domain with caching support.
-        Properly raises exceptions for network errors to allow core logic retries.
-        """
-        if not _is_valid_domain(domain):
-            return []
-        
-        if self._stop_event and self._stop_event.is_set():
-            return []
-        
-        cache_file = self._get_cache_file_path(domain)
-        cache_status = self._get_cache_status(cache_file)
-        
-        certificates = []
-        
-        try:
-            if cache_status == "fresh":
-                certificates = self._load_cached_certificates(cache_file)
-                self.logger.logger.info(f"Using cached data for {domain} ({len(certificates)} certificates)")
-                
-            elif cache_status == "not_found":
-                # Fresh query from DB, create new cache
-                certificates = self._query_crtsh(domain)
-                if certificates:
-                    self._create_cache_file(cache_file, domain, self._serialize_certs_for_cache(certificates))
-                else:
-                    self.logger.logger.info(f"No certificates found for {domain}, not caching")
-                    
-            elif cache_status == "stale":
-                try:
-                    new_certificates = self._query_crtsh(domain)
-                    if new_certificates:
-                        certificates = self._append_to_cache(cache_file, self._serialize_certs_for_cache(new_certificates))
-                    else:
-                        certificates = self._load_cached_certificates(cache_file)
-                except requests.exceptions.RequestException:
-                    certificates = self._load_cached_certificates(cache_file)
-                    if certificates:
-                        self.logger.logger.warning(f"DB query failed for {domain}, using stale cache data.")
-                    else:
-                        raise
-            
-        except requests.exceptions.RequestException as e:
-            # Re-raise so core logic can retry
-            self.logger.logger.error(f"DB query failed for {domain}: {e}")
-            raise e
-        except json.JSONDecodeError as e:
-            # JSON parsing errors from cache should also be handled
-            self.logger.logger.error(f"Failed to parse JSON from cache for {domain}: {e}")
-            raise e
-        
-        if self._stop_event and self._stop_event.is_set():
-            return []
-        
-        if not certificates:
-            return []
-        
-        return self._process_certificates_to_relationships(domain, certificates)
-        
-    def _serialize_certs_for_cache(self, certificates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """
-        Serialize certificate data for JSON caching, converting datetime objects to ISO strings.
-        """
-        serialized_certs = []
-        for cert in certificates:
-            serialized_cert = cert.copy()
-            for key in ['not_before', 'not_after', 'entry_timestamp']:
-                if isinstance(serialized_cert.get(key), datetime):
-                    # Ensure datetime is timezone-aware before converting
-                    dt_obj = serialized_cert[key]
-                    if dt_obj.tzinfo is None:
-                        dt_obj = dt_obj.replace(tzinfo=timezone.utc)
-                    serialized_cert[key] = dt_obj.isoformat()
-            serialized_certs.append(serialized_cert)
-        return serialized_certs
-
-    # --- All methods below are copied directly from the original CrtShProvider ---
-    # They are compatible because _query_crtsh returns data in the same format
-    # as the original _query_crtsh_api method. A small adjustment is made to
-    # _parse_certificate_date to handle datetime objects directly from the DB.
-
     def _get_cache_file_path(self, domain: str) -> Path:
         """Generate cache file path for a domain."""
+        # Sanitize domain for filename safety
         safe_domain = domain.replace('.', '_').replace('/', '_').replace('\\', '_')
         return self.cache_dir / f"{safe_domain}.json"
     
     def _get_cache_status(self, cache_file_path: Path) -> str:
-        """Check cache status for a domain."""
+        """
+        Check cache status for a domain.
+        Returns: 'not_found', 'fresh', or 'stale'
+        """
         if not cache_file_path.exists():
             return "not_found"
         
@@ -230,7 +78,7 @@ class CrtShProvider(BaseProvider):
             
             last_query_str = cache_data.get("last_upstream_query")
             if not last_query_str:
-                return "stale"
+                return "stale"  # Invalid cache format
             
             last_query = datetime.fromisoformat(last_query_str.replace('Z', '+00:00'))
             hours_since_query = (datetime.now(timezone.utc) - last_query).total_seconds() / 3600
@@ -255,6 +103,24 @@ class CrtShProvider(BaseProvider):
             self.logger.logger.error(f"Failed to load cached certificates from {cache_file_path}: {e}")
             return []
     
+    def _query_crtsh_api(self, domain: str) -> List[Dict[str, Any]]:
+        """
+        Query crt.sh API for raw certificate data.
+        Raises exceptions for network errors to allow core logic to retry.
+        """
+        url = f"{self.base_url}?q={quote(domain)}&output=json"
+        response = self.make_request(url, target_indicator=domain)
+        
+        if not response or response.status_code != 200:
+            # This could be a temporary error - raise exception so core can retry
+            raise requests.exceptions.RequestException(f"crt.sh API returned status {response.status_code if response else 'None'}")
+        
+        certificates = response.json()
+        if not certificates:
+            return []
+        
+        return certificates
+    
     def _create_cache_file(self, cache_file_path: Path, domain: str, certificates: List[Dict[str, Any]]) -> None:
         """Create new cache file with certificates."""
         try:
@@ -265,20 +131,27 @@ class CrtShProvider(BaseProvider):
                 "upstream_query_count": 1,
                 "certificates": certificates
             }
+            
             cache_file_path.parent.mkdir(parents=True, exist_ok=True)
             with open(cache_file_path, 'w') as f:
                 json.dump(cache_data, f, separators=(',', ':'))
+                
             self.logger.logger.info(f"Created cache file for {domain} with {len(certificates)} certificates")
+            
         except Exception as e:
             self.logger.logger.warning(f"Failed to create cache file for {domain}: {e}")
     
     def _append_to_cache(self, cache_file_path: Path, new_certificates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
         """Append new certificates to existing cache and return all certificates."""
         try:
+            # Load existing cache
             with open(cache_file_path, 'r') as f:
                 cache_data = json.load(f)
             
+            # Track existing certificate IDs to avoid duplicates
             existing_ids = {cert.get('id') for cert in cache_data.get('certificates', [])}
+            
+            # Add only new certificates
             added_count = 0
             for cert in new_certificates:
                 cert_id = cert.get('id')
@@ -287,141 +160,314 @@ class CrtShProvider(BaseProvider):
                     existing_ids.add(cert_id)
                     added_count += 1
             
+            # Update metadata
             cache_data['last_upstream_query'] = datetime.now(timezone.utc).isoformat()
             cache_data['upstream_query_count'] = cache_data.get('upstream_query_count', 0) + 1
             
+            # Write updated cache
             with open(cache_file_path, 'w') as f:
                 json.dump(cache_data, f, separators=(',', ':'))
             
             total_certs = len(cache_data['certificates'])
             self.logger.logger.info(f"Appended {added_count} new certificates to cache. Total: {total_certs}")
+            
             return cache_data['certificates']
+            
         except Exception as e:
             self.logger.logger.warning(f"Failed to append to cache: {e}")
-            return new_certificates
+            return new_certificates  # Fallback to new certificates only
     
     def _parse_issuer_organization(self, issuer_dn: str) -> str:
-        """Parse the issuer Distinguished Name to extract just the organization name."""
-        if not issuer_dn: return issuer_dn
+        """
+        Parse the issuer Distinguished Name to extract just the organization name.
+        
+        Args:
+            issuer_dn: Full issuer DN string (e.g., "C=US, O=Let's Encrypt, CN=R11")
+            
+        Returns:
+            Organization name (e.g., "Let's Encrypt") or original string if parsing fails
+        """
+        if not issuer_dn:
+            return issuer_dn
+        
         try:
+            # Split by comma and look for O= component
             components = [comp.strip() for comp in issuer_dn.split(',')]
+            
             for component in components:
                 if component.startswith('O='):
+                    # Extract the value after O=
                     org_name = component[2:].strip()
+                    # Remove quotes if present
                     if org_name.startswith('"') and org_name.endswith('"'):
                         org_name = org_name[1:-1]
                     return org_name
+            
+            # If no O= component found, return the original string
             return issuer_dn
+            
         except Exception as e:
             self.logger.logger.debug(f"Failed to parse issuer DN '{issuer_dn}': {e}")
             return issuer_dn
     
-    def _parse_certificate_date(self, date_input: Any) -> datetime:
+    def _parse_certificate_date(self, date_string: str) -> datetime:
         """
-        Parse certificate date from various formats (string from cache, datetime from DB).
-        """
-        if isinstance(date_input, datetime):
-            # If it's already a datetime object from the DB, just ensure it's UTC
-            if date_input.tzinfo is None:
-                return date_input.replace(tzinfo=timezone.utc)
-            return date_input
+        Parse certificate date from crt.sh format.
 
-        date_string = str(date_input)
+        Args:
+            date_string: Date string from crt.sh API
+
+        Returns:
+            Parsed datetime object in UTC
+        """
         if not date_string:
             raise ValueError("Empty date string")
 
         try:
-            if 'Z' in date_string:
-                return datetime.fromisoformat(date_string.replace('Z', '+00:00'))
-            # Handle standard ISO format with or without timezone
-            dt = datetime.fromisoformat(date_string)
-            if dt.tzinfo is None:
-                return dt.replace(tzinfo=timezone.utc)
-            return dt
-        except ValueError as e:
+            # Handle various possible formats from crt.sh
+            if date_string.endswith('Z'):
+                return datetime.fromisoformat(date_string[:-1]).replace(tzinfo=timezone.utc)
+            elif '+' in date_string or date_string.endswith('UTC'):
+                # Handle timezone-aware strings
+                date_string = date_string.replace('UTC', '').strip()
+                if '+' in date_string:
+                    date_string = date_string.split('+')[0]
+                return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc)
+            else:
+                # Assume UTC if no timezone specified
+                return datetime.fromisoformat(date_string).replace(tzinfo=timezone.utc)
+        except Exception as e:
+            # Fallback: try parsing without timezone info and assume UTC
             try:
-                # Fallback for other formats
                 return datetime.strptime(date_string[:19], "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc)
             except Exception:
                 raise ValueError(f"Unable to parse date: {date_string}") from e
 
     def _is_cert_valid(self, cert_data: Dict[str, Any]) -> bool:
-        """Check if a certificate is currently valid based on its expiry date."""
+        """
+        Check if a certificate is currently valid based on its expiry date.
+
+        Args:
+            cert_data: Certificate data from crt.sh
+
+        Returns:
+            True if certificate is currently valid (not expired)
+        """
         try:
             not_after_str = cert_data.get('not_after')
-            if not not_after_str: return False
+            if not not_after_str:
+                return False
 
             not_after_date = self._parse_certificate_date(not_after_str)
             not_before_str = cert_data.get('not_before')
+
             now = datetime.now(timezone.utc)
+
+            # Check if certificate is within valid date range
             is_not_expired = not_after_date > now
 
             if not_before_str:
                 not_before_date = self._parse_certificate_date(not_before_str)
                 is_not_before_valid = not_before_date <= now
                 return is_not_expired and is_not_before_valid
+
             return is_not_expired
+
         except Exception as e:
             self.logger.logger.debug(f"Certificate validity check failed: {e}")
             return False
 
     def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]:
-        # This method works as-is.
+        """
+        Extract comprehensive metadata from certificate data.
+        
+        Args:
+            cert_data: Raw certificate data from crt.sh
+            
+        Returns:
+            Comprehensive certificate metadata dictionary
+        """
+        # Parse the issuer name to get just the organization
         raw_issuer_name = cert_data.get('issuer_name', '')
         parsed_issuer_name = self._parse_issuer_organization(raw_issuer_name)
+        
         metadata = {
             'certificate_id': cert_data.get('id'),
             'serial_number': cert_data.get('serial_number'),
-            'issuer_name': parsed_issuer_name,
+            'issuer_name': parsed_issuer_name,  # Use parsed organization name
+            #'issuer_name_full': raw_issuer_name,  # deliberately left out, because its not useful in most cases
             'issuer_ca_id': cert_data.get('issuer_ca_id'),
             'common_name': cert_data.get('common_name'),
             'not_before': cert_data.get('not_before'),
             'not_after': cert_data.get('not_after'),
             'entry_timestamp': cert_data.get('entry_timestamp'),
-            'source': 'crt.sh (DB)'
+            'source': 'crt.sh'
         }
+        
         try:
             if metadata['not_before'] and metadata['not_after']:
                 not_before = self._parse_certificate_date(metadata['not_before'])
                 not_after = self._parse_certificate_date(metadata['not_after'])
+                
                 metadata['validity_period_days'] = (not_after - not_before).days
                 metadata['is_currently_valid'] = self._is_cert_valid(cert_data)
                 metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30
+                
+                # Add human-readable dates
                 metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC')
                 metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC')
+                
         except Exception as e:
             self.logger.logger.debug(f"Error computing certificate metadata: {e}")
             metadata['is_currently_valid'] = False
             metadata['expires_soon'] = False
+        
         return metadata
 
+    def query_domain(self, domain: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query crt.sh for certificates containing the domain with caching support.
+        Properly raises exceptions for network errors to allow core logic retries.
+        """
+        if not _is_valid_domain(domain):
+            return []
+        
+        # Check for cancellation before starting
+        if self._stop_event and self._stop_event.is_set():
+            print(f"CrtSh query cancelled before start for domain: {domain}")
+            return []
+        
+        # === CACHING LOGIC ===
+        cache_file = self._get_cache_file_path(domain)
+        cache_status = self._get_cache_status(cache_file)
+        
+        certificates = []
+        
+        try:
+            if cache_status == "fresh":
+                # Use cached data
+                certificates = self._load_cached_certificates(cache_file)
+                self.logger.logger.info(f"Using cached data for {domain} ({len(certificates)} certificates)")
+                
+            elif cache_status == "not_found":
+                # Fresh query, create new cache
+                certificates = self._query_crtsh_api(domain)
+                if certificates:  # Only cache if we got results
+                    self._create_cache_file(cache_file, domain, certificates)
+                    self.logger.logger.info(f"Cached fresh data for {domain} ({len(certificates)} certificates)")
+                else:
+                    self.logger.logger.info(f"No certificates found for {domain}, not caching")
+                    
+            elif cache_status == "stale":
+                # Append query, update existing cache
+                try:
+                    new_certificates = self._query_crtsh_api(domain)
+                    if new_certificates:
+                        certificates = self._append_to_cache(cache_file, new_certificates)
+                        self.logger.logger.info(f"Refreshed and appended cache for {domain}")
+                    else:
+                        # Use existing cache if API returns no results
+                        certificates = self._load_cached_certificates(cache_file)
+                        self.logger.logger.info(f"API returned no new results, using existing cache for {domain}")
+                except requests.exceptions.RequestException:
+                    # If API call fails for stale cache, use cached data and re-raise for retry logic
+                    certificates = self._load_cached_certificates(cache_file)
+                    if certificates:
+                        self.logger.logger.warning(f"API call failed for {domain}, using stale cache data ({len(certificates)} certificates)")
+                        # Don't re-raise here, just use cached data
+                    else:
+                        # No cached data and API failed - re-raise for retry
+                        raise
+            
+        except requests.exceptions.RequestException as e:
+            # Network/API errors should be re-raised so core logic can retry
+            self.logger.logger.error(f"API query failed for {domain}: {e}")
+            raise e
+        except json.JSONDecodeError as e:
+            # JSON parsing errors should also be raised for retry
+            self.logger.logger.error(f"Failed to parse JSON response from crt.sh for {domain}: {e}")
+            raise e
+        
+        # Check for cancellation after cache operations
+        if self._stop_event and self._stop_event.is_set():
+            print(f"CrtSh query cancelled after cache operations for domain: {domain}")
+            return []
+        
+        if not certificates:
+            return []
+        
+        return self._process_certificates_to_relationships(domain, certificates)
+    
     def _process_certificates_to_relationships(self, domain: str, certificates: List[Dict[str, Any]]) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
-        # This method works as-is.
+        """
+        Process certificates to relationships using existing logic.
+        This method contains the original processing logic from query_domain.
+        """
         relationships = []
-        if self._stop_event and self._stop_event.is_set(): return []
+        
+        # Check for cancellation before processing
+        if self._stop_event and self._stop_event.is_set():
+            print(f"CrtSh processing cancelled before processing for domain: {domain}")
+            return []
+
+        # Aggregate certificate data by domain
         domain_certificates = {}
         all_discovered_domains = set()
+        
+        # Process certificates with cancellation checking
         for i, cert_data in enumerate(certificates):
-            if i % 5 == 0 and self._stop_event and self._stop_event.is_set(): break
+            # Check for cancellation every 5 certificates for faster response
+            if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
+                print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}")
+                break
+                
             cert_metadata = self._extract_certificate_metadata(cert_data)
             cert_domains = self._extract_domains_from_certificate(cert_data)
+            
+            # Add all domains from this certificate to our tracking
             all_discovered_domains.update(cert_domains)
             for cert_domain in cert_domains:
-                if not _is_valid_domain(cert_domain): continue
+                if not _is_valid_domain(cert_domain):
+                    continue
+                
+                # Initialize domain certificate list if needed
                 if cert_domain not in domain_certificates:
                     domain_certificates[cert_domain] = []
+                
+                # Add this certificate to the domain's certificate list
                 domain_certificates[cert_domain].append(cert_metadata)
-        if self._stop_event and self._stop_event.is_set(): return []
+        
+        # Final cancellation check before creating relationships
+        if self._stop_event and self._stop_event.is_set():
+            print(f"CrtSh query cancelled before relationship creation for domain: {domain}")
+            return []
+
+        # Create relationships from query domain to ALL discovered domains with stop checking
         for i, discovered_domain in enumerate(all_discovered_domains):
-            if discovered_domain == domain: continue
-            if i % 10 == 0 and self._stop_event and self._stop_event.is_set(): break
-            if not _is_valid_domain(discovered_domain): continue
+            if discovered_domain == domain:
+                continue  # Skip self-relationships
+            
+            # Check for cancellation every 10 relationships
+            if i % 10 == 0 and self._stop_event and self._stop_event.is_set():
+                print(f"CrtSh relationship creation cancelled for domain: {domain}")
+                break
+
+            if not _is_valid_domain(discovered_domain):
+                continue
+            
+            # Get certificates for both domains
             query_domain_certs = domain_certificates.get(domain, [])
             discovered_domain_certs = domain_certificates.get(discovered_domain, [])
+            
+            # Find shared certificates (for metadata purposes)
             shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs)
+            
+            # Calculate confidence based on relationship type and shared certificates
             confidence = self._calculate_domain_relationship_confidence(
                 domain, discovered_domain, shared_certificates, all_discovered_domains
             )
+            
+            # Create comprehensive raw data for the relationship
             relationship_raw_data = {
                 'relationship_type': 'certificate_discovery',
                 'shared_certificates': shared_certificates,
@@ -432,82 +478,267 @@ class CrtShProvider(BaseProvider):
                     discovered_domain: self._summarize_certificates(discovered_domain_certs)
                 }
             }
+            
+            # Create domain -> domain relationship
             relationships.append((
-                domain, discovered_domain, 'san_certificate', confidence, relationship_raw_data
+                domain,
+                discovered_domain,
+                'san_certificate',
+                confidence,
+                relationship_raw_data
             ))
+            
+            # Log the relationship discovery
             self.log_relationship_discovery(
-                source_node=domain, target_node=discovered_domain, relationship_type='san_certificate',
-                confidence_score=confidence, raw_data=relationship_raw_data,
+                source_node=domain,
+                target_node=discovered_domain,
+                relationship_type='san_certificate',
+                confidence_score=confidence,
+                raw_data=relationship_raw_data,
                 discovery_method="certificate_transparency_analysis"
             )
+
         return relationships
 
-    # --- All remaining helper methods are identical to the original and fully compatible ---
-    # They are included here for completeness.
-
     def _find_shared_certificates(self, certs1: List[Dict[str, Any]], certs2: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """
+        Find certificates that are shared between two domain certificate lists.
+        
+        Args:
+            certs1: First domain's certificates
+            certs2: Second domain's certificates
+            
+        Returns:
+            List of shared certificate metadata
+        """
+        shared = []
+        
+        # Create a set of certificate IDs from the first list for quick lookup
         cert1_ids = {cert.get('certificate_id') for cert in certs1 if cert.get('certificate_id')}
-        return [cert for cert in certs2 if cert.get('certificate_id') in cert1_ids]
+        
+        # Find certificates in the second list that match
+        for cert in certs2:
+            if cert.get('certificate_id') in cert1_ids:
+                shared.append(cert)
+        
+        return shared
 
     def _summarize_certificates(self, certificates: List[Dict[str, Any]]) -> Dict[str, Any]:
-        if not certificates: return {'total_certificates': 0, 'valid_certificates': 0, 'expired_certificates': 0, 'expires_soon_count': 0, 'unique_issuers': [], 'latest_certificate': None, 'has_valid_cert': False}
+        """
+        Create a summary of certificates for a domain.
+        
+        Args:
+            certificates: List of certificate metadata
+            
+        Returns:
+            Summary dictionary with aggregate statistics
+        """
+        if not certificates:
+            return {
+                'total_certificates': 0,
+                'valid_certificates': 0,
+                'expired_certificates': 0,
+                'expires_soon_count': 0,
+                'unique_issuers': [],
+                'latest_certificate': None,
+                'has_valid_cert': False
+            }
+        
         valid_count = sum(1 for cert in certificates if cert.get('is_currently_valid'))
+        expired_count = len(certificates) - valid_count
         expires_soon_count = sum(1 for cert in certificates if cert.get('expires_soon'))
+        
+        # Get unique issuers (using parsed organization names)
         unique_issuers = list(set(cert.get('issuer_name') for cert in certificates if cert.get('issuer_name')))
-        latest_cert, latest_date = None, None
+        
+        # Find the most recent certificate
+        latest_cert = None
+        latest_date = None
+        
         for cert in certificates:
             try:
                 if cert.get('not_before'):
                     cert_date = self._parse_certificate_date(cert['not_before'])
                     if latest_date is None or cert_date > latest_date:
-                        latest_date, latest_cert = cert_date, cert
-            except Exception: continue
-        return {'total_certificates': len(certificates), 'valid_certificates': valid_count, 'expired_certificates': len(certificates) - valid_count, 'expires_soon_count': expires_soon_count, 'unique_issuers': unique_issuers, 'latest_certificate': latest_cert, 'has_valid_cert': valid_count > 0, 'certificate_details': certificates}
+                        latest_date = cert_date
+                        latest_cert = cert
+            except Exception:
+                continue
         
-    def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, shared_certificates: List[Dict[str, Any]], all_discovered_domains: Set[str]) -> float:
-        base_confidence, context_bonus, shared_bonus, validity_bonus, issuer_bonus = 0.9, 0.0, 0.0, 0.0, 0.0
+        return {
+            'total_certificates': len(certificates),
+            'valid_certificates': valid_count,
+            'expired_certificates': expired_count,
+            'expires_soon_count': expires_soon_count,
+            'unique_issuers': unique_issuers,
+            'latest_certificate': latest_cert,
+            'has_valid_cert': valid_count > 0,
+            'certificate_details': certificates  # Full details for forensic analysis
+        }
+
+    def _calculate_domain_relationship_confidence(self, domain1: str, domain2: str, 
+                                                shared_certificates: List[Dict[str, Any]],
+                                                all_discovered_domains: Set[str]) -> float:
+        """
+        Calculate confidence score for domain relationship based on various factors.
+        
+        Args:
+            domain1: Source domain (query domain)
+            domain2: Target domain (discovered domain)
+            shared_certificates: List of shared certificate metadata
+            all_discovered_domains: All domains discovered in this query
+            
+        Returns:
+            Confidence score between 0.0 and 1.0
+        """
+        base_confidence = 0.9
+        
+        # Adjust confidence based on domain relationship context
         relationship_context = self._determine_relationship_context(domain2, domain1)
-        if relationship_context == 'subdomain': context_bonus = 0.1
-        elif relationship_context == 'parent_domain': context_bonus = 0.05
+        
+        if relationship_context == 'exact_match':
+            context_bonus = 0.0  # This shouldn't happen, but just in case
+        elif relationship_context == 'subdomain':
+            context_bonus = 0.1  # High confidence for subdomains
+        elif relationship_context == 'parent_domain':
+            context_bonus = 0.05  # Medium confidence for parent domains
+        else:
+            context_bonus = 0.0  # Related domains get base confidence
+        
+        # Adjust confidence based on shared certificates
+        if shared_certificates:
+            shared_count = len(shared_certificates)
+            if shared_count >= 3:
+                shared_bonus = 0.1
+            elif shared_count >= 2:
+                shared_bonus = 0.05
+            else:
+                shared_bonus = 0.02
+            
+            # Additional bonus for valid shared certificates
+            valid_shared = sum(1 for cert in shared_certificates if cert.get('is_currently_valid'))
+            if valid_shared > 0:
+                validity_bonus = 0.05
+            else:
+                validity_bonus = 0.0
+        else:
+            # Even without shared certificates, domains found in the same query have some relationship
+            shared_bonus = 0.0
+            validity_bonus = 0.0
+        
+        # Adjust confidence based on certificate issuer reputation (if shared certificates exist)
+        issuer_bonus = 0.0
         if shared_certificates:
-            if len(shared_certificates) >= 3: shared_bonus = 0.1
-            elif len(shared_certificates) >= 2: shared_bonus = 0.05
-            else: shared_bonus = 0.02
-            if any(cert.get('is_currently_valid') for cert in shared_certificates): validity_bonus = 0.05
             for cert in shared_certificates:
-                if any(ca in cert.get('issuer_name', '').lower() for ca in ['let\'s encrypt', 'digicert', 'sectigo', 'globalsign']):
+                issuer = cert.get('issuer_name', '').lower()
+                if any(trusted_ca in issuer for trusted_ca in ['let\'s encrypt', 'digicert', 'sectigo', 'globalsign']):
                     issuer_bonus = max(issuer_bonus, 0.03)
                     break
-        return max(0.1, min(1.0, base_confidence + context_bonus + shared_bonus + validity_bonus + issuer_bonus))
+        
+        # Calculate final confidence
+        final_confidence = base_confidence + context_bonus + shared_bonus + validity_bonus + issuer_bonus
+        return max(0.1, min(1.0, final_confidence))  # Clamp between 0.1 and 1.0
 
     def _determine_relationship_context(self, cert_domain: str, query_domain: str) -> str:
-        if cert_domain == query_domain: return 'exact_match'
-        if cert_domain.endswith(f'.{query_domain}'): return 'subdomain'
-        if query_domain.endswith(f'.{cert_domain}'): return 'parent_domain'
-        return 'related_domain'
+        """
+        Determine the context of the relationship between certificate domain and query domain.
+        
+        Args:
+            cert_domain: Domain found in certificate
+            query_domain: Original query domain
+            
+        Returns:
+            String describing the relationship context
+        """
+        if cert_domain == query_domain:
+            return 'exact_match'
+        elif cert_domain.endswith(f'.{query_domain}'):
+            return 'subdomain'
+        elif query_domain.endswith(f'.{cert_domain}'):
+            return 'parent_domain'
+        else:
+            return 'related_domain'
     
     def query_ip(self, ip: str) -> List[Tuple[str, str, str, float, Dict[str, Any]]]:
+        """
+        Query crt.sh for certificates containing the IP address.
+        Note: crt.sh doesn't typically index by IP, so this returns empty results.
+        
+        Args:
+            ip: IP address to investigate
+            
+        Returns:
+            Empty list (crt.sh doesn't support IP-based certificate queries effectively)
+        """
+        # crt.sh doesn't effectively support IP-based certificate queries
         return []
     
     def _extract_domains_from_certificate(self, cert_data: Dict[str, Any]) -> Set[str]:
+        """
+        Extract all domains from certificate data.
+        
+        Args:
+            cert_data: Certificate data from crt.sh API
+            
+        Returns:
+            Set of unique domain names found in the certificate
+        """
         domains = set()
-        if cn := cert_data.get('common_name'):
-            if cleaned := self._clean_domain_name(cn):
-                domains.update(cleaned)
-        if nv := cert_data.get('name_value'):
-            for line in nv.split('\n'):
-                if cleaned := self._clean_domain_name(line.strip()):
-                    domains.update(cleaned)
+        
+        # Extract from common name
+        common_name = cert_data.get('common_name', '')
+        if common_name:
+            cleaned_cn = self._clean_domain_name(common_name)
+            if cleaned_cn:
+                domains.update(cleaned_cn)
+        
+        # Extract from name_value field (contains SANs)
+        name_value = cert_data.get('name_value', '')
+        if name_value:
+            # Split by newlines and clean each domain
+            for line in name_value.split('\n'):
+                cleaned_domains = self._clean_domain_name(line.strip())
+                if cleaned_domains:
+                    domains.update(cleaned_domains)
+        
         return domains
     
     def _clean_domain_name(self, domain_name: str) -> List[str]:
-        if not domain_name: return []
-        domain = domain_name.strip().lower().split('://', 1)[-1].split('/', 1)[0]
-        if ':' in domain and not domain.count(':') > 1: domain = domain.split(':', 1)[0]
-        cleaned_domains = [domain, domain[2:]] if domain.startswith('*.') else [domain]
+        """
+        Clean and normalize domain name from certificate data.
+        Now returns a list to handle wildcards correctly.
+        """
+        if not domain_name:
+            return []
+
+        domain = domain_name.strip().lower()
+
+        # Remove protocol if present
+        if domain.startswith(('http://', 'https://')):
+            domain = domain.split('://', 1)[1]
+
+        # Remove path if present
+        if '/' in domain:
+            domain = domain.split('/', 1)[0]
+
+        # Remove port if present
+        if ':' in domain and not domain.count(':') > 1:  # Avoid breaking IPv6
+            domain = domain.split(':', 1)[0]
+
+        # Handle wildcard domains
+        cleaned_domains = []
+        if domain.startswith('*.'):
+            # Add both the wildcard and the base domain
+            cleaned_domains.append(domain)
+            cleaned_domains.append(domain[2:])
+        else:
+            cleaned_domains.append(domain)
+
+        # Remove any remaining invalid characters and validate
         final_domains = []
         for d in cleaned_domains:
             d = re.sub(r'[^\w\-\.]', '', d)
             if d and not d.startswith(('.', '-')) and not d.endswith(('.', '-')):
                 final_domains.append(d)
+
         return [d for d in final_domains if _is_valid_domain(d)]

requirements.txt

@@ -8,4 +8,3 @@ dnspython>=2.4.2
 gunicorn
 redis
 python-dotenv
-psycopg2-binary