new node types

core/graph_manager.py

@@ -43,12 +43,13 @@ class GraphManager:
         # Compile regex for date filtering for efficiency
         self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
         
-        # These are the actual attribute names created in providers, WITHOUT provider prefix
+        # FIXED: Exclude cert_issuer_name since we already create proper CA relationships
         self.EXCLUDED_KEYS = [
-            # Certificate metadata that creates noise
+            # Certificate metadata that creates noise or has dedicated node types
             'cert_source',                    # Always 'crtsh' for crtsh provider
             'cert_common_name',
             'cert_validity_period_days',      # Numerical, not useful for correlation
+            'cert_issuer_name',              # FIXED: Has dedicated CA nodes, don't correlate
             #'cert_certificate_id',            # Unique per certificate
             #'cert_serial_number',            # Unique per certificate
             'cert_entry_timestamp',          # Timestamp, filtered by date regex anyway
@@ -211,7 +212,7 @@ class GraphManager:
     def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
         """
         Check if there's a direct edge between two nodes in either direction.
-        Returns True if node_aâ†'node_b OR node_bâ†'node_a exists.
+        Returns True if node_aâ†'node_b OR node_bâ†'node_a exists.
         """
         return (self.graph.has_edge(node_a, node_b) or 
                 self.graph.has_edge(node_b, node_a))

core/scanner.py

@@ -547,9 +547,10 @@ class Scanner:
             return None
 
     def _process_provider_result_unified(self, target: str, provider: BaseProvider,
-                                       provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
+                                    provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
         """
         Process a unified ProviderResult object to update the graph.
+        VERIFIED: Proper ISP and CA node type assignment.
         """
         provider_name = provider.get_name()
         discovered_targets = set()
@@ -557,6 +558,7 @@ class Scanner:
         if self._is_stop_requested():
             return discovered_targets, False
 
+        # Process all attributes first, grouping by target node
         attributes_by_node = defaultdict(list)
         for attribute in provider_result.attributes:
             attr_dict = {
@@ -569,8 +571,10 @@ class Scanner:
             }
             attributes_by_node[attribute.target_node].append(attr_dict)
 
+        # Add attributes to existing nodes (important for ISP nodes to get ASN attributes)
         for node_id, node_attributes_list in attributes_by_node.items():
             if self.graph.graph.has_node(node_id):
+                # Node already exists, just add attributes
                 if _is_valid_ip(node_id):
                     node_type = NodeType.IP
                 else:
@@ -578,10 +582,12 @@ class Scanner:
 
                 self.graph.add_node(node_id, node_type, attributes=node_attributes_list)
 
+        # Check if this should be a large entity
         if provider_result.get_relationship_count() > self.config.large_entity_threshold:
             members = self._create_large_entity_from_provider_result(target, provider_name, provider_result, current_depth)
             return members, True
 
+        # Process relationships and create nodes with proper types
         for i, relationship in enumerate(provider_result.relationships):
             if i % 5 == 0 and self._is_stop_requested():
                 break
@@ -589,20 +595,24 @@ class Scanner:
             source_node = relationship.source_node
             target_node = relationship.target_node
 
+            # VERIFIED: Determine source node type
             source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN
             
-            if provider_name == 'shodan' and relationship.relationship_type == 'ip_to_isp':
-                target_type = NodeType.ISP
-            elif provider_name == 'crtsh' and relationship.relationship_type == 'issued_by':
-                target_type = NodeType.CA
+            # VERIFIED: Determine target node type based on provider and relationship
+            if provider_name == 'shodan' and relationship.relationship_type == 'shodan_isp':
+                target_type = NodeType.ISP  # ISP node for Shodan organization data
+            elif provider_name == 'crtsh' and relationship.relationship_type == 'crtsh_cert_issuer':
+                target_type = NodeType.CA   # CA node for certificate issuers
             elif _is_valid_ip(target_node):
                 target_type = NodeType.IP
             else:
                 target_type = NodeType.DOMAIN
 
+            # Create or update nodes with proper types
             self.graph.add_node(source_node, source_type)
             self.graph.add_node(target_node, target_type)
 
+            # Add the relationship edge
             if self.graph.add_edge(
                 source_node, target_node,
                 relationship.relationship_type,
@@ -610,8 +620,9 @@ class Scanner:
                 provider_name,
                 relationship.raw_data
             ):
-                pass
+                pass  # Edge was successfully added
 
+            # Add target to discovered nodes for further processing
             if _is_valid_domain(target_node) or _is_valid_ip(target_node):
                 discovered_targets.add(target_node)