core/graph_manager.py

@@ -41,7 +41,7 @@ class GraphManager:
         self.correlation_index = {}
         # Compile regex for date filtering for efficiency
         self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
-        self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days','crtsh_cert_source']
+        self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','cert_validity_period_days','cert_source']
 
     def __getstate__(self):
         """Prepare GraphManager for pickling, excluding compiled regex."""
@@ -73,7 +73,7 @@ class GraphManager:
             attr_provider = attr.get('provider', 'unknown')
 
             # Skip excluded attributes and invalid values
-            if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
+            if any(excluded_key in attr_name for excluded_key in self.EXCLUDED_KEYS) or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
                 continue
 
             if isinstance(attr_value, bool):

core/scanner.py

@@ -373,6 +373,7 @@ class Scanner:
 
                 task_tuple = (provider_name, target_item)
                 if task_tuple in processed_tasks:
+                    self.indicators_completed += 1
                     continue
 
                 if depth > max_depth:
@@ -661,7 +662,7 @@ class Scanner:
             target_node_type = NodeType.DOMAIN if node_type == 'domain' else NodeType.IP
             self.graph.add_node(target, target_node_type)
 
-        attributes = {
+        attributes_dict = {
             'count': len(targets),
             'nodes': targets,
             'node_type': node_type,
@@ -669,9 +670,21 @@ class Scanner:
             'discovery_depth': current_depth,
             'threshold_exceeded': self.config.large_entity_threshold,
         }
+        
+        attributes_list = []
+        for key, value in attributes_dict.items():
+            attributes_list.append({
+                "name": key,
+                "value": value,
+                "type": "large_entity_info",
+                "provider": provider_name,
+                "confidence": 0.9,
+                "metadata": {}
+            })
+
         description = f'Large entity created due to {len(targets)} relationships from {provider_name}'
         
-        self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes, description=description)
+        self.graph.add_node(entity_id, NodeType.LARGE_ENTITY, attributes=attributes_list, description=description)
         
         # Create edge from source to large entity
         if provider_result.relationships: