misp_analyzer.py aktualisiert

misp_analyzer.py

@@ -31,6 +31,8 @@ class MispAnalyzer(interface.BaseAnalyzer):
         self._attr = kwargs.get("attr")
         self._timesketch_attr = kwargs.get("timesketch_attr")
         self.ip_pattern = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b')
+        # Track marked events to prevent duplicates
+        self.marked_events = set()
 
     @staticmethod
     def get_kwargs():
@@ -58,17 +60,12 @@ class MispAnalyzer(interface.BaseAnalyzer):
             },
             {
                 "query_string": "message:*",
-                "attr": "ip-src",
+                "attr": "ip",  # Generic IP instead of ip-src/ip-dst
-                "timesketch_attr": "message",
-            },
-            {
-                "query_string": "message:*",
-                "attr": "ip-dst",
                 "timesketch_attr": "message",
             },
             {
                 "query_string": "source_ip:*",
-                "attr": "ip-src",
+                "attr": "ip",
                 "timesketch_attr": "source_ip",
             },
         ]
@@ -80,7 +77,6 @@ class MispAnalyzer(interface.BaseAnalyzer):
             import ipaddress
             ip_str = ip_str.strip()
             ipaddress.ip_address(ip_str)
-            # Filter out invalid ranges
             if ip_str.startswith(('0.', '127.', '255.255.255.255')):
                 return False
             return True
@@ -103,41 +99,96 @@ class MispAnalyzer(interface.BaseAnalyzer):
         return False
 
     def query_misp_single(self, value, attr):
-        """Query MISP for a single value."""
+        """Query MISP for a single value - ENHANCED for cross-org visibility."""
+        results = []
+        
+        # Query both ip-src and ip-dst for IPs, include cross-org events
+        if attr == "ip":
+            search_types = ["ip-src", "ip-dst"]
+        else:
+            search_types = [attr]
+        
+        for search_type in search_types:
             try:
+                # Include events from other organizations
+                payload = {
+                    "returnFormat": "json", 
+                    "value": value, 
+                    "type": search_type,
+                    # Include events from all organizations with proper distribution
+                    "enforceWarninglist": False,  # Don't filter known-good IPs
+                    "includeDecayScore": False,   # Skip decay scores for speed
+                    "includeFullModel": False,    # Skip full model for speed
+                    "decayingModel": [],          # No decaying model filters
+                    "excludeDecayed": False,      # Include older indicators
+                    # Distribution levels: 0=Own org, 1=Community, 2=Connected, 3=All, 5=Inherit
+                    "distribution": [0, 1, 2, 3, 5]  # Include all distribution levels
+                }
+                
                 response = requests.post(
                     f"{self.misp_url}/attributes/restSearch/",
-                json={"returnFormat": "json", "value": value, "type": attr},
+                    json=payload,
                     headers={"Authorization": self.misp_api_key},
                     verify=False,
                     timeout=30,
                 )
 
-            if response.status_code != 200:
+                if response.status_code == 200:
-                return []
-            
                     data = response.json()
-            return data.get("response", {}).get("Attribute", [])
+                    attributes = data.get("response", {}).get("Attribute", [])
+                    results.extend(attributes)
                     
             except Exception:
-            return []
+                continue
+                
+        return results
 
     def mark_event(self, event, result, attr):
-        """Add MISP intelligence to event."""
+        """Add MISP intelligence to event - FIXED to prevent duplicates."""
         try:
-            if attr.startswith("ip-"):
+            # Check if event already marked
-                msg = "MISP: Malicious IP - "
+            event_id = event.source.get('_id', '')
+            if event_id in self.marked_events:
+                return
+            
+            self.marked_events.add(event_id)
+            
+            # Show organization info for cross-org awareness
+            if attr == "ip":
+                msg = "MISP: Malicious IP detected - "
             else:
                 msg = "MISP: Known indicator - "
             
-            event_info = result[0].get("Event", {}).get("info", "Unknown")
+            # Collect unique events and organizations
-            msg += event_info
+            events_info = {}
+            orgs_info = set()
             
-            if len(result) > 1:
+            for misp_attr in result:
-                msg += f" (+{len(result)-1} more)"
+                event_info = misp_attr.get("Event", {})
+                event_id = event_info.get("id", "")
+                event_desc = event_info.get("info", "Unknown")
+                org_name = event_info.get("Orgc", {}).get("name", "Unknown Org")
+                
+                events_info[event_id] = f'"{event_desc}"'
+                orgs_info.add(org_name)
+            
+            # Build message with org info
+            event_descriptions = list(events_info.values())[:2]  # First 2 events
+            msg += " | ".join(event_descriptions)
+            
+            if len(result) > 2:
+                msg += f" | +{len(result)-2} more"
+            
+            # Add organization information
+            if len(orgs_info) > 1:
+                msg += f" | Orgs: {', '.join(list(orgs_info)[:3])}"
+            elif orgs_info:
+                org_name = list(orgs_info)[0]
+                if org_name != "Unknown Org":
+                    msg += f" | Org: {org_name}"
 
             event.add_comment(msg)
-            event.add_tags([f"MISP-{attr}", "threat-intel"])
+            event.add_tags([f"MISP-{attr}", "threat-intel", "cross-org-intel"])
             event.commit()
             
         except Exception as e:
@@ -145,7 +196,7 @@ class MispAnalyzer(interface.BaseAnalyzer):
 
     def query_misp(self, query, attr, timesketch_attr):
         """Extract indicators and query MISP."""
-        events = self.event_stream(query_string=query, return_fields=[timesketch_attr])
+        events = self.event_stream(query_string=query, return_fields=[timesketch_attr, '_id'])
         query_list = []
         events_list = []
         processed = 0
@@ -153,7 +204,7 @@ class MispAnalyzer(interface.BaseAnalyzer):
         # Extract indicators from events
         for event in events:
             processed += 1
-            if processed > 5000:  # Reasonable limit
+            if processed > 5000:
                 break
                 
             loc = event.source.get(timesketch_attr)
@@ -163,24 +214,20 @@ class MispAnalyzer(interface.BaseAnalyzer):
             events_list.append(event)
             indicators = []
 
-            # Extract based on attribute type - STRICT VALIDATION
+            # Extract based on attribute type
-            if attr.startswith("ip-") and timesketch_attr == "message":
+            if attr == "ip" and timesketch_attr == "message":
-                # Extract IPs from message field
                 ip_matches = self.ip_pattern.findall(str(loc))
                 indicators = [ip for ip in ip_matches if self._is_valid_ip(ip)]
                 
-            elif attr.startswith("ip-") and timesketch_attr in ["source_ip", "src_ip", "client_ip"]:
+            elif attr == "ip" and timesketch_attr in ["source_ip", "src_ip", "client_ip"]:
-                # Direct IP field
                 if self._is_valid_ip(str(loc)):
                     indicators = [str(loc)]
                     
             elif attr in ["md5", "sha1", "sha256"]:
-                # Hash fields
                 if self._is_valid_hash(str(loc), attr):
                     indicators = [str(loc)]
                     
             elif attr == "filename":
-                # Filename extraction
                 filename = ntpath.basename(str(loc))
                 if filename and len(filename) > 1:
                     indicators = [filename]
@@ -201,7 +248,12 @@ class MispAnalyzer(interface.BaseAnalyzer):
             result = self.query_misp_single(indicator, attr)
             if result:
                 self.result_dict[f"{attr}:{indicator}"] = result
-                logger.info(f"MISP hit: {indicator}")
+                # Log organization diversity
+                orgs = set()
+                for r in result:
+                    org = r.get("Event", {}).get("Orgc", {}).get("name", "Unknown")
+                    orgs.add(org)
+                logger.info(f"MISP hit: {indicator} ({len(result)} indicators from {len(orgs)} orgs)")
 
         # Mark matching events 
         for event in events_list:
@@ -209,13 +261,18 @@ class MispAnalyzer(interface.BaseAnalyzer):
             if not loc:
                 continue
 
+            # Check if event already processed
+            event_id = event.source.get('_id', '')
+            if event_id in self.marked_events:
+                continue
+
             # Re-extract indicators from this event
             event_indicators = []
             
-            if attr.startswith("ip-") and timesketch_attr == "message":
+            if attr == "ip" and timesketch_attr == "message":
                 ip_matches = self.ip_pattern.findall(str(loc))
                 event_indicators = [ip for ip in ip_matches if self._is_valid_ip(ip)]
-            elif attr.startswith("ip-") and timesketch_attr in ["source_ip", "src_ip", "client_ip"]:
+            elif attr == "ip" and timesketch_attr in ["source_ip", "src_ip", "client_ip"]:
                 if self._is_valid_ip(str(loc)):
                     event_indicators = [str(loc)]
             elif attr in ["md5", "sha1", "sha256"]:
@@ -237,9 +294,9 @@ class MispAnalyzer(interface.BaseAnalyzer):
         # Create view if we found matches
         if self.total_event_counter > 0:
             self.sketch.add_view(
-                view_name="MISP Threat Intelligence",
+                view_name="MISP Cross-Org Threat Intel",
                 analyzer_name=self.NAME,
-                query_string='tag:"MISP-*" OR tag:"threat-intel"',
+                query_string='tag:"MISP-*" OR tag:"threat-intel" OR tag:"cross-org-intel"',
             )
 
     def run(self):