From bff763e4bb78dc10923b4adc4ca569266e1903ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20St=C3=B6ckl?= Date: Wed, 30 Jul 2025 14:26:28 +0000 Subject: [PATCH] misp_analyzer.py aktualisiert --- misp_analyzer.py | 157 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 24 deletions(-) diff --git a/misp_analyzer.py b/misp_analyzer.py index 3f18218..9252dce 100644 --- a/misp_analyzer.py +++ b/misp_analyzer.py @@ -32,8 +32,7 @@ class MispAnalyzer(interface.BaseAnalyzer): self._attr = kwargs.get("attr") self._timesketch_attr = kwargs.get("timesketch_attr") - # Simple configuration for reliability - self.include_community = kwargs.get("include_community", False) # Default to false for reliability + self.include_community = kwargs.get("include_community", False) self.chunk_size = kwargs.get("chunk_size", 1000) # Process in chunks self.max_retries = kwargs.get("max_retries", 2) # Minimal retries self.request_delay = kwargs.get("request_delay", 0.5) # Small delay between requests @@ -50,18 +49,19 @@ class MispAnalyzer(interface.BaseAnalyzer): 'indicators_found': 0, 'api_calls': 0, 'api_timeouts': 0, - 'events_marked': 0 + 'events_marked': 0, + 'community_hits': 0, + 'own_org_hits': 0 } @staticmethod def get_kwargs(): - """Get kwargs for the analyzer - keeping original working structure.""" to_query = [ { "query_string": "md5_hash:*", "attr": "md5", "timesketch_attr": "md5_hash", - "include_community": True, + "include_community": True, }, { "query_string": "sha1_hash:*", @@ -91,7 +91,6 @@ class MispAnalyzer(interface.BaseAnalyzer): return to_query def _is_valid_ip(self, ip_str): - """Simple IP validation - keeping original working version.""" try: import ipaddress ip_str = ip_str.strip() @@ -119,17 +118,34 @@ class MispAnalyzer(interface.BaseAnalyzer): return False def query_misp_single(self, value, attr, retry_count=0): - """Query MISP for a single value - enhanced with minimal retry logic.""" + """Query MISP for a single value - enhanced with community search.""" if value in self.failed_indicators: return [] try: - # Build basic payload - payload = {"returnFormat": "json", "value": value, "type": attr} + # Build enhanced payload for community search + payload = { + "returnFormat": "json", + "value": value, + "type": attr, + "enforceWarninglist": False, # Don't filter known-good indicators + "includeEventTags": True, # Include event tags for context + "includeContext": True, # Include context information + } - # Add community search if enabled + # community search - include ALL distribution levels if self.include_community: - payload["distribution"] = [0, 1, 2] # Own, community, connected + payload.update({ + "distribution": [0, 1, 2, 3, 5], # Own, community, connected, all, inherit + "includeEventUuid": True, # Include event UUIDs + "includeCorrelations": True, # Include correlations + "includeDecayScore": False, # Skip decay for speed + "includeFullModel": False, # Skip full model for speed + }) + logger.debug(f"Community search enabled for {value} ({attr})") + else: + payload["distribution"] = [0] # Own org only + logger.debug(f"Own org search only for {value} ({attr})") self.stats['api_calls'] += 1 @@ -138,7 +154,7 @@ class MispAnalyzer(interface.BaseAnalyzer): json=payload, headers={"Authorization": self.misp_api_key}, verify=False, - timeout=45, # Slightly increased from original 30s + timeout=45, ) if response.status_code != 200: @@ -146,13 +162,24 @@ class MispAnalyzer(interface.BaseAnalyzer): return [] data = response.json() - return data.get("response", {}).get("Attribute", []) + attributes = data.get("response", {}).get("Attribute", []) + + # Log community sources for debugging + if attributes and self.include_community: + orgs = set() + for attr_data in attributes: + org = attr_data.get("Event", {}).get("Orgc", {}).get("name", "Unknown") + orgs.add(org) + if len(orgs) > 1 or (orgs and list(orgs)[0] not in ["Unknown", "Your Org"]): + logger.info(f"Community hit for {value}: {len(attributes)} matches from {', '.join(list(orgs)[:3])}") + + return attributes except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: self.stats['api_timeouts'] += 1 if retry_count < self.max_retries: - wait_time = (retry_count + 1) * 2 # Simple backoff: 2s, 4s + wait_time = (retry_count + 1) * 2 # backoff: 2s, 4s logger.warning(f"Timeout for {value}, retrying in {wait_time}s (attempt {retry_count + 1})") time.sleep(wait_time) return self.query_misp_single(value, attr, retry_count + 1) @@ -166,21 +193,44 @@ class MispAnalyzer(interface.BaseAnalyzer): return [] def mark_event(self, event, result, attr): - """Mark event with MISP intelligence - keeping original working version.""" + """Mark event with MISP intelligence - enhanced with community info.""" try: if attr.startswith("ip-"): - msg = "MISP: Malicious IP - " + msg = "MISP: Malicious IP" else: - msg = "MISP: Known indicator - " + msg = "MISP: Known indicator" + # Extract event and organization information event_info = result[0].get("Event", {}).get("info", "Unknown") - msg += event_info + org_info = result[0].get("Event", {}).get("Orgc", {}).get("name", "Unknown") + + msg += f" - {event_info}" if len(result) > 1: msg += f" (+{len(result)-1} more)" + + # Add organization information for community awareness + if self.include_community and org_info != "Unknown": + # Collect unique organizations + orgs = set() + for res in result[:3]: + org = res.get("Event", {}).get("Orgc", {}).get("name", "Unknown") + if org != "Unknown": + orgs.add(org) + + if len(orgs) > 1: + msg += f" | Sources: {', '.join(list(orgs)[:2])}" + if len(orgs) > 2: + msg += f" +{len(orgs)-2} more" + elif orgs: + msg += f" | Source: {list(orgs)[0]}" + tags = [f"MISP-{attr}", "threat-intel"] + if self.include_community: + tags.append("community-intel") + event.add_comment(msg) - event.add_tags([f"MISP-{attr}", "threat-intel"]) + event.add_tags(tags) event.commit() self.stats['events_marked'] += 1 @@ -202,7 +252,7 @@ class MispAnalyzer(interface.BaseAnalyzer): indicators = [] - # Extract based on attribute type - SAME AS ORIGINAL + # Extract based on attribute type if attr.startswith("ip-") and timesketch_attr == "message": ip_matches = self.ip_pattern.findall(str(loc)) indicators = [ip for ip in ip_matches if self._is_valid_ip(ip)] @@ -256,16 +306,65 @@ class MispAnalyzer(interface.BaseAnalyzer): result = self.query_misp_single(indicator, attr) if result: self.result_dict[f"{attr}:{indicator}"] = result - logger.info(f"MISP hit: {indicator} ({len(result)} matches)") + + # Track community vs own org hits + orgs = set() + for res in result: + org = res.get("Event", {}).get("Orgc", {}).get("name", "Unknown") + orgs.add(org) + + if len(orgs) > 1 or any(org not in ["Unknown", "Your Organization"] for org in orgs): + self.stats['community_hits'] += 1 + logger.info(f"Community MISP hit: {indicator} ({len(result)} matches from {', '.join(list(orgs)[:3])})") + else: + self.stats['own_org_hits'] += 1 + logger.info(f"Own org MISP hit: {indicator} ({len(result)} matches)") self.processed_indicators.add(indicator) - # Small delay to be nice to MISP server time.sleep(self.request_delay) # Mark events that have matches self.check_existing_matches(events_with_indicators, attr) + def test_community_connectivity(self): + """Test if community feeds are accessible.""" + if not self.include_community: + return "Community search disabled" + + try: + test_payload = { + "returnFormat": "json", + "distribution": [1, 2, 3], + "limit": 1, + "enforceWarninglist": False, + } + + response = requests.post( + f"{self.misp_url}/attributes/restSearch/", + json=test_payload, + headers={"Authorization": self.misp_api_key}, + verify=False, + timeout=30, + ) + + if response.status_code == 200: + data = response.json() + attributes = data.get("response", {}).get("Attribute", []) + if attributes: + orgs = set() + for attr in attributes[:5]: + org = attr.get("Event", {}).get("Orgc", {}).get("name", "Unknown") + orgs.add(org) + return f"Community access OK - {len(attributes)} indicators from {len(orgs)} orgs: {', '.join(list(orgs)[:3])}" + else: + return "Community access OK but no community indicators found" + else: + return f"Community test failed: HTTP {response.status_code}" + + except Exception as e: + return f"Community test error: {e}" + def check_existing_matches(self, events_with_indicators, attr): """Check events against existing MISP results.""" for event, indicators in events_with_indicators: @@ -332,6 +431,11 @@ class MispAnalyzer(interface.BaseAnalyzer): start_time = time.time() + # Test community connectivity if enabled + if self.include_community: + community_status = self.test_community_connectivity() + logger.info(f"Community connectivity test: {community_status}") + try: self.query_misp(self._query_string, self._attr, self._timesketch_attr) @@ -339,10 +443,15 @@ class MispAnalyzer(interface.BaseAnalyzer): success_rate = ((self.stats['api_calls'] - self.stats['api_timeouts']) / max(1, self.stats['api_calls']) * 100) + # Enhanced results with community statistics result = (f"[{self._timesketch_attr}] MISP Analysis Complete: " f"{self.stats['events_marked']}/{self.stats['events_processed']} events marked | " - f"{self.stats['api_calls']} API calls ({success_rate:.1f}% success) | " - f"{elapsed:.0f}s") + f"{self.stats['api_calls']} API calls ({success_rate:.1f}% success) | ") + + if self.include_community: + result += f"Community hits: {self.stats['community_hits']}, Own org: {self.stats['own_org_hits']} | " + + result += f"{elapsed:.0f}s" logger.info(result) return result