new data model refinement
This commit is contained in:
overcuriousity
parent
97aa18f788
commit
733e1da640
@ -40,6 +40,7 @@ class GraphManager:
|
||||
self.correlation_index = {}
|
||||
# Compile regex for date filtering for efficiency
|
||||
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
|
||||
self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type']
|
||||
|
||||
def __getstate__(self):
|
||||
"""Prepare GraphManager for pickling, excluding compiled regex."""
|
||||
@ -54,145 +55,44 @@ class GraphManager:
|
||||
self.__dict__.update(state)
|
||||
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
|
||||
|
||||
def _update_correlation_index(self, node_id: str, data: Any, path: List[str] = [], parent_attr: str = ""):
|
||||
"""Recursively traverse metadata and add hashable values to the index with better path tracking."""
|
||||
if path is None:
|
||||
path = []
|
||||
|
||||
if isinstance(data, dict):
|
||||
for key, value in data.items():
|
||||
self._update_correlation_index(node_id, value, path + [key], key)
|
||||
elif isinstance(data, list):
|
||||
for i, item in enumerate(data):
|
||||
# Instead of just using [i], include the parent attribute context
|
||||
list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
|
||||
self._update_correlation_index(node_id, item, path + [list_path_component], parent_attr)
|
||||
else:
|
||||
self._add_to_correlation_index(node_id, data, ".".join(path), parent_attr)
|
||||
|
||||
def _add_to_correlation_index(self, node_id: str, value: Any, path_str: str, parent_attr: str = ""):
|
||||
"""Add a hashable value to the correlation index, filtering out noise."""
|
||||
if not isinstance(value, (str, int, float, bool)) or value is None:
|
||||
def process_correlations_for_node(self, node_id: str):
|
||||
"""Process correlations for a given node based on its attributes."""
|
||||
if not self.graph.has_node(node_id):
|
||||
return
|
||||
|
||||
# Ignore certain paths that contain noisy, non-unique identifiers
|
||||
if any(keyword in path_str.lower() for keyword in ['count', 'total', 'timestamp', 'date']):
|
||||
return
|
||||
node_attributes = self.graph.nodes[node_id].get('attributes', [])
|
||||
for attr in node_attributes:
|
||||
attr_name = attr.get('name')
|
||||
attr_value = attr.get('value')
|
||||
|
||||
# Filter out common low-entropy values and date-like strings
|
||||
if isinstance(value, str):
|
||||
# FIXED: Prevent correlation on date/time strings.
|
||||
if self.date_pattern.match(value):
|
||||
return
|
||||
if len(value) < 4 or value.lower() in ['true', 'false', 'unknown', 'none', 'crt.sh']:
|
||||
return
|
||||
elif isinstance(value, int) and (abs(value) < 1024 or abs(value) > 65535):
|
||||
return # Ignore small integers and common port numbers
|
||||
elif isinstance(value, bool):
|
||||
return # Ignore boolean values
|
||||
if attr_name in self.EXCLUDED_KEYS or not isinstance(attr_value, (str, int, float, bool)) or attr_value is None:
|
||||
continue
|
||||
|
||||
# Add the valuable correlation data to the index
|
||||
if value not in self.correlation_index:
|
||||
self.correlation_index[value] = {}
|
||||
if node_id not in self.correlation_index[value]:
|
||||
self.correlation_index[value][node_id] = []
|
||||
|
||||
# Store both the full path and the parent attribute for better edge labeling
|
||||
correlation_entry = {
|
||||
'path': path_str,
|
||||
'parent_attr': parent_attr,
|
||||
'meaningful_attr': self._extract_meaningful_attribute(path_str, parent_attr)
|
||||
}
|
||||
|
||||
if correlation_entry not in self.correlation_index[value][node_id]:
|
||||
self.correlation_index[value][node_id].append(correlation_entry)
|
||||
|
||||
def _extract_meaningful_attribute(self, path_str: str, parent_attr: str = "") -> str:
|
||||
"""Extract the most meaningful attribute name from a path string."""
|
||||
if not path_str:
|
||||
return "unknown"
|
||||
|
||||
path_parts = path_str.split('.')
|
||||
|
||||
# Look for the last non-array-index part
|
||||
for part in reversed(path_parts):
|
||||
# Skip array indices like [0], [1], etc.
|
||||
if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
|
||||
# Clean up compound names like "hostnames[0]" to just "hostnames"
|
||||
clean_part = re.sub(r'\[\d+\]$', '', part)
|
||||
if clean_part:
|
||||
return clean_part
|
||||
|
||||
# Fallback to parent attribute if available
|
||||
if parent_attr:
|
||||
return parent_attr
|
||||
|
||||
# Last resort - use the first meaningful part
|
||||
for part in path_parts:
|
||||
if not (part.startswith('[') and part.endswith(']') and part[1:-1].isdigit()):
|
||||
clean_part = re.sub(r'\[\d+\]$', '', part)
|
||||
if clean_part:
|
||||
return clean_part
|
||||
|
||||
return "correlation"
|
||||
|
||||
def _check_for_correlations(self, new_node_id: str, data: Any, path: List[str] = [], parent_attr: str = "") -> List[Dict]:
|
||||
"""Recursively traverse metadata to find correlations with existing data."""
|
||||
if path is None:
|
||||
path = []
|
||||
|
||||
all_correlations = []
|
||||
if isinstance(data, dict):
|
||||
for key, value in data.items():
|
||||
if key == 'source': # Avoid correlating on the provider name
|
||||
continue
|
||||
all_correlations.extend(self._check_for_correlations(new_node_id, value, path + [key], key))
|
||||
elif isinstance(data, list):
|
||||
for i, item in enumerate(data):
|
||||
list_path_component = f"[{i}]" if not parent_attr else f"{parent_attr}[{i}]"
|
||||
all_correlations.extend(self._check_for_correlations(new_node_id, item, path + [list_path_component], parent_attr))
|
||||
else:
|
||||
value = data
|
||||
if value in self.correlation_index:
|
||||
existing_nodes_with_paths = self.correlation_index[value]
|
||||
unique_nodes = set(existing_nodes_with_paths.keys())
|
||||
unique_nodes.add(new_node_id)
|
||||
|
||||
if len(unique_nodes) < 2:
|
||||
return all_correlations # Correlation must involve at least two distinct nodes
|
||||
|
||||
new_source = {
|
||||
'node_id': new_node_id,
|
||||
'path': ".".join(path),
|
||||
'parent_attr': parent_attr,
|
||||
'meaningful_attr': self._extract_meaningful_attribute(".".join(path), parent_attr)
|
||||
}
|
||||
all_sources = [new_source]
|
||||
if isinstance(attr_value, bool):
|
||||
continue
|
||||
|
||||
for node_id, path_entries in existing_nodes_with_paths.items():
|
||||
for entry in path_entries:
|
||||
if isinstance(entry, dict):
|
||||
all_sources.append({
|
||||
'node_id': node_id,
|
||||
'path': entry['path'],
|
||||
'parent_attr': entry.get('parent_attr', ''),
|
||||
'meaningful_attr': entry.get('meaningful_attr', self._extract_meaningful_attribute(entry['path'], entry.get('parent_attr', '')))
|
||||
})
|
||||
else:
|
||||
# Handle legacy string-only entries
|
||||
all_sources.append({
|
||||
'node_id': node_id,
|
||||
'path': str(entry),
|
||||
'parent_attr': '',
|
||||
'meaningful_attr': self._extract_meaningful_attribute(str(entry))
|
||||
})
|
||||
if isinstance(attr_value, str) and (len(attr_value) < 4 or self.date_pattern.match(attr_value)):
|
||||
continue
|
||||
|
||||
if attr_value not in self.correlation_index:
|
||||
self.correlation_index[attr_value] = set()
|
||||
|
||||
self.correlation_index[attr_value].add(node_id)
|
||||
|
||||
if len(self.correlation_index[attr_value]) > 1:
|
||||
self._create_correlation_node_and_edges(attr_value, self.correlation_index[attr_value])
|
||||
|
||||
def _create_correlation_node_and_edges(self, value, nodes):
|
||||
"""Create a correlation node and edges to the correlated nodes."""
|
||||
correlation_node_id = f"corr_{value}"
|
||||
if not self.graph.has_node(correlation_node_id):
|
||||
self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
|
||||
metadata={'value': value, 'correlated_nodes': list(nodes)})
|
||||
|
||||
for node_id in nodes:
|
||||
if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id):
|
||||
self.add_edge(node_id, correlation_node_id, "correlation", confidence_score=0.9)
|
||||
|
||||
all_correlations.append({
|
||||
'value': value,
|
||||
'sources': all_sources,
|
||||
'nodes': list(unique_nodes)
|
||||
})
|
||||
return all_correlations
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
|
||||
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
@ -232,78 +132,9 @@ class GraphManager:
|
||||
existing_metadata.update(metadata)
|
||||
self.graph.nodes[node_id]['metadata'] = existing_metadata
|
||||
|
||||
if attributes and node_type != NodeType.CORRELATION_OBJECT:
|
||||
correlations = self._check_for_correlations(node_id, attributes)
|
||||
for corr in correlations:
|
||||
value = corr['value']
|
||||
|
||||
# STEP 1: Substring check against all existing nodes
|
||||
if self._correlation_value_matches_existing_node(value):
|
||||
# Skip creating correlation node - would be redundant
|
||||
continue
|
||||
|
||||
eligible_nodes = set(corr['nodes'])
|
||||
|
||||
if len(eligible_nodes) < 2:
|
||||
# Need at least 2 nodes to create a correlation
|
||||
continue
|
||||
|
||||
# STEP 3: Check for existing correlation node with same connection pattern
|
||||
correlation_nodes_with_pattern = self._find_correlation_nodes_with_same_pattern(eligible_nodes)
|
||||
|
||||
if correlation_nodes_with_pattern:
|
||||
# STEP 4: Merge with existing correlation node
|
||||
target_correlation_node = correlation_nodes_with_pattern[0]
|
||||
self._merge_correlation_values(target_correlation_node, value, corr)
|
||||
else:
|
||||
# STEP 5: Create new correlation node for eligible nodes only
|
||||
correlation_node_id = f"corr_{abs(hash(str(sorted(eligible_nodes))))}"
|
||||
self.add_node(correlation_node_id, NodeType.CORRELATION_OBJECT,
|
||||
metadata={'values': [value], 'sources': corr['sources'],
|
||||
'correlated_nodes': list(eligible_nodes)})
|
||||
|
||||
# Create edges from eligible nodes to this correlation node with better labeling
|
||||
for c_node_id in eligible_nodes:
|
||||
if self.graph.has_node(c_node_id):
|
||||
# Find the best attribute name for this node
|
||||
meaningful_attr = self._find_best_attribute_name_for_node(c_node_id, corr['sources'])
|
||||
relationship_type = f"c_{meaningful_attr}"
|
||||
self.add_edge(c_node_id, correlation_node_id, relationship_type, confidence_score=0.9)
|
||||
|
||||
self._update_correlation_index(node_id, attributes)
|
||||
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return is_new_node
|
||||
|
||||
def _find_best_attribute_name_for_node(self, node_id: str, sources: List[Dict]) -> str:
|
||||
"""Find the best attribute name for a correlation edge by looking at the sources."""
|
||||
node_sources = [s for s in sources if s['node_id'] == node_id]
|
||||
|
||||
if not node_sources:
|
||||
return "correlation"
|
||||
|
||||
# Use the meaningful_attr if available
|
||||
for source in node_sources:
|
||||
meaningful_attr = source.get('meaningful_attr')
|
||||
if meaningful_attr and meaningful_attr != "unknown":
|
||||
return meaningful_attr
|
||||
|
||||
# Fallback to parent_attr
|
||||
for source in node_sources:
|
||||
parent_attr = source.get('parent_attr')
|
||||
if parent_attr:
|
||||
return parent_attr
|
||||
|
||||
# Last resort - extract from path
|
||||
for source in node_sources:
|
||||
path = source.get('path', '')
|
||||
if path:
|
||||
extracted = self._extract_meaningful_attribute(path)
|
||||
if extracted != "unknown":
|
||||
return extracted
|
||||
|
||||
return "correlation"
|
||||
|
||||
def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
|
||||
"""
|
||||
Check if there's a direct edge between two nodes in either direction.
|
||||
|
||||
@ -506,6 +506,7 @@ class Scanner:
|
||||
large_entity_members.update(discovered)
|
||||
else:
|
||||
new_targets.update(discovered)
|
||||
self.graph.process_correlations_for_node(target)
|
||||
else:
|
||||
print(f"Stop requested after processing results from {provider.get_name()}")
|
||||
except Exception as e:
|
||||
|
||||
@ -134,7 +134,7 @@ class CrtShProvider(BaseProvider):
|
||||
self.logger.logger.info(f"Refreshed and merged cache for {domain}")
|
||||
else: # "not_found"
|
||||
# Create new result from processed certs
|
||||
result = self._process_certificates_to_result(domain, current_processed_certs)
|
||||
result = self._process_certificates_to_result(domain, raw_certificates)
|
||||
self.logger.logger.info(f"Created fresh result for {domain} ({result.get_relationship_count()} relationships)")
|
||||
|
||||
# Save the result to cache
|
||||
@ -272,109 +272,73 @@ class CrtShProvider(BaseProvider):
|
||||
Process certificates to create ProviderResult with relationships and attributes.
|
||||
"""
|
||||
result = ProviderResult()
|
||||
|
||||
|
||||
if self._stop_event and self._stop_event.is_set():
|
||||
print(f"CrtSh processing cancelled before processing for domain: {domain}")
|
||||
return result
|
||||
|
||||
# Aggregate certificate data by domain
|
||||
domain_certificates = {}
|
||||
all_discovered_domains = set()
|
||||
|
||||
# Process certificates with cancellation checking
|
||||
|
||||
for i, cert_data in enumerate(certificates):
|
||||
if i % 5 == 0 and self._stop_event and self._stop_event.is_set():
|
||||
print(f"CrtSh processing cancelled at certificate {i} for domain: {domain}")
|
||||
break
|
||||
|
||||
cert_metadata = self._extract_certificate_metadata(cert_data)
|
||||
|
||||
cert_domains = self._extract_domains_from_certificate(cert_data)
|
||||
|
||||
all_discovered_domains.update(cert_domains)
|
||||
|
||||
for cert_domain in cert_domains:
|
||||
if not _is_valid_domain(cert_domain):
|
||||
continue
|
||||
|
||||
if cert_domain not in domain_certificates:
|
||||
domain_certificates[cert_domain] = []
|
||||
|
||||
domain_certificates[cert_domain].append(cert_metadata)
|
||||
|
||||
|
||||
for key, value in self._extract_certificate_metadata(cert_data).items():
|
||||
if value is not None:
|
||||
result.add_attribute(
|
||||
target_node=cert_domain,
|
||||
name=f"cert_{key}",
|
||||
value=value,
|
||||
attr_type='certificate_data',
|
||||
provider=self.name,
|
||||
confidence=0.9
|
||||
)
|
||||
|
||||
if self._stop_event and self._stop_event.is_set():
|
||||
print(f"CrtSh query cancelled before relationship creation for domain: {domain}")
|
||||
return result
|
||||
|
||||
# Create relationships from query domain to ALL discovered domains
|
||||
for i, discovered_domain in enumerate(all_discovered_domains):
|
||||
if discovered_domain == domain:
|
||||
continue # Skip self-relationships
|
||||
|
||||
continue
|
||||
|
||||
if i % 10 == 0 and self._stop_event and self._stop_event.is_set():
|
||||
print(f"CrtSh relationship creation cancelled for domain: {domain}")
|
||||
break
|
||||
|
||||
if not _is_valid_domain(discovered_domain):
|
||||
continue
|
||||
|
||||
# Get certificates for both domains
|
||||
query_domain_certs = domain_certificates.get(domain, [])
|
||||
discovered_domain_certs = domain_certificates.get(discovered_domain, [])
|
||||
|
||||
# Find shared certificates
|
||||
shared_certificates = self._find_shared_certificates(query_domain_certs, discovered_domain_certs)
|
||||
|
||||
# Calculate confidence
|
||||
|
||||
confidence = self._calculate_domain_relationship_confidence(
|
||||
domain, discovered_domain, shared_certificates, all_discovered_domains
|
||||
domain, discovered_domain, [], all_discovered_domains
|
||||
)
|
||||
|
||||
# Create comprehensive raw data for the relationship
|
||||
relationship_raw_data = {
|
||||
'relationship_type': 'certificate_discovery',
|
||||
'shared_certificates': shared_certificates,
|
||||
'total_shared_certs': len(shared_certificates),
|
||||
'discovery_context': self._determine_relationship_context(discovered_domain, domain),
|
||||
'domain_certificates': {
|
||||
domain: self._summarize_certificates(query_domain_certs),
|
||||
discovered_domain: self._summarize_certificates(discovered_domain_certs)
|
||||
}
|
||||
}
|
||||
|
||||
# Add relationship
|
||||
|
||||
result.add_relationship(
|
||||
source_node=domain,
|
||||
target_node=discovered_domain,
|
||||
relationship_type='san_certificate',
|
||||
provider=self.name,
|
||||
confidence=confidence,
|
||||
raw_data=relationship_raw_data
|
||||
raw_data={'relationship_type': 'certificate_discovery'}
|
||||
)
|
||||
|
||||
# Log the relationship discovery
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=domain,
|
||||
target_node=discovered_domain,
|
||||
relationship_type='san_certificate',
|
||||
confidence_score=confidence,
|
||||
raw_data=relationship_raw_data,
|
||||
raw_data={'relationship_type': 'certificate_discovery'},
|
||||
discovery_method="certificate_transparency_analysis"
|
||||
)
|
||||
|
||||
# Add certificate summary as attributes for all domains that have certificates
|
||||
for cert_domain, cert_list in domain_certificates.items():
|
||||
if cert_list:
|
||||
cert_summary = self._summarize_certificates(cert_list)
|
||||
|
||||
result.add_attribute(
|
||||
target_node=cert_domain,
|
||||
name='certificates',
|
||||
value=cert_summary,
|
||||
attr_type='certificate_data',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'total_certificates': len(cert_list)}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def _extract_certificate_metadata(self, cert_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
||||
@ -222,110 +222,62 @@ class ShodanProvider(BaseProvider):
|
||||
"""
|
||||
result = ProviderResult()
|
||||
|
||||
# Extract hostname relationships
|
||||
hostnames = data.get('hostnames', [])
|
||||
for hostname in hostnames:
|
||||
if _is_valid_domain(hostname):
|
||||
for key, value in data.items():
|
||||
if key == 'hostnames':
|
||||
for hostname in value:
|
||||
if _is_valid_domain(hostname):
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='a_record',
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
raw_data=data
|
||||
)
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='a_record',
|
||||
confidence_score=0.8,
|
||||
raw_data=data,
|
||||
discovery_method="shodan_host_lookup"
|
||||
)
|
||||
elif key == 'asn':
|
||||
asn_name = f"AS{value[2:]}" if isinstance(value, str) and value.startswith('AS') else f"AS{value}"
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='a_record',
|
||||
target_node=asn_name,
|
||||
relationship_type='asn_membership',
|
||||
provider=self.name,
|
||||
confidence=0.8,
|
||||
confidence=0.7,
|
||||
raw_data=data
|
||||
)
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=hostname,
|
||||
relationship_type='a_record',
|
||||
confidence_score=0.8,
|
||||
target_node=asn_name,
|
||||
relationship_type='asn_membership',
|
||||
confidence_score=0.7,
|
||||
raw_data=data,
|
||||
discovery_method="shodan_host_lookup"
|
||||
discovery_method="shodan_asn_lookup"
|
||||
)
|
||||
elif key == 'ports':
|
||||
for port in value:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='open_port',
|
||||
value=port,
|
||||
attr_type='network_info',
|
||||
provider=self.name,
|
||||
confidence=0.9
|
||||
)
|
||||
elif isinstance(value, (str, int, float, bool)) and value is not None:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name=f"shodan_{key}",
|
||||
value=value,
|
||||
attr_type='shodan_info',
|
||||
provider=self.name,
|
||||
confidence=0.9
|
||||
)
|
||||
|
||||
# Extract ASN relationship
|
||||
asn = data.get('asn')
|
||||
if asn:
|
||||
asn_name = f"AS{asn[2:]}" if isinstance(asn, str) and asn.startswith('AS') else f"AS{asn}"
|
||||
result.add_relationship(
|
||||
source_node=ip,
|
||||
target_node=asn_name,
|
||||
relationship_type='asn_membership',
|
||||
provider=self.name,
|
||||
confidence=0.7,
|
||||
raw_data=data
|
||||
)
|
||||
|
||||
self.log_relationship_discovery(
|
||||
source_node=ip,
|
||||
target_node=asn_name,
|
||||
relationship_type='asn_membership',
|
||||
confidence_score=0.7,
|
||||
raw_data=data,
|
||||
discovery_method="shodan_asn_lookup"
|
||||
)
|
||||
|
||||
# Add comprehensive Shodan host information as attributes
|
||||
if 'ports' in data:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='ports',
|
||||
value=data['ports'],
|
||||
attr_type='network_info',
|
||||
provider=self.name,
|
||||
confidence=0.9
|
||||
)
|
||||
|
||||
if 'os' in data and data['os']:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='operating_system',
|
||||
value=data['os'],
|
||||
attr_type='system_info',
|
||||
provider=self.name,
|
||||
confidence=0.8
|
||||
)
|
||||
|
||||
if 'org' in data:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='organization',
|
||||
value=data['org'],
|
||||
attr_type='network_info',
|
||||
provider=self.name,
|
||||
confidence=0.8
|
||||
)
|
||||
|
||||
if 'country_name' in data:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='country',
|
||||
value=data['country_name'],
|
||||
attr_type='location_info',
|
||||
provider=self.name,
|
||||
confidence=0.9
|
||||
)
|
||||
|
||||
if 'city' in data:
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='city',
|
||||
value=data['city'],
|
||||
attr_type='location_info',
|
||||
provider=self.name,
|
||||
confidence=0.8
|
||||
)
|
||||
|
||||
# Store complete Shodan data as a comprehensive attribute
|
||||
result.add_attribute(
|
||||
target_node=ip,
|
||||
name='shodan_host_info',
|
||||
value=data, # Complete Shodan response for full forensic detail
|
||||
attr_type='comprehensive_data',
|
||||
provider=self.name,
|
||||
confidence=0.9,
|
||||
metadata={'data_source': 'shodan_api', 'query_type': 'host_lookup'}
|
||||
)
|
||||
|
||||
return result
|
||||
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Graph visualization module for DNSRecon
|
||||
* Handles network graph rendering using vis.js with proper large entity node hiding
|
||||
* UPDATED: Now compatible with unified data model (StandardAttribute objects)
|
||||
* UPDATED: Now compatible with a strictly flat, unified data model for attributes.
|
||||
*/
|
||||
const contextMenuCSS = `
|
||||
.graph-context-menu {
|
||||
@ -484,7 +484,7 @@ class GraphManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Process node data with styling and metadata for unified data model
|
||||
* UPDATED: Process node data with styling and metadata for the flat data model
|
||||
* @param {Object} node - Raw node data with standardized attributes
|
||||
* @returns {Object} Processed node data
|
||||
*/
|
||||
@ -508,14 +508,6 @@ class GraphManager {
|
||||
if (node.confidence) {
|
||||
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
|
||||
}
|
||||
|
||||
// UPDATED: Style based on certificate validity using unified data model
|
||||
if (node.type === 'domain') {
|
||||
const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates');
|
||||
if (certificatesAttr && certificatesAttr.value && certificatesAttr.value.has_valid_cert === false) {
|
||||
processedNode.color = { background: '#888888', border: '#666666' };
|
||||
}
|
||||
}
|
||||
|
||||
// Handle merged correlation objects (similar to large entities)
|
||||
if (node.type === 'correlation_object') {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Main application logic for DNSRecon web interface
|
||||
* Handles UI interactions, API communication, and data flow
|
||||
* UPDATED: Now compatible with unified data model (StandardAttribute objects)
|
||||
* UPDATED: Now compatible with a strictly flat, unified data model for attributes.
|
||||
*/
|
||||
|
||||
class DNSReconApp {
|
||||
@ -879,21 +879,9 @@ class DNSReconApp {
|
||||
// Relationships sections
|
||||
html += this.generateRelationshipsSection(node);
|
||||
|
||||
// UPDATED: Enhanced attributes section with special certificate handling for unified model
|
||||
// UPDATED: Simplified attributes section for the flat model
|
||||
if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) {
|
||||
// Find certificate attribute separately
|
||||
const certificatesAttr = this.findAttributeByName(node.attributes, 'certificates');
|
||||
|
||||
// Handle certificates separately with enhanced display
|
||||
if (certificatesAttr) {
|
||||
html += this.generateCertificateSection(certificatesAttr);
|
||||
}
|
||||
|
||||
// Handle other attributes normally (excluding certificates to avoid duplication)
|
||||
const otherAttributes = node.attributes.filter(attr => attr.name !== 'certificates');
|
||||
if (otherAttributes.length > 0) {
|
||||
html += this.generateAttributesSection(otherAttributes);
|
||||
}
|
||||
html += this.generateAttributesSection(node.attributes);
|
||||
}
|
||||
|
||||
// Description section
|
||||
@ -905,213 +893,6 @@ class DNSReconApp {
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Enhanced certificate section generation for unified data model
|
||||
*/
|
||||
generateCertificateSection(certificatesAttr) {
|
||||
const certificates = certificatesAttr.value;
|
||||
if (!certificates || typeof certificates !== 'object') {
|
||||
return '';
|
||||
}
|
||||
|
||||
let html = `
|
||||
<div class="modal-section">
|
||||
<details>
|
||||
<summary>🔒 SSL/TLS Certificates</summary>
|
||||
<div class="modal-section-content">
|
||||
`;
|
||||
|
||||
// Certificate summary using existing grid pattern
|
||||
html += this.generateCertificateSummary(certificates);
|
||||
|
||||
// Latest certificate info using existing attribute display
|
||||
if (certificates.latest_certificate) {
|
||||
html += this.generateLatestCertificateInfo(certificates.latest_certificate);
|
||||
}
|
||||
|
||||
// Detailed certificate list if available
|
||||
if (certificates.certificate_details && Array.isArray(certificates.certificate_details)) {
|
||||
html += this.generateCertificateList(certificates.certificate_details);
|
||||
}
|
||||
|
||||
html += '</div></details></div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate latest certificate info using existing attribute list
|
||||
*/
|
||||
generateLatestCertificateInfo(latest) {
|
||||
const isValid = latest.is_currently_valid;
|
||||
const statusText = isValid ? 'Valid' : 'Invalid/Expired';
|
||||
const statusColor = isValid ? '#00ff41' : '#ff6b6b';
|
||||
|
||||
let html = `
|
||||
<div style="margin-bottom: 1rem; padding: 0.75rem; background: rgba(255, 255, 255, 0.02); border-radius: 4px; border: 1px solid #333;">
|
||||
<h5 style="margin: 0 0 0.5rem 0; color: #00ff41; font-size: 0.9rem;">Most Recent Certificate</h5>
|
||||
<div class="attribute-list">
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Status:</span>
|
||||
<span class="attribute-value-compact" style="color: ${statusColor}; font-weight: 600;">${statusText}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Issued:</span>
|
||||
<span class="attribute-value-compact">${latest.not_before || 'Unknown'}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Expires:</span>
|
||||
<span class="attribute-value-compact">${latest.not_after || 'Unknown'}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Issuer:</span>
|
||||
<span class="attribute-value-compact">${this.escapeHtml(latest.issuer_name || 'Unknown')}</span>
|
||||
</div>
|
||||
${latest.certificate_id ? `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Certificate:</span>
|
||||
<span class="attribute-value-compact">
|
||||
<a href="https://crt.sh/?id=${latest.certificate_id}" target="_blank" class="cert-link">
|
||||
View on crt.sh ↗
|
||||
</a>
|
||||
</span>
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate certificate list using existing collapsible structure
|
||||
*/
|
||||
generateCertificateList(certificateDetails) {
|
||||
if (!certificateDetails || certificateDetails.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Limit display to prevent overwhelming the UI
|
||||
const maxDisplay = 8;
|
||||
const certificates = certificateDetails.slice(0, maxDisplay);
|
||||
const remaining = certificateDetails.length - maxDisplay;
|
||||
|
||||
let html = `
|
||||
<details style="margin-top: 1rem;">
|
||||
<summary>📋 Certificate Details (${certificates.length}${remaining > 0 ? ` of ${certificateDetails.length}` : ''})</summary>
|
||||
<div style="margin-top: 0.75rem;">
|
||||
`;
|
||||
|
||||
certificates.forEach((cert, index) => {
|
||||
const isValid = cert.is_currently_valid;
|
||||
let statusText = isValid ? '✅ Valid' : '❌ Invalid/Expired';
|
||||
let statusColor = isValid ? '#00ff41' : '#ff6b6b';
|
||||
|
||||
if (cert.expires_soon && isValid) {
|
||||
statusText = '⚠️ Valid (Expiring Soon)';
|
||||
statusColor = '#ff9900';
|
||||
}
|
||||
|
||||
html += `
|
||||
<div style="margin-bottom: 0.75rem; padding: 0.75rem; background: rgba(255, 255, 255, 0.02); border: 1px solid #333; border-radius: 4px;">
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 0.5rem; border-bottom: 1px solid #333; padding-bottom: 0.5rem;">
|
||||
<span style="font-weight: 600; color: #999;">#${index + 1}</span>
|
||||
<span style="color: ${statusColor}; font-size: 0.85rem; font-weight: 500;">${statusText}</span>
|
||||
${cert.certificate_id ? `
|
||||
<a href="https://crt.sh/?id=${cert.certificate_id}" target="_blank" class="cert-link">crt.sh ↗</a>
|
||||
` : ''}
|
||||
</div>
|
||||
<div class="attribute-list">
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Common Name:</span>
|
||||
<span class="attribute-value-compact">${this.escapeHtml(cert.common_name || 'N/A')}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Issuer:</span>
|
||||
<span class="attribute-value-compact">${this.escapeHtml(cert.issuer_name || 'Unknown')}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Valid From:</span>
|
||||
<span class="attribute-value-compact">${cert.not_before || 'Unknown'}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Valid Until:</span>
|
||||
<span class="attribute-value-compact">${cert.not_after || 'Unknown'}</span>
|
||||
</div>
|
||||
${cert.validity_period_days ? `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Period:</span>
|
||||
<span class="attribute-value-compact">${cert.validity_period_days} days</span>
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
if (remaining > 0) {
|
||||
html += `
|
||||
<div style="text-align: center; padding: 1rem; color: #ff9900; background: rgba(255, 153, 0, 0.1); border: 1px solid #ff9900; border-radius: 4px;">
|
||||
📋 ${remaining} additional certificate${remaining > 1 ? 's' : ''} not shown.<br>
|
||||
<small style="color: #999;">Use the export function to see all certificates.</small>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
html += '</div></details>';
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate certificate summary using minimal new CSS
|
||||
*/
|
||||
generateCertificateSummary(certificates) {
|
||||
const total = certificates.total_certificates || 0;
|
||||
const valid = certificates.valid_certificates || 0;
|
||||
const expired = certificates.expired_certificates || 0;
|
||||
const expiringSoon = certificates.expires_soon_count || 0;
|
||||
const issuers = certificates.unique_issuers || [];
|
||||
|
||||
let html = `
|
||||
<div class="cert-summary-grid">
|
||||
<div class="cert-stat-item">
|
||||
<div class="cert-stat-value">${total}</div>
|
||||
<div class="cert-stat-label">Total</div>
|
||||
</div>
|
||||
<div class="cert-stat-item">
|
||||
<div class="cert-stat-value" style="color: #00ff41">${valid}</div>
|
||||
<div class="cert-stat-label">Valid</div>
|
||||
</div>
|
||||
<div class="cert-stat-item">
|
||||
<div class="cert-stat-value" style="color: #ff6b6b">${expired}</div>
|
||||
<div class="cert-stat-label">Expired</div>
|
||||
</div>
|
||||
<div class="cert-stat-item">
|
||||
<div class="cert-stat-value" style="color: #ff9900">${expiringSoon}</div>
|
||||
<div class="cert-stat-label">Expiring Soon</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Certificate authorities using existing array display
|
||||
if (issuers.length > 0) {
|
||||
html += `
|
||||
<div class="attribute-item-compact" style="margin-bottom: 1rem;">
|
||||
<span class="attribute-key-compact">Certificate Authorities:</span>
|
||||
<span class="attribute-value-compact">
|
||||
<div class="array-display">
|
||||
`;
|
||||
|
||||
issuers.forEach(issuer => {
|
||||
html += `<div class="array-display-item">${this.escapeHtml(issuer)}</div>`;
|
||||
});
|
||||
|
||||
html += '</div></span></div>';
|
||||
}
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Generate large entity details using unified data model
|
||||
*/
|
||||
@ -1356,71 +1137,32 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Generate attributes section for unified data model
|
||||
* Now processes StandardAttribute objects instead of key-value pairs
|
||||
* UPDATED: Generate attributes section for the new flat data model
|
||||
*/
|
||||
generateAttributesSection(attributes) {
|
||||
if (!Array.isArray(attributes) || attributes.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const categorized = this.categorizeStandardAttributes(attributes);
|
||||
let html = '';
|
||||
|
||||
Object.entries(categorized).forEach(([category, attrs]) => {
|
||||
if (attrs.length === 0) return;
|
||||
|
||||
html += `
|
||||
<div class="modal-section">
|
||||
<details>
|
||||
<summary>📊 ${category}</summary>
|
||||
<div class="modal-section-content">
|
||||
`;
|
||||
|
||||
html += '<div class="attribute-list">';
|
||||
attrs.forEach(attr => {
|
||||
html += `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">${this.formatLabel(attr.name)}</span>
|
||||
<span class="attribute-value-compact">${this.formatStandardAttributeValue(attr)}</span>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
html += '</div>';
|
||||
|
||||
html += '</div></details></div>';
|
||||
});
|
||||
|
||||
return html;
|
||||
}
|
||||
let html = `
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📊 Attributes (${attributes.length})</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
`;
|
||||
|
||||
/**
|
||||
* UPDATED: Categorize StandardAttribute objects by type and content
|
||||
*/
|
||||
categorizeStandardAttributes(attributes) {
|
||||
const categories = {
|
||||
'DNS Records': [],
|
||||
'Network Info': [],
|
||||
'Provider Data': [],
|
||||
'Other': []
|
||||
};
|
||||
|
||||
attributes.forEach(attr => {
|
||||
const lowerName = attr.name.toLowerCase();
|
||||
const attrType = attr.type ? attr.type.toLowerCase() : '';
|
||||
|
||||
if (lowerName.includes('dns') || lowerName.includes('record') || attrType.includes('dns')) {
|
||||
categories['DNS Records'].push(attr);
|
||||
} else if (lowerName.includes('ip') || lowerName.includes('asn') || lowerName.includes('network') || attrType.includes('network')) {
|
||||
categories['Network Info'].push(attr);
|
||||
} else if (lowerName.includes('shodan') || lowerName.includes('crtsh') || lowerName.includes('provider') || attrType.includes('provider')) {
|
||||
categories['Provider Data'].push(attr);
|
||||
} else {
|
||||
categories['Other'].push(attr);
|
||||
}
|
||||
html += `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">${this.formatLabel(attr.name)}</span>
|
||||
<span class="attribute-value-compact">${this.formatStandardAttributeValue(attr)}</span>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
return categories;
|
||||
|
||||
html += '</div></div></details></div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user