attempt to fix some logic
This commit is contained in:
parent
47ce7ff883
commit
f2db739fa1
@ -41,7 +41,7 @@ class GraphManager:
|
|||||||
self.correlation_index = {}
|
self.correlation_index = {}
|
||||||
# Compile regex for date filtering for efficiency
|
# Compile regex for date filtering for efficiency
|
||||||
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
|
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
|
||||||
self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days']
|
self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days','crtsh_cert_source']
|
||||||
|
|
||||||
def __getstate__(self):
|
def __getstate__(self):
|
||||||
"""Prepare GraphManager for pickling, excluding compiled regex."""
|
"""Prepare GraphManager for pickling, excluding compiled regex."""
|
||||||
@ -112,7 +112,7 @@ class GraphManager:
|
|||||||
|
|
||||||
def _create_enhanced_correlation_node_and_edges(self, value, correlation_data):
|
def _create_enhanced_correlation_node_and_edges(self, value, correlation_data):
|
||||||
"""
|
"""
|
||||||
UPDATED: Create correlation node and edges with detailed provider tracking.
|
UPDATED: Create correlation node and edges with raw provider data (no formatting).
|
||||||
"""
|
"""
|
||||||
correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
|
correlation_node_id = f"corr_{hash(str(value)) & 0x7FFFFFFF}"
|
||||||
nodes = correlation_data['nodes']
|
nodes = correlation_data['nodes']
|
||||||
@ -120,13 +120,14 @@ class GraphManager:
|
|||||||
|
|
||||||
# Create or update correlation node
|
# Create or update correlation node
|
||||||
if not self.graph.has_node(correlation_node_id):
|
if not self.graph.has_node(correlation_node_id):
|
||||||
# Determine the most common provider/attribute combination
|
# Use raw provider/attribute data - no formatting
|
||||||
provider_counts = {}
|
provider_counts = {}
|
||||||
for source in sources:
|
for source in sources:
|
||||||
|
# Keep original provider and attribute names
|
||||||
key = f"{source['provider']}_{source['attribute']}"
|
key = f"{source['provider']}_{source['attribute']}"
|
||||||
provider_counts[key] = provider_counts.get(key, 0) + 1
|
provider_counts[key] = provider_counts.get(key, 0) + 1
|
||||||
|
|
||||||
# Use the most common provider/attribute as the primary label
|
# Use the most common provider/attribute as the primary label (raw)
|
||||||
primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation"
|
primary_source = max(provider_counts.items(), key=lambda x: x[1])[0] if provider_counts else "unknown_correlation"
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
@ -305,16 +306,16 @@ class GraphManager:
|
|||||||
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
|
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
|
||||||
confidence_score: float = 0.5, source_provider: str = "unknown",
|
confidence_score: float = 0.5, source_provider: str = "unknown",
|
||||||
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
||||||
"""Add or update an edge between two nodes, ensuring nodes exist."""
|
"""
|
||||||
|
UPDATED: Add or update an edge between two nodes with raw relationship labels.
|
||||||
|
"""
|
||||||
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
|
if not self.graph.has_node(source_id) or not self.graph.has_node(target_id):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
new_confidence = confidence_score
|
new_confidence = confidence_score
|
||||||
|
|
||||||
if relationship_type.startswith("c_"):
|
# UPDATED: Use raw relationship type - no formatting
|
||||||
edge_label = relationship_type
|
edge_label = relationship_type
|
||||||
else:
|
|
||||||
edge_label = f"{source_provider}_{relationship_type}"
|
|
||||||
|
|
||||||
if self.graph.has_edge(source_id, target_id):
|
if self.graph.has_edge(source_id, target_id):
|
||||||
# If edge exists, update confidence if the new score is higher.
|
# If edge exists, update confidence if the new score is higher.
|
||||||
@ -324,7 +325,7 @@ class GraphManager:
|
|||||||
self.graph.edges[source_id, target_id]['updated_by'] = source_provider
|
self.graph.edges[source_id, target_id]['updated_by'] = source_provider
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Add a new edge with all attributes.
|
# Add a new edge with raw attributes
|
||||||
self.graph.add_edge(source_id, target_id,
|
self.graph.add_edge(source_id, target_id,
|
||||||
relationship_type=edge_label,
|
relationship_type=edge_label,
|
||||||
confidence_score=new_confidence,
|
confidence_score=new_confidence,
|
||||||
@ -417,73 +418,45 @@ class GraphManager:
|
|||||||
def get_graph_data(self) -> Dict[str, Any]:
|
def get_graph_data(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Export graph data formatted for frontend visualization.
|
Export graph data formatted for frontend visualization.
|
||||||
UPDATED: Fixed certificate validity styling logic for unified data model.
|
SIMPLIFIED: No certificate styling - frontend handles all visual styling.
|
||||||
"""
|
"""
|
||||||
nodes = []
|
nodes = []
|
||||||
for node_id, attrs in self.graph.nodes(data=True):
|
for node_id, attrs in self.graph.nodes(data=True):
|
||||||
node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'),
|
node_data = {
|
||||||
'attributes': attrs.get('attributes', []), # Ensure attributes is a list
|
'id': node_id,
|
||||||
|
'label': node_id,
|
||||||
|
'type': attrs.get('type', 'unknown'),
|
||||||
|
'attributes': attrs.get('attributes', []), # Raw attributes list
|
||||||
'description': attrs.get('description', ''),
|
'description': attrs.get('description', ''),
|
||||||
'metadata': attrs.get('metadata', {}),
|
'metadata': attrs.get('metadata', {}),
|
||||||
'added_timestamp': attrs.get('added_timestamp')}
|
'added_timestamp': attrs.get('added_timestamp')
|
||||||
|
}
|
||||||
# UPDATED: Fixed certificate validity styling logic
|
|
||||||
node_type = node_data['type']
|
|
||||||
attributes_list = node_data['attributes']
|
|
||||||
|
|
||||||
if node_type == 'domain' and isinstance(attributes_list, list):
|
|
||||||
# Check for certificate-related attributes
|
|
||||||
has_certificates = False
|
|
||||||
has_valid_certificates = False
|
|
||||||
has_expired_certificates = False
|
|
||||||
|
|
||||||
for attr in attributes_list:
|
|
||||||
attr_name = attr.get('name', '').lower()
|
|
||||||
attr_provider = attr.get('provider', '').lower()
|
|
||||||
attr_value = attr.get('value')
|
|
||||||
|
|
||||||
# Look for certificate attributes from crt.sh provider
|
|
||||||
if attr_provider == 'crtsh' or 'cert' in attr_name:
|
|
||||||
has_certificates = True
|
|
||||||
|
|
||||||
# Check certificate validity
|
|
||||||
if attr_name == 'cert_is_currently_valid':
|
|
||||||
if attr_value is True:
|
|
||||||
has_valid_certificates = True
|
|
||||||
elif attr_value is False:
|
|
||||||
has_expired_certificates = True
|
|
||||||
|
|
||||||
# Also check for certificate expiry indicators
|
|
||||||
elif 'expires_soon' in attr_name and attr_value is True:
|
|
||||||
has_expired_certificates = True
|
|
||||||
elif 'expired' in attr_name and attr_value is True:
|
|
||||||
has_expired_certificates = True
|
|
||||||
|
|
||||||
# Apply styling based on certificate status
|
|
||||||
if has_expired_certificates and not has_valid_certificates:
|
|
||||||
# Red for expired/invalid certificates
|
|
||||||
node_data['color'] = {'background': '#ff6b6b', 'border': '#cc5555'}
|
|
||||||
elif not has_certificates:
|
|
||||||
# Grey for domains with no certificates
|
|
||||||
node_data['color'] = {'background': '#c7c7c7', 'border': '#999999'}
|
|
||||||
# Default green styling is handled by the frontend for domains with valid certificates
|
|
||||||
|
|
||||||
# Add incoming and outgoing edges to node data
|
# Add incoming and outgoing edges to node data
|
||||||
if self.graph.has_node(node_id):
|
if self.graph.has_node(node_id):
|
||||||
node_data['incoming_edges'] = [{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)]
|
node_data['incoming_edges'] = [
|
||||||
node_data['outgoing_edges'] = [{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)]
|
{'from': u, 'data': d} for u, _, d in self.graph.in_edges(node_id, data=True)
|
||||||
|
]
|
||||||
|
node_data['outgoing_edges'] = [
|
||||||
|
{'to': v, 'data': d} for _, v, d in self.graph.out_edges(node_id, data=True)
|
||||||
|
]
|
||||||
|
|
||||||
nodes.append(node_data)
|
nodes.append(node_data)
|
||||||
|
|
||||||
edges = []
|
edges = []
|
||||||
for source, target, attrs in self.graph.edges(data=True):
|
for source, target, attrs in self.graph.edges(data=True):
|
||||||
edges.append({'from': source, 'to': target,
|
edges.append({
|
||||||
|
'from': source,
|
||||||
|
'to': target,
|
||||||
'label': attrs.get('relationship_type', ''),
|
'label': attrs.get('relationship_type', ''),
|
||||||
'confidence_score': attrs.get('confidence_score', 0),
|
'confidence_score': attrs.get('confidence_score', 0),
|
||||||
'source_provider': attrs.get('source_provider', ''),
|
'source_provider': attrs.get('source_provider', ''),
|
||||||
'discovery_timestamp': attrs.get('discovery_timestamp')})
|
'discovery_timestamp': attrs.get('discovery_timestamp')
|
||||||
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'nodes': nodes, 'edges': edges,
|
'nodes': nodes,
|
||||||
|
'edges': edges,
|
||||||
'statistics': self.get_statistics()['basic_metrics']
|
'statistics': self.get_statistics()['basic_metrics']
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -355,7 +355,7 @@ class CrtShProvider(BaseProvider):
|
|||||||
'not_before': cert_data.get('not_before'),
|
'not_before': cert_data.get('not_before'),
|
||||||
'not_after': cert_data.get('not_after'),
|
'not_after': cert_data.get('not_after'),
|
||||||
'entry_timestamp': cert_data.get('entry_timestamp'),
|
'entry_timestamp': cert_data.get('entry_timestamp'),
|
||||||
'source': 'crt.sh'
|
'source': 'crtsh'
|
||||||
}
|
}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@ -367,8 +367,9 @@ class CrtShProvider(BaseProvider):
|
|||||||
metadata['is_currently_valid'] = self._is_cert_valid(cert_data)
|
metadata['is_currently_valid'] = self._is_cert_valid(cert_data)
|
||||||
metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30
|
metadata['expires_soon'] = (not_after - datetime.now(timezone.utc)).days <= 30
|
||||||
|
|
||||||
metadata['not_before'] = not_before.strftime('%Y-%m-%d %H:%M:%S UTC')
|
# UPDATED: Keep raw date format or convert to standard format
|
||||||
metadata['not_after'] = not_after.strftime('%Y-%m-%d %H:%M:%S UTC')
|
metadata['not_before'] = not_before.isoformat()
|
||||||
|
metadata['not_after'] = not_after.isoformat()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.logger.debug(f"Error computing certificate metadata: {e}")
|
self.logger.logger.debug(f"Error computing certificate metadata: {e}")
|
||||||
|
|||||||
@ -155,12 +155,7 @@ class DNSProvider(BaseProvider):
|
|||||||
|
|
||||||
def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None:
|
def _query_record(self, domain: str, record_type: str, result: ProviderResult) -> None:
|
||||||
"""
|
"""
|
||||||
Query a specific type of DNS record for the domain and add results to ProviderResult.
|
UPDATED: Query DNS records with minimal formatting - keep raw values.
|
||||||
|
|
||||||
Args:
|
|
||||||
domain: Domain to query
|
|
||||||
record_type: DNS record type (A, AAAA, CNAME, etc.)
|
|
||||||
result: ProviderResult to populate
|
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self.total_requests += 1
|
self.total_requests += 1
|
||||||
@ -180,13 +175,14 @@ class DNSProvider(BaseProvider):
|
|||||||
elif record_type == 'SOA':
|
elif record_type == 'SOA':
|
||||||
target = str(record.mname).rstrip('.')
|
target = str(record.mname).rstrip('.')
|
||||||
elif record_type in ['TXT']:
|
elif record_type in ['TXT']:
|
||||||
# TXT records are treated as attributes, not relationships
|
# UPDATED: Keep raw TXT record value
|
||||||
txt_value = str(record).strip('"')
|
txt_value = str(record).strip('"')
|
||||||
dns_records.append(f"TXT: {txt_value}")
|
dns_records.append(f"TXT: {txt_value}")
|
||||||
continue
|
continue
|
||||||
elif record_type == 'SRV':
|
elif record_type == 'SRV':
|
||||||
target = str(record.target).rstrip('.')
|
target = str(record.target).rstrip('.')
|
||||||
elif record_type == 'CAA':
|
elif record_type == 'CAA':
|
||||||
|
# UPDATED: Keep raw CAA record format
|
||||||
caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
|
caa_value = f"{record.flags} {record.tag.decode('utf-8')} \"{record.value.decode('utf-8')}\""
|
||||||
dns_records.append(f"CAA: {caa_value}")
|
dns_records.append(f"CAA: {caa_value}")
|
||||||
continue
|
continue
|
||||||
@ -200,8 +196,8 @@ class DNSProvider(BaseProvider):
|
|||||||
'value': target,
|
'value': target,
|
||||||
'ttl': response.ttl
|
'ttl': response.ttl
|
||||||
}
|
}
|
||||||
relationship_type = f"{record_type.lower()}_record"
|
relationship_type = f"{record_type.lower()}_record" # Raw relationship type
|
||||||
confidence = 0.8 # Standard confidence for DNS records
|
confidence = 0.8
|
||||||
|
|
||||||
# Add relationship
|
# Add relationship
|
||||||
result.add_relationship(
|
result.add_relationship(
|
||||||
@ -213,7 +209,7 @@ class DNSProvider(BaseProvider):
|
|||||||
raw_data=raw_data
|
raw_data=raw_data
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add DNS record as attribute to the source domain
|
# UPDATED: Keep raw DNS record format
|
||||||
dns_records.append(f"{record_type}: {target}")
|
dns_records.append(f"{record_type}: {target}")
|
||||||
|
|
||||||
# Log relationship discovery
|
# Log relationship discovery
|
||||||
@ -226,7 +222,7 @@ class DNSProvider(BaseProvider):
|
|||||||
discovery_method=f"dns_{record_type.lower()}_record"
|
discovery_method=f"dns_{record_type.lower()}_record"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add DNS records as a consolidated attribute
|
# Add DNS records as a consolidated attribute (raw format)
|
||||||
if dns_records:
|
if dns_records:
|
||||||
result.add_attribute(
|
result.add_attribute(
|
||||||
target_node=domain,
|
target_node=domain,
|
||||||
@ -241,5 +237,5 @@ class DNSProvider(BaseProvider):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.failed_requests += 1
|
self.failed_requests += 1
|
||||||
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
|
self.logger.logger.debug(f"{record_type} record query failed for {domain}: {e}")
|
||||||
# Re-raise the exception so the scanner can handle it
|
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
|
|||||||
@ -211,14 +211,7 @@ class ShodanProvider(BaseProvider):
|
|||||||
|
|
||||||
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
|
def _process_shodan_data(self, ip: str, data: Dict[str, Any]) -> ProviderResult:
|
||||||
"""
|
"""
|
||||||
Process Shodan data to extract relationships and attributes.
|
UPDATED: Process Shodan data with raw attribute names and values.
|
||||||
|
|
||||||
Args:
|
|
||||||
ip: IP address queried
|
|
||||||
data: Raw Shodan response data
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
ProviderResult with relationships and attributes
|
|
||||||
"""
|
"""
|
||||||
result = ProviderResult()
|
result = ProviderResult()
|
||||||
|
|
||||||
@ -271,9 +264,10 @@ class ShodanProvider(BaseProvider):
|
|||||||
confidence=0.9
|
confidence=0.9
|
||||||
)
|
)
|
||||||
elif isinstance(value, (str, int, float, bool)) and value is not None:
|
elif isinstance(value, (str, int, float, bool)) and value is not None:
|
||||||
|
# UPDATED: Keep raw Shodan field names (no "shodan_" prefix)
|
||||||
result.add_attribute(
|
result.add_attribute(
|
||||||
target_node=ip,
|
target_node=ip,
|
||||||
name=f"shodan_{key}",
|
name=key, # Raw field name from Shodan API
|
||||||
value=value,
|
value=value,
|
||||||
attr_type='shodan_info',
|
attr_type='shodan_info',
|
||||||
provider=self.name,
|
provider=self.name,
|
||||||
|
|||||||
@ -382,7 +382,6 @@ class GraphManager {
|
|||||||
|
|
||||||
graphData.nodes.forEach(node => {
|
graphData.nodes.forEach(node => {
|
||||||
if (node.type === 'large_entity' && node.attributes) {
|
if (node.type === 'large_entity' && node.attributes) {
|
||||||
// UPDATED: Handle unified data model - look for 'nodes' attribute in the attributes list
|
|
||||||
const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes');
|
const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes');
|
||||||
if (nodesAttribute && Array.isArray(nodesAttribute.value)) {
|
if (nodesAttribute && Array.isArray(nodesAttribute.value)) {
|
||||||
nodesAttribute.value.forEach(nodeId => {
|
nodesAttribute.value.forEach(nodeId => {
|
||||||
@ -394,15 +393,30 @@ class GraphManager {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const filteredNodes = graphData.nodes.filter(node => {
|
const filteredNodes = graphData.nodes.filter(node => {
|
||||||
// Only include nodes that are NOT members of large entities, but always include the container itself
|
|
||||||
return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity';
|
return !this.largeEntityMembers.has(node.id) || node.type === 'large_entity';
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`);
|
console.log(`Filtered ${graphData.nodes.length - filteredNodes.length} large entity member nodes from visualization`);
|
||||||
|
|
||||||
// Process only the filtered nodes
|
// FIXED: Process nodes with proper certificate coloring
|
||||||
const processedNodes = filteredNodes.map(node => {
|
const processedNodes = filteredNodes.map(node => {
|
||||||
return this.processNode(node);
|
const processed = this.processNode(node);
|
||||||
|
|
||||||
|
// FIXED: Apply certificate-based coloring here in frontend
|
||||||
|
if (node.type === 'domain' && Array.isArray(node.attributes)) {
|
||||||
|
const certInfo = this.analyzeCertificateInfo(node.attributes);
|
||||||
|
|
||||||
|
if (certInfo.hasExpiredOnly) {
|
||||||
|
// Red for domains with only expired/invalid certificates
|
||||||
|
processed.color = { background: '#ff6b6b', border: '#cc5555' };
|
||||||
|
} else if (!certInfo.hasCertificates) {
|
||||||
|
// Grey for domains with no certificates
|
||||||
|
processed.color = { background: '#c7c7c7', border: '#999999' };
|
||||||
|
}
|
||||||
|
// Valid certificates use default green (handled by processNode)
|
||||||
|
}
|
||||||
|
|
||||||
|
return processed;
|
||||||
});
|
});
|
||||||
|
|
||||||
const mergedEdges = {};
|
const mergedEdges = {};
|
||||||
@ -439,24 +453,19 @@ class GraphManager {
|
|||||||
const existingNodeIds = this.nodes.getIds();
|
const existingNodeIds = this.nodes.getIds();
|
||||||
const existingEdgeIds = this.edges.getIds();
|
const existingEdgeIds = this.edges.getIds();
|
||||||
|
|
||||||
// Add new nodes with fade-in animation
|
|
||||||
const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
|
const newNodes = processedNodes.filter(node => !existingNodeIds.includes(node.id));
|
||||||
const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
|
const newEdges = processedEdges.filter(edge => !existingEdgeIds.includes(edge.id));
|
||||||
|
|
||||||
// Update existing data
|
|
||||||
this.nodes.update(processedNodes);
|
this.nodes.update(processedNodes);
|
||||||
this.edges.update(processedEdges);
|
this.edges.update(processedEdges);
|
||||||
|
|
||||||
// After data is loaded, apply filters
|
|
||||||
this.updateFilterControls();
|
this.updateFilterControls();
|
||||||
this.applyAllFilters();
|
this.applyAllFilters();
|
||||||
|
|
||||||
// Highlight new additions briefly
|
|
||||||
if (newNodes.length > 0 || newEdges.length > 0) {
|
if (newNodes.length > 0 || newEdges.length > 0) {
|
||||||
setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
|
setTimeout(() => this.highlightNewElements(newNodes, newEdges), 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto-fit view for small graphs or first update
|
|
||||||
if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
|
if (processedNodes.length <= 10 || existingNodeIds.length === 0) {
|
||||||
setTimeout(() => this.fitView(), 800);
|
setTimeout(() => this.fitView(), 800);
|
||||||
}
|
}
|
||||||
@ -470,6 +479,46 @@ class GraphManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
analyzeCertificateInfo(attributes) {
|
||||||
|
let hasCertificates = false;
|
||||||
|
let hasValidCertificates = false;
|
||||||
|
let hasExpiredCertificates = false;
|
||||||
|
|
||||||
|
for (const attr of attributes) {
|
||||||
|
const attrName = (attr.name || '').toLowerCase();
|
||||||
|
const attrProvider = (attr.provider || '').toLowerCase();
|
||||||
|
const attrValue = attr.value;
|
||||||
|
|
||||||
|
// Look for certificate attributes from crtsh provider
|
||||||
|
if (attrProvider === 'crtsh' || attrName.startsWith('cert_')) {
|
||||||
|
hasCertificates = true;
|
||||||
|
|
||||||
|
// Check certificate validity using raw attribute names
|
||||||
|
if (attrName === 'cert_is_currently_valid') {
|
||||||
|
if (attrValue === true) {
|
||||||
|
hasValidCertificates = true;
|
||||||
|
} else if (attrValue === false) {
|
||||||
|
hasExpiredCertificates = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Check for expiry indicators
|
||||||
|
else if (attrName === 'cert_expires_soon' && attrValue === true) {
|
||||||
|
hasExpiredCertificates = true;
|
||||||
|
}
|
||||||
|
else if (attrName.includes('expired') && attrValue === true) {
|
||||||
|
hasExpiredCertificates = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
hasCertificates,
|
||||||
|
hasValidCertificates,
|
||||||
|
hasExpiredCertificates,
|
||||||
|
hasExpiredOnly: hasExpiredCertificates && !hasValidCertificates
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* UPDATED: Helper method to find an attribute by name in the standardized attributes list
|
* UPDATED: Helper method to find an attribute by name in the standardized attributes list
|
||||||
* @param {Array} attributes - List of StandardAttribute objects
|
* @param {Array} attributes - List of StandardAttribute objects
|
||||||
@ -496,7 +545,7 @@ class GraphManager {
|
|||||||
size: this.getNodeSize(node.type),
|
size: this.getNodeSize(node.type),
|
||||||
borderColor: this.getNodeBorderColor(node.type),
|
borderColor: this.getNodeBorderColor(node.type),
|
||||||
shape: this.getNodeShape(node.type),
|
shape: this.getNodeShape(node.type),
|
||||||
attributes: node.attributes || [], // Keep as standardized attributes list
|
attributes: node.attributes || [],
|
||||||
description: node.description || '',
|
description: node.description || '',
|
||||||
metadata: node.metadata || {},
|
metadata: node.metadata || {},
|
||||||
type: node.type,
|
type: node.type,
|
||||||
@ -509,19 +558,33 @@ class GraphManager {
|
|||||||
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
|
processedNode.borderWidth = Math.max(2, Math.floor(node.confidence * 5));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle merged correlation objects (similar to large entities)
|
// FIXED: Certificate-based domain coloring
|
||||||
|
if (node.type === 'domain' && Array.isArray(node.attributes)) {
|
||||||
|
const certInfo = this.analyzeCertificateInfo(node.attributes);
|
||||||
|
|
||||||
|
if (certInfo.hasExpiredOnly) {
|
||||||
|
// Red for domains with only expired/invalid certificates
|
||||||
|
processedNode.color = '#ff6b6b';
|
||||||
|
processedNode.borderColor = '#cc5555';
|
||||||
|
} else if (!certInfo.hasCertificates) {
|
||||||
|
// Grey for domains with no certificates
|
||||||
|
processedNode.color = '#c7c7c7';
|
||||||
|
processedNode.borderColor = '#999999';
|
||||||
|
}
|
||||||
|
// Green for valid certificates (default color)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle merged correlation objects
|
||||||
if (node.type === 'correlation_object') {
|
if (node.type === 'correlation_object') {
|
||||||
const metadata = node.metadata || {};
|
const metadata = node.metadata || {};
|
||||||
const values = metadata.values || [];
|
const values = metadata.values || [];
|
||||||
const mergeCount = metadata.merge_count || 1;
|
const mergeCount = metadata.merge_count || 1;
|
||||||
|
|
||||||
if (mergeCount > 1) {
|
if (mergeCount > 1) {
|
||||||
// Display as merged correlation container
|
|
||||||
processedNode.label = `Correlations (${mergeCount})`;
|
processedNode.label = `Correlations (${mergeCount})`;
|
||||||
processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
|
processedNode.title = `Merged correlation container with ${mergeCount} values: ${values.slice(0, 3).join(', ')}${values.length > 3 ? '...' : ''}`;
|
||||||
processedNode.borderWidth = 3; // Thicker border for merged nodes
|
processedNode.borderWidth = 3;
|
||||||
} else {
|
} else {
|
||||||
// Single correlation value
|
|
||||||
const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
|
const value = Array.isArray(values) && values.length > 0 ? values[0] : (metadata.value || 'Unknown');
|
||||||
const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
|
const displayValue = typeof value === 'string' && value.length > 20 ? value.substring(0, 17) + '...' : value;
|
||||||
processedNode.label = `${displayValue}`;
|
processedNode.label = `${displayValue}`;
|
||||||
@ -532,6 +595,7 @@ class GraphManager {
|
|||||||
return processedNode;
|
return processedNode;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process edge data with styling and metadata
|
* Process edge data with styling and metadata
|
||||||
* @param {Object} edge - Raw edge data
|
* @param {Object} edge - Raw edge data
|
||||||
|
|||||||
@ -868,20 +868,14 @@ class DNSReconApp {
|
|||||||
return html;
|
return html;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* NEW: Organized attributes section with provider/semantic grouping (no formatting)
|
|
||||||
*/
|
|
||||||
generateOrganizedAttributesSection(attributes, nodeType) {
|
generateOrganizedAttributesSection(attributes, nodeType) {
|
||||||
if (!Array.isArray(attributes) || attributes.length === 0) {
|
if (!Array.isArray(attributes) || attributes.length === 0) {
|
||||||
return '';
|
return '';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Group attributes intelligently
|
|
||||||
const groups = this.groupAttributesByProviderAndType(attributes, nodeType);
|
const groups = this.groupAttributesByProviderAndType(attributes, nodeType);
|
||||||
|
|
||||||
let html = '';
|
let html = '';
|
||||||
|
|
||||||
// Sort groups by priority
|
|
||||||
const sortedGroups = Object.entries(groups).sort((a, b) => {
|
const sortedGroups = Object.entries(groups).sort((a, b) => {
|
||||||
const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 };
|
const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 };
|
||||||
return priorityOrder[a[1].priority] - priorityOrder[b[1].priority];
|
return priorityOrder[a[1].priority] - priorityOrder[b[1].priority];
|
||||||
@ -904,22 +898,10 @@ class DNSReconApp {
|
|||||||
`;
|
`;
|
||||||
|
|
||||||
groupData.attributes.forEach(attr => {
|
groupData.attributes.forEach(attr => {
|
||||||
// Format the value appropriately
|
|
||||||
let displayValue = '';
|
|
||||||
if (attr.value === null || attr.value === undefined) {
|
|
||||||
displayValue = 'N/A';
|
|
||||||
} else if (Array.isArray(attr.value)) {
|
|
||||||
displayValue = attr.value.length > 0 ? `Array (${attr.value.length} items)` : 'Empty Array';
|
|
||||||
} else if (typeof attr.value === 'object') {
|
|
||||||
displayValue = 'Object';
|
|
||||||
} else {
|
|
||||||
displayValue = String(attr.value);
|
|
||||||
}
|
|
||||||
|
|
||||||
html += `
|
html += `
|
||||||
<div class="attribute-item-compact">
|
<div class="attribute-item-compact">
|
||||||
<span class="attribute-key-compact">${this.escapeHtml(attr.name || 'Unknown')}</span>
|
<span class="attribute-key-compact">${this.escapeHtml(attr.name || 'Unknown')}</span>
|
||||||
<span class="attribute-value-compact">${this.escapeHtml(displayValue)}</span>
|
<span class="attribute-value-compact">${this.formatAttributeValue(attr)}</span>
|
||||||
</div>
|
</div>
|
||||||
`;
|
`;
|
||||||
});
|
});
|
||||||
@ -930,12 +912,49 @@ class DNSReconApp {
|
|||||||
return html;
|
return html;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
formatAttributeValue(attr) {
|
||||||
* NEW: Group attributes by provider and semantic meaning (no formatting)
|
const value = attr.value;
|
||||||
*/
|
const name = attr.name || '';
|
||||||
|
|
||||||
|
if (value === null || value === undefined) {
|
||||||
|
return 'N/A';
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
if (value.length === 0) {
|
||||||
|
return 'Empty Array';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Special handling for DNS records and similar arrays
|
||||||
|
if (name === 'dns_records' || name.includes('record') || name.includes('hostname')) {
|
||||||
|
// Show DNS records as a readable list
|
||||||
|
if (value.length <= 5) {
|
||||||
|
return this.escapeHtml(value.join(', '));
|
||||||
|
} else {
|
||||||
|
const preview = value.slice(0, 3).join(', ');
|
||||||
|
return this.escapeHtml(`${preview} ... (+${value.length - 3} more)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// For other arrays
|
||||||
|
if (value.length <= 3) {
|
||||||
|
return this.escapeHtml(value.join(', '));
|
||||||
|
} else {
|
||||||
|
const preview = value.slice(0, 2).join(', ');
|
||||||
|
return this.escapeHtml(`${preview} ... (${value.length} total)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value === 'object') {
|
||||||
|
return 'Object';
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.escapeHtml(String(value));
|
||||||
|
}
|
||||||
|
|
||||||
groupAttributesByProviderAndType(attributes, nodeType) {
|
groupAttributesByProviderAndType(attributes, nodeType) {
|
||||||
const groups = {
|
const groups = {
|
||||||
'DNS Records': { icon: '🔍', priority: 'high', attributes: [] },
|
'DNS Records': { icon: '📋', priority: 'high', attributes: [] },
|
||||||
'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] },
|
'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] },
|
||||||
'Network Information': { icon: '🌐', priority: 'high', attributes: [] },
|
'Network Information': { icon: '🌐', priority: 'high', attributes: [] },
|
||||||
'Provider Data': { icon: '📊', priority: 'medium', attributes: [] },
|
'Provider Data': { icon: '📊', priority: 'medium', attributes: [] },
|
||||||
@ -943,23 +962,29 @@ class DNSReconApp {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for (const attr of attributes) {
|
for (const attr of attributes) {
|
||||||
const provider = attr.provider?.toLowerCase() || '';
|
const provider = (attr.provider || '').toLowerCase();
|
||||||
const name = attr.name?.toLowerCase() || '';
|
const name = (attr.name || '').toLowerCase();
|
||||||
|
const type = (attr.type || '').toLowerCase();
|
||||||
|
|
||||||
let assigned = false;
|
let assigned = false;
|
||||||
|
|
||||||
// DNS-related attributes
|
// DNS-related attributes (better detection)
|
||||||
if (provider === 'dns' || ['dns', 'record', 'ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) {
|
if (provider === 'dns' ||
|
||||||
|
name === 'dns_records' ||
|
||||||
|
name.includes('record') ||
|
||||||
|
['ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) {
|
||||||
groups['DNS Records'].attributes.push(attr);
|
groups['DNS Records'].attributes.push(attr);
|
||||||
assigned = true;
|
assigned = true;
|
||||||
}
|
}
|
||||||
// Certificate-related attributes
|
// Certificate-related attributes
|
||||||
else if (provider === 'crtsh' || ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) {
|
else if (provider === 'crtsh' || name.startsWith('cert_') ||
|
||||||
|
['certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) {
|
||||||
groups['Certificate Information'].attributes.push(attr);
|
groups['Certificate Information'].attributes.push(attr);
|
||||||
assigned = true;
|
assigned = true;
|
||||||
}
|
}
|
||||||
// Network/Shodan attributes
|
// Network/Shodan attributes
|
||||||
else if (provider === 'shodan' || ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) {
|
else if (provider === 'shodan' ||
|
||||||
|
['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) {
|
||||||
groups['Network Information'].attributes.push(attr);
|
groups['Network Information'].attributes.push(attr);
|
||||||
assigned = true;
|
assigned = true;
|
||||||
}
|
}
|
||||||
@ -985,6 +1010,33 @@ class DNSReconApp {
|
|||||||
return groups;
|
return groups;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
formatEdgeLabel(relationshipType, confidence) {
|
||||||
|
if (!relationshipType) return '';
|
||||||
|
|
||||||
|
// UPDATED: No formatting of relationship type - use raw values
|
||||||
|
const confidenceText = confidence >= 0.8 ? '●' : confidence >= 0.6 ? '◐' : '○';
|
||||||
|
return `${relationshipType} ${confidenceText}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
createEdgeTooltip(edge) {
|
||||||
|
let tooltip = `<div style="font-family: 'Roboto Mono', monospace; font-size: 11px;">`;
|
||||||
|
tooltip += `<div style="color: #00ff41; font-weight: bold; margin-bottom: 4px;">${edge.label || 'Relationship'}</div>`;
|
||||||
|
tooltip += `<div style="color: #999; margin-bottom: 2px;">Confidence: ${(edge.confidence_score * 100).toFixed(1)}%</div>`;
|
||||||
|
|
||||||
|
// UPDATED: Use raw provider name (no formatting)
|
||||||
|
if (edge.source_provider) {
|
||||||
|
tooltip += `<div style="color: #999; margin-bottom: 2px;">Provider: ${edge.source_provider}</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (edge.discovery_timestamp) {
|
||||||
|
const date = new Date(edge.discovery_timestamp);
|
||||||
|
tooltip += `<div style="color: #666; font-size: 10px;">Discovered: ${date.toLocaleString()}</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
tooltip += `</div>`;
|
||||||
|
return tooltip;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* UPDATED: Enhanced correlation details showing the correlated attribute clearly (no formatting)
|
* UPDATED: Enhanced correlation details showing the correlated attribute clearly (no formatting)
|
||||||
*/
|
*/
|
||||||
@ -996,7 +1048,7 @@ class DNSReconApp {
|
|||||||
|
|
||||||
let html = '';
|
let html = '';
|
||||||
|
|
||||||
// Show what attribute is being correlated
|
// Show what attribute is being correlated (raw names)
|
||||||
const primarySource = metadata.primary_source || 'unknown';
|
const primarySource = metadata.primary_source || 'unknown';
|
||||||
|
|
||||||
html += `
|
html += `
|
||||||
@ -1013,7 +1065,7 @@ class DNSReconApp {
|
|||||||
<span class="attribute-value-compact"><code>${this.escapeHtml(String(value))}</code></span>
|
<span class="attribute-value-compact"><code>${this.escapeHtml(String(value))}</code></span>
|
||||||
</div>
|
</div>
|
||||||
<div class="attribute-item-compact">
|
<div class="attribute-item-compact">
|
||||||
<span class="attribute-key-compact">Attribute Type</span>
|
<span class="attribute-key-compact">Attribute Source</span>
|
||||||
<span class="attribute-value-compact">${primarySource}</span>
|
<span class="attribute-value-compact">${primarySource}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="attribute-item-compact">
|
<div class="attribute-item-compact">
|
||||||
@ -1050,10 +1102,6 @@ class DNSReconApp {
|
|||||||
return html;
|
return html;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* UPDATED: Generate large entity details using unified data model
|
* UPDATED: Generate large entity details using unified data model
|
||||||
*/
|
*/
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user