format keys reduction
This commit is contained in:
overcuriousity
parent
229746e1ec
commit
47ce7ff883
@ -4,6 +4,7 @@
|
||||
Graph data model for DNSRecon using NetworkX.
|
||||
Manages in-memory graph storage with confidence scoring and forensic metadata.
|
||||
Now fully compatible with the unified ProviderResult data model.
|
||||
UPDATED: Fixed certificate styling and correlation edge labeling.
|
||||
"""
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
@ -40,7 +41,7 @@ class GraphManager:
|
||||
self.correlation_index = {}
|
||||
# Compile regex for date filtering for efficiency
|
||||
self.date_pattern = re.compile(r'^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}')
|
||||
self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type']
|
||||
self.EXCLUDED_KEYS = ['confidence', 'provider', 'timestamp', 'type','crtsh_cert_validity_period_days']
|
||||
|
||||
def __getstate__(self):
|
||||
"""Prepare GraphManager for pickling, excluding compiled regex."""
|
||||
@ -101,7 +102,7 @@ class GraphManager:
|
||||
|
||||
# Add source if not already present (avoid duplicates)
|
||||
existing_sources = [s for s in self.correlation_index[attr_value]['sources']
|
||||
if s['node_id'] == node_id and s['path'] == source_info['path']]
|
||||
if s['node_id'] == node_id and s['path'] == source_info['path']]
|
||||
if not existing_sources:
|
||||
self.correlation_index[attr_value]['sources'].append(source_info)
|
||||
|
||||
@ -146,11 +147,8 @@ class GraphManager:
|
||||
attribute = source['attribute']
|
||||
|
||||
if self.graph.has_node(node_id) and not self.graph.has_edge(node_id, correlation_node_id):
|
||||
# Format relationship label as "provider: attribute"
|
||||
display_provider = provider
|
||||
display_attribute = attribute.replace('_', ' ').replace('cert ', '').strip()
|
||||
|
||||
relationship_label = f"{display_provider}: {display_attribute}"
|
||||
# Format relationship label as "corr_provider_attribute"
|
||||
relationship_label = f"corr_{provider}_{attribute}"
|
||||
|
||||
self.add_edge(
|
||||
source_id=node_id,
|
||||
@ -167,46 +165,6 @@ class GraphManager:
|
||||
|
||||
print(f"Added correlation edge: {node_id} -> {correlation_node_id} ({relationship_label})")
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
|
||||
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Add a node to the graph, update attributes, and process correlations.
|
||||
Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects.
|
||||
"""
|
||||
is_new_node = not self.graph.has_node(node_id)
|
||||
if is_new_node:
|
||||
self.graph.add_node(node_id, type=node_type.value,
|
||||
added_timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
attributes=attributes or [], # Store as a list from the start
|
||||
description=description,
|
||||
metadata=metadata or {})
|
||||
else:
|
||||
# Safely merge new attributes into the existing list of attributes
|
||||
if attributes:
|
||||
existing_attributes = self.graph.nodes[node_id].get('attributes', [])
|
||||
|
||||
# Handle cases where old data might still be in dictionary format
|
||||
if not isinstance(existing_attributes, list):
|
||||
existing_attributes = []
|
||||
|
||||
# Create a set of existing attribute names for efficient duplicate checking
|
||||
existing_attr_names = {attr['name'] for attr in existing_attributes}
|
||||
|
||||
for new_attr in attributes:
|
||||
if new_attr['name'] not in existing_attr_names:
|
||||
existing_attributes.append(new_attr)
|
||||
existing_attr_names.add(new_attr['name'])
|
||||
|
||||
self.graph.nodes[node_id]['attributes'] = existing_attributes
|
||||
if description:
|
||||
self.graph.nodes[node_id]['description'] = description
|
||||
if metadata:
|
||||
existing_metadata = self.graph.nodes[node_id].get('metadata', {})
|
||||
existing_metadata.update(metadata)
|
||||
self.graph.nodes[node_id]['metadata'] = existing_metadata
|
||||
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return is_new_node
|
||||
|
||||
def _has_direct_edge_bidirectional(self, node_a: str, node_b: str) -> bool:
|
||||
"""
|
||||
@ -303,6 +261,47 @@ class GraphManager:
|
||||
f"across {node_count} nodes"
|
||||
)
|
||||
|
||||
def add_node(self, node_id: str, node_type: NodeType, attributes: Optional[List[Dict[str, Any]]] = None,
|
||||
description: str = "", metadata: Optional[Dict[str, Any]] = None) -> bool:
|
||||
"""
|
||||
Add a node to the graph, update attributes, and process correlations.
|
||||
Now compatible with unified data model - attributes are dictionaries from converted StandardAttribute objects.
|
||||
"""
|
||||
is_new_node = not self.graph.has_node(node_id)
|
||||
if is_new_node:
|
||||
self.graph.add_node(node_id, type=node_type.value,
|
||||
added_timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
attributes=attributes or [], # Store as a list from the start
|
||||
description=description,
|
||||
metadata=metadata or {})
|
||||
else:
|
||||
# Safely merge new attributes into the existing list of attributes
|
||||
if attributes:
|
||||
existing_attributes = self.graph.nodes[node_id].get('attributes', [])
|
||||
|
||||
# Handle cases where old data might still be in dictionary format
|
||||
if not isinstance(existing_attributes, list):
|
||||
existing_attributes = []
|
||||
|
||||
# Create a set of existing attribute names for efficient duplicate checking
|
||||
existing_attr_names = {attr['name'] for attr in existing_attributes}
|
||||
|
||||
for new_attr in attributes:
|
||||
if new_attr['name'] not in existing_attr_names:
|
||||
existing_attributes.append(new_attr)
|
||||
existing_attr_names.add(new_attr['name'])
|
||||
|
||||
self.graph.nodes[node_id]['attributes'] = existing_attributes
|
||||
if description:
|
||||
self.graph.nodes[node_id]['description'] = description
|
||||
if metadata:
|
||||
existing_metadata = self.graph.nodes[node_id].get('metadata', {})
|
||||
existing_metadata.update(metadata)
|
||||
self.graph.nodes[node_id]['metadata'] = existing_metadata
|
||||
|
||||
self.last_modified = datetime.now(timezone.utc).isoformat()
|
||||
return is_new_node
|
||||
|
||||
def add_edge(self, source_id: str, target_id: str, relationship_type: str,
|
||||
confidence_score: float = 0.5, source_provider: str = "unknown",
|
||||
raw_data: Optional[Dict[str, Any]] = None) -> bool:
|
||||
@ -369,11 +368,21 @@ class GraphManager:
|
||||
|
||||
# Clean up the correlation index
|
||||
keys_to_delete = []
|
||||
for value, nodes in self.correlation_index.items():
|
||||
if node_id in nodes:
|
||||
del nodes[node_id]
|
||||
if not nodes: # If no other nodes are associated with this value, remove it
|
||||
keys_to_delete.append(value)
|
||||
for value, data in self.correlation_index.items():
|
||||
if isinstance(data, dict) and 'nodes' in data:
|
||||
# Updated correlation structure
|
||||
if node_id in data['nodes']:
|
||||
data['nodes'].discard(node_id)
|
||||
# Remove sources for this node
|
||||
data['sources'] = [s for s in data['sources'] if s['node_id'] != node_id]
|
||||
if not data['nodes']: # If no other nodes are associated, remove it
|
||||
keys_to_delete.append(value)
|
||||
else:
|
||||
# Legacy correlation structure (fallback)
|
||||
if isinstance(data, set) and node_id in data:
|
||||
data.discard(node_id)
|
||||
if not data:
|
||||
keys_to_delete.append(value)
|
||||
|
||||
for key in keys_to_delete:
|
||||
if key in self.correlation_index:
|
||||
@ -413,10 +422,10 @@ class GraphManager:
|
||||
nodes = []
|
||||
for node_id, attrs in self.graph.nodes(data=True):
|
||||
node_data = {'id': node_id, 'label': node_id, 'type': attrs.get('type', 'unknown'),
|
||||
'attributes': attrs.get('attributes', []), # Ensure attributes is a list
|
||||
'description': attrs.get('description', ''),
|
||||
'metadata': attrs.get('metadata', {}),
|
||||
'added_timestamp': attrs.get('added_timestamp')}
|
||||
'attributes': attrs.get('attributes', []), # Ensure attributes is a list
|
||||
'description': attrs.get('description', ''),
|
||||
'metadata': attrs.get('metadata', {}),
|
||||
'added_timestamp': attrs.get('added_timestamp')}
|
||||
|
||||
# UPDATED: Fixed certificate validity styling logic
|
||||
node_type = node_data['type']
|
||||
@ -469,10 +478,10 @@ class GraphManager:
|
||||
edges = []
|
||||
for source, target, attrs in self.graph.edges(data=True):
|
||||
edges.append({'from': source, 'to': target,
|
||||
'label': attrs.get('relationship_type', ''),
|
||||
'confidence_score': attrs.get('confidence_score', 0),
|
||||
'source_provider': attrs.get('source_provider', ''),
|
||||
'discovery_timestamp': attrs.get('discovery_timestamp')})
|
||||
'label': attrs.get('relationship_type', ''),
|
||||
'confidence_score': attrs.get('confidence_score', 0),
|
||||
'source_provider': attrs.get('source_provider', ''),
|
||||
'discovery_timestamp': attrs.get('discovery_timestamp')})
|
||||
return {
|
||||
'nodes': nodes, 'edges': edges,
|
||||
'statistics': self.get_statistics()['basic_metrics']
|
||||
|
||||
@ -484,18 +484,6 @@ class DNSReconApp {
|
||||
console.log('- Nodes:', graphData.nodes ? graphData.nodes.length : 0);
|
||||
console.log('- Edges:', graphData.edges ? graphData.edges.length : 0);
|
||||
|
||||
/*if (graphData.nodes) {
|
||||
graphData.nodes.forEach(node => {
|
||||
console.log(` Node: ${node.id} (${node.type})`);
|
||||
});
|
||||
}
|
||||
|
||||
if (graphData.edges) {
|
||||
graphData.edges.forEach(edge => {
|
||||
console.log(` Edge: ${edge.from} -> ${edge.to} (${edge.label})`);
|
||||
});
|
||||
}*/
|
||||
|
||||
// Only update if data has changed
|
||||
if (this.hasGraphChanged(graphData)) {
|
||||
console.log('*** GRAPH DATA CHANGED - UPDATING VISUALIZATION ***');
|
||||
@ -809,21 +797,8 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Helper method to find an attribute by name in the standardized attributes list
|
||||
* @param {Array} attributes - List of StandardAttribute objects
|
||||
* @param {string} name - Attribute name to find
|
||||
* @returns {Object|null} The attribute object if found, null otherwise
|
||||
*/
|
||||
findAttributeByName(attributes, name) {
|
||||
if (!Array.isArray(attributes)) {
|
||||
return null;
|
||||
}
|
||||
return attributes.find(attr => attr.name === name) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Node details HTML generation for unified data model
|
||||
* processes StandardAttribute objects
|
||||
* UPDATED: Enhanced node details HTML generation for unified data model
|
||||
* Now properly groups attributes by provider/type with organized sections
|
||||
*/
|
||||
generateNodeDetailsHtml(node) {
|
||||
if (!node) return '<div class="detail-row"><span class="detail-value">Details not available.</span></div>';
|
||||
@ -857,7 +832,7 @@ class DNSReconApp {
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Handle different node types with collapsible sections
|
||||
// Handle different node types
|
||||
if (node.type === 'correlation_object') {
|
||||
detailsHtml += this.generateCorrelationDetails(node);
|
||||
} else if (node.type === 'large_entity') {
|
||||
@ -871,7 +846,7 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate details for standard nodes using unified data model
|
||||
* UPDATED: Generate details for standard nodes with organized attribute grouping
|
||||
*/
|
||||
generateStandardNodeDetails(node) {
|
||||
let html = '';
|
||||
@ -879,9 +854,9 @@ class DNSReconApp {
|
||||
// Relationships sections
|
||||
html += this.generateRelationshipsSection(node);
|
||||
|
||||
// Attributes section with grouping
|
||||
// UPDATED: Enhanced attributes section with intelligent grouping (no formatting)
|
||||
if (node.attributes && Array.isArray(node.attributes) && node.attributes.length > 0) {
|
||||
html += this.generateEnhancedAttributesSection(node.attributes, node.type);
|
||||
html += this.generateOrganizedAttributesSection(node.attributes, node.type);
|
||||
}
|
||||
|
||||
// Description section
|
||||
@ -893,35 +868,58 @@ class DNSReconApp {
|
||||
return html;
|
||||
}
|
||||
|
||||
generateEnhancedAttributesSection(attributes, nodeType) {
|
||||
/**
|
||||
* NEW: Organized attributes section with provider/semantic grouping (no formatting)
|
||||
*/
|
||||
generateOrganizedAttributesSection(attributes, nodeType) {
|
||||
if (!Array.isArray(attributes) || attributes.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Group attributes by provider and type for better organization
|
||||
const groupedAttributes = this.groupAttributesIntelligently(attributes, nodeType);
|
||||
// Group attributes intelligently
|
||||
const groups = this.groupAttributesByProviderAndType(attributes, nodeType);
|
||||
|
||||
let html = '';
|
||||
|
||||
for (const [groupName, groupData] of Object.entries(groupedAttributes)) {
|
||||
const isDefaultOpen = groupData.priority === 'high';
|
||||
// Sort groups by priority
|
||||
const sortedGroups = Object.entries(groups).sort((a, b) => {
|
||||
const priorityOrder = { 'high': 0, 'medium': 1, 'low': 2 };
|
||||
return priorityOrder[a[1].priority] - priorityOrder[b[1].priority];
|
||||
});
|
||||
|
||||
for (const [groupName, groupData] of sortedGroups) {
|
||||
if (groupData.attributes.length === 0) continue;
|
||||
|
||||
const isOpen = groupData.priority === 'high';
|
||||
|
||||
html += `
|
||||
<div class="modal-section">
|
||||
<details ${isDefaultOpen ? 'open' : ''}>
|
||||
<details ${isOpen ? 'open' : ''}>
|
||||
<summary>
|
||||
<span>${groupData.icon} ${groupName}</span>
|
||||
<span class="count-badge">${groupData.attributes.length}</span>
|
||||
<span class="merge-badge">${groupData.attributes.length}</span>
|
||||
</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
`;
|
||||
|
||||
groupData.attributes.forEach(attr => {
|
||||
// Format the value appropriately
|
||||
let displayValue = '';
|
||||
if (attr.value === null || attr.value === undefined) {
|
||||
displayValue = 'N/A';
|
||||
} else if (Array.isArray(attr.value)) {
|
||||
displayValue = attr.value.length > 0 ? `Array (${attr.value.length} items)` : 'Empty Array';
|
||||
} else if (typeof attr.value === 'object') {
|
||||
displayValue = 'Object';
|
||||
} else {
|
||||
displayValue = String(attr.value);
|
||||
}
|
||||
|
||||
html += `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">${this.formatAttributeLabel(attr.name)}</span>
|
||||
<span class="attribute-value-compact">${this.formatAttributeValueEnhanced(attr)}</span>
|
||||
<span class="attribute-key-compact">${this.escapeHtml(attr.name || 'Unknown')}</span>
|
||||
<span class="attribute-value-compact">${this.escapeHtml(displayValue)}</span>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
@ -932,192 +930,50 @@ class DNSReconApp {
|
||||
return html;
|
||||
}
|
||||
|
||||
formatAttributeValueEnhanced(attr) {
|
||||
const value = attr.value;
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
return '<em>None</em>';
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) return '<em>None</em>';
|
||||
if (value.length === 1) return this.escapeHtml(String(value[0]));
|
||||
|
||||
// Complex array - make it collapsible
|
||||
const previewItems = value.slice(0, 2);
|
||||
const hasMore = value.length > 2;
|
||||
|
||||
let html = '<div class="expandable-array">';
|
||||
html += `<div class="array-preview">`;
|
||||
|
||||
previewItems.forEach(item => {
|
||||
html += `<div class="array-item-preview">${this.escapeHtml(String(item))}</div>`;
|
||||
});
|
||||
|
||||
if (hasMore) {
|
||||
html += `
|
||||
<button class="expand-array-btn" onclick="this.parentElement.style.display='none'; this.parentElement.nextElementSibling.style.display='block';">
|
||||
+${value.length - 2} more...
|
||||
</button>
|
||||
`;
|
||||
}
|
||||
|
||||
html += '</div>';
|
||||
|
||||
if (hasMore) {
|
||||
html += `<div class="array-full" style="display: none;">`;
|
||||
value.forEach(item => {
|
||||
html += `<div class="array-item-full">${this.escapeHtml(String(item))}</div>`;
|
||||
});
|
||||
html += `
|
||||
<button class="collapse-array-btn" onclick="this.parentElement.style.display='none'; this.parentElement.previousElementSibling.style.display='block';">
|
||||
Show less
|
||||
</button>
|
||||
`;
|
||||
html += '</div>';
|
||||
}
|
||||
|
||||
html += '</div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return this.formatObjectExpandable(value);
|
||||
}
|
||||
|
||||
return this.escapeHtml(String(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* NEW: Format objects as expandable content
|
||||
* NEW: Group attributes by provider and semantic meaning (no formatting)
|
||||
*/
|
||||
formatObjectExpandable(obj) {
|
||||
if (!obj || typeof obj !== 'object') return '';
|
||||
|
||||
const entries = Object.entries(obj);
|
||||
if (entries.length === 0) return '<em>Empty</em>';
|
||||
|
||||
if (entries.length <= 3) {
|
||||
// Simple inline display for small objects
|
||||
let html = '<div class="simple-object">';
|
||||
entries.forEach(([key, value]) => {
|
||||
html += `<div><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`;
|
||||
});
|
||||
html += '</div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
// Expandable display for complex objects
|
||||
let html = '<div class="expandable-object">';
|
||||
html += `
|
||||
<div class="object-preview">
|
||||
<strong>${entries[0][0]}:</strong> ${this.escapeHtml(String(entries[0][1]))}
|
||||
<button class="expand-object-btn" onclick="this.parentElement.style.display='none'; this.parentElement.nextElementSibling.style.display='block';">
|
||||
+${entries.length - 1} more properties...
|
||||
</button>
|
||||
</div>
|
||||
<div class="object-full" style="display: none;">
|
||||
`;
|
||||
|
||||
entries.forEach(([key, value]) => {
|
||||
html += `<div class="object-property"><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`;
|
||||
});
|
||||
|
||||
html += `
|
||||
<button class="collapse-object-btn" onclick="this.parentElement.style.display='none'; this.parentElement.previousElementSibling.style.display='block';">
|
||||
Show less
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
formatAttributeLabel(name) {
|
||||
// Handle provider prefixed attributes
|
||||
if (name.includes('_')) {
|
||||
const parts = name.split('_');
|
||||
if (parts.length >= 2) {
|
||||
const provider = parts[0];
|
||||
const attribute = parts.slice(1).join('_');
|
||||
return `${this.provider}: ${this.formatLabel(attribute)}`;
|
||||
}
|
||||
}
|
||||
|
||||
return this.formatLabel(name);
|
||||
}
|
||||
|
||||
groupAttributesIntelligently(attributes, nodeType) {
|
||||
const groups = {};
|
||||
|
||||
// Define group configurations
|
||||
const groupConfigs = {
|
||||
'DNS Records': {
|
||||
icon: '🔍',
|
||||
priority: 'high',
|
||||
keywords: ['dns', 'record', 'a_record', 'cname', 'mx', 'ns', 'txt', 'ptr'],
|
||||
providers: ['dns']
|
||||
},
|
||||
'Certificate Information': {
|
||||
icon: '🔒',
|
||||
priority: 'high',
|
||||
keywords: ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'],
|
||||
providers: ['crtsh']
|
||||
},
|
||||
'Network Information': {
|
||||
icon: '🌐',
|
||||
priority: 'high',
|
||||
keywords: ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city'],
|
||||
providers: ['shodan']
|
||||
},
|
||||
'Correlation Data': {
|
||||
icon: '🔗',
|
||||
priority: 'medium',
|
||||
keywords: ['correlation', 'shared', 'common'],
|
||||
providers: []
|
||||
}
|
||||
groupAttributesByProviderAndType(attributes, nodeType) {
|
||||
const groups = {
|
||||
'DNS Records': { icon: '🔍', priority: 'high', attributes: [] },
|
||||
'Certificate Information': { icon: '🔒', priority: 'high', attributes: [] },
|
||||
'Network Information': { icon: '🌐', priority: 'high', attributes: [] },
|
||||
'Provider Data': { icon: '📊', priority: 'medium', attributes: [] },
|
||||
'Technical Details': { icon: '⚙️', priority: 'low', attributes: [] }
|
||||
};
|
||||
|
||||
// Initialize groups
|
||||
Object.entries(groupConfigs).forEach(([name, config]) => {
|
||||
groups[name] = {
|
||||
...config,
|
||||
attributes: []
|
||||
};
|
||||
});
|
||||
for (const attr of attributes) {
|
||||
const provider = attr.provider?.toLowerCase() || '';
|
||||
const name = attr.name?.toLowerCase() || '';
|
||||
|
||||
// Add a catch-all group
|
||||
groups['Other Information'] = {
|
||||
icon: '📋',
|
||||
priority: 'low',
|
||||
attributes: []
|
||||
};
|
||||
|
||||
// Classify attributes into groups
|
||||
attributes.forEach(attr => {
|
||||
let assigned = false;
|
||||
|
||||
// Try to assign to a specific group based on provider or keywords
|
||||
for (const [groupName, config] of Object.entries(groupConfigs)) {
|
||||
const matchesProvider = config.providers.includes(attr.provider);
|
||||
const matchesKeyword = config.keywords.some(keyword =>
|
||||
attr.name.toLowerCase().includes(keyword) ||
|
||||
attr.type.toLowerCase().includes(keyword)
|
||||
);
|
||||
|
||||
if (matchesProvider || matchesKeyword) {
|
||||
groups[groupName].attributes.push(attr);
|
||||
assigned = true;
|
||||
break;
|
||||
}
|
||||
|
||||
// DNS-related attributes
|
||||
if (provider === 'dns' || ['dns', 'record', 'ptr', 'mx', 'cname', 'ns', 'txt', 'soa'].some(keyword => name.includes(keyword))) {
|
||||
groups['DNS Records'].attributes.push(attr);
|
||||
assigned = true;
|
||||
}
|
||||
// Certificate-related attributes
|
||||
else if (provider === 'crtsh' || ['cert', 'certificate', 'ssl', 'tls', 'issuer', 'validity', 'san'].some(keyword => name.includes(keyword))) {
|
||||
groups['Certificate Information'].attributes.push(attr);
|
||||
assigned = true;
|
||||
}
|
||||
// Network/Shodan attributes
|
||||
else if (provider === 'shodan' || ['port', 'service', 'banner', 'asn', 'organization', 'country', 'city', 'network'].some(keyword => name.includes(keyword))) {
|
||||
groups['Network Information'].attributes.push(attr);
|
||||
assigned = true;
|
||||
}
|
||||
// Provider-specific data
|
||||
else if (provider && ['shodan_', 'crtsh_', 'dns_'].some(prefix => name.startsWith(prefix))) {
|
||||
groups['Provider Data'].attributes.push(attr);
|
||||
assigned = true;
|
||||
}
|
||||
|
||||
// If not assigned to any specific group, put in "Other"
|
||||
// If not assigned to any specific group, put in technical details
|
||||
if (!assigned) {
|
||||
groups['Other Information'].attributes.push(attr);
|
||||
groups['Technical Details'].attributes.push(attr);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Remove empty groups
|
||||
Object.keys(groups).forEach(groupName => {
|
||||
@ -1130,119 +986,47 @@ class DNSReconApp {
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Generate large entity details using unified data model
|
||||
* UPDATED: Enhanced correlation details showing the correlated attribute clearly (no formatting)
|
||||
*/
|
||||
generateLargeEntityDetails(node) {
|
||||
// UPDATED: Look for attributes in the unified model structure
|
||||
const nodesAttribute = this.findAttributeByName(node.attributes, 'nodes');
|
||||
const countAttribute = this.findAttributeByName(node.attributes, 'count');
|
||||
const nodeTypeAttribute = this.findAttributeByName(node.attributes, 'node_type');
|
||||
const sourceProviderAttribute = this.findAttributeByName(node.attributes, 'source_provider');
|
||||
const discoveryDepthAttribute = this.findAttributeByName(node.attributes, 'discovery_depth');
|
||||
generateCorrelationDetails(node) {
|
||||
const metadata = node.metadata || {};
|
||||
const value = metadata.value;
|
||||
const correlatedNodes = metadata.correlated_nodes || [];
|
||||
const sources = metadata.sources || [];
|
||||
|
||||
const nodes = nodesAttribute ? nodesAttribute.value : [];
|
||||
const count = countAttribute ? countAttribute.value : 0;
|
||||
const nodeType = nodeTypeAttribute ? nodeTypeAttribute.value : 'nodes';
|
||||
const sourceProvider = sourceProviderAttribute ? sourceProviderAttribute.value : 'Unknown';
|
||||
const discoveryDepth = discoveryDepthAttribute ? discoveryDepthAttribute.value : 'Unknown';
|
||||
let html = '';
|
||||
|
||||
let html = `
|
||||
// Show what attribute is being correlated
|
||||
const primarySource = metadata.primary_source || 'unknown';
|
||||
|
||||
html += `
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📦 Entity Summary</summary>
|
||||
<summary>
|
||||
<span>🔗 Correlation: ${primarySource}</span>
|
||||
<span class="merge-badge">${correlatedNodes.length}</span>
|
||||
</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Contains:</span>
|
||||
<span class="attribute-value-compact">${count} ${nodeType}s</span>
|
||||
<span class="attribute-key-compact">Shared Value</span>
|
||||
<span class="attribute-value-compact"><code>${this.escapeHtml(String(value))}</code></span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Provider:</span>
|
||||
<span class="attribute-value-compact">${sourceProvider}</span>
|
||||
<span class="attribute-key-compact">Attribute Type</span>
|
||||
<span class="attribute-value-compact">${primarySource}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Depth:</span>
|
||||
<span class="attribute-value-compact">${discoveryDepth}</span>
|
||||
<span class="attribute-key-compact">Correlated Nodes</span>
|
||||
<span class="attribute-value-compact">${correlatedNodes.length} nodes</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📋 Contained ${nodeType}s (${nodes.length})</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="relationship-compact">
|
||||
`;
|
||||
|
||||
// Use node.id for the large_entity_id
|
||||
const largeEntityId = node.id;
|
||||
|
||||
if (Array.isArray(nodes)) {
|
||||
nodes.forEach(innerNodeId => {
|
||||
html += `
|
||||
<div class="relationship-compact-item">
|
||||
<span class="node-link-compact" data-node-id="${innerNodeId}">${innerNodeId}</span>
|
||||
<button class="btn-icon-small extract-node-btn"
|
||||
title="Extract to graph"
|
||||
data-large-entity-id="${largeEntityId}"
|
||||
data-node-id="${innerNodeId}">[+]</button>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
}
|
||||
|
||||
html += '</div></div></details></div>';
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
generateCorrelationDetails(node) {
|
||||
const metadata = node.metadata || {};
|
||||
const values = metadata.values || [];
|
||||
const sources = metadata.sources || [];
|
||||
const mergeCount = metadata.merge_count || 1;
|
||||
|
||||
let html = '';
|
||||
|
||||
// Correlation values section with meaningful labels - reuses existing modal structure
|
||||
html += `
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>
|
||||
<span>🔗 Correlation Values</span>
|
||||
<span class="merge-badge">${mergeCount} value${mergeCount > 1 ? 's' : ''}</span>
|
||||
</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
`;
|
||||
|
||||
// Create a map of values to their source attributes for better labeling
|
||||
const valueSourceMap = this.createValueSourceMap(values, sources);
|
||||
|
||||
values.forEach((value, index) => {
|
||||
const sourceInfo = valueSourceMap[index] || {};
|
||||
const attributeName = sourceInfo.meaningfulName || `Value ${index + 1}`;
|
||||
const sourceDetails = sourceInfo.details || '';
|
||||
|
||||
html += `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">
|
||||
<span class="correlation-attr-name">${this.escapeHtml(attributeName)}</span>
|
||||
${sourceDetails ? `<span class="correlation-hint" title="${this.escapeHtml(sourceDetails)}"> ℹ️</span>` : ''}
|
||||
</span>
|
||||
<span class="attribute-value-compact">
|
||||
<code>${this.escapeHtml(String(value))}</code>
|
||||
</span>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
html += '</div></div></details></div>';
|
||||
|
||||
// Correlated nodes section - reuses existing relationship display
|
||||
const correlatedNodes = metadata.correlated_nodes || [];
|
||||
// Show the correlated nodes
|
||||
if (correlatedNodes.length > 0) {
|
||||
html += `
|
||||
<div class="modal-section">
|
||||
@ -1266,42 +1050,77 @@ class DNSReconApp {
|
||||
return html;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Create a mapping of values to their source attribute information
|
||||
* UPDATED: Generate large entity details using unified data model
|
||||
*/
|
||||
createValueSourceMap(values, sources) {
|
||||
const valueSourceMap = {};
|
||||
generateLargeEntityDetails(node) {
|
||||
// Look for attributes in the unified model structure
|
||||
const attributes = node.attributes || [];
|
||||
const nodesAttribute = attributes.find(attr => attr.name === 'nodes');
|
||||
const countAttribute = attributes.find(attr => attr.name === 'count');
|
||||
const nodeTypeAttribute = attributes.find(attr => attr.name === 'node_type');
|
||||
const sourceProviderAttribute = attributes.find(attr => attr.name === 'source_provider');
|
||||
const discoveryDepthAttribute = attributes.find(attr => attr.name === 'discovery_depth');
|
||||
|
||||
// Group sources by their meaningful attributes
|
||||
const attrGroups = {};
|
||||
sources.forEach(source => {
|
||||
const meaningfulAttr = source.meaningful_attr || source.parent_attr || 'correlation';
|
||||
const nodes = nodesAttribute ? nodesAttribute.value : [];
|
||||
const count = countAttribute ? countAttribute.value : 0;
|
||||
const nodeType = nodeTypeAttribute ? nodeTypeAttribute.value : 'nodes';
|
||||
const sourceProvider = sourceProviderAttribute ? sourceProviderAttribute.value : 'Unknown';
|
||||
const discoveryDepth = discoveryDepthAttribute ? discoveryDepthAttribute.value : 'Unknown';
|
||||
|
||||
let html = `
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📦 Entity Summary</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Contains</span>
|
||||
<span class="attribute-value-compact">${count} ${nodeType}s</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Provider</span>
|
||||
<span class="attribute-value-compact">${sourceProvider}</span>
|
||||
</div>
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">Depth</span>
|
||||
<span class="attribute-value-compact">${discoveryDepth}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</div>
|
||||
|
||||
if (!attrGroups[meaningfulAttr]) {
|
||||
attrGroups[meaningfulAttr] = {
|
||||
nodeIds: [],
|
||||
paths: []
|
||||
};
|
||||
}
|
||||
attrGroups[meaningfulAttr].nodeIds.push(source.node_id);
|
||||
attrGroups[meaningfulAttr].paths.push(source.path || '');
|
||||
});
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📋 Contained ${nodeType}s (${Array.isArray(nodes) ? nodes.length : 0})</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="relationship-compact">
|
||||
`;
|
||||
|
||||
// Map values to their best attribute names
|
||||
values.forEach((value, index) => {
|
||||
// Find the most meaningful attribute name
|
||||
const attrNames = Object.keys(attrGroups);
|
||||
const bestAttr = attrNames.find(attr => attr !== 'correlation' && attr !== 'unknown') || attrNames[0] || 'correlation';
|
||||
|
||||
if (attrGroups[bestAttr]) {
|
||||
valueSourceMap[index] = {
|
||||
meaningfulName: bestAttr,
|
||||
details: `Found in: ${[...new Set(attrGroups[bestAttr].nodeIds)].join(', ')}`
|
||||
};
|
||||
}
|
||||
});
|
||||
const largeEntityId = node.id;
|
||||
|
||||
if (Array.isArray(nodes)) {
|
||||
nodes.forEach(innerNodeId => {
|
||||
html += `
|
||||
<div class="relationship-compact-item">
|
||||
<span class="node-link-compact" data-node-id="${innerNodeId}">${innerNodeId}</span>
|
||||
<button class="btn-icon-small extract-node-btn"
|
||||
title="Extract to graph"
|
||||
data-large-entity-id="${largeEntityId}"
|
||||
data-node-id="${innerNodeId}">[+]</button>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
}
|
||||
|
||||
return valueSourceMap;
|
||||
html += '</div></div></details></div>';
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
generateRelationshipsSection(node) {
|
||||
@ -1372,85 +1191,30 @@ class DNSReconApp {
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Generate attributes section for the new flat data model
|
||||
*/
|
||||
generateAttributesSection(attributes) {
|
||||
if (!Array.isArray(attributes) || attributes.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
let html = `
|
||||
<div class="modal-section">
|
||||
<details open>
|
||||
<summary>📊 Attributes (${attributes.length})</summary>
|
||||
<div class="modal-section-content">
|
||||
<div class="attribute-list">
|
||||
`;
|
||||
|
||||
attributes.forEach(attr => {
|
||||
html += `
|
||||
<div class="attribute-item-compact">
|
||||
<span class="attribute-key-compact">${this.formatLabel(attr.name)}</span>
|
||||
<span class="attribute-value-compact">${this.formatStandardAttributeValue(attr)}</span>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
html += '</div></div></details></div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* UPDATED: Format StandardAttribute value for display
|
||||
*/
|
||||
formatStandardAttributeValue(attr) {
|
||||
const value = attr.value;
|
||||
|
||||
if (value === null || value === undefined) {
|
||||
return '<em>None</em>';
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) return '<em>None</em>';
|
||||
if (value.length === 1) return this.escapeHtml(String(value[0]));
|
||||
|
||||
let html = '<div class="array-display">';
|
||||
value.forEach((item, index) => {
|
||||
html += `<div class="array-display-item">${this.escapeHtml(String(item))}</div>`;
|
||||
});
|
||||
html += '</div>';
|
||||
return html;
|
||||
}
|
||||
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
return `<div class="object-display">${this.formatObjectCompact(value)}</div>`;
|
||||
}
|
||||
|
||||
return this.escapeHtml(String(value));
|
||||
}
|
||||
|
||||
formatObjectCompact(obj) {
|
||||
if (!obj || typeof obj !== 'object') return '';
|
||||
|
||||
let html = '';
|
||||
const entries = Object.entries(obj);
|
||||
if (entries.length <= 2) {
|
||||
let html = '';
|
||||
entries.forEach(([key, value]) => {
|
||||
html += `<div><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`;
|
||||
});
|
||||
return html;
|
||||
}
|
||||
|
||||
entries.forEach(([key, value]) => {
|
||||
html += `<div><strong>${key}:</strong> `;
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
if (Array.isArray(value)) {
|
||||
html += `[${value.length} items]`;
|
||||
} else {
|
||||
html += `{${Object.keys(value).length} properties}`;
|
||||
}
|
||||
} else {
|
||||
html += this.escapeHtml(String(value));
|
||||
}
|
||||
html += '</div>';
|
||||
});
|
||||
|
||||
return html;
|
||||
// For complex objects, show first entry with expansion
|
||||
return `
|
||||
<div><strong>${entries[0][0]}:</strong> ${this.escapeHtml(String(entries[0][1]))}</div>
|
||||
<details class="object-more">
|
||||
<summary>+${entries.length - 1} more properties...</summary>
|
||||
<div class="object-display">
|
||||
${entries.slice(1).map(([key, value]) =>
|
||||
`<div><strong>${key}:</strong> ${this.escapeHtml(String(value))}</div>`
|
||||
).join('')}
|
||||
</div>
|
||||
</details>
|
||||
`;
|
||||
}
|
||||
|
||||
generateDescriptionSection(node) {
|
||||
@ -1660,8 +1424,8 @@ class DNSReconApp {
|
||||
*/
|
||||
getNodeTypeIcon(nodeType) {
|
||||
const icons = {
|
||||
'domain': '🌐',
|
||||
'ip': '🔍',
|
||||
'domain': '🌍',
|
||||
'ip': '📍',
|
||||
'asn': '🏢',
|
||||
'large_entity': '📦',
|
||||
'correlation_object': '🔗'
|
||||
@ -1710,28 +1474,6 @@ class DNSReconApp {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle all entity nodes in large entity view
|
||||
*/
|
||||
toggleAllEntities() {
|
||||
const entityCards = this.elements.modalDetails.querySelectorAll('.entity-node-card');
|
||||
const allExpanded = Array.from(entityCards).every(card => card.classList.contains('expanded'));
|
||||
|
||||
entityCards.forEach(card => {
|
||||
if (allExpanded) {
|
||||
card.classList.remove('expanded');
|
||||
} else {
|
||||
card.classList.add('expanded');
|
||||
}
|
||||
});
|
||||
|
||||
// Update button text
|
||||
const toggleBtn = this.elements.modalDetails.querySelector('.toggle-all-btn');
|
||||
if (toggleBtn) {
|
||||
toggleBtn.textContent = allExpanded ? 'Expand All' : 'Collapse All';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhanced keyboard navigation for modals
|
||||
*/
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user