data-model #2
@ -18,7 +18,8 @@ class NodeType(Enum):
|
|||||||
"""Enumeration of supported node types."""
|
"""Enumeration of supported node types."""
|
||||||
DOMAIN = "domain"
|
DOMAIN = "domain"
|
||||||
IP = "ip"
|
IP = "ip"
|
||||||
ASN = "asn"
|
ISP = "isp"
|
||||||
|
CA = "ca"
|
||||||
LARGE_ENTITY = "large_entity"
|
LARGE_ENTITY = "large_entity"
|
||||||
CORRELATION_OBJECT = "correlation_object"
|
CORRELATION_OBJECT = "correlation_object"
|
||||||
|
|
||||||
|
|||||||
@ -546,7 +546,7 @@ class Scanner:
|
|||||||
self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time)
|
self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _process_provider_result_unified(self, target: str, provider: BaseProvider,
|
def _process_provider_result_unified(self, target: str, provider: BaseProvider,
|
||||||
provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
|
provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
|
||||||
"""
|
"""
|
||||||
Process a unified ProviderResult object to update the graph.
|
Process a unified ProviderResult object to update the graph.
|
||||||
@ -573,11 +573,9 @@ class Scanner:
|
|||||||
if self.graph.graph.has_node(node_id):
|
if self.graph.graph.has_node(node_id):
|
||||||
if _is_valid_ip(node_id):
|
if _is_valid_ip(node_id):
|
||||||
node_type = NodeType.IP
|
node_type = NodeType.IP
|
||||||
elif node_id.startswith('AS') and node_id[2:].isdigit():
|
|
||||||
node_type = NodeType.ASN
|
|
||||||
else:
|
else:
|
||||||
node_type = NodeType.DOMAIN
|
node_type = NodeType.DOMAIN
|
||||||
|
|
||||||
self.graph.add_node(node_id, node_type, attributes=node_attributes_list)
|
self.graph.add_node(node_id, node_type, attributes=node_attributes_list)
|
||||||
|
|
||||||
if provider_result.get_relationship_count() > self.config.large_entity_threshold:
|
if provider_result.get_relationship_count() > self.config.large_entity_threshold:
|
||||||
@ -590,27 +588,30 @@ class Scanner:
|
|||||||
|
|
||||||
source_node = relationship.source_node
|
source_node = relationship.source_node
|
||||||
target_node = relationship.target_node
|
target_node = relationship.target_node
|
||||||
|
|
||||||
source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN
|
source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN
|
||||||
if target_node.startswith('AS') and target_node[2:].isdigit():
|
|
||||||
target_type = NodeType.ASN
|
if provider_name == 'shodan' and relationship.relationship_type == 'ip_to_isp':
|
||||||
|
target_type = NodeType.ISP
|
||||||
|
elif provider_name == 'crtsh' and relationship.relationship_type == 'issued_by':
|
||||||
|
target_type = NodeType.CA
|
||||||
elif _is_valid_ip(target_node):
|
elif _is_valid_ip(target_node):
|
||||||
target_type = NodeType.IP
|
target_type = NodeType.IP
|
||||||
else:
|
else:
|
||||||
target_type = NodeType.DOMAIN
|
target_type = NodeType.DOMAIN
|
||||||
|
|
||||||
self.graph.add_node(source_node, source_type)
|
self.graph.add_node(source_node, source_type)
|
||||||
self.graph.add_node(target_node, target_type)
|
self.graph.add_node(target_node, target_type)
|
||||||
|
|
||||||
if self.graph.add_edge(
|
if self.graph.add_edge(
|
||||||
source_node, target_node,
|
source_node, target_node,
|
||||||
relationship.relationship_type,
|
relationship.relationship_type,
|
||||||
relationship.confidence,
|
relationship.confidence,
|
||||||
provider_name,
|
provider_name,
|
||||||
relationship.raw_data
|
relationship.raw_data
|
||||||
):
|
):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if _is_valid_domain(target_node) or _is_valid_ip(target_node):
|
if _is_valid_domain(target_node) or _is_valid_ip(target_node):
|
||||||
discovered_targets.add(target_node)
|
discovered_targets.add(target_node)
|
||||||
|
|
||||||
|
|||||||
@ -293,6 +293,16 @@ class CrtShProvider(BaseProvider):
|
|||||||
cert_domains = self._extract_domains_from_certificate(cert_data)
|
cert_domains = self._extract_domains_from_certificate(cert_data)
|
||||||
all_discovered_domains.update(cert_domains)
|
all_discovered_domains.update(cert_domains)
|
||||||
|
|
||||||
|
issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', ''))
|
||||||
|
if issuer_name:
|
||||||
|
result.add_relationship(
|
||||||
|
source_node=domain,
|
||||||
|
target_node=issuer_name,
|
||||||
|
relationship_type='issued_by',
|
||||||
|
provider=self.name,
|
||||||
|
confidence=0.95
|
||||||
|
)
|
||||||
|
|
||||||
for cert_domain in cert_domains:
|
for cert_domain in cert_domains:
|
||||||
if not _is_valid_domain(cert_domain):
|
if not _is_valid_domain(cert_domain):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@ -215,6 +215,27 @@ class ShodanProvider(BaseProvider):
|
|||||||
"""
|
"""
|
||||||
result = ProviderResult()
|
result = ProviderResult()
|
||||||
|
|
||||||
|
isp_name = data.get('org')
|
||||||
|
asn_value = data.get('asn')
|
||||||
|
|
||||||
|
if isp_name and asn_value:
|
||||||
|
result.add_relationship(
|
||||||
|
source_node=ip,
|
||||||
|
target_node=isp_name,
|
||||||
|
relationship_type='ip_to_isp',
|
||||||
|
provider=self.name,
|
||||||
|
confidence=0.9,
|
||||||
|
raw_data={'asn': asn_value}
|
||||||
|
)
|
||||||
|
result.add_attribute(
|
||||||
|
target_node=isp_name,
|
||||||
|
name='asn',
|
||||||
|
value=asn_value,
|
||||||
|
attr_type='isp_info',
|
||||||
|
provider=self.name,
|
||||||
|
confidence=0.9
|
||||||
|
)
|
||||||
|
|
||||||
for key, value in data.items():
|
for key, value in data.items():
|
||||||
if key == 'hostnames':
|
if key == 'hostnames':
|
||||||
for hostname in value:
|
for hostname in value:
|
||||||
@ -235,24 +256,6 @@ class ShodanProvider(BaseProvider):
|
|||||||
raw_data=data,
|
raw_data=data,
|
||||||
discovery_method="shodan_host_lookup"
|
discovery_method="shodan_host_lookup"
|
||||||
)
|
)
|
||||||
elif key == 'asn':
|
|
||||||
asn_name = f"AS{value[2:]}" if isinstance(value, str) and value.startswith('AS') else f"AS{value}"
|
|
||||||
result.add_relationship(
|
|
||||||
source_node=ip,
|
|
||||||
target_node=asn_name,
|
|
||||||
relationship_type='shodan_asn_membership',
|
|
||||||
provider=self.name,
|
|
||||||
confidence=0.7,
|
|
||||||
raw_data=data
|
|
||||||
)
|
|
||||||
self.log_relationship_discovery(
|
|
||||||
source_node=ip,
|
|
||||||
target_node=asn_name,
|
|
||||||
relationship_type='shodan_asn_membership',
|
|
||||||
confidence_score=0.7,
|
|
||||||
raw_data=data,
|
|
||||||
discovery_method="shodan_asn_lookup"
|
|
||||||
)
|
|
||||||
elif key == 'ports':
|
elif key == 'ports':
|
||||||
for port in value:
|
for port in value:
|
||||||
result.add_attribute(
|
result.add_attribute(
|
||||||
|
|||||||
@ -665,7 +665,8 @@ class GraphManager {
|
|||||||
const colors = {
|
const colors = {
|
||||||
'domain': '#00ff41', // Green
|
'domain': '#00ff41', // Green
|
||||||
'ip': '#ff9900', // Amber
|
'ip': '#ff9900', // Amber
|
||||||
'asn': '#00aaff', // Blue
|
'isp': '#00aaff', // Blue
|
||||||
|
'ca': '#ff6b6b', // Red
|
||||||
'large_entity': '#ff6b6b', // Red for large entities
|
'large_entity': '#ff6b6b', // Red for large entities
|
||||||
'correlation_object': '#9620c0ff'
|
'correlation_object': '#9620c0ff'
|
||||||
};
|
};
|
||||||
@ -681,7 +682,8 @@ class GraphManager {
|
|||||||
const borderColors = {
|
const borderColors = {
|
||||||
'domain': '#00aa2e',
|
'domain': '#00aa2e',
|
||||||
'ip': '#cc7700',
|
'ip': '#cc7700',
|
||||||
'asn': '#0088cc',
|
'isp': '#0088cc',
|
||||||
|
'ca': '#cc5555',
|
||||||
'correlation_object': '#c235c9ff'
|
'correlation_object': '#c235c9ff'
|
||||||
};
|
};
|
||||||
return borderColors[nodeType] || '#666666';
|
return borderColors[nodeType] || '#666666';
|
||||||
@ -696,9 +698,10 @@ class GraphManager {
|
|||||||
const sizes = {
|
const sizes = {
|
||||||
'domain': 12,
|
'domain': 12,
|
||||||
'ip': 14,
|
'ip': 14,
|
||||||
'asn': 16,
|
'isp': 16,
|
||||||
|
'ca': 16,
|
||||||
'correlation_object': 8,
|
'correlation_object': 8,
|
||||||
'large_entity': 5
|
'large_entity': 25
|
||||||
};
|
};
|
||||||
return sizes[nodeType] || 12;
|
return sizes[nodeType] || 12;
|
||||||
}
|
}
|
||||||
@ -712,9 +715,10 @@ class GraphManager {
|
|||||||
const shapes = {
|
const shapes = {
|
||||||
'domain': 'dot',
|
'domain': 'dot',
|
||||||
'ip': 'square',
|
'ip': 'square',
|
||||||
'asn': 'triangle',
|
'isp': 'triangle',
|
||||||
|
'ca': 'diamond',
|
||||||
'correlation_object': 'hexagon',
|
'correlation_object': 'hexagon',
|
||||||
'large_entity': 'database'
|
'large_entity': 'dot'
|
||||||
};
|
};
|
||||||
return shapes[nodeType] || 'dot';
|
return shapes[nodeType] || 'dot';
|
||||||
}
|
}
|
||||||
|
|||||||
@ -137,6 +137,14 @@
|
|||||||
<div class="legend-color" style="background-color: #ff9900;"></div>
|
<div class="legend-color" style="background-color: #ff9900;"></div>
|
||||||
<span>IP Addresses</span>
|
<span>IP Addresses</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="legend-item">
|
||||||
|
<div class="legend-color" style="background-color: #00aaff;"></div>
|
||||||
|
<span>ISPs</span>
|
||||||
|
</div>
|
||||||
|
<div class="legend-item">
|
||||||
|
<div class="legend-color" style="background-color: #ff6b6b;"></div>
|
||||||
|
<span>Certificate Authorities</span>
|
||||||
|
</div>
|
||||||
<div class="legend-item">
|
<div class="legend-item">
|
||||||
<div class="legend-color" style="background-color: #c7c7c7;"></div>
|
<div class="legend-color" style="background-color: #c7c7c7;"></div>
|
||||||
<span>Domain (invalid cert)</span>
|
<span>Domain (invalid cert)</span>
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user