data-model #2

Merged
mstoeck3 merged 20 commits from data-model into main 2025-09-17 21:56:18 +00:00
6 changed files with 66 additions and 39 deletions
Showing only changes of commit 284660ab8c - Show all commits

View File

@ -18,7 +18,8 @@ class NodeType(Enum):
"""Enumeration of supported node types.""" """Enumeration of supported node types."""
DOMAIN = "domain" DOMAIN = "domain"
IP = "ip" IP = "ip"
ASN = "asn" ISP = "isp"
CA = "ca"
LARGE_ENTITY = "large_entity" LARGE_ENTITY = "large_entity"
CORRELATION_OBJECT = "correlation_object" CORRELATION_OBJECT = "correlation_object"

View File

@ -546,7 +546,7 @@ class Scanner:
self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time) self._update_provider_state(target, provider_name, 'failed', 0, str(e), start_time)
return None return None
def _process_provider_result_unified(self, target: str, provider: BaseProvider, def _process_provider_result_unified(self, target: str, provider: BaseProvider,
provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]: provider_result: ProviderResult, current_depth: int) -> Tuple[Set[str], bool]:
""" """
Process a unified ProviderResult object to update the graph. Process a unified ProviderResult object to update the graph.
@ -573,11 +573,9 @@ class Scanner:
if self.graph.graph.has_node(node_id): if self.graph.graph.has_node(node_id):
if _is_valid_ip(node_id): if _is_valid_ip(node_id):
node_type = NodeType.IP node_type = NodeType.IP
elif node_id.startswith('AS') and node_id[2:].isdigit():
node_type = NodeType.ASN
else: else:
node_type = NodeType.DOMAIN node_type = NodeType.DOMAIN
self.graph.add_node(node_id, node_type, attributes=node_attributes_list) self.graph.add_node(node_id, node_type, attributes=node_attributes_list)
if provider_result.get_relationship_count() > self.config.large_entity_threshold: if provider_result.get_relationship_count() > self.config.large_entity_threshold:
@ -590,27 +588,30 @@ class Scanner:
source_node = relationship.source_node source_node = relationship.source_node
target_node = relationship.target_node target_node = relationship.target_node
source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN source_type = NodeType.IP if _is_valid_ip(source_node) else NodeType.DOMAIN
if target_node.startswith('AS') and target_node[2:].isdigit():
target_type = NodeType.ASN if provider_name == 'shodan' and relationship.relationship_type == 'ip_to_isp':
target_type = NodeType.ISP
elif provider_name == 'crtsh' and relationship.relationship_type == 'issued_by':
target_type = NodeType.CA
elif _is_valid_ip(target_node): elif _is_valid_ip(target_node):
target_type = NodeType.IP target_type = NodeType.IP
else: else:
target_type = NodeType.DOMAIN target_type = NodeType.DOMAIN
self.graph.add_node(source_node, source_type) self.graph.add_node(source_node, source_type)
self.graph.add_node(target_node, target_type) self.graph.add_node(target_node, target_type)
if self.graph.add_edge( if self.graph.add_edge(
source_node, target_node, source_node, target_node,
relationship.relationship_type, relationship.relationship_type,
relationship.confidence, relationship.confidence,
provider_name, provider_name,
relationship.raw_data relationship.raw_data
): ):
pass pass
if _is_valid_domain(target_node) or _is_valid_ip(target_node): if _is_valid_domain(target_node) or _is_valid_ip(target_node):
discovered_targets.add(target_node) discovered_targets.add(target_node)

View File

@ -293,6 +293,16 @@ class CrtShProvider(BaseProvider):
cert_domains = self._extract_domains_from_certificate(cert_data) cert_domains = self._extract_domains_from_certificate(cert_data)
all_discovered_domains.update(cert_domains) all_discovered_domains.update(cert_domains)
issuer_name = self._parse_issuer_organization(cert_data.get('issuer_name', ''))
if issuer_name:
result.add_relationship(
source_node=domain,
target_node=issuer_name,
relationship_type='issued_by',
provider=self.name,
confidence=0.95
)
for cert_domain in cert_domains: for cert_domain in cert_domains:
if not _is_valid_domain(cert_domain): if not _is_valid_domain(cert_domain):
continue continue

View File

@ -215,6 +215,27 @@ class ShodanProvider(BaseProvider):
""" """
result = ProviderResult() result = ProviderResult()
isp_name = data.get('org')
asn_value = data.get('asn')
if isp_name and asn_value:
result.add_relationship(
source_node=ip,
target_node=isp_name,
relationship_type='ip_to_isp',
provider=self.name,
confidence=0.9,
raw_data={'asn': asn_value}
)
result.add_attribute(
target_node=isp_name,
name='asn',
value=asn_value,
attr_type='isp_info',
provider=self.name,
confidence=0.9
)
for key, value in data.items(): for key, value in data.items():
if key == 'hostnames': if key == 'hostnames':
for hostname in value: for hostname in value:
@ -235,24 +256,6 @@ class ShodanProvider(BaseProvider):
raw_data=data, raw_data=data,
discovery_method="shodan_host_lookup" discovery_method="shodan_host_lookup"
) )
elif key == 'asn':
asn_name = f"AS{value[2:]}" if isinstance(value, str) and value.startswith('AS') else f"AS{value}"
result.add_relationship(
source_node=ip,
target_node=asn_name,
relationship_type='shodan_asn_membership',
provider=self.name,
confidence=0.7,
raw_data=data
)
self.log_relationship_discovery(
source_node=ip,
target_node=asn_name,
relationship_type='shodan_asn_membership',
confidence_score=0.7,
raw_data=data,
discovery_method="shodan_asn_lookup"
)
elif key == 'ports': elif key == 'ports':
for port in value: for port in value:
result.add_attribute( result.add_attribute(

View File

@ -665,7 +665,8 @@ class GraphManager {
const colors = { const colors = {
'domain': '#00ff41', // Green 'domain': '#00ff41', // Green
'ip': '#ff9900', // Amber 'ip': '#ff9900', // Amber
'asn': '#00aaff', // Blue 'isp': '#00aaff', // Blue
'ca': '#ff6b6b', // Red
'large_entity': '#ff6b6b', // Red for large entities 'large_entity': '#ff6b6b', // Red for large entities
'correlation_object': '#9620c0ff' 'correlation_object': '#9620c0ff'
}; };
@ -681,7 +682,8 @@ class GraphManager {
const borderColors = { const borderColors = {
'domain': '#00aa2e', 'domain': '#00aa2e',
'ip': '#cc7700', 'ip': '#cc7700',
'asn': '#0088cc', 'isp': '#0088cc',
'ca': '#cc5555',
'correlation_object': '#c235c9ff' 'correlation_object': '#c235c9ff'
}; };
return borderColors[nodeType] || '#666666'; return borderColors[nodeType] || '#666666';
@ -696,9 +698,10 @@ class GraphManager {
const sizes = { const sizes = {
'domain': 12, 'domain': 12,
'ip': 14, 'ip': 14,
'asn': 16, 'isp': 16,
'ca': 16,
'correlation_object': 8, 'correlation_object': 8,
'large_entity': 5 'large_entity': 25
}; };
return sizes[nodeType] || 12; return sizes[nodeType] || 12;
} }
@ -712,9 +715,10 @@ class GraphManager {
const shapes = { const shapes = {
'domain': 'dot', 'domain': 'dot',
'ip': 'square', 'ip': 'square',
'asn': 'triangle', 'isp': 'triangle',
'ca': 'diamond',
'correlation_object': 'hexagon', 'correlation_object': 'hexagon',
'large_entity': 'database' 'large_entity': 'dot'
}; };
return shapes[nodeType] || 'dot'; return shapes[nodeType] || 'dot';
} }

View File

@ -137,6 +137,14 @@
<div class="legend-color" style="background-color: #ff9900;"></div> <div class="legend-color" style="background-color: #ff9900;"></div>
<span>IP Addresses</span> <span>IP Addresses</span>
</div> </div>
<div class="legend-item">
<div class="legend-color" style="background-color: #00aaff;"></div>
<span>ISPs</span>
</div>
<div class="legend-item">
<div class="legend-color" style="background-color: #ff6b6b;"></div>
<span>Certificate Authorities</span>
</div>
<div class="legend-item"> <div class="legend-item">
<div class="legend-color" style="background-color: #c7c7c7;"></div> <div class="legend-color" style="background-color: #c7c7c7;"></div>
<span>Domain (invalid cert)</span> <span>Domain (invalid cert)</span>