noisedestroyers/StreamLens
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tabbed frametype filtering

Browse Source
This commit is contained in:
noisedestroyers
2025-07-30 23:48:32 -04:00
parent 8d883f25c3
commit bb3eeb79d0
92 changed files with 33696 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
analyzer/analysis/background_analyzer.py
View File

@@ -37,7 +37,7 @@ class BackgroundAnalyzer:
"""Analyzer that processes PCAP files in background threads"""
def __init__(self, analyzer: EthernetAnalyzer,
num_threads: int = 4,
num_threads: int = 1, # Force single-threaded to avoid race conditions
batch_size: int = 1000,
progress_callback: Optional[Callable[[ParsingProgress], None]] = None,
flow_update_callback: Optional[Callable[[], None]] = None):
@@ -74,7 +74,7 @@ class BackgroundAnalyzer:
# Flow update batching
self.packets_since_update = 0
self.update_batch_size = 50 # Update UI every 50 packets (more frequent)
self.update_batch_size = 100 # Update UI every 100 packets (slower for less frequent updates)
self.update_lock = threading.Lock()
logging.basicConfig(level=logging.INFO)
@@ -87,6 +87,7 @@ class BackgroundAnalyzer:
return
self.is_parsing = True
self.analyzer.is_parsing = True # Set parsing flag on analyzer
self.stop_event.clear()
self.start_time = time.time()
self.processed_packets = 0
@@ -221,8 +222,8 @@ class BackgroundAnalyzer:
try:
current_time = time.time()
# Update every 0.5 seconds
if current_time - last_update_time >= 0.5:
# Update every 2.0 seconds (slower progress updates)
if current_time - last_update_time >= 2.0:
with self.parse_lock:
current_packets = self.processed_packets
@@ -246,7 +247,7 @@ class BackgroundAnalyzer:
if all(f.done() for f in futures):
break
time.sleep(0.1)
time.sleep(0.5) # Slower monitoring loop
except KeyboardInterrupt:
self.logger.info("Monitor thread interrupted")
break
@@ -256,6 +257,7 @@ class BackgroundAnalyzer:
# Final update
self.is_parsing = False
self.analyzer.is_parsing = False # Clear parsing flag on analyzer
self._report_progress(is_complete=True)
# Final flow update
@@ -267,7 +269,7 @@ class BackgroundAnalyzer:
# Calculate final statistics
with self.flow_lock:
self.analyzer.statistics_engine.calculate_all_statistics()
self.analyzer.statistics_engine.calculate_flow_statistics(self.analyzer.flows)
def _report_progress(self, packets_per_second: float = 0,
elapsed_time: float = 0,

1
analyzer/analysis/core.py
View File

@@ -26,6 +26,7 @@ class EthernetAnalyzer:
self.all_packets: List[Packet] = []
self.is_live = False
self.stop_capture = False
self.is_parsing = False # Flag to track parsing state
# Expose flows for backward compatibility
self.flows = self.flow_manager.flows

36
analyzer/analysis/flow_manager.py
View File

@@ -256,21 +256,31 @@ class FlowManager:
decoded = ch10_info['decoded_payload']
data_type_name = decoded.get('data_type_name', 'CH10-Data')
# Simplify timing frame names for display
# For timing analysis purposes, group frames by their actual timing behavior
# rather than their semantic meaning. Based on debug analysis:
# - Some timing frames have ~26s intervals (high-level timing)
# - Other frames (including some timing) have ~100ms intervals (data stream)
# Keep high-level timing frames separate (they have very different timing)
if 'ACTTS' in data_type_name:
return 'CH10-ACTTS'
# Note: Extended Timing frames often have the same ~100ms timing as data frames
# so they should be grouped with CH10-Data for accurate timing analysis
elif 'Sync' in data_type_name and 'Custom' in data_type_name:
return 'CH10-Sync'
elif 'Clock' in data_type_name and 'Custom' in data_type_name:
return 'CH10-Clock'
elif ('Time' in data_type_name or 'Timing' in data_type_name) and 'Custom' in data_type_name:
# Custom timing frames often have the 26s interval pattern
if 'Time' in data_type_name:
return 'CH10-Time'
else:
return 'CH10-Timing'
# Special data types that should remain separate
elif 'GPS NMEA' in data_type_name:
return 'CH10-GPS'
elif 'EAG ACMI' in data_type_name:
return 'CH10-ACMI'
elif 'Custom' in data_type_name and 'Timing' in data_type_name:
# Extract variant for custom timing
if 'Variant 0x04' in data_type_name:
return 'CH10-ACTTS'
elif 'Extended Timing' in data_type_name:
return 'CH10-ExtTiming'
else:
return 'CH10-Timing'
elif 'Ethernet' in data_type_name:
return 'CH10-Ethernet'
elif 'Image' in data_type_name:
@@ -279,10 +289,10 @@ class FlowManager:
return 'CH10-UART'
elif 'CAN' in data_type_name:
return 'CH10-CAN'
elif 'Unknown' not in data_type_name:
# Extract first word for other known types
first_word = data_type_name.split()[0]
return f'CH10-{first_word}'
# Everything else gets grouped as CH10-Data for consistent timing analysis
# This includes: Multi-Source, regular timing frames, custom data types, etc.
else:
return 'CH10-Data'
return 'CH10-Data'

22
analyzer/analysis/statistics.py
View File

@@ -27,6 +27,13 @@ class StatisticsEngine:
for flow in flows.values():
self._calculate_single_flow_statistics(flow)
def calculate_all_statistics(self, analyzer=None) -> None:
"""Calculate statistics for all flows (called by background analyzer)"""
# This is called by the background analyzer
# The analyzer parameter should be passed in
if analyzer and hasattr(analyzer, 'flows'):
self.calculate_flow_statistics(analyzer.flows)
def _calculate_single_flow_statistics(self, flow: FlowStats) -> None:
"""Calculate statistics for a single flow"""
# Ensure timeline statistics are calculated
@@ -77,11 +84,18 @@ class StatisticsEngine:
# Detect outliers for this frame type
ft_threshold = ft_stats.avg_inter_arrival + (self.outlier_threshold_sigma * ft_stats.std_inter_arrival)
# Clear existing outliers to recalculate
ft_stats.outlier_frames.clear()
ft_stats.outlier_details.clear()
ft_stats.enhanced_outlier_details.clear()
for i, inter_time in enumerate(ft_stats.inter_arrival_times):
if inter_time > ft_threshold:
frame_number = ft_stats.frame_numbers[i + 1]
frame_number = ft_stats.frame_numbers[i + 1] # Current frame
prev_frame_number = ft_stats.frame_numbers[i] # Previous frame
ft_stats.outlier_frames.append(frame_number)
ft_stats.outlier_details.append((frame_number, inter_time))
ft_stats.outlier_details.append((frame_number, inter_time)) # Legacy format
ft_stats.enhanced_outlier_details.append((frame_number, prev_frame_number, inter_time)) # Enhanced format
def get_flow_summary_statistics(self, flows: Dict[tuple, FlowStats]) -> Dict[str, float]:
"""Get summary statistics across all flows"""
@@ -232,9 +246,11 @@ class StatisticsEngine:
threshold = avg + (self.outlier_threshold_sigma * std)
if new_time > threshold:
frame_number = ft_stats.frame_numbers[-1]
prev_frame_number = ft_stats.frame_numbers[-2] if len(ft_stats.frame_numbers) > 1 else 0
if frame_number not in ft_stats.outlier_frames:
ft_stats.outlier_frames.append(frame_number)
ft_stats.outlier_details.append((frame_number, new_time))
ft_stats.outlier_details.append((frame_number, new_time)) # Legacy format
ft_stats.enhanced_outlier_details.append((frame_number, prev_frame_number, new_time)) # Enhanced format
stats['outlier_count'] += 1
stats['last_avg'] = avg