tabbed frametype filtering

2025-07-30 23:48:32 -04:00
parent 8d883f25c3
commit bb3eeb79d0
92 changed files with 33696 additions and 139 deletions
--- a/analyzer/tui/textual/app_v2.py
+++ b/analyzer/tui/textual/app_v2.py
@@ -5,10 +5,11 @@ Modern TUI with real-time metrics, sparklines, and professional monitoring aesth

 from textual.app import App, ComposeResult
 from textual.containers import Container, Horizontal, Vertical, ScrollableContainer
-from textual.widgets import Header, Footer, Static, DataTable, Label
+from textual.widgets import Header, Footer, Static, DataTable, Label, TabPane
 from textual.reactive import reactive
 from textual.timer import Timer
 from textual.events import MouseDown, MouseMove
+from textual.binding import Binding
 from typing import TYPE_CHECKING
 from rich.text import Text
 from rich.console import Group
@@ -17,14 +18,30 @@ from rich.table import Table
 import time
 import signal
 import sys
+import datetime
+from pathlib import Path
+import subprocess
+import platform

 from .widgets.sparkline import SparklineWidget
 from .widgets.metric_card import MetricCard
 from .widgets.flow_table_v2 import EnhancedFlowTable
+from .widgets.filtered_flow_view import FilteredFlowView
+from ...reporting import FlowReportGenerator
 from .widgets.split_flow_details import FlowMainDetailsPanel, SubFlowDetailsPanel
 from .widgets.debug_panel import DebugPanel
+from .widgets.progress_bar import ParsingProgressBar
 from ...analysis.background_analyzer import BackgroundAnalyzer

+
+# Debugging imports
+try:
+    from textual_state_visualizer import TextualStateMonitor, TextualStateWebServer
+    from textual_inspector import inspect_textual_app, print_widget_tree
+    DEBUGGING_AVAILABLE = True
+except ImportError:
+    DEBUGGING_AVAILABLE = False
+
 if TYPE_CHECKING:
    from ...analysis.core import EthernetAnalyzer

@@ -47,14 +64,35 @@ class StreamLensAppV2(App):
    
    BINDINGS = [
        ("q", "quit", "Quit"),
-        ("1", "sort('flows')", "Sort Flows"),
-        ("2", "sort('packets')", "Sort Packets"),
-        ("3", "sort('volume')", "Sort Volume"),
-        ("4", "sort('quality')", "Sort Quality"),
+        ("1", "select_filter('1')", "Overview"),
+        ("2", "select_filter('2')", "Frame Type 2"),
+        ("3", "select_filter('3')", "Frame Type 3"),
+        ("4", "select_filter('4')", "Frame Type 4"),
+        ("5", "select_filter('5')", "Frame Type 5"),
+        ("6", "select_filter('6')", "Frame Type 6"),
+        ("7", "select_filter('7')", "Frame Type 7"),
+        ("8", "select_filter('8')", "Frame Type 8"),
+        ("9", "select_filter('9')", "Frame Type 9"),
+        ("0", "select_filter('0')", "Frame Type 10"),
+        ("alt+1", "sort_table_column(0)", "Sort by column 1"),
+        ("alt+2", "sort_table_column(1)", "Sort by column 2"),
+        ("alt+3", "sort_table_column(2)", "Sort by column 3"),
+        ("alt+4", "sort_table_column(3)", "Sort by column 4"),
+        ("alt+5", "sort_table_column(4)", "Sort by column 5"),
+        ("alt+6", "sort_table_column(5)", "Sort by column 6"),
+        ("alt+7", "sort_table_column(6)", "Sort by column 7"),
+        ("alt+8", "sort_table_column(7)", "Sort by column 8"),
+        ("alt+9", "sort_table_column(8)", "Sort by column 9"),
+        ("alt+0", "sort_table_column(9)", "Sort by column 10"),
        ("p", "toggle_pause", "Pause"),
        ("d", "show_details", "Details"),
        ("v", "toggle_view_mode", "Toggle View"),
+        ("r", "generate_report", "Generate Report"),
+        ("o", "copy_outliers", "Copy Outliers"),
        ("?", "toggle_help", "Help"),
+        Binding("ctrl+d,t", "debug_tree", "Debug: Widget Tree", show=False),
+        Binding("ctrl+d,f", "debug_focus", "Debug: Focused Widget", show=False),
+        Binding("ctrl+d,w", "start_web_debug", "Debug: Web Interface", show=False),
    ]
    
    # Reactive attributes
@@ -77,12 +115,12 @@ class StreamLensAppV2(App):
        self.sub_title = "Network Flow Analysis"
        self.paused = False
        
-        # Background parsing support
+        # Background parsing support - Use single thread to avoid race conditions in frame reference tracking
        self.background_analyzer = BackgroundAnalyzer(
            analyzer=analyzer,
-            num_threads=4,
+            num_threads=1,  # Single-threaded to prevent race conditions in outlier frame references
            batch_size=1000,
-            progress_callback=None,
+            progress_callback=self._on_progress_update,
            flow_update_callback=self._on_flow_update
        )
        self.pcap_file = None
@@ -99,6 +137,9 @@ class StreamLensAppV2(App):
        yield Header()
        
        with Container(id="main-container"):
+            # Progress bar for PCAP loading (initially hidden)
+            yield ParsingProgressBar(id="progress-bar")
+            
            # Ultra-compact metrics bar 
            with Horizontal(id="metrics-bar"):
                yield MetricCard("Flows", f"{self.total_flows}", id="flows-metric")
@@ -109,10 +150,10 @@ class StreamLensAppV2(App):
            
            # Main content area with conditional debug panel
            with Horizontal(id="content-area"):
-                # Left - Enhanced flow table 
-                yield EnhancedFlowTable(
+                # Left - Filtered flow view with frame type buttons
+                yield FilteredFlowView(
                    self.analyzer,
-                    id="flow-table",
+                    id="filtered-flow-view",
                    classes="panel-wide"
                )
                
@@ -153,9 +194,9 @@ class StreamLensAppV2(App):
        
        self.update_metrics()
        
-        # Set up update intervals like TipTop (reduced frequency since we have real-time updates)
-        self.metric_timer = self.set_interval(2.0, self.update_metrics)  # 0.5Hz for background updates
-        self.flow_timer = self.set_interval(5.0, self.update_flows)     # 0.2Hz for fallback flow updates
+        # Set up update intervals (slower during parsing to reduce CPU usage)
+        self.metric_timer = self.set_interval(5.0, self.update_metrics)  # 0.2Hz for slower background updates
+        self.flow_timer = self.set_interval(10.0, self.update_flows)     # 0.1Hz for slower fallback flow updates
        
        # Initialize sparkline history
        self._initialize_history()
@@ -164,13 +205,12 @@ class StreamLensAppV2(App):
        self.call_after_refresh(self._set_initial_focus)
    
    def _set_initial_focus(self):
-        """Set initial focus to the flow table after widgets are ready"""
+        """Set initial focus to the filtered flow view after widgets are ready"""
        try:
-            flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-            data_table = flow_table.query_one("#flows-data-table", DataTable)
-            data_table.focus()
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            flow_view.flow_table.focus()
        except Exception:
-            # If table isn't ready yet, try again after a short delay
+            # If flow view isn't ready yet, try again after a short delay
            self.set_timer(0.1, self._set_initial_focus)

    def _initialize_history(self):
@@ -210,13 +250,15 @@ class StreamLensAppV2(App):
            for flow in flows.values():
                if flow.enhanced_analysis.decoder_type != "Standard":
                    enhanced += 1
-                outliers += len(flow.outlier_frames)
+                # Use frame-type-specific outliers instead of flow-level outliers
+                outliers += sum(len(ft_stats.outlier_frames) for ft_stats in flow.frame_types.values())
        except Exception:
            # Fallback to direct access if background analyzer not available
            for flow in self.analyzer.flows.values():
                if flow.enhanced_analysis.decoder_type != "Standard":
                    enhanced += 1
-                outliers += len(flow.outlier_frames)
+                # Use frame-type-specific outliers instead of flow-level outliers
+                outliers += sum(len(ft_stats.outlier_frames) for ft_stats in flow.frame_types.values())
        
        self.enhanced_flows = enhanced
        self.outlier_count = outliers
@@ -286,10 +328,45 @@ class StreamLensAppV2(App):
        if self.paused:
            return
            
-        # Update flow table
-        flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-        flow_table.refresh_data()
+        # Update filtered flow view
+        flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+        flow_view.refresh_frame_types()
+        flow_view.refresh_flow_data()
+    def _on_progress_update(self, progress):
+        """Handle progress updates from background parser"""
+        try:
+            # Use call_from_thread to safely update UI from background thread
+            self.call_from_thread(self._update_progress_ui, progress)
+        except Exception:
+            # Ignore errors during shutdown
+            pass
    
+    def _update_progress_ui(self, progress):
+        """Update progress UI (called from main thread)"""
+        try:
+            progress_bar = self.query_one("#progress-bar", ParsingProgressBar)
+            
+            if progress.error:
+                progress_bar.show_error(progress.error)
+            elif progress.is_complete:
+                progress_bar.complete_parsing()
+                # Trigger frame type button creation now that parsing is complete
+                self._create_frame_type_buttons()
+            else:
+                # Start progress if this is the first update
+                if not progress_bar.is_visible and progress.total_packets > 0:
+                    progress_bar.start_parsing(progress.total_packets)
+                
+                # Update progress
+                progress_bar.update_progress(
+                    progress.processed_packets,
+                    progress.total_packets,
+                    progress.packets_per_second,
+                    progress.estimated_time_remaining
+                )
+        except Exception as e:
+            # Progress bar widget may not be available yet
+            pass
    
    def _on_flow_update(self):
        """Handle flow data updates from background parser"""
@@ -303,14 +380,30 @@ class StreamLensAppV2(App):
    def _update_flow_ui(self):
        """Update flow UI (called from main thread)"""
        try:
-            # Update flow table
-            flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-            flow_table.refresh_data()
+            # Update filtered flow view - frame types first for dynamic button creation
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            flow_view.refresh_frame_types()  # This will create buttons as frame types are detected
+            flow_view.refresh_flow_data()
            
+            # Also trigger button creation if parsing is complete but buttons haven't been created yet
+            if not self.analyzer.is_parsing and not getattr(flow_view, '_buttons_created', False):
+                self._create_frame_type_buttons()
+                
            # Also update metrics in real-time
            self.update_metrics()
        except Exception:
-            # Flow table widget may not be available yet
+            # Flow view widget may not be available yet
+            pass
+    
+    def _create_frame_type_buttons(self):
+        """Create frame type buttons now that parsing is complete"""
+        try:
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            # Force refresh of frame types now that parsing is complete
+            flow_view.refresh_frame_types()
+            flow_view.refresh_flow_data()
+        except Exception as e:
+            # Flow view widget may not be available yet
            pass
    
    def start_background_parsing(self, pcap_file: str):
@@ -372,18 +465,24 @@ class StreamLensAppV2(App):
        self.paused = not self.paused
        status = "PAUSED" if self.paused else "LIVE"
        
-        # Get current view mode to maintain it in subtitle
-        try:
-            flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-            view_mode = flow_table.get_current_view_mode()
-            self.sub_title = f"Network Flow Analysis - {status} - {view_mode} VIEW"
-        except:
-            self.sub_title = f"Network Flow Analysis - {status}"
+        # Update subtitle
+        self.sub_title = f"Network Flow Analysis - {status}"
    
-    def action_sort(self, key: str) -> None:
-        """Sort flow table by specified key"""
-        flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-        flow_table.sort_by(key)
+    def action_select_filter(self, number: str) -> None:
+        """Select frame type filter by number key"""
+        try:
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            flow_view.action_select_filter(number)
+        except Exception:
+            pass
+    
+    def action_sort_table_column(self, column_index: int) -> None:
+        """Sort table by column index"""
+        try:
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            flow_view.action_sort_column(column_index)
+        except Exception:
+            pass
    
    def action_show_details(self) -> None:
        """Show detailed view for selected flow"""
@@ -391,14 +490,11 @@ class StreamLensAppV2(App):
        pass
    
    def action_toggle_view_mode(self) -> None:
-        """Toggle between simplified and detailed view modes"""
-        flow_table = self.query_one("#flow-table", EnhancedFlowTable)
-        flow_table.toggle_view_mode()
-        
-        # Update subtitle to show current view mode
-        view_mode = flow_table.get_current_view_mode()
-        status = "PAUSED" if self.paused else "LIVE"
-        self.sub_title = f"Network Flow Analysis - {status} - {view_mode} VIEW"
+        """Toggle between different display modes"""
+        # For now, this could cycle through different column layouts
+        # or show more/less detail in the frame type views
+        pass
+    
    
    def on_mouse_down(self, event: MouseDown) -> None:
        """Prevent default mouse down behavior to disable mouse interaction."""
@@ -408,6 +504,126 @@ class StreamLensAppV2(App):
        """Prevent default mouse move behavior to disable mouse interaction."""
        event.prevent_default()
    
+    def action_generate_report(self) -> None:
+        """Generate comprehensive flow analysis report"""
+        try:
+            # Generate timestamp-based filename
+            timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+            output_file = f"streamlens_flow_report_{timestamp}.md"
+            
+            # Create report generator
+            report_generator = FlowReportGenerator(self.analyzer)
+            
+            # Generate report (markdown format)
+            report_content = report_generator.generate_report(output_file, "markdown")
+            
+            # Show success notification in the footer
+            self.sub_title = f"✅ Report generated: {output_file}"
+            
+            # Set a timer to restore the original subtitle
+            self.set_timer(3.0, self._restore_subtitle)
+            
+        except Exception as e:
+            # Show error notification
+            self.sub_title = f"❌ Report generation failed: {str(e)}"
+            self.set_timer(3.0, self._restore_subtitle)
+    
+    def _restore_subtitle(self) -> None:
+        """Restore the original subtitle"""
+        status = "PAUSED" if self.paused else "LIVE"
+        self.sub_title = f"Network Flow Analysis - {status}"
+    
+    def action_copy_outliers(self) -> None:
+        """Copy outlier frame information to clipboard"""
+        try:
+            # Get selected flow from the filtered view
+            flow_view = self.query_one("#filtered-flow-view", FilteredFlowView)
+            # For now, get the first flow (could be improved to use actual selection)
+            flows = list(self.analyzer.flows.values())
+            selected_flow = flows[0] if flows else None
+            
+            if not selected_flow:
+                self.sub_title = "⚠️ No flow selected"
+                self.set_timer(2.0, self._restore_subtitle)
+                return
+            
+            # Build frame-type-specific outlier information
+            outlier_info = []
+            outlier_info.append(f"Flow: {selected_flow.src_ip}:{selected_flow.src_port} → {selected_flow.dst_ip}:{selected_flow.dst_port}")
+            outlier_info.append(f"Protocol: {selected_flow.transport_protocol}")
+            outlier_info.append(f"Total Packets: {selected_flow.frame_count}")
+            
+            # Calculate total frame-type-specific outliers
+            total_frame_type_outliers = sum(len(ft_stats.outlier_frames) for ft_stats in selected_flow.frame_types.values())
+            outlier_info.append(f"Total Frame-Type Outliers: {total_frame_type_outliers}")
+            
+            if total_frame_type_outliers > 0:
+                outlier_info.append(f"\n=== Frame Type Outlier Analysis ===")
+                
+                # Show outliers per frame type
+                for frame_type, ft_stats in sorted(selected_flow.frame_types.items(), key=lambda x: len(x[1].outlier_frames), reverse=True):
+                    if ft_stats.outlier_frames:
+                        outlier_info.append(f"\n{frame_type}: {len(ft_stats.outlier_frames)} outliers")
+                        outlier_info.append(f"  Frames: {', '.join(map(str, sorted(ft_stats.outlier_frames)))}")
+                        outlier_info.append(f"  Avg ΔT: {ft_stats.avg_inter_arrival * 1000:.3f} ms")
+                        outlier_info.append(f"  Std σ: {ft_stats.std_inter_arrival * 1000:.3f} ms")
+                        outlier_info.append(f"  3σ Threshold: {(ft_stats.avg_inter_arrival + 3 * ft_stats.std_inter_arrival) * 1000:.3f} ms")
+                        
+                        # Show enhanced outlier information for this frame type
+                        if hasattr(ft_stats, 'enhanced_outlier_details') and ft_stats.enhanced_outlier_details:
+                            outlier_info.append(f"  Enhanced Outlier Details:")
+                            for frame_num, prev_frame_num, inter_time in sorted(ft_stats.enhanced_outlier_details[:5]):
+                                deviation = (inter_time - ft_stats.avg_inter_arrival) / ft_stats.std_inter_arrival if ft_stats.std_inter_arrival > 0 else 0
+                                outlier_info.append(f"    Frame {frame_num} (from {prev_frame_num}): {inter_time * 1000:.3f} ms ({deviation:.1f}σ)")
+                            if len(ft_stats.enhanced_outlier_details) > 5:
+                                outlier_info.append(f"    ... and {len(ft_stats.enhanced_outlier_details) - 5} more")
+                        elif ft_stats.outlier_details:
+                            outlier_info.append(f"  Outlier Details:")
+                            for frame_num, inter_time in sorted(ft_stats.outlier_details[:5]):
+                                deviation = (inter_time - ft_stats.avg_inter_arrival) / ft_stats.std_inter_arrival if ft_stats.std_inter_arrival > 0 else 0
+                                outlier_info.append(f"    Frame {frame_num}: {inter_time * 1000:.3f} ms ({deviation:.1f}σ)")
+                            if len(ft_stats.outlier_details) > 5:
+                                outlier_info.append(f"    ... and {len(ft_stats.outlier_details) - 5} more")
+            else:
+                outlier_info.append("\nNo frame-type-specific timing outliers detected.")
+            
+            # Copy to clipboard
+            clipboard_text = "\n".join(outlier_info)
+            self._copy_to_clipboard(clipboard_text)
+            
+            # Show success notification
+            total_frame_type_outliers = sum(len(ft_stats.outlier_frames) for ft_stats in selected_flow.frame_types.values())
+            self.sub_title = f"✅ Copied {total_frame_type_outliers} frame-type outliers to clipboard"
+            self.set_timer(2.0, self._restore_subtitle)
+            
+        except Exception as e:
+            self.sub_title = f"❌ Failed to copy: {str(e)}"
+            self.set_timer(2.0, self._restore_subtitle)
+    
+    def _copy_to_clipboard(self, text: str) -> None:
+        """Copy text to system clipboard"""
+        system = platform.system()
+        
+        if system == "Darwin":  # macOS
+            process = subprocess.Popen(['pbcopy'], stdin=subprocess.PIPE)
+            process.communicate(text.encode('utf-8'))
+        elif system == "Linux":
+            # Try xclip first, then xsel
+            try:
+                process = subprocess.Popen(['xclip', '-selection', 'clipboard'], stdin=subprocess.PIPE)
+                process.communicate(text.encode('utf-8'))
+            except FileNotFoundError:
+                try:
+                    process = subprocess.Popen(['xsel', '--clipboard', '--input'], stdin=subprocess.PIPE)
+                    process.communicate(text.encode('utf-8'))
+                except FileNotFoundError:
+                    raise Exception("Neither xclip nor xsel found. Please install one.")
+        elif system == "Windows":
+            process = subprocess.Popen(['clip'], stdin=subprocess.PIPE, shell=True)
+            process.communicate(text.encode('utf-8'))
+        else:
+            raise Exception(f"Unsupported platform: {system}")
+
    def action_quit(self) -> None:
        """Quit the application with proper cleanup"""
        self.cleanup()
@@ -415,4 +631,68 @@ class StreamLensAppV2(App):
    
    def on_unmount(self) -> None:
        """Called when app is being unmounted - ensure cleanup"""
-        self.cleanup()
+        self.cleanup()
+
+    
+    # Debugging methods
+    def start_debugging(self, web_interface: bool = True, port: int = 8080):
+        """Start debugging tools"""
+        if not DEBUGGING_AVAILABLE:
+            print("❌ Debugging tools not available. Run: pip install watchdog")
+            return
+            
+        self._debug_monitor = TextualStateMonitor(self)
+        self._debug_monitor.start_monitoring()
+        
+        if web_interface:
+            self._debug_server = TextualStateWebServer(self._debug_monitor, port)
+            self._debug_server.start()
+            
+        print(f"🔍 Debug monitoring started!")
+        if web_interface:
+            print(f"🌐 Web interface: http://localhost:{port}")
+            
+    def stop_debugging(self):
+        """Stop debugging tools"""
+        if hasattr(self, '_debug_monitor') and self._debug_monitor:
+            self._debug_monitor.stop_monitoring()
+        if hasattr(self, '_debug_server') and self._debug_server:
+            self._debug_server.stop()
+            
+    def debug_widget_tree(self):
+        """Print current widget tree to console"""
+        if not DEBUGGING_AVAILABLE:
+            print("❌ Debugging tools not available")
+            return
+            
+        data = inspect_textual_app(self)
+        print("🔍 TEXTUAL APP INSPECTION")
+        print("=" * 50)
+        print_widget_tree(data.get('current_screen', {}))
+        
+    def debug_focused_widget(self):
+        """Print info about currently focused widget"""
+        focused = self.focused
+        if focused:
+            print(f"🎯 Focused widget: {focused.__class__.__name__}")
+            if hasattr(focused, 'id'):
+                print(f"   ID: {focused.id}")
+            if hasattr(focused, 'classes'):
+                print(f"   Classes: {list(focused.classes)}")
+            if hasattr(focused, 'label'):
+                print(f"   Label: {focused.label}")
+        else:
+            print("🎯 No widget has focus")
+            
+    # Debugging key bindings
+    def action_debug_tree(self):
+        """Debug action: Print widget tree"""
+        self.debug_widget_tree()
+        
+    def action_debug_focus(self):
+        """Debug action: Print focused widget"""
+        self.debug_focused_widget()
+        
+    def action_start_web_debug(self):
+        """Debug action: Start web debugging interface"""
+        self.start_debugging()