Merge pull request 'fixed refactored code and validated inputs' (#2) from validate-inputs into main

Fixed the refactored version, removed app.py, added error feedback on bad input files.

Reviewed-on: godber/embedding-buddy#2

.claude/settings.local.json

@@ -6,6 +6,7 @@
       "Bash(uv add:*)"
     ],
     "deny": [],
-    "ask": []
+    "ask": [],
+    "defaultMode": "acceptEdits"
   }
 }

README.md

@@ -90,7 +90,7 @@ uv run python main.py
 
 The application follows a modular architecture for improved maintainability and testability:
 
-```
+```text
 src/embeddingbuddy/
 ├── config/          # Configuration management
 │   └── settings.py  # Centralized app settings
@@ -115,8 +115,8 @@ src/embeddingbuddy/
 Run the test suite to verify functionality:
 
 ```bash
-# Install pytest
-uv add pytest
+# Install test dependencies
+uv sync --extra test
 
 # Run all tests
 uv run pytest tests/ -v
@@ -128,6 +128,31 @@ uv run pytest tests/test_data_processing.py -v
 uv run pytest tests/ --cov=src/embeddingbuddy
 ```
 
+### Development Tools
+
+Install development dependencies for linting, type checking, and security:
+
+```bash
+# Install all dev dependencies
+uv sync --extra dev
+
+# Or install specific groups
+uv sync --extra test        # Testing tools
+uv sync --extra lint        # Linting and formatting
+uv sync --extra security    # Security scanning tools
+
+# Run linting
+uv run ruff check src/ tests/
+uv run ruff format src/ tests/
+
+# Run type checking
+uv run mypy src/embeddingbuddy/
+
+# Run security scans
+uv run bandit -r src/
+uv run safety check
+```
+
 ### Adding New Features
 
 The modular architecture makes it easy to extend functionality:

app.py

@@ -1,515 +0,0 @@
-import json
-import uuid
-from io import StringIO
-import base64
-
-import dash
-from dash import dcc, html, Input, Output, State, callback
-import dash_bootstrap_components as dbc
-import plotly.express as px
-import plotly.graph_objects as go
-import pandas as pd
-import numpy as np
-from sklearn.decomposition import PCA
-import umap
-from openTSNE import TSNE
-
-
-app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
-
-def parse_ndjson(contents):
-    """Parse NDJSON content and return list of documents."""
-    content_type, content_string = contents.split(',')
-    decoded = base64.b64decode(content_string)
-    text_content = decoded.decode('utf-8')
-    
-    documents = []
-    for line in text_content.strip().split('\n'):
-        if line.strip():
-            doc = json.loads(line)
-            if 'id' not in doc:
-                doc['id'] = str(uuid.uuid4())
-            documents.append(doc)
-    return documents
-
-def apply_dimensionality_reduction(embeddings, method='pca', n_components=3):
-    """Apply dimensionality reduction to embeddings."""
-    if method == 'pca':
-        reducer = PCA(n_components=n_components)
-        reduced = reducer.fit_transform(embeddings)
-        variance_explained = reducer.explained_variance_ratio_
-        return reduced, variance_explained
-    elif method == 'tsne':
-        reducer = TSNE(n_components=n_components, random_state=42)
-        reduced = reducer.fit(embeddings)
-        return reduced, None
-    elif method == 'umap':
-        reducer = umap.UMAP(n_components=n_components, random_state=42)
-        reduced = reducer.fit_transform(embeddings)
-        return reduced, None
-    else:
-        raise ValueError(f"Unknown method: {method}")
-
-def create_color_mapping(documents, color_by):
-    """Create color mapping for documents based on specified field."""
-    if color_by == 'category':
-        values = [doc.get('category', 'Unknown') for doc in documents]
-    elif color_by == 'subcategory':
-        values = [doc.get('subcategory', 'Unknown') for doc in documents]
-    elif color_by == 'tags':
-        values = [', '.join(doc.get('tags', [])) if doc.get('tags') else 'No tags' for doc in documents]
-    else:
-        values = ['All'] * len(documents)
-    
-    return values
-
-def create_plot(df, dimensions='3d', color_by='category', method='PCA'):
-    """Create plotly scatter plot."""
-    color_values = create_color_mapping(df.to_dict('records'), color_by)
-    
-    # Truncate text for hover display
-    df_display = df.copy()
-    df_display['text_preview'] = df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x)
-    
-    # Include all metadata fields in hover
-    hover_fields = ['id', 'text_preview', 'category', 'subcategory']
-    # Add tags as a string for hover
-    df_display['tags_str'] = df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None')
-    hover_fields.append('tags_str')
-    
-    if dimensions == '3d':
-        fig = px.scatter_3d(
-            df_display, x='dim_1', y='dim_2', z='dim_3',
-            color=color_values,
-            hover_data=hover_fields,
-            title=f'3D Embedding Visualization - {method} (colored by {color_by})'
-        )
-        fig.update_traces(marker=dict(size=5))
-    else:
-        fig = px.scatter(
-            df_display, x='dim_1', y='dim_2',
-            color=color_values,
-            hover_data=hover_fields,
-            title=f'2D Embedding Visualization - {method} (colored by {color_by})'
-        )
-        fig.update_traces(marker=dict(size=8))
-    
-    fig.update_layout(
-        height=None,  # Let CSS height control this
-        autosize=True,
-        margin=dict(l=0, r=0, t=50, b=0)
-    )
-    return fig
-
-def create_dual_plot(doc_df, prompt_df, dimensions='3d', color_by='category', method='PCA', show_prompts=None):
-    """Create plotly scatter plot with separate traces for documents and prompts."""
-    
-    # Create the base figure
-    fig = go.Figure()
-    
-    # Helper function to convert colors to grayscale
-    def to_grayscale_hex(color_str):
-        """Convert a color to grayscale while maintaining some distinction."""
-        import plotly.colors as pc
-        # Try to get RGB values from the color
-        try:
-            if color_str.startswith('#'):
-                # Hex color
-                rgb = tuple(int(color_str[i:i+2], 16) for i in (1, 3, 5))
-            else:
-                # Named color or other format - convert through plotly
-                rgb = pc.hex_to_rgb(pc.convert_colors_to_same_type([color_str], colortype='hex')[0][0])
-            
-            # Convert to grayscale using luminance formula, but keep some color
-            gray_value = int(0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2])
-            # Make it a bit more gray but not completely
-            gray_rgb = (gray_value * 0.7 + rgb[0] * 0.3, 
-                       gray_value * 0.7 + rgb[1] * 0.3, 
-                       gray_value * 0.7 + rgb[2] * 0.3)
-            return f'rgb({int(gray_rgb[0])},{int(gray_rgb[1])},{int(gray_rgb[2])})'
-        except:
-            return 'rgb(128,128,128)'  # fallback gray
-    
-    # Create document plot using plotly express for consistent colors
-    doc_color_values = create_color_mapping(doc_df.to_dict('records'), color_by)
-    doc_df_display = doc_df.copy()
-    doc_df_display['text_preview'] = doc_df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x)
-    doc_df_display['tags_str'] = doc_df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None')
-    
-    hover_fields = ['id', 'text_preview', 'category', 'subcategory', 'tags_str']
-    
-    # Create documents plot to get the color mapping
-    if dimensions == '3d':
-        doc_fig = px.scatter_3d(
-            doc_df_display, x='dim_1', y='dim_2', z='dim_3',
-            color=doc_color_values,
-            hover_data=hover_fields
-        )
-    else:
-        doc_fig = px.scatter(
-            doc_df_display, x='dim_1', y='dim_2',
-            color=doc_color_values,
-            hover_data=hover_fields
-        )
-    
-    # Add document traces to main figure
-    for trace in doc_fig.data:
-        trace.name = f'Documents - {trace.name}'
-        if dimensions == '3d':
-            trace.marker.size = 5
-            trace.marker.symbol = 'circle'
-        else:
-            trace.marker.size = 8
-            trace.marker.symbol = 'circle'
-        trace.marker.opacity = 1.0
-        fig.add_trace(trace)
-    
-    # Add prompt traces if they exist
-    if prompt_df is not None and show_prompts and 'show' in show_prompts:
-        prompt_color_values = create_color_mapping(prompt_df.to_dict('records'), color_by)
-        prompt_df_display = prompt_df.copy()
-        prompt_df_display['text_preview'] = prompt_df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x)
-        prompt_df_display['tags_str'] = prompt_df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None')
-        
-        # Create prompts plot to get consistent color grouping
-        if dimensions == '3d':
-            prompt_fig = px.scatter_3d(
-                prompt_df_display, x='dim_1', y='dim_2', z='dim_3',
-                color=prompt_color_values,
-                hover_data=hover_fields
-            )
-        else:
-            prompt_fig = px.scatter(
-                prompt_df_display, x='dim_1', y='dim_2',
-                color=prompt_color_values,
-                hover_data=hover_fields
-            )
-        
-        # Add prompt traces with grayed colors
-        for trace in prompt_fig.data:
-            # Convert the color to grayscale
-            original_color = trace.marker.color
-            if hasattr(trace.marker, 'color') and isinstance(trace.marker.color, str):
-                trace.marker.color = to_grayscale_hex(trace.marker.color)
-            
-            trace.name = f'Prompts - {trace.name}'
-            if dimensions == '3d':
-                trace.marker.size = 6
-                trace.marker.symbol = 'diamond'
-            else:
-                trace.marker.size = 10
-                trace.marker.symbol = 'diamond'
-            trace.marker.opacity = 0.8
-            fig.add_trace(trace)
-    
-    title = f'{dimensions.upper()} Embedding Visualization - {method} (colored by {color_by})'
-    fig.update_layout(
-        title=title,
-        height=None,
-        autosize=True,
-        margin=dict(l=0, r=0, t=50, b=0)
-    )
-    
-    return fig
-
-# Layout
-app.layout = dbc.Container([
-    dbc.Row([
-        dbc.Col([
-            html.H1("EmbeddingBuddy", className="text-center mb-4"),
-        ], width=12)
-    ]),
-    
-    dbc.Row([
-        # Left sidebar with controls
-        dbc.Col([
-            html.H5("Upload Data", className="mb-3"),
-            dcc.Upload(
-                id='upload-data',
-                children=html.Div([
-                    'Drag and Drop or ',
-                    html.A('Select Files')
-                ]),
-                style={
-                    'width': '100%',
-                    'height': '60px',
-                    'lineHeight': '60px',
-                    'borderWidth': '1px',
-                    'borderStyle': 'dashed',
-                    'borderRadius': '5px',
-                    'textAlign': 'center',
-                    'margin-bottom': '20px'
-                },
-                multiple=False
-            ),
-            
-            dcc.Upload(
-                id='upload-prompts',
-                children=html.Div([
-                    'Drag and Drop Prompts or ',
-                    html.A('Select Files')
-                ]),
-                style={
-                    'width': '100%',
-                    'height': '60px',
-                    'lineHeight': '60px',
-                    'borderWidth': '1px',
-                    'borderStyle': 'dashed',
-                    'borderRadius': '5px',
-                    'textAlign': 'center',
-                    'margin-bottom': '20px',
-                    'borderColor': '#28a745'
-                },
-                multiple=False
-            ),
-            
-            dbc.Button(
-                "Reset All Data",
-                id='reset-button',
-                color='danger',
-                outline=True,
-                size='sm',
-                className='mb-3',
-                style={'width': '100%'}
-            ),
-            
-            html.H5("Visualization Controls", className="mb-3"),
-            
-            dbc.Label("Method:"),
-            dcc.Dropdown(
-                id='method-dropdown',
-                options=[
-                    {'label': 'PCA', 'value': 'pca'},
-                    {'label': 't-SNE', 'value': 'tsne'},
-                    {'label': 'UMAP', 'value': 'umap'}
-                ],
-                value='pca',
-                style={'margin-bottom': '15px'}
-            ),
-            
-            dbc.Label("Color by:"),
-            dcc.Dropdown(
-                id='color-dropdown',
-                options=[
-                    {'label': 'Category', 'value': 'category'},
-                    {'label': 'Subcategory', 'value': 'subcategory'},
-                    {'label': 'Tags', 'value': 'tags'}
-                ],
-                value='category',
-                style={'margin-bottom': '15px'}
-            ),
-            
-            dbc.Label("Dimensions:"),
-            dcc.RadioItems(
-                id='dimension-toggle',
-                options=[
-                    {'label': '2D', 'value': '2d'},
-                    {'label': '3D', 'value': '3d'}
-                ],
-                value='3d',
-                style={'margin-bottom': '20px'}
-            ),
-            
-            dbc.Label("Show Prompts:"),
-            dcc.Checklist(
-                id='show-prompts-toggle',
-                options=[{'label': 'Show prompts on plot', 'value': 'show'}],
-                value=['show'],
-                style={'margin-bottom': '20px'}
-            ),
-            
-            html.H5("Point Details", className="mb-3"),
-            html.Div(id='point-details', children="Click on a point to see details")
-            
-        ], width=3, style={'padding-right': '20px'}),
-        
-        # Main visualization area
-        dbc.Col([
-            dcc.Graph(
-                id='embedding-plot',
-                style={'height': '85vh', 'width': '100%'},
-                config={'responsive': True, 'displayModeBar': True}
-            )
-        ], width=9)
-    ]),
-    
-    dcc.Store(id='processed-data'),
-    dcc.Store(id='processed-prompts')
-], fluid=True)
-
-@callback(
-    Output('processed-data', 'data'),
-    Input('upload-data', 'contents'),
-    State('upload-data', 'filename')
-)
-def process_uploaded_file(contents, filename):
-    if contents is None:
-        return None
-    
-    try:
-        documents = parse_ndjson(contents)
-        embeddings = np.array([doc['embedding'] for doc in documents])
-        
-        # Store original embeddings and documents
-        return {
-            'documents': documents,
-            'embeddings': embeddings.tolist()
-        }
-    except Exception as e:
-        return {'error': str(e)}
-
-@callback(
-    Output('processed-prompts', 'data'),
-    Input('upload-prompts', 'contents'),
-    State('upload-prompts', 'filename')
-)
-def process_uploaded_prompts(contents, filename):
-    if contents is None:
-        return None
-    
-    try:
-        prompts = parse_ndjson(contents)
-        embeddings = np.array([prompt['embedding'] for prompt in prompts])
-        
-        # Store original embeddings and prompts
-        return {
-            'prompts': prompts,
-            'embeddings': embeddings.tolist()
-        }
-    except Exception as e:
-        return {'error': str(e)}
-
-@callback(
-    Output('embedding-plot', 'figure'),
-    [Input('processed-data', 'data'),
-     Input('processed-prompts', 'data'),
-     Input('method-dropdown', 'value'),
-     Input('color-dropdown', 'value'),
-     Input('dimension-toggle', 'value'),
-     Input('show-prompts-toggle', 'value')]
-)
-def update_plot(data, prompts_data, method, color_by, dimensions, show_prompts):
-    if not data or 'error' in data:
-        return go.Figure().add_annotation(
-            text="Upload a valid NDJSON file to see visualization",
-            xref="paper", yref="paper",
-            x=0.5, y=0.5, xanchor='center', yanchor='middle',
-            showarrow=False, font=dict(size=16)
-        )
-    
-    # Prepare embeddings for dimensionality reduction
-    doc_embeddings = np.array(data['embeddings'])
-    all_embeddings = doc_embeddings
-    has_prompts = prompts_data and 'error' not in prompts_data and prompts_data.get('prompts')
-    
-    if has_prompts:
-        prompt_embeddings = np.array(prompts_data['embeddings'])
-        all_embeddings = np.vstack([doc_embeddings, prompt_embeddings])
-    
-    n_components = 3 if dimensions == '3d' else 2
-    
-    # Apply dimensionality reduction to combined data
-    reduced, variance_explained = apply_dimensionality_reduction(
-        all_embeddings, method=method, n_components=n_components
-    )
-    
-    # Split reduced embeddings back
-    doc_reduced = reduced[:len(doc_embeddings)]
-    prompt_reduced = reduced[len(doc_embeddings):] if has_prompts else None
-    
-    # Create dataframes
-    doc_df_data = []
-    for i, doc in enumerate(data['documents']):
-        row = {
-            'id': doc['id'],
-            'text': doc['text'],
-            'category': doc.get('category', 'Unknown'),
-            'subcategory': doc.get('subcategory', 'Unknown'),
-            'tags': doc.get('tags', []),
-            'dim_1': doc_reduced[i, 0],
-            'dim_2': doc_reduced[i, 1],
-            'type': 'document'
-        }
-        if dimensions == '3d':
-            row['dim_3'] = doc_reduced[i, 2]
-        doc_df_data.append(row)
-    
-    doc_df = pd.DataFrame(doc_df_data)
-    
-    prompt_df = None
-    if has_prompts and prompt_reduced is not None:
-        prompt_df_data = []
-        for i, prompt in enumerate(prompts_data['prompts']):
-            row = {
-                'id': prompt['id'],
-                'text': prompt['text'],
-                'category': prompt.get('category', 'Unknown'),
-                'subcategory': prompt.get('subcategory', 'Unknown'),
-                'tags': prompt.get('tags', []),
-                'dim_1': prompt_reduced[i, 0],
-                'dim_2': prompt_reduced[i, 1],
-                'type': 'prompt'
-            }
-            if dimensions == '3d':
-                row['dim_3'] = prompt_reduced[i, 2]
-            prompt_df_data.append(row)
-        
-        prompt_df = pd.DataFrame(prompt_df_data)
-    
-    return create_dual_plot(doc_df, prompt_df, dimensions, color_by, method.upper(), show_prompts)
-
-@callback(
-    Output('point-details', 'children'),
-    Input('embedding-plot', 'clickData'),
-    [State('processed-data', 'data'),
-     State('processed-prompts', 'data')]
-)
-def display_click_data(clickData, data, prompts_data):
-    if not clickData or not data:
-        return "Click on a point to see details"
-    
-    # Get point info from click
-    point_data = clickData['points'][0]
-    trace_name = point_data.get('fullData', {}).get('name', 'Documents')
-    
-    if 'pointIndex' in point_data:
-        point_index = point_data['pointIndex']
-    elif 'pointNumber' in point_data:
-        point_index = point_data['pointNumber']
-    else:
-        return "Could not identify clicked point"
-    
-    # Determine which dataset this point belongs to
-    if trace_name == 'Prompts' and prompts_data and 'prompts' in prompts_data:
-        item = prompts_data['prompts'][point_index]
-        item_type = 'Prompt'
-    else:
-        item = data['documents'][point_index]
-        item_type = 'Document'
-    
-    return dbc.Card([
-        dbc.CardBody([
-            html.H5(f"{item_type}: {item['id']}", className="card-title"),
-            html.P(f"Text: {item['text']}", className="card-text"),
-            html.P(f"Category: {item.get('category', 'Unknown')}", className="card-text"),
-            html.P(f"Subcategory: {item.get('subcategory', 'Unknown')}", className="card-text"),
-            html.P(f"Tags: {', '.join(item.get('tags', [])) if item.get('tags') else 'None'}", className="card-text"),
-            html.P(f"Type: {item_type}", className="card-text text-muted")
-        ])
-    ])
-
-@callback(
-    [Output('processed-data', 'data', allow_duplicate=True),
-     Output('processed-prompts', 'data', allow_duplicate=True),
-     Output('point-details', 'children', allow_duplicate=True)],
-    Input('reset-button', 'n_clicks'),
-    prevent_initial_call=True
-)
-def reset_data(n_clicks):
-    if n_clicks is None or n_clicks == 0:
-        return dash.no_update, dash.no_update, dash.no_update
-    
-    return None, None, "Click on a point to see details"
-
-if __name__ == '__main__':
-    app.run(debug=True)

example/bad/bad_binary_content.ndjson

@@ -0,0 +1,2 @@
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "Binary junk at start"}
+{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Normal line"}<7D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

example/bad/bad_empty_lines.ndjson

@@ -0,0 +1,6 @@
+{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "First line"}
+
+{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "After empty line"}
+
+
+{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1], "text": "After multiple empty lines"}

example/bad/bad_inconsistent_dimensions.ndjson

@@ -0,0 +1,4 @@
+{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "4D embedding"}
+{"id": "doc_002", "embedding": [0.5, 0.1, -0.2], "text": "3D embedding"}
+{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1, 0.8], "text": "5D embedding"}
+{"id": "doc_004", "embedding": [0.2, 0.1], "text": "2D embedding"}

example/bad/bad_invalid_embeddings.ndjson

@@ -0,0 +1,8 @@
+{"id": "doc_001", "embedding": "not_an_array", "text": "Embedding as string"}
+{"id": "doc_002", "embedding": [0.1, "text", 0.7, 0.2], "text": "Mixed types in embedding"}
+{"id": "doc_003", "embedding": [], "text": "Empty embedding array"}
+{"id": "doc_004", "embedding": [0.1], "text": "Single dimension embedding"}
+{"id": "doc_005", "embedding": null, "text": "Null embedding"}
+{"id": "doc_006", "embedding": [0.1, 0.2, null, 0.4], "text": "Null value in embedding"}
+{"id": "doc_007", "embedding": [0.1, 0.2, "NaN", 0.4], "text": "String NaN in embedding"}
+{"id": "doc_008", "embedding": [0.1, 0.2, Infinity, 0.4], "text": "Infinity in embedding"}

example/bad/bad_malformed_json.ndjson

@@ -0,0 +1,5 @@
+{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, "text": "Valid line"}
+{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Missing closing brace"
+{"id": "doc_003" "embedding": [0.3, 0.4, 0.1, -0.1], "text": "Missing colon after id"}
+{id: "doc_004", "embedding": [0.2, 0.1, 0.3, 0.4], "text": "Unquoted key"}
+{"id": "doc_005", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "Valid line again"}

example/bad/bad_missing_embedding.ndjson

@@ -0,0 +1,3 @@
+{"id": "doc_001", "text": "Sample text without embedding field", "category": "test"}
+{"id": "doc_002", "text": "Another text without embedding", "category": "test"}
+{"id": "doc_003", "text": "Third text missing embedding", "category": "test"}

example/bad/bad_missing_text.ndjson

@@ -0,0 +1,3 @@
+{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "category": "test"}
+{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "category": "test"}
+{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1], "category": "test"}

example/bad/bad_not_ndjson.json

@@ -0,0 +1,4 @@
+[
+  {"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "Regular JSON array"},
+  {"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Instead of NDJSON"}
+]

src/embeddingbuddy/data/parser.py

@@ -16,11 +16,22 @@ class NDJSONParser:
     @staticmethod
     def parse_text(text_content: str) -> List[Document]:
         documents = []
-        for line in text_content.strip().split("\n"):
+        for line_num, line in enumerate(text_content.strip().split("\n"), 1):
             if line.strip():
-                doc_dict = json.loads(line)
-                doc = NDJSONParser._dict_to_document(doc_dict)
-                documents.append(doc)
+                try:
+                    doc_dict = json.loads(line)
+                    doc = NDJSONParser._dict_to_document(doc_dict)
+                    documents.append(doc)
+                except json.JSONDecodeError as e:
+                    raise json.JSONDecodeError(
+                        f"Invalid JSON on line {line_num}: {e.msg}", e.doc, e.pos
+                    )
+                except KeyError as e:
+                    raise KeyError(f"Missing required field {e} on line {line_num}")
+                except (TypeError, ValueError) as e:
+                    raise ValueError(
+                        f"Invalid data format on line {line_num}: {str(e)}"
+                    )
         return documents
 
     @staticmethod
@@ -28,10 +39,33 @@ class NDJSONParser:
         if "id" not in doc_dict:
             doc_dict["id"] = str(uuid.uuid4())
 
+        # Validate required fields
+        if "text" not in doc_dict:
+            raise KeyError("'text'")
+        if "embedding" not in doc_dict:
+            raise KeyError("'embedding'")
+
+        # Validate embedding format
+        embedding = doc_dict["embedding"]
+        if not isinstance(embedding, list):
+            raise ValueError(
+                f"Embedding must be a list, got {type(embedding).__name__}"
+            )
+
+        if not embedding:
+            raise ValueError("Embedding cannot be empty")
+
+        # Check that all embedding values are numbers
+        for i, val in enumerate(embedding):
+            if not isinstance(val, (int, float)) or val != val:  # NaN check
+                raise ValueError(
+                    f"Embedding contains invalid value at index {i}: {val}"
+                )
+
         return Document(
             id=doc_dict["id"],
             text=doc_dict["text"],
-            embedding=doc_dict["embedding"],
+            embedding=embedding,
             category=doc_dict.get("category"),
             subcategory=doc_dict.get("subcategory"),
             tags=doc_dict.get("tags"),

src/embeddingbuddy/ui/callbacks/data_processing.py

@@ -9,30 +9,47 @@ class DataProcessingCallbacks:
 
     def _register_callbacks(self):
         @callback(
-            Output("processed-data", "data"),
+            [
+                Output("processed-data", "data", allow_duplicate=True),
+                Output("upload-error-alert", "children", allow_duplicate=True),
+                Output("upload-error-alert", "is_open", allow_duplicate=True),
+            ],
             Input("upload-data", "contents"),
             State("upload-data", "filename"),
+            prevent_initial_call=True,
         )
         def process_uploaded_file(contents, filename):
             if contents is None:
-                return None
+                return None, "", False
 
             processed_data = self.processor.process_upload(contents, filename)
 
             if processed_data.error:
-                return {"error": processed_data.error}
+                error_message = self._format_error_message(
+                    processed_data.error, filename
+                )
+                return (
+                    {"error": processed_data.error},
+                    error_message,
+                    True,  # Show error alert
+                )
 
-            return {
-                "documents": [
-                    self._document_to_dict(doc) for doc in processed_data.documents
-                ],
-                "embeddings": processed_data.embeddings.tolist(),
-            }
+            return (
+                {
+                    "documents": [
+                        self._document_to_dict(doc) for doc in processed_data.documents
+                    ],
+                    "embeddings": processed_data.embeddings.tolist(),
+                },
+                "",
+                False,  # Hide error alert
+            )
 
         @callback(
-            Output("processed-prompts", "data"),
+            Output("processed-prompts", "data", allow_duplicate=True),
             Input("upload-prompts", "contents"),
             State("upload-prompts", "filename"),
+            prevent_initial_call=True,
         )
         def process_uploaded_prompts(contents, filename):
             if contents is None:
@@ -60,3 +77,44 @@ class DataProcessingCallbacks:
             "subcategory": doc.subcategory,
             "tags": doc.tags,
         }
+
+    @staticmethod
+    def _format_error_message(error: str, filename: str | None = None) -> str:
+        """Format error message with helpful guidance for users."""
+        file_part = f" in file '{filename}'" if filename else ""
+
+        # Check for common error patterns and provide helpful messages
+        if "embedding" in error.lower() and (
+            "key" in error.lower() or "required field" in error.lower()
+        ):
+            return (
+                f"❌ Missing 'embedding' field{file_part}. "
+                "Each line must contain an 'embedding' field with a list of numbers."
+            )
+        elif "text" in error.lower() and (
+            "key" in error.lower() or "required field" in error.lower()
+        ):
+            return (
+                f"❌ Missing 'text' field{file_part}. "
+                "Each line must contain a 'text' field with the document content."
+            )
+        elif "json" in error.lower() and "decode" in error.lower():
+            return (
+                f"❌ Invalid JSON format{file_part}. "
+                "Please check that each line is valid JSON with proper syntax (quotes, braces, etc.)."
+            )
+        elif "unicode" in error.lower() or "decode" in error.lower():
+            return (
+                f"❌ File encoding issue{file_part}. "
+                "Please ensure the file is saved in UTF-8 format and contains no binary data."
+            )
+        elif "array" in error.lower() or "list" in error.lower():
+            return (
+                f"❌ Invalid embedding format{file_part}. "
+                "Embeddings must be arrays/lists of numbers, not strings or other types."
+            )
+        else:
+            return (
+                f"❌ Error processing file{file_part}: {error}. "
+                "Please check that your file is valid NDJSON with required 'text' and 'embedding' fields."
+            )

src/embeddingbuddy/ui/components/sidebar.py

@@ -11,14 +11,17 @@ class SidebarComponent:
         return dbc.Col(
             [
                 html.H5("Upload Data", className="mb-3"),
+                self.upload_component.create_error_alert(),
                 self.upload_component.create_data_upload(),
                 self.upload_component.create_prompts_upload(),
                 self.upload_component.create_reset_button(),
                 html.H5("Visualization Controls", className="mb-3"),
-                self._create_method_dropdown(),
-                self._create_color_dropdown(),
-                self._create_dimension_toggle(),
-                self._create_prompts_toggle(),
+            ]
+            + self._create_method_dropdown()
+            + self._create_color_dropdown()
+            + self._create_dimension_toggle()
+            + self._create_prompts_toggle()
+            + [
                 html.H5("Point Details", className="mb-3"),
                 html.Div(
                     id="point-details", children="Click on a point to see details"

src/embeddingbuddy/ui/components/upload.py

@@ -51,3 +51,14 @@ class UploadComponent:
             className="mb-3",
             style={"width": "100%"},
         )
+
+    @staticmethod
+    def create_error_alert():
+        """Create error alert component for data upload issues."""
+        return dbc.Alert(
+            id="upload-error-alert",
+            dismissable=True,
+            is_open=False,
+            color="danger",
+            className="mb-3",
+        )

src/embeddingbuddy/ui/layout.py

@@ -9,7 +9,8 @@ class AppLayout:
 
     def create_layout(self):
         return dbc.Container(
-            [self._create_header(), self._create_main_content(), self._create_stores()],
+            [self._create_header(), self._create_main_content()]
+            + self._create_stores(),
             fluid=True,
         )
 

tests/test_bad_data.py

@@ -0,0 +1,197 @@
+"""Tests for handling bad/invalid data files."""
+
+import pytest
+import json
+import base64
+from src.embeddingbuddy.data.parser import NDJSONParser
+from src.embeddingbuddy.data.processor import DataProcessor
+
+
+class TestBadDataHandling:
+    """Test suite for various types of invalid input data."""
+
+    def setup_method(self):
+        """Set up test fixtures."""
+        self.parser = NDJSONParser()
+        self.processor = DataProcessor()
+
+    def _create_upload_contents(self, text_content: str) -> str:
+        """Helper to create upload contents format."""
+        encoded = base64.b64encode(text_content.encode("utf-8")).decode("utf-8")
+        return f"data:application/json;base64,{encoded}"
+
+    def test_missing_embedding_field(self):
+        """Test files missing required embedding field."""
+        bad_content = '{"id": "doc_001", "text": "Sample text", "category": "test"}'
+
+        with pytest.raises(KeyError, match="embedding"):
+            self.parser.parse_text(bad_content)
+
+        # Test processor error handling
+        upload_contents = self._create_upload_contents(bad_content)
+        result = self.processor.process_upload(upload_contents)
+        assert result.error is not None
+        assert "embedding" in result.error
+
+    def test_missing_text_field(self):
+        """Test files missing required text field."""
+        bad_content = (
+            '{"id": "doc_001", "embedding": [0.1, 0.2, 0.3], "category": "test"}'
+        )
+
+        with pytest.raises(KeyError, match="text"):
+            self.parser.parse_text(bad_content)
+
+        # Test processor error handling
+        upload_contents = self._create_upload_contents(bad_content)
+        result = self.processor.process_upload(upload_contents)
+        assert result.error is not None
+        assert "text" in result.error
+
+    def test_malformed_json_lines(self):
+        """Test files with malformed JSON syntax."""
+        # Missing closing brace
+        bad_content = '{"id": "doc_001", "embedding": [0.1, 0.2], "text": "test"'
+
+        with pytest.raises(json.JSONDecodeError):
+            self.parser.parse_text(bad_content)
+
+        # Test processor error handling
+        upload_contents = self._create_upload_contents(bad_content)
+        result = self.processor.process_upload(upload_contents)
+        assert result.error is not None
+
+    def test_invalid_embedding_types(self):
+        """Test files with invalid embedding data types."""
+        test_cases = [
+            # String instead of array
+            '{"id": "doc_001", "embedding": "not_an_array", "text": "test"}',
+            # Mixed types in array
+            '{"id": "doc_002", "embedding": [0.1, "text", 0.3], "text": "test"}',
+            # Empty array
+            '{"id": "doc_003", "embedding": [], "text": "test"}',
+            # Null embedding
+            '{"id": "doc_004", "embedding": null, "text": "test"}',
+        ]
+
+        for bad_content in test_cases:
+            upload_contents = self._create_upload_contents(bad_content)
+            result = self.processor.process_upload(upload_contents)
+            assert result.error is not None, f"Should fail for: {bad_content}"
+
+    def test_inconsistent_embedding_dimensions(self):
+        """Test files with embeddings of different dimensions."""
+        bad_content = """{"id": "doc_001", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "4D embedding"}
+{"id": "doc_002", "embedding": [0.1, 0.2, 0.3], "text": "3D embedding"}"""
+
+        upload_contents = self._create_upload_contents(bad_content)
+        result = self.processor.process_upload(upload_contents)
+
+        # This might succeed parsing but fail in processing
+        # The error depends on where dimension validation occurs
+        if result.error is None:
+            # If parsing succeeds, check that embeddings have inconsistent shapes
+            assert len(result.documents) == 2
+            assert len(result.documents[0].embedding) != len(
+                result.documents[1].embedding
+            )
+
+    def test_empty_lines_in_ndjson(self):
+        """Test files with empty lines mixed in."""
+        content_with_empty_lines = """{"id": "doc_001", "embedding": [0.1, 0.2], "text": "First line"}
+
+{"id": "doc_002", "embedding": [0.3, 0.4], "text": "After empty line"}"""
+
+        # This should work - empty lines should be skipped
+        documents = self.parser.parse_text(content_with_empty_lines)
+        assert len(documents) == 2
+        assert documents[0].id == "doc_001"
+        assert documents[1].id == "doc_002"
+
+    def test_not_ndjson_format(self):
+        """Test regular JSON array instead of NDJSON."""
+        json_array = """[
+  {"id": "doc_001", "embedding": [0.1, 0.2], "text": "First"},
+  {"id": "doc_002", "embedding": [0.3, 0.4], "text": "Second"}
+]"""
+
+        with pytest.raises(json.JSONDecodeError):
+            self.parser.parse_text(json_array)
+
+    def test_binary_content_in_file(self):
+        """Test files with binary content mixed in."""
+        # Simulate binary content that can't be decoded
+        binary_content = (
+            b'\x00\x01\x02{"id": "doc_001", "embedding": [0.1], "text": "test"}'
+        )
+
+        # This should result in an error when processing
+        encoded = base64.b64encode(binary_content).decode("utf-8")
+        upload_contents = f"data:application/json;base64,{encoded}"
+        result = self.processor.process_upload(upload_contents)
+
+        # Should either fail with UnicodeDecodeError or JSON parsing error
+        assert result.error is not None
+
+    def test_extremely_large_embeddings(self):
+        """Test embeddings with very large dimensions."""
+        large_embedding = [0.1] * 10000  # 10k dimensions
+        content = json.dumps(
+            {
+                "id": "doc_001",
+                "embedding": large_embedding,
+                "text": "Large embedding test",
+            }
+        )
+
+        # This should work but might be slow
+        upload_contents = self._create_upload_contents(content)
+        result = self.processor.process_upload(upload_contents)
+
+        if result.error is None:
+            assert len(result.documents) == 1
+            assert len(result.documents[0].embedding) == 10000
+
+    def test_special_characters_in_text(self):
+        """Test handling of special characters and unicode."""
+        special_content = json.dumps(
+            {
+                "id": "doc_001",
+                "embedding": [0.1, 0.2],
+                "text": 'Special chars: 🚀 ñoñó 中文 \n\t"',
+            },
+            ensure_ascii=False,
+        )
+
+        upload_contents = self._create_upload_contents(special_content)
+        result = self.processor.process_upload(upload_contents)
+
+        assert result.error is None
+        assert len(result.documents) == 1
+        assert "🚀" in result.documents[0].text
+
+    def test_processor_error_structure(self):
+        """Test that processor returns proper error structure."""
+        bad_content = '{"invalid": "json"'  # Missing closing brace
+        upload_contents = self._create_upload_contents(bad_content)
+
+        result = self.processor.process_upload(upload_contents)
+
+        # Check error structure
+        assert result.error is not None
+        assert isinstance(result.error, str)
+        assert len(result.documents) == 0
+        assert result.embeddings.size == 0
+
+    def test_multiple_errors_in_file(self):
+        """Test file with multiple different types of errors."""
+        multi_error_content = """{"id": "doc_001", "text": "Missing embedding"}
+{"id": "doc_002", "embedding": "wrong_type", "text": "Wrong embedding type"}
+{"id": "doc_003", "embedding": [0.1, 0.2], "text": "Valid line"}
+{"id": "doc_004", "embedding": [0.3, 0.4]"""  # Missing text and closing brace
+
+        upload_contents = self._create_upload_contents(multi_error_content)
+        result = self.processor.process_upload(upload_contents)
+
+        # Should fail on first error encountered
+        assert result.error is not None