From 7b81c20a26ac45fca3fa935e43a41e3123b5e8da Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Thu, 14 Aug 2025 07:55:40 -0700 Subject: [PATCH 1/3] fixed refactored code --- README.md | 29 +- app.py | 515 ------------------ example/bad/bad_binary_content.ndjson | 2 + example/bad/bad_empty_lines.ndjson | 6 + .../bad/bad_inconsistent_dimensions.ndjson | 4 + example/bad/bad_invalid_embeddings.ndjson | 8 + example/bad/bad_malformed_json.ndjson | 5 + example/bad/bad_missing_embedding.ndjson | 3 + example/bad/bad_missing_text.ndjson | 3 + example/bad/bad_not_ndjson.json | 4 + .../sample_data.ndjson | 0 .../sample_prompts.ndjson | 0 src/embeddingbuddy/data/parser.py | 44 +- .../ui/callbacks/data_processing.py | 79 ++- src/embeddingbuddy/ui/components/sidebar.py | 12 +- src/embeddingbuddy/ui/components/upload.py | 22 + src/embeddingbuddy/ui/layout.py | 2 +- tests/test_bad_data.py | 185 +++++++ 18 files changed, 386 insertions(+), 537 deletions(-) delete mode 100644 app.py create mode 100644 example/bad/bad_binary_content.ndjson create mode 100644 example/bad/bad_empty_lines.ndjson create mode 100644 example/bad/bad_inconsistent_dimensions.ndjson create mode 100644 example/bad/bad_invalid_embeddings.ndjson create mode 100644 example/bad/bad_malformed_json.ndjson create mode 100644 example/bad/bad_missing_embedding.ndjson create mode 100644 example/bad/bad_missing_text.ndjson create mode 100644 example/bad/bad_not_ndjson.json rename sample_data.ndjson => example/sample_data.ndjson (100%) rename sample_prompts.ndjson => example/sample_prompts.ndjson (100%) create mode 100644 tests/test_bad_data.py diff --git a/README.md b/README.md index 75ee3ff..0bc814b 100644 --- a/README.md +++ b/README.md @@ -115,8 +115,8 @@ src/embeddingbuddy/ Run the test suite to verify functionality: ```bash -# Install pytest -uv add pytest +# Install test dependencies +uv sync --extra test # Run all tests uv run pytest tests/ -v @@ -128,6 +128,31 @@ uv run pytest tests/test_data_processing.py -v uv run pytest tests/ --cov=src/embeddingbuddy ``` +### Development Tools + +Install development dependencies for linting, type checking, and security: + +```bash +# Install all dev dependencies +uv sync --extra dev + +# Or install specific groups +uv sync --extra test # Testing tools +uv sync --extra lint # Linting and formatting +uv sync --extra security # Security scanning tools + +# Run linting +uv run ruff check src/ tests/ +uv run ruff format src/ tests/ + +# Run type checking +uv run mypy src/embeddingbuddy/ + +# Run security scans +uv run bandit -r src/ +uv run safety check +``` + ### Adding New Features The modular architecture makes it easy to extend functionality: diff --git a/app.py b/app.py deleted file mode 100644 index e34874d..0000000 --- a/app.py +++ /dev/null @@ -1,515 +0,0 @@ -import json -import uuid -from io import StringIO -import base64 - -import dash -from dash import dcc, html, Input, Output, State, callback -import dash_bootstrap_components as dbc -import plotly.express as px -import plotly.graph_objects as go -import pandas as pd -import numpy as np -from sklearn.decomposition import PCA -import umap -from openTSNE import TSNE - - -app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) - -def parse_ndjson(contents): - """Parse NDJSON content and return list of documents.""" - content_type, content_string = contents.split(',') - decoded = base64.b64decode(content_string) - text_content = decoded.decode('utf-8') - - documents = [] - for line in text_content.strip().split('\n'): - if line.strip(): - doc = json.loads(line) - if 'id' not in doc: - doc['id'] = str(uuid.uuid4()) - documents.append(doc) - return documents - -def apply_dimensionality_reduction(embeddings, method='pca', n_components=3): - """Apply dimensionality reduction to embeddings.""" - if method == 'pca': - reducer = PCA(n_components=n_components) - reduced = reducer.fit_transform(embeddings) - variance_explained = reducer.explained_variance_ratio_ - return reduced, variance_explained - elif method == 'tsne': - reducer = TSNE(n_components=n_components, random_state=42) - reduced = reducer.fit(embeddings) - return reduced, None - elif method == 'umap': - reducer = umap.UMAP(n_components=n_components, random_state=42) - reduced = reducer.fit_transform(embeddings) - return reduced, None - else: - raise ValueError(f"Unknown method: {method}") - -def create_color_mapping(documents, color_by): - """Create color mapping for documents based on specified field.""" - if color_by == 'category': - values = [doc.get('category', 'Unknown') for doc in documents] - elif color_by == 'subcategory': - values = [doc.get('subcategory', 'Unknown') for doc in documents] - elif color_by == 'tags': - values = [', '.join(doc.get('tags', [])) if doc.get('tags') else 'No tags' for doc in documents] - else: - values = ['All'] * len(documents) - - return values - -def create_plot(df, dimensions='3d', color_by='category', method='PCA'): - """Create plotly scatter plot.""" - color_values = create_color_mapping(df.to_dict('records'), color_by) - - # Truncate text for hover display - df_display = df.copy() - df_display['text_preview'] = df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x) - - # Include all metadata fields in hover - hover_fields = ['id', 'text_preview', 'category', 'subcategory'] - # Add tags as a string for hover - df_display['tags_str'] = df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None') - hover_fields.append('tags_str') - - if dimensions == '3d': - fig = px.scatter_3d( - df_display, x='dim_1', y='dim_2', z='dim_3', - color=color_values, - hover_data=hover_fields, - title=f'3D Embedding Visualization - {method} (colored by {color_by})' - ) - fig.update_traces(marker=dict(size=5)) - else: - fig = px.scatter( - df_display, x='dim_1', y='dim_2', - color=color_values, - hover_data=hover_fields, - title=f'2D Embedding Visualization - {method} (colored by {color_by})' - ) - fig.update_traces(marker=dict(size=8)) - - fig.update_layout( - height=None, # Let CSS height control this - autosize=True, - margin=dict(l=0, r=0, t=50, b=0) - ) - return fig - -def create_dual_plot(doc_df, prompt_df, dimensions='3d', color_by='category', method='PCA', show_prompts=None): - """Create plotly scatter plot with separate traces for documents and prompts.""" - - # Create the base figure - fig = go.Figure() - - # Helper function to convert colors to grayscale - def to_grayscale_hex(color_str): - """Convert a color to grayscale while maintaining some distinction.""" - import plotly.colors as pc - # Try to get RGB values from the color - try: - if color_str.startswith('#'): - # Hex color - rgb = tuple(int(color_str[i:i+2], 16) for i in (1, 3, 5)) - else: - # Named color or other format - convert through plotly - rgb = pc.hex_to_rgb(pc.convert_colors_to_same_type([color_str], colortype='hex')[0][0]) - - # Convert to grayscale using luminance formula, but keep some color - gray_value = int(0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2]) - # Make it a bit more gray but not completely - gray_rgb = (gray_value * 0.7 + rgb[0] * 0.3, - gray_value * 0.7 + rgb[1] * 0.3, - gray_value * 0.7 + rgb[2] * 0.3) - return f'rgb({int(gray_rgb[0])},{int(gray_rgb[1])},{int(gray_rgb[2])})' - except: - return 'rgb(128,128,128)' # fallback gray - - # Create document plot using plotly express for consistent colors - doc_color_values = create_color_mapping(doc_df.to_dict('records'), color_by) - doc_df_display = doc_df.copy() - doc_df_display['text_preview'] = doc_df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x) - doc_df_display['tags_str'] = doc_df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None') - - hover_fields = ['id', 'text_preview', 'category', 'subcategory', 'tags_str'] - - # Create documents plot to get the color mapping - if dimensions == '3d': - doc_fig = px.scatter_3d( - doc_df_display, x='dim_1', y='dim_2', z='dim_3', - color=doc_color_values, - hover_data=hover_fields - ) - else: - doc_fig = px.scatter( - doc_df_display, x='dim_1', y='dim_2', - color=doc_color_values, - hover_data=hover_fields - ) - - # Add document traces to main figure - for trace in doc_fig.data: - trace.name = f'Documents - {trace.name}' - if dimensions == '3d': - trace.marker.size = 5 - trace.marker.symbol = 'circle' - else: - trace.marker.size = 8 - trace.marker.symbol = 'circle' - trace.marker.opacity = 1.0 - fig.add_trace(trace) - - # Add prompt traces if they exist - if prompt_df is not None and show_prompts and 'show' in show_prompts: - prompt_color_values = create_color_mapping(prompt_df.to_dict('records'), color_by) - prompt_df_display = prompt_df.copy() - prompt_df_display['text_preview'] = prompt_df_display['text'].apply(lambda x: x[:100] + "..." if len(x) > 100 else x) - prompt_df_display['tags_str'] = prompt_df_display['tags'].apply(lambda x: ', '.join(x) if x else 'None') - - # Create prompts plot to get consistent color grouping - if dimensions == '3d': - prompt_fig = px.scatter_3d( - prompt_df_display, x='dim_1', y='dim_2', z='dim_3', - color=prompt_color_values, - hover_data=hover_fields - ) - else: - prompt_fig = px.scatter( - prompt_df_display, x='dim_1', y='dim_2', - color=prompt_color_values, - hover_data=hover_fields - ) - - # Add prompt traces with grayed colors - for trace in prompt_fig.data: - # Convert the color to grayscale - original_color = trace.marker.color - if hasattr(trace.marker, 'color') and isinstance(trace.marker.color, str): - trace.marker.color = to_grayscale_hex(trace.marker.color) - - trace.name = f'Prompts - {trace.name}' - if dimensions == '3d': - trace.marker.size = 6 - trace.marker.symbol = 'diamond' - else: - trace.marker.size = 10 - trace.marker.symbol = 'diamond' - trace.marker.opacity = 0.8 - fig.add_trace(trace) - - title = f'{dimensions.upper()} Embedding Visualization - {method} (colored by {color_by})' - fig.update_layout( - title=title, - height=None, - autosize=True, - margin=dict(l=0, r=0, t=50, b=0) - ) - - return fig - -# Layout -app.layout = dbc.Container([ - dbc.Row([ - dbc.Col([ - html.H1("EmbeddingBuddy", className="text-center mb-4"), - ], width=12) - ]), - - dbc.Row([ - # Left sidebar with controls - dbc.Col([ - html.H5("Upload Data", className="mb-3"), - dcc.Upload( - id='upload-data', - children=html.Div([ - 'Drag and Drop or ', - html.A('Select Files') - ]), - style={ - 'width': '100%', - 'height': '60px', - 'lineHeight': '60px', - 'borderWidth': '1px', - 'borderStyle': 'dashed', - 'borderRadius': '5px', - 'textAlign': 'center', - 'margin-bottom': '20px' - }, - multiple=False - ), - - dcc.Upload( - id='upload-prompts', - children=html.Div([ - 'Drag and Drop Prompts or ', - html.A('Select Files') - ]), - style={ - 'width': '100%', - 'height': '60px', - 'lineHeight': '60px', - 'borderWidth': '1px', - 'borderStyle': 'dashed', - 'borderRadius': '5px', - 'textAlign': 'center', - 'margin-bottom': '20px', - 'borderColor': '#28a745' - }, - multiple=False - ), - - dbc.Button( - "Reset All Data", - id='reset-button', - color='danger', - outline=True, - size='sm', - className='mb-3', - style={'width': '100%'} - ), - - html.H5("Visualization Controls", className="mb-3"), - - dbc.Label("Method:"), - dcc.Dropdown( - id='method-dropdown', - options=[ - {'label': 'PCA', 'value': 'pca'}, - {'label': 't-SNE', 'value': 'tsne'}, - {'label': 'UMAP', 'value': 'umap'} - ], - value='pca', - style={'margin-bottom': '15px'} - ), - - dbc.Label("Color by:"), - dcc.Dropdown( - id='color-dropdown', - options=[ - {'label': 'Category', 'value': 'category'}, - {'label': 'Subcategory', 'value': 'subcategory'}, - {'label': 'Tags', 'value': 'tags'} - ], - value='category', - style={'margin-bottom': '15px'} - ), - - dbc.Label("Dimensions:"), - dcc.RadioItems( - id='dimension-toggle', - options=[ - {'label': '2D', 'value': '2d'}, - {'label': '3D', 'value': '3d'} - ], - value='3d', - style={'margin-bottom': '20px'} - ), - - dbc.Label("Show Prompts:"), - dcc.Checklist( - id='show-prompts-toggle', - options=[{'label': 'Show prompts on plot', 'value': 'show'}], - value=['show'], - style={'margin-bottom': '20px'} - ), - - html.H5("Point Details", className="mb-3"), - html.Div(id='point-details', children="Click on a point to see details") - - ], width=3, style={'padding-right': '20px'}), - - # Main visualization area - dbc.Col([ - dcc.Graph( - id='embedding-plot', - style={'height': '85vh', 'width': '100%'}, - config={'responsive': True, 'displayModeBar': True} - ) - ], width=9) - ]), - - dcc.Store(id='processed-data'), - dcc.Store(id='processed-prompts') -], fluid=True) - -@callback( - Output('processed-data', 'data'), - Input('upload-data', 'contents'), - State('upload-data', 'filename') -) -def process_uploaded_file(contents, filename): - if contents is None: - return None - - try: - documents = parse_ndjson(contents) - embeddings = np.array([doc['embedding'] for doc in documents]) - - # Store original embeddings and documents - return { - 'documents': documents, - 'embeddings': embeddings.tolist() - } - except Exception as e: - return {'error': str(e)} - -@callback( - Output('processed-prompts', 'data'), - Input('upload-prompts', 'contents'), - State('upload-prompts', 'filename') -) -def process_uploaded_prompts(contents, filename): - if contents is None: - return None - - try: - prompts = parse_ndjson(contents) - embeddings = np.array([prompt['embedding'] for prompt in prompts]) - - # Store original embeddings and prompts - return { - 'prompts': prompts, - 'embeddings': embeddings.tolist() - } - except Exception as e: - return {'error': str(e)} - -@callback( - Output('embedding-plot', 'figure'), - [Input('processed-data', 'data'), - Input('processed-prompts', 'data'), - Input('method-dropdown', 'value'), - Input('color-dropdown', 'value'), - Input('dimension-toggle', 'value'), - Input('show-prompts-toggle', 'value')] -) -def update_plot(data, prompts_data, method, color_by, dimensions, show_prompts): - if not data or 'error' in data: - return go.Figure().add_annotation( - text="Upload a valid NDJSON file to see visualization", - xref="paper", yref="paper", - x=0.5, y=0.5, xanchor='center', yanchor='middle', - showarrow=False, font=dict(size=16) - ) - - # Prepare embeddings for dimensionality reduction - doc_embeddings = np.array(data['embeddings']) - all_embeddings = doc_embeddings - has_prompts = prompts_data and 'error' not in prompts_data and prompts_data.get('prompts') - - if has_prompts: - prompt_embeddings = np.array(prompts_data['embeddings']) - all_embeddings = np.vstack([doc_embeddings, prompt_embeddings]) - - n_components = 3 if dimensions == '3d' else 2 - - # Apply dimensionality reduction to combined data - reduced, variance_explained = apply_dimensionality_reduction( - all_embeddings, method=method, n_components=n_components - ) - - # Split reduced embeddings back - doc_reduced = reduced[:len(doc_embeddings)] - prompt_reduced = reduced[len(doc_embeddings):] if has_prompts else None - - # Create dataframes - doc_df_data = [] - for i, doc in enumerate(data['documents']): - row = { - 'id': doc['id'], - 'text': doc['text'], - 'category': doc.get('category', 'Unknown'), - 'subcategory': doc.get('subcategory', 'Unknown'), - 'tags': doc.get('tags', []), - 'dim_1': doc_reduced[i, 0], - 'dim_2': doc_reduced[i, 1], - 'type': 'document' - } - if dimensions == '3d': - row['dim_3'] = doc_reduced[i, 2] - doc_df_data.append(row) - - doc_df = pd.DataFrame(doc_df_data) - - prompt_df = None - if has_prompts and prompt_reduced is not None: - prompt_df_data = [] - for i, prompt in enumerate(prompts_data['prompts']): - row = { - 'id': prompt['id'], - 'text': prompt['text'], - 'category': prompt.get('category', 'Unknown'), - 'subcategory': prompt.get('subcategory', 'Unknown'), - 'tags': prompt.get('tags', []), - 'dim_1': prompt_reduced[i, 0], - 'dim_2': prompt_reduced[i, 1], - 'type': 'prompt' - } - if dimensions == '3d': - row['dim_3'] = prompt_reduced[i, 2] - prompt_df_data.append(row) - - prompt_df = pd.DataFrame(prompt_df_data) - - return create_dual_plot(doc_df, prompt_df, dimensions, color_by, method.upper(), show_prompts) - -@callback( - Output('point-details', 'children'), - Input('embedding-plot', 'clickData'), - [State('processed-data', 'data'), - State('processed-prompts', 'data')] -) -def display_click_data(clickData, data, prompts_data): - if not clickData or not data: - return "Click on a point to see details" - - # Get point info from click - point_data = clickData['points'][0] - trace_name = point_data.get('fullData', {}).get('name', 'Documents') - - if 'pointIndex' in point_data: - point_index = point_data['pointIndex'] - elif 'pointNumber' in point_data: - point_index = point_data['pointNumber'] - else: - return "Could not identify clicked point" - - # Determine which dataset this point belongs to - if trace_name == 'Prompts' and prompts_data and 'prompts' in prompts_data: - item = prompts_data['prompts'][point_index] - item_type = 'Prompt' - else: - item = data['documents'][point_index] - item_type = 'Document' - - return dbc.Card([ - dbc.CardBody([ - html.H5(f"{item_type}: {item['id']}", className="card-title"), - html.P(f"Text: {item['text']}", className="card-text"), - html.P(f"Category: {item.get('category', 'Unknown')}", className="card-text"), - html.P(f"Subcategory: {item.get('subcategory', 'Unknown')}", className="card-text"), - html.P(f"Tags: {', '.join(item.get('tags', [])) if item.get('tags') else 'None'}", className="card-text"), - html.P(f"Type: {item_type}", className="card-text text-muted") - ]) - ]) - -@callback( - [Output('processed-data', 'data', allow_duplicate=True), - Output('processed-prompts', 'data', allow_duplicate=True), - Output('point-details', 'children', allow_duplicate=True)], - Input('reset-button', 'n_clicks'), - prevent_initial_call=True -) -def reset_data(n_clicks): - if n_clicks is None or n_clicks == 0: - return dash.no_update, dash.no_update, dash.no_update - - return None, None, "Click on a point to see details" - -if __name__ == '__main__': - app.run(debug=True) \ No newline at end of file diff --git a/example/bad/bad_binary_content.ndjson b/example/bad/bad_binary_content.ndjson new file mode 100644 index 0000000..4f3ecbb --- /dev/null +++ b/example/bad/bad_binary_content.ndjson @@ -0,0 +1,2 @@ +��������{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "Binary junk at start"} +{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Normal line"}�������� \ No newline at end of file diff --git a/example/bad/bad_empty_lines.ndjson b/example/bad/bad_empty_lines.ndjson new file mode 100644 index 0000000..91b2076 --- /dev/null +++ b/example/bad/bad_empty_lines.ndjson @@ -0,0 +1,6 @@ +{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "First line"} + +{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "After empty line"} + + +{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1], "text": "After multiple empty lines"} \ No newline at end of file diff --git a/example/bad/bad_inconsistent_dimensions.ndjson b/example/bad/bad_inconsistent_dimensions.ndjson new file mode 100644 index 0000000..b70b9ee --- /dev/null +++ b/example/bad/bad_inconsistent_dimensions.ndjson @@ -0,0 +1,4 @@ +{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "4D embedding"} +{"id": "doc_002", "embedding": [0.5, 0.1, -0.2], "text": "3D embedding"} +{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1, 0.8], "text": "5D embedding"} +{"id": "doc_004", "embedding": [0.2, 0.1], "text": "2D embedding"} \ No newline at end of file diff --git a/example/bad/bad_invalid_embeddings.ndjson b/example/bad/bad_invalid_embeddings.ndjson new file mode 100644 index 0000000..a5ebb17 --- /dev/null +++ b/example/bad/bad_invalid_embeddings.ndjson @@ -0,0 +1,8 @@ +{"id": "doc_001", "embedding": "not_an_array", "text": "Embedding as string"} +{"id": "doc_002", "embedding": [0.1, "text", 0.7, 0.2], "text": "Mixed types in embedding"} +{"id": "doc_003", "embedding": [], "text": "Empty embedding array"} +{"id": "doc_004", "embedding": [0.1], "text": "Single dimension embedding"} +{"id": "doc_005", "embedding": null, "text": "Null embedding"} +{"id": "doc_006", "embedding": [0.1, 0.2, null, 0.4], "text": "Null value in embedding"} +{"id": "doc_007", "embedding": [0.1, 0.2, "NaN", 0.4], "text": "String NaN in embedding"} +{"id": "doc_008", "embedding": [0.1, 0.2, Infinity, 0.4], "text": "Infinity in embedding"} \ No newline at end of file diff --git a/example/bad/bad_malformed_json.ndjson b/example/bad/bad_malformed_json.ndjson new file mode 100644 index 0000000..6108f57 --- /dev/null +++ b/example/bad/bad_malformed_json.ndjson @@ -0,0 +1,5 @@ +{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, "text": "Valid line"} +{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Missing closing brace" +{"id": "doc_003" "embedding": [0.3, 0.4, 0.1, -0.1], "text": "Missing colon after id"} +{id: "doc_004", "embedding": [0.2, 0.1, 0.3, 0.4], "text": "Unquoted key"} +{"id": "doc_005", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "Valid line again"} \ No newline at end of file diff --git a/example/bad/bad_missing_embedding.ndjson b/example/bad/bad_missing_embedding.ndjson new file mode 100644 index 0000000..6123c6c --- /dev/null +++ b/example/bad/bad_missing_embedding.ndjson @@ -0,0 +1,3 @@ +{"id": "doc_001", "text": "Sample text without embedding field", "category": "test"} +{"id": "doc_002", "text": "Another text without embedding", "category": "test"} +{"id": "doc_003", "text": "Third text missing embedding", "category": "test"} \ No newline at end of file diff --git a/example/bad/bad_missing_text.ndjson b/example/bad/bad_missing_text.ndjson new file mode 100644 index 0000000..4f9a7ff --- /dev/null +++ b/example/bad/bad_missing_text.ndjson @@ -0,0 +1,3 @@ +{"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "category": "test"} +{"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "category": "test"} +{"id": "doc_003", "embedding": [0.3, 0.4, 0.1, -0.1], "category": "test"} \ No newline at end of file diff --git a/example/bad/bad_not_ndjson.json b/example/bad/bad_not_ndjson.json new file mode 100644 index 0000000..7544f2d --- /dev/null +++ b/example/bad/bad_not_ndjson.json @@ -0,0 +1,4 @@ +[ + {"id": "doc_001", "embedding": [0.1, -0.3, 0.7, 0.2], "text": "Regular JSON array"}, + {"id": "doc_002", "embedding": [0.5, 0.1, -0.2, 0.8], "text": "Instead of NDJSON"} +] \ No newline at end of file diff --git a/sample_data.ndjson b/example/sample_data.ndjson similarity index 100% rename from sample_data.ndjson rename to example/sample_data.ndjson diff --git a/sample_prompts.ndjson b/example/sample_prompts.ndjson similarity index 100% rename from sample_prompts.ndjson rename to example/sample_prompts.ndjson diff --git a/src/embeddingbuddy/data/parser.py b/src/embeddingbuddy/data/parser.py index ed76bb2..3b8cc42 100644 --- a/src/embeddingbuddy/data/parser.py +++ b/src/embeddingbuddy/data/parser.py @@ -16,11 +16,26 @@ class NDJSONParser: @staticmethod def parse_text(text_content: str) -> List[Document]: documents = [] - for line in text_content.strip().split("\n"): + for line_num, line in enumerate(text_content.strip().split("\n"), 1): if line.strip(): - doc_dict = json.loads(line) - doc = NDJSONParser._dict_to_document(doc_dict) - documents.append(doc) + try: + doc_dict = json.loads(line) + doc = NDJSONParser._dict_to_document(doc_dict) + documents.append(doc) + except json.JSONDecodeError as e: + raise json.JSONDecodeError( + f"Invalid JSON on line {line_num}: {e.msg}", + e.doc, + e.pos + ) + except KeyError as e: + raise KeyError( + f"Missing required field {e} on line {line_num}" + ) + except (TypeError, ValueError) as e: + raise ValueError( + f"Invalid data format on line {line_num}: {str(e)}" + ) return documents @staticmethod @@ -28,10 +43,29 @@ class NDJSONParser: if "id" not in doc_dict: doc_dict["id"] = str(uuid.uuid4()) + # Validate required fields + if "text" not in doc_dict: + raise KeyError("'text'") + if "embedding" not in doc_dict: + raise KeyError("'embedding'") + + # Validate embedding format + embedding = doc_dict["embedding"] + if not isinstance(embedding, list): + raise ValueError(f"Embedding must be a list, got {type(embedding).__name__}") + + if not embedding: + raise ValueError("Embedding cannot be empty") + + # Check that all embedding values are numbers + for i, val in enumerate(embedding): + if not isinstance(val, (int, float)) or val != val: # NaN check + raise ValueError(f"Embedding contains invalid value at index {i}: {val}") + return Document( id=doc_dict["id"], text=doc_dict["text"], - embedding=doc_dict["embedding"], + embedding=embedding, category=doc_dict.get("category"), subcategory=doc_dict.get("subcategory"), tags=doc_dict.get("tags"), diff --git a/src/embeddingbuddy/ui/callbacks/data_processing.py b/src/embeddingbuddy/ui/callbacks/data_processing.py index 2a2bf7a..eec3a82 100644 --- a/src/embeddingbuddy/ui/callbacks/data_processing.py +++ b/src/embeddingbuddy/ui/callbacks/data_processing.py @@ -9,30 +9,52 @@ class DataProcessingCallbacks: def _register_callbacks(self): @callback( - Output("processed-data", "data"), + [ + Output("processed-data", "data", allow_duplicate=True), + Output("upload-error-alert", "children", allow_duplicate=True), + Output("upload-error-alert", "is_open", allow_duplicate=True), + Output("upload-success-alert", "children", allow_duplicate=True), + Output("upload-success-alert", "is_open", allow_duplicate=True), + ], Input("upload-data", "contents"), State("upload-data", "filename"), + prevent_initial_call=True, ) def process_uploaded_file(contents, filename): if contents is None: - return None + return None, "", False, "", False processed_data = self.processor.process_upload(contents, filename) if processed_data.error: - return {"error": processed_data.error} + error_message = self._format_error_message(processed_data.error, filename) + return ( + {"error": processed_data.error}, + error_message, + True, # Show error alert + "", + False, # Hide success alert + ) - return { - "documents": [ - self._document_to_dict(doc) for doc in processed_data.documents - ], - "embeddings": processed_data.embeddings.tolist(), - } + success_message = f"Successfully loaded {len(processed_data.documents)} documents from {filename or 'file'}" + return ( + { + "documents": [ + self._document_to_dict(doc) for doc in processed_data.documents + ], + "embeddings": processed_data.embeddings.tolist(), + }, + "", + False, # Hide error alert + success_message, + True, # Show success alert + ) @callback( - Output("processed-prompts", "data"), + Output("processed-prompts", "data", allow_duplicate=True), Input("upload-prompts", "contents"), State("upload-prompts", "filename"), + prevent_initial_call=True, ) def process_uploaded_prompts(contents, filename): if contents is None: @@ -60,3 +82,40 @@ class DataProcessingCallbacks: "subcategory": doc.subcategory, "tags": doc.tags, } + + @staticmethod + def _format_error_message(error: str, filename: str | None = None) -> str: + """Format error message with helpful guidance for users.""" + file_part = f" in file '{filename}'" if filename else "" + + # Check for common error patterns and provide helpful messages + if "embedding" in error.lower() and ("key" in error.lower() or "required field" in error.lower()): + return ( + f"❌ Missing 'embedding' field{file_part}. " + "Each line must contain an 'embedding' field with a list of numbers." + ) + elif "text" in error.lower() and ("key" in error.lower() or "required field" in error.lower()): + return ( + f"❌ Missing 'text' field{file_part}. " + "Each line must contain a 'text' field with the document content." + ) + elif "json" in error.lower() and "decode" in error.lower(): + return ( + f"❌ Invalid JSON format{file_part}. " + "Please check that each line is valid JSON with proper syntax (quotes, braces, etc.)." + ) + elif "unicode" in error.lower() or "decode" in error.lower(): + return ( + f"❌ File encoding issue{file_part}. " + "Please ensure the file is saved in UTF-8 format and contains no binary data." + ) + elif "array" in error.lower() or "list" in error.lower(): + return ( + f"❌ Invalid embedding format{file_part}. " + "Embeddings must be arrays/lists of numbers, not strings or other types." + ) + else: + return ( + f"❌ Error processing file{file_part}: {error}. " + "Please check that your file is valid NDJSON with required 'text' and 'embedding' fields." + ) diff --git a/src/embeddingbuddy/ui/components/sidebar.py b/src/embeddingbuddy/ui/components/sidebar.py index 2ea6487..3e8e49a 100644 --- a/src/embeddingbuddy/ui/components/sidebar.py +++ b/src/embeddingbuddy/ui/components/sidebar.py @@ -11,14 +11,18 @@ class SidebarComponent: return dbc.Col( [ html.H5("Upload Data", className="mb-3"), + self.upload_component.create_error_alert(), + self.upload_component.create_success_alert(), self.upload_component.create_data_upload(), self.upload_component.create_prompts_upload(), self.upload_component.create_reset_button(), html.H5("Visualization Controls", className="mb-3"), - self._create_method_dropdown(), - self._create_color_dropdown(), - self._create_dimension_toggle(), - self._create_prompts_toggle(), + ] + + self._create_method_dropdown() + + self._create_color_dropdown() + + self._create_dimension_toggle() + + self._create_prompts_toggle() + + [ html.H5("Point Details", className="mb-3"), html.Div( id="point-details", children="Click on a point to see details" diff --git a/src/embeddingbuddy/ui/components/upload.py b/src/embeddingbuddy/ui/components/upload.py index 9a25092..94a7bbf 100644 --- a/src/embeddingbuddy/ui/components/upload.py +++ b/src/embeddingbuddy/ui/components/upload.py @@ -51,3 +51,25 @@ class UploadComponent: className="mb-3", style={"width": "100%"}, ) + + @staticmethod + def create_error_alert(): + """Create error alert component for data upload issues.""" + return dbc.Alert( + id="upload-error-alert", + dismissable=True, + is_open=False, + color="danger", + className="mb-3", + ) + + @staticmethod + def create_success_alert(): + """Create success alert component for successful uploads.""" + return dbc.Alert( + id="upload-success-alert", + dismissable=True, + is_open=False, + color="success", + className="mb-3", + ) diff --git a/src/embeddingbuddy/ui/layout.py b/src/embeddingbuddy/ui/layout.py index 960bc4a..cabfe55 100644 --- a/src/embeddingbuddy/ui/layout.py +++ b/src/embeddingbuddy/ui/layout.py @@ -9,7 +9,7 @@ class AppLayout: def create_layout(self): return dbc.Container( - [self._create_header(), self._create_main_content(), self._create_stores()], + [self._create_header(), self._create_main_content()] + self._create_stores(), fluid=True, ) diff --git a/tests/test_bad_data.py b/tests/test_bad_data.py new file mode 100644 index 0000000..76c213c --- /dev/null +++ b/tests/test_bad_data.py @@ -0,0 +1,185 @@ +"""Tests for handling bad/invalid data files.""" +import pytest +import json +import base64 +from src.embeddingbuddy.data.parser import NDJSONParser +from src.embeddingbuddy.data.processor import DataProcessor + + +class TestBadDataHandling: + """Test suite for various types of invalid input data.""" + + def setup_method(self): + """Set up test fixtures.""" + self.parser = NDJSONParser() + self.processor = DataProcessor() + + def _create_upload_contents(self, text_content: str) -> str: + """Helper to create upload contents format.""" + encoded = base64.b64encode(text_content.encode('utf-8')).decode('utf-8') + return f"data:application/json;base64,{encoded}" + + def test_missing_embedding_field(self): + """Test files missing required embedding field.""" + bad_content = '{"id": "doc_001", "text": "Sample text", "category": "test"}' + + with pytest.raises(KeyError, match="embedding"): + self.parser.parse_text(bad_content) + + # Test processor error handling + upload_contents = self._create_upload_contents(bad_content) + result = self.processor.process_upload(upload_contents) + assert result.error is not None + assert "embedding" in result.error + + def test_missing_text_field(self): + """Test files missing required text field.""" + bad_content = '{"id": "doc_001", "embedding": [0.1, 0.2, 0.3], "category": "test"}' + + with pytest.raises(KeyError, match="text"): + self.parser.parse_text(bad_content) + + # Test processor error handling + upload_contents = self._create_upload_contents(bad_content) + result = self.processor.process_upload(upload_contents) + assert result.error is not None + assert "text" in result.error + + def test_malformed_json_lines(self): + """Test files with malformed JSON syntax.""" + # Missing closing brace + bad_content = '{"id": "doc_001", "embedding": [0.1, 0.2], "text": "test"' + + with pytest.raises(json.JSONDecodeError): + self.parser.parse_text(bad_content) + + # Test processor error handling + upload_contents = self._create_upload_contents(bad_content) + result = self.processor.process_upload(upload_contents) + assert result.error is not None + + def test_invalid_embedding_types(self): + """Test files with invalid embedding data types.""" + test_cases = [ + # String instead of array + '{"id": "doc_001", "embedding": "not_an_array", "text": "test"}', + # Mixed types in array + '{"id": "doc_002", "embedding": [0.1, "text", 0.3], "text": "test"}', + # Empty array + '{"id": "doc_003", "embedding": [], "text": "test"}', + # Null embedding + '{"id": "doc_004", "embedding": null, "text": "test"}', + ] + + for bad_content in test_cases: + upload_contents = self._create_upload_contents(bad_content) + result = self.processor.process_upload(upload_contents) + assert result.error is not None, f"Should fail for: {bad_content}" + + def test_inconsistent_embedding_dimensions(self): + """Test files with embeddings of different dimensions.""" + bad_content = '''{"id": "doc_001", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "4D embedding"} +{"id": "doc_002", "embedding": [0.1, 0.2, 0.3], "text": "3D embedding"}''' + + upload_contents = self._create_upload_contents(bad_content) + result = self.processor.process_upload(upload_contents) + + # This might succeed parsing but fail in processing + # The error depends on where dimension validation occurs + if result.error is None: + # If parsing succeeds, check that embeddings have inconsistent shapes + assert len(result.documents) == 2 + assert len(result.documents[0].embedding) != len(result.documents[1].embedding) + + def test_empty_lines_in_ndjson(self): + """Test files with empty lines mixed in.""" + content_with_empty_lines = '''{"id": "doc_001", "embedding": [0.1, 0.2], "text": "First line"} + +{"id": "doc_002", "embedding": [0.3, 0.4], "text": "After empty line"}''' + + # This should work - empty lines should be skipped + documents = self.parser.parse_text(content_with_empty_lines) + assert len(documents) == 2 + assert documents[0].id == "doc_001" + assert documents[1].id == "doc_002" + + def test_not_ndjson_format(self): + """Test regular JSON array instead of NDJSON.""" + json_array = '''[ + {"id": "doc_001", "embedding": [0.1, 0.2], "text": "First"}, + {"id": "doc_002", "embedding": [0.3, 0.4], "text": "Second"} +]''' + + with pytest.raises(json.JSONDecodeError): + self.parser.parse_text(json_array) + + def test_binary_content_in_file(self): + """Test files with binary content mixed in.""" + # Simulate binary content that can't be decoded + binary_content = b'\x00\x01\x02{"id": "doc_001", "embedding": [0.1], "text": "test"}' + + # This should result in an error when processing + encoded = base64.b64encode(binary_content).decode('utf-8') + upload_contents = f"data:application/json;base64,{encoded}" + result = self.processor.process_upload(upload_contents) + + # Should either fail with UnicodeDecodeError or JSON parsing error + assert result.error is not None + + def test_extremely_large_embeddings(self): + """Test embeddings with very large dimensions.""" + large_embedding = [0.1] * 10000 # 10k dimensions + content = json.dumps({ + "id": "doc_001", + "embedding": large_embedding, + "text": "Large embedding test" + }) + + # This should work but might be slow + upload_contents = self._create_upload_contents(content) + result = self.processor.process_upload(upload_contents) + + if result.error is None: + assert len(result.documents) == 1 + assert len(result.documents[0].embedding) == 10000 + + def test_special_characters_in_text(self): + """Test handling of special characters and unicode.""" + special_content = json.dumps({ + "id": "doc_001", + "embedding": [0.1, 0.2], + "text": "Special chars: 🚀 ñoñó 中文 \n\t\"" + }, ensure_ascii=False) + + upload_contents = self._create_upload_contents(special_content) + result = self.processor.process_upload(upload_contents) + + assert result.error is None + assert len(result.documents) == 1 + assert "🚀" in result.documents[0].text + + def test_processor_error_structure(self): + """Test that processor returns proper error structure.""" + bad_content = '{"invalid": "json"' # Missing closing brace + upload_contents = self._create_upload_contents(bad_content) + + result = self.processor.process_upload(upload_contents) + + # Check error structure + assert result.error is not None + assert isinstance(result.error, str) + assert len(result.documents) == 0 + assert result.embeddings.size == 0 + + def test_multiple_errors_in_file(self): + """Test file with multiple different types of errors.""" + multi_error_content = '''{"id": "doc_001", "text": "Missing embedding"} +{"id": "doc_002", "embedding": "wrong_type", "text": "Wrong embedding type"} +{"id": "doc_003", "embedding": [0.1, 0.2], "text": "Valid line"} +{"id": "doc_004", "embedding": [0.3, 0.4]''' # Missing text and closing brace + + upload_contents = self._create_upload_contents(multi_error_content) + result = self.processor.process_upload(upload_contents) + + # Should fail on first error encountered + assert result.error is not None \ No newline at end of file From 6a995635ac321d025d23ca3d8afb024a8cad254f Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Thu, 14 Aug 2025 08:00:47 -0700 Subject: [PATCH 2/3] remove upload success alert --- .claude/settings.local.json | 3 ++- README.md | 2 +- src/embeddingbuddy/ui/callbacks/data_processing.py | 9 +-------- src/embeddingbuddy/ui/components/sidebar.py | 1 - src/embeddingbuddy/ui/components/upload.py | 10 ---------- 5 files changed, 4 insertions(+), 21 deletions(-) diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 88bfffc..895a257 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -6,6 +6,7 @@ "Bash(uv add:*)" ], "deny": [], - "ask": [] + "ask": [], + "defaultMode": "acceptEdits" } } \ No newline at end of file diff --git a/README.md b/README.md index 0bc814b..507961c 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ uv run python main.py The application follows a modular architecture for improved maintainability and testability: -``` +```text src/embeddingbuddy/ ├── config/ # Configuration management │ └── settings.py # Centralized app settings diff --git a/src/embeddingbuddy/ui/callbacks/data_processing.py b/src/embeddingbuddy/ui/callbacks/data_processing.py index eec3a82..bd9edfd 100644 --- a/src/embeddingbuddy/ui/callbacks/data_processing.py +++ b/src/embeddingbuddy/ui/callbacks/data_processing.py @@ -13,8 +13,6 @@ class DataProcessingCallbacks: Output("processed-data", "data", allow_duplicate=True), Output("upload-error-alert", "children", allow_duplicate=True), Output("upload-error-alert", "is_open", allow_duplicate=True), - Output("upload-success-alert", "children", allow_duplicate=True), - Output("upload-success-alert", "is_open", allow_duplicate=True), ], Input("upload-data", "contents"), State("upload-data", "filename"), @@ -22,7 +20,7 @@ class DataProcessingCallbacks: ) def process_uploaded_file(contents, filename): if contents is None: - return None, "", False, "", False + return None, "", False processed_data = self.processor.process_upload(contents, filename) @@ -32,11 +30,8 @@ class DataProcessingCallbacks: {"error": processed_data.error}, error_message, True, # Show error alert - "", - False, # Hide success alert ) - success_message = f"Successfully loaded {len(processed_data.documents)} documents from {filename or 'file'}" return ( { "documents": [ @@ -46,8 +41,6 @@ class DataProcessingCallbacks: }, "", False, # Hide error alert - success_message, - True, # Show success alert ) @callback( diff --git a/src/embeddingbuddy/ui/components/sidebar.py b/src/embeddingbuddy/ui/components/sidebar.py index 3e8e49a..0766f02 100644 --- a/src/embeddingbuddy/ui/components/sidebar.py +++ b/src/embeddingbuddy/ui/components/sidebar.py @@ -12,7 +12,6 @@ class SidebarComponent: [ html.H5("Upload Data", className="mb-3"), self.upload_component.create_error_alert(), - self.upload_component.create_success_alert(), self.upload_component.create_data_upload(), self.upload_component.create_prompts_upload(), self.upload_component.create_reset_button(), diff --git a/src/embeddingbuddy/ui/components/upload.py b/src/embeddingbuddy/ui/components/upload.py index 94a7bbf..8e4db92 100644 --- a/src/embeddingbuddy/ui/components/upload.py +++ b/src/embeddingbuddy/ui/components/upload.py @@ -63,13 +63,3 @@ class UploadComponent: className="mb-3", ) - @staticmethod - def create_success_alert(): - """Create success alert component for successful uploads.""" - return dbc.Alert( - id="upload-success-alert", - dismissable=True, - is_open=False, - color="success", - className="mb-3", - ) From 48676144741fd15e0bf0daf6a2f3545f0bd15dff Mon Sep 17 00:00:00 2001 From: Austin Godber Date: Thu, 14 Aug 2025 08:07:50 -0700 Subject: [PATCH 3/3] reformat --- src/embeddingbuddy/data/parser.py | 20 ++-- .../ui/callbacks/data_processing.py | 14 ++- src/embeddingbuddy/ui/components/upload.py | 1 - src/embeddingbuddy/ui/layout.py | 3 +- tests/test_bad_data.py | 102 ++++++++++-------- 5 files changed, 79 insertions(+), 61 deletions(-) diff --git a/src/embeddingbuddy/data/parser.py b/src/embeddingbuddy/data/parser.py index 3b8cc42..a1aa1fe 100644 --- a/src/embeddingbuddy/data/parser.py +++ b/src/embeddingbuddy/data/parser.py @@ -24,14 +24,10 @@ class NDJSONParser: documents.append(doc) except json.JSONDecodeError as e: raise json.JSONDecodeError( - f"Invalid JSON on line {line_num}: {e.msg}", - e.doc, - e.pos + f"Invalid JSON on line {line_num}: {e.msg}", e.doc, e.pos ) except KeyError as e: - raise KeyError( - f"Missing required field {e} on line {line_num}" - ) + raise KeyError(f"Missing required field {e} on line {line_num}") except (TypeError, ValueError) as e: raise ValueError( f"Invalid data format on line {line_num}: {str(e)}" @@ -52,15 +48,19 @@ class NDJSONParser: # Validate embedding format embedding = doc_dict["embedding"] if not isinstance(embedding, list): - raise ValueError(f"Embedding must be a list, got {type(embedding).__name__}") - + raise ValueError( + f"Embedding must be a list, got {type(embedding).__name__}" + ) + if not embedding: raise ValueError("Embedding cannot be empty") - + # Check that all embedding values are numbers for i, val in enumerate(embedding): if not isinstance(val, (int, float)) or val != val: # NaN check - raise ValueError(f"Embedding contains invalid value at index {i}: {val}") + raise ValueError( + f"Embedding contains invalid value at index {i}: {val}" + ) return Document( id=doc_dict["id"], diff --git a/src/embeddingbuddy/ui/callbacks/data_processing.py b/src/embeddingbuddy/ui/callbacks/data_processing.py index bd9edfd..e0491d3 100644 --- a/src/embeddingbuddy/ui/callbacks/data_processing.py +++ b/src/embeddingbuddy/ui/callbacks/data_processing.py @@ -25,7 +25,9 @@ class DataProcessingCallbacks: processed_data = self.processor.process_upload(contents, filename) if processed_data.error: - error_message = self._format_error_message(processed_data.error, filename) + error_message = self._format_error_message( + processed_data.error, filename + ) return ( {"error": processed_data.error}, error_message, @@ -80,14 +82,18 @@ class DataProcessingCallbacks: def _format_error_message(error: str, filename: str | None = None) -> str: """Format error message with helpful guidance for users.""" file_part = f" in file '{filename}'" if filename else "" - + # Check for common error patterns and provide helpful messages - if "embedding" in error.lower() and ("key" in error.lower() or "required field" in error.lower()): + if "embedding" in error.lower() and ( + "key" in error.lower() or "required field" in error.lower() + ): return ( f"❌ Missing 'embedding' field{file_part}. " "Each line must contain an 'embedding' field with a list of numbers." ) - elif "text" in error.lower() and ("key" in error.lower() or "required field" in error.lower()): + elif "text" in error.lower() and ( + "key" in error.lower() or "required field" in error.lower() + ): return ( f"❌ Missing 'text' field{file_part}. " "Each line must contain a 'text' field with the document content." diff --git a/src/embeddingbuddy/ui/components/upload.py b/src/embeddingbuddy/ui/components/upload.py index 8e4db92..b9271d7 100644 --- a/src/embeddingbuddy/ui/components/upload.py +++ b/src/embeddingbuddy/ui/components/upload.py @@ -62,4 +62,3 @@ class UploadComponent: color="danger", className="mb-3", ) - diff --git a/src/embeddingbuddy/ui/layout.py b/src/embeddingbuddy/ui/layout.py index cabfe55..71a0402 100644 --- a/src/embeddingbuddy/ui/layout.py +++ b/src/embeddingbuddy/ui/layout.py @@ -9,7 +9,8 @@ class AppLayout: def create_layout(self): return dbc.Container( - [self._create_header(), self._create_main_content()] + self._create_stores(), + [self._create_header(), self._create_main_content()] + + self._create_stores(), fluid=True, ) diff --git a/tests/test_bad_data.py b/tests/test_bad_data.py index 76c213c..0260c1c 100644 --- a/tests/test_bad_data.py +++ b/tests/test_bad_data.py @@ -1,4 +1,5 @@ """Tests for handling bad/invalid data files.""" + import pytest import json import base64 @@ -16,16 +17,16 @@ class TestBadDataHandling: def _create_upload_contents(self, text_content: str) -> str: """Helper to create upload contents format.""" - encoded = base64.b64encode(text_content.encode('utf-8')).decode('utf-8') + encoded = base64.b64encode(text_content.encode("utf-8")).decode("utf-8") return f"data:application/json;base64,{encoded}" def test_missing_embedding_field(self): """Test files missing required embedding field.""" bad_content = '{"id": "doc_001", "text": "Sample text", "category": "test"}' - + with pytest.raises(KeyError, match="embedding"): self.parser.parse_text(bad_content) - + # Test processor error handling upload_contents = self._create_upload_contents(bad_content) result = self.processor.process_upload(upload_contents) @@ -34,11 +35,13 @@ class TestBadDataHandling: def test_missing_text_field(self): """Test files missing required text field.""" - bad_content = '{"id": "doc_001", "embedding": [0.1, 0.2, 0.3], "category": "test"}' - + bad_content = ( + '{"id": "doc_001", "embedding": [0.1, 0.2, 0.3], "category": "test"}' + ) + with pytest.raises(KeyError, match="text"): self.parser.parse_text(bad_content) - + # Test processor error handling upload_contents = self._create_upload_contents(bad_content) result = self.processor.process_upload(upload_contents) @@ -49,10 +52,10 @@ class TestBadDataHandling: """Test files with malformed JSON syntax.""" # Missing closing brace bad_content = '{"id": "doc_001", "embedding": [0.1, 0.2], "text": "test"' - + with pytest.raises(json.JSONDecodeError): self.parser.parse_text(bad_content) - + # Test processor error handling upload_contents = self._create_upload_contents(bad_content) result = self.processor.process_upload(upload_contents) @@ -70,7 +73,7 @@ class TestBadDataHandling: # Null embedding '{"id": "doc_004", "embedding": null, "text": "test"}', ] - + for bad_content in test_cases: upload_contents = self._create_upload_contents(bad_content) result = self.processor.process_upload(upload_contents) @@ -78,25 +81,27 @@ class TestBadDataHandling: def test_inconsistent_embedding_dimensions(self): """Test files with embeddings of different dimensions.""" - bad_content = '''{"id": "doc_001", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "4D embedding"} -{"id": "doc_002", "embedding": [0.1, 0.2, 0.3], "text": "3D embedding"}''' - + bad_content = """{"id": "doc_001", "embedding": [0.1, 0.2, 0.3, 0.4], "text": "4D embedding"} +{"id": "doc_002", "embedding": [0.1, 0.2, 0.3], "text": "3D embedding"}""" + upload_contents = self._create_upload_contents(bad_content) result = self.processor.process_upload(upload_contents) - + # This might succeed parsing but fail in processing # The error depends on where dimension validation occurs if result.error is None: # If parsing succeeds, check that embeddings have inconsistent shapes assert len(result.documents) == 2 - assert len(result.documents[0].embedding) != len(result.documents[1].embedding) + assert len(result.documents[0].embedding) != len( + result.documents[1].embedding + ) def test_empty_lines_in_ndjson(self): """Test files with empty lines mixed in.""" - content_with_empty_lines = '''{"id": "doc_001", "embedding": [0.1, 0.2], "text": "First line"} + content_with_empty_lines = """{"id": "doc_001", "embedding": [0.1, 0.2], "text": "First line"} + +{"id": "doc_002", "embedding": [0.3, 0.4], "text": "After empty line"}""" -{"id": "doc_002", "embedding": [0.3, 0.4], "text": "After empty line"}''' - # This should work - empty lines should be skipped documents = self.parser.parse_text(content_with_empty_lines) assert len(documents) == 2 @@ -105,55 +110,62 @@ class TestBadDataHandling: def test_not_ndjson_format(self): """Test regular JSON array instead of NDJSON.""" - json_array = '''[ + json_array = """[ {"id": "doc_001", "embedding": [0.1, 0.2], "text": "First"}, {"id": "doc_002", "embedding": [0.3, 0.4], "text": "Second"} -]''' - +]""" + with pytest.raises(json.JSONDecodeError): self.parser.parse_text(json_array) def test_binary_content_in_file(self): """Test files with binary content mixed in.""" # Simulate binary content that can't be decoded - binary_content = b'\x00\x01\x02{"id": "doc_001", "embedding": [0.1], "text": "test"}' - + binary_content = ( + b'\x00\x01\x02{"id": "doc_001", "embedding": [0.1], "text": "test"}' + ) + # This should result in an error when processing - encoded = base64.b64encode(binary_content).decode('utf-8') + encoded = base64.b64encode(binary_content).decode("utf-8") upload_contents = f"data:application/json;base64,{encoded}" result = self.processor.process_upload(upload_contents) - + # Should either fail with UnicodeDecodeError or JSON parsing error assert result.error is not None def test_extremely_large_embeddings(self): """Test embeddings with very large dimensions.""" large_embedding = [0.1] * 10000 # 10k dimensions - content = json.dumps({ - "id": "doc_001", - "embedding": large_embedding, - "text": "Large embedding test" - }) - + content = json.dumps( + { + "id": "doc_001", + "embedding": large_embedding, + "text": "Large embedding test", + } + ) + # This should work but might be slow upload_contents = self._create_upload_contents(content) result = self.processor.process_upload(upload_contents) - + if result.error is None: assert len(result.documents) == 1 assert len(result.documents[0].embedding) == 10000 def test_special_characters_in_text(self): """Test handling of special characters and unicode.""" - special_content = json.dumps({ - "id": "doc_001", - "embedding": [0.1, 0.2], - "text": "Special chars: 🚀 ñoñó 中文 \n\t\"" - }, ensure_ascii=False) - + special_content = json.dumps( + { + "id": "doc_001", + "embedding": [0.1, 0.2], + "text": 'Special chars: 🚀 ñoñó 中文 \n\t"', + }, + ensure_ascii=False, + ) + upload_contents = self._create_upload_contents(special_content) result = self.processor.process_upload(upload_contents) - + assert result.error is None assert len(result.documents) == 1 assert "🚀" in result.documents[0].text @@ -162,9 +174,9 @@ class TestBadDataHandling: """Test that processor returns proper error structure.""" bad_content = '{"invalid": "json"' # Missing closing brace upload_contents = self._create_upload_contents(bad_content) - + result = self.processor.process_upload(upload_contents) - + # Check error structure assert result.error is not None assert isinstance(result.error, str) @@ -173,13 +185,13 @@ class TestBadDataHandling: def test_multiple_errors_in_file(self): """Test file with multiple different types of errors.""" - multi_error_content = '''{"id": "doc_001", "text": "Missing embedding"} + multi_error_content = """{"id": "doc_001", "text": "Missing embedding"} {"id": "doc_002", "embedding": "wrong_type", "text": "Wrong embedding type"} {"id": "doc_003", "embedding": [0.1, 0.2], "text": "Valid line"} -{"id": "doc_004", "embedding": [0.3, 0.4]''' # Missing text and closing brace - +{"id": "doc_004", "embedding": [0.3, 0.4]""" # Missing text and closing brace + upload_contents = self._create_upload_contents(multi_error_content) result = self.processor.process_upload(upload_contents) - + # Should fail on first error encountered - assert result.error is not None \ No newline at end of file + assert result.error is not None