initial implementation works.
This commit is contained in:
238
app.py
Normal file
238
app.py
Normal file
@@ -0,0 +1,238 @@
|
||||
import json
|
||||
import uuid
|
||||
from io import StringIO
|
||||
import base64
|
||||
|
||||
import dash
|
||||
from dash import dcc, html, Input, Output, State, callback
|
||||
import dash_bootstrap_components as dbc
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from sklearn.decomposition import PCA
|
||||
|
||||
|
||||
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
|
||||
|
||||
def parse_ndjson(contents):
|
||||
"""Parse NDJSON content and return list of documents."""
|
||||
content_type, content_string = contents.split(',')
|
||||
decoded = base64.b64decode(content_string)
|
||||
text_content = decoded.decode('utf-8')
|
||||
|
||||
documents = []
|
||||
for line in text_content.strip().split('\n'):
|
||||
if line.strip():
|
||||
doc = json.loads(line)
|
||||
if 'id' not in doc:
|
||||
doc['id'] = str(uuid.uuid4())
|
||||
documents.append(doc)
|
||||
return documents
|
||||
|
||||
def apply_pca(embeddings, n_components=3):
|
||||
"""Apply PCA to embeddings."""
|
||||
pca = PCA(n_components=n_components)
|
||||
reduced = pca.fit_transform(embeddings)
|
||||
return reduced, pca.explained_variance_ratio_
|
||||
|
||||
def create_color_mapping(documents, color_by):
|
||||
"""Create color mapping for documents based on specified field."""
|
||||
if color_by == 'category':
|
||||
values = [doc.get('category', 'Unknown') for doc in documents]
|
||||
elif color_by == 'subcategory':
|
||||
values = [doc.get('subcategory', 'Unknown') for doc in documents]
|
||||
elif color_by == 'tags':
|
||||
values = [', '.join(doc.get('tags', [])) if doc.get('tags') else 'No tags' for doc in documents]
|
||||
else:
|
||||
values = ['All'] * len(documents)
|
||||
|
||||
return values
|
||||
|
||||
def create_plot(df, dimensions='3d', color_by='category'):
|
||||
"""Create plotly scatter plot."""
|
||||
color_values = create_color_mapping(df.to_dict('records'), color_by)
|
||||
|
||||
if dimensions == '3d':
|
||||
fig = px.scatter_3d(
|
||||
df, x='pca_1', y='pca_2', z='pca_3',
|
||||
color=color_values,
|
||||
hover_data=['id', 'text'],
|
||||
title=f'3D Embedding Visualization (colored by {color_by})'
|
||||
)
|
||||
fig.update_traces(marker=dict(size=5))
|
||||
else:
|
||||
fig = px.scatter(
|
||||
df, x='pca_1', y='pca_2',
|
||||
color=color_values,
|
||||
hover_data=['id', 'text'],
|
||||
title=f'2D Embedding Visualization (colored by {color_by})'
|
||||
)
|
||||
fig.update_traces(marker=dict(size=8))
|
||||
|
||||
fig.update_layout(height=600)
|
||||
return fig
|
||||
|
||||
# Layout
|
||||
app.layout = dbc.Container([
|
||||
dbc.Row([
|
||||
dbc.Col([
|
||||
html.H1("EmbeddingBuddy", className="text-center mb-4"),
|
||||
html.P("Upload NDJSON file with embeddings to visualize", className="text-center text-muted")
|
||||
])
|
||||
]),
|
||||
|
||||
dbc.Row([
|
||||
dbc.Col([
|
||||
dcc.Upload(
|
||||
id='upload-data',
|
||||
children=html.Div([
|
||||
'Drag and Drop or ',
|
||||
html.A('Select Files')
|
||||
]),
|
||||
style={
|
||||
'width': '100%',
|
||||
'height': '60px',
|
||||
'lineHeight': '60px',
|
||||
'borderWidth': '1px',
|
||||
'borderStyle': 'dashed',
|
||||
'borderRadius': '5px',
|
||||
'textAlign': 'center',
|
||||
'margin': '10px'
|
||||
},
|
||||
multiple=False
|
||||
)
|
||||
])
|
||||
]),
|
||||
|
||||
dbc.Row([
|
||||
dbc.Col([
|
||||
dbc.Label("Color by:"),
|
||||
dcc.Dropdown(
|
||||
id='color-dropdown',
|
||||
options=[
|
||||
{'label': 'Category', 'value': 'category'},
|
||||
{'label': 'Subcategory', 'value': 'subcategory'},
|
||||
{'label': 'Tags', 'value': 'tags'}
|
||||
],
|
||||
value='category',
|
||||
style={'margin-bottom': '10px'}
|
||||
)
|
||||
], width=6),
|
||||
dbc.Col([
|
||||
dbc.Label("Dimensions:"),
|
||||
dcc.RadioItems(
|
||||
id='dimension-toggle',
|
||||
options=[
|
||||
{'label': '2D', 'value': '2d'},
|
||||
{'label': '3D', 'value': '3d'}
|
||||
],
|
||||
value='3d',
|
||||
inline=True
|
||||
)
|
||||
], width=6)
|
||||
], className="mb-3"),
|
||||
|
||||
dbc.Row([
|
||||
dbc.Col([
|
||||
dcc.Graph(id='embedding-plot')
|
||||
])
|
||||
]),
|
||||
|
||||
dbc.Row([
|
||||
dbc.Col([
|
||||
html.Div(id='point-details', style={'margin-top': '20px'})
|
||||
])
|
||||
]),
|
||||
|
||||
dcc.Store(id='processed-data')
|
||||
], fluid=True)
|
||||
|
||||
@callback(
|
||||
Output('processed-data', 'data'),
|
||||
Input('upload-data', 'contents'),
|
||||
State('upload-data', 'filename')
|
||||
)
|
||||
def process_uploaded_file(contents, filename):
|
||||
if contents is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
documents = parse_ndjson(contents)
|
||||
embeddings = np.array([doc['embedding'] for doc in documents])
|
||||
|
||||
# Apply PCA
|
||||
pca_2d, var_2d = apply_pca(embeddings, n_components=2)
|
||||
pca_3d, var_3d = apply_pca(embeddings, n_components=3)
|
||||
|
||||
# Create dataframe
|
||||
df_data = []
|
||||
for i, doc in enumerate(documents):
|
||||
df_data.append({
|
||||
'id': doc['id'],
|
||||
'text': doc['text'],
|
||||
'category': doc.get('category', 'Unknown'),
|
||||
'subcategory': doc.get('subcategory', 'Unknown'),
|
||||
'tags': doc.get('tags', []),
|
||||
'pca_1': pca_3d[i, 0],
|
||||
'pca_2': pca_3d[i, 1],
|
||||
'pca_3': pca_3d[i, 2],
|
||||
'pca_1_2d': pca_2d[i, 0],
|
||||
'pca_2_2d': pca_2d[i, 1]
|
||||
})
|
||||
|
||||
return {
|
||||
'documents': df_data,
|
||||
'variance_explained_2d': var_2d.tolist(),
|
||||
'variance_explained_3d': var_3d.tolist()
|
||||
}
|
||||
except Exception as e:
|
||||
return {'error': str(e)}
|
||||
|
||||
@callback(
|
||||
Output('embedding-plot', 'figure'),
|
||||
[Input('processed-data', 'data'),
|
||||
Input('color-dropdown', 'value'),
|
||||
Input('dimension-toggle', 'value')]
|
||||
)
|
||||
def update_plot(data, color_by, dimensions):
|
||||
if not data or 'error' in data:
|
||||
return go.Figure().add_annotation(
|
||||
text="Upload a valid NDJSON file to see visualization",
|
||||
xref="paper", yref="paper",
|
||||
x=0.5, y=0.5, xanchor='center', yanchor='middle',
|
||||
showarrow=False, font=dict(size=16)
|
||||
)
|
||||
|
||||
df = pd.DataFrame(data['documents'])
|
||||
|
||||
if dimensions == '2d':
|
||||
df['pca_1'] = df['pca_1_2d']
|
||||
df['pca_2'] = df['pca_2_2d']
|
||||
|
||||
return create_plot(df, dimensions, color_by)
|
||||
|
||||
@callback(
|
||||
Output('point-details', 'children'),
|
||||
Input('embedding-plot', 'clickData'),
|
||||
State('processed-data', 'data')
|
||||
)
|
||||
def display_click_data(clickData, data):
|
||||
if not clickData or not data:
|
||||
return "Click on a point to see details"
|
||||
|
||||
point_index = clickData['points'][0]['pointIndex']
|
||||
doc = data['documents'][point_index]
|
||||
|
||||
return dbc.Card([
|
||||
dbc.CardBody([
|
||||
html.H5(f"Document: {doc['id']}", className="card-title"),
|
||||
html.P(f"Text: {doc['text']}", className="card-text"),
|
||||
html.P(f"Category: {doc['category']}", className="card-text"),
|
||||
html.P(f"Subcategory: {doc['subcategory']}", className="card-text"),
|
||||
html.P(f"Tags: {', '.join(doc['tags']) if doc['tags'] else 'None'}", className="card-text")
|
||||
])
|
||||
])
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True)
|
||||
Reference in New Issue
Block a user