this will load data from Opensearch.
it doesn't have prompts as well
This commit is contained in:
@@ -1,10 +1,14 @@
|
||||
from dash import callback, Input, Output, State
|
||||
from dash import callback, Input, Output, State, no_update
|
||||
from ...data.processor import DataProcessor
|
||||
from ...data.sources.opensearch import OpenSearchClient
|
||||
from ...models.field_mapper import FieldMapper
|
||||
from ...config.settings import AppSettings
|
||||
|
||||
|
||||
class DataProcessingCallbacks:
|
||||
def __init__(self):
|
||||
self.processor = DataProcessor()
|
||||
self.opensearch_client = OpenSearchClient()
|
||||
self._register_callbacks()
|
||||
|
||||
def _register_callbacks(self):
|
||||
@@ -67,6 +71,283 @@ class DataProcessingCallbacks:
|
||||
"embeddings": processed_data.embeddings.tolist(),
|
||||
}
|
||||
|
||||
# OpenSearch callbacks
|
||||
@callback(
|
||||
[
|
||||
Output("tab-content", "children"),
|
||||
],
|
||||
[Input("data-source-tabs", "active_tab")],
|
||||
prevent_initial_call=False,
|
||||
)
|
||||
def render_tab_content(active_tab):
|
||||
from ...ui.components.datasource import DataSourceComponent
|
||||
|
||||
datasource = DataSourceComponent()
|
||||
|
||||
if active_tab == "opensearch-tab":
|
||||
return [datasource.create_opensearch_tab()]
|
||||
else:
|
||||
return [datasource.create_file_upload_tab()]
|
||||
|
||||
@callback(
|
||||
Output("auth-collapse", "is_open"),
|
||||
[Input("auth-toggle", "n_clicks")],
|
||||
[State("auth-collapse", "is_open")],
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def toggle_auth(n_clicks, is_open):
|
||||
if n_clicks:
|
||||
return not is_open
|
||||
return is_open
|
||||
|
||||
@callback(
|
||||
Output("auth-toggle", "children"),
|
||||
[Input("auth-collapse", "is_open")],
|
||||
prevent_initial_call=False,
|
||||
)
|
||||
def update_auth_button_text(is_open):
|
||||
return "Hide Authentication" if is_open else "Show Authentication"
|
||||
|
||||
@callback(
|
||||
[
|
||||
Output("connection-status", "children"),
|
||||
Output("field-mapping-section", "children"),
|
||||
Output("field-mapping-section", "style"),
|
||||
Output("load-data-section", "style"),
|
||||
Output("load-opensearch-data-btn", "disabled"),
|
||||
Output("embedding-field-dropdown", "options"),
|
||||
Output("text-field-dropdown", "options"),
|
||||
Output("id-field-dropdown", "options"),
|
||||
Output("category-field-dropdown", "options"),
|
||||
Output("subcategory-field-dropdown", "options"),
|
||||
Output("tags-field-dropdown", "options"),
|
||||
],
|
||||
[Input("test-connection-btn", "n_clicks")],
|
||||
[
|
||||
State("opensearch-url", "value"),
|
||||
State("opensearch-index", "value"),
|
||||
State("opensearch-username", "value"),
|
||||
State("opensearch-password", "value"),
|
||||
State("opensearch-api-key", "value"),
|
||||
],
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def test_opensearch_connection(
|
||||
n_clicks, url, index_name, username, password, api_key
|
||||
):
|
||||
if not n_clicks or not url or not index_name:
|
||||
return no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update, no_update
|
||||
|
||||
# Test connection
|
||||
success, message = self.opensearch_client.connect(
|
||||
url=url,
|
||||
username=username,
|
||||
password=password,
|
||||
api_key=api_key,
|
||||
verify_certs=AppSettings.OPENSEARCH_VERIFY_CERTS,
|
||||
)
|
||||
|
||||
if not success:
|
||||
return (
|
||||
self._create_status_alert(f"❌ {message}", "danger"),
|
||||
[],
|
||||
{"display": "none"},
|
||||
{"display": "none"},
|
||||
True,
|
||||
[], # empty options for hidden dropdowns
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
# Analyze fields
|
||||
success, field_analysis, analysis_message = (
|
||||
self.opensearch_client.analyze_fields(index_name)
|
||||
)
|
||||
|
||||
if not success:
|
||||
return (
|
||||
self._create_status_alert(f"❌ {analysis_message}", "danger"),
|
||||
[],
|
||||
{"display": "none"},
|
||||
{"display": "none"},
|
||||
True,
|
||||
[], # empty options for hidden dropdowns
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
|
||||
# Generate field suggestions
|
||||
field_suggestions = FieldMapper.suggest_mappings(field_analysis)
|
||||
|
||||
from ...ui.components.datasource import DataSourceComponent
|
||||
|
||||
datasource = DataSourceComponent()
|
||||
field_mapping_ui = datasource.create_field_mapping_interface(
|
||||
field_suggestions
|
||||
)
|
||||
|
||||
return (
|
||||
self._create_status_alert(f"✅ {message}", "success"),
|
||||
field_mapping_ui,
|
||||
{"display": "block"},
|
||||
{"display": "block"},
|
||||
False,
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("embedding", [])],
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("text", [])],
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("id", [])],
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("category", [])],
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("subcategory", [])],
|
||||
[{"label": field, "value": field} for field in field_suggestions.get("tags", [])],
|
||||
)
|
||||
|
||||
@callback(
|
||||
[
|
||||
Output("processed-data", "data", allow_duplicate=True),
|
||||
Output("opensearch-success-alert", "children", allow_duplicate=True),
|
||||
Output("opensearch-success-alert", "is_open", allow_duplicate=True),
|
||||
Output("opensearch-error-alert", "children", allow_duplicate=True),
|
||||
Output("opensearch-error-alert", "is_open", allow_duplicate=True),
|
||||
],
|
||||
[Input("load-opensearch-data-btn", "n_clicks")],
|
||||
[
|
||||
State("opensearch-index", "value"),
|
||||
State("embedding-field-dropdown", "value"),
|
||||
State("text-field-dropdown", "value"),
|
||||
State("id-field-dropdown", "value"),
|
||||
State("category-field-dropdown", "value"),
|
||||
State("subcategory-field-dropdown", "value"),
|
||||
State("tags-field-dropdown", "value"),
|
||||
],
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def load_opensearch_data(
|
||||
n_clicks,
|
||||
index_name,
|
||||
embedding_field,
|
||||
text_field,
|
||||
id_field,
|
||||
category_field,
|
||||
subcategory_field,
|
||||
tags_field,
|
||||
):
|
||||
if not n_clicks or not index_name or not embedding_field or not text_field:
|
||||
return no_update, no_update, no_update, no_update, no_update
|
||||
|
||||
try:
|
||||
# Create field mapping
|
||||
field_mapping = FieldMapper.create_mapping_from_dict(
|
||||
{
|
||||
"embedding": embedding_field,
|
||||
"text": text_field,
|
||||
"id": id_field,
|
||||
"category": category_field,
|
||||
"subcategory": subcategory_field,
|
||||
"tags": tags_field,
|
||||
}
|
||||
)
|
||||
|
||||
# Fetch data from OpenSearch
|
||||
success, raw_documents, message = self.opensearch_client.fetch_data(
|
||||
index_name, size=AppSettings.OPENSEARCH_DEFAULT_SIZE
|
||||
)
|
||||
|
||||
if not success:
|
||||
return (
|
||||
no_update,
|
||||
"",
|
||||
False,
|
||||
f"❌ Failed to fetch data: {message}",
|
||||
True,
|
||||
)
|
||||
|
||||
# Process the data
|
||||
processed_data = self.processor.process_opensearch_data(
|
||||
raw_documents, field_mapping
|
||||
)
|
||||
|
||||
if processed_data.error:
|
||||
return (
|
||||
{"error": processed_data.error},
|
||||
"",
|
||||
False,
|
||||
f"❌ Data processing error: {processed_data.error}",
|
||||
True,
|
||||
)
|
||||
|
||||
success_message = f"✅ Successfully loaded {len(processed_data.documents)} documents from OpenSearch"
|
||||
|
||||
return (
|
||||
{
|
||||
"documents": [
|
||||
self._document_to_dict(doc)
|
||||
for doc in processed_data.documents
|
||||
],
|
||||
"embeddings": processed_data.embeddings.tolist(),
|
||||
},
|
||||
success_message,
|
||||
True,
|
||||
"",
|
||||
False,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return (no_update, "", False, f"❌ Unexpected error: {str(e)}", True)
|
||||
|
||||
# Sync callbacks to update hidden dropdowns from UI dropdowns
|
||||
@callback(
|
||||
Output("embedding-field-dropdown", "value"),
|
||||
Input("embedding-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_embedding_dropdown(value):
|
||||
return value
|
||||
|
||||
@callback(
|
||||
Output("text-field-dropdown", "value"),
|
||||
Input("text-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_text_dropdown(value):
|
||||
return value
|
||||
|
||||
@callback(
|
||||
Output("id-field-dropdown", "value"),
|
||||
Input("id-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_id_dropdown(value):
|
||||
return value
|
||||
|
||||
@callback(
|
||||
Output("category-field-dropdown", "value"),
|
||||
Input("category-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_category_dropdown(value):
|
||||
return value
|
||||
|
||||
@callback(
|
||||
Output("subcategory-field-dropdown", "value"),
|
||||
Input("subcategory-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_subcategory_dropdown(value):
|
||||
return value
|
||||
|
||||
@callback(
|
||||
Output("tags-field-dropdown", "value"),
|
||||
Input("tags-field-dropdown-ui", "value"),
|
||||
prevent_initial_call=True,
|
||||
)
|
||||
def sync_tags_dropdown(value):
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _document_to_dict(doc):
|
||||
return {
|
||||
@@ -118,3 +399,10 @@ class DataProcessingCallbacks:
|
||||
f"❌ Error processing file{file_part}: {error}. "
|
||||
"Please check that your file is valid NDJSON with required 'text' and 'embedding' fields."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _create_status_alert(message: str, color: str):
|
||||
"""Create a status alert component."""
|
||||
import dash_bootstrap_components as dbc
|
||||
|
||||
return dbc.Alert(message, color=color, className="mb-2")
|
||||
|
||||
320
src/embeddingbuddy/ui/components/datasource.py
Normal file
320
src/embeddingbuddy/ui/components/datasource.py
Normal file
@@ -0,0 +1,320 @@
|
||||
from dash import dcc, html
|
||||
import dash_bootstrap_components as dbc
|
||||
from .upload import UploadComponent
|
||||
|
||||
|
||||
class DataSourceComponent:
|
||||
def __init__(self):
|
||||
self.upload_component = UploadComponent()
|
||||
|
||||
def create_tabbed_interface(self):
|
||||
"""Create tabbed interface for different data sources."""
|
||||
return dbc.Card(
|
||||
[
|
||||
dbc.CardHeader(
|
||||
[
|
||||
dbc.Tabs(
|
||||
[
|
||||
dbc.Tab(label="File Upload", tab_id="file-tab"),
|
||||
dbc.Tab(label="OpenSearch", tab_id="opensearch-tab"),
|
||||
],
|
||||
id="data-source-tabs",
|
||||
active_tab="file-tab",
|
||||
)
|
||||
]
|
||||
),
|
||||
dbc.CardBody([html.Div(id="tab-content")]),
|
||||
]
|
||||
)
|
||||
|
||||
def create_file_upload_tab(self):
|
||||
"""Create file upload tab content."""
|
||||
return html.Div(
|
||||
[
|
||||
self.upload_component.create_error_alert(),
|
||||
self.upload_component.create_data_upload(),
|
||||
self.upload_component.create_prompts_upload(),
|
||||
self.upload_component.create_reset_button(),
|
||||
]
|
||||
)
|
||||
|
||||
def create_opensearch_tab(self):
|
||||
"""Create OpenSearch tab content."""
|
||||
return html.Div(
|
||||
[
|
||||
# Connection section
|
||||
html.H6("Connection", className="mb-2"),
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("OpenSearch URL:"),
|
||||
dbc.Input(
|
||||
id="opensearch-url",
|
||||
type="text",
|
||||
placeholder="https://opensearch.example.com:9200",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=12,
|
||||
),
|
||||
]
|
||||
),
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Index Name:"),
|
||||
dbc.Input(
|
||||
id="opensearch-index",
|
||||
type="text",
|
||||
placeholder="my-embeddings-index",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Button(
|
||||
"Test Connection",
|
||||
id="test-connection-btn",
|
||||
color="primary",
|
||||
size="sm",
|
||||
className="mt-4",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
className="d-flex align-items-end",
|
||||
),
|
||||
]
|
||||
),
|
||||
# Authentication section (collapsible)
|
||||
dbc.Collapse(
|
||||
[
|
||||
html.Hr(),
|
||||
html.H6("Authentication (Optional)", className="mb-2"),
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Username:"),
|
||||
dbc.Input(
|
||||
id="opensearch-username",
|
||||
type="text",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Password:"),
|
||||
dbc.Input(
|
||||
id="opensearch-password",
|
||||
type="password",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
]
|
||||
),
|
||||
dbc.Label("OR"),
|
||||
dbc.Input(
|
||||
id="opensearch-api-key",
|
||||
type="text",
|
||||
placeholder="API Key",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
id="auth-collapse",
|
||||
is_open=False,
|
||||
),
|
||||
dbc.Button(
|
||||
"Show Authentication",
|
||||
id="auth-toggle",
|
||||
color="link",
|
||||
size="sm",
|
||||
className="p-0 mb-3",
|
||||
),
|
||||
# Connection status
|
||||
html.Div(id="connection-status", className="mb-3"),
|
||||
# Field mapping section (hidden initially)
|
||||
html.Div(id="field-mapping-section", style={"display": "none"}),
|
||||
|
||||
# Hidden dropdowns to prevent callback errors
|
||||
html.Div([
|
||||
dcc.Dropdown(id="embedding-field-dropdown", style={"display": "none"}),
|
||||
dcc.Dropdown(id="text-field-dropdown", style={"display": "none"}),
|
||||
dcc.Dropdown(id="id-field-dropdown", style={"display": "none"}),
|
||||
dcc.Dropdown(id="category-field-dropdown", style={"display": "none"}),
|
||||
dcc.Dropdown(id="subcategory-field-dropdown", style={"display": "none"}),
|
||||
dcc.Dropdown(id="tags-field-dropdown", style={"display": "none"}),
|
||||
], style={"display": "none"}),
|
||||
# Load data button (hidden initially)
|
||||
html.Div(
|
||||
[
|
||||
dbc.Button(
|
||||
"Load Data",
|
||||
id="load-opensearch-data-btn",
|
||||
color="success",
|
||||
className="mb-2",
|
||||
disabled=True,
|
||||
),
|
||||
],
|
||||
id="load-data-section",
|
||||
style={"display": "none"},
|
||||
),
|
||||
# OpenSearch status/results
|
||||
html.Div(id="opensearch-status", className="mb-3"),
|
||||
]
|
||||
)
|
||||
|
||||
def create_field_mapping_interface(self, field_suggestions):
|
||||
"""Create field mapping interface based on detected fields."""
|
||||
return html.Div(
|
||||
[
|
||||
html.Hr(),
|
||||
html.H6("Field Mapping", className="mb-2"),
|
||||
html.P(
|
||||
"Map your OpenSearch fields to the required format:",
|
||||
className="text-muted small",
|
||||
),
|
||||
# Required fields
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label(
|
||||
"Embedding Field (required):", className="fw-bold"
|
||||
),
|
||||
dcc.Dropdown(
|
||||
id="embedding-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("embedding", [])
|
||||
],
|
||||
value=field_suggestions.get("embedding", [None])[0], # Default to first suggestion
|
||||
placeholder="Select embedding field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label(
|
||||
"Text Field (required):", className="fw-bold"
|
||||
),
|
||||
dcc.Dropdown(
|
||||
id="text-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("text", [])
|
||||
],
|
||||
value=field_suggestions.get("text", [None])[0], # Default to first suggestion
|
||||
placeholder="Select text field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
]
|
||||
),
|
||||
# Optional fields
|
||||
html.H6("Optional Fields", className="mb-2 mt-3"),
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("ID Field:"),
|
||||
dcc.Dropdown(
|
||||
id="id-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("id", [])
|
||||
],
|
||||
value=field_suggestions.get("id", [None])[0], # Default to first suggestion
|
||||
placeholder="Select ID field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Category Field:"),
|
||||
dcc.Dropdown(
|
||||
id="category-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("category", [])
|
||||
],
|
||||
value=field_suggestions.get("category", [None])[0], # Default to first suggestion
|
||||
placeholder="Select category field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
]
|
||||
),
|
||||
dbc.Row(
|
||||
[
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Subcategory Field:"),
|
||||
dcc.Dropdown(
|
||||
id="subcategory-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("subcategory", [])
|
||||
],
|
||||
value=field_suggestions.get("subcategory", [None])[0], # Default to first suggestion
|
||||
placeholder="Select subcategory field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
dbc.Col(
|
||||
[
|
||||
dbc.Label("Tags Field:"),
|
||||
dcc.Dropdown(
|
||||
id="tags-field-dropdown-ui",
|
||||
options=[
|
||||
{"label": field, "value": field}
|
||||
for field in field_suggestions.get("tags", [])
|
||||
],
|
||||
value=field_suggestions.get("tags", [None])[0], # Default to first suggestion
|
||||
placeholder="Select tags field...",
|
||||
className="mb-2",
|
||||
),
|
||||
],
|
||||
width=6,
|
||||
),
|
||||
]
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
def create_error_alert(self):
|
||||
"""Create error alert component for OpenSearch issues."""
|
||||
return dbc.Alert(
|
||||
id="opensearch-error-alert",
|
||||
dismissable=True,
|
||||
is_open=False,
|
||||
color="danger",
|
||||
className="mb-3",
|
||||
)
|
||||
|
||||
def create_success_alert(self):
|
||||
"""Create success alert component for OpenSearch operations."""
|
||||
return dbc.Alert(
|
||||
id="opensearch-success-alert",
|
||||
dismissable=True,
|
||||
is_open=False,
|
||||
color="success",
|
||||
className="mb-3",
|
||||
)
|
||||
@@ -1,21 +1,22 @@
|
||||
from dash import dcc, html
|
||||
import dash_bootstrap_components as dbc
|
||||
from .upload import UploadComponent
|
||||
from .datasource import DataSourceComponent
|
||||
|
||||
|
||||
class SidebarComponent:
|
||||
def __init__(self):
|
||||
self.upload_component = UploadComponent()
|
||||
self.datasource_component = DataSourceComponent()
|
||||
|
||||
def create_layout(self):
|
||||
return dbc.Col(
|
||||
[
|
||||
html.H5("Upload Data", className="mb-3"),
|
||||
self.upload_component.create_error_alert(),
|
||||
self.upload_component.create_data_upload(),
|
||||
self.upload_component.create_prompts_upload(),
|
||||
self.upload_component.create_reset_button(),
|
||||
html.H5("Visualization Controls", className="mb-3"),
|
||||
html.H5("Data Sources", className="mb-3"),
|
||||
self.datasource_component.create_error_alert(),
|
||||
self.datasource_component.create_success_alert(),
|
||||
self.datasource_component.create_tabbed_interface(),
|
||||
html.H5("Visualization Controls", className="mb-3 mt-4"),
|
||||
]
|
||||
+ self._create_method_dropdown()
|
||||
+ self._create_color_dropdown()
|
||||
|
||||
Reference in New Issue
Block a user