diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fcebe89 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,76 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +.venv/ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +coverage.xml +*.cover + +# Development tools +.mypy_cache/ +.ruff_cache/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Git +.git/ +.gitignore + +# Documentation +*.md +!README.md + +# Docker +Dockerfile* +docker-compose*.yml +.dockerignore + +# Data files (may contain sensitive information) +*.ndjson +*.ldjson +*.json + +# Reports +*-report.json +bandit-report.json +safety-report.json + +# Screenshots +*.png +*.jpg +*.jpeg +*.gif + +# Logs +*.log + +# Temporary files +*.tmp +*.temp \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index ac3cab9..ebfb035 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,8 +21,23 @@ uv sync **Run the application:** +Development mode (with auto-reload): ```bash -uv run python main.py +uv run run_dev.py +``` + +Production mode (with Gunicorn WSGI server): +```bash +# First install production dependencies +uv sync --extra prod + +# Then run in production mode +uv run run_prod.py +``` + +Legacy mode (basic Dash server): +```bash +uv run main.py ``` The app will be available at http://127.0.0.1:8050 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f968012 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,73 @@ +# Two-stage Dockerfile for EmbeddingBuddy +# Stage 1: Builder +FROM python:3.11-slim as builder + +# Install system dependencies for building Python packages +RUN apt-get update && apt-get install -y \ + build-essential \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Install uv for dependency management +RUN pip install uv + +# Set working directory +WORKDIR /app + +# Copy dependency files +COPY pyproject.toml uv.lock ./ + +# Copy source code (needed for editable install) +COPY src/ src/ +COPY main.py . +COPY wsgi.py . +COPY run_prod.py . +COPY assets/ assets/ + +# Create virtual environment and install dependencies (including production extras) +RUN uv venv .venv +RUN uv sync --frozen --extra prod + +# Stage 2: Runtime +FROM python:3.11-slim as runtime + +# Install runtime dependencies for compiled packages +RUN apt-get update && apt-get install -y \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy virtual environment from builder stage +COPY --from=builder /app/.venv /app/.venv + +# Copy application files from builder stage +COPY --from=builder /app/src /app/src +COPY --from=builder /app/main.py /app/main.py +COPY --from=builder /app/assets /app/assets +COPY --from=builder /app/wsgi.py /app/wsgi.py +COPY --from=builder /app/run_prod.py /app/run_prod.py + +# Make sure the virtual environment is in PATH +ENV PATH="/app/.venv/bin:$PATH" + +# Set Python path +ENV PYTHONPATH="/app/src:$PYTHONPATH" + +# Environment variables for production +ENV EMBEDDINGBUDDY_HOST=0.0.0.0 +ENV EMBEDDINGBUDDY_PORT=8050 +ENV EMBEDDINGBUDDY_DEBUG=false +ENV EMBEDDINGBUDDY_ENV=production + +# Expose port +EXPOSE 8050 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD python -c "import requests; requests.get('http://localhost:8050/', timeout=5)" || exit 1 + +# Run application with Gunicorn in production +CMD ["python", "run_prod.py"] \ No newline at end of file diff --git a/README.md b/README.md index 507961c..1e42d41 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ EmbeddingBuddy provides an intuitive web interface for analyzing high-dimensiona embedding vectors by applying various dimensionality reduction algorithms and visualizing the results in interactive 2D and 3D plots. The application features a clean, modular architecture that makes it easy to test, maintain, and extend -with new features. It supports dual dataset visualization, allowing you to compare +with new features. It supports dual dataset visualization, allowing you to compare documents and prompts to understand how queries relate to your content. ## Features @@ -73,17 +73,77 @@ uv sync 2. **Run the application:** +**Development mode** (with auto-reload): + ```bash -uv run python main.py +uv run run_dev.py ``` -3. **Open your browser** to http://127.0.0.1:8050 +**Production mode** (with Gunicorn WSGI server): + +```bash +# Install production dependencies +uv sync --extra prod + +# Run in production mode +uv run run_prod.py +``` + +**Legacy mode** (basic Dash server): + +```bash +uv run main.py +``` + +3. **Open your browser** to 4. **Test with sample data**: - Upload `sample_data.ndjson` (documents) - Upload `sample_prompts.ndjson` (prompts) to see dual visualization - Use the "Show prompts" toggle to compare how prompts relate to documents +## Docker + +You can also run EmbeddingBuddy using Docker: + +### Basic Usage + +```bash +# Run in the background +docker compose up -d +``` + +The application will be available at + +### With OpenSearch + +To run with OpenSearch for enhanced search capabilities: + +```bash +# Run in the background with OpenSearch +docker compose --profile opensearch up -d +``` + +This will start both the EmbeddingBuddy application and an OpenSearch instance. +OpenSearch will be available at + +### Docker Commands + +```bash +# Stop all services +docker compose down + +# Stop and remove volumes +docker compose down -v + +# View logs +docker compose logs embeddingbuddy +docker compose logs opensearch + +# Rebuild containers +docker compose build +``` + ## Development ### Project Structure diff --git a/bump_version.py b/bump_version.py new file mode 100755 index 0000000..3c2c5e8 --- /dev/null +++ b/bump_version.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Version bump script for EmbeddingBuddy. +Automatically updates version in pyproject.toml following semantic versioning. +""" +import argparse +import re +import sys +from pathlib import Path + + +def get_current_version(pyproject_path: Path) -> str: + """Extract current version from pyproject.toml.""" + content = pyproject_path.read_text() + match = re.search(r'version\s*=\s*"([^"]+)"', content) + if not match: + raise ValueError("Could not find version in pyproject.toml") + return match.group(1) + + +def parse_version(version_str: str) -> tuple[int, int, int]: + """Parse semantic version string into major, minor, patch tuple.""" + match = re.match(r'(\d+)\.(\d+)\.(\d+)', version_str) + if not match: + raise ValueError(f"Invalid version format: {version_str}") + return int(match.group(1)), int(match.group(2)), int(match.group(3)) + + +def bump_version(current: str, bump_type: str) -> str: + """Bump version based on type (major, minor, patch).""" + major, minor, patch = parse_version(current) + + if bump_type == "major": + return f"{major + 1}.0.0" + elif bump_type == "minor": + return f"{major}.{minor + 1}.0" + elif bump_type == "patch": + return f"{major}.{minor}.{patch + 1}" + else: + raise ValueError(f"Invalid bump type: {bump_type}") + + +def update_version_in_file(pyproject_path: Path, new_version: str) -> None: + """Update version in pyproject.toml file.""" + content = pyproject_path.read_text() + updated_content = re.sub( + r'version\s*=\s*"[^"]+"', + f'version = "{new_version}"', + content + ) + pyproject_path.write_text(updated_content) + + +def main(): + """Main version bump function.""" + parser = argparse.ArgumentParser( + description="Bump version in pyproject.toml", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python bump_version.py patch # 0.3.0 -> 0.3.1 + python bump_version.py minor # 0.3.0 -> 0.4.0 + python bump_version.py major # 0.3.0 -> 1.0.0 + python bump_version.py --set 1.2.3 # Set specific version + +Semantic versioning guide: + - patch: Bug fixes, no API changes + - minor: New features, backward compatible + - major: Breaking changes, not backward compatible + """ + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "bump_type", + nargs="?", + choices=["major", "minor", "patch"], + help="Type of version bump" + ) + group.add_argument( + "--set", + dest="set_version", + help="Set specific version (e.g., 1.2.3)" + ) + + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be changed without making changes" + ) + + args = parser.parse_args() + + # Find pyproject.toml + pyproject_path = Path("pyproject.toml") + if not pyproject_path.exists(): + print("āŒ pyproject.toml not found in current directory") + sys.exit(1) + + try: + current_version = get_current_version(pyproject_path) + print(f"šŸ“¦ Current version: {current_version}") + + if args.set_version: + # Validate the set version format + parse_version(args.set_version) + new_version = args.set_version + else: + new_version = bump_version(current_version, args.bump_type) + + print(f"šŸš€ New version: {new_version}") + + if args.dry_run: + print("šŸ” Dry run - no changes made") + else: + update_version_in_file(pyproject_path, new_version) + print("āœ… Version updated in pyproject.toml") + print() + print("šŸ’” Next steps:") + print(" 1. Review changes: git diff") + print(" 2. Commit changes: git add . && git commit -m 'bump version to {}'".format(new_version)) + print(" 3. Tag release: git tag v{}".format(new_version)) + + except ValueError as e: + print(f"āŒ Error: {e}") + sys.exit(1) + except Exception as e: + print(f"āŒ Unexpected error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..488be2a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,69 @@ +services: + opensearch: + image: opensearchproject/opensearch:2 + container_name: embeddingbuddy-opensearch + profiles: + - opensearch + environment: + - cluster.name=embeddingbuddy-cluster + - node.name=embeddingbuddy-node + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - "DISABLE_INSTALL_DEMO_CONFIG=true" + - "DISABLE_SECURITY_PLUGIN=true" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data:/usr/share/opensearch/data + ports: + - "9200:9200" + - "9600:9600" + networks: + - embeddingbuddy + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + + embeddingbuddy: + build: . + container_name: embeddingbuddy-app + environment: + - EMBEDDINGBUDDY_HOST=0.0.0.0 + - EMBEDDINGBUDDY_PORT=8050 + - EMBEDDINGBUDDY_DEBUG=false + - OPENSEARCH_HOST=opensearch + - OPENSEARCH_PORT=9200 + - OPENSEARCH_SCHEME=http + - OPENSEARCH_VERIFY_CERTS=false + ports: + - "8050:8050" + networks: + - embeddingbuddy + depends_on: + opensearch: + condition: service_healthy + required: false + healthcheck: + test: ["CMD-SHELL", "python -c 'import requests; requests.get(\"http://localhost:8050/\", timeout=5)'"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + restart: unless-stopped + +volumes: + opensearch-data: + driver: local + +networks: + embeddingbuddy: + driver: bridge \ No newline at end of file diff --git a/embedding-buddy-screenshot.png b/embedding-buddy-screenshot.png index 8ecee66..12efa76 100644 Binary files a/embedding-buddy-screenshot.png and b/embedding-buddy-screenshot.png differ diff --git a/pyproject.toml b/pyproject.toml index d3fbf7b..a17ec5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "embeddingbuddy" -version = "0.3.0" +version = "0.4.0" description = "A Python Dash application for interactive exploration and visualization of embedding vectors through dimensionality reduction techniques." readme = "README.md" requires-python = ">=3.11" @@ -12,7 +12,6 @@ dependencies = [ "scikit-learn>=1.3.2", "dash-bootstrap-components>=1.5.0", "umap-learn>=0.5.8", - "numba>=0.56.4", "openTSNE>=1.0.0", "mypy>=1.17.1", "opensearch-py>=3.0.0", @@ -32,11 +31,14 @@ security = [ "safety>=2.3.0", "pip-audit>=2.6.0", ] +prod = [ + "gunicorn>=21.2.0", +] dev = [ "embeddingbuddy[test,lint,security]", ] all = [ - "embeddingbuddy[test,lint,security]", + "embeddingbuddy[test,lint,security,prod]", ] [build-system] diff --git a/run_dev.py b/run_dev.py new file mode 100644 index 0000000..525efea --- /dev/null +++ b/run_dev.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +""" +Development runner with auto-reload enabled. +This runs the Dash development server with hot reloading. +""" +import os +from src.embeddingbuddy.app import create_app, run_app + +def main(): + """Run the application in development mode with auto-reload.""" + # Force development settings + os.environ["EMBEDDINGBUDDY_ENV"] = "development" + os.environ["EMBEDDINGBUDDY_DEBUG"] = "true" + + print("šŸš€ Starting EmbeddingBuddy in development mode...") + print("šŸ“ Auto-reload enabled - changes will trigger restart") + print("šŸŒ Server will be available at http://127.0.0.1:8050") + print("ā¹ļø Press Ctrl+C to stop") + + app = create_app() + + # Run with development server (includes auto-reload when debug=True) + run_app(app, debug=True) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/run_prod.py b/run_prod.py new file mode 100644 index 0000000..0ee464d --- /dev/null +++ b/run_prod.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Production runner using Gunicorn WSGI server. +This provides better performance and stability for production deployments. +""" +import os +import subprocess +import sys +from src.embeddingbuddy.config.settings import AppSettings + +def main(): + """Run the application in production mode with Gunicorn.""" + # Force production settings + os.environ["EMBEDDINGBUDDY_ENV"] = "production" + os.environ["EMBEDDINGBUDDY_DEBUG"] = "false" + + print("šŸš€ Starting EmbeddingBuddy in production mode...") + print(f"āš™ļø Workers: {AppSettings.GUNICORN_WORKERS}") + print(f"šŸŒ Server will be available at http://{AppSettings.GUNICORN_BIND}") + print("ā¹ļø Press Ctrl+C to stop") + + # Gunicorn command + cmd = [ + "gunicorn", + "--workers", str(AppSettings.GUNICORN_WORKERS), + "--bind", AppSettings.GUNICORN_BIND, + "--timeout", str(AppSettings.GUNICORN_TIMEOUT), + "--keep-alive", str(AppSettings.GUNICORN_KEEPALIVE), + "--access-logfile", "-", + "--error-logfile", "-", + "--log-level", "info", + "wsgi:application" + ] + + try: + subprocess.run(cmd, check=True) + except KeyboardInterrupt: + print("\nšŸ›‘ Shutting down...") + sys.exit(0) + except subprocess.CalledProcessError as e: + print(f"āŒ Error running Gunicorn: {e}") + sys.exit(1) + except FileNotFoundError: + print("āŒ Gunicorn not found. Install it with: uv add gunicorn") + print("šŸ’” Or run in development mode with: python run_dev.py") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/embeddingbuddy/config/settings.py b/src/embeddingbuddy/config/settings.py index 3f60ab2..1c345c5 100644 --- a/src/embeddingbuddy/config/settings.py +++ b/src/embeddingbuddy/config/settings.py @@ -73,6 +73,17 @@ class AppSettings: HOST = os.getenv("EMBEDDINGBUDDY_HOST", "127.0.0.1") PORT = int(os.getenv("EMBEDDINGBUDDY_PORT", "8050")) + # Environment Configuration + ENVIRONMENT = os.getenv( + "EMBEDDINGBUDDY_ENV", "development" + ) # development, production + + # WSGI Server Configuration (for production) + GUNICORN_WORKERS = int(os.getenv("GUNICORN_WORKERS", "4")) + GUNICORN_BIND = os.getenv("GUNICORN_BIND", f"{HOST}:{PORT}") + GUNICORN_TIMEOUT = int(os.getenv("GUNICORN_TIMEOUT", "120")) + GUNICORN_KEEPALIVE = int(os.getenv("GUNICORN_KEEPALIVE", "5")) + # OpenSearch Configuration OPENSEARCH_DEFAULT_SIZE = 100 OPENSEARCH_SAMPLE_SIZE = 5 diff --git a/uv.lock b/uv.lock index 0bff82e..c9424db 100644 --- a/uv.lock +++ b/uv.lock @@ -412,13 +412,12 @@ wheels = [ [[package]] name = "embeddingbuddy" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "dash" }, { name = "dash-bootstrap-components" }, { name = "mypy" }, - { name = "numba" }, { name = "numpy" }, { name = "opensearch-py" }, { name = "opentsne" }, @@ -431,6 +430,7 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "bandit" }, + { name = "gunicorn" }, { name = "mypy" }, { name = "pip-audit" }, { name = "pytest" }, @@ -451,6 +451,9 @@ lint = [ { name = "mypy" }, { name = "ruff" }, ] +prod = [ + { name = "gunicorn" }, +] security = [ { name = "bandit" }, { name = "pip-audit" }, @@ -466,11 +469,11 @@ requires-dist = [ { name = "bandit", extras = ["toml"], marker = "extra == 'security'", specifier = ">=1.7.5" }, { name = "dash", specifier = ">=2.17.1" }, { name = "dash-bootstrap-components", specifier = ">=1.5.0" }, - { name = "embeddingbuddy", extras = ["test", "lint", "security"], marker = "extra == 'all'" }, { name = "embeddingbuddy", extras = ["test", "lint", "security"], marker = "extra == 'dev'" }, + { name = "embeddingbuddy", extras = ["test", "lint", "security", "prod"], marker = "extra == 'all'" }, + { name = "gunicorn", marker = "extra == 'prod'", specifier = ">=21.2.0" }, { name = "mypy", specifier = ">=1.17.1" }, { name = "mypy", marker = "extra == 'lint'", specifier = ">=1.5.0" }, - { name = "numba", specifier = ">=0.56.4" }, { name = "numpy", specifier = ">=1.24.4" }, { name = "opensearch-py", specifier = ">=3.0.0" }, { name = "opentsne", specifier = ">=1.0.0" }, @@ -484,7 +487,7 @@ requires-dist = [ { name = "scikit-learn", specifier = ">=1.3.2" }, { name = "umap-learn", specifier = ">=0.5.8" }, ] -provides-extras = ["test", "lint", "security", "dev", "all"] +provides-extras = ["test", "lint", "security", "prod", "dev", "all"] [[package]] name = "events" @@ -520,6 +523,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/68/9d4508e893976286d2ead7f8f571314af6c2037af34853a30fd769c02e9d/flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c", size = 103305, upload-time = "2025-05-13T15:01:15.591Z" }, ] +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031, upload-time = "2024-08-10T20:25:27.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029, upload-time = "2024-08-10T20:25:24.996Z" }, +] + [[package]] name = "h11" version = "0.16.0" diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..0c26210 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,20 @@ +""" +WSGI entry point for production deployment. +Use this with a production WSGI server like Gunicorn. +""" +from src.embeddingbuddy.app import create_app + +# Create the application instance +application = create_app() + +# For compatibility with different WSGI servers +app = application + +if __name__ == "__main__": + # This won't be used in production, but useful for testing + from src.embeddingbuddy.config.settings import AppSettings + application.run( + host=AppSettings.HOST, + port=AppSettings.PORT, + debug=False + ) \ No newline at end of file