add in browser embedding generation

assets/embeddings.js

@@ -0,0 +1,278 @@
+// Text input embedding generation using Transformers.js
+// This module runs entirely in the browser for privacy and performance
+
+// Global flag to track initialization
+window.transformersLoading = false;
+window.transformersLoaded = false;
+
+class TransformersEmbedder {
+    constructor() {
+        this.extractor = null;
+        this.currentModel = null;
+        this.modelCache = new Map();
+        this.isLoading = false;
+    }
+    
+    async initializeModel(modelName = 'Xenova/all-MiniLM-L6-v2') {
+        try {
+            if (this.modelCache.has(modelName)) {
+                this.extractor = this.modelCache.get(modelName);
+                this.currentModel = modelName;
+                return { success: true, model: modelName };
+            }
+            
+            if (this.isLoading) {
+                return { success: false, error: 'Model loading already in progress' };
+            }
+            
+            this.isLoading = true;
+            
+            // Use globally loaded Transformers.js pipeline
+            if (!window.transformers) {
+                if (!window.transformersPipeline) {
+                    // Wait for the pipeline to load
+                    let attempts = 0;
+                    while (!window.transformersPipeline && attempts < 50) { // Wait up to 5 seconds
+                        await new Promise(resolve => setTimeout(resolve, 100));
+                        attempts++;
+                    }
+                    if (!window.transformersPipeline) {
+                        throw new Error('Transformers.js pipeline not available. Please refresh the page.');
+                    }
+                }
+                window.transformers = { pipeline: window.transformersPipeline };
+                window.transformersLoaded = true;
+                console.log('✅ Using globally loaded Transformers.js pipeline');
+            }
+            
+            // Show loading progress to user
+            if (window.updateModelLoadingProgress) {
+                window.updateModelLoadingProgress(0, `Loading ${modelName}...`);
+            }
+            
+            this.extractor = await window.transformers.pipeline('feature-extraction', modelName, {
+                progress_callback: (data) => {
+                    if (window.updateModelLoadingProgress && data.progress !== undefined) {
+                        const progress = Math.round(data.progress);
+                        window.updateModelLoadingProgress(progress, data.status || 'Loading...');
+                    }
+                }
+            });
+            
+            this.modelCache.set(modelName, this.extractor);
+            this.currentModel = modelName;
+            this.isLoading = false;
+            
+            if (window.updateModelLoadingProgress) {
+                window.updateModelLoadingProgress(100, 'Model loaded successfully');
+            }
+            
+            return { success: true, model: modelName };
+        } catch (error) {
+            this.isLoading = false;
+            console.error('Model initialization error:', error);
+            return { success: false, error: error.message };
+        }
+    }
+    
+    async generateEmbeddings(texts, options = {}) {
+        if (!this.extractor) {
+            throw new Error('Model not initialized. Call initializeModel() first.');
+        }
+        
+        if (!texts || texts.length === 0) {
+            throw new Error('No texts provided for embedding generation.');
+        }
+        
+        const embeddings = [];
+        const defaultOptions = { 
+            pooling: 'mean', 
+            normalize: true,
+            ...options 
+        };
+        
+        // Process in batches to avoid memory issues
+        const batchSize = options.batchSize || 8;
+        
+        try {
+            for (let i = 0; i < texts.length; i += batchSize) {
+                const batch = texts.slice(i, i + batchSize);
+                
+                const batchResults = await Promise.all(
+                    batch.map(text => {
+                        if (!text || text.trim().length === 0) {
+                            throw new Error('Empty text found in batch');
+                        }
+                        return this.extractor(text.trim(), defaultOptions);
+                    })
+                );
+                
+                // Convert tensor output to arrays
+                batchResults.forEach((result, idx) => {
+                    if (result && result.data) {
+                        embeddings.push(Array.from(result.data));
+                    } else {
+                        throw new Error(`Invalid embedding result for text: ${batch[idx]}`);
+                    }
+                });
+                
+                // Update progress
+                const progress = Math.min(100, ((i + batch.length) / texts.length) * 100);
+                if (window.updateEmbeddingProgress) {
+                    window.updateEmbeddingProgress(progress, `Processing ${i + batch.length}/${texts.length} texts`);
+                }
+            }
+            
+            if (window.updateEmbeddingProgress) {
+                window.updateEmbeddingProgress(100, `Generated ${embeddings.length} embeddings successfully`);
+            }
+            
+            return embeddings;
+        } catch (error) {
+            console.error('Embedding generation error:', error);
+            throw error;
+        }
+    }
+}
+
+// Global instance
+window.transformersEmbedder = new TransformersEmbedder();
+console.log('📦 TransformersEmbedder instance created');
+
+// Global progress update functions
+window.updateModelLoadingProgress = function(progress, status) {
+    const progressBar = document.getElementById('model-loading-progress');
+    const statusText = document.getElementById('model-loading-status');
+    if (progressBar) {
+        progressBar.style.width = progress + '%';
+        progressBar.setAttribute('aria-valuenow', progress);
+    }
+    if (statusText) {
+        statusText.textContent = status;
+    }
+};
+
+window.updateEmbeddingProgress = function(progress, status) {
+    const progressBar = document.getElementById('embedding-progress');
+    const statusText = document.getElementById('embedding-status');
+    if (progressBar) {
+        progressBar.style.width = progress + '%';
+        progressBar.setAttribute('aria-valuenow', progress);
+    }
+    if (statusText) {
+        statusText.textContent = status;
+    }
+};
+
+// Dash clientside callback functions
+window.dash_clientside = window.dash_clientside || {};
+console.log('🔧 Setting up window.dash_clientside.transformers');
+window.dash_clientside.transformers = {
+    generateEmbeddings: async function(nClicks, textContent, modelName, tokenizationMethod, category, subcategory) {
+        console.log('🚀 generateEmbeddings called with:', { nClicks, modelName, tokenizationMethod, textLength: textContent?.length });
+        
+        if (!nClicks || !textContent || textContent.trim().length === 0) {
+            console.log('⚠️ Early return - missing required parameters');
+            return window.dash_clientside.no_update;
+        }
+        
+        try {
+            // Initialize model if needed
+            const initResult = await window.transformersEmbedder.initializeModel(modelName);
+            if (!initResult.success) {
+                return [
+                    { error: initResult.error },
+                    `❌ Model loading error: ${initResult.error}`,
+                    "danger",
+                    false
+                ];
+            }
+            
+            // Tokenize text based on method
+            let textChunks;
+            const trimmedText = textContent.trim();
+            
+            switch (tokenizationMethod) {
+                case 'sentence':
+                    // Simple sentence splitting - can be enhanced with proper NLP
+                    textChunks = trimmedText
+                        .split(/[.!?]+/)
+                        .map(s => s.trim())
+                        .filter(s => s.length > 0);
+                    break;
+                case 'paragraph':
+                    textChunks = trimmedText
+                        .split(/\n\s*\n/)
+                        .map(s => s.trim())
+                        .filter(s => s.length > 0);
+                    break;
+                case 'manual':
+                    textChunks = trimmedText
+                        .split('\n')
+                        .map(s => s.trim())
+                        .filter(s => s.length > 0);
+                    break;
+                default:
+                    textChunks = [trimmedText];
+            }
+            
+            if (textChunks.length === 0) {
+                return [
+                    { error: 'No valid text chunks found after tokenization' },
+                    '❌ Error: No valid text chunks found after tokenization',
+                    "danger",
+                    false
+                ];
+            }
+            
+            // Generate embeddings
+            const embeddings = await window.transformersEmbedder.generateEmbeddings(textChunks);
+            
+            if (!embeddings || embeddings.length !== textChunks.length) {
+                return [
+                    { error: 'Embedding generation failed - mismatch in text chunks and embeddings' },
+                    '❌ Error: Embedding generation failed',
+                    "danger",
+                    false
+                ];
+            }
+            
+            // Create documents structure
+            const documents = textChunks.map((text, i) => ({
+                id: `text_input_${Date.now()}_${i}`,
+                text: text,
+                embedding: embeddings[i],
+                category: category || "Text Input",
+                subcategory: subcategory || "Generated",
+                tags: []
+            }));
+            
+            return [
+                {
+                    documents: documents,
+                    embeddings: embeddings
+                },
+                `✅ Generated embeddings for ${documents.length} text chunks using ${modelName}`,
+                "success",
+                false
+            ];
+            
+        } catch (error) {
+            console.error('Client-side embedding error:', error);
+            return [
+                { error: error.message },
+                `❌ Error: ${error.message}`,
+                "danger", 
+                false
+            ];
+        }
+    }
+};
+
+console.log('✅ Transformers.js client-side setup complete');
+console.log('Available:', {
+    transformersEmbedder: !!window.transformersEmbedder,
+    dashClientside: !!window.dash_clientside,
+    transformersModule: !!window.dash_clientside?.transformers,
+    generateFunction: typeof window.dash_clientside?.transformers?.generateEmbeddings
+});

assets/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "embeddingbuddy-assets",
+  "version": "1.0.0",
+  "description": "JavaScript dependencies for EmbeddingBuddy text input functionality",
+  "dependencies": {
+    "@huggingface/transformers": "^3.0.0"
+  },
+  "type": "module"
+}

assets/sample-txt.md

@@ -0,0 +1,106 @@
+The sun peeked through the clouds after a drizzly morning.
+A gentle breeze rustled the leaves as we walked along the shoreline.
+Heavy rains caused flooding in several low-lying neighborhoods.
+It was so hot that even the birds sought shade under the palm trees.
+By midnight, the temperature had dropped below freezing.
+Thunderstorms lit up the sky with flashes of lightning.
+A thick fog settled over the city streets at dawn.
+The air smelled of ozone after the sudden hailstorm.
+I watched the snowflakes drift silently onto the ground.
+A double rainbow appeared after the rain shower.
+The humidity soared to uncomfortable levels by midday.
+Dust devils formed in the dry desert plains.
+The barometer readings indicated an approaching front.
+A sudden gust of wind knocked over the garden chairs.
+Light drizzle turned into a torrential downpour within minutes.
+The new smartphone features a foldable display and 5G connectivity.
+In the world of AI, transformers have revolutionized natural language processing.
+Quantum computing promises to solve problems beyond classical computers' reach.
+Blockchain technology is being explored for secure voting systems.
+Virtual reality headsets are becoming more affordable and accessible.
+The rise of electric vehicles is reshaping the automotive industry.
+Cloud computing allows businesses to scale resources dynamically.
+Machine learning algorithms can now predict stock market trends with surprising accuracy.
+Augmented reality applications are transforming retail experiences.
+The Internet of Things connects everyday devices to the web for smarter living.
+Cybersecurity threats are evolving, requiring constant vigilance.
+3D printing is enabling rapid prototyping and custom manufacturing.
+Edge computing reduces latency by processing data closer to the source.
+Biometric authentication methods are enhancing security in devices.
+Wearable technology is tracking health metrics in real-time.
+Artificial intelligence is being used to create realistic deepfakes.
+Preheat the oven to 375°F before you start mixing the batter.
+She finely chopped the garlic and sautéed it in two tablespoons of olive oil.
+A pinch of saffron adds a beautiful color and aroma to traditional paella.
+If the soup is too salty, add a peeled potato to absorb excess sodium.
+Let the bread dough rise for at least an hour in a warm, draft-free spot.
+Marinate the chicken overnight in a blend of citrus and spices.
+Use a cast-iron skillet to sear the steak on high heat.
+Whisk the egg whites until they form stiff peaks.
+Fold in the chocolate chips gently to keep the batter airy.
+Brush the pastry with an egg wash for a golden finish.
+Slow-roast the pork shoulder until it falls off the bone.
+Garnish the salad with toasted nuts and fresh herbs.
+Deglaze the pan with white wine for a rich sauce.
+Simmer the curry paste until the aroma intensifies.
+Let the risotto rest before serving to thicken slightly.
+He dribbled past two defenders and sank a three-pointer at the buzzer.
+The marathon runner kept a steady pace despite the sweltering heat.
+Their home team clinched the championship with a last-minute goal.
+NASCAR fans cheered as the cars roared around the oval track.
+She landed a perfect triple axel at the figure skating championship.
+The cyclist pedaled up the steep hill in record time.
+He pitched a no-hitter during the high school baseball game.
+The quarterback threw a touchdown pass under heavy pressure.
+They scored a hat-trick in the hockey final.
+The boxer delivered a swift uppercut in the final round.
+Surfers caught massive waves at dawn on the Pacific coast.
+Fans erupted when the underdog scored the winning goal.
+The swimmer broke the national record in the 200m freestyle.
+The gymnast executed a flawless routine on the balance beam.
+The rugby team celebrated their victory with a traditional haka.
+The stock market rallied after positive earnings reports.
+Investors are closely watching interest rate changes by the Federal Reserve.
+Cryptocurrency prices have been extremely volatile this year.
+Diversification is key to managing investment risk effectively.
+Inflation rates have reached a 40-year high, impacting consumer spending.
+Many companies are adopting ESG criteria to attract socially conscious investors.
+The bond market is reacting to geopolitical tensions and supply chain disruptions.
+Venture capital funding for startups has surged in the tech sector.
+Exchange-traded funds (ETFs) offer a way to invest in diversified portfolios.
+The global economy is recovering from the pandemic, but challenges remain.
+Central banks are exploring digital currencies to modernize payment systems.
+Retail investors are increasingly participating in the stock market through apps.
+Hedge funds are using complex algorithms to gain an edge in trading.
+Real estate prices have skyrocketed in urban areas due to low inventory.
+The startup raised $10 million in its Series A funding round.
+The symphony orchestra played a hauntingly beautiful melody.
+She strummed her guitar softly, filling the room with a warm sound.
+The DJ mixed tracks seamlessly, keeping the crowd dancing all night.
+His voice soared during the high notes of the ballad.
+The band played an acoustic set in the intimate coffee shop.
+Jazz musicians often improvise solos based on the chord changes.
+The opera singer hit the high C with perfect pitch.
+The choir harmonized beautifully, filling the church with sound.
+He composed a symphony that was performed at the concert hall.
+The singer-songwriter wrote heartfelt lyrics about love and loss.
+The rock band headlined the festival, drawing a massive crowd.
+Hip-hop artists use rhythm and rhyme to tell powerful stories.
+The violinist played a virtuosic solo that left the audience in awe.
+Folk music often reflects the culture and traditions of a community.
+The gospel choir lifted spirits with their uplifting performance.
+The fall of the Berlin Wall in 1989 marked the end of the Cold War.
+Ancient Egypt's pyramids are a testament to their architectural prowess.
+Europe's Renaissance period sparked a revival in art and science.
+The signing of the Declaration of Independence in 1776 established the United States.
+The Industrial Revolution transformed economies and societies worldwide.
+Rome was the center of a vast empire that influenced law and governance.
+The discovery of the New World by Christopher Columbus in 1492 changed global trade.
+The French Revolution in 1789 led to significant political and social change.
+World War II was a global conflict that reshaped international relations.
+The fall of the Roman Empire in 476 AD marked the beginning of the Middle Ages.
+The invention of the printing press revolutionized the spread of knowledge.
+The Cold War was characterized by political tension between the U.S. and the Soviet Union.
+The ancient Silk Road connected East and West through trade routes.
+The signing of the Magna Carta in 1215 established principles of due process.
+Exploration during the Age of Discovery expanded European empires across the globe.

assets/transformers-loader.js

@@ -0,0 +1,172 @@
+// Simple script to load Transformers.js from CDN and initialize embedding functionality
+// This approach uses traditional script loading instead of ES6 modules
+
+console.log('🔧 Transformers.js loader starting...');
+
+// Global state
+window.transformersLibraryLoaded = false;
+window.transformersLibraryLoading = false;
+
+// Function to dynamically load a script
+function loadScript(src) {
+    return new Promise((resolve, reject) => {
+        const script = document.createElement('script');
+        script.src = src;
+        script.type = 'module';
+        script.onload = () => resolve();
+        script.onerror = () => reject(new Error(`Failed to load script: ${src}`));
+        document.head.appendChild(script);
+    });
+}
+
+// Function to initialize Transformers.js
+async function initializeTransformers() {
+    if (window.transformersLibraryLoaded) {
+        console.log('✅ Transformers.js already loaded');
+        return true;
+    }
+    
+    if (window.transformersLibraryLoading) {
+        console.log('⏳ Transformers.js already loading, waiting...');
+        // Wait for loading to complete
+        while (window.transformersLibraryLoading) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        return window.transformersLibraryLoaded;
+    }
+    
+    window.transformersLibraryLoading = true;
+    
+    try {
+        console.log('📦 Loading Transformers.js from CDN...');
+        
+        // Use dynamic import since this is more reliable with ES modules
+        const transformers = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0');
+        window.transformersLibrary = transformers;
+        window.transformersLibraryLoaded = true;
+        
+        console.log('✅ Transformers.js loaded successfully');
+        return true;
+    } catch (error) {
+        console.error('❌ Failed to load Transformers.js:', error);
+        return false;
+    } finally {
+        window.transformersLibraryLoading = false;
+    }
+}
+
+// Simple embeddings class
+class SimpleEmbedder {
+    constructor() {
+        this.pipeline = null;
+        this.modelCache = new Map();
+    }
+    
+    async generateEmbeddings(texts, modelName = 'Xenova/all-MiniLM-L6-v2') {
+        console.log('🔄 Generating embeddings for', texts.length, 'texts with model', modelName);
+        
+        // Ensure Transformers.js is loaded
+        if (!window.transformersLibraryLoaded) {
+            const loaded = await initializeTransformers();
+            if (!loaded) {
+                throw new Error('Failed to load Transformers.js');
+            }
+        }
+        
+        // Create pipeline if not cached
+        if (!this.modelCache.has(modelName)) {
+            console.log('🏗️ Creating pipeline for', modelName);
+            const { pipeline } = window.transformersLibrary;
+            this.pipeline = await pipeline('feature-extraction', modelName);
+            this.modelCache.set(modelName, this.pipeline);
+        } else {
+            this.pipeline = this.modelCache.get(modelName);
+        }
+        
+        // Generate embeddings
+        const embeddings = [];
+        for (let i = 0; i < texts.length; i++) {
+            console.log(`Processing text ${i + 1}/${texts.length}...`);
+            const result = await this.pipeline(texts[i], { pooling: 'mean', normalize: true });
+            embeddings.push(Array.from(result.data));
+        }
+        
+        console.log('✅ Generated', embeddings.length, 'embeddings');
+        return embeddings;
+    }
+}
+
+// Create global instance
+window.simpleEmbedder = new SimpleEmbedder();
+
+// Set up Dash clientside callbacks
+window.dash_clientside = window.dash_clientside || {};
+window.dash_clientside.transformers = {
+    generateEmbeddings: async function(nClicks, textContent, modelName, tokenizationMethod, category, subcategory) {
+        console.log('🚀 Client-side generateEmbeddings called');
+        
+        if (!nClicks || !textContent || textContent.trim().length === 0) {
+            console.log('⚠️ Missing required parameters');
+            return window.dash_clientside.no_update;
+        }
+        
+        try {
+            // Tokenize text
+            let textChunks;
+            const trimmedText = textContent.trim();
+            
+            switch (tokenizationMethod) {
+                case 'sentence':
+                    textChunks = trimmedText.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 0);
+                    break;
+                case 'paragraph':
+                    textChunks = trimmedText.split(/\n\s*\n/).map(s => s.trim()).filter(s => s.length > 0);
+                    break;
+                case 'manual':
+                    textChunks = trimmedText.split('\n').map(s => s.trim()).filter(s => s.length > 0);
+                    break;
+                default:
+                    textChunks = [trimmedText];
+            }
+            
+            if (textChunks.length === 0) {
+                throw new Error('No valid text chunks after tokenization');
+            }
+            
+            // Generate embeddings
+            const embeddings = await window.simpleEmbedder.generateEmbeddings(textChunks, modelName);
+            
+            // Create documents
+            const documents = textChunks.map((text, i) => ({
+                id: `text_input_${Date.now()}_${i}`,
+                text: text,
+                embedding: embeddings[i],
+                category: category || "Text Input",
+                subcategory: subcategory || "Generated", 
+                tags: []
+            }));
+            
+            return [
+                {
+                    documents: documents,
+                    embeddings: embeddings
+                },
+                `✅ Generated embeddings for ${documents.length} text chunks using ${modelName}`,
+                "success",
+                false
+            ];
+            
+        } catch (error) {
+            console.error('❌ Error generating embeddings:', error);
+            return [
+                { error: error.message },
+                `❌ Error: ${error.message}`,
+                "danger",
+                false
+            ];
+        }
+    }
+};
+
+console.log('✅ Simple Transformers.js setup complete');
+console.log('Available functions:', Object.keys(window.dash_clientside.transformers));