relocate assets so they get budled correctly
All checks were successful
All checks were successful
This commit is contained in:
@@ -16,9 +16,9 @@ def create_app():
|
||||
from .ui.callbacks.visualization import VisualizationCallbacks
|
||||
from .ui.callbacks.interactions import InteractionCallbacks
|
||||
|
||||
# Get the project root directory (two levels up from this file)
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
|
||||
assets_path = os.path.join(project_root, "assets")
|
||||
# Get the assets directory relative to this module
|
||||
module_dir = os.path.dirname(__file__)
|
||||
assets_path = os.path.join(module_dir, "assets")
|
||||
|
||||
app = dash.Dash(
|
||||
__name__,
|
||||
@@ -167,8 +167,15 @@ def serve(host=None, port=None, dev=False, debug=False):
|
||||
|
||||
# Only print startup messages in main process (not in Flask reloader)
|
||||
if not os.environ.get("WERKZEUG_RUN_MAIN"):
|
||||
from importlib.metadata import version
|
||||
|
||||
try:
|
||||
pkg_version = version("embeddingbuddy")
|
||||
except Exception:
|
||||
pkg_version = "unknown"
|
||||
|
||||
mode = "development" if dev else ("debug" if debug else "production")
|
||||
print(f"Starting EmbeddingBuddy in {mode} mode...")
|
||||
print(f"Starting EmbeddingBuddy v{pkg_version} in {mode} mode...")
|
||||
print("Loading dependencies (this may take a few seconds)...")
|
||||
print(f"Server will start at http://{actual_host}:{actual_port}")
|
||||
if use_reloader:
|
||||
|
17
src/embeddingbuddy/assets/custom.css
Normal file
17
src/embeddingbuddy/assets/custom.css
Normal file
@@ -0,0 +1,17 @@
|
||||
/* CSS override for transparent hover boxes in Plotly plots */
|
||||
|
||||
/* Make hover boxes transparent while preserving text readability */
|
||||
.hovertext {
|
||||
fill-opacity: 0.8 !important;
|
||||
stroke-opacity: 1 !important;
|
||||
}
|
||||
|
||||
/* Alternative selector for different Plotly versions */
|
||||
g.hovertext > path {
|
||||
opacity: 0.8 !important;
|
||||
}
|
||||
|
||||
/* Ensure text remains fully visible */
|
||||
.hovertext text {
|
||||
opacity: 1 !important;
|
||||
}
|
225
src/embeddingbuddy/assets/embeddings.js
Normal file
225
src/embeddingbuddy/assets/embeddings.js
Normal file
@@ -0,0 +1,225 @@
|
||||
// Text input embedding generation using Transformers.js
|
||||
// This module runs entirely in the browser for privacy and performance
|
||||
|
||||
// Global flag to track initialization
|
||||
window.transformersLoading = false;
|
||||
window.transformersLoaded = false;
|
||||
|
||||
class TransformersEmbedder {
|
||||
constructor() {
|
||||
this.extractor = null;
|
||||
this.currentModel = null;
|
||||
this.modelCache = new Map();
|
||||
this.isLoading = false;
|
||||
}
|
||||
|
||||
async initializeModel(modelName = 'Xenova/all-MiniLM-L6-v2') {
|
||||
try {
|
||||
if (this.modelCache.has(modelName)) {
|
||||
this.extractor = this.modelCache.get(modelName);
|
||||
this.currentModel = modelName;
|
||||
return { success: true, model: modelName };
|
||||
}
|
||||
|
||||
if (this.isLoading) {
|
||||
return { success: false, error: 'Model loading already in progress' };
|
||||
}
|
||||
|
||||
this.isLoading = true;
|
||||
|
||||
// Use globally loaded Transformers.js pipeline
|
||||
if (!window.transformers) {
|
||||
if (!window.transformersPipeline) {
|
||||
// Wait for the pipeline to load
|
||||
let attempts = 0;
|
||||
while (!window.transformersPipeline && attempts < 50) { // Wait up to 5 seconds
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
attempts++;
|
||||
}
|
||||
if (!window.transformersPipeline) {
|
||||
throw new Error('Transformers.js pipeline not available. Please refresh the page.');
|
||||
}
|
||||
}
|
||||
window.transformers = { pipeline: window.transformersPipeline };
|
||||
window.transformersLoaded = true;
|
||||
console.log('✅ Using globally loaded Transformers.js pipeline');
|
||||
}
|
||||
|
||||
this.extractor = await window.transformers.pipeline('feature-extraction', modelName);
|
||||
|
||||
this.modelCache.set(modelName, this.extractor);
|
||||
this.currentModel = modelName;
|
||||
this.isLoading = false;
|
||||
|
||||
return { success: true, model: modelName };
|
||||
} catch (error) {
|
||||
this.isLoading = false;
|
||||
console.error('Model initialization error:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async generateEmbeddings(texts, options = {}) {
|
||||
if (!this.extractor) {
|
||||
throw new Error('Model not initialized. Call initializeModel() first.');
|
||||
}
|
||||
|
||||
if (!texts || texts.length === 0) {
|
||||
throw new Error('No texts provided for embedding generation.');
|
||||
}
|
||||
|
||||
const embeddings = [];
|
||||
const defaultOptions = {
|
||||
pooling: 'mean',
|
||||
normalize: true,
|
||||
...options
|
||||
};
|
||||
|
||||
// Process in batches to avoid memory issues
|
||||
const batchSize = options.batchSize || 8;
|
||||
|
||||
try {
|
||||
for (let i = 0; i < texts.length; i += batchSize) {
|
||||
const batch = texts.slice(i, i + batchSize);
|
||||
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(text => {
|
||||
if (!text || text.trim().length === 0) {
|
||||
throw new Error('Empty text found in batch');
|
||||
}
|
||||
return this.extractor(text.trim(), defaultOptions);
|
||||
})
|
||||
);
|
||||
|
||||
// Convert tensor output to arrays
|
||||
batchResults.forEach((result, idx) => {
|
||||
if (result && result.data) {
|
||||
embeddings.push(Array.from(result.data));
|
||||
} else {
|
||||
throw new Error(`Invalid embedding result for text: ${batch[idx]}`);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
return embeddings;
|
||||
} catch (error) {
|
||||
console.error('Embedding generation error:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global instance
|
||||
window.transformersEmbedder = new TransformersEmbedder();
|
||||
console.log('📦 TransformersEmbedder instance created');
|
||||
|
||||
|
||||
// Dash clientside callback functions
|
||||
window.dash_clientside = window.dash_clientside || {};
|
||||
console.log('🔧 Setting up window.dash_clientside.transformers');
|
||||
window.dash_clientside.transformers = {
|
||||
generateEmbeddings: async function(nClicks, textContent, modelName, tokenizationMethod, category, subcategory) {
|
||||
console.log('🚀 generateEmbeddings called with:', { nClicks, modelName, tokenizationMethod, textLength: textContent?.length });
|
||||
|
||||
if (!nClicks || !textContent || textContent.trim().length === 0) {
|
||||
console.log('⚠️ Early return - missing required parameters');
|
||||
return window.dash_clientside.no_update;
|
||||
}
|
||||
|
||||
try {
|
||||
// Initialize model if needed
|
||||
const initResult = await window.transformersEmbedder.initializeModel(modelName);
|
||||
if (!initResult.success) {
|
||||
return [
|
||||
{ error: `Model loading error: ${initResult.error}` },
|
||||
false
|
||||
];
|
||||
}
|
||||
|
||||
// Tokenize text based on method
|
||||
let textChunks;
|
||||
const trimmedText = textContent.trim();
|
||||
|
||||
switch (tokenizationMethod) {
|
||||
case 'sentence':
|
||||
textChunks = trimmedText
|
||||
.split(/[.!?]+/)
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0);
|
||||
break;
|
||||
case 'paragraph':
|
||||
textChunks = trimmedText
|
||||
.split(/\n\s*\n/)
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0);
|
||||
break;
|
||||
case 'manual':
|
||||
textChunks = trimmedText
|
||||
.split('\n')
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0);
|
||||
break;
|
||||
default:
|
||||
textChunks = [trimmedText];
|
||||
}
|
||||
|
||||
if (textChunks.length === 0) {
|
||||
return [
|
||||
{ error: 'No valid text chunks found after tokenization' },
|
||||
false
|
||||
];
|
||||
}
|
||||
|
||||
// Generate embeddings
|
||||
const embeddings = await window.transformersEmbedder.generateEmbeddings(textChunks);
|
||||
|
||||
if (!embeddings || embeddings.length !== textChunks.length) {
|
||||
return [
|
||||
{ error: 'Embedding generation failed' },
|
||||
false
|
||||
];
|
||||
}
|
||||
|
||||
// Create documents structure
|
||||
const documents = textChunks.map((text, i) => ({
|
||||
id: `text_input_${Date.now()}_${i}`,
|
||||
text: text,
|
||||
embedding: embeddings[i],
|
||||
category: category || "Text Input",
|
||||
subcategory: subcategory || "Generated",
|
||||
tags: []
|
||||
}));
|
||||
|
||||
// Return the successful embeddings data
|
||||
const embeddingsData = {
|
||||
documents: documents,
|
||||
embeddings: embeddings
|
||||
};
|
||||
|
||||
console.log('✅ Embeddings generated successfully:', embeddingsData);
|
||||
|
||||
return [
|
||||
embeddingsData,
|
||||
false
|
||||
];
|
||||
|
||||
} catch (error) {
|
||||
console.error('Client-side embedding error:', error);
|
||||
return [
|
||||
{ error: error.message },
|
||||
false
|
||||
];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
console.log('✅ Transformers.js client-side setup complete');
|
||||
console.log('Available:', {
|
||||
transformersEmbedder: !!window.transformersEmbedder,
|
||||
dashClientside: !!window.dash_clientside,
|
||||
transformersModule: !!window.dash_clientside?.transformers,
|
||||
generateFunction: typeof window.dash_clientside?.transformers?.generateEmbeddings,
|
||||
processAsync: typeof window.processEmbeddingsAsync
|
||||
});
|
2
src/embeddingbuddy/assets/fontawesome.css
vendored
Normal file
2
src/embeddingbuddy/assets/fontawesome.css
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/* Load Font Awesome from local assets */
|
||||
@import url("/assets/fontawesome/css/all.min.css");
|
165
src/embeddingbuddy/assets/fontawesome/LICENSE.txt
Normal file
165
src/embeddingbuddy/assets/fontawesome/LICENSE.txt
Normal file
@@ -0,0 +1,165 @@
|
||||
Fonticons, Inc. (https://fontawesome.com)
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
Font Awesome Free License
|
||||
|
||||
Font Awesome Free is free, open source, and GPL friendly. You can use it for
|
||||
commercial projects, open source projects, or really almost whatever you want.
|
||||
Full Font Awesome Free license: https://fontawesome.com/license/free.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
# Icons: CC BY 4.0 License (https://creativecommons.org/licenses/by/4.0/)
|
||||
|
||||
The Font Awesome Free download is licensed under a Creative Commons
|
||||
Attribution 4.0 International License and applies to all icons packaged
|
||||
as SVG and JS file types.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
# Fonts: SIL OFL 1.1 License
|
||||
|
||||
In the Font Awesome Free download, the SIL OFL license applies to all icons
|
||||
packaged as web and desktop font files.
|
||||
|
||||
Copyright (c) 2023 Fonticons, Inc. (https://fontawesome.com)
|
||||
with Reserved Font Name: "Font Awesome".
|
||||
|
||||
This Font Software is licensed under the SIL Open Font License, Version 1.1.
|
||||
This license is copied below, and is also available with a FAQ at:
|
||||
http://scripts.sil.org/OFL
|
||||
|
||||
SIL OPEN FONT LICENSE
|
||||
Version 1.1 - 26 February 2007
|
||||
|
||||
PREAMBLE
|
||||
The goals of the Open Font License (OFL) are to stimulate worldwide
|
||||
development of collaborative font projects, to support the font creation
|
||||
efforts of academic and linguistic communities, and to provide a free and
|
||||
open framework in which fonts may be shared and improved in partnership
|
||||
with others.
|
||||
|
||||
The OFL allows the licensed fonts to be used, studied, modified and
|
||||
redistributed freely as long as they are not sold by themselves. The
|
||||
fonts, including any derivative works, can be bundled, embedded,
|
||||
redistributed and/or sold with any software provided that any reserved
|
||||
names are not used by derivative works. The fonts and derivatives,
|
||||
however, cannot be released under any other type of license. The
|
||||
requirement for fonts to remain under this license does not apply
|
||||
to any document created using the fonts or their derivatives.
|
||||
|
||||
DEFINITIONS
|
||||
"Font Software" refers to the set of files released by the Copyright
|
||||
Holder(s) under this license and clearly marked as such. This may
|
||||
include source files, build scripts and documentation.
|
||||
|
||||
"Reserved Font Name" refers to any names specified as such after the
|
||||
copyright statement(s).
|
||||
|
||||
"Original Version" refers to the collection of Font Software components as
|
||||
distributed by the Copyright Holder(s).
|
||||
|
||||
"Modified Version" refers to any derivative made by adding to, deleting,
|
||||
or substituting — in part or in whole — any of the components of the
|
||||
Original Version, by changing formats or by porting the Font Software to a
|
||||
new environment.
|
||||
|
||||
"Author" refers to any designer, engineer, programmer, technical
|
||||
writer or other person who contributed to the Font Software.
|
||||
|
||||
PERMISSION & CONDITIONS
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of the Font Software, to use, study, copy, merge, embed, modify,
|
||||
redistribute, and sell modified and unmodified copies of the Font
|
||||
Software, subject to the following conditions:
|
||||
|
||||
1) Neither the Font Software nor any of its individual components,
|
||||
in Original or Modified Versions, may be sold by itself.
|
||||
|
||||
2) Original or Modified Versions of the Font Software may be bundled,
|
||||
redistributed and/or sold with any software, provided that each copy
|
||||
contains the above copyright notice and this license. These can be
|
||||
included either as stand-alone text files, human-readable headers or
|
||||
in the appropriate machine-readable metadata fields within text or
|
||||
binary files as long as those fields can be easily viewed by the user.
|
||||
|
||||
3) No Modified Version of the Font Software may use the Reserved Font
|
||||
Name(s) unless explicit written permission is granted by the corresponding
|
||||
Copyright Holder. This restriction only applies to the primary font name as
|
||||
presented to the users.
|
||||
|
||||
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
|
||||
Software shall not be used to promote, endorse or advertise any
|
||||
Modified Version, except to acknowledge the contribution(s) of the
|
||||
Copyright Holder(s) and the Author(s) or with their explicit written
|
||||
permission.
|
||||
|
||||
5) The Font Software, modified or unmodified, in part or in whole,
|
||||
must be distributed entirely under this license, and must not be
|
||||
distributed under any other license. The requirement for fonts to
|
||||
remain under this license does not apply to any document created
|
||||
using the Font Software.
|
||||
|
||||
TERMINATION
|
||||
This license becomes null and void if any of the above conditions are
|
||||
not met.
|
||||
|
||||
DISCLAIMER
|
||||
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
|
||||
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
|
||||
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
|
||||
OTHER DEALINGS IN THE FONT SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
# Code: MIT License (https://opensource.org/licenses/MIT)
|
||||
|
||||
In the Font Awesome Free download, the MIT license applies to all non-font and
|
||||
non-icon files.
|
||||
|
||||
Copyright 2023 Fonticons, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in the
|
||||
Software without restriction, including without limitation the rights to use, copy,
|
||||
modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
and to permit persons to whom the Software is furnished to do so, subject to the
|
||||
following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
||||
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
# Attribution
|
||||
|
||||
Attribution is required by MIT, SIL OFL, and CC BY licenses. Downloaded Font
|
||||
Awesome Free files already contain embedded comments with sufficient
|
||||
attribution, so you shouldn't need to do anything additional when using these
|
||||
files normally.
|
||||
|
||||
We've kept attribution comments terse, so we ask that you do not actively work
|
||||
to remove them from files, especially code. They're a great way for folks to
|
||||
learn about Font Awesome.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
# Brand Icons
|
||||
|
||||
All brand icons are trademarks of their respective owners. The use of these
|
||||
trademarks does not indicate endorsement of the trademark holder by Font
|
||||
Awesome, nor vice versa. **Please do not use brand logos for any purpose except
|
||||
to represent the company, product, or service to which they refer.**
|
9
src/embeddingbuddy/assets/fontawesome/css/all.min.css
vendored
Normal file
9
src/embeddingbuddy/assets/fontawesome/css/all.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
BIN
src/embeddingbuddy/assets/fontawesome/webfonts/fa-brands-400.ttf
Normal file
BIN
src/embeddingbuddy/assets/fontawesome/webfonts/fa-brands-400.ttf
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
src/embeddingbuddy/assets/fontawesome/webfonts/fa-solid-900.ttf
Normal file
BIN
src/embeddingbuddy/assets/fontawesome/webfonts/fa-solid-900.ttf
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
9
src/embeddingbuddy/assets/package.json
Normal file
9
src/embeddingbuddy/assets/package.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"name": "embeddingbuddy-assets",
|
||||
"version": "1.0.0",
|
||||
"description": "JavaScript dependencies for EmbeddingBuddy text input functionality",
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.0.0"
|
||||
},
|
||||
"type": "module"
|
||||
}
|
106
src/embeddingbuddy/assets/sample-txt.md
Normal file
106
src/embeddingbuddy/assets/sample-txt.md
Normal file
@@ -0,0 +1,106 @@
|
||||
The sun peeked through the clouds after a drizzly morning.
|
||||
A gentle breeze rustled the leaves as we walked along the shoreline.
|
||||
Heavy rains caused flooding in several low-lying neighborhoods.
|
||||
It was so hot that even the birds sought shade under the palm trees.
|
||||
By midnight, the temperature had dropped below freezing.
|
||||
Thunderstorms lit up the sky with flashes of lightning.
|
||||
A thick fog settled over the city streets at dawn.
|
||||
The air smelled of ozone after the sudden hailstorm.
|
||||
I watched the snowflakes drift silently onto the ground.
|
||||
A double rainbow appeared after the rain shower.
|
||||
The humidity soared to uncomfortable levels by midday.
|
||||
Dust devils formed in the dry desert plains.
|
||||
The barometer readings indicated an approaching front.
|
||||
A sudden gust of wind knocked over the garden chairs.
|
||||
Light drizzle turned into a torrential downpour within minutes.
|
||||
The new smartphone features a foldable display and 5G connectivity.
|
||||
In the world of AI, transformers have revolutionized natural language processing.
|
||||
Quantum computing promises to solve problems beyond classical computers' reach.
|
||||
Blockchain technology is being explored for secure voting systems.
|
||||
Virtual reality headsets are becoming more affordable and accessible.
|
||||
The rise of electric vehicles is reshaping the automotive industry.
|
||||
Cloud computing allows businesses to scale resources dynamically.
|
||||
Machine learning algorithms can now predict stock market trends with surprising accuracy.
|
||||
Augmented reality applications are transforming retail experiences.
|
||||
The Internet of Things connects everyday devices to the web for smarter living.
|
||||
Cybersecurity threats are evolving, requiring constant vigilance.
|
||||
3D printing is enabling rapid prototyping and custom manufacturing.
|
||||
Edge computing reduces latency by processing data closer to the source.
|
||||
Biometric authentication methods are enhancing security in devices.
|
||||
Wearable technology is tracking health metrics in real-time.
|
||||
Artificial intelligence is being used to create realistic deepfakes.
|
||||
Preheat the oven to 375°F before you start mixing the batter.
|
||||
She finely chopped the garlic and sautéed it in two tablespoons of olive oil.
|
||||
A pinch of saffron adds a beautiful color and aroma to traditional paella.
|
||||
If the soup is too salty, add a peeled potato to absorb excess sodium.
|
||||
Let the bread dough rise for at least an hour in a warm, draft-free spot.
|
||||
Marinate the chicken overnight in a blend of citrus and spices.
|
||||
Use a cast-iron skillet to sear the steak on high heat.
|
||||
Whisk the egg whites until they form stiff peaks.
|
||||
Fold in the chocolate chips gently to keep the batter airy.
|
||||
Brush the pastry with an egg wash for a golden finish.
|
||||
Slow-roast the pork shoulder until it falls off the bone.
|
||||
Garnish the salad with toasted nuts and fresh herbs.
|
||||
Deglaze the pan with white wine for a rich sauce.
|
||||
Simmer the curry paste until the aroma intensifies.
|
||||
Let the risotto rest before serving to thicken slightly.
|
||||
He dribbled past two defenders and sank a three-pointer at the buzzer.
|
||||
The marathon runner kept a steady pace despite the sweltering heat.
|
||||
Their home team clinched the championship with a last-minute goal.
|
||||
NASCAR fans cheered as the cars roared around the oval track.
|
||||
She landed a perfect triple axel at the figure skating championship.
|
||||
The cyclist pedaled up the steep hill in record time.
|
||||
He pitched a no-hitter during the high school baseball game.
|
||||
The quarterback threw a touchdown pass under heavy pressure.
|
||||
They scored a hat-trick in the hockey final.
|
||||
The boxer delivered a swift uppercut in the final round.
|
||||
Surfers caught massive waves at dawn on the Pacific coast.
|
||||
Fans erupted when the underdog scored the winning goal.
|
||||
The swimmer broke the national record in the 200m freestyle.
|
||||
The gymnast executed a flawless routine on the balance beam.
|
||||
The rugby team celebrated their victory with a traditional haka.
|
||||
The stock market rallied after positive earnings reports.
|
||||
Investors are closely watching interest rate changes by the Federal Reserve.
|
||||
Cryptocurrency prices have been extremely volatile this year.
|
||||
Diversification is key to managing investment risk effectively.
|
||||
Inflation rates have reached a 40-year high, impacting consumer spending.
|
||||
Many companies are adopting ESG criteria to attract socially conscious investors.
|
||||
The bond market is reacting to geopolitical tensions and supply chain disruptions.
|
||||
Venture capital funding for startups has surged in the tech sector.
|
||||
Exchange-traded funds (ETFs) offer a way to invest in diversified portfolios.
|
||||
The global economy is recovering from the pandemic, but challenges remain.
|
||||
Central banks are exploring digital currencies to modernize payment systems.
|
||||
Retail investors are increasingly participating in the stock market through apps.
|
||||
Hedge funds are using complex algorithms to gain an edge in trading.
|
||||
Real estate prices have skyrocketed in urban areas due to low inventory.
|
||||
The startup raised $10 million in its Series A funding round.
|
||||
The symphony orchestra played a hauntingly beautiful melody.
|
||||
She strummed her guitar softly, filling the room with a warm sound.
|
||||
The DJ mixed tracks seamlessly, keeping the crowd dancing all night.
|
||||
His voice soared during the high notes of the ballad.
|
||||
The band played an acoustic set in the intimate coffee shop.
|
||||
Jazz musicians often improvise solos based on the chord changes.
|
||||
The opera singer hit the high C with perfect pitch.
|
||||
The choir harmonized beautifully, filling the church with sound.
|
||||
He composed a symphony that was performed at the concert hall.
|
||||
The singer-songwriter wrote heartfelt lyrics about love and loss.
|
||||
The rock band headlined the festival, drawing a massive crowd.
|
||||
Hip-hop artists use rhythm and rhyme to tell powerful stories.
|
||||
The violinist played a virtuosic solo that left the audience in awe.
|
||||
Folk music often reflects the culture and traditions of a community.
|
||||
The gospel choir lifted spirits with their uplifting performance.
|
||||
The fall of the Berlin Wall in 1989 marked the end of the Cold War.
|
||||
Ancient Egypt's pyramids are a testament to their architectural prowess.
|
||||
Europe's Renaissance period sparked a revival in art and science.
|
||||
The signing of the Declaration of Independence in 1776 established the United States.
|
||||
The Industrial Revolution transformed economies and societies worldwide.
|
||||
Rome was the center of a vast empire that influenced law and governance.
|
||||
The discovery of the New World by Christopher Columbus in 1492 changed global trade.
|
||||
The French Revolution in 1789 led to significant political and social change.
|
||||
World War II was a global conflict that reshaped international relations.
|
||||
The fall of the Roman Empire in 476 AD marked the beginning of the Middle Ages.
|
||||
The invention of the printing press revolutionized the spread of knowledge.
|
||||
The Cold War was characterized by political tension between the U.S. and the Soviet Union.
|
||||
The ancient Silk Road connected East and West through trade routes.
|
||||
The signing of the Magna Carta in 1215 established principles of due process.
|
||||
Exploration during the Age of Discovery expanded European empires across the globe.
|
188
src/embeddingbuddy/assets/transformers-loader.js
Normal file
188
src/embeddingbuddy/assets/transformers-loader.js
Normal file
@@ -0,0 +1,188 @@
|
||||
// Simple script to load Transformers.js from CDN and initialize embedding functionality
|
||||
// This approach uses traditional script loading instead of ES6 modules
|
||||
|
||||
console.log('🔧 Transformers.js loader starting...');
|
||||
|
||||
// Global state
|
||||
window.transformersLibraryLoaded = false;
|
||||
window.transformersLibraryLoading = false;
|
||||
|
||||
// Function to dynamically load a script
|
||||
function loadScript(src) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const script = document.createElement('script');
|
||||
script.src = src;
|
||||
script.type = 'module';
|
||||
script.onload = () => resolve();
|
||||
script.onerror = () => reject(new Error(`Failed to load script: ${src}`));
|
||||
document.head.appendChild(script);
|
||||
});
|
||||
}
|
||||
|
||||
// Function to initialize Transformers.js
|
||||
async function initializeTransformers() {
|
||||
if (window.transformersLibraryLoaded) {
|
||||
console.log('✅ Transformers.js already loaded');
|
||||
return true;
|
||||
}
|
||||
|
||||
if (window.transformersLibraryLoading) {
|
||||
console.log('⏳ Transformers.js already loading, waiting...');
|
||||
// Wait for loading to complete
|
||||
while (window.transformersLibraryLoading) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
return window.transformersLibraryLoaded;
|
||||
}
|
||||
|
||||
window.transformersLibraryLoading = true;
|
||||
|
||||
try {
|
||||
console.log('📦 Loading Transformers.js from CDN...');
|
||||
|
||||
// Use dynamic import since this is more reliable with ES modules
|
||||
const transformers = await import('https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.0.0');
|
||||
window.transformersLibrary = transformers;
|
||||
window.transformersLibraryLoaded = true;
|
||||
|
||||
console.log('✅ Transformers.js loaded successfully');
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to load Transformers.js:', error);
|
||||
return false;
|
||||
} finally {
|
||||
window.transformersLibraryLoading = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Simple embeddings class
|
||||
class SimpleEmbedder {
|
||||
constructor() {
|
||||
this.pipeline = null;
|
||||
this.modelCache = new Map();
|
||||
}
|
||||
|
||||
async generateEmbeddings(texts, modelName = 'Xenova/all-MiniLM-L6-v2') {
|
||||
console.log('🔄 Generating embeddings for', texts.length, 'texts with model', modelName);
|
||||
|
||||
// Ensure Transformers.js is loaded
|
||||
if (!window.transformersLibraryLoaded) {
|
||||
const loaded = await initializeTransformers();
|
||||
if (!loaded) {
|
||||
throw new Error('Failed to load Transformers.js');
|
||||
}
|
||||
}
|
||||
|
||||
// Create pipeline if not cached
|
||||
if (!this.modelCache.has(modelName)) {
|
||||
console.log('🏗️ Creating pipeline for', modelName);
|
||||
const { pipeline } = window.transformersLibrary;
|
||||
this.pipeline = await pipeline('feature-extraction', modelName);
|
||||
this.modelCache.set(modelName, this.pipeline);
|
||||
} else {
|
||||
this.pipeline = this.modelCache.get(modelName);
|
||||
}
|
||||
|
||||
// Generate embeddings
|
||||
const embeddings = [];
|
||||
for (let i = 0; i < texts.length; i++) {
|
||||
console.log(`Processing text ${i + 1}/${texts.length}...`);
|
||||
const result = await this.pipeline(texts[i], { pooling: 'mean', normalize: true });
|
||||
embeddings.push(Array.from(result.data));
|
||||
}
|
||||
|
||||
console.log('✅ Generated', embeddings.length, 'embeddings');
|
||||
return embeddings;
|
||||
}
|
||||
}
|
||||
|
||||
// Create global instance
|
||||
window.simpleEmbedder = new SimpleEmbedder();
|
||||
|
||||
// Set up Dash clientside callbacks
|
||||
window.dash_clientside = window.dash_clientside || {};
|
||||
window.dash_clientside.transformers = {
|
||||
generateEmbeddings: async function(nClicks, textContent, modelName, tokenizationMethod, category, subcategory) {
|
||||
console.log('🚀 Client-side generateEmbeddings called');
|
||||
|
||||
if (!nClicks || !textContent || textContent.trim().length === 0) {
|
||||
console.log('⚠️ Missing required parameters');
|
||||
return window.dash_clientside.no_update;
|
||||
}
|
||||
|
||||
try {
|
||||
// Ensure Transformers.js is loaded
|
||||
if (!window.transformersLibraryLoaded) {
|
||||
const loaded = await initializeTransformers();
|
||||
if (!loaded) {
|
||||
return [
|
||||
{ error: 'Failed to load Transformers.js' },
|
||||
false
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Tokenize text
|
||||
let textChunks;
|
||||
const trimmedText = textContent.trim();
|
||||
|
||||
switch (tokenizationMethod) {
|
||||
case 'sentence':
|
||||
textChunks = trimmedText.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 0);
|
||||
break;
|
||||
case 'paragraph':
|
||||
textChunks = trimmedText.split(/\n\s*\n/).map(s => s.trim()).filter(s => s.length > 0);
|
||||
break;
|
||||
case 'manual':
|
||||
textChunks = trimmedText.split('\n').map(s => s.trim()).filter(s => s.length > 0);
|
||||
break;
|
||||
default:
|
||||
textChunks = [trimmedText];
|
||||
}
|
||||
|
||||
if (textChunks.length === 0) {
|
||||
return [
|
||||
{ error: 'No valid text chunks after tokenization' },
|
||||
false
|
||||
];
|
||||
}
|
||||
|
||||
// Generate embeddings
|
||||
const embeddings = await window.simpleEmbedder.generateEmbeddings(textChunks, modelName);
|
||||
|
||||
// Create documents
|
||||
const documents = textChunks.map((text, i) => ({
|
||||
id: `text_input_${Date.now()}_${i}`,
|
||||
text: text,
|
||||
embedding: embeddings[i],
|
||||
category: category || "Text Input",
|
||||
subcategory: subcategory || "Generated",
|
||||
tags: []
|
||||
}));
|
||||
|
||||
// Return the successful embeddings data
|
||||
const embeddingsData = {
|
||||
documents: documents,
|
||||
embeddings: embeddings
|
||||
};
|
||||
|
||||
console.log('✅ Embeddings generated successfully:', embeddingsData);
|
||||
|
||||
return [
|
||||
embeddingsData,
|
||||
false
|
||||
];
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Error generating embeddings:', error);
|
||||
return [
|
||||
{ error: error.message },
|
||||
false
|
||||
];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
console.log('✅ Simple Transformers.js setup complete');
|
||||
console.log('Available functions:', Object.keys(window.dash_clientside.transformers));
|
Reference in New Issue
Block a user