Cosine Similarity and Vector Databases

import math

def cosine_similarity(vector1, vector2):
    """
    Calculate cosine similarity between two vectors
    Formula: cos(θ) = (A · B) / (||A|| × ||B||)
    
    Args:
        vector1: List of numbers
        vector2: List of numbers
    
    Returns:
        Float between -1 and 1
    """
    # Check if vectors have same length
    if len(vector1) != len(vector2):
        raise ValueError("Vectors must have the same length")
    
    # Calculate dot product
    dot_product = sum(a * b for a, b in zip(vector1, vector2))
    
    # Calculate magnitudes
    magnitude1 = math.sqrt(sum(x * x for x in vector1))
    magnitude2 = math.sqrt(sum(x * x for x in vector2))
    
    # Avoid division by zero
    if magnitude1 == 0 or magnitude2 == 0:
        return 0.0
    
    # Calculate cosine similarity
    return dot_product / (magnitude1 * magnitude2)

# Example usage
if __name__ == "__main__":
    # Test vectors
    vec1 = [1, 2, 3, 4, 5]
    vec2 = [2, 4, 6, 8, 10]
    vec3 = [1, 0, -1, 0, 1]
    
    print(f"Vector 1: {vec1}")
    print(f"Vector 2: {vec2}")
    print(f"Vector 3: {vec3}")
    print()
    
    print(f"Cosine similarity (vec1, vec2): {cosine_similarity(vec1, vec2):.4f}")
    print(f"Cosine similarity (vec1, vec3): {cosine_similarity(vec1, vec3):.4f}")
    print(f"Cosine similarity (vec2, vec3): {cosine_similarity(vec2, vec3):.4f}")

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

# Manual implementation of cosine similarity
def calculate_cosine_similarity(vec1, vec2):
    # Calculate dot product
    dot_product = np.dot(vec1, vec2)
    
    # Calculate magnitudes (norms)
    norm_vec1 = np.linalg.norm(vec1)
    norm_vec2 = np.linalg.norm(vec2)
    
    # Calculate cosine similarity
    cosine_sim = dot_product / (norm_vec1 * norm_vec2)
    return cosine_sim

# Example vectors (could represent document embeddings)
doc1 = np.array([1, 2, 3, 4, 5])
doc2 = np.array([2, 4, 6, 8, 10])  # Similar direction, different magnitude
doc3 = np.array([1, 0, -1, 0, 1])  # Different direction

print("Manual calculation:")
print(f"Similarity between document 1 and document 2: {calculate_cosine_similarity(doc1, doc2):.3f}")
print(f"Similarity between document 1 and document 3: {calculate_cosine_similarity(doc1, doc3):.3f}")

# Using sklearn for comparison
print("\nUsing sklearn:")
vectors = np.array([doc1, doc2, doc3])
similarity_matrix = cosine_similarity(vectors)
print(f"Similarity matrix:\n{similarity_matrix}")

# Simple vector store example
class SimpleVectorStore:
    def __init__(self):
        self.vectors = []
        self.metadata = []
    
    def add_document(self, vector, metadata):
        self.vectors.append(vector)
        self.metadata.append(metadata)
    
    def search(self, query_vector, top_k=3):
        if not self.vectors:
            return []
        
        # Calculate similarities with all stored vectors
        similarities = []
        for i, stored_vector in enumerate(self.vectors):
            sim = calculate_cosine_similarity(query_vector, stored_vector)
            similarities.append((sim, i, self.metadata[i]))
        
        # Sort by similarity (highest first) and return top_k
        similarities.sort(key=lambda x: x[0], reverse=True)
        return similarities[:top_k]

# Example usage of the vector store
store = SimpleVectorStore()

# Add some "documents" (represented as vectors)
store.add_document(np.array([1, 2, 3, 4, 5]), "Document about AI")
store.add_document(np.array([2, 4, 6, 8, 10]), "Document about machine learning")
store.add_document(np.array([1, 0, -1, 0, 1]), "Document about cooking")
store.add_document(np.array([5, 4, 3, 2, 1]), "Document about data science")

# Search with a query vector
query = np.array([1.5, 3, 4.5, 6, 7.5])
results = store.search(query, top_k=2)

print("\nVector store search results:")
for similarity, index, metadata in results:
    print(f"Similarity: {similarity:.3f} - {metadata}")

# Simplified attention calculation
attention_scores = torch.matmul(query, key.transpose(-2, -1)) / sqrt(d_k)
# This dot product is related to cosine similarity

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.utils.data import DataLoader, Dataset

# 1. Cosine Similarity Loss for Training Embeddings
class CosineSimilarityLoss(nn.Module):
    def __init__(self, margin=0.5):
        super().__init__()
        self.margin = margin
        
    def forward(self, anchor, positive, negative):
        """
        Triplet loss using cosine similarity
        anchor: current word embedding
        positive: context word (should be similar)
        negative: random word (should be dissimilar)
        """
        pos_sim = F.cosine_similarity(anchor, positive, dim=1)
        neg_sim = F.cosine_similarity(anchor, negative, dim=1)
        
        # We want positive similarity to be higher than negative by margin
        loss = torch.clamp(neg_sim - pos_sim + self.margin, min=0.0)
        return loss.mean()

# 2. Attention Mechanism Using Cosine Similarity
class CosineAttention(nn.Module):
    def __init__(self, embed_dim):
        super().__init__()
        self.embed_dim = embed_dim
        self.query_proj = nn.Linear(embed_dim, embed_dim)
        self.key_proj = nn.Linear(embed_dim, embed_dim)
        self.value_proj = nn.Linear(embed_dim, embed_dim)
        
    def forward(self, x):
        batch_size, seq_len, embed_dim = x.shape
        
        # Project to query, key, value
        Q = self.query_proj(x)  # [batch, seq_len, embed_dim]
        K = self.key_proj(x)
        V = self.value_proj(x)
        
        # Compute cosine similarity between queries and keys
        # Normalize Q and K to unit vectors
        Q_norm = F.normalize(Q, p=2, dim=-1)
        K_norm = F.normalize(K, p=2, dim=-1)
        
        # Cosine similarity = normalized dot product
        attention_scores = torch.matmul(Q_norm, K_norm.transpose(-2, -1))
        
        # Apply softmax to get attention weights
        attention_weights = F.softmax(attention_scores, dim=-1)
        
        # Apply attention to values
        output = torch.matmul(attention_weights, V)
        return output, attention_weights

# 3. Contrastive Learning for Language Model Pre-training
class ContrastiveLM(nn.Module):
    def __init__(self, vocab_size, embed_dim, hidden_dim):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.transformer = nn.TransformerEncoder(
            nn.TransformerEncoderLayer(embed_dim, nhead=8, dim_feedforward=hidden_dim),
            num_layers=6
        )
        self.projection_head = nn.Linear(embed_dim, embed_dim)
        
    def forward(self, input_ids):
        # Get embeddings and pass through transformer
        x = self.embedding(input_ids)
        x = self.transformer(x)
        
        # Use [CLS] token representation (first token)
        sentence_embedding = x[:, 0, :]
        
        # Project to contrastive learning space
        projected = self.projection_head(sentence_embedding)
        projected = F.normalize(projected, p=2, dim=1)  # L2 normalize
        
        return projected

def contrastive_loss(embeddings, temperature=0.1):
    """
    InfoNCE loss using cosine similarity
    Used in models like SimCSE for learning sentence representations
    """
    batch_size = embeddings.shape[0]
    
    # Compute cosine similarities between all pairs
    similarity_matrix = F.cosine_similarity(
        embeddings.unsqueeze(1), 
        embeddings.unsqueeze(0), 
        dim=2
    )
    
    # Scale by temperature
    similarity_matrix = similarity_matrix / temperature
    
    # Create labels (diagonal elements are positive pairs)
    labels = torch.arange(batch_size, device=embeddings.device)
    
    # Compute cross-entropy loss
    loss = F.cross_entropy(similarity_matrix, labels)
    return loss

# 4. Knowledge Distillation Using Cosine Similarity
class DistillationLoss(nn.Module):
    def __init__(self, temperature=3.0, alpha=0.7):
        super().__init__()
        self.temperature = temperature
        self.alpha = alpha
        
    def forward(self, student_logits, teacher_logits, true_labels):
        # Standard cross-entropy loss
        ce_loss = F.cross_entropy(student_logits, true_labels)
        
        # Distillation loss using cosine similarity between representations
        student_probs = F.softmax(student_logits / self.temperature, dim=1)
        teacher_probs = F.softmax(teacher_logits / self.temperature, dim=1)
        
        # Cosine similarity between probability distributions
        cosine_sim = F.cosine_similarity(student_probs, teacher_probs, dim=1)
        distill_loss = 1 - cosine_sim.mean()  # Convert similarity to loss
        
        # Combined loss
        total_loss = self.alpha * distill_loss + (1 - self.alpha) * ce_loss
        return total_loss

# 5. Training Loop Example
def train_contrastive_model():
    # Initialize model and optimizer
    model = ContrastiveLM(vocab_size=30000, embed_dim=768, hidden_dim=3072)
    optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5)
    
    # Dummy training loop
    for epoch in range(5):
        model.train()
        total_loss = 0
        
        # Simulate batch of sentence pairs (for contrastive learning)
        for batch_idx in range(100):  # 100 batches per epoch
            # Simulate input (batch of tokenized sentences)
            batch_size = 32
            seq_len = 128
            input_ids = torch.randint(0, 30000, (batch_size, seq_len))
            
            # Forward pass
            embeddings = model(input_ids)
            
            # Compute contrastive loss
            loss = contrastive_loss(embeddings)
            
            # Backward pass
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            total_loss += loss.item()
            
        avg_loss = total_loss / 100
        print(f"Epoch {epoch+1}, Average Loss: {avg_loss:.4f}")

# 6. Similarity-based Evaluation During Training
def evaluate_similarity_tasks(model, test_pairs):
    """
    Evaluate model on semantic similarity tasks during training
    """
    model.eval()
    similarities = []
    
    with torch.no_grad():
        for sentence1, sentence2, human_score in test_pairs:
            # Get embeddings for both sentences
            emb1 = model(sentence1.unsqueeze(0))
            emb2 = model(sentence2.unsqueeze(0))
            
            # Compute cosine similarity
            sim = F.cosine_similarity(emb1, emb2, dim=1).item()
            similarities.append((sim, human_score))
    
    # Compute correlation with human judgments
    model_scores = [s[0] for s in similarities]
    human_scores = [s[1] for s in similarities]
    correlation = np.corrcoef(model_scores, human_scores)[0, 1]
    
    return correlation

# 7. Regularization Using Cosine Similarity
class CosineRegularizer(nn.Module):
    def __init__(self, lambda_reg=0.01):
        super().__init__()
        self.lambda_reg = lambda_reg
        
    def forward(self, embeddings):
        """
        Encourage diversity in embeddings by penalizing high cosine similarities
        """
        # Compute pairwise cosine similarities
        norm_embeddings = F.normalize(embeddings, p=2, dim=1)
        similarity_matrix = torch.matmul(norm_embeddings, norm_embeddings.T)
        
        # Zero out diagonal (self-similarities)
        similarity_matrix = similarity_matrix - torch.diag(torch.diag(similarity_matrix))
        
        # Penalize high similarities (encourage diversity)
        reg_loss = self.lambda_reg * similarity_matrix.abs().mean()
        return reg_loss

# Example usage in training
if __name__ == "__main__":
    print("Training contrastive language model with cosine similarity...")
    train_contrastive_model()
    
    # Example of using cosine attention
    print("\nTesting cosine attention mechanism...")
    attention_layer = CosineAttention(embed_dim=768)
    sample_input = torch.randn(2, 10, 768)  # batch_size=2, seq_len=10, embed_dim=768
    output, weights = attention_layer(sample_input)
    print(f"Attention output shape: {output.shape}")
    print(f"Attention weights shape: {weights.shape}")

import torch
import torch.nn.functional as F
import numpy as np
from transformers import AutoTokenizer, AutoModel
import faiss
from typing import List, Tuple
import json

class RAGSystem:
    def __init__(self, encoder_model_name="sentence-transformers/all-MiniLM-L6-v2"):
        # Initialize encoder for creating embeddings
        self.tokenizer = AutoTokenizer.from_pretrained(encoder_model_name)
        self.encoder = AutoModel.from_pretrained(encoder_model_name)
        self.encoder.eval()
        
        # Vector store for efficient similarity search
        self.vector_store = None
        self.documents = []
        self.embeddings = []
        
    def encode_text(self, texts: List[str]) -> torch.Tensor:
        """Convert text to embeddings using the encoder model"""
        encoded = self.tokenizer(texts, padding=True, truncation=True, 
                                return_tensors='pt', max_length=512)
        
        with torch.no_grad():
            outputs = self.encoder(**encoded)
            # Use mean pooling of token embeddings
            embeddings = outputs.last_hidden_state.mean(dim=1)
            # Normalize for cosine similarity
            embeddings = F.normalize(embeddings, p=2, dim=1)
            
        return embeddings
    
    def build_index(self, documents: List[str]):
        """Build vector index from documents"""
        print(f"Indexing {len(documents)} documents...")
        
        # Store documents
        self.documents = documents
        
        # Create embeddings in batches
        batch_size = 32
        all_embeddings = []
        
        for i in range(0, len(documents), batch_size):
            batch = documents[i:i + batch_size]
            batch_embeddings = self.encode_text(batch)
            all_embeddings.append(batch_embeddings)
        
        # Concatenate all embeddings
        self.embeddings = torch.cat(all_embeddings, dim=0)
        
        # Build FAISS index for fast cosine similarity search
        embedding_dim = self.embeddings.shape[1]
        
        # Use inner product index (equivalent to cosine similarity for normalized vectors)
        self.vector_store = faiss.IndexFlatIP(embedding_dim)
        self.vector_store.add(self.embeddings.numpy())
        
        print(f"Index built with {len(documents)} documents")
    
    def retrieve_similar_docs(self, query: str, top_k: int = 5) -> List[Tuple[str, float]]:
        """Retrieve most similar documents using cosine similarity"""
        if self.vector_store is None:
            raise ValueError("Index not built. Call build_index() first.")
        
        # Encode query
        query_embedding = self.encode_text([query])
        
        # Search using cosine similarity (inner product on normalized vectors)
        similarities, indices = self.vector_store.search(
            query_embedding.numpy(), top_k
        )
        
        # Return documents with similarity scores
        results = []
        for i, (idx, sim) in enumerate(zip(indices[0], similarities[0])):
            if idx != -1:  # Valid index
                results.append((self.documents[idx], float(sim)))
        
        return results
    
    def manual_cosine_search(self, query: str, top_k: int = 5) -> List[Tuple[str, float]]:
        """Manual implementation of cosine similarity search (for demonstration)"""
        query_embedding = self.encode_text([query])
        
        # Compute cosine similarities with all documents
        similarities = F.cosine_similarity(
            query_embedding, self.embeddings, dim=1
        )
        
        # Get top-k most similar
        top_similarities, top_indices = torch.topk(similarities, top_k)
        
        results = []
        for idx, sim in zip(top_indices, top_similarities):
            results.append((self.documents[idx.item()], sim.item()))
        
        return results

# Advanced RAG with re-ranking and filtering
class AdvancedRAG(RAGSystem):
    def __init__(self, encoder_model_name="sentence-transformers/all-MiniLM-L6-v2", 
                 similarity_threshold=0.3):
        super().__init__(encoder_model_name)
        self.similarity_threshold = similarity_threshold
        
    def retrieve_with_filtering(self, query: str, top_k: int = 10) -> List[Tuple[str, float]]:
        """Retrieve documents with similarity threshold filtering"""
        # Get more candidates than needed
        candidates = self.retrieve_similar_docs(query, top_k * 2)
        
        # Filter by similarity threshold
        filtered = [(doc, sim) for doc, sim in candidates 
                   if sim >= self.similarity_threshold]
        
        # Return top-k after filtering
        return filtered[:top_k]
    
    def diverse_retrieval(self, query: str, top_k: int = 5, 
                         diversity_lambda: float = 0.5) -> List[Tuple[str, float]]:
        """Retrieve diverse documents using MMR (Maximal Marginal Relevance)"""
        # Get initial candidates
        candidates = self.retrieve_similar_docs(query, top_k * 3)
        query_embedding = self.encode_text([query])
        
        selected = []
        candidate_docs = [doc for doc, _ in candidates]
        candidate_embeddings = self.encode_text(candidate_docs)
        
        for _ in range(min(top_k, len(candidates))):
            if not selected:
                # First document: highest similarity to query
                best_idx = 0
                selected.append(candidates[best_idx])
            else:
                # Subsequent documents: balance relevance and diversity
                best_score = -float('inf')
                best_idx = -1
                
                for i, (doc, query_sim) in enumerate(candidates):
                    if (doc, query_sim) in selected:
                        continue
                    
                    # Compute max similarity to already selected documents
                    selected_embeddings = self.encode_text([s[0] for s in selected])
                    doc_embedding = candidate_embeddings[i:i+1]
                    
                    max_sim_to_selected = F.cosine_similarity(
                        doc_embedding, selected_embeddings, dim=1
                    ).max().item()
                    
                    # MMR score: balance relevance and diversity
                    mmr_score = (diversity_lambda * query_sim - 
                               (1 - diversity_lambda) * max_sim_to_selected)
                    
                    if mmr_score > best_score:
                        best_score = mmr_score
                        best_idx = i
                
                if best_idx != -1:
                    selected.append(candidates[best_idx])
        
        return selected

# Evaluation and analysis tools
class RAGAnalyzer:
    def __init__(self, rag_system: RAGSystem):
        self.rag_system = rag_system
    
    def analyze_retrieval_distribution(self, queries: List[str]) -> dict:
        """Analyze the distribution of similarity scores"""
        all_similarities = []
        
        for query in queries:
            results = self.rag_system.retrieve_similar_docs(query, top_k=10)
            similarities = [sim for _, sim in results]
            all_similarities.extend(similarities)
        
        return {
            'mean_similarity': np.mean(all_similarities),
            'std_similarity': np.std(all_similarities),
            'min_similarity': np.min(all_similarities),
            'max_similarity': np.max(all_similarities),
            'percentiles': {
                '25': np.percentile(all_similarities, 25),
                '50': np.percentile(all_similarities, 50),
                '75': np.percentile(all_similarities, 75),
                '90': np.percentile(all_similarities, 90)
            }
        }
    
    def find_optimal_threshold(self, queries: List[str], 
                             ground_truth_relevant: List[List[str]]) -> float:
        """Find optimal similarity threshold using precision-recall analysis"""
        thresholds = np.arange(0.1, 1.0, 0.05)
        best_f1 = 0
        best_threshold = 0.5
        
        for threshold in thresholds:
            total_precision = 0
            total_recall = 0
            valid_queries = 0
            
            for query, relevant_docs in zip(queries, ground_truth_relevant):
                results = self.rag_system.retrieve_similar_docs(query, top_k=20)
                retrieved = [doc for doc, sim in results if sim >= threshold]
                
                if len(retrieved) > 0 and len(relevant_docs) > 0:
                    relevant_retrieved = set(retrieved) & set(relevant_docs)
                    precision = len(relevant_retrieved) / len(retrieved)
                    recall = len(relevant_retrieved) / len(relevant_docs)
                    
                    total_precision += precision
                    total_recall += recall
                    valid_queries += 1
            
            if valid_queries > 0:
                avg_precision = total_precision / valid_queries
                avg_recall = total_recall / valid_queries
                
                if avg_precision + avg_recall > 0:
                    f1 = 2 * (avg_precision * avg_recall) / (avg_precision + avg_recall)
                    if f1 > best_f1:
                        best_f1 = f1
                        best_threshold = threshold
        
        return best_threshold

# Example usage and demonstration
def demonstrate_rag_with_cosine_similarity():
    # Sample knowledge base
    documents = [
        "Machine learning is a subset of artificial intelligence that focuses on algorithms.",
        "Deep learning uses neural networks with multiple layers to process data.",
        "Natural language processing helps computers understand human language.",
        "Computer vision enables machines to interpret and understand visual information.",
        "Reinforcement learning teaches agents to make decisions through trial and error.",
        "Python is a popular programming language for data science and AI.",
        "Transformers are a type of neural network architecture used in NLP.",
        "Gradient descent is an optimization algorithm used in machine learning.",
        "Overfitting occurs when a model performs well on training data but poorly on new data.",
        "Cross-validation is a technique for evaluating model performance."
    ]
    
    # Initialize RAG system
    rag = AdvancedRAG()
    
    # Build index
    rag.build_index(documents)
    
    # Test queries
    queries = [
        "What is machine learning?",
        "How do neural networks work?",
        "What programming language is best for AI?",
        "How to prevent overfitting in models?"
    ]
    
    print("=== RAG Retrieval Results ===")
    for query in queries:
        print(f"\nQuery: {query}")
        
        # Standard retrieval
        results = rag.retrieve_similar_docs(query, top_k=3)
        print("Standard retrieval:")
        for i, (doc, sim) in enumerate(results, 1):
            print(f"  {i}. (sim: {sim:.3f}) {doc}")
        
        # Diverse retrieval
        diverse_results = rag.diverse_retrieval(query, top_k=3)
        print("Diverse retrieval:")
        for i, (doc, sim) in enumerate(diverse_results, 1):
            print(f"  {i}. (sim: {sim:.3f}) {doc}")
    
    # Analysis
    analyzer = RAGAnalyzer(rag)
    stats = analyzer.analyze_retrieval_distribution(queries)
    print(f"\n=== Similarity Statistics ===")
    print(f"Mean similarity: {stats['mean_similarity']:.3f}")
    print(f"Std deviation: {stats['std_similarity']:.3f}")
    print(f"90th percentile: {stats['percentiles']['90']:.3f}")

if __name__ == "__main__":
    demonstrate_rag_with_cosine_similarity()

import torch
import torch.nn.functional as F
import numpy as np
from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
import faiss
from typing import List, Tuple, Dict
import json
import time
from dataclasses import dataclass

@dataclass
class Document:
    id: str
    content: str
    metadata: Dict = None

class EmbeddingModel:
    """Handles text encoding for RAG retrieval"""
    
    def __init__(self, model_name="sentence-transformers/all-MiniLM-L6-v2"):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
        self.model = AutoModel.from_pretrained(model_name)
        self.model.eval()
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.model.to(self.device)
        
    def encode(self, texts: List[str], batch_size: int = 32) -> np.ndarray:
        """Encode texts into normalized embeddings for cosine similarity"""
        all_embeddings = []
        
        for i in range(0, len(texts), batch_size):
            batch_texts = texts[i:i + batch_size]
            
            # Tokenize
            encoded = self.tokenizer(
                batch_texts,
                padding=True,
                truncation=True,
                max_length=512,
                return_tensors='pt'
            ).to(self.device)
            
            with torch.no_grad():
                # Get embeddings
                outputs = self.model(**encoded)
                
                # Mean pooling
                attention_mask = encoded['attention_mask']
                token_embeddings = outputs.last_hidden_state
                input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
                
                embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
                
                # L2 normalize for cosine similarity
                embeddings = F.normalize(embeddings, p=2, dim=1)
                
                all_embeddings.append(embeddings.cpu().numpy())
        
        return np.vstack(all_embeddings)

class VectorStore:
    """Vector database using FAISS for fast cosine similarity search"""
    
    def __init__(self, embedding_dim: int):
        # Use Inner Product index - equivalent to cosine similarity for normalized vectors
        self.index = faiss.IndexFlatIP(embedding_dim)
        self.documents = []
        
    def add_documents(self, documents: List[Document], embeddings: np.ndarray):
        """Add documents and their embeddings to the store"""
        self.documents.extend(documents)
        self.index.add(embeddings.astype(np.float32))
        
    def search(self, query_embedding: np.ndarray, top_k: int = 5, 
               similarity_threshold: float = 0.0) -> List[Tuple[Document, float]]:
        """Search for most similar documents using cosine similarity"""
        
        # Ensure query embedding is normalized
        query_embedding = query_embedding / np.linalg.norm(query_embedding)
        query_embedding = query_embedding.reshape(1, -1).astype(np.float32)
        
        # Search using inner product (cosine similarity for normalized vectors)
        similarities, indices = self.index.search(query_embedding, top_k)
        
        results = []
        for idx, sim in zip(indices[0], similarities[0]):
            if idx != -1 and sim >= similarity_threshold:
                results.append((self.documents[idx], float(sim)))
                
        return results
    
    def manual_cosine_search(self, query_embedding: np.ndarray, top_k: int = 5) -> List[Tuple[Document, float]]:
        """Manual cosine similarity computation for demonstration"""
        if self.index.ntotal == 0:
            return []
        
        # Get all stored embeddings
        all_embeddings = self.index.reconstruct_n(0, self.index.ntotal)
        
        # Compute cosine similarities manually
        query_norm = query_embedding / np.linalg.norm(query_embedding)
        similarities = []
        
        for i, doc_embedding in enumerate(all_embeddings):
            doc_norm = doc_embedding / np.linalg.norm(doc_embedding)
            cosine_sim = np.dot(query_norm, doc_norm)
            similarities.append((i, cosine_sim))
        
        # Sort by similarity (descending)
        similarities.sort(key=lambda x: x[1], reverse=True)
        
        # Return top-k results
        results = []
        for i, sim in similarities[:top_k]:
            results.append((self.documents[i], float(sim)))
            
        return results

class RAGSystem:
    """Complete RAG implementation with cosine similarity retrieval"""
    
    def __init__(self, encoder_model: str = "sentence-transformers/all-MiniLM-L6-v2",
                 generator_model: str = "microsoft/DialoGPT-small"):
        
        # Initialize components
        self.encoder = EmbeddingModel(encoder_model)
        self.vector_store = None
        
        # Initialize generator (simplified - in practice you'd use a larger model)
        self.generator_tokenizer = AutoTokenizer.from_pretrained(generator_model)
        self.generator_model = AutoModelForCausalLM.from_pretrained(generator_model)
        
        if self.generator_tokenizer.pad_token is None:
            self.generator_tokenizer.pad_token = self.generator_tokenizer.eos_token
            
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        self.generator_model.to(self.device)
        
    def index_documents(self, documents: List[Document]):
        """Index documents for retrieval"""
        print(f"Indexing {len(documents)} documents...")
        
        # Extract text content
        texts = [doc.content for doc in documents]
        
        # Generate embeddings
        embeddings = self.encoder.encode(texts)
        
        # Initialize vector store
        embedding_dim = embeddings.shape[1]
        self.vector_store = VectorStore(embedding_dim)
        
        # Add to vector store
        self.vector_store.add_documents(documents, embeddings)
        
        print(f"Indexing complete. Embedding dimension: {embedding_dim}")
        
    def retrieve(self, query: str, top_k: int = 5, similarity_threshold: float = 0.3) -> List[Tuple[Document, float]]:
        """Retrieve relevant documents using cosine similarity"""
        if self.vector_store is None:
            raise ValueError("No documents indexed. Call index_documents() first.")
        
        # Encode query
        query_embedding = self.encoder.encode([query])[0]
        
        # Search for similar documents
        results = self.vector_store.search(
            query_embedding, 
            top_k=top_k, 
            similarity_threshold=similarity_threshold
        )
        
        return results
    
    def retrieve_with_reranking(self, query: str, initial_k: int = 20, final_k: int = 5) -> List[Tuple[Document, float]]:
        """Advanced retrieval with re-ranking based on query-document similarity"""
        
        # Initial broad retrieval
        candidates = self.retrieve(query, top_k=initial_k, similarity_threshold=0.1)
        
        if not candidates:
            return []
        
        # Re-rank by computing more detailed similarity
        query_embedding = self.encoder.encode([query])[0]
        reranked = []
        
        for doc, initial_sim in candidates:
            # Compute similarity between query and document
            doc_embedding = self.encoder.encode([doc.content])[0]
            
            # Detailed cosine similarity computation
            detailed_sim = np.dot(query_embedding, doc_embedding) / (
                np.linalg.norm(query_embedding) * np.linalg.norm(doc_embedding)
            )
            
            reranked.append((doc, float(detailed_sim)))
        
        # Sort by re-ranked similarity
        reranked.sort(key=lambda x: x[1], reverse=True)
        
        return reranked[:final_k]
    
    def generate_response(self, query: str, retrieved_docs: List[Tuple[Document, float]], 
                         max_length: int = 200) -> str:
        """Generate response using retrieved context"""
        
        # Prepare context from retrieved documents
        context_parts = []
        for doc, similarity in retrieved_docs:
            context_parts.append(f"Context (similarity: {similarity:.3f}): {doc.content}")
        
        context = "\n".join(context_parts)
        
        # Create prompt
        prompt = f"Context:\n{context}\n\nQuestion: {query}\nAnswer:"
        
        # Tokenize and generate
        inputs = self.generator_tokenizer.encode(prompt, return_tensors='pt').to(self.device)
        
        with torch.no_grad():
            outputs = self.generator_model.generate(
                inputs,
                max_length=len(inputs[0]) + max_length,
                num_return_sequences=1,
                temperature=0.7,
                pad_token_id=self.generator_tokenizer.eos_token_id,
                do_sample=True
            )
        
        # Decode response
        full_response = self.generator_tokenizer.decode(outputs[0], skip_special_tokens=True)
        
        # Extract just the answer part
        answer_start = full_response.find("Answer:") + len("Answer:")
        answer = full_response[answer_start:].strip()
        
        return answer
    
    def query(self, question: str, top_k: int = 3, use_reranking: bool = True) -> Dict:
        """Complete RAG query pipeline"""
        start_time = time.time()
        
        # Retrieve relevant documents
        if use_reranking:
            retrieved_docs = self.retrieve_with_reranking(question, initial_k=10, final_k=top_k)
        else:
            retrieved_docs = self.retrieve(question, top_k=top_k)
        
        retrieval_time = time.time() - start_time
        
        # Generate response
        generation_start = time.time()
        response = self.generate_response(question, retrieved_docs)
        generation_time = time.time() - generation_start
        
        return {
            'question': question,
            'answer': response,
            'retrieved_documents': [
                {
                    'content': doc.content,
                    'similarity': sim,
                    'metadata': doc.metadata
                }
                for doc, sim in retrieved_docs
            ],
            'timing': {
                'retrieval_time': retrieval_time,
                'generation_time': generation_time,
                'total_time': retrieval_time + generation_time
            }
        }

class RAGEvaluator:
    """Evaluation tools for RAG system performance"""
    
    def __init__(self, rag_system: RAGSystem):
        self.rag_system = rag_system
        
    def evaluate_retrieval_quality(self, test_queries: List[str], 
                                 ground_truth: List[List[str]]) -> Dict:
        """Evaluate retrieval using cosine similarity thresholds"""
        
        total_precision = 0
        total_recall = 0
        similarity_scores = []
        
        for query, relevant_docs in zip(test_queries, ground_truth):
            retrieved = self.rag_system.retrieve(query, top_k=10)
            
            retrieved_contents = [doc.content for doc, _ in retrieved]
            similarities = [sim for _, sim in retrieved]
            similarity_scores.extend(similarities)
            
            # Calculate precision and recall
            relevant_retrieved = len(set(retrieved_contents) & set(relevant_docs))
            
            precision = relevant_retrieved / len(retrieved_contents) if retrieved_contents else 0
            recall = relevant_retrieved / len(relevant_docs) if relevant_docs else 0
            
            total_precision += precision
            total_recall += recall
        
        avg_precision = total_precision / len(test_queries)
        avg_recall = total_recall / len(test_queries)
        f1_score = 2 * (avg_precision * avg_recall) / (avg_precision + avg_recall) if (avg_precision + avg_recall) > 0 else 0
        
        return {
            'precision': avg_precision,
            'recall': avg_recall,
            'f1_score': f1_score,
            'avg_similarity': np.mean(similarity_scores),
            'similarity_std': np.std(similarity_scores),
            'similarity_distribution': {
                'min': np.min(similarity_scores),
                'max': np.max(similarity_scores),
                '25th_percentile': np.percentile(similarity_scores, 25),
                '50th_percentile': np.percentile(similarity_scores, 50),
                '75th_percentile': np.percentile(similarity_scores, 75),
                '90th_percentile': np.percentile(similarity_scores, 90)
            }
        }

# Demo and example usage
def create_sample_knowledge_base() -> List[Document]:
    """Create a sample knowledge base for testing"""
    knowledge_base = [
        Document("1", "Machine learning is a method of data analysis that automates analytical model building using algorithms that iteratively learn from data without being explicitly programmed.", {"topic": "ML"}),
        Document("2", "Deep learning is part of machine learning methods based on artificial neural networks with representation learning.", {"topic": "DL"}),
        Document("3", "Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence concerned with interactions between computers and human language.", {"topic": "NLP"}),
        Document("4", "Computer vision is an interdisciplinary scientific field that deals with how computers can gain high-level understanding from digital images or videos.", {"topic": "CV"}),
        Document("5", "Reinforcement learning is an area of machine learning concerned with how intelligent agents ought to take actions in an environment to maximize cumulative reward.", {"topic": "RL"}),
        Document("6", "Python is an interpreted, high-level programming language with dynamic semantics and is widely used for artificial intelligence and machine learning applications.", {"topic": "Programming"}),
        Document("7", "TensorFlow is an end-to-end open source platform for machine learning with comprehensive tools, libraries and community resources.", {"topic": "Tools"}),
        Document("8", "PyTorch is an open source machine learning library based on the Torch library, used for applications such as computer vision and natural language processing.", {"topic": "Tools"}),
        Document("9", "Neural networks are computing systems vaguely inspired by biological neural networks and are used to estimate functions that depend on a large number of inputs.", {"topic": "ML"}),
        Document("10", "Transformers are a neural network architecture that has become dominant in natural language processing tasks since 2017.", {"topic": "NLP"})
    ]
    return knowledge_base

def demonstrate_cosine_similarity_in_rag():
    """Comprehensive demonstration of cosine similarity in RAG"""
    
    print("=== RAG System with Cosine Similarity Demo ===\n")
    
    # Create knowledge base
    documents = create_sample_knowledge_base()
    
    # Initialize RAG system
    rag = RAGSystem()
    
    # Index documents
    rag.index_documents(documents)
    
    # Test queries
    test_queries = [
        "What is machine learning?",
        "How do neural networks work?",
        "What tools are available for ML?",
        "Explain reinforcement learning",
        "What programming language is best for AI?"
    ]
    
    print("=== Query Results ===")
    for query in test_queries:
        print(f"\nQuery: {query}")
        
        # Show detailed retrieval process
        print("\n--- Retrieval Process ---")
        retrieved_docs = rag.retrieve(query, top_k=3)
        
        for i, (doc, similarity) in enumerate(retrieved_docs, 1):
            print(f"{i}. Similarity: {similarity:.4f}")
            print(f"   Content: {doc.content[:100]}...")
            print(f"   Metadata: {doc.metadata}")
        
        # Show manual cosine similarity calculation
        if rag.vector_store:
            print("\n--- Manual Cosine Similarity Verification ---")
            query_embedding = rag.encoder.encode([query])[0]
            manual_results = rag.vector_store.manual_cosine_search(query_embedding, top_k=2)
            
            for i, (doc, sim) in enumerate(manual_results, 1):
                print(f"Manual calc {i}: {sim:.4f} - {doc.content[:50]}...")
        
        # Full RAG response
        print("\n--- RAG Response ---")
        result = rag.query(query, top_k=2)
        print(f"Answer: {result['answer']}")
        print(f"Retrieval time: {result['timing']['retrieval_time']:.3f}s")
        print(f"Generation time: {result['timing']['generation_time']:.3f}s")
        print("-" * 80)
    
    # Evaluation
    print("\n=== System Evaluation ===")
    evaluator = RAGEvaluator(rag)
    
    # Simple ground truth for demonstration
    ground_truth = [
        ["Machine learning is a method of data analysis that automates analytical model building"],
        ["Neural networks are computing systems vaguely inspired by biological neural networks"],
        ["TensorFlow is an end-to-end open source platform", "PyTorch is an open source machine learning library"],
        ["Reinforcement learning is an area of machine learning concerned with how intelligent agents"],
        ["Python is an interpreted, high-level programming language"]
    ]
    
    eval_results = evaluator.evaluate_retrieval_quality(test_queries, ground_truth)
    
    print(f"Precision: {eval_results['precision']:.3f}")
    print(f"Recall: {eval_results['recall']:.3f}")
    print(f"F1 Score: {eval_results['f1_score']:.3f}")
    print(f"Average Similarity: {eval_results['avg_similarity']:.3f}")
    print(f"Similarity Std Dev: {eval_results['similarity_std']:.3f}")
    print(f"Similarity Range: {eval_results['similarity_distribution']['min']:.3f} - {eval_results['similarity_distribution']['max']:.3f}")

if __name__ == "__main__":
    demonstrate_cosine_similarity_in_rag()

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoTokenizer, AutoModel
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import seaborn as sns
from typing import List, Dict, Tuple

class EmbeddingDemo:
    """Comprehensive demonstration of embeddings and cosine similarity"""
    
    def __init__(self, model_name="sentence-transformers/all-MiniLM-L6-v2"):
        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
        self.model = AutoModel.from_pretrained(model_name)
        self.model.eval()
        
    def create_word_embeddings(self, words: List[str]) -> np.ndarray:
        """Create embeddings for individual words"""
        embeddings = []
        
        for word in words:
            # Tokenize and encode
            inputs = self.tokenizer(word, return_tensors='pt', padding=True, truncation=True)
            
            with torch.no_grad():
                outputs = self.model(**inputs)
                # Use mean pooling for word representation
                embedding = outputs.last_hidden_state.mean(dim=1)
                # Normalize for cosine similarity
                embedding = F.normalize(embedding, p=2, dim=1)
                embeddings.append(embedding.squeeze().numpy())
        
        return np.array(embeddings)
    
    def create_sentence_embeddings(self, sentences: List[str]) -> np.ndarray:
        """Create embeddings for sentences"""
        embeddings = []
        
        for sentence in sentences:
            inputs = self.tokenizer(sentence, return_tensors='pt', padding=True, 
                                  truncation=True, max_length=512)
            
            with torch.no_grad():
                outputs = self.model(**inputs)
                # Mean pooling with attention mask
                attention_mask = inputs['attention_mask']
                token_embeddings = outputs.last_hidden_state
                
                # Weighted average based on attention mask
                input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
                sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
                sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
                
                embedding = sum_embeddings / sum_mask
                embedding = F.normalize(embedding, p=2, dim=1)
                embeddings.append(embedding.squeeze().numpy())
        
        return np.array(embeddings)

def compute_cosine_similarity_matrix(embeddings: np.ndarray) -> np.ndarray:
    """Compute pairwise cosine similarities between embeddings"""
    # Normalize embeddings (though they should already be normalized)
    normalized = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
    
    # Compute cosine similarity matrix
    similarity_matrix = np.dot(normalized, normalized.T)
    
    return similarity_matrix

def demonstrate_word_embeddings():
    """Show how word embeddings capture semantic relationships"""
    
    print("=== Word Embeddings and Cosine Similarity ===\n")
    
    demo = EmbeddingDemo()
    
    # Related word groups
    words = [
        # Animals
        'cat', 'dog', 'lion', 'tiger',
        # Technology
        'computer', 'laptop', 'smartphone', 'tablet',
        # Colors
        'red', 'blue', 'green', 'yellow',
        # Sports
        'football', 'basketball', 'tennis', 'golf'
    ]
    
    # Create embeddings
    embeddings = demo.create_word_embeddings(words)
    
    # Compute similarity matrix
    similarity_matrix = compute_cosine_similarity_matrix(embeddings)
    
    print("Word Similarity Matrix (Cosine Similarity):")
    print("Words:", words)
    print(similarity_matrix.round(3))
    
    # Find most similar word pairs
    print("\nMost Similar Word Pairs:")
    for i in range(len(words)):
        for j in range(i+1, len(words)):
            sim = similarity_matrix[i, j]
            if sim > 0.7:  # High similarity threshold
                print(f"{words[i]} - {words[j]}: {sim:.3f}")
    
    return words, embeddings, similarity_matrix

def demonstrate_semantic_relationships():
    """Show how embeddings capture semantic relationships"""
    
    print("\n=== Semantic Relationships in Embeddings ===\n")
    
    demo = EmbeddingDemo()
    
    # Analogy examples: A is to B as C is to D
    analogies = [
        ("king", "queen", "man", "woman"),
        ("paris", "france", "london", "england"),
        ("big", "bigger", "small", "smaller"),
        ("run", "running", "walk", "walking")
    ]
    
    for a, b, c, d in analogies:
        words = [a, b, c, d]
        embeddings = demo.create_word_embeddings(words)
        
        # Vector arithmetic: king - man + woman ≈ queen
        vector_ab = embeddings[1] - embeddings[0]  # queen - king
        vector_cd = embeddings[3] - embeddings[2]  # woman - man
        
        # Compute cosine similarity between the relationship vectors
        relationship_similarity = np.dot(vector_ab, vector_cd) / (
            np.linalg.norm(vector_ab) * np.linalg.norm(vector_cd)
        )
        
        print(f"Analogy: {a}:{b} :: {c}:{d}")
        print(f"Relationship similarity: {relationship_similarity:.3f}")
        
        # Test the analogy: compute a - b + d and see if it's close to c
        predicted_c = embeddings[0] - embeddings[1] + embeddings[3]
        predicted_c = predicted_c / np.linalg.norm(predicted_c)
        
        actual_c = embeddings[2] / np.linalg.norm(embeddings[2])
        analogy_score = np.dot(predicted_c, actual_c)
        
        print(f"Analogy completion accuracy: {analogy_score:.3f}")
        print()

def demonstrate_sentence_embeddings():
    """Show sentence-level embeddings and similarity"""
    
    print("=== Sentence Embeddings and Similarity ===\n")
    
    demo = EmbeddingDemo()
    
    sentences = [
        "The cat sits on the mat",
        "A feline rests on the carpet",
        "Dogs are loyal animals",
        "Canines are faithful pets",
        "I love programming in Python",
        "Python is my favorite coding language",
        "The weather is sunny today",
        "It's a beautiful day outside",
        "Machine learning is fascinating",
        "AI algorithms are interesting"
    ]
    
    # Create sentence embeddings
    embeddings = demo.create_sentence_embeddings(sentences)
    
    # Compute similarity matrix
    similarity_matrix = compute_cosine_similarity_matrix(embeddings)
    
    print("Sentence Similarity Analysis:")
    for i in range(len(sentences)):
        print(f"{i}: {sentences[i]}")
    
    print("\nMost Similar Sentence Pairs:")
    pairs = []
    for i in range(len(sentences)):
        for j in range(i+1, len(sentences)):
            sim = similarity_matrix[i, j]
            pairs.append((sim, i, j))
    
    # Sort by similarity
    pairs.sort(reverse=True)
    
    # Show top 5 most similar pairs
    for sim, i, j in pairs[:5]:
        print(f"Similarity: {sim:.3f}")
        print(f"  Sentence {i}: {sentences[i]}")
        print(f"  Sentence {j}: {sentences[j]}")
        print()
    
    return sentences, embeddings, similarity_matrix

class EmbeddingVisualizer:
    """Visualize embeddings in 2D space"""
    
    @staticmethod
    def visualize_embeddings(embeddings: np.ndarray, labels: List[str], 
                           title: str = "Embedding Visualization"):
        """Visualize high-dimensional embeddings in 2D"""
        
        # Reduce dimensions using t-SNE
        tsne = TSNE(n_components=2, random_state=42, perplexity=min(30, len(embeddings)-1))
        embeddings_2d = tsne.fit_transform(embeddings)
        
        # Create visualization
        plt.figure(figsize=(12, 8))
        scatter = plt.scatter(embeddings_2d[:, 0], embeddings_2d[:, 1], 
                             c=range(len(labels)), cmap='tab20', s=100)
        
        # Add labels
        for i, label in enumerate(labels):
            plt.annotate(label, (embeddings_2d[i, 0], embeddings_2d[i, 1]), 
                        xytext=(5, 5), textcoords='offset points', fontsize=10)
        
        plt.title(title)
        plt.xlabel('t-SNE Dimension 1')
        plt.ylabel('t-SNE Dimension 2')
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.show()
    
    @staticmethod
    def plot_similarity_heatmap(similarity_matrix: np.ndarray, labels: List[str], 
                               title: str = "Cosine Similarity Heatmap"):
        """Plot cosine similarity matrix as heatmap"""
        
        plt.figure(figsize=(10, 8))
        sns.heatmap(similarity_matrix, 
                   xticklabels=labels, 
                   yticklabels=labels,
                   annot=True, 
                   cmap='coolwarm', 
                   center=0,
                   square=True,
                   fmt='.3f')
        plt.title(title)
        plt.tight_layout()
        plt.show()

def demonstrate_embedding_arithmetic():
    """Show vector arithmetic with embeddings"""
    
    print("=== Embedding Arithmetic ===\n")
    
    demo = EmbeddingDemo()
    
    # Create embeddings for arithmetic examples
    words = ['king', 'man', 'woman', 'queen', 'prince', 'princess', 'boy', 'girl']
    embeddings = demo.create_word_embeddings(words)
    
    word_to_embedding = {word: emb for word, emb in zip(words, embeddings)}
    
    def find_closest_word(target_embedding, word_embeddings, words, exclude=[]):
        """Find the word with embedding closest to target"""
        target_norm = target_embedding / np.linalg.norm(target_embedding)
        
        best_similarity = -1
        best_word = None
        
        for word, embedding in word_embeddings.items():
            if word in exclude:
                continue
            
            embedding_norm = embedding / np.linalg.norm(embedding)
            similarity = np.dot(target_norm, embedding_norm)
            
            if similarity > best_similarity:
                best_similarity = similarity
                best_word = word
        
        return best_word, best_similarity
    
    # Perform vector arithmetic
    arithmetic_examples = [
        ("king", "man", "woman", "Should be close to 'queen'"),
        ("prince", "boy", "girl", "Should be close to 'princess'")
    ]
    
    for word_a, word_b, word_c, description in arithmetic_examples:
        # Compute: word_a - word_b + word_c
        result_embedding = (word_to_embedding[word_a] - 
                          word_to_embedding[word_b] + 
                          word_to_embedding[word_c])
        
        closest_word, similarity = find_closest_word(
            result_embedding, word_to_embedding, 
            exclude=[word_a, word_b, word_c]
        )
        
        print(f"Vector arithmetic: {word_a} - {word_b} + {word_c}")
        print(f"Closest word: {closest_word} (similarity: {similarity:.3f})")
        print(f"Expected: {description}")
        print()

def comprehensive_embedding_demo():
    """Run complete demonstration of embeddings and cosine similarity"""
    
    print("🔍 COMPREHENSIVE EMBEDDINGS AND COSINE SIMILARITY DEMO\n")
    print("=" * 60)
    
    # 1. Word embeddings
    words, word_embeddings, word_similarity = demonstrate_word_embeddings()
    
    # 2. Semantic relationships
    demonstrate_semantic_relationships()
    
    # 3. Sentence embeddings
    sentences, sentence_embeddings, sentence_similarity = demonstrate_sentence_embeddings()
    
    # 4. Vector arithmetic
    demonstrate_embedding_arithmetic()
    
    # 5. Visualizations (optional - requires matplotlib)
    try:
        print("=== Visualizations ===\n")
        
        visualizer = EmbeddingVisualizer()
        
        # Visualize word embeddings
        print("Generating word embedding visualization...")
        visualizer.visualize_embeddings(word_embeddings, words, "Word Embeddings (t-SNE)")
        
        # Plot similarity heatmap
        print("Generating similarity heatmap...")
        visualizer.plot_similarity_heatmap(word_similarity[:8, :8], words[:8], 
                                         "Word Cosine Similarity Heatmap")
        
    except ImportError:
        print("Visualization libraries not available. Skipping plots.")
    
    # 6. Summary insights
    print("\n=== Key Insights ===")
    print("1. Embeddings map discrete data to continuous vector spaces")
    print("2. Cosine similarity captures semantic relationships effectively")
    print("3. Similar concepts cluster together in embedding space")
    print("4. Vector arithmetic can capture analogical relationships")
    print("5. Sentence embeddings capture meaning beyond individual words")
    print("6. Similarity thresholds help filter relevant vs irrelevant matches")

if __name__ == "__main__":
    comprehensive_embedding_demo()