How do you compute semantic similarity between texts using Hugging Face and OpenAI embeddings?

Semantic similarity compares text meaning rather than surface words. This powers search engines, duplicate detection, recommendation systems, and the retrieval step in RAG. The standard approach embeds both texts into a high-dimensional vector space and measures the angle between them via cosine similarity — texts with similar meaning land close together in this space, regardless of wording.

import numpy as np
from sklearn.metrics.pairwise import cosine_similarity

# ── OpenAI text-embedding-3 (cloud-based, best quality)
from openai import OpenAI
client = OpenAI()

def openai_embed(texts: list[str]) -> np.ndarray:
    resp = client.embeddings.create(
        model='text-embedding-3-small',  # 1536-dim, fast and cheap
        input=texts,
    )
    return np.array([d.embedding for d in resp.data])

# ── Sentence Transformers (local, open-source, fast)
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('BAAI/bge-small-en-v1.5')

sentences = [
    'The quick brown fox jumps over the lazy dog.',
    'A fast auburn fox leaps above a sleeping hound.',
    'Machine learning is a subset of AI.',
]

embeds = model.encode(sentences, normalize_embeddings=True)  # unit vectors

# Cosine similarity via dot product (normalised vectors)
sim_matrix = embeds @ embeds.T
print(sim_matrix)
# [[1.00, 0.92, 0.31],
#  [0.92, 1.00, 0.29],   <- sentences 0 and 1 are highly similar (0.92)
#  [0.31, 0.29, 1.00]]   <- sentence 2 is unrelated (0.29-0.31)

# ── Semantic search: find most similar to a query
query = 'fox jumping'
q_embed = model.encode([query], normalize_embeddings=True)
scores  = (q_embed @ embeds.T)[0]
ranked  = sorted(zip(scores, sentences), reverse=True)
for score, sent in ranked:
    print(f'{score:.3f}: {sent}')