llm-caching

Name: llm-caching
Author: BagelHole

42stars4forks

How to install llm-caching?

bagelhole/devops-security-agent-skills/llm-caching

$npx -y skills add BagelHole/DevOps-Security-Agent-Skills --skill llm-caching

Installs into the current project.

Summary

Implement multi-layer LLM caching with exact match, semantic similarity, and provider-side prompt caching. Reduce API costs by 30–70%, cut latency, and improve throughput using Redis, GPTCache, and provider caching APIs.

Files · 1

View on GitHub

SKILL.md

1	# LLM Caching
2
3	Cut LLM costs and latency with exact match, semantic, and provider-side caching layers.
4
5	## When to Use This Skill
6
7	Use this skill when:
8	- The same or similar queries are asked repeatedly (FAQ bots, support tools)
9	- LLM API costs are growing and you need immediate savings
10	- Serving high request volumes where repeated queries cause bottlenecks
11	- Implementing prompt caching for long system prompts (Anthropic/OpenAI)
12	- Building offline-capable AI features that need response persistence
13
14	## Caching Layers
15
16	```
17	Request → Exact Cache → Semantic Cache → Provider Cache → LLM API
18	↓ hit ↓ hit ↓ hit
19	instant ~5ms 50-80% cheaper
20	```
21
22	## Layer 1: Exact Match Cache (Redis)
23
24	```python
25	import hashlib
26	import json
27	import redis
28	from openai import OpenAI
29
30	r = redis.Redis(host="localhost", port=6379, decode_responses=True)
31	client = OpenAI()
32
33	def build_cache_key(model: str, messages: list, temperature: float) -> str:
34	"""Deterministic key from request parameters."""
35	payload = json.dumps({
36	"model": model,
37	"messages": messages,
38	"temperature": temperature,
39	}, sort_keys=True)
40	return f"llm:exact:{hashlib.sha256(payload.encode()).hexdigest()}"
41
42	def cached_completion(model: str, messages: list, temperature: float = 0.0,
43	ttl: int = 3600) -> dict:
44	key = build_cache_key(model, messages, temperature)
45
46	# Check cache
47	if cached := r.get(key):
48	return json.loads(cached)
49
50	# Call API
51	response = client.chat.completions.create(
52	model=model, messages=messages, temperature=temperature
53	)
54	result = response.model_dump()
55
56	# Cache result (only cache deterministic responses)
57	if temperature == 0.0:
58	r.setex(key, ttl, json.dumps(result))
59
60	return result
61	```
62
63	## Layer 2: Semantic Cache (GPTCache)
64
65	```python
66	from gptcache import cache, Config
67	from gptcache.adapter import openai
68	from gptcache.embedding import Onnx
69	from gptcache.manager import CacheBase, VectorBase, get_data_manager
70	from gptcache.similarity_evaluation.distance import SearchDistanceEvaluation
71
72	# Configure GPTCache with Qdrant backend
73	def init_gptcache(cache_obj, llm: str):
74	onnx = Onnx() # local embedding model
75	data_manager = get_data_manager(
76	CacheBase("redis"), # metadata store
77	VectorBase("qdrant",
78	host="localhost",
79	port=6333,
80	collection_name=f"llm-cache-{llm}",
81	dimension=onnx.dimension),
82	)
83	cache_obj.init(
84	embedding_func=onnx.to_embeddings,
85	data_manager=data_manager,
86	similarity_evaluation=SearchDistanceEvaluation(),
87	config=Config(similarity_threshold=0.80), # 80% similarity = cache hit
88	)
89
90	cache.set_openai_key()
91	init_gptcache(cache, "gpt-4o-mini")
92
93	# Now openai calls are automatically cached
94	response = openai.ChatCompletion.create(
95	model="gpt-4o-mini",
96	messages=[{"role": "user", "content": "What is machine learning?"}],
97	)
98	# Second call with similar question ("Explain machine learning") → cache hit
99	```
100
101	## Custom Semantic Cache (Production-Grade)
102
103	```python
104	from sentence_transformers import SentenceTransformer
105	from qdrant_client import QdrantClient
106	from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, Range
107	import numpy as np
108	import uuid
109	import time
110
111	embed_model = SentenceTransformer("BAAI/bge-small-en-v1.5") # fast, 33M params
112	qdrant = QdrantClient("http://localhost:6333")
113
114	CACHE_COLLECTION = "semantic-cache"
115	SIMILARITY_THRESHOLD = 0.88
116	CACHE_TTL_SECONDS = 86400 # 24h
117
118	# Create collection once
119	qdrant.create_collection(
120	collection_name=CACHE_COLLECTION,
121	vectors_config=VectorParams(size=384, distance=Distance.COSINE),
122	on_disk_payload=True,
123	)
124
125	def semantic_cache_lookup(query: str, model: str) -> str \| None:
126	embedding = embed_model.encode(query).tolist()
127	results = qdrant.query_points(
128	collection_name=CACHE_COLLECTION,
129	query=embedding,
130	query_filter=Filter(must=[
131	FieldCondition(key="model", match={"value": model}),
132	FieldCondition(key="expires_at", range=Range(gte=time.time())),
133	]),
134	limit=1,
135	score_threshold=SIMILARITY_THRESHOLD,
136	)
137	if results.points:
138	return results.points[0].payload["response"]
139	return None
140
141	def semantic_cache_store(query: str, response: str, model: str):
142	embedding = embed_model.encode(query).tolist()
143	qdrant.upsert(
144	collection_name=CACHE_COLLECTION,
145	points=[PointStruct(
146	id=str(uuid.uuid4

People who liked this also viewed

6 picks

Type

llm-caching

42stars4forks

View on GitHub

How to install llm-caching?

bagelhole/devops-security-agent-skills/llm-caching

$npx -y skills add BagelHole/DevOps-Security-Agent-Skills --skill llm-caching

Installs into the current project.

Summary

Files · 1

View on GitHub

SKILL.md

1	# LLM Caching
2
3	Cut LLM costs and latency with exact match, semantic, and provider-side caching layers.
4
5	## When to Use This Skill
6
7	Use this skill when:
8	- The same or similar queries are asked repeatedly (FAQ bots, support tools)
9	- LLM API costs are growing and you need immediate savings
10	- Serving high request volumes where repeated queries cause bottlenecks
11	- Implementing prompt caching for long system prompts (Anthropic/OpenAI)
12	- Building offline-capable AI features that need response persistence
13
14	## Caching Layers
15
16	```
17	Request → Exact Cache → Semantic Cache → Provider Cache → LLM API
18	↓ hit ↓ hit ↓ hit
19	instant ~5ms 50-80% cheaper
20	```
21
22	## Layer 1: Exact Match Cache (Redis)
23
24	```python
25	import hashlib
26	import json
27	import redis
28	from openai import OpenAI
29
30	r = redis.Redis(host="localhost", port=6379, decode_responses=True)
31	client = OpenAI()
32
33	def build_cache_key(model: str, messages: list, temperature: float) -> str:
34	"""Deterministic key from request parameters."""
35	payload = json.dumps({
36	"model": model,
37	"messages": messages,
38	"temperature": temperature,
39	}, sort_keys=True)
40	return f"llm:exact:{hashlib.sha256(payload.encode()).hexdigest()}"
41
42	def cached_completion(model: str, messages: list, temperature: float = 0.0,
43	ttl: int = 3600) -> dict:
44	key = build_cache_key(model, messages, temperature)
45
46	# Check cache
47	if cached := r.get(key):
48	return json.loads(cached)
49
50	# Call API
51	response = client.chat.completions.create(
52	model=model, messages=messages, temperature=temperature
53	)
54	result = response.model_dump()
55
56	# Cache result (only cache deterministic responses)
57	if temperature == 0.0:
58	r.setex(key, ttl, json.dumps(result))
59
60	return result
61	```
62
63	## Layer 2: Semantic Cache (GPTCache)
64
65	```python
66	from gptcache import cache, Config
67	from gptcache.adapter import openai
68	from gptcache.embedding import Onnx
69	from gptcache.manager import CacheBase, VectorBase, get_data_manager
70	from gptcache.similarity_evaluation.distance import SearchDistanceEvaluation
71
72	# Configure GPTCache with Qdrant backend
73	def init_gptcache(cache_obj, llm: str):
74	onnx = Onnx() # local embedding model
75	data_manager = get_data_manager(
76	CacheBase("redis"), # metadata store
77	VectorBase("qdrant",
78	host="localhost",
79	port=6333,
80	collection_name=f"llm-cache-{llm}",
81	dimension=onnx.dimension),
82	)
83	cache_obj.init(
84	embedding_func=onnx.to_embeddings,
85	data_manager=data_manager,
86	similarity_evaluation=SearchDistanceEvaluation(),
87	config=Config(similarity_threshold=0.80), # 80% similarity = cache hit
88	)
89
90	cache.set_openai_key()
91	init_gptcache(cache, "gpt-4o-mini")
92
93	# Now openai calls are automatically cached
94	response = openai.ChatCompletion.create(
95	model="gpt-4o-mini",
96	messages=[{"role": "user", "content": "What is machine learning?"}],
97	)
98	# Second call with similar question ("Explain machine learning") → cache hit
99	```
100
101	## Custom Semantic Cache (Production-Grade)
102
103	```python
104	from sentence_transformers import SentenceTransformer
105	from qdrant_client import QdrantClient
106	from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, Range
107	import numpy as np
108	import uuid
109	import time
110
111	embed_model = SentenceTransformer("BAAI/bge-small-en-v1.5") # fast, 33M params
112	qdrant = QdrantClient("http://localhost:6333")
113
114	CACHE_COLLECTION = "semantic-cache"
115	SIMILARITY_THRESHOLD = 0.88
116	CACHE_TTL_SECONDS = 86400 # 24h
117
118	# Create collection once
119	qdrant.create_collection(
120	collection_name=CACHE_COLLECTION,
121	vectors_config=VectorParams(size=384, distance=Distance.COSINE),
122	on_disk_payload=True,
123	)
124
125	def semantic_cache_lookup(query: str, model: str) -> str \| None:
126	embedding = embed_model.encode(query).tolist()
127	results = qdrant.query_points(
128	collection_name=CACHE_COLLECTION,
129	query=embedding,
130	query_filter=Filter(must=[
131	FieldCondition(key="model", match={"value": model}),
132	FieldCondition(key="expires_at", range=Range(gte=time.time())),
133	]),
134	limit=1,
135	score_threshold=SIMILARITY_THRESHOLD,
136	)
137	if results.points:
138	return results.points[0].payload["response"]
139	return None
140
141	def semantic_cache_store(query: str, response: str, model: str):
142	embedding = embed_model.encode(query).tolist()
143	qdrant.upsert(
144	collection_name=CACHE_COLLECTION,
145	points=[PointStruct(
146	id=str(uuid.uuid4

People who liked this also viewed

6 picks

Type