LLM 缓存

# 方式 1：完整 URL
export REDIS_URL="redis://localhost:6379"

# 方式 2：单独组件
export REDIS_HOST="redis"
export REDIS_PORT="6379"
export REDIS_PASSWORD="your-password"  # 可选

prompt_cache:
  enabled: true
  ttl_seconds: 3600           # 默认 1 小时
  max_cache_size_mb: 2048     # 内存限制
  similarity_threshold: 0.95   # 语义相似度阈值

from shannon import Client

client = Client()
response = client.complete(
    messages=[{"role": "user", "content": "解释量子计算"}],
    cache_key="quantum-intro-cn",  # 自定义缓存键
    cache_ttl=7200                  # 此请求 2 小时
)

# 检查响应是否来自缓存
if response.cached:
    print("响应来自缓存")

key_data = {
    "messages": messages,
    "model_tier": model_tier,
    "model": model,
    "temperature": temperature,
    "max_tokens": max_tokens,
    "functions": functions,
    "seed": seed,
}
# 排序后 JSON 的 SHA-256 哈希

def _should_cache_response(self, request, response) -> bool:
    # 检查 finish_reason
    fr = (response.finish_reason or "").lower()
    if fr in {"length", "content_filter"}:
        return False

    # 严格 JSON 模式
    if is_strict_json_mode(request):
        try:
            obj = json.loads(response.content or "")
            if not isinstance(obj, dict):
                return False
        except:
            return False

    # 要求内容或 function_call
    if not response.content.strip() and not response.function_call:
        return False

    return True

# 访问命中率
cache_hit_rate = manager.cache.hit_rate

# 响应包含缓存状态
response = await manager.complete(...)
print(f"缓存: {response.cached}")

{
  "content": "...",
  "cached": true,
  "finish_reason": "stop",
  "usage": {
    "prompt_tokens": 50,
    "completion_tokens": 100
  }
}

# 好：具体、可复用的缓存键
cache_key="product-summary-v1-{product_id}"

# 避免：可能冲突的通用键
cache_key="summary"

# 带持久化的 Redis
redis:
  appendonly: yes
  maxmemory: 2gb
  maxmemory-policy: allkeys-lru