v2.7.0: Usage Dashboard redesign (OpenUsage-inspired), TCP_NODELAY streaming, Anthropic prompt caching

src/translate-proxy.py

@@ -11,7 +11,7 @@ Usage:
   python3 translate-proxy.py --backend openai-compat --target-url https://... --api-key sk-...
 """
 
-import json, http.server, urllib.request, time, uuid, os, sys, argparse
+import json, http.server, urllib.request, time, uuid, os, sys, argparse, threading, socket
 
 # ═══════════════════════════════════════════════════════════════════
 # Config
@@ -141,6 +141,8 @@ _pool = uuid.uuid4().hex[:8]
 _response_store = {}
 _MAX_STORED = 50
 
+_LOG_DIR = os.path.join(os.path.expanduser("~"), ".cache", "codex-proxy")
+os.makedirs(_LOG_DIR, exist_ok=True)
 _stats_path = os.path.join(_LOG_DIR, "usage-stats.json")
 _stats_lock = threading.Lock()
 
@@ -961,9 +963,6 @@ def cc_stream_to_sse(cc_stream, model, req_id):
 # HTTP Server
 # ═══════════════════════════════════════════════════════════════════
 
-_LOG_DIR = os.path.join(os.path.expanduser("~"), ".cache", "codex-proxy")
-os.makedirs(_LOG_DIR, exist_ok=True)
-
 def _log_resp(resp_id, status, output):
     try:
         import datetime as _dt
@@ -1209,6 +1208,11 @@ class Handler(http.server.BaseHTTPRequestHandler):
             self.send_header("Cache-Control", "no-cache")
             self.send_header("Connection", "keep-alive")
             self.end_headers()
+            if hasattr(self, 'connection') and self.connection:
+                try:
+                    self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+                except Exception:
+                    pass
 
             collected_events = []
             last_resp_id = None
@@ -1290,6 +1294,11 @@ class Handler(http.server.BaseHTTPRequestHandler):
         self.send_header("Cache-Control", "no-cache")
         self.send_header("Connection", "keep-alive")
         self.end_headers()
+        if hasattr(self, 'connection') and self.connection:
+            try:
+                self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+            except Exception:
+                pass
 
         last_resp_id = None
         last_output = None
@@ -1322,7 +1331,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
                    "max_tokens": body.get("max_output_tokens", 8192)}
         instructions = body.get("instructions", "").strip()
         if instructions:
-            an_body["system"] = instructions
+            an_body["system"] = [{"type": "text", "text": instructions,
+                                   "cache_control": {"type": "ephemeral"}}]
         for k in ("temperature", "top_p"):
             if k in body:
                 an_body[k] = body[k]
@@ -1429,6 +1439,11 @@ class Handler(http.server.BaseHTTPRequestHandler):
             self.send_header("Cache-Control", "no-cache")
             self.send_header("Connection", "keep-alive")
             self.end_headers()
+            if hasattr(self, 'connection') and self.connection:
+                try:
+                    self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+                except Exception:
+                    pass
             last_resp_id = None
             last_output = None
             for event in cc_stream_to_sse(upstream, model, body.get("request_id") or body.get("id")):
@@ -1476,6 +1491,11 @@ class Handler(http.server.BaseHTTPRequestHandler):
             self.send_header("Cache-Control", "no-cache")
             self.send_header("Connection", "keep-alive")
             self.end_headers()
+            if hasattr(self, 'connection') and self.connection:
+                try:
+                    self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+                except Exception:
+                    pass
             last_resp_id = None
             last_output = None
             last_status = None