llm: switch to single-model Qwen (Fireworks), drop heartbeats; throttle to avoid 429

llm_provider.py

@@ -1,16 +1,12 @@
 from __future__ import annotations
 """
 llm_provider.py
-- Primary: Fireworks OpenAI-compatible API (chat first, fallback to completions)
-- Secondary: Hugging Face Inference Router (provider="fireworks-ai") if HF_TOKEN present
-- Throttle & retry tuned for demo stability
+- Single provider: Fireworks (OpenAI-compatible) with Qwen3-Coder-30B-A3B-Instruct
+- Optional fallback: Hugging Face Inference Router (provider="fireworks-ai") if HF_TOKEN present
+- No heartbeats (avoid rate hits); conservative throttling & retries
 """
 
-import os
-import time
-import random
-import threading
-import logging
+import os, time, random, threading, logging
 from typing import List
 
 from dotenv import load_dotenv
@@ -21,7 +17,7 @@ from langchain_core.outputs import ChatGeneration, ChatResult
 
 # Fireworks (OpenAI-compatible)
 from openai import OpenAI
-from openai import RateLimitError, BadRequestError
+from openai import RateLimitError
 
 # HF Router (provider routing)
 from huggingface_hub import InferenceClient
@@ -30,10 +26,8 @@ load_dotenv()
 log = logging.getLogger("fraud-analyst")
 logging.basicConfig(level=logging.INFO)
 
-# --------- Public constant used by app.py when LLM is not available ----------
 SUMMARY_NOTICE = "🔌 Please connect to an inference point to generate summary."
 
-# --------- Read secrets (support your HF repo secret name) ----------
 def _first_env(*names: List[str]):
     for n in names:
         v = os.getenv(n)
@@ -41,31 +35,25 @@ def _first_env(*names: List[str]):
             return v
     return None
 
+# Secrets
 FIREWORKS_API_KEY = _first_env(
-    "fireworks_api_huggingface",      # your HF Space/Repo secret
-    "FIREWORKS_API_HUGGINGFACE",
-    "FIREWORKS_API_KEY",
-    "OPENAI_API_KEY",                 # allow reuse
+    "fireworks_api_huggingface", "FIREWORKS_API_HUGGINGFACE",
+    "FIREWORKS_API_KEY", "OPENAI_API_KEY"
 )
 HF_TOKEN = _first_env("HF_TOKEN", "HUGGINGFACE_TOKEN")
 
-# --------- Models ----------
-# Fireworks (use accounts/fireworks path)
-FW_PRIMARY_MODEL   = os.getenv("FW_PRIMARY_MODEL",   "accounts/fireworks/models/gpt-oss-20b")
-FW_SECONDARY_MODEL = os.getenv("FW_SECONDARY_MODEL", "accounts/fireworks/models/qwen3-coder-30b-a3b-instruct")
+# Models (Qwen only)
+FW_PRIMARY_MODEL = os.getenv("FW_PRIMARY_MODEL", "accounts/fireworks/models/qwen3-coder-30b-a3b-instruct")
+HF_PRIMARY_MODEL = os.getenv("HF_PRIMARY_MODEL", "Qwen/Qwen3-Coder-30B-A3B-Instruct")
 
-# Hugging Face Router IDs (plain slugs)
-HF_PRIMARY_MODEL   = os.getenv("HF_PRIMARY_MODEL",   "fireworks/gpt-oss-20b")
-HF_SECONDARY_MODEL = os.getenv("HF_SECONDARY_MODEL", "Qwen/Qwen3-Coder-30B-A3B-Instruct")
+# Throttle / Retry (conservative for demo)
+MAX_NEW_TOKENS  = int(os.getenv("LLM_MAX_NEW_TOKENS", "96"))
+TEMP            = float(os.getenv("LLM_TEMPERATURE", "0.2"))
+MAX_RETRIES     = int(os.getenv("LLM_MAX_RETRIES", "2"))
+MIN_INTERVAL_S  = float(os.getenv("LLM_MIN_INTERVAL_S", "1.0"))
+MAX_CONCURRENCY = int(os.getenv("LLM_MAX_CONCURRENCY", "1"))
 
-# --------- Throttle / Retry knobs (tuned for demo stability) ----------
-MAX_NEW_TOKENS   = int(os.getenv("LLM_MAX_NEW_TOKENS", "96"))
-TEMP             = float(os.getenv("LLM_TEMPERATURE", "0.2"))
-MAX_RETRIES      = int(os.getenv("LLM_MAX_RETRIES", "2"))
-MIN_INTERVAL_S   = float(os.getenv("LLM_MIN_INTERVAL_S", "1.0"))
-MAX_CONCURRENCY  = int(os.getenv("LLM_MAX_CONCURRENCY", "1"))
-
-# Ensure OpenAI SDK itself doesn't add extra retries
+# Ensure OpenAI SDK itself doesn't also retry
 os.environ.setdefault("OPENAI_MAX_RETRIES", "0")
 
 # Global throttle across all instances
@@ -74,7 +62,7 @@ _last_call_ts = 0.0
 _ts_lock = threading.Lock()
 
 def _pace():
-    """Global pacing to avoid hitting 429 on demo plans."""
+    """Global pacing to avoid hitting 429s."""
     global _last_call_ts
     with _ts_lock:
         now = time.monotonic()
@@ -99,11 +87,8 @@ def _with_retries(fn):
 # ========================== Fireworks (OpenAI-compatible) ==========================
 FW_BASE = os.getenv("OPENAI_API_BASE", "https://api.fireworks.ai/inference/v1")
 
-class _ChatIncompatible(Exception):
-    """Raise to switch a model to /completions pathway (e.g., gpt-oss-20b)."""
-
 class FireworksOpenAIChat(BaseChatModel):
-    """Normal chat.completions driver."""
+    """Qwen on Fireworks via /chat/completions."""
     model: str
     api_key: str | None = None
     temperature: float = TEMP
@@ -149,102 +134,15 @@ class FireworksOpenAIChat(BaseChatModel):
                     text = ch.message.content
             return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text or ""))],
                               llm_output={"model": self.model, "endpoint": "chat"})
-        except BadRequestError as e:
-            # Fireworks returns this when a model is not chat-friendly.
-            if "Failed to format non-streaming choice" in str(e) or "invalid_request_error" in str(e):
-                raise _ChatIncompatible()
-            log.warning(f"FW chat BadRequest for {self.model}: {str(e)[:200]}")
-            return ChatResult(generations=[ChatGeneration(message=AIMessage(content=""))],
-                              llm_output={"error": str(e)})
-        except Exception as e:
-            log.warning(f"FW chat failed for {self.model}: {type(e).__name__}: {str(e)[:200]}")
-            return ChatResult(generations=[ChatGeneration(message=AIMessage(content=""))],
-                              llm_output={"error": str(e)})
-
-class FireworksOpenAICompletionChat(BaseChatModel):
-    """Wrap /completions as a chat model (robust for gpt-oss-20b)."""
-    model: str
-    api_key: str | None = None
-    temperature: float = TEMP
-    max_new_tokens: int = MAX_NEW_TOKENS
-
-    def __init__(self, **data):
-        super().__init__(**data)
-        self._client = OpenAI(base_url=FW_BASE, api_key=self.api_key, max_retries=0)
-
-    @property
-    def _llm_type(self) -> str:
-        return "fireworks_openai_completion_chat"
-
-    def _to_prompt(self, messages) -> str:
-        parts=[]
-        for m in messages:
-            if isinstance(m, SystemMessage):
-                parts.append(f"[System] {m.content}")
-            elif isinstance(m, HumanMessage):
-                parts.append(f"[User] {m.content}")
-            elif isinstance(m, AIMessage):
-                parts.append(f"[Assistant] {m.content}")
-            else:
-                parts.append(f"[User] {str(getattr(m,'content',m))}")
-        parts.append("[Assistant]")
-        return "\n".join(parts)
-
-    def _generate(self, messages, stop=None, run_manager=None, **kwargs) -> ChatResult:
-        if not self.api_key:
-            return ChatResult(generations=[ChatGeneration(message=AIMessage(content=""))],
-                              llm_output={"error": "no_api_key"})
-        prompt = self._to_prompt(messages)
-        def _call():
-            with _CALL_LOCK:
-                _pace()
-                return self._client.completions.create(
-                    model=self.model,
-                    prompt=prompt,
-                    temperature=kwargs.get("temperature", self.temperature),
-                    max_tokens=kwargs.get("max_tokens", self.max_new_tokens),
-                )
-        try:
-            resp = _with_retries(_call)
-            text = ""
-            if getattr(resp, "choices", None):
-                ch = resp.choices[0]
-                if getattr(ch, "text", None):
-                    text = ch.text
-            return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text or ""))],
-                              llm_output={"model": self.model, "endpoint": "completions"})
         except Exception as e:
-            log.warning(f"FW completion failed for {self.model}: {type(e).__name__}: {str(e)[:200]}")
+            # Return empty output; UI will show notice if needed
+            logging.warning(f"Fireworks(Qwen) failed: {type(e).__name__}: {str(e)[:200]}")
             return ChatResult(generations=[ChatGeneration(message=AIMessage(content=""))],
                               llm_output={"error": str(e)})
 
-def _heartbeat_fw_chat(model_id: str) -> bool:
-    if not FIREWORKS_API_KEY:
-        return False
-    try:
-        cli = OpenAI(base_url=FW_BASE, api_key=FIREWORKS_API_KEY, max_retries=0)
-        _ = cli.chat.completions.create(model=model_id, messages=[{"role":"user","content":"ping"}], max_tokens=1)
-        return True
-    except BadRequestError as e:
-        # chat-incompatible → let caller try completions heartbeat
-        if "Failed to format non-streaming choice" in str(e) or "invalid_request_error" in str(e):
-            return False
-        return False
-    except Exception:
-        return False
-
-def _heartbeat_fw_completion(model_id: str) -> bool:
-    if not FIREWORKS_API_KEY:
-        return False
-    try:
-        cli = OpenAI(base_url=FW_BASE, api_key=FIREWORKS_API_KEY, max_retries=0)
-        _ = cli.completions.create(model=model_id, prompt="ping", max_tokens=1)
-        return True
-    except Exception:
-        return False
-
 # ========================== HF Router (provider="fireworks-ai") ==========================
 class HFRouterChat(BaseChatModel):
+    """Fallback only if FIREWORKS_API_KEY is absent but HF_TOKEN is set."""
     model: str
     hf_token: str | None = None
     temperature: float = TEMP
@@ -252,7 +150,6 @@ class HFRouterChat(BaseChatModel):
 
     def __init__(self, **data):
         super().__init__(**data)
-        # This reaches Fireworks through HF Router. Needs HF_TOKEN.
         self._client = InferenceClient(provider="fireworks-ai", api_key=self.hf_token)
 
     @property
@@ -276,7 +173,7 @@ class HFRouterChat(BaseChatModel):
             with _CALL_LOCK:
                 _pace()
                 return self._client.chat.completions.create(
-                    model=self.model,  # e.g. "fireworks/gpt-oss-20b"
+                    model=self.model,  # "Qwen/Qwen3-Coder-30B-A3B-Instruct"
                     messages=self._convert(messages),
                     stream=False,
                     max_tokens=kwargs.get("max_tokens", self.max_new_tokens),
@@ -294,50 +191,23 @@ class HFRouterChat(BaseChatModel):
             return ChatResult(generations=[ChatGeneration(message=AIMessage(content=text or ""))],
                               llm_output={"model": self.model})
         except Exception as e:
-            log.warning(f"HF Router call failed for {self.model}: {type(e).__name__}: {str(e)[:200]}")
+            logging.warning(f"HF Router(Qwen) failed: {type(e).__name__}: {str(e)[:200]}")
             return ChatResult(generations=[ChatGeneration(message=AIMessage(content=""))],
                               llm_output={"error": str(e)})
 
-def _heartbeat_hf_router(model_id: str) -> bool:
-    if not HF_TOKEN:
-        return False
-    try:
-        cli = InferenceClient(provider="fireworks-ai", api_key=HF_TOKEN)
-        _ = cli.chat.completions.create(model=model_id, messages=[{"role":"user","content":"ping"}], stream=False, max_tokens=1)
-        return True
-    except Exception:
-        return False
-
 # =============================== Selection ===============================
 def build_chat_llm():
-    # Prefer Fireworks direct
+    # Prefer Fireworks direct (Qwen)
     if FIREWORKS_API_KEY:
-        # Try primary as chat, then completions
-        if _heartbeat_fw_chat(FW_PRIMARY_MODEL):
-            log.info(f"Using Fireworks chat model: {FW_PRIMARY_MODEL}")
-            return FireworksOpenAIChat(model=FW_PRIMARY_MODEL, api_key=FIREWORKS_API_KEY)
-        elif _heartbeat_fw_completion(FW_PRIMARY_MODEL):
-            log.info(f"Using Fireworks COMPLETION-wrapped model: {FW_PRIMARY_MODEL}")
-            return FireworksOpenAICompletionChat(model=FW_PRIMARY_MODEL, api_key=FIREWORKS_API_KEY)
-
-        # Secondary chain
-        if _heartbeat_fw_chat(FW_SECONDARY_MODEL):
-            log.info(f"Using Fireworks chat model (fallback): {FW_SECONDARY_MODEL}")
-            return FireworksOpenAIChat(model=FW_SECONDARY_MODEL, api_key=FIREWORKS_API_KEY)
-        elif _heartbeat_fw_completion(FW_SECONDARY_MODEL):
-            log.info(f"Using Fireworks COMPLETION-wrapped model (fallback): {FW_SECONDARY_MODEL}")
-            return FireworksOpenAICompletionChat(model=FW_SECONDARY_MODEL, api_key=FIREWORKS_API_KEY)
+        log.info(f"Using Fireworks chat model: {FW_PRIMARY_MODEL}")
+        return FireworksOpenAIChat(model=FW_PRIMARY_MODEL, api_key=FIREWORKS_API_KEY)
 
-    # Else try HF Router (requires HF_TOKEN)
-    if HF_TOKEN and _heartbeat_hf_router(HF_PRIMARY_MODEL):
+    # Fallback to HF Router (if provided)
+    if HF_TOKEN:
         log.info(f"Using HF Router chat model: {HF_PRIMARY_MODEL}")
         return HFRouterChat(model=HF_PRIMARY_MODEL, hf_token=HF_TOKEN)
-    if HF_TOKEN and _heartbeat_hf_router(HF_SECONDARY_MODEL):
-        log.info(f"Using HF Router chat model (fallback): {HF_SECONDARY_MODEL}")
-        return HFRouterChat(model=HF_SECONDARY_MODEL, hf_token=HF_TOKEN)
 
     log.warning("No working chat model; notice will be shown.")
     return None
 
-# Singleton used by app.py and agent.py
 CHAT_LLM = build_chat_llm()