diff --git a/opus_orchestrator/__init__.py b/opus_orchestrator/__init__.py index 674312d..8946338 100644 --- a/opus_orchestrator/__init__.py +++ b/opus_orchestrator/__init__.py @@ -8,3 +8,138 @@ Quick Start: For full documentation, see README.md """ + +# Lazy imports to avoid circular dependency on cold starts +# See: https://peps.python.org/pep-0562/ +def __getattr__(name: str): + """Lazy import for module-level attributes to break circular imports.""" + + # Core exports + if name == "OpusOrchestrator": + from opus_orchestrator.orchestrator import OpusOrchestrator + return OpusOrchestrator + if name == "run_opus": + from opus_orchestrator.langgraph_workflow import run_opus + return run_opus + if name == "OpusConfig": + from opus_orchestrator.config import OpusConfig + return OpusConfig + if name == "get_config": + from opus_orchestrator.config import get_config + return get_config + if name == "set_config": + from opus_orchestrator.config import set_config + return set_config + if name == "OpusState": + from opus_orchestrator.state import OpusState + return OpusState + if name == "NonfictionGenerator": + from opus_orchestrator.nonfiction_generator import NonfictionGenerator + return NonfictionGenerator + if name == "OpusLogger": + from opus_orchestrator.logging import OpusLogger + return OpusLogger + if name == "get_logger": + from opus_orchestrator.logging import get_logger + return get_logger + + # Frameworks + if name == "StoryFramework": + from opus_orchestrator.frameworks import StoryFramework + return StoryFramework + if name == "FRAMEWORKS": + from opus_orchestrator.frameworks import FRAMEWORKS + return FRAMEWORKS + if name == "get_framework_for_genre": + from opus_orchestrator.frameworks import get_framework_for_genre + return get_framework_for_genre + if name == "get_framework_prompt": + from opus_orchestrator.frameworks import get_framework_prompt + return get_framework_prompt + + # Schemas + if name == "BookBlueprint": + from opus_orchestrator.schemas import BookBlueprint + return BookBlueprint + if name == "BookIntent": + from opus_orchestrator.schemas import BookIntent + return BookIntent + if name == "BookType": + from opus_orchestrator.schemas import BookType + return BookType + if name == "Chapter": + from opus_orchestrator.schemas import Chapter + return Chapter + if name == "ChapterBlueprint": + from opus_orchestrator.schemas import ChapterBlueprint + return ChapterBlueprint + if name == "ChapterCritique": + from opus_orchestrator.schemas import ChapterCritique + return ChapterCritique + if name == "ChapterDraft": + from opus_orchestrator.schemas import ChapterDraft + return ChapterDraft + if name == "Manuscript": + from opus_orchestrator.schemas import Manuscript + return Manuscript + if name == "RawContent": + from opus_orchestrator.schemas import RawContent + return RawContent + + # Utilities + if name == "LLMClient": + from opus_orchestrator.utils.llm import LLMClient + return LLMClient + if name == "get_llm_client": + from opus_orchestrator.utils.llm import get_llm_client + return get_llm_client + if name == "RetryHandler": + from opus_orchestrator.utils.retry import RetryHandler + return RetryHandler + if name == "CircuitBreaker": + from opus_orchestrator.utils.retry import CircuitBreaker + return CircuitBreaker + if name == "with_retry": + from opus_orchestrator.utils.retry import with_retry + return with_retry + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__version__ = "0.2.0" + +# Explicit __all__ for static analysis and IDE support +__all__ = [ + # Core + "OpusOrchestrator", + "OpusConfig", + "get_config", + "set_config", + "OpusState", + "NonfictionGenerator", + "run_opus", + # Logging + "OpusLogger", + "get_logger", + # Frameworks + "StoryFramework", + "FRAMEWORKS", + "get_framework_for_genre", + "get_framework_prompt", + # Schemas + "BookBlueprint", + "BookIntent", + "BookType", + "Chapter", + "ChapterBlueprint", + "ChapterCritique", + "ChapterDraft", + "Manuscript", + "RawContent", + # Utilities + "LLMClient", + "get_llm_client", + "RetryHandler", + "CircuitBreaker", + "with_retry", +] diff --git a/opus_orchestrator/agents/base.py b/opus_orchestrator/agents/base.py index e44dcfc..92b9a72 100644 --- a/opus_orchestrator/agents/base.py +++ b/opus_orchestrator/agents/base.py @@ -87,7 +87,8 @@ class BaseAgent(ABC, Generic[T]): """ temp = temperature if temperature is not None else self.config.temperature - return await self.llm_client.complete( + # Use async version for async context + return await self.llm_client.complete_async( system_prompt=system_prompt, user_prompt=user_prompt, temperature=temp, diff --git a/opus_orchestrator/agents/fiction/editor.py b/opus_orchestrator/agents/fiction/editor.py index 9ce094d..de4571a 100644 --- a/opus_orchestrator/agents/fiction/editor.py +++ b/opus_orchestrator/agents/fiction/editor.py @@ -182,7 +182,7 @@ Provide: if numbers: score = float(numbers[0]) break - except: + except (ValueError, IndexError): pass return AgentResponse( diff --git a/opus_orchestrator/agents/research.py b/opus_orchestrator/agents/research.py index 9315b4a..8310a03 100644 --- a/opus_orchestrator/agents/research.py +++ b/opus_orchestrator/agents/research.py @@ -8,7 +8,6 @@ from typing import Any, Optional from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.agents.base import BaseAgent, AgentResponse from opus_orchestrator.utils.research import ( diff --git a/opus_orchestrator/autogen_critique.py b/opus_orchestrator/autogen_critique.py index aa241ed..53eb281 100644 --- a/opus_orchestrator/autogen_critique.py +++ b/opus_orchestrator/autogen_critique.py @@ -9,7 +9,6 @@ from typing import Any, Optional from dotenv import load_dotenv -load_dotenv() from autogen import ConversableAgent, GroupChat, GroupChatManager diff --git a/opus_orchestrator/crews/base_crew.py b/opus_orchestrator/crews/base_crew.py index 6c3983f..cbe2906 100644 --- a/opus_orchestrator/crews/base_crew.py +++ b/opus_orchestrator/crews/base_crew.py @@ -9,7 +9,6 @@ from typing import Any, Optional from crewai import Agent, Crew, LLM, Process, Task from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.config import get_config diff --git a/opus_orchestrator/crews/fiction_crew.py b/opus_orchestrator/crews/fiction_crew.py index aaac191..6150410 100644 --- a/opus_orchestrator/crews/fiction_crew.py +++ b/opus_orchestrator/crews/fiction_crew.py @@ -8,7 +8,6 @@ from typing import Any, Optional from crewai import Agent, Process from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.crews.base_crew import OpusCrew from opus_orchestrator.config import get_config diff --git a/opus_orchestrator/crews/nonfiction_crew.py b/opus_orchestrator/crews/nonfiction_crew.py index f4c47fb..23932aa 100644 --- a/opus_orchestrator/crews/nonfiction_crew.py +++ b/opus_orchestrator/crews/nonfiction_crew.py @@ -8,7 +8,6 @@ from typing import Any, Optional from crewai import Agent, Process from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.crews.base_crew import OpusCrew from opus_orchestrator.config import get_config diff --git a/opus_orchestrator/langgraph_workflow.py b/opus_orchestrator/langgraph_workflow.py index c2316df..35aa98e 100644 --- a/opus_orchestrator/langgraph_workflow.py +++ b/opus_orchestrator/langgraph_workflow.py @@ -140,11 +140,15 @@ class OpusGraph: self.framework = framework self.genre = genre self.target_word_count = target_word_count - self.api_key = api_key or os.environ.get("OPENAI_API_KEY") + + # Get API key and provider from environment + self.api_key = api_key or os.environ.get("MINIMAX_API_KEY") or os.environ.get("OPENAI_API_KEY") + self.provider = os.environ.get("OPUS_PROVIDER", "minimax") + self.model = os.environ.get("OPUS_MODEL", "MiniMax-M2.5") self.use_autogen = use_autogen - # Use synchronous LLM - self.llm = LLMClient(api_key=self.api_key, provider="openai", model="gpt-4o") + # Use synchronous LLM with config + self.llm = LLMClient(api_key=self.api_key, provider=self.provider, model=self.model) # AutoGen critique crew self.critique_crew = None diff --git a/opus_orchestrator/logging.py b/opus_orchestrator/logging.py index f5f937b..567133f 100644 --- a/opus_orchestrator/logging.py +++ b/opus_orchestrator/logging.py @@ -1,61 +1,145 @@ -"""Logging configuration for Opus. +"""Structured Logging for Opus Orchestrator. -Structured logging with levels, formats, and handlers. +Provides JSON-formatted logging for production environments. """ +import json import logging import sys -from pathlib import Path +from datetime import datetime +from typing import Any, Optional +from enum import Enum -def setup_logging( - level: str = "INFO", - log_file: str = None, - format: str = "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s", -) -> logging.Logger: - """Setup structured logging for Opus. +class LogLevel(str, Enum): + """Log levels.""" + DEBUG = "DEBUG" + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + CRITICAL = "CRITICAL" + + +class StructuredFormatter(logging.Formatter): + """JSON formatter for structured logging.""" + + def format(self, record: logging.LogRecord) -> str: + """Format log record as JSON.""" + log_data = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "module": record.module, + "function": record.funcName, + "line": record.lineno, + } + + # Add exception info if present + if record.exc_info: + log_data["exception"] = self.formatException(record.exc_info) + + # Add extra fields + if hasattr(record, "extra"): + log_data.update(record.extra) + + return json.dumps(log_data) + + +class OpusLogger: + """Structured logger for Opus.""" + + def __init__(self, name: str, level: str = "INFO"): + """Initialize logger. + + Args: + name: Logger name (usually module name) + level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + """ + self.logger = logging.getLogger(name) + self.logger.setLevel(getattr(logging, level.upper())) + + # Only add handler once + if not self.logger.handlers: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(StructuredFormatter()) + self.logger.addHandler(handler) + + def debug(self, message: str, **kwargs: Any) -> None: + """Log debug message.""" + self.logger.debug(message, extra={"extra": kwargs} if kwargs else {}) + + def info(self, message: str, **kwargs: Any) -> None: + """Log info message.""" + self.logger.info(message, extra={"extra": kwargs} if kwargs else {}) + + def warning(self, message: str, **kwargs: Any) -> None: + """Log warning message.""" + self.logger.warning(message, extra={"extra": kwargs} if kwargs else {}) + + def error(self, message: str, **kwargs: Any) -> None: + """Log error message.""" + self.logger.error(message, extra={"extra": kwargs} if kwargs else {}) + + def critical(self, message: str, **kwargs: Any) -> None: + """Log critical message.""" + self.logger.critical(message, extra={"extra": kwargs} if kwargs else {}) + + def log_request(self, method: str, path: str, status_code: int, duration_ms: float) -> None: + """Log HTTP request.""" + self.info( + "HTTP Request", + method=method, + path=path, + status_code=status_code, + duration_ms=duration_ms, + ) + + def log_llm_request(self, provider: str, model: str, duration_ms: float, success: bool) -> None: + """Log LLM API request.""" + self.info( + "LLM Request", + provider=provider, + model=model, + duration_ms=duration_ms, + success=success, + ) + + def log_generation(self, book_type: str, genre: str, word_count: int, duration_s: float) -> None: + """Log book generation.""" + self.info( + "Book Generation", + book_type=book_type, + genre=genre, + word_count=word_count, + duration_s=duration_s, + ) + + +def get_logger(name: str, level: Optional[str] = None) -> OpusLogger: + """Get a structured logger. Args: - level: DEBUG, INFO, WARNING, ERROR - log_file: Optional file path - format: Log message format + name: Logger name + level: Optional log level override Returns: - Configured logger + OpusLogger instance """ - # Create logger - logger = logging.getLogger("opus") - logger.setLevel(getattr(logging, level.upper())) - - # Clear existing handlers - logger.handlers.clear() - - # Console handler - console = logging.StreamHandler(sys.stdout) - console.setLevel(getattr(logging, level.upper())) - console.setFormatter(logging.Formatter(format)) - logger.addHandler(console) - - # File handler (optional) - if log_file: - log_path = Path(log_file) - log_path.parent.mkdir(parents=True, exist_ok=True) - - file_handler = logging.FileHandler(log_file) - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(logging.Formatter( - "%(asctime)s | %(levelname)-8s | %(name)s:%(lineno)d | %(message)s" - )) - logger.addHandler(file_handler) - - return logger + import os + env_level = os.environ.get("OPUS_LOG_LEVEL", level or "INFO") + return OpusLogger(name, env_level) -# Default logger -logger = setup_logging() - - -# Usage in modules: -# from opus_orchestrator.logging import logger -# logger.info("Starting generation") -# logger.error(f"Failed: {e}") +# Convenience function for quick logging +def log(name: str, level: str, message: str, **kwargs: Any) -> None: + """Quick logging function. + + Args: + name: Logger name + level: Log level + message: Log message + **kwargs: Additional context + """ + logger = get_logger(name) + getattr(logger, level.lower())(message, **kwargs) diff --git a/opus_orchestrator/nonfiction/classifier.py b/opus_orchestrator/nonfiction/classifier.py index be4bc1f..af15871 100644 --- a/opus_orchestrator/nonfiction/classifier.py +++ b/opus_orchestrator/nonfiction/classifier.py @@ -7,7 +7,7 @@ Usage: from opus_orchestrator.nonfiction.classifier import PurposeClassifier, ReaderPurpose classifier = PurposeClassifier() - result = await classifier.classify( + result = classifier._keyword_classify( concept="Leadership for introverts", target_audience="Introverted professionals who want to develop leadership skills", intended_outcome="Learn to lead with quiet confidence" @@ -19,20 +19,11 @@ Usage: """ import re -import json from dataclasses import dataclass -from enum import Enum from typing import Optional - -class ReaderPurpose(str, Enum): - """Why is the reader reading this book?""" - LEARN_HANDS_ON = "learn_hands_on" - UNDERSTAND = "understand" - TRANSFORM = "transform" - DECIDE = "decide" - REFERENCE = "reference" - BE_INSPIRED = "be_inspired" +# Import ReaderPurpose from taxonomy to avoid duplicate enum definitions +from opus_orchestrator.nonfiction_taxonomy import ReaderPurpose @dataclass diff --git a/opus_orchestrator/nonfiction/creative_frameworks.py b/opus_orchestrator/nonfiction/creative_frameworks.py index 45f3523..38b9cb5 100644 --- a/opus_orchestrator/nonfiction/creative_frameworks.py +++ b/opus_orchestrator/nonfiction/creative_frameworks.py @@ -22,7 +22,7 @@ CREATIVE_FRAMEWORKS = { "description": "Branching narrative where reader choices determine the story path. Multiple endings based on decisions.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Introduction - Set the scene", "Opening Choice - First decision point", @@ -59,7 +59,7 @@ Include: "description": "RPG-style adventure with combat, stats, and inventory. Reader/player is the protagonist.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Character Creation - Stats, skills", "Equipment List - Starting items", @@ -95,7 +95,7 @@ Include: "description": "Game script with character dialogues, scene descriptions, and choice points. Anime/VN style.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Title Screen / Opening", "Prologue", @@ -135,7 +135,7 @@ Include: "description": "Story told entirely through documents: letters, emails, texts, diary entries, etc.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Editor's Note - Frame narrative", "Document 1: Letter/Email/Text", @@ -170,7 +170,7 @@ Include: "description": "A story revealed through discovered materials: redacted files, journal fragments, annotated maps.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Foreword - How these were found", "Artifact 1: Document type", @@ -207,7 +207,7 @@ Include: "description": "Revolutionary call to action. A passionate argument for change.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.ARGUMENT, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Opening - The problem we've ignored", "Part 1: What Is - Current state", @@ -242,7 +242,7 @@ Include: "description": "Public letter to a specific person/entity. Makes a point through direct address.", "purpose": ReaderPurpose.DECIDE, "structure": StructuralPattern.ARGUMENT, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Salutation - Dear [Name/Entity]", "Opening - Why I'm writing publicly", @@ -279,7 +279,7 @@ Include: "description": "Endlessly extensible story. Each chapter ends with a new beginning. For serialization.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Episode 1: Complete arc", "Episode 1 Cliffhanger - Link to next", @@ -313,7 +313,7 @@ Include: "description": "Story that repeats at different scales. Chapter mirrors scene mirrors paragraph.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Macro Level: The Overall Arc", "Structure mirrors down", @@ -344,7 +344,7 @@ Include: "description": "Non-linear collection of fragments: memories, images (described), ticket stubs, recipes, etc.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.SPIRAL, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Opening Spread - First impressions", "Fragment 1: Memory/Image", @@ -381,7 +381,7 @@ Include: "description": "Script for spoken audio. Includes banter, segments, transitions.", "purpose": ReaderPurpose.LEARN_HANDS_ON, "structure": StructuralPattern.SEQUENTIAL, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Intro - Hook + branding", "Cold Open - Tease", @@ -415,7 +415,7 @@ Include: "description": "Hollywood standard screenplay format. Visual storytelling through action and dialogue.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Title Page", "Fade In", @@ -451,7 +451,7 @@ Include: "description": "Theatrical script with scenes, stage directions, and dialogue.", "purpose": ReaderPurpose.BE_INSPIRED, "structure": StructuralPattern.NARRATIVE, - "category": NonfictionCategory.CREATIVE, + "category": NonfictionCategory.CREATIVITY, "stages": [ "Title Page", "Dramatis Personae - Characters", diff --git a/opus_orchestrator/nonfiction_generator.py b/opus_orchestrator/nonfiction_generator.py index dfd2199..994d154 100644 --- a/opus_orchestrator/nonfiction_generator.py +++ b/opus_orchestrator/nonfiction_generator.py @@ -8,7 +8,6 @@ from typing import Any, Optional from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.nonfiction_frameworks import ( NonfictionFramework, diff --git a/opus_orchestrator/nonfiction_taxonomy.py b/opus_orchestrator/nonfiction_taxonomy.py index 5e8ff72..ddec38f 100644 --- a/opus_orchestrator/nonfiction_taxonomy.py +++ b/opus_orchestrator/nonfiction_taxonomy.py @@ -58,6 +58,9 @@ class NonfictionCategory(str, Enum): CREATIVITY = "creativity" SPIRITUALITY = "spirituality" HOW_TO = "how_to" + EDUCATION = "education" + ACADEMIC = "academic" + RPG = "rpg" # ============================================================================ diff --git a/opus_orchestrator/orchestrator.py b/opus_orchestrator/orchestrator.py index 421a7bc..f0c4135 100644 --- a/opus_orchestrator/orchestrator.py +++ b/opus_orchestrator/orchestrator.py @@ -10,7 +10,6 @@ from typing import Any, Optional from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.agents.fiction import ( ArchitectAgent, @@ -347,8 +346,6 @@ Generate a detailed outline with: Returns: RawContent with the combined text from the repo """ - from opus_orchestrator.utils.github_ingest import GitHubIngestor - print(f"📥 Loading from GitHub: {repo}") github_token = self.config.github_token or os.environ.get("GITHUB_TOKEN") diff --git a/opus_orchestrator/pydanticai_agent.py b/opus_orchestrator/pydanticai_agent.py index dde2b1b..52044db 100644 --- a/opus_orchestrator/pydanticai_agent.py +++ b/opus_orchestrator/pydanticai_agent.py @@ -10,7 +10,6 @@ from pydantic import BaseModel from pydantic_ai import Agent from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.config import get_config diff --git a/opus_orchestrator/server.py b/opus_orchestrator/server.py index e45f619..fa82ce1 100644 --- a/opus_orchestrator/server.py +++ b/opus_orchestrator/server.py @@ -7,18 +7,46 @@ import os from typing import Any, Optional from contextlib import asynccontextmanager -from fastapi import FastAPI, HTTPException, BackgroundTasks, UploadFile, File, StreamingResponse +from fastapi import FastAPI, HTTPException, BackgroundTasks, UploadFile, File, StreamingResponse, Depends, Security from fastapi.responses import JSONResponse, RedirectResponse +from fastapi.security import APIKeyHeader from pydantic import BaseModel, Field from dotenv import load_dotenv -load_dotenv() from opus_orchestrator.config import get_config from opus_orchestrator import run_opus, OpusOrchestrator from opus_orchestrator.frameworks import FRAMEWORKS +# ============================================================================= +# AUTHENTICATION +# ============================================================================= + +API_KEY_HEADER = APIKeyHeader(name="X-API-Key", auto_error=False) + +async def get_api_key(api_key: str = Security(API_KEY_HEADER)) -> str: + """Validate API key from header or environment. + + If no API key is configured (for development), allow all requests. + Set OPUS_API_KEY environment variable to protect production endpoints. + """ + configured_key = os.environ.get("OPUS_API_KEY") + + # No key configured - allow all (development mode) + if not configured_key: + return "dev" + + # Key configured - validate + if api_key is None: + raise HTTPException(status_code=401, detail="API key required. Set X-API-Key header.") + + if api_key != configured_key: + raise HTTPException(status_code=403, detail="Invalid API key") + + return api_key + + # ============================================================================= # REQUEST/RESPONSE MODELS # ============================================================================= @@ -194,7 +222,7 @@ async def list_frameworks(): @app.post("/generate", response_model=GenerateResponse, tags=["generate"]) -async def generate(request: GenerateRequest, background_tasks: BackgroundTasks): +async def generate(request: GenerateRequest, background_tasks: BackgroundTasks, api_key: str = Depends(get_api_key)): """Generate a manuscript from concept or GitHub repo.""" import traceback try: @@ -302,7 +330,7 @@ async def generate_stream(request: GenerateRequest): return StreamingResponse(event_generator(), media_type="text/event-stream") @app.post("/ingest", response_model=IngestResponse, tags=["ingest"]) -async def ingest(request: IngestRequest): +async def ingest(request: IngestRequest, api_key: str = Depends(get_api_key)): """Ingest content from a GitHub repository.""" try: orch = OpusOrchestrator(book_type="fiction") @@ -351,7 +379,7 @@ class S3UploadResponse(BaseModel): @app.post("/upload", response_model=UploadResponse, tags=["upload"]) -async def upload_file(file: UploadFile = File(...)): +async def upload_file(file: UploadFile = File(...), api_key: str = Depends(get_api_key)): """Upload a file for processing.""" try: content = await file.read() @@ -368,7 +396,7 @@ async def upload_file(file: UploadFile = File(...)): @app.post("/upload/s3", response_model=S3UploadResponse, tags=["upload"]) -async def upload_to_s3(request: S3UploadRequest): +async def upload_to_s3(request: S3UploadRequest, api_key: str = Depends(get_api_key)): """Upload content to S3-compatible storage.""" try: from opus_orchestrator import S3Ingestor @@ -441,3 +469,64 @@ if __name__ == "__main__": port = int(sys.argv[1]) if len(sys.argv) > 1 else 8000 uvicorn.run(app, host="0.0.0.0", port=port) + +# ============================================================================= +# RATE LIMITING +# ============================================================================= +from fastapi import Request +from starlette.middleware.base import BaseHTTPMiddleware +import time +from collections import defaultdict + +class RateLimitMiddleware(BaseHTTPMiddleware): + """Simple in-memory rate limiter.""" + + def __init__(self, app, requests_per_minute: int = 30): + super().__init__(app) + self.requests_per_minute = requests_per_minute + self.requests = defaultdict(list) + + async def dispatch(self, request: Request, call_next): + # Skip rate limiting for health check + if request.url.path == "/health": + return await call_next(request) + + client_ip = request.client.host if request.client else "unknown" + current_time = time.time() + + # Clean old requests (older than 1 minute) + self.requests[client_ip] = [ + t for t in self.requests[client_ip] + if current_time - t < 60 + ] + + # Check rate limit + if len(self.requests[client_ip]) >= self.requests_per_minute: + return JSONResponse( + status_code=429, + content={"detail": "Rate limit exceeded. Please try again later."} + ) + + # Record this request + self.requests[client_ip].append(current_time) + + return await call_next(request) + + +# Get rate limit from environment, default to 30/minute +_rate_limit = int(os.environ.get("RATE_LIMIT_PER_MINUTE", "30")) +app.add_middleware(RateLimitMiddleware, requests_per_minute=_rate_limit) + +# CORS middleware - secure configuration +from fastapi.middleware.cors import CORSMiddleware + +# Get allowed origins from environment, default to restricted set +_cors_origins = os.environ.get("CORS_ORIGINS", "").split(",") if os.environ.get("CORS_ORIGINS") else [] + +app.add_middleware( + CORSMiddleware, + allow_origins=_cors_origins if _cors_origins else ["http://localhost:3000", "http://localhost:8000"], + allow_credentials=True if _cors_origins else False, + allow_methods=["GET", "POST", "PUT", "DELETE"], + allow_headers=["Content-Type", "Authorization"], +) diff --git a/opus_orchestrator/utils/agent_crawler.py b/opus_orchestrator/utils/agent_crawler.py index 665356e..5a14c11 100644 --- a/opus_orchestrator/utils/agent_crawler.py +++ b/opus_orchestrator/utils/agent_crawler.py @@ -284,7 +284,7 @@ Just respond with a number between 0.0 and 1.0.""" try: return float(response.strip()) - except: + except (ValueError, TypeError): return 0.5 def _simple_relevance(self, content: str, purpose: CrawlPurpose) -> float: diff --git a/opus_orchestrator/utils/github_ingest.py b/opus_orchestrator/utils/github_ingest.py index a33c2b2..5d8d6d8 100644 --- a/opus_orchestrator/utils/github_ingest.py +++ b/opus_orchestrator/utils/github_ingest.py @@ -9,9 +9,7 @@ import re from typing import Any, Optional import requests -from dotenv import load_dotenv - -load_dotenv() +# Note: dotenv loading removed - set GITHUB_TOKEN environment variable directly class GitHubIngestor: diff --git a/opus_orchestrator/utils/llm.py b/opus_orchestrator/utils/llm.py index fc4e1a6..1bb5804 100644 --- a/opus_orchestrator/utils/llm.py +++ b/opus_orchestrator/utils/llm.py @@ -1,6 +1,7 @@ """LLM client for Opus Orchestrator. Supports MiniMax and OpenAI providers - both async and sync. +Includes retry logic with exponential backoff and circuit breaker. """ import os @@ -10,9 +11,14 @@ from typing import Any, Optional import httpx import requests +from opus_orchestrator.utils.retry import RetryHandler, RetryConfig + class LLMClient: - """Simple LLM client for making API calls - supports both sync and async.""" + """Simple LLM client for making API calls - supports both sync and async. + + Includes built-in retry logic with circuit breaker for resilience. + """ def __init__( self, @@ -20,8 +26,17 @@ class LLMClient: provider: str = "minimax", model: str = "MiniMax/MiniMax-M2.1", base_url: Optional[str] = None, + max_retries: int = 3, ): - """Initialize LLM client.""" + """Initialize LLM client. + + Args: + api_key: API key for the provider + provider: LLM provider (minimax, openai) + model: Model name + base_url: Optional custom base URL + max_retries: Maximum retry attempts (default 3) + """ self.api_key = api_key or os.environ.get("MINIMAX_API_KEY") or os.environ.get("OPENAI_API_KEY") self.provider = provider self.model = model @@ -34,7 +49,8 @@ class LLMClient: if base_url: self.base_url = base_url elif provider == "minimax": - self.base_url = "https://api.minimax.chat/v1" + # Use Anthropic-compatible API (like OpenClaw uses) + self.base_url = "https://api.minimax.io/anthropic" elif provider == "openai": self.base_url = "https://api.openai.com/v1" else: @@ -42,6 +58,16 @@ class LLMClient: # Async client self._async_client = httpx.AsyncClient(timeout=120.0) + + # Initialize retry handler + retry_config = RetryConfig( + max_attempts=max_retries, + base_delay=1.0, + max_delay=30.0, + exponential_base=2.0, + jitter=True, + ) + self._retry_handler = RetryHandler(retry_config) def complete( self, @@ -74,24 +100,33 @@ class LLMClient: temperature: float = 0.7, max_tokens: Optional[int] = None, ) -> str: - """Make a completion request (ASYNC).""" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - } + """Make a completion request (ASYNC) with retry logic.""" - if self.provider == "minimax": - return await self._complete_minimax_async( - system_prompt, user_prompt, temperature, max_tokens, headers - ) - elif self.provider == "openai": - return await self._complete_openai_async( - system_prompt, user_prompt, temperature, max_tokens, headers - ) - else: - raise ValueError(f"Unsupported provider: {self.provider}") + async def _make_request(): + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + if self.provider == "minimax": + return await self._complete_minimax_async( + system_prompt, user_prompt, temperature, max_tokens, headers + ) + elif self.provider == "openai": + return await self._complete_openai_async( + system_prompt, user_prompt, temperature, max_tokens, headers + ) + else: + raise ValueError(f"Unsupported provider: {self.provider}") + + # Use retry handler for resilience + try: + return await self._retry_handler.execute_with_retry(_make_request) + except Exception as e: + # Log and re-raise with context + raise RuntimeError(f"LLM request failed after retries: {e}") from e - async def _complete_minimax( + async def _complete_minimax_async( self, system_prompt: str, user_prompt: str, @@ -99,8 +134,8 @@ class LLMClient: max_tokens: Optional[int], headers: dict, ) -> str: - """Call MiniMax API.""" - # MiniMax chat completion format + """Call MiniMax API using Anthropic-compatible endpoint.""" + # Anthropic-compatible format payload = { "model": self.minimax_model, "messages": [ @@ -113,9 +148,10 @@ class LLMClient: if max_tokens: payload["max_tokens"] = max_tokens + # Use Anthropic-compatible endpoint response = await self._async_client.post( - f"{self.base_url}/text/chatcompletion_v2", - headers=headers, + f"{self.base_url}/v1/messages", + headers={**headers, "Content-Type": "application/json"}, json=payload, ) @@ -127,13 +163,13 @@ class LLMClient: data = response.json() - # Handle different response formats - if "choices" in data: - return data["choices"][0]["message"]["content"] - elif "choices" in data.get("data", {}): - return data["data"]["choices"][0]["message"]["content"] + # Handle Anthropic-compatible response format + if "content" in data: + # Return the text content + if isinstance(data["content"], list) and len(data["content"]) > 0: + return data["content"][0].get("text", str(data["content"][0])) + return str(data["content"]) else: - # Try to find content in response raise Exception(f"Unexpected MiniMax response: {data}") async def _complete_openai( @@ -183,7 +219,7 @@ class LLMClient: max_tokens: Optional[int], headers: dict, ) -> str: - """Call MiniMax API (sync).""" + """Call MiniMax API (sync) using Anthropic-compatible endpoint.""" payload = { "model": self.minimax_model, "messages": [ @@ -196,9 +232,10 @@ class LLMClient: if max_tokens: payload["max_tokens"] = max_tokens + # Use Anthropic-compatible endpoint response = requests.post( - f"{self.base_url}/text/chatcompletion_v2", - headers=headers, + f"{self.base_url}/v1/messages", + headers={**headers, "Content-Type": "application/json"}, json=payload, timeout=120, ) @@ -210,10 +247,19 @@ class LLMClient: data = response.json() - if "choices" in data: - return data["choices"][0]["message"]["content"] - elif "choices" in data.get("data", {}): - return data["data"]["choices"][0]["message"]["content"] + # Handle Anthropic-compatible response format + if "content" in data: + if isinstance(data["content"], list) and len(data["content"]) > 0: + # Look for text content, skip thinking + text_parts = [] + for item in data["content"]: + if item.get("type") == "text": + text_parts.append(item.get("text", "")) + if text_parts: + return "".join(text_parts) + # If no text found, return first item as string + return str(data["content"][0]) + return str(data["content"]) else: raise Exception(f"Unexpected MiniMax response: {data}") diff --git a/opus_orchestrator/utils/multi_source_ingest.py b/opus_orchestrator/utils/multi_source_ingest.py index 82326ee..332c24c 100644 --- a/opus_orchestrator/utils/multi_source_ingest.py +++ b/opus_orchestrator/utils/multi_source_ingest.py @@ -193,7 +193,7 @@ class MultiSourceIngestor: text = f.read_text(encoding='utf-8', errors='ignore') rel_path = f.relative_to(path) content_parts.append(f"## {rel_path}\n\n{text}\n") - except: + except OSError: pass merged = "\n\n".join(content_parts) diff --git a/opus_orchestrator/utils/research.py b/opus_orchestrator/utils/research.py index aa7715f..e74ff0c 100644 --- a/opus_orchestrator/utils/research.py +++ b/opus_orchestrator/utils/research.py @@ -178,7 +178,7 @@ class WikipediaTool: "summary": page.summary[:500], "content": page.content[:2000], }) - except: + except Exception: continue return articles except Exception as e: diff --git a/opus_orchestrator/web_ui.py b/opus_orchestrator/web_ui.py index ebbf356..2910f42 100644 --- a/opus_orchestrator/web_ui.py +++ b/opus_orchestrator/web_ui.py @@ -14,7 +14,6 @@ from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from dotenv import load_dotenv -load_dotenv() # HTML Template for the UI diff --git a/test_memoir.py b/test_memoir.py new file mode 100644 index 0000000..447f132 --- /dev/null +++ b/test_memoir.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Test memoir ingestion.""" + +import asyncio +import os + +# Set token +os.environ["GITHUB_TOKEN"] = "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" + +async def main(): + from opus_orchestrator.nonfiction.intake import determine_intake + from opus_orchestrator.nonfiction import ReaderPurpose + + print("=== Testing Memoir Sources ===\n") + + # 1. Determine purpose + result = await determine_intake( + concept="A memoir about love, loss, and transformation", + purpose="transform", + category="memoir" + ) + print(f"1. PURPOSE: {result.purpose.value}") + print(f" Framework: {result.framework.get('name')}") + print(f" Stages: {len(result.framework.get('stages', []))}") + print(f" Source: {result.source}") + + # 2. Try GitHub sources + print("\n2. Ingesting from GitHub...") + from opus_orchestrator.utils.multi_source_ingest import ingest_multiple + + sources = [ + {"type": "github", "repo": "mrhavens/The-Last-Love-Story"}, + ] + + result = await ingest_multiple(sources) + print(f" Success: {result.successful_sources}/{result.total_sources}") + print(f" Content: {len(result.merged_content)} chars") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index d2082d7..d568544 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -186,7 +186,7 @@ class TestIntegration: """Integration tests - skip if no API key.""" @pytest.mark.skipif( - not __import__('os')..environ.get('OPENAI_API_KEY'), + not __import__('os').environ.get('OPENAI_API_KEY'), reason="No API key" ) def test_real_api_call(self):