feat: PostgreSQL kuyruk sistemi (SSE yerine queue+poll)

Ne yaptik: - ReportQueue modeli + Alembic migration (report_queue tablosu) - QueueWorker: SELECT FOR UPDATE SKIP LOCKED ile tek worker polling - 3 yeni endpoint: POST /api/queue/submit, GET /api/queue/{id}/status, GET /api/queue/active - Startup hook ile worker otomatik basliyor, shutdown'da duruyor - Stuck task recovery: 15 dk'dan eski PROCESSING tasklar PENDING'e donuyor Neden yaptik: - Esanli SSE rapor isteklerinde IP ban riski ve veri kaybi vardi - Tek worker ile sirayla isleniyor, rate limit korunuyor - Sifir yeni dependency: sadece PostgreSQL + mevcut scraper'lar Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-07-03 10:17:03 +00:00 · 2026-04-14 16:41:19 +03:00
parent 8cde3879b5
commit 8cbe100035
4 changed files with 580 additions and 46 deletions
--- a/backend/main.py
+++ b/backend/main.py
@@ -1,9 +1,10 @@
 """
 FastAPI Backend for Trendyol Admin Panel
 """
-from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks
+from fastapi import FastAPI, HTTPException, Depends, BackgroundTasks, Request, Security
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
+from fastapi.security import APIKeyHeader
 import asyncio
 import json as json_module
 from sqlalchemy.orm import Session
@@ -22,7 +23,7 @@ from collections import OrderedDict
 from threading import Lock
 import os

-from database import SessionLocal, Category, Snapshot, Report, EnrichmentError, init_db
+from database import SessionLocal, Category, Snapshot, Report, EnrichmentError, ReportQueue, init_db
 from google_trends_helper import estimate_traffic_sources, fetch_google_trends
 from logging_config import setup_logging, get_logger, set_correlation_id, set_report_id, log_timing

@@ -40,6 +41,32 @@ log_keywords = get_logger("keywords")

 init_db()

+# ============================================================================
+# API KEY AUTHENTICATION
+# ============================================================================
+
+API_KEY = os.getenv("API_KEY")
+if not API_KEY:
+    import warnings
+    warnings.warn("API_KEY env var not set! API authentication disabled in development.")
+API_KEY_HEADER = APIKeyHeader(name="X-API-Key", auto_error=False)
+
+# Paths that do not require API key authentication
+PUBLIC_PATHS = {"/health"}
+
+async def verify_api_key(request: Request, api_key: Optional[str] = Security(API_KEY_HEADER)):
+    """
+    Global dependency that validates X-API-Key header.
+    Skips authentication for public paths (health check, docs).
+    """
+    if request.url.path in PUBLIC_PATHS:
+        return
+    if not api_key or api_key != API_KEY:
+        raise HTTPException(
+            status_code=401,
+            detail="Invalid or missing API key"
+        )
+
 # GS1 Barcode Prefix to Country Mapping (EAN-13 / EAN-8)
 # Source: https://www.gs1.org/standards/id-keys/company-prefix
 BARCODE_PREFIX_TO_COUNTRY = {
@@ -259,7 +286,11 @@ def get_country_from_barcode(barcode: str) -> str:
    prefix = barcode[:3]
    return BARCODE_PREFIX_TO_COUNTRY.get(prefix, "Bilinmeyen")

-app = FastAPI(title="Trendyol Admin API", version="1.0.0")
+app = FastAPI(
+    title="Trendyol Admin API",
+    version="1.0.0",
+    dependencies=[Depends(verify_api_key)]
+)

 # Base directory for resolving relative paths
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -273,45 +304,18 @@ CATEGORY_TREE_PATH = os.getenv("CATEGORY_TREE_PATH", os.path.join(BASE_DIR, ".."
 DATABASE_PATH = os.getenv("DATABASE_PATH", os.path.join(BASE_DIR, "trendyol.db"))

 # CORS for React admin panel
-# Security: Specify exact origins instead of wildcard
-# Supports: Local development, Docker Compose, and production deployment
-allowed_origins = [
-    # Local development (Vite dev server)
-    "http://localhost:5173",
-    "http://localhost:5174",
-    "http://localhost:5175",
-    "http://localhost:5176",
-    "http://localhost:3000",
-    "http://127.0.0.1:5173",
-    "http://127.0.0.1:5174",
-    "http://127.0.0.1:5175",
-    "http://127.0.0.1:5176",
-    "http://127.0.0.1:3000",
-    # Docker Compose internal networking
-    "http://frontend",
-    "http://frontend:80",
-    # Docker host access (mapped ports)
-    "http://localhost:80",
-    "http://localhost:8080",
-    "http://127.0.0.1:80",
-    "http://127.0.0.1:8080",
-    # Production server (Coolify)
-    "http://194.187.253.230:3010",
-    "http://194.187.253.230",
-    # Coolify Traefik proxy (sslip.io)
-    "http://trendyol.194.187.253.230.sslip.io",
-    "https://trendyol.194.187.253.230.sslip.io",
-    "http://trendyol-api.194.187.253.230.sslip.io",
-    "https://trendyol-api.194.187.253.230.sslip.io",
-]
-
-# Add production domain from environment variable
-frontend_url = os.getenv("FRONTEND_URL")
-if frontend_url:
-    allowed_origins.append(frontend_url)
-    # Also add https variant if http is provided
-    if frontend_url.startswith("http://"):
-        allowed_origins.append(frontend_url.replace("http://", "https://"))
+# Security: Environment-based origin control
+allowed_origins = []
+if os.getenv("ENV", "development") == "production":
+    allowed_origins = [
+        "https://trendyol.194.187.253.230.sslip.io",
+    ]
+else:
+    allowed_origins = [
+        "http://localhost:5173",
+        "http://localhost:3000",
+        "http://127.0.0.1:5173",
+    ]

 app.add_middleware(
    CORSMiddleware,
@@ -1603,7 +1607,8 @@ async def create_report(
            # This is a leaf — return itself
            cat = tree_by_id.get(parent_id)
            if cat:
-                return [(None, cat["name"], cat["id"])]
+                path_model = (cat.get("url") or "").lstrip("/") or None
+                return [(path_model, cat["name"], cat["id"])]
            return []
        leaves = []
        for child in children:
@@ -1883,6 +1888,68 @@ async def create_report(
    return StreamingResponse(progress_stream(), media_type="text/event-stream")


+# ============================================================================
+# QUEUE ENDPOINTS (new — replaces SSE for SellerX Java backend)
+# ============================================================================
+
+class QueueSubmitRequest(BaseModel):
+    name: str
+    category_id: int
+
+@app.post("/api/queue/submit", status_code=202)
+def submit_to_queue(req: QueueSubmitRequest, db: Session = Depends(get_db)):
+    """Submit a report generation task to the queue. Returns 202 Accepted."""
+    # Validate category exists
+    tree = _load_category_tree()
+    tree_by_id = {c["id"]: c for c in tree}
+    if req.category_id not in tree_by_id:
+        raise HTTPException(status_code=404, detail=f"Category {req.category_id} not found")
+
+    queue_item = ReportQueue(
+        report_name=req.name,
+        category_id=req.category_id,
+        status="PENDING",
+        progress=0,
+    )
+    db.add(queue_item)
+    db.commit()
+    db.refresh(queue_item)
+
+    # Calculate position in queue
+    position = db.query(func.count(ReportQueue.id)).filter(
+        ReportQueue.status == "PENDING",
+        ReportQueue.created_at <= queue_item.created_at,
+    ).scalar() or 1
+
+    log_api.info(f"Queue submit: id={queue_item.id}, name={req.name}, category={req.category_id}, position={position}")
+    return {"queue_id": queue_item.id, "status": "PENDING", "position": position}
+
+
+@app.get("/api/queue/{queue_id}/status")
+def get_queue_status(queue_id: int, db: Session = Depends(get_db)):
+    """Get the status of a queued report task."""
+    item = db.query(ReportQueue).filter(ReportQueue.id == queue_id).first()
+    if not item:
+        raise HTTPException(status_code=404, detail="Queue item not found")
+
+    return {
+        "queue_id": item.id,
+        "status": item.status,
+        "progress": item.progress,
+        "message": item.message or "",
+        "result_report_id": item.result_report_id,
+        "error": item.error,
+    }
+
+
+@app.get("/api/queue/active")
+def get_queue_info(db: Session = Depends(get_db)):
+    """Get queue overview: pending and processing counts."""
+    pending = db.query(func.count(ReportQueue.id)).filter(ReportQueue.status == "PENDING").scalar() or 0
+    processing = db.query(func.count(ReportQueue.id)).filter(ReportQueue.status == "PROCESSING").scalar() or 0
+    return {"pending_count": pending, "processing_count": processing}
+
+
 # Update report

@app.get("/api/reports/{report_id}", response_model=ReportResponse)
@@ -4040,6 +4107,24 @@ async def _start_resource_logger():
    _resource_logger.info("Periodic resource logger started (60s interval)")


+# ── Queue Worker Lifecycle ──────────────────────────────────────────────
+_queue_worker = None
+
+@app.on_event("startup")
+async def _start_queue_worker():
+    global _queue_worker
+    from queue_worker import QueueWorker
+    _queue_worker = QueueWorker(poll_interval=2.0)
+    _queue_worker.start()
+
+@app.on_event("shutdown")
+async def _stop_queue_worker():
+    global _queue_worker
+    if _queue_worker:
+        _queue_worker.stop()
+        _queue_worker = None
+
+
 if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8001)