mirror of
https://github.com/nethunterzist/trendyol-analiz
synced 2026-07-01 01:17:04 +00:00
feat: PostgreSQL kuyruk sistemi (SSE yerine queue+poll)
Ne yaptik:
- ReportQueue modeli + Alembic migration (report_queue tablosu)
- QueueWorker: SELECT FOR UPDATE SKIP LOCKED ile tek worker polling
- 3 yeni endpoint: POST /api/queue/submit, GET /api/queue/{id}/status, GET /api/queue/active
- Startup hook ile worker otomatik basliyor, shutdown'da duruyor
- Stuck task recovery: 15 dk'dan eski PROCESSING tasklar PENDING'e donuyor
Neden yaptik:
- Esanli SSE rapor isteklerinde IP ban riski ve veri kaybi vardi
- Tek worker ile sirayla isleniyor, rate limit korunuyor
- Sifir yeni dependency: sadece PostgreSQL + mevcut scraper'lar
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
Database setup and models - PostgreSQL
|
||||
"""
|
||||
from sqlalchemy import create_engine, Column, Integer, String, DateTime, Boolean, ForeignKey
|
||||
from sqlalchemy import create_engine, Column, Integer, String, DateTime, Boolean, ForeignKey, Text
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker, relationship
|
||||
from datetime import datetime
|
||||
@@ -12,8 +12,7 @@ log = get_logger("db")
|
||||
|
||||
# PostgreSQL database - configurable via environment variable
|
||||
# Default: Local PostgreSQL for development
|
||||
# Docker: postgresql://postgres:trendyol123@postgres:5432/trendyol_db
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:trendyol123@localhost:5433/trendyol_db")
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres@localhost:5433/trendyol_db")
|
||||
|
||||
engine = create_engine(DATABASE_URL, pool_pre_ping=True)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
@@ -84,6 +83,25 @@ class EnrichmentError(Base):
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ReportQueue(Base):
|
||||
"""Queue for report generation tasks"""
|
||||
__tablename__ = "report_queue"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
report_name = Column(String, nullable=False)
|
||||
category_id = Column(Integer, nullable=False)
|
||||
status = Column(String, nullable=False, default="PENDING")
|
||||
progress = Column(Integer, nullable=False, default=0)
|
||||
message = Column(Text, nullable=True)
|
||||
result_report_id = Column(Integer, nullable=True)
|
||||
error = Column(Text, nullable=True)
|
||||
worker_id = Column(String, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
started_at = Column(DateTime, nullable=True)
|
||||
completed_at = Column(DateTime, nullable=True)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
def init_db():
|
||||
"""Initialize database - create tables"""
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
Reference in New Issue
Block a user