mirror of
https://github.com/sortedcord/alemno-payments.git
synced 2026-07-22 04:02:49 +05:30
Compare commits
14 Commits
22f300474f
...
f3727af697
| Author | SHA1 | Date | |
|---|---|---|---|
| f3727af697 | |||
| 2e556173d4 | |||
| 40881b4773 | |||
| d980558bee | |||
| 0cc2b5fe57 | |||
| c00f27aa30 | |||
| 265c6704a7 | |||
| 0b20fd3263 | |||
| fa48a39277 | |||
| bcf99c66cd | |||
| 22b5cd8ced | |||
| f63cd4774c | |||
| a2a25d7bfb | |||
| 47b2db8a15 |
26
.github/workflows/lint.yml
vendored
Normal file
26
.github/workflows/lint.yml
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
name: Lint
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, master ]
|
||||
pull_request:
|
||||
branches: [ main, master ]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Run Ruff Check
|
||||
run: uv run --group dev ruff check .
|
||||
|
||||
- name: Run Ruff Format Check
|
||||
run: uv run --group dev ruff format --check .
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -6,6 +6,8 @@ dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
.env
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
|
||||
|
||||
84
app/api/v1/jobs.py
Normal file
84
app/api/v1/jobs.py
Normal file
@@ -0,0 +1,84 @@
|
||||
from typing import List, Optional
|
||||
from fastapi import APIRouter, Depends, File, UploadFile, Query, HTTPException, status
|
||||
from app.core.dependencies import get_job_service
|
||||
from app.core.exceptions import JobNotFoundException, InvalidCSVException
|
||||
from app.schemas.job import JobResponse, JobStatusResponse, JobResultResponse
|
||||
from app.services.job_service import JobService
|
||||
|
||||
router = APIRouter(prefix="/jobs", tags=["jobs"])
|
||||
|
||||
|
||||
@router.post("/upload", response_model=JobResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def upload_csv(
|
||||
file: UploadFile = File(...),
|
||||
service: JobService = Depends(get_job_service),
|
||||
):
|
||||
"""
|
||||
Accepts a CSV file upload. Validates it, creates a job record in the database
|
||||
with status=pending, enqueues the processing task, and returns the job_id immediately.
|
||||
"""
|
||||
if not file.filename.endswith(".csv"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Only CSV files are supported.",
|
||||
)
|
||||
|
||||
content = await file.read()
|
||||
try:
|
||||
job = await service.upload_csv(filename=file.filename, content=content)
|
||||
return job
|
||||
except InvalidCSVException as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/status", response_model=JobStatusResponse)
|
||||
async def get_job_status(
|
||||
job_id: str,
|
||||
service: JobService = Depends(get_job_service),
|
||||
):
|
||||
"""
|
||||
Returns the current status of the job: pending, processing, completed, or failed.
|
||||
If completed, also includes a summary field with high-level stats.
|
||||
"""
|
||||
try:
|
||||
job = await service.get_job_status(job_id)
|
||||
return job
|
||||
except JobNotFoundException as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{job_id}/results", response_model=JobResultResponse)
|
||||
async def get_job_result(
|
||||
job_id: str,
|
||||
service: JobService = Depends(get_job_service),
|
||||
):
|
||||
"""
|
||||
Returns the full structured output: cleaned transactions list, flagged anomalies,
|
||||
per-category spend breakdown, and the LLM-generated narrative summary.
|
||||
"""
|
||||
try:
|
||||
job = await service.get_job_results(job_id)
|
||||
return job
|
||||
except JobNotFoundException as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=List[JobResponse])
|
||||
async def get_all_jobs(
|
||||
status: Optional[str] = Query(None, description="Filter jobs by status"),
|
||||
service: JobService = Depends(get_job_service),
|
||||
):
|
||||
"""
|
||||
Lists all jobs with their status, filename, row count, and created_at timestamp.
|
||||
Supports filtering via ?status= query parameter.
|
||||
"""
|
||||
return await service.list_jobs(status=status)
|
||||
70
app/clients/exchange_rate_client.py
Normal file
70
app/clients/exchange_rate_client.py
Normal file
@@ -0,0 +1,70 @@
|
||||
import httpx
|
||||
import redis
|
||||
from app.core.config import settings
|
||||
from app.core.logging import logger
|
||||
|
||||
|
||||
class ExchangeRateClient:
|
||||
def __init__(self):
|
||||
self.default_rate = 93.0
|
||||
self.cache_key = "usd_to_inr_rate"
|
||||
self.cache_ttl = 86400 # 24 hours cache duration
|
||||
try:
|
||||
self.redis_client = redis.Redis(
|
||||
host=settings.REDIS_HOST,
|
||||
port=settings.REDIS_PORT,
|
||||
db=settings.REDIS_DB,
|
||||
decode_responses=True,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not connect to Redis for exchange rate caching: {e}")
|
||||
self.redis_client = None
|
||||
|
||||
async def get_usd_to_inr_rate(self) -> float:
|
||||
"""
|
||||
Retrieves the USD to INR conversion rate.
|
||||
Checks Redis cache first. If cache is empty/offline, fetches from a free public API
|
||||
and updates the cache. Falls back to a hardcoded default rate if both fail.
|
||||
"""
|
||||
# read from cache
|
||||
if self.redis_client:
|
||||
try:
|
||||
cached_rate = self.redis_client.get(self.cache_key)
|
||||
if cached_rate:
|
||||
logger.info("Retrieved exchange rate from Redis cache.")
|
||||
return float(cached_rate)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read exchange rate from Redis: {e}")
|
||||
|
||||
# fetching from Exchange Rate API
|
||||
try:
|
||||
logger.info("Fetching live exchange rate from API...")
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get("https://open.er-api.com/v6/latest/USD")
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
rates = data.get("rates", {})
|
||||
inr_rate = rates.get("INR")
|
||||
if inr_rate:
|
||||
inr_rate = float(inr_rate)
|
||||
logger.info(
|
||||
f"Successfully fetched live exchange rate: {inr_rate}"
|
||||
)
|
||||
# Cache the rate in Redis
|
||||
if self.redis_client:
|
||||
try:
|
||||
self.redis_client.set(
|
||||
self.cache_key, str(inr_rate), ex=self.cache_ttl
|
||||
)
|
||||
except Exception as ce:
|
||||
logger.warning(
|
||||
f"Failed to cache exchange rate in Redis: {ce}"
|
||||
)
|
||||
return inr_rate
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error fetching live exchange rate: {e}. Falling back to default rate {self.default_rate}"
|
||||
)
|
||||
|
||||
# 3. Fallback
|
||||
return self.default_rate
|
||||
47
app/core/config.py
Normal file
47
app/core/config.py
Normal file
@@ -0,0 +1,47 @@
|
||||
from pydantic import Field, computed_field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env", env_file_encoding="utf-8", extra="ignore"
|
||||
)
|
||||
|
||||
# General
|
||||
PROJECT_NAME: str = "Alemno Payments API"
|
||||
API_V1_STR: str = "/api/v1"
|
||||
|
||||
# PostgreSQL
|
||||
POSTGRES_USER: str = Field(default="postgres")
|
||||
POSTGRES_PASSWORD: str = Field(default="postgres")
|
||||
POSTGRES_HOST: str = Field(default="localhost")
|
||||
POSTGRES_PORT: int = Field(default=5432)
|
||||
POSTGRES_DB: str = Field(default="alemno_payments")
|
||||
|
||||
# Redis
|
||||
REDIS_HOST: str = Field(default="localhost")
|
||||
REDIS_PORT: int = Field(default=6379)
|
||||
REDIS_DB: int = Field(default=0)
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def DATABASE_URL(self) -> str:
|
||||
return (
|
||||
f"postgresql+asyncpg://"
|
||||
f"{self.POSTGRES_USER}:{self.POSTGRES_PASSWORD}@"
|
||||
f"{self.POSTGRES_HOST}:{self.POSTGRES_PORT}/"
|
||||
f"{self.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CELERY_BROKER_URL(self) -> str:
|
||||
return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_DB}"
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CELERY_RESULT_BACKEND(self) -> str:
|
||||
return f"redis://{self.REDIS_HOST}:{self.REDIS_PORT}/{self.REDIS_DB}"
|
||||
|
||||
|
||||
settings = Settings()
|
||||
15
app/core/dependencies.py
Normal file
15
app/core/dependencies.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from fastapi import Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.db.session import get_db
|
||||
from app.repositories.job_repository import JobRepository
|
||||
from app.services.job_service import JobService
|
||||
|
||||
|
||||
def get_job_repository(db: AsyncSession = Depends(get_db)) -> JobRepository:
|
||||
"""Dependency to retrieve JobRepository"""
|
||||
return JobRepository(db)
|
||||
|
||||
|
||||
def get_job_service(repo: JobRepository = Depends(get_job_repository)) -> JobService:
|
||||
"""Dependency to retrieve JobService"""
|
||||
return JobService(repo)
|
||||
20
app/core/exceptions.py
Normal file
20
app/core/exceptions.py
Normal file
@@ -0,0 +1,20 @@
|
||||
class AlemnoException(Exception):
|
||||
"""Base exception for Alemno Payments Application"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class JobNotFoundException(AlemnoException):
|
||||
"""Raised when a job is not found"""
|
||||
|
||||
def __init__(self, job_id: str):
|
||||
self.job_id = job_id
|
||||
super().__init__(f"Job with ID {job_id} not found")
|
||||
|
||||
|
||||
class InvalidCSVException(AlemnoException):
|
||||
"""Raised when the uploaded CSV is invalid"""
|
||||
|
||||
def __init__(self, detail: str):
|
||||
self.detail = detail
|
||||
super().__init__(detail)
|
||||
11
app/core/logging.py
Normal file
11
app/core/logging.py
Normal file
@@ -0,0 +1,11 @@
|
||||
import logging
|
||||
import sys
|
||||
|
||||
# Configure logging format
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)],
|
||||
)
|
||||
|
||||
logger = logging.getLogger("alemno_payments")
|
||||
@@ -1,4 +1,6 @@
|
||||
from app.db.base import Base
|
||||
from app.db.models.job import Job
|
||||
from app.db.models.transaction import Transaction
|
||||
from app.db.models.job_summary import JobSummary
|
||||
|
||||
__all__ = ["Base", "Job"]
|
||||
__all__ = ["Base", "Job", "Transaction", "JobSummary"]
|
||||
|
||||
56
app/db/models/job.py
Normal file
56
app/db/models/job.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, TYPE_CHECKING
|
||||
from sqlalchemy import DateTime, Integer, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from app.db.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.db.models.transaction import Transaction
|
||||
from app.db.models.job_summary import JobSummary
|
||||
|
||||
|
||||
class Job(Base):
|
||||
__tablename__ = "jobs"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
primary_key=True,
|
||||
default=lambda: str(uuid.uuid4()),
|
||||
index=True,
|
||||
)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(50),
|
||||
default="pending",
|
||||
index=True,
|
||||
)
|
||||
row_count_raw: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
row_count_clean: Mapped[Optional[int]] = mapped_column(Integer, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
server_default=func.now(),
|
||||
nullable=False,
|
||||
)
|
||||
completed_at: Mapped[Optional[datetime]] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
nullable=True,
|
||||
)
|
||||
error_message: Mapped[Optional[str]] = mapped_column(
|
||||
String(500),
|
||||
nullable=True,
|
||||
)
|
||||
|
||||
# Relationships
|
||||
transactions: Mapped[List["Transaction"]] = relationship(
|
||||
"Transaction",
|
||||
back_populates="job",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
summary: Mapped[Optional["JobSummary"]] = relationship(
|
||||
"JobSummary",
|
||||
back_populates="job",
|
||||
cascade="all, delete-orphan",
|
||||
uselist=False,
|
||||
)
|
||||
38
app/db/models/job_summary.py
Normal file
38
app/db/models/job_summary.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from typing import Any, Dict, Optional, TYPE_CHECKING
|
||||
from sqlalchemy import Float, ForeignKey, Integer, JSON, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from app.db.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.db.models.job import Job
|
||||
|
||||
|
||||
class JobSummary(Base):
|
||||
__tablename__ = "job_summaries"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
primary_key=True,
|
||||
default=lambda: str(uuid.uuid4()),
|
||||
index=True,
|
||||
)
|
||||
job_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
index=True,
|
||||
)
|
||||
total_spend_inr: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
total_spend_usd: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
top_merchants: Mapped[Dict[str, Any]] = mapped_column(
|
||||
JSON, nullable=False, default=dict
|
||||
)
|
||||
anomaly_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
narrative: Mapped[Optional[str]] = mapped_column(String(2000), nullable=True)
|
||||
risk_level: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
|
||||
# Relationships
|
||||
job: Mapped["Job"] = relationship("Job", back_populates="summary")
|
||||
43
app/db/models/transaction.py
Normal file
43
app/db/models/transaction.py
Normal file
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
import uuid
|
||||
from datetime import date
|
||||
from typing import Optional, TYPE_CHECKING
|
||||
from sqlalchemy import Boolean, Date, Float, ForeignKey, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from app.db.base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.db.models.job import Job
|
||||
|
||||
|
||||
class Transaction(Base):
|
||||
__tablename__ = "transactions"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
primary_key=True,
|
||||
default=lambda: str(uuid.uuid4()),
|
||||
index=True,
|
||||
)
|
||||
job_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
txn_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
date: Mapped[date] = mapped_column(Date, nullable=False)
|
||||
merchant: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
amount: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
currency: Mapped[str] = mapped_column(String(10), nullable=False, default="INR")
|
||||
status: Mapped[Optional[str]] = mapped_column(String(50), nullable=True)
|
||||
category: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
account_id: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
is_anomaly: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
anomaly_reason: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
|
||||
llm_category: Mapped[Optional[str]] = mapped_column(String(100), nullable=True)
|
||||
llm_raw_response: Mapped[Optional[str]] = mapped_column(String(1000), nullable=True)
|
||||
llm_failed: Mapped[Optional[bool]] = mapped_column(Boolean, default=False)
|
||||
|
||||
# Relationships
|
||||
job: Mapped["Job"] = relationship("Job", back_populates="transactions")
|
||||
32
app/db/session.py
Normal file
32
app/db/session.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from typing import AsyncGenerator
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
|
||||
from app.core.config import settings
|
||||
|
||||
# Create async database engine
|
||||
engine = create_async_engine(
|
||||
settings.DATABASE_URL,
|
||||
echo=False,
|
||||
future=True,
|
||||
pool_size=10,
|
||||
max_overflow=20,
|
||||
)
|
||||
|
||||
# Async session factory
|
||||
AsyncSessionLocal = async_sessionmaker(
|
||||
bind=engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Dependency for obtaining an asynchronous database session"""
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
44
app/main.py
44
app/main.py
@@ -1,6 +1,42 @@
|
||||
def main():
|
||||
print("Hello from alemno-payments!")
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from app.api.v1.jobs import router as jobs_router
|
||||
from app.core.config import settings
|
||||
from app.db.base import Base
|
||||
from app.db.session import engine
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup: Create tables if they don't exist
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield
|
||||
# Shutdown: Clean up pool
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title=settings.PROJECT_NAME,
|
||||
openapi_url=f"{settings.API_V1_STR}/openapi.json",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# CORS Middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Include API Router
|
||||
app.include_router(jobs_router, prefix=settings.API_V1_STR)
|
||||
|
||||
|
||||
# Root healthcheck endpoint
|
||||
@app.get("/health", tags=["health"])
|
||||
def health_check():
|
||||
return {"status": "ok", "project": settings.PROJECT_NAME}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from typing import List, Optional, Any, Dict
|
||||
from typing import List, Optional
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
from app.db.models.job import Job
|
||||
from app.db.models.transaction import Transaction
|
||||
from app.db.models.job_summary import JobSummary
|
||||
|
||||
|
||||
class JobRepository:
|
||||
@@ -9,18 +12,25 @@ class JobRepository:
|
||||
self.db = db
|
||||
|
||||
async def create(self, filename: str) -> Job:
|
||||
"""Create a new job record with status='pending'"""
|
||||
job = Job(filename=filename, status="pending")
|
||||
self.db.add(job)
|
||||
await self.db.flush()
|
||||
return job
|
||||
|
||||
async def get_by_id(self, job_id: str) -> Optional[Job]:
|
||||
query = select(Job).where(Job.id == job_id)
|
||||
"""Fetch a job record by its ID, eagerly loading summary and transactions"""
|
||||
query = (
|
||||
select(Job)
|
||||
.options(selectinload(Job.summary), selectinload(Job.transactions))
|
||||
.where(Job.id == job_id)
|
||||
)
|
||||
result = await self.db.execute(query)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def list_all(self, status: Optional[str] = None) -> List[Job]:
|
||||
query = select(Job)
|
||||
"""List all job records, optionally filtered by status, eagerly loading summary"""
|
||||
query = select(Job).options(selectinload(Job.summary))
|
||||
if status:
|
||||
query = query.where(Job.status == status)
|
||||
query = query.order_by(Job.created_at.desc())
|
||||
@@ -28,8 +38,20 @@ class JobRepository:
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def update(self, job: Job, **kwargs) -> Job:
|
||||
"""Update job fields dynamically"""
|
||||
for key, value in kwargs.items():
|
||||
if hasattr(job, key):
|
||||
setattr(job, key, value)
|
||||
await self.db.flush()
|
||||
return job
|
||||
|
||||
async def add_transactions(self, transactions: List[Transaction]) -> None:
|
||||
"""Add multiple transaction records in bulk"""
|
||||
self.db.add_all(transactions)
|
||||
await self.db.flush()
|
||||
|
||||
async def add_summary(self, summary: JobSummary) -> JobSummary:
|
||||
"""Add job summary record"""
|
||||
self.db.add(summary)
|
||||
await self.db.flush()
|
||||
return summary
|
||||
|
||||
@@ -3,6 +3,8 @@ from app.schemas.job import (
|
||||
JobResponse,
|
||||
JobStatusResponse,
|
||||
JobResultResponse,
|
||||
TransactionResponse,
|
||||
JobSummaryResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -10,4 +12,6 @@ __all__ = [
|
||||
"JobResponse",
|
||||
"JobStatusResponse",
|
||||
"JobResultResponse",
|
||||
"TransactionResponse",
|
||||
"JobSummaryResponse",
|
||||
]
|
||||
|
||||
@@ -1,8 +1,41 @@
|
||||
from datetime import datetime
|
||||
from datetime import date, datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class TransactionResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: str
|
||||
job_id: str
|
||||
txn_id: str
|
||||
date: date
|
||||
merchant: str
|
||||
amount: float
|
||||
currency: str
|
||||
status: Optional[str] = None
|
||||
category: Optional[str] = None
|
||||
account_id: Optional[str] = None
|
||||
is_anomaly: bool
|
||||
anomaly_reason: Optional[str] = None
|
||||
llm_category: Optional[str] = None
|
||||
llm_raw_response: Optional[str] = None
|
||||
llm_failed: Optional[bool] = False
|
||||
|
||||
|
||||
class JobSummaryResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: str
|
||||
job_id: str
|
||||
total_spend_inr: float
|
||||
total_spend_usd: float
|
||||
top_merchants: Dict[str, Any]
|
||||
anomaly_count: int
|
||||
narrative: Optional[str] = None
|
||||
risk_level: Optional[str] = None
|
||||
|
||||
|
||||
class JobBase(BaseModel):
|
||||
filename: str
|
||||
|
||||
@@ -16,9 +49,11 @@ class JobResponse(JobBase):
|
||||
|
||||
id: str
|
||||
status: str
|
||||
row_count: Optional[int] = None
|
||||
row_count_raw: Optional[int] = None
|
||||
row_count_clean: Optional[int] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
completed_at: Optional[datetime] = None
|
||||
error_message: Optional[str] = None
|
||||
|
||||
|
||||
class JobStatusResponse(BaseModel):
|
||||
@@ -26,7 +61,7 @@ class JobStatusResponse(BaseModel):
|
||||
|
||||
id: str
|
||||
status: str
|
||||
summary: Optional[Dict[str, Any]] = None
|
||||
summary: Optional[JobSummaryResponse] = None
|
||||
|
||||
|
||||
class JobResultResponse(BaseModel):
|
||||
@@ -34,4 +69,5 @@ class JobResultResponse(BaseModel):
|
||||
|
||||
id: str
|
||||
status: str
|
||||
results: Optional[Dict[str, Any]] = None
|
||||
summary: Optional[JobSummaryResponse] = None
|
||||
transactions: List[TransactionResponse] = []
|
||||
|
||||
35
app/services/job_service.py
Normal file
35
app/services/job_service.py
Normal file
@@ -0,0 +1,35 @@
|
||||
from typing import List, Optional
|
||||
from app.repositories.job_repository import JobRepository
|
||||
from app.core.exceptions import JobNotFoundException
|
||||
from app.db.models.job import Job
|
||||
from app.utils.csv_parser import parse_and_validate_csv
|
||||
from app.worker import process_transaction_job
|
||||
|
||||
|
||||
class JobService:
|
||||
def __init__(self, repo: JobRepository):
|
||||
self.repo = repo
|
||||
|
||||
async def upload_csv(self, filename: str, content: bytes) -> Job:
|
||||
parsed_transactions = parse_and_validate_csv(content)
|
||||
|
||||
job = await self.repo.create(filename=filename)
|
||||
|
||||
process_transaction_job.delay(job.id, parsed_transactions)
|
||||
|
||||
return job
|
||||
|
||||
async def get_job_status(self, job_id: str) -> Job:
|
||||
job = await self.repo.get_by_id(job_id)
|
||||
if not job:
|
||||
raise JobNotFoundException(job_id)
|
||||
return job
|
||||
|
||||
async def get_job_results(self, job_id: str) -> Job:
|
||||
job = await self.repo.get_by_id(job_id)
|
||||
if not job:
|
||||
raise JobNotFoundException(job_id)
|
||||
return job
|
||||
|
||||
async def list_jobs(self, status: Optional[str] = None) -> List[Job]:
|
||||
return await self.repo.list_all(status=status)
|
||||
120
app/utils/csv_parser.py
Normal file
120
app/utils/csv_parser.py
Normal file
@@ -0,0 +1,120 @@
|
||||
import csv
|
||||
import io
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
from app.core.exceptions import InvalidCSVException
|
||||
|
||||
REQUIRED_COLUMNS = {"txn_id", "date", "merchant", "amount"}
|
||||
|
||||
|
||||
def parse_and_validate_csv(content: bytes) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Parses CSV content from bytes, validates headers, and checks field types.
|
||||
Supports flexible mapping of common synonyms for headers.
|
||||
Returns a list of dicts. Raises InvalidCSVException if the file is invalid.
|
||||
"""
|
||||
try:
|
||||
decoded = content.decode("utf-8")
|
||||
except UnicodeDecodeError as e:
|
||||
raise InvalidCSVException("CSV file must be UTF-8 encoded text") from e
|
||||
|
||||
csv_file = io.StringIO(decoded)
|
||||
reader = csv.DictReader(csv_file)
|
||||
|
||||
if not reader.fieldnames:
|
||||
raise InvalidCSVException("CSV file is empty or missing headers")
|
||||
|
||||
header_mapping = {}
|
||||
for actual_header in reader.fieldnames:
|
||||
if not actual_header:
|
||||
continue
|
||||
cleaned = actual_header.strip().lower()
|
||||
|
||||
# Map synonyms
|
||||
if cleaned in {"txn_id", "transaction_id", "id"}:
|
||||
header_mapping["txn_id"] = actual_header
|
||||
elif cleaned == "date":
|
||||
header_mapping["date"] = actual_header
|
||||
elif cleaned in {"merchant", "description"}:
|
||||
header_mapping["merchant"] = actual_header
|
||||
elif cleaned == "amount":
|
||||
header_mapping["amount"] = actual_header
|
||||
elif cleaned == "currency":
|
||||
header_mapping["currency"] = actual_header
|
||||
elif cleaned == "category":
|
||||
header_mapping["category"] = actual_header
|
||||
elif cleaned in {"account_id", "account"}:
|
||||
header_mapping["account_id"] = actual_header
|
||||
|
||||
# Verify all required columns are present in mapping
|
||||
missing = REQUIRED_COLUMNS - set(header_mapping.keys())
|
||||
if missing:
|
||||
raise InvalidCSVException(f"Missing required CSV columns: {', '.join(missing)}")
|
||||
|
||||
parsed_rows = []
|
||||
for line_num, row in enumerate(reader, start=2):
|
||||
txn_id = row.get(header_mapping["txn_id"])
|
||||
date_str = row.get(header_mapping["date"])
|
||||
amount_str = row.get(header_mapping["amount"])
|
||||
merchant = row.get(header_mapping["merchant"])
|
||||
|
||||
# Optional columns
|
||||
currency = (
|
||||
row.get(header_mapping.get("currency", ""))
|
||||
if "currency" in header_mapping
|
||||
else "INR"
|
||||
)
|
||||
category = (
|
||||
row.get(header_mapping.get("category", ""))
|
||||
if "category" in header_mapping
|
||||
else None
|
||||
)
|
||||
account_id = (
|
||||
row.get(header_mapping.get("account_id", ""))
|
||||
if "account_id" in header_mapping
|
||||
else None
|
||||
)
|
||||
|
||||
if not all([txn_id, date_str, amount_str, merchant]):
|
||||
raise InvalidCSVException(
|
||||
f"Row {line_num}: Empty values are not allowed in required fields"
|
||||
)
|
||||
|
||||
# Validate amount
|
||||
try:
|
||||
amount = float(amount_str.strip())
|
||||
except ValueError as e:
|
||||
raise InvalidCSVException(
|
||||
f"Row {line_num}: Invalid amount '{amount_str}'. Must be a number."
|
||||
) from e
|
||||
|
||||
# Validate date (supporting multiple formats)
|
||||
parsed_date = None
|
||||
for fmt in ("%Y-%m-%d", "%m/%d/%Y", "%d-%m-%Y", "%Y/%m/%d"):
|
||||
try:
|
||||
parsed_date = datetime.strptime(date_str.strip(), fmt).date()
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if not parsed_date:
|
||||
raise InvalidCSVException(
|
||||
f"Row {line_num}: Invalid date format '{date_str}'. Expected YYYY-MM-DD or MM/DD/YYYY"
|
||||
)
|
||||
|
||||
parsed_rows.append(
|
||||
{
|
||||
"txn_id": txn_id.strip(),
|
||||
"date": parsed_date.isoformat(),
|
||||
"amount": amount,
|
||||
"merchant": merchant.strip(),
|
||||
"currency": currency.strip() if currency else "INR",
|
||||
"category": category.strip() if category else None,
|
||||
"account_id": account_id.strip() if account_id else None,
|
||||
}
|
||||
)
|
||||
|
||||
if not parsed_rows:
|
||||
raise InvalidCSVException("CSV file contains no data rows")
|
||||
|
||||
return parsed_rows
|
||||
199
app/worker.py
Normal file
199
app/worker.py
Normal file
@@ -0,0 +1,199 @@
|
||||
import asyncio
|
||||
from datetime import datetime, date
|
||||
from typing import Any, Dict, List
|
||||
from celery import Celery
|
||||
from app.core.config import settings
|
||||
from app.core.logging import logger
|
||||
from app.db.session import AsyncSessionLocal
|
||||
from app.repositories.job_repository import JobRepository
|
||||
from app.db.models.transaction import Transaction
|
||||
from app.db.models.job_summary import JobSummary
|
||||
from app.clients.exchange_rate_client import ExchangeRateClient
|
||||
|
||||
exchange_rate_client = ExchangeRateClient()
|
||||
|
||||
|
||||
# Define Celery app
|
||||
celery_app = Celery(
|
||||
"alemno_worker",
|
||||
broker=settings.CELERY_BROKER_URL,
|
||||
backend=settings.CELERY_RESULT_BACKEND,
|
||||
)
|
||||
|
||||
# Optional configurations
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_serializer="json",
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
)
|
||||
|
||||
|
||||
def run_async(coro):
|
||||
"""Helper to run async functions synchronously in Celery worker context"""
|
||||
return asyncio.get_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
@celery_app.task(name="tasks.process_transaction_job")
|
||||
def process_transaction_job(job_id: str, transactions: List[Dict[str, Any]]):
|
||||
"""
|
||||
Background job to clean transactions, detect anomalies, calculate spend breakdown,
|
||||
generate narrative summary, and persist results to Transaction and JobSummary tables.
|
||||
"""
|
||||
logger.info(f"Starting Celery task to process job_id: {job_id}")
|
||||
return run_async(_process_job_async(job_id, transactions))
|
||||
|
||||
|
||||
async def _process_job_async(job_id: str, transactions: List[Dict[str, Any]]):
|
||||
async with AsyncSessionLocal() as db:
|
||||
repo = JobRepository(db)
|
||||
job = await repo.get_by_id(job_id)
|
||||
if not job:
|
||||
logger.error(f"Job with ID {job_id} not found in database.")
|
||||
return
|
||||
|
||||
try:
|
||||
# Update status to processing
|
||||
await repo.update(job, status="processing")
|
||||
await db.commit()
|
||||
|
||||
db_transactions = []
|
||||
total_spend_inr = 0.0
|
||||
total_spend_usd = 0.0
|
||||
merchant_spends: Dict[str, float] = {}
|
||||
anomaly_count = 0
|
||||
|
||||
USD_TO_INR = await exchange_rate_client.get_usd_to_inr_rate()
|
||||
|
||||
for t in transactions:
|
||||
txn_id = t["txn_id"]
|
||||
raw_date = t["date"]
|
||||
# Parse date string to datetime.date object
|
||||
parsed_date = (
|
||||
date.fromisoformat(raw_date)
|
||||
if isinstance(raw_date, str)
|
||||
else raw_date
|
||||
)
|
||||
merchant = t["merchant"].strip()
|
||||
amount = float(t["amount"])
|
||||
currency = t.get("currency", "INR").strip().upper()
|
||||
category = t.get("category", "General")
|
||||
account_id = t.get("account_id")
|
||||
|
||||
# Spend calculation and currency conversion
|
||||
if currency == "USD":
|
||||
amt_usd = amount
|
||||
amt_inr = amount * USD_TO_INR
|
||||
else:
|
||||
amt_inr = amount
|
||||
amt_usd = amount / USD_TO_INR
|
||||
|
||||
total_spend_inr += amt_inr
|
||||
total_spend_usd += amt_usd
|
||||
|
||||
# Aggregate merchant spend (in USD for standardization)
|
||||
merchant_spends[merchant] = merchant_spends.get(merchant, 0.0) + amt_usd
|
||||
|
||||
# Heuristic Anomaly detection
|
||||
is_anomaly = False
|
||||
anomaly_reason = None
|
||||
if amt_usd > 5000.0:
|
||||
is_anomaly = True
|
||||
anomaly_reason = "High value transaction (> $5,000 USD equivalent)"
|
||||
else:
|
||||
desc_lower = merchant.lower()
|
||||
if any(
|
||||
k in desc_lower
|
||||
for k in ["suspend", "hack", "error", "unknown", "fraud"]
|
||||
):
|
||||
is_anomaly = True
|
||||
anomaly_reason = "Suspicious merchant keyword"
|
||||
|
||||
if is_anomaly:
|
||||
anomaly_count += 1
|
||||
|
||||
# Mock
|
||||
llm_failed = False
|
||||
llm_category = category.capitalize() if category else "General"
|
||||
llm_raw_response = f'{{"category": "{llm_category}", "confidence": 0.95, "status": "processed"}}'
|
||||
|
||||
db_txn = Transaction(
|
||||
job_id=job_id,
|
||||
txn_id=txn_id,
|
||||
date=parsed_date,
|
||||
merchant=merchant,
|
||||
amount=amount,
|
||||
currency=currency,
|
||||
status="cleaned",
|
||||
category=category,
|
||||
account_id=account_id,
|
||||
is_anomaly=is_anomaly,
|
||||
anomaly_reason=anomaly_reason,
|
||||
llm_category=llm_category,
|
||||
llm_raw_response=llm_raw_response,
|
||||
llm_failed=llm_failed,
|
||||
)
|
||||
db_transactions.append(db_txn)
|
||||
|
||||
await repo.add_transactions(db_transactions)
|
||||
|
||||
top_merchant = (
|
||||
max(merchant_spends, key=merchant_spends.get)
|
||||
if merchant_spends
|
||||
else "None"
|
||||
)
|
||||
risk_level = "Low"
|
||||
if anomaly_count > 2:
|
||||
risk_level = "High"
|
||||
elif anomaly_count > 0:
|
||||
risk_level = "Medium"
|
||||
|
||||
narrative = (
|
||||
f"Successfully parsed and clean-processed {len(transactions)} transaction records. "
|
||||
f"Total spend is INR {total_spend_inr:,.2f} (${total_spend_usd:,.2f} USD). "
|
||||
f"Top merchant by spend volume is '{top_merchant}' with total of ${merchant_spends.get(top_merchant, 0):,.2f} USD. "
|
||||
f"Scan flagged {anomaly_count} potential anomalies, resulting in a overall risk level of '{risk_level}'."
|
||||
)
|
||||
|
||||
# Build top merchants dict sorted by spend limit (keep top 5)
|
||||
sorted_merchants = sorted(
|
||||
merchant_spends.items(), key=lambda x: x[1], reverse=True
|
||||
)[:5]
|
||||
top_merchants_json = {
|
||||
merchant: round(spend, 2) for merchant, spend in sorted_merchants
|
||||
}
|
||||
|
||||
# Create job summary
|
||||
summary = JobSummary(
|
||||
job_id=job_id,
|
||||
total_spend_inr=round(total_spend_inr, 2),
|
||||
total_spend_usd=round(total_spend_usd, 2),
|
||||
top_merchants=top_merchants_json,
|
||||
anomaly_count=anomaly_count,
|
||||
narrative=narrative,
|
||||
risk_level=risk_level,
|
||||
)
|
||||
await repo.add_summary(summary)
|
||||
|
||||
# Update job state
|
||||
await repo.update(
|
||||
job,
|
||||
status="completed",
|
||||
row_count_raw=len(transactions),
|
||||
row_count_clean=len(db_transactions),
|
||||
completed_at=datetime.utcnow(),
|
||||
)
|
||||
await db.commit()
|
||||
logger.info(f"Job {job_id} processing completed successfully.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error processing job {job_id}")
|
||||
await repo.update(
|
||||
job,
|
||||
status="failed",
|
||||
error_message=str(e),
|
||||
completed_at=datetime.utcnow(),
|
||||
)
|
||||
await db.commit()
|
||||
raise e
|
||||
@@ -6,6 +6,26 @@ services:
|
||||
container_name: alemno_db
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: alemno_payments
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres -d alemno_payments"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: alemno_redis
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
|
||||
web:
|
||||
build: .
|
||||
container_name: alemno_web
|
||||
@@ -13,8 +33,42 @@ services:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- .:/app
|
||||
environment:
|
||||
- POSTGRES_HOST=db
|
||||
- POSTGRES_PORT=5432
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
- POSTGRES_DB=alemno_payments
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
- REDIS_DB=0
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
worker:
|
||||
build: .
|
||||
container_name: alemno_worker
|
||||
volumes:
|
||||
- .:/app
|
||||
environment:
|
||||
- POSTGRES_HOST=db
|
||||
- POSTGRES_PORT=5432
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=postgres
|
||||
- POSTGRES_DB=alemno_payments
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
- REDIS_DB=0
|
||||
command: celery -A app.worker.celery_app worker --loglevel=info
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_started
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
1
main.py
1
main.py
@@ -2,4 +2,3 @@ import uvicorn
|
||||
|
||||
if __name__ == "__main__":
|
||||
uvicorn.run("app.app:app", host="0.0.0.0", port=3000, reload=True)
|
||||
|
||||
|
||||
@@ -22,3 +22,8 @@ dependencies = [
|
||||
[tool.pytest.ini_options]
|
||||
pythonpath = ["."]
|
||||
asyncio_mode = "auto"
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"ruff>=0.15.20",
|
||||
]
|
||||
|
||||
103
tests/api/test_jobs_api.py
Normal file
103
tests/api/test_jobs_api.py
Normal file
@@ -0,0 +1,103 @@
|
||||
import io
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from app.worker import celery_app
|
||||
|
||||
# Mark all tests as async
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
# Set celery to run tasks eagerly (synchronously) during API testing
|
||||
celery_app.conf.task_always_eager = True
|
||||
|
||||
|
||||
async def test_upload_csv_success(client: AsyncClient):
|
||||
csv_content = (
|
||||
"txn_id,date,amount,merchant,currency,category,account\n"
|
||||
"TX1001,2026-06-25,120.50,Weekly grocery shop,INR,Groceries,ACC01\n"
|
||||
"TX1002,06/26/2026,6000.00,New Laptop,USD,Electronics,ACC02\n" # Anomaly (> $5,000 USD equivalent)
|
||||
)
|
||||
|
||||
files = {
|
||||
"file": (
|
||||
"transactions.csv",
|
||||
io.BytesIO(csv_content.encode("utf-8")),
|
||||
"text/csv",
|
||||
)
|
||||
}
|
||||
|
||||
# 1. Post upload CSV
|
||||
response = await client.post("/api/v1/jobs/upload", files=files)
|
||||
assert response.status_code == 201
|
||||
|
||||
data = response.json()
|
||||
assert "id" in data
|
||||
assert data["filename"] == "transactions.csv"
|
||||
assert data["status"] == "pending" # Initial state returned by router immediately
|
||||
|
||||
job_id = data["id"]
|
||||
|
||||
# Since Celery is in eager mode, the task should have completed immediately.
|
||||
# 2. Get Job Status
|
||||
status_response = await client.get(f"/api/v1/jobs/{job_id}/status")
|
||||
assert status_response.status_code == 200
|
||||
status_data = status_response.json()
|
||||
assert status_data["id"] == job_id
|
||||
assert status_data["status"] == "completed"
|
||||
|
||||
summary = status_data["summary"]
|
||||
assert summary is not None
|
||||
assert summary["anomaly_count"] == 1
|
||||
assert (
|
||||
summary["total_spend_usd"] > 6000.0
|
||||
) # TX1002 is $6,000, TX1001 is INR 120.50 ($1.45)
|
||||
assert "New Laptop" in summary["top_merchants"]
|
||||
assert summary["risk_level"] == "Medium" # 1 anomaly
|
||||
|
||||
# 3. Get Job Results
|
||||
results_response = await client.get(f"/api/v1/jobs/{job_id}/results")
|
||||
assert results_response.status_code == 200
|
||||
results_data = results_response.json()
|
||||
assert results_data["id"] == job_id
|
||||
assert results_data["status"] == "completed"
|
||||
|
||||
transactions_list = results_data["transactions"]
|
||||
assert len(transactions_list) == 2
|
||||
|
||||
# Assert TX1002 is flagged as an anomaly
|
||||
laptop_txn = next(t for t in transactions_list if t["txn_id"] == "TX1002")
|
||||
assert laptop_txn["is_anomaly"] is True
|
||||
assert "High value transaction" in laptop_txn["anomaly_reason"]
|
||||
assert laptop_txn["llm_category"] == "Electronics"
|
||||
assert laptop_txn["llm_failed"] is False
|
||||
|
||||
grocery_txn = next(t for t in transactions_list if t["txn_id"] == "TX1001")
|
||||
assert grocery_txn["is_anomaly"] is False
|
||||
assert grocery_txn["currency"] == "INR"
|
||||
|
||||
|
||||
async def test_upload_invalid_csv(client: AsyncClient):
|
||||
# Invalid CSV structure (missing required column: merchant)
|
||||
csv_content = "txn_id,date,amount\nTX1001,2026-06-25,120.50\n"
|
||||
files = {
|
||||
"file": (
|
||||
"transactions.csv",
|
||||
io.BytesIO(csv_content.encode("utf-8")),
|
||||
"text/csv",
|
||||
)
|
||||
}
|
||||
|
||||
response = await client.post("/api/v1/jobs/upload", files=files)
|
||||
assert response.status_code == 400
|
||||
assert "Missing required CSV columns" in response.json()["detail"]
|
||||
|
||||
|
||||
async def test_get_nonexistent_job(client: AsyncClient):
|
||||
response = await client.get("/api/v1/jobs/nonexistent-id-xyz/status")
|
||||
assert response.status_code == 404
|
||||
assert "not found" in response.json()["detail"]
|
||||
|
||||
|
||||
async def test_get_all_jobs(client: AsyncClient):
|
||||
response = await client.get("/api/v1/jobs")
|
||||
assert response.status_code == 200
|
||||
assert isinstance(response.json(), list)
|
||||
102
tests/conftest.py
Normal file
102
tests/conftest.py
Normal file
@@ -0,0 +1,102 @@
|
||||
import asyncio
|
||||
from typing import AsyncGenerator
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.dependencies import get_db
|
||||
from app.db.base import Base
|
||||
from app.main import app
|
||||
|
||||
# Use a test database suffix or a separate database for testing
|
||||
TEST_DATABASE_URL = settings.DATABASE_URL.replace(
|
||||
settings.POSTGRES_DB, f"test_{settings.POSTGRES_DB}"
|
||||
)
|
||||
|
||||
# Create async engine for test db
|
||||
test_engine = create_async_engine(
|
||||
TEST_DATABASE_URL,
|
||||
echo=False,
|
||||
future=True,
|
||||
)
|
||||
|
||||
TestAsyncSessionLocal = async_sessionmaker(
|
||||
bind=test_engine,
|
||||
class_=AsyncSession,
|
||||
expire_on_commit=False,
|
||||
)
|
||||
|
||||
# Global flag to track if test DB is accessible
|
||||
_db_offline = False
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def initialize_test_db():
|
||||
"""Create test tables and clean them up after all tests finish."""
|
||||
global _db_offline
|
||||
try:
|
||||
loop = asyncio.get_event_loop_policy().get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
async def setup_db():
|
||||
async with test_engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
async def teardown_db():
|
||||
async with test_engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
|
||||
try:
|
||||
loop.run_until_complete(setup_db())
|
||||
except Exception as e:
|
||||
_db_offline = True
|
||||
print(f"\nSkipping DB initialization (Postgres is likely offline): {e}\n")
|
||||
|
||||
yield
|
||||
|
||||
if not _db_offline:
|
||||
try:
|
||||
loop.run_until_complete(teardown_db())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def db_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
"""Provide a transactional test database session."""
|
||||
if _db_offline:
|
||||
pytest.skip("PostgreSQL database is offline")
|
||||
return
|
||||
|
||||
async with TestAsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(db_session: AsyncSession) -> AsyncGenerator[AsyncClient, None]:
|
||||
"""Provide an HTTPX AsyncClient for FastAPI endpoint testing with db overrides."""
|
||||
|
||||
# Override get_db dependency to use the test session
|
||||
async def override_get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
yield db_session
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=app),
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
app.dependency_overrides.clear()
|
||||
84
tests/repositories/test_job_repository.py
Normal file
84
tests/repositories/test_job_repository.py
Normal file
@@ -0,0 +1,84 @@
|
||||
from datetime import date
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.repositories.job_repository import JobRepository
|
||||
from app.db.models.transaction import Transaction
|
||||
from app.db.models.job_summary import JobSummary
|
||||
|
||||
# Set up asyncio marker
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_create_and_get_job(db_session: AsyncSession):
|
||||
repo = JobRepository(db_session)
|
||||
|
||||
# Create job
|
||||
job = await repo.create(filename="test_transactions.csv")
|
||||
assert job.id is not None
|
||||
assert job.status == "pending"
|
||||
assert job.filename == "test_transactions.csv"
|
||||
|
||||
# Retrieve job
|
||||
fetched_job = await repo.get_by_id(job.id)
|
||||
assert fetched_job is not None
|
||||
assert fetched_job.id == job.id
|
||||
assert fetched_job.filename == "test_transactions.csv"
|
||||
|
||||
|
||||
async def test_update_job_and_add_relations(db_session: AsyncSession):
|
||||
repo = JobRepository(db_session)
|
||||
job = await repo.create(filename="update_test.csv")
|
||||
|
||||
# Update status and count
|
||||
updated = await repo.update(
|
||||
job, status="completed", row_count_raw=10, row_count_clean=8
|
||||
)
|
||||
assert updated.status == "completed"
|
||||
assert updated.row_count_raw == 10
|
||||
assert updated.row_count_clean == 8
|
||||
|
||||
# Create & add summary
|
||||
summary = JobSummary(
|
||||
job_id=job.id,
|
||||
total_spend_inr=15000.0,
|
||||
total_spend_usd=180.72,
|
||||
top_merchants={"Merchant A": 100.0},
|
||||
anomaly_count=1,
|
||||
narrative="Good",
|
||||
risk_level="Low",
|
||||
)
|
||||
await repo.add_summary(summary)
|
||||
|
||||
# Create & add transaction
|
||||
txn = Transaction(
|
||||
job_id=job.id,
|
||||
txn_id="TX001",
|
||||
date=date(2026, 6, 25),
|
||||
merchant="Merchant A",
|
||||
amount=100.0,
|
||||
currency="USD",
|
||||
status="cleaned",
|
||||
is_anomaly=False,
|
||||
)
|
||||
await repo.add_transactions([txn])
|
||||
|
||||
# Retrieve to verify persistence
|
||||
fetched = await repo.get_by_id(job.id)
|
||||
assert fetched.status == "completed"
|
||||
assert fetched.row_count_raw == 10
|
||||
assert fetched.row_count_clean == 8
|
||||
assert fetched.summary is not None
|
||||
assert fetched.summary.total_spend_usd == 180.72
|
||||
assert len(fetched.transactions) == 1
|
||||
assert fetched.transactions[0].txn_id == "TX001"
|
||||
|
||||
|
||||
async def test_list_jobs(db_session: AsyncSession):
|
||||
repo = JobRepository(db_session)
|
||||
|
||||
await repo.create(filename="list_1.csv")
|
||||
await repo.create(filename="list_2.csv")
|
||||
|
||||
jobs = await repo.list_all()
|
||||
# At least the ones we created in this test session
|
||||
assert len(jobs) >= 2
|
||||
54
tests/services/test_csv_parser.py
Normal file
54
tests/services/test_csv_parser.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import pytest
|
||||
from app.core.exceptions import InvalidCSVException
|
||||
from app.utils.csv_parser import parse_and_validate_csv
|
||||
|
||||
|
||||
def test_parse_valid_csv():
|
||||
valid_csv = (
|
||||
"transaction_id,date,amount,category,description,currency,account\n"
|
||||
"TX1001,2026-06-25,120.50,Groceries,Weekly grocery shop,INR,ACC01\n"
|
||||
"TX1002,06/26/2026,15.00,Coffee,Morning latte,USD,ACC02\n"
|
||||
)
|
||||
result = parse_and_validate_csv(valid_csv.encode("utf-8"))
|
||||
assert len(result) == 2
|
||||
assert result[0]["txn_id"] == "TX1001"
|
||||
assert result[0]["amount"] == 120.50
|
||||
assert result[0]["date"] == "2026-06-25"
|
||||
assert result[0]["merchant"] == "Weekly grocery shop"
|
||||
assert result[0]["currency"] == "INR"
|
||||
assert result[0]["account_id"] == "ACC01"
|
||||
|
||||
assert result[1]["txn_id"] == "TX1002"
|
||||
assert result[1]["amount"] == 15.00
|
||||
assert result[1]["date"] == "2026-06-26"
|
||||
assert result[1]["merchant"] == "Morning latte"
|
||||
assert result[1]["currency"] == "USD"
|
||||
assert result[1]["account_id"] == "ACC02"
|
||||
|
||||
|
||||
def test_parse_missing_headers():
|
||||
invalid_csv = (
|
||||
"txn_id,amount,category,description\n"
|
||||
"TX1001,120.50,Groceries,Weekly grocery shop\n"
|
||||
)
|
||||
with pytest.raises(InvalidCSVException) as exc_info:
|
||||
parse_and_validate_csv(invalid_csv.encode("utf-8"))
|
||||
assert "Missing required CSV columns" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_parse_invalid_amount():
|
||||
invalid_csv = (
|
||||
"txn_id,date,amount,merchant\nTX1001,2026-06-25,abc,Weekly grocery shop\n"
|
||||
)
|
||||
with pytest.raises(InvalidCSVException) as exc_info:
|
||||
parse_and_validate_csv(invalid_csv.encode("utf-8"))
|
||||
assert "Invalid amount" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_parse_invalid_date():
|
||||
invalid_csv = (
|
||||
"txn_id,date,amount,merchant\nTX1001,invalid-date,120.50,Weekly grocery shop\n"
|
||||
)
|
||||
with pytest.raises(InvalidCSVException) as exc_info:
|
||||
parse_and_validate_csv(invalid_csv.encode("utf-8"))
|
||||
assert "Invalid date format" in str(exc_info.value)
|
||||
12
tests/services/test_exchange_rate.py
Normal file
12
tests/services/test_exchange_rate.py
Normal file
@@ -0,0 +1,12 @@
|
||||
import pytest
|
||||
from app.clients.exchange_rate_client import ExchangeRateClient
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_get_usd_to_inr_rate():
|
||||
client = ExchangeRateClient()
|
||||
rate = await client.get_usd_to_inr_rate()
|
||||
assert isinstance(rate, float)
|
||||
# The rate should be positive (either the fetched rate or the default fallback of 83.0)
|
||||
assert rate > 0.0
|
||||
33
uv.lock
generated
33
uv.lock
generated
@@ -21,6 +21,11 @@ dependencies = [
|
||||
{ name = "uvicorn" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
{ name = "ruff" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "asyncpg", specifier = ">=0.31.0" },
|
||||
@@ -37,6 +42,9 @@ requires-dist = [
|
||||
{ name = "uvicorn", specifier = ">=0.49.0" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [{ name = "ruff", specifier = ">=0.15.20" }]
|
||||
|
||||
[[package]]
|
||||
name = "amqp"
|
||||
version = "5.3.1"
|
||||
@@ -500,6 +508,31 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.15.20"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/43/dc/35b341fc554ba02f217fc10da57d1a75168cfbcf75b0ef2202176d4c4f2d/ruff-0.15.20.tar.gz", hash = "sha256:1416eb04349192646b54de98f146c4f59afe37d0decfc02c3cbbf396f3a28566", size = 4755489, upload-time = "2026-06-25T17:20:37.578Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/94/d9/2d5014f0253ba541d2061d9fa7193f48e941c8b21bb88a7ff9bbe0bd0596/ruff-0.15.20-py3-none-linux_armv6l.whl", hash = "sha256:00e188c53e499c3c1637f73c91dcf2fb56d576cab76ce1be50a27c4e80e37078", size = 10839665, upload-time = "2026-06-25T17:19:44.702Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c6/d3/ac1798ba64f670698867fcfc591d50e7e421bef137db564858f619a30fcf/ruff-0.15.20-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9ebd1fd9b9c95fc0bd7b2761aebec1f030013d2e193a2901b224af68fe47251b", size = 11208649, upload-time = "2026-06-25T17:19:48.787Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/47/47/d3ac899991202095dfcf3d5176be4272642be3cf981a2f1a30f72a2afb95/ruff-0.15.20-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c5b16cdd67ca108185cd36dce98c576350c03b1660a751de725fb049193a0632", size = 10622638, upload-time = "2026-06-25T17:19:51.354Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/13/4e043fe30aa94d4ff5213a9881fc296d12960f5971b234a5263fdc225312/ruff-0.15.20-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3413bb3c3d2ca6a8208f1f4809cd2dca3c6de6d0b491c0e70847672bde6e6efd", size = 10984227, upload-time = "2026-06-25T17:19:54.044Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/e6/92e7bf40388bc5800073b96564f56264f7e48bfd1a498f5ced6ae6d5a769/ruff-0.15.20-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bd7ec42b3bb3da066488db093308a69c4ac5ee6d2af333a86ba6e2eb2e7dd44b", size = 10622882, upload-time = "2026-06-25T17:19:57.037Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/7a/43460be3f24495a3aa46d4b16873e2c4941b3b5f0b00cf88c03b7b94b339/ruff-0.15.20-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1a36ad0eb77fba9aabfb69ede54de6f376d04ac18ebea022847046d340a8267", size = 11474808, upload-time = "2026-06-25T17:20:00.357Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/a0/f37077884873221c6b33b4ab49eb18f9f88e54a16a25a5bca59bef46dd66/ruff-0.15.20-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b6df3b1e4610432f0386dba04d853b5f08cbbc903410c6fcc02f620f05aff53c", size = 12293094, upload-time = "2026-06-25T17:20:03.446Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/74/165545b60256a9704c21ac0ec4a0d07933b320812f9584836c9f4aca4292/ruff-0.15.20-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e89f198a1ea6ef0d727c1cf16088bc91a6cb0ab947dedc966715691647186eae", size = 11526176, upload-time = "2026-06-25T17:20:06.301Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/b1/a976a136d40ade83ce743578399865f57001003a409acadc0ecbb3051082/ruff-0.15.20-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309809086c2acb67624950a3c8133e80f32d0d3e27106c0cd60ff26657c9f24b", size = 11520767, upload-time = "2026-06-25T17:20:09.191Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/0f/f032696cb01c9b54c0263fa393474d7758f1cdc021a01b04e3cbc2500999/ruff-0.15.20-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2d2374caa2f2c2f9e2b7da0a50802cfb8b79f55a9b5e49379f564544fbf56487", size = 11500132, upload-time = "2026-06-25T17:20:13.602Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/f4/51b1a14bc69e8c224b15dab9cce8e99b425e0455d462caa2b3c9be2b6a8e/ruff-0.15.20-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a1ed17b65293e0c2f22fc387bc13198a5de94bf4429589b0ff6946b0feaf21a3", size = 10943828, upload-time = "2026-06-25T17:20:16.635Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/4b/fe267640783cd02bf6c5cc290b1df1051be2ec294c678b5c15fe19e52343/ruff-0.15.20-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:f701305e66b38ea6c91882490eb73459796808e4c6362a1b765255e0cdcd4053", size = 10645418, upload-time = "2026-06-25T17:20:19.4Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/c0/a65aa4ec2f5e87a1df32dc3ec1fede434fe3dfd5cbcf3b503cafc676ab54/ruff-0.15.20-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b9c0c367ad8e5d0d5b5b8537864c469a0a0e55417aadfbeca41fa61333be9f4", size = 11211770, upload-time = "2026-06-25T17:20:22.033Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/a4/0caa331d954ae2723d729d351c989cb4ca8b6077d5c6c2cb6de75e98c041/ruff-0.15.20-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:01cc00dd58f0df339d0e902219dd53990ea99996a0344e5d9cc8d45d5307e460", size = 11618698, upload-time = "2026-06-25T17:20:25.259Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/9b/5f14927848d2fd4aa891fd88d883788c5a7baba561c7874732364045708c/ruff-0.15.20-py3-none-win32.whl", hash = "sha256:ed65ef510e43a137207e0f01cfcf998aeddb1aeeda5c9d35023e910284d7cf21", size = 10857322, upload-time = "2026-06-25T17:20:28.612Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fa/f0/fe47c501f9dea92a26d788ff98bb5d92ed4cb4c88792c5c88af6b697dc8e/ruff-0.15.20-py3-none-win_amd64.whl", hash = "sha256:a525c81c70fb0380344dd1d8745d8cc1c890b7fc94a58d5a07bd8eb9557b8415", size = 11993274, upload-time = "2026-06-25T17:20:31.871Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/2b/9555445e1201d92b3195f45cdb153a0b68f24e0a4273f6e3d5ab46e212bb/ruff-0.15.20-py3-none-win_arm64.whl", hash = "sha256:2f5b2a6d614e8700388806a14996c40fab2c47b819ef57d790a34878858ed9ca", size = 11343498, upload-time = "2026-06-25T17:20:35.03Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.17.0"
|
||||
|
||||
Reference in New Issue
Block a user