"""
API Server for Land Classification Model Training
Cho phép chọn dữ liệu và cấu hình training qua giao diện web
"""

from fastapi import FastAPI, BackgroundTasks, HTTPException, UploadFile, File, Form
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from fastapi.responses import HTMLResponse, FileResponse
from pydantic import BaseModel
from typing import Optional, List
import uvicorn
import joblib
import json
from datetime import datetime
from pathlib import Path
import sys
import numpy as np
import xarray as xr
import rasterio
from rasterio.transform import from_bounds
import asyncio
import hashlib
import traceback
import socket
import urllib.request

# Import report generator
from report_generator import generate_training_report, generate_prediction_report

# Import Model Manager
from model_manager import ModelManager, get_model_manager

# Import Vietnam provinces data
from vietnam_provinces import get_all_provinces, get_provinces_by_region, get_province_bbox, search_province
from vietnam_provinces_merged import (
    get_all_provinces_32, get_provinces_by_region_32, get_province_bbox_32, 
    search_province_32, get_merged_info, get_provinces_statistics
)

# Import cloud removal module
from cloud_removal import process_cloud_removal, get_available_methods

# Import planetary computer libraries (conditional)
try:
    from pystac_client import Client
    import planetary_computer
    import odc.stac
except ImportError:
    Client = None
    planetary_computer = None
    odc = None

app = FastAPI(title="Land Classification Training API", version="1.0.0")

# Enable CORS
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Global training status``
training_status = {
    "is_training": False,
    "progress": "",
    "error": None,
    "result": None,
    "start_time": None,
    "end_time": None,
    "cancel_requested": False
}

# Global prediction status
prediction_status = {
    "is_predicting": False,
    "progress": "",
    "error": None,
    "result": None,
    "output_file": None,
    "start_time": None,
    "end_time": None
}

# Batch prediction queue
batch_queue = []
batch_results = []

# Label mapping from training data (from 01.train_ODC.ipynb)
DEFAULT_LABEL_MAPPING = {
    "Lua tom": "0",
    "Lua": "1",
    "CHN": "2",
    "CLN": "3",
    "TS": "4",
    "Song": "5",
    "Dat xay dung": "6",
    "Rung": "7",
}

DEFAULT_LABEL_NAMES = {
    0: "Lua tom",
    1: "Lua",
    2: "CHN",
    3: "CLN",
    4: "TS",
    5: "Song",
    6: "Dat xay dung",
    7: "Rung",
}


# ============ NETWORK CONNECTIVITY HELPERS ============
def check_internet_connectivity(timeout=5):
    """
    Check if we have internet connectivity to common hosts
    Returns: (is_connected: bool, error_message: str)
    """
    test_hosts = [
        ("8.8.8.8", 53),  # Google DNS
        ("1.1.1.1", 53),  # Cloudflare DNS
    ]
    
    for host, port in test_hosts:
        try:
            socket.setdefaulttimeout(timeout)
            socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
            return True, None
        except (socket.timeout, socket.error):
            continue
    
    return False, "No internet connectivity detected. Please check your network connection."


def check_planetary_computer_access(timeout=10):
    """
    Check if we can access Microsoft Planetary Computer
    Returns: (is_accessible: bool, error_message: str)
    """
    test_urls = [
        "https://planetarycomputer.microsoft.com/api/stac/v1",
    ]
    
    for url in test_urls:
        try:
            req = urllib.request.Request(url)
            response = urllib.request.urlopen(req, timeout=timeout)
            if response.status == 200:
                return True, None
        except urllib.error.HTTPError as e:
            # 405 Method Not Allowed is OK - means the service is reachable
            if e.code in [200, 405]:
                return True, None
            return False, f"HTTP Error {e.code}: {e.reason}"
        except urllib.error.URLError as e:
            if "Could not resolve host" in str(e):
                return False, f"Cannot resolve Planetary Computer hosts. DNS issue detected: {str(e)}"
            return False, f"URL Error: {str(e)}"
        except Exception as e:
            return False, f"Unexpected error: {str(e)}"
    
    return False, "Cannot access Microsoft Planetary Computer. Service may be down or blocked by firewall."


class TrainingConfig(BaseModel):
    """Cấu hình training - Tất cả bắt buộc nhập từ giao diện"""
    # Khu vực (bbox)
    min_lon: float
    min_lat: float
    max_lon: float
    max_lat: float
    
    # Thời gian
    start_date: str
    end_date: str
    
    # Dữ liệu
    max_scenes: int
    cloud_cover: int
    resolution: int  # 10m hoặc 20m
    
    # Model parameters
    model_type: str  # xgboost, random_forest, decision_tree, svm, cnn, swin-unet, mobilenet-lraspp
    n_estimators: int
    max_depth: int
    learning_rate: float
    use_gpu: bool
    
    # Train/test split
    test_size: float  # Tỷ lệ dữ liệu dùng làm test (0-1)
    
    # Cache
    use_cache: bool  # Cache dataset để test nhanh hơn
    
    # Training data
    training_shapefile: str


class PredictionConfig(BaseModel):
    """Cấu hình dự đoán - Tất cả bắt buộc nhập từ giao diện"""
    # Model to use
    model_filename: str
    
    # Khu vực (bbox)
    min_lon: float
    min_lat: float
    max_lon: float
    max_lat: float
    
    # Thời gian
    start_date: str
    end_date: str
    
    # Dữ liệu
    max_scenes: int
    cloud_cover: int
    resolution: int
    
    # GPU support for deep learning models
    use_gpu: bool
    
    # Cloud removal strategy
    cloud_removal_method: str = "classic"
    cloud_removal_model: Optional[str] = None  # Optional: .pth model filename for deep learning cloud removal
    
    # Shapefile overlay for visualization
    shapefile_overlay: Optional[str] = None  # Path to shapefile for overlaying boundaries


class TrainingStatus(BaseModel):
    """Trạng thái training"""
    is_training: bool
    progress: str
    error: Optional[str]
    result: Optional[dict]
    start_time: Optional[str]
    end_time: Optional[str]


class NDVIConfig(BaseModel):
    """Cấu hình tính NDVI time series"""
    bbox: List[float]  # [min_lon, min_lat, max_lon, max_lat]
    start_date: str
    end_date: str
    max_cloud_cover: int = 30
    resolution: int = 20
    cloud_removal_method: str = "classic"


class ChangeDetectionWorkflowRequest(BaseModel):
    """Request for change detection workflow"""
    prediction_result: dict
    bbox: List[float]


class ComparePeriodsPredictionConfig(BaseModel):
    """Compare predictions between two time periods"""
    model_filename: str
    min_lon: float
    min_lat: float
    max_lon: float
    max_lat: float
    current_period: dict  # {start_date, end_date}
    prediction_period: dict  # {start_date, end_date}
    max_scenes: int = 12
    cloud_cover: int = 30
    resolution: int = 20
    export_ndvi: bool = True
    export_classification: bool = True
    cloud_removal_method: str = "classic"


class PredictionWithNDVIConfig(BaseModel):
    """Cấu hình predict kết hợp land classification và NDVI"""
    model_filename: str
    min_lon: float
    min_lat: float
    max_lon: float
    max_lat: float
    start_date: str
    end_date: str
    max_scenes: int = 12
    cloud_cover: int = 30
    resolution: int = 20
    use_gpu: bool = False  # Use GPU for deep learning models
    export_ndvi: bool = True  # Export NDVI raster
    export_classification: bool = True  # Export classification raster
    cloud_removal_method: str = "classic"
    cloud_removal_model: Optional[str] = None  # Optional: .pth model filename for deep learning cloud removal
    shapefile_overlay: Optional[str] = None  # Path to shapefile for overlaying boundaries


class CloudRemovalTrainingConfig(BaseModel):
    """Cấu hình train cloud removal model"""
    data_dir: str = "winter_dataset"
    use_s1: bool = True  # Sử dụng Sentinel-1 radar data
    batch_size: int = 8
    num_epochs: int = 50
    learning_rate: float = 1e-4
    use_gpu: bool = True
    model_name: str = "cloud_removal_unet"  # Tên model để lưu


# Serve change detection interface page (moved here after app is defined)
@app.get("/change-detection", response_class=HTMLResponse)
async def change_detection_page():
    html_file = Path(__file__).parent / "change_detection_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        return HTMLResponse("<h2>Change Detection Interface not found.</h2>")


@app.get("/api/network/check")
async def check_network():
    """
    Check network connectivity and access to Planetary Computer
    Useful for diagnosing connection issues before attempting data downloads
    """
    result = {
        "timestamp": datetime.now().isoformat(),
        "internet_connected": False,
        "planetary_computer_accessible": False,
        "internet_error": None,
        "pc_error": None,
        "status": "unknown"
    }
    
    # Check basic internet connectivity
    is_connected, conn_error = check_internet_connectivity(timeout=5)
    result["internet_connected"] = is_connected
    result["internet_error"] = conn_error
    
    # Check Planetary Computer access
    if is_connected:
        is_accessible, access_error = check_planetary_computer_access(timeout=10)
        result["planetary_computer_accessible"] = is_accessible
        result["pc_error"] = access_error
        
        if is_accessible:
            result["status"] = "ready"
        else:
            result["status"] = "pc_unavailable"
    else:
        result["status"] = "no_internet"
        result["planetary_computer_accessible"] = False
        result["pc_error"] = "Cannot check - no internet connection"
    
    return result


@app.get("/", response_class=HTMLResponse)
async def root():
    """Serve main index page with tabs"""
    html_file = Path(__file__).parent / "index.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        return HTMLResponse("""
        <html>
            <head><title>Land Classification System</title></head>
            <body>
                <h1>Land Classification System</h1>
                <p>API Documentation: <a href="/docs">/docs</a></p>
                <p>Training: <a href="/training">/training</a></p>
                <p>Prediction: <a href="/prediction">/prediction</a></p>
                <p>Dashboard: <a href="/dashboard">/dashboard</a></p>
            </body>
        </html>
        """)


@app.get("/training", response_class=HTMLResponse)
async def training_page():
    """Serve training interface"""
    html_file = Path(__file__).parent / "training_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Training interface không tồn tại")


@app.get("/cloud-training", response_class=HTMLResponse)
async def cloud_training_page():
    """Serve cloud removal training interface"""
    html_file = Path(__file__).parent / "cloud_training_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Cloud training interface không tồn tại")


@app.get("/prediction", response_class=HTMLResponse)
async def prediction_page():
    """Serve prediction interface"""
    html_file = Path(__file__).parent / "prediction_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Prediction interface không tồn tại")


@app.get("/dashboard", response_class=HTMLResponse)
async def dashboard():
    """Serve dashboard visualization"""
    html_file = Path(__file__).parent / "dashboard.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Dashboard không tồn tại")


@app.get("/batch", response_class=HTMLResponse)
async def batch_page():
    """Serve batch processing interface"""
    html_file = Path(__file__).parent / "batch_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Batch interface không tồn tại")


@app.get("/ndvi", response_class=HTMLResponse)
async def ndvi_page():
    """Serve NDVI time series interface"""
    html_file = Path(__file__).parent / "ndvi_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="NDVI interface không tồn tại")


@app.get("/reports", response_class=HTMLResponse)
async def reports_page():
    """Serve reports management interface"""
    html_file = Path(__file__).parent / "reports_interface.html"
    if html_file.exists():
        return FileResponse(html_file)
    else:
        raise HTTPException(status_code=404, detail="Reports interface không tồn tại")


@app.get("/api/models/list")
async def list_models():
    """Liệt kê tất cả models có sẵn với metadata"""
    try:
        model_manager = get_model_manager()
        models = model_manager.list_models()
        return {
            "success": True,
            "models": models,
            "count": len(models)
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e),
            "models": []
        }


@app.get("/api/models/{model_filename}/info")
async def get_model_info(model_filename: str):
    """Lấy thông tin chi tiết về model"""
    try:
        model_manager = get_model_manager()
        info = model_manager.get_model_info(model_filename)
        if info is None:
            raise HTTPException(status_code=404, detail=f"Model không tồn tại: {model_filename}")
        return {
            "success": True,
            "model": info
        }
    except HTTPException:
        raise
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }


@app.get("/api/models/{model_filename}/validate")
async def validate_model(model_filename: str):
    """Validate model file"""
    try:
        model_manager = get_model_manager()
        validation = model_manager.validate_model(model_filename)
        return {
            "success": True,
            "validation": validation
        }
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }


@app.get("/api/cloud-removal/methods")
async def get_cloud_removal_methods():
    """Lấy danh sách các phương pháp xử lý mây có sẵn"""
    methods = get_available_methods()
    return {
        "success": True,
        "methods": methods,
        "default": "classic",
        "description": "Cloud removal strategies for Sentinel-2 data processing"
    }


@app.get("/api/cloud-removal/models")
async def list_cloud_removal_models():
    """Liệt kê các cloud removal models đã train"""
    model_dir = Path("cloud_removal_model")
    if not model_dir.exists():
        return {"models": [], "count": 0}
    
    models = []
    # Search for ALL .pth files in cloud_removal_model and subdirectories
    for model_file in model_dir.rglob("*.pth"):
        # Skip non-cloud-removal models (keep land classification models separate)
        if any(x in model_file.name.lower() for x in ['mobilenet', 'cnn_', 'swin', 'xgboost', 'random_forest']):
            continue
            
        try:
            import torch
            import json
            
            # Try to load metadata from .json sidecar file first
            metadata_file = model_file.with_suffix('.json')
            if metadata_file.exists():
                try:
                    with open(metadata_file, 'r') as f:
                        metadata = json.load(f)
                    
                    models.append({
                        "filename": model_file.name,
                        "path": str(model_file),
                        "relative_path": str(model_file.relative_to(model_dir)),
                        "epoch": metadata.get('epoch', 0),
                        "train_loss": metadata.get('train_loss', 0),
                        "val_loss": metadata.get('val_loss', 0),
                        "use_s1": metadata.get('use_s1', True),
                        "in_channels": metadata.get('in_channels', 6),
                        "out_channels": metadata.get('out_channels', 4),
                        "description": metadata.get('description', ''),
                        "created": model_file.stat().st_mtime,
                        "size_mb": model_file.stat().st_size / (1024 * 1024),
                        "has_metadata": True
                    })
                    continue
                except Exception as e:
                    print(f"[Cloud Models] Failed to read metadata file {metadata_file}: {e}")
            
            # Try to load checkpoint metadata from .pth file
            try:
                checkpoint = torch.load(model_file, map_location='cpu')
                epoch = checkpoint.get('epoch', 0) if isinstance(checkpoint, dict) else 0
                train_loss = checkpoint.get('train_loss', 0) if isinstance(checkpoint, dict) else 0
                val_loss = checkpoint.get('val_loss', 0) if isinstance(checkpoint, dict) else 0
                use_s1 = checkpoint.get('use_s1', True) if isinstance(checkpoint, dict) else True
                in_channels = checkpoint.get('in_channels', 6) if isinstance(checkpoint, dict) else 6
                out_channels = checkpoint.get('out_channels', 4) if isinstance(checkpoint, dict) else 4
            except:
                # If checkpoint format is different or corrupted, use defaults
                epoch = 0
                train_loss = 0
                val_loss = 0
                use_s1 = True
                in_channels = 3
                out_channels = 3
            
            models.append({
                "filename": model_file.name,
                "path": str(model_file),
                "relative_path": str(model_file.relative_to(model_dir)),
                "epoch": epoch,
                "train_loss": train_loss,
                "val_loss": val_loss,
                "use_s1": use_s1,
                "in_channels": in_channels,
                "out_channels": out_channels,
                "description": "",
                "created": model_file.stat().st_mtime,
                "size_mb": model_file.stat().st_size / (1024 * 1024),
                "has_metadata": False
            })
        except Exception as e:
            print(f"[Cloud Models] Error loading {model_file}: {e}")
            # Still add the file even if we can't load metadata
            models.append({
                "filename": model_file.name,
                "path": str(model_file),
                "relative_path": str(model_file.relative_to(model_dir)),
                "epoch": 0,
                "train_loss": 0,
                "val_loss": 0,
                "use_s1": False,
                "in_channels": 3,
                "out_channels": 3,
                "description": "",
                "created": model_file.stat().st_mtime,
                "size_mb": model_file.stat().st_size / (1024 * 1024),
                "has_metadata": False
            })
    
    models.sort(key=lambda x: x['created'], reverse=True)
    return {"models": models, "count": len(models)}


@app.post("/api/cloud-removal/train")
async def train_cloud_removal(config: CloudRemovalTrainingConfig, background_tasks: BackgroundTasks):
    """Bắt đầu train cloud removal model"""
    
    # Check if data directory exists
    data_dir = Path(config.data_dir)
    if not data_dir.exists():
        raise HTTPException(
            status_code=404,
            detail=f"Data directory not found: {config.data_dir}"
        )
    
    # Create status tracking
    training_id = datetime.now().strftime("%Y%m%d_%H%M%S")
    
    async def run_cloud_training():
        try:
            from train_cloud_removal import train_cloud_removal_model
            
            print(f"[CLOUD REMOVAL TRAINING] Starting training {training_id}")
            
            model, train_losses, val_losses = train_cloud_removal_model(
                data_dir=config.data_dir,
                use_s1=config.use_s1,
                batch_size=config.batch_size,
                num_epochs=config.num_epochs,
                learning_rate=config.learning_rate,
                device="cuda" if config.use_gpu else "cpu",
                save_dir="cloud_removal_model"
            )
            
            print(f"[CLOUD REMOVAL TRAINING] Completed {training_id}")
            
            return {
                "success": True,
                "training_id": training_id,
                "final_train_loss": train_losses[-1],
                "final_val_loss": val_losses[-1],
                "epochs": len(train_losses)
            }
            
        except Exception as e:
            print(f"[CLOUD REMOVAL TRAINING ERROR] {e}")
            import traceback
            traceback.print_exc()
            return {
                "success": False,
                "error": str(e),
                "training_id": training_id
            }
    
    # Run in background
    background_tasks.add_task(run_cloud_training)
    
    return {
        "message": "Cloud removal training started",
        "training_id": training_id,
        "config": {
            "data_dir": config.data_dir,
            "use_s1": config.use_s1,
            "batch_size": config.batch_size,
            "num_epochs": config.num_epochs,
            "learning_rate": config.learning_rate,
            "use_gpu": config.use_gpu
        }
    }


@app.post("/api/cloud-removal/upload")
async def upload_cloud_removal_model(
    file: UploadFile = File(...), 
    epoch: int = Form(0),
    train_loss: float = Form(0.0),
    val_loss: float = Form(0.0),
    in_channels: int = Form(6),
    out_channels: int = Form(4),
    use_s1: bool = Form(True),
    description: str = Form("")
):
    """Upload cloud removal .pth model with optional metadata"""
    
    # Debug logging
    print(f"[Upload] Received parameters:")
    print(f"  File: {file.filename}")
    print(f"  Epoch: {epoch} (type: {type(epoch)})")
    print(f"  Train Loss: {train_loss} (type: {type(train_loss)})")
    print(f"  Val Loss: {val_loss} (type: {type(val_loss)})")
    print(f"  In Channels: {in_channels} (type: {type(in_channels)})")
    print(f"  Out Channels: {out_channels} (type: {type(out_channels)})")
    print(f"  Use S1: {use_s1} (type: {type(use_s1)})")
    print(f"  Description: {description}")
    
    # Validate file extension
    if not file.filename.endswith('.pth'):
        raise HTTPException(status_code=400, detail="Only .pth files are allowed")
    
    # Security check
    if ".." in file.filename or "/" in file.filename or "\\" in file.filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    try:
        model_dir = Path("cloud_removal_model")
        model_dir.mkdir(exist_ok=True)
        
        # Save uploaded file
        file_path = model_dir / file.filename
        
        # Check if file already exists
        if file_path.exists():
            raise HTTPException(status_code=400, detail=f"Model {file.filename} already exists")
        
        # Write file
        with open(file_path, "wb") as f:
            content = await file.read()
            f.write(content)
        
        file_size = file_path.stat().st_size
        
        # Save metadata as JSON sidecar file
        import json
        metadata_file = file_path.with_suffix('.json')
        
        metadata_dict = {
            "filename": file.filename,
            "epoch": epoch,
            "train_loss": train_loss,
            "val_loss": val_loss,
            "in_channels": in_channels,
            "out_channels": out_channels,
            "use_s1": use_s1,
            "description": description,
            "uploaded_at": datetime.now().isoformat()
        }
        
        with open(metadata_file, 'w') as f:
            json.dump(metadata_dict, f, indent=2)
        
        print(f"[Upload] Saved model: {file_path}")
        print(f"[Upload] Saved metadata: {metadata_file}")
        print(f"[Upload] Metadata: {metadata_dict}")
        
        return {
            "message": f"Successfully uploaded {file.filename}",
            "filename": file.filename,
            "size_mb": round(file_size / 1024 / 1024, 2),
            "path": str(file_path),
            "metadata": metadata_dict
        }
    except HTTPException:
        raise
    except Exception as e:
        print(f"[Upload] Error: {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")


@app.post("/api/land-classification/upload")
async def upload_land_classification_model(
    file: UploadFile = File(...),
    model_type: str = Form("mobilenet"),  # mobilenet, cnn, swin, xgboost, etc.
    epoch: int = Form(0),
    train_accuracy: float = Form(0.0),
    val_accuracy: float = Form(0.0),
    train_loss: float = Form(0.0),
    val_loss: float = Form(0.0),
    num_classes: int = Form(10),
    input_size: int = Form(64),
    description: str = Form("")
):
    """Upload land classification model with metadata"""
    
    # Debug logging
    print(f"[Land Upload] Received parameters:")
    print(f"  File: {file.filename}")
    print(f"  Model Type: {model_type}")
    print(f"  Epoch: {epoch}")
    print(f"  Train Accuracy: {train_accuracy}")
    print(f"  Val Accuracy: {val_accuracy}")
    print(f"  Train Loss: {train_loss}")
    print(f"  Val Loss: {val_loss}")
    print(f"  Num Classes: {num_classes}")
    print(f"  Input Size: {input_size}")
    print(f"  Description: {description}")
    
    # Validate file extension
    valid_extensions = ['.pth', '.pkl', '.joblib', '.h5', '.keras']
    if not any(file.filename.endswith(ext) for ext in valid_extensions):
        raise HTTPException(
            status_code=400, 
            detail=f"Only {', '.join(valid_extensions)} files are allowed"
        )
    
    # Security check
    if ".." in file.filename or "/" in file.filename or "\\" in file.filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    try:
        model_dir = Path("land_classification_model")
        model_dir.mkdir(exist_ok=True)
        
        # Save uploaded file
        file_path = model_dir / file.filename
        
        # Check if file already exists
        if file_path.exists():
            raise HTTPException(
                status_code=400, 
                detail=f"Model {file.filename} already exists"
            )
        
        # Write file
        with open(file_path, "wb") as f:
            content = await file.read()
            f.write(content)
        
        file_size = file_path.stat().st_size
        
        # Save metadata as JSON sidecar file
        metadata_file = file_path.with_suffix(file_path.suffix + '.json')
        
        metadata_dict = {
            "filename": file.filename,
            "model_type": model_type,
            "epoch": epoch,
            "train_accuracy": train_accuracy,
            "val_accuracy": val_accuracy,
            "train_loss": train_loss,
            "val_loss": val_loss,
            "num_classes": num_classes,
            "input_size": input_size,
            "description": description,
            "uploaded_at": datetime.now().isoformat()
        }
        
        with open(metadata_file, 'w') as f:
            json.dump(metadata_dict, f, indent=2)
        
        print(f"[Land Upload] Saved model: {file_path}")
        print(f"[Land Upload] Saved metadata: {metadata_file}")
        
        return {
            "message": f"Successfully uploaded {file.filename}",
            "filename": file.filename,
            "size_mb": round(file_size / 1024 / 1024, 2),
            "path": str(file_path),
            "metadata": metadata_dict
        }
    except HTTPException:
        raise
    except Exception as e:
        print(f"[Land Upload] Error: {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")


@app.get("/api/land-classification/models")
async def list_land_classification_models():
    """List all uploaded land classification models"""
    model_dir = Path("land_classification_model")
    if not model_dir.exists():
        return {"models": [], "count": 0}
    
    models = []
    # Search for model files
    for model_file in model_dir.rglob("*"):
        # Skip JSON metadata files
        if model_file.suffix == '.json':
            continue
        
        # Only include model files
        valid_extensions = ['.pth', '.pkl', '.joblib', '.h5', '.keras']
        if not any(model_file.name.endswith(ext) for ext in valid_extensions):
            continue
        
        try:
            # Try to load metadata from JSON sidecar file
            metadata_file = Path(str(model_file) + '.json')
            if metadata_file.exists():
                with open(metadata_file, 'r') as f:
                    metadata = json.load(f)
                
                models.append({
                    "filename": model_file.name,
                    "path": str(model_file),
                    "relative_path": str(model_file.relative_to(model_dir)),
                    "model_type": metadata.get('model_type', 'unknown'),
                    "epoch": metadata.get('epoch', 0),
                    "train_accuracy": metadata.get('train_accuracy', 0),
                    "val_accuracy": metadata.get('val_accuracy', 0),
                    "train_loss": metadata.get('train_loss', 0),
                    "val_loss": metadata.get('val_loss', 0),
                    "num_classes": metadata.get('num_classes', 10),
                    "input_size": metadata.get('input_size', 64),
                    "description": metadata.get('description', ''),
                    "created": model_file.stat().st_mtime,
                    "size_mb": model_file.stat().st_size / (1024 * 1024),
                    "has_metadata": True
                })
            else:
                # No metadata file, use defaults
                models.append({
                    "filename": model_file.name,
                    "path": str(model_file),
                    "relative_path": str(model_file.relative_to(model_dir)),
                    "model_type": "unknown",
                    "epoch": 0,
                    "train_accuracy": 0,
                    "val_accuracy": 0,
                    "train_loss": 0,
                    "val_loss": 0,
                    "num_classes": 10,
                    "input_size": 64,
                    "description": "",
                    "created": model_file.stat().st_mtime,
                    "size_mb": model_file.stat().st_size / (1024 * 1024),
                    "has_metadata": False
                })
        except Exception as e:
            print(f"[Land Models] Error loading {model_file}: {e}")
            continue
    
    # Sort by creation time (newest first)
    models.sort(key=lambda x: x['created'], reverse=True)
    
    return {
        "models": models,
        "count": len(models)
    }


@app.delete("/api/land-classification/models/{filename}")
async def delete_land_classification_model(filename: str):
    """Delete a land classification model"""
    model_dir = Path("land_classification_model")
    model_path = model_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not model_path.exists():
        raise HTTPException(status_code=404, detail=f"Model not found: {filename}")
    
    try:
        # Delete model file
        model_path.unlink()
        
        # Delete metadata file if exists
        metadata_file = Path(str(model_path) + '.json')
        if metadata_file.exists():
            metadata_file.unlink()
        
        return {
            "success": True,
            "message": f"Deleted land classification model: {filename}"
        }
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to delete: {str(e)}")


@app.delete("/api/cloud-removal/models/{filename}")
async def delete_cloud_removal_model(filename: str):
    """Xóa cloud removal model"""
    model_dir = Path("cloud_removal_model")
    model_path = model_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not model_path.exists():
        raise HTTPException(status_code=404, detail=f"Model not found: {filename}")
    
    try:
        model_path.unlink()
        return {
            "success": True,
            "message": f"Deleted cloud removal model: {filename}"
        }
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to delete: {str(e)}")


@app.delete("/api/models/{model_filename}")
async def delete_model(model_filename: str):
    """Xóa model"""
    try:
        model_manager = get_model_manager()
        success = model_manager.delete_model(model_filename)
        if not success:
            raise HTTPException(status_code=404, detail=f"Model không tồn tại: {model_filename}")
        return {
            "success": True,
            "message": f"Đã xóa model: {model_filename}"
        }
    except HTTPException:
        raise
    except Exception as e:
        return {
            "success": False,
            "error": str(e)
        }


@app.get("/api/config/presets")
async def get_presets():
    """Lấy các preset cấu hình sẵn"""
    return {
        "presets": [
            {
                "name": "PC - Nhỏ (3 tháng, 20m, 12 scenes)",
                "config": {
                    "min_lon": 105.5, "min_lat": 9.2, "max_lon": 106.4, "max_lat": 10.0,
                    "start_date": "2023-03-01", "end_date": "2023-05-31",
                    "max_scenes": 12, "cloud_cover": 30, "resolution": 20,
                    "test_size": 0.2
                }
            },
            {
                "name": "Server - Trung bình (6 tháng, 10m, 30 scenes)",
                "config": {
                    "min_lon": 105.5, "min_lat": 9.2, "max_lon": 106.4, "max_lat": 10.0,
                    "start_date": "2023-01-01", "end_date": "2023-06-30",
                    "max_scenes": 30, "cloud_cover": 30, "resolution": 10,
                    "test_size": 0.2
                }
            },
            {
                "name": "Full - ODC (10 tháng, 10m, 1 scene) - từ 01.train_ODC.ipynb",
                "config": {
                    "min_lon": 105.5, "min_lat": 9.2, "max_lon": 106.4, "max_lat": 10.0,
                    "start_date": "2023-03-01", "end_date": "2023-12-31",
                    "max_scenes": 1, "cloud_cover": 50, "resolution": 10,
                    "test_size": 0.2
                }
            }
        ]
    }


@app.get("/api/provinces/list")
async def list_provinces():
    """Lấy danh sách tất cả các tỉnh thành Việt Nam"""
    return {
        "provinces": get_all_provinces(),
        "count": len(get_all_provinces())
    }


@app.get("/api/provinces/by-region")
async def list_provinces_by_region():
    """Lấy danh sách tỉnh thành theo vùng miền"""
    return get_provinces_by_region()


@app.get("/api/provinces/{province_name}/bbox")
async def get_province_bbox_api(province_name: str):
    """Lấy bbox của một tỉnh thành"""
    bbox = get_province_bbox(province_name)
    if bbox is None:
        raise HTTPException(status_code=404, detail=f"Không tìm thấy tỉnh: {province_name}")
    return {
        "province": province_name,
        "bbox": bbox,
        "min_lon": bbox[0],
        "min_lat": bbox[1],
        "max_lon": bbox[2],
        "max_lat": bbox[3]
    }


@app.get("/api/provinces/search/{query}")
async def search_provinces(query: str):
    """Tìm kiếm tỉnh thành theo tên"""
    results = search_province(query)
    return {
        "query": query,
        "results": results,
        "count": len(results)
    }


@app.get("/api/provinces-32/list")
async def list_provinces_32():
    """Lấy danh sách 32 tỉnh thành sau sáp nhập"""
    return {
        "provinces": get_all_provinces_32(),
        "count": len(get_all_provinces_32()),
        "note": "32 tỉnh thành sau sáp nhập theo Nghị quyết 1211/2023"
    }


@app.get("/api/provinces-32/by-region")
async def list_provinces_by_region_32():
    """Lấy danh sách 32 tỉnh thành theo vùng miền"""
    return get_provinces_by_region_32()


@app.get("/api/provinces-32/{province_name}/bbox")
async def get_province_bbox_api_32(province_name: str):
    """Lấy bbox của một tỉnh thành (32 tỉnh)"""
    bbox = get_province_bbox_32(province_name)
    if bbox is None:
        raise HTTPException(status_code=404, detail=f"Không tìm thấy tỉnh: {province_name}")
    
    # Get merged info
    info = get_merged_info(province_name)
    
    return {
        "province": province_name,
        "bbox": bbox,
        "min_lon": bbox[0],
        "min_lat": bbox[1],
        "max_lon": bbox[2],
        "max_lat": bbox[3],
        "merged_from": info.get("merged_from"),
        "area_km2": info.get("area_km2"),
        "region": info.get("region")
    }


@app.get("/api/provinces-32/search/{query}")
async def search_provinces_32(query: str):
    """Tìm kiếm tỉnh thành theo tên (32 tỉnh)"""
    results = search_province_32(query)
    return {
        "query": query,
        "results": results,
        "count": len(results)
    }


@app.get("/api/provinces-32/statistics")
async def get_provinces_stats():
    """Thống kê các tỉnh đã sáp nhập"""
    return get_provinces_statistics()


@app.get("/api/training/labels")
async def get_training_labels():
    """Lấy danh sách các label từ training data"""
    return {
        "label_mapping": DEFAULT_LABEL_MAPPING,
        "label_names": DEFAULT_LABEL_NAMES,
        "count": len(DEFAULT_LABEL_MAPPING),
        "labels": [
            {"code": code, "name": name, "description": name}
            for name, code in DEFAULT_LABEL_MAPPING.items()
        ]
    }


@app.get("/api/training/files")
async def list_training_files():
    """Liệt kê các file training shapefile có sẵn"""
    train_dir = Path("train")
    if not train_dir.exists():
        raise HTTPException(status_code=404, detail="Thư mục train không tồn tại")
    
    shapefiles = []
    for shp_file in train_dir.glob("*.shp"):
        try:
            # Get file info
            file_size = shp_file.stat().st_size
            file_modified = datetime.fromtimestamp(shp_file.stat().st_mtime).isoformat()
            
            # Try to read shapefile to get point count and unique labels
            try:
                import geopandas as gpd
                gdf = gpd.read_file(str(shp_file))
                
                # Convert to WGS84 if not already
                if gdf.crs and gdf.crs.to_epsg() != 4326:
                    gdf = gdf.to_crs("EPSG:4326")
                
                point_count = len(gdf)
                
                # Try to find label column (Hientrang, class, label, etc.)
                label_column = None
                for col in ['Hientrang', 'class', 'label', 'Class', 'Label']:
                    if col in gdf.columns:
                        label_column = col
                        break
                
                unique_labels = []
                if label_column:
                    unique_labels = sorted(gdf[label_column].unique().tolist())
                
                shapefiles.append({
                    "filename": shp_file.name,
                    "path": f"train/{shp_file.name}",
                    "size_bytes": file_size,
                    "size_mb": round(file_size / 1024 / 1024, 2),
                    "modified": file_modified,
                    "point_count": point_count,
                    "label_column": label_column,
                    "unique_labels": unique_labels,
                    "label_count": len(unique_labels)
                })
            except Exception as e:
                # If cannot read shapefile, just add basic info
                shapefiles.append({
                    "filename": shp_file.name,
                    "path": f"train/{shp_file.name}",
                    "size_bytes": file_size,
                    "size_mb": round(file_size / 1024 / 1024, 2),
                    "modified": file_modified,
                    "error": f"Cannot read shapefile: {str(e)}"
                })
        except Exception as e:
            continue
    
    return {
        "files": shapefiles,
        "count": len(shapefiles),
        "directory": "train/"
    }


@app.get("/api/overlay/shapefiles")
async def list_overlay_shapefiles():
    """Liệt kê các shapefile có sẵn cho overlay trên prediction"""
    overlay_dirs = ["region", "ChauThanh", "ThuanHoa"]
    shapefiles = []
    
    for overlay_dir in overlay_dirs:
        dir_path = Path(overlay_dir)
        if not dir_path.exists():
            continue
            
        # Find all .shp files in this directory and subdirectories
        for shp_file in dir_path.rglob("*.shp"):
            try:
                file_size = shp_file.stat().st_size
                file_modified = datetime.fromtimestamp(shp_file.stat().st_mtime).isoformat()
                
                # Try to read shapefile to get feature count and bbox
                import geopandas as gpd
                gdf = gpd.read_file(str(shp_file))
                feature_count = len(gdf)
                
                # Calculate bbox (always in EPSG:4326 for consistency)
                bbox = None
                if not gdf.empty and gdf.crs:
                    try:
                        # Reproject to EPSG:4326 if needed
                        if gdf.crs != "EPSG:4326":
                            gdf_4326 = gdf.to_crs("EPSG:4326")
                        else:
                            gdf_4326 = gdf
                        
                        # Get total bounds [minx, miny, maxx, maxy]
                        bounds = gdf_4326.total_bounds
                        if len(bounds) == 4:
                            bbox = [
                                float(bounds[0]),  # min_lon
                                float(bounds[1]),  # min_lat
                                float(bounds[2]),  # max_lon
                                float(bounds[3])   # max_lat
                            ]
                    except Exception as bbox_error:
                        print(f"[WARNING] Cannot calculate bbox for {shp_file}: {bbox_error}")
                
                # Get relative path from workspace root
                relative_path = str(shp_file)
                
                shapefiles.append({
                    "filename": shp_file.name,
                    "path": relative_path,
                    "directory": overlay_dir,
                    "size_bytes": file_size,
                    "size_mb": round(file_size / 1024 / 1024, 2),
                    "modified": file_modified,
                    "feature_count": feature_count,
                    "crs": str(gdf.crs) if gdf.crs else "Unknown",
                    "bbox": bbox  # [min_lon, min_lat, max_lon, max_lat] in EPSG:4326
                })
            except Exception as e:
                # If cannot read shapefile, just add basic info
                file_size = shp_file.stat().st_size
                file_modified = datetime.fromtimestamp(shp_file.stat().st_mtime).isoformat()
                relative_path = str(shp_file)
                
                shapefiles.append({
                    "filename": shp_file.name,
                    "path": relative_path,
                    "directory": overlay_dir,
                    "size_bytes": file_size,
                    "size_mb": round(file_size / 1024 / 1024, 2),
                    "modified": file_modified,
                    "error": f"Cannot read shapefile: {str(e)}"
                })
    
    return {
        "shapefiles": shapefiles,
        "count": len(shapefiles),
        "directories": overlay_dirs
    }


@app.get("/api/training/shapefile/{filename}/labels")
async def get_shapefile_labels(filename: str):
    """Lấy các label từ một shapefile cụ thể"""
    train_dir = Path("train")
    shp_file = train_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not shp_file.exists():
        raise HTTPException(status_code=404, detail=f"File {filename} không tồn tại")
    
    try:
        import geopandas as gpd
        gdf = gpd.read_file(str(shp_file))
        
        # Convert to WGS84 if not already
        if gdf.crs and gdf.crs.to_epsg() != 4326:
            print(f"📍 Converting shapefile from {gdf.crs} to WGS84 (EPSG:4326)")
            gdf = gdf.to_crs("EPSG:4326")
        
        # Try to find label column
        label_column = None
        for col in ['Hientrang', 'class', 'label', 'Class', 'Label']:
            if col in gdf.columns:
                label_column = col
                break
        
        if not label_column:
            return {
                "filename": filename,
                "error": "No label column found",
                "columns": list(gdf.columns),
                "point_count": len(gdf),
                "bbox": gdf.total_bounds.tolist()  # Still return bbox even without labels
            }
        
        # Get unique labels and their counts
        label_counts = gdf[label_column].value_counts().to_dict()
        unique_labels = sorted(gdf[label_column].unique().tolist())
        
        # Map to default labels if possible
        mapped_labels = []
        for label in unique_labels:
            code = DEFAULT_LABEL_MAPPING.get(label, "unknown")
            mapped_labels.append({
                "name": label,
                "code": code,
                "count": int(label_counts.get(label, 0)),
                "mapped": label in DEFAULT_LABEL_MAPPING
            })
        
        # Get bbox in WGS84 coordinates
        bbox = gdf.total_bounds.tolist()  # [minx, miny, maxx, maxy]
        print(f"✅ Shapefile bbox (WGS84): {bbox}")
        
        return {
            "filename": filename,
            "label_column": label_column,
            "point_count": len(gdf),
            "unique_labels": unique_labels,
            "label_count": len(unique_labels),
            "labels": mapped_labels,
            "bbox": bbox,  # Now in WGS84 lat/lon
            "columns": list(gdf.columns)
        }
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error reading shapefile: {str(e)}")


@app.get("/api/training/status", response_model=TrainingStatus)
async def get_training_status():
    """Kiểm tra trạng thái training"""
    return training_status


@app.post("/api/training/start")
async def start_training(config: TrainingConfig, background_tasks: BackgroundTasks):
    """Bắt đầu training với config đã chọn"""
    global training_status
    
    if training_status["is_training"]:
        raise HTTPException(status_code=400, detail="Training đang chạy, vui lòng đợi")
    
    # Reset status
    training_status = {
        "is_training": True,
        "progress": "Đang khởi tạo...",
        "error": None,
        "result": None,
        "start_time": datetime.now().isoformat(),
        "end_time": None
    }
    
    # Run training in background
    background_tasks.add_task(run_training, config)
    
    return {"message": "Training đã bắt đầu", "status": training_status}


@app.post("/api/training/stop")
async def stop_training():
    """Dừng training (nếu đang chạy)"""
    global training_status
    
    if not training_status["is_training"]:
        return {"message": "Không có training nào đang chạy"}
    
    # Set cancel flag - the training will check this and stop
    training_status["cancel_requested"] = True
    training_status["progress"] = "Đang hủy training..."
    
    return {"message": "Đang dừng training..."}


@app.post("/api/cache/clear")
async def clear_cache():
    """Xóa cache dataset"""
    import shutil
    cache_dir = Path("dataset_cache")
    
    if not cache_dir.exists():
        return {"message": "Không có cache để xóa", "deleted": 0}
    
    # Count files
    cache_files = list(cache_dir.glob("*.joblib"))
    count = len(cache_files)
    
    # Delete all cache files
    for cache_file in cache_files:
        try:
            cache_file.unlink()
        except:
            pass
    
    return {"message": f"Đã xóa {count} file cache", "deleted": count}


@app.get("/api/cache/info")
async def get_cache_info():
    """Lấy thông tin về cache với metadata đầy đủ, tự động xóa cache cũ có lazy data"""
    cache_dir = Path("dataset_cache")
    
    if not cache_dir.exists():
        return {"exists": False, "files": [], "total_size_mb": 0}
    
    cache_files = []
    total_size = 0
    deleted_count = 0
    
    for cache_file in cache_dir.glob("*.joblib"):
        # Skip if file doesn't exist (race condition)
        if not cache_file.exists():
            continue
            
        size = cache_file.stat().st_size
        
        # Try to load metadata from cache and check if it's valid
        metadata = {}
        is_valid = True
        try:
            cached_data = joblib.load(cache_file)
            
            # Check if cache contains lazy data (will cause 403 errors)
            if isinstance(cached_data, dict) and "s2_data" in cached_data:
                s2_data_temp = cached_data["s2_data"]
                is_lazy = False
                try:
                    is_lazy = any(hasattr(s2_data_temp[var].data, 'chunks') for var in s2_data_temp.data_vars)
                except:
                    pass
                
                if is_lazy:
                    print(f"[CLEANUP] Deleting cache with lazy data: {cache_file.name}")
                    cache_file.unlink()
                    deleted_count += 1
                    is_valid = False
            
            if is_valid and isinstance(cached_data, dict):
                metadata = {
                    "bbox": cached_data.get("bbox", []),
                    "time_range": cached_data.get("time_range", ""),
                    "resolution": cached_data.get("resolution", 20),
                    "n_samples": len(cached_data.get("features", [])),
                    "created": cached_data.get("timestamp", "")
                }
                # Parse time_range to get start/end dates
                if metadata["time_range"]:
                    time_parts = metadata["time_range"].split("/")
                    if len(time_parts) == 2:
                        metadata["start_date"] = time_parts[0]
                        metadata["end_date"] = time_parts[1]
                # Parse bbox to get min/max lon/lat
                if metadata["bbox"] and len(metadata["bbox"]) == 4:
                    metadata["min_lon"] = metadata["bbox"][0]
                    metadata["min_lat"] = metadata["bbox"][1]
                    metadata["max_lon"] = metadata["bbox"][2]
                    metadata["max_lat"] = metadata["bbox"][3]
        except Exception as e:
            print(f"[CLEANUP] Error loading cache {cache_file.name}: {e}. Deleting...")
            try:
                cache_file.unlink()
                deleted_count += 1
                is_valid = False
            except:
                pass
        
        # Only add valid cache files to the list
        if is_valid:
            total_size += size
            cache_files.append({
                "filename": cache_file.name,
                "size_mb": round(size / 1024 / 1024, 2),
                "modified": datetime.fromtimestamp(cache_file.stat().st_mtime).isoformat(),
                "metadata": metadata
            })
    
    # Sort by modified time (newest first)
    cache_files.sort(key=lambda x: x["modified"], reverse=True)
    
    if deleted_count > 0:
        print(f"[CLEANUP] Deleted {deleted_count} invalid cache files")
    
    return {
        "exists": True,
        "files": cache_files,
        "count": len(cache_files),
        "total_size_mb": round(total_size / 1024 / 1024, 2),
        "deleted_invalid": deleted_count
    }



@app.get("/api/models/list")
async def list_models():
    """Liệt kê các model đã train"""
    model_dir = Path("model_train")
    if not model_dir.exists():
        return {"models": []}

    models = []
    # List all .joblib model files (actual trained models)
    for model_file in model_dir.glob("*.joblib"):
        # Skip any file that contains '_info' in its name
        if '_info' in model_file.stem:
            continue
            
        info = {}
        # Try to find corresponding .json info file
        # Remove .joblib and try with _info.json
        base_name = model_file.stem  # e.g., "model_cnn_20251221_163841"
        info_file = model_dir / f"{base_name}_info.json"
        
        if info_file.exists():
            try:
                with open(info_file) as f:
                    info = json.load(f)
            except Exception as e:
                info = {"error": str(e)}
        
        size_mb = round(model_file.stat().st_size / 1024 / 1024, 2)
        created = datetime.fromtimestamp(model_file.stat().st_mtime).isoformat()
        
        models.append({
            "filename": model_file.name,
            "created": created,
            "size_mb": size_mb,
            "info": info
        })

    # Sort by creation time (newest first)
    models.sort(key=lambda x: x["created"], reverse=True)
    return {"models": models}


# ============ REPORTS API ============

@app.get("/api/reports/list")
async def list_reports():
    """Liệt kê các báo cáo đã tạo"""
    reports_dir = Path("reports")
    reports_dir.mkdir(exist_ok=True)
    
    reports = []
    for report_file in reports_dir.glob("*.html"):
        # Determine report type from filename
        if "training" in report_file.name:
            report_type = "training"
        elif "prediction" in report_file.name:
            report_type = "prediction"
        else:
            report_type = "unknown"
        
        report_info = {
            "filename": report_file.name,
            "type": report_type,
            "created": datetime.fromtimestamp(report_file.stat().st_mtime).isoformat(),
            "size_kb": round(report_file.stat().st_size / 1024, 2),
            "view_url": f"/api/reports/view/{report_file.name}",
            "download_url": f"/api/reports/download/{report_file.name}",
            "is_batch_job": False,
            "batch_metadata": None
        }
        
        # Check if this is a batch job report
        if report_type == "prediction":
            predictions_dir = Path("predictions")
            # Look for batch metadata JSON files that reference this report
            for json_file in predictions_dir.glob("batch_*.json"):
                try:
                    import json
                    with open(json_file, 'r') as f:
                        metadata = json.load(f)
                        if metadata.get("report_filename") == report_file.name or \
                           (metadata.get("batch_job_id") and report_file.name.endswith('.html')):
                            report_info["is_batch_job"] = True
                            report_info["batch_metadata"] = {
                                "batch_job_id": metadata.get("batch_job_id"),
                                "batch_name": metadata.get("batch_name"),
                                "batch_timestamp": metadata.get("batch_timestamp")
                            }
                            break
                except Exception as e:
                    pass
        
        reports.append(report_info)
    
    # Sort by creation time (newest first)
    reports.sort(key=lambda x: x["created"], reverse=True)
    return {"reports": reports, "count": len(reports)}


@app.get("/api/reports/view/{filename}", response_class=HTMLResponse)
async def view_report(filename: str):
    """Xem báo cáo HTML trực tiếp"""
    reports_dir = Path("reports")
    file_path = reports_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
    
    with open(file_path, 'r', encoding='utf-8') as f:
        html_content = f.read()
    
    return HTMLResponse(content=html_content)


@app.get("/api/reports/download/{filename}")
async def download_report(filename: str):
    """Download báo cáo HTML"""
    reports_dir = Path("reports")
    file_path = reports_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
    
    return FileResponse(
        path=str(file_path),
        filename=filename,
        media_type="text/html",
        headers={
            "Content-Disposition": f"attachment; filename={filename}"
        }
    )


@app.delete("/api/reports/delete/{filename}")
async def delete_report(filename: str):
    """Xóa một báo cáo"""
    reports_dir = Path("reports")
    file_path = reports_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
    
    try:
        file_path.unlink()
        return {"message": f"Đã xóa báo cáo: {filename}", "success": True}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Không thể xóa: {str(e)}")


@app.post("/api/prediction/start")
async def start_prediction(config: PredictionConfig, background_tasks: BackgroundTasks):
    """Bắt đầu dự đoán"""
    global prediction_status
    
    if prediction_status["is_predicting"]:
        raise HTTPException(status_code=400, detail="Đang có dự đoán khác đang chạy")
    
    # Reset status
    prediction_status = {
        "is_predicting": True,
        "progress": "Đang khởi động...",
        "error": None,
        "result": None,
        "output_file": None,
        "start_time": datetime.now().isoformat(),
        "end_time": None
    }
    
    # Run prediction in background
    background_tasks.add_task(run_prediction, config)
    
    return {"message": "Đã bắt đầu dự đoán", "status": prediction_status}


@app.get("/api/prediction/status")
async def get_prediction_status():
    """Kiểm tra trạng thái dự đoán"""
    return prediction_status


async def run_training(config: TrainingConfig):
    """Chạy training process"""
    global training_status
    
    try:
        training_status["cancel_requested"] = False
        training_status["progress"] = "Đang import thư viện..."
        
        # Import training module
        from train_module import train_model
        
        training_status["progress"] = "Đang load dữ liệu Sentinel-2..."
        
        # Function to check if training should be cancelled
        def should_cancel():
            return training_status.get("cancel_requested", False)
        
        # Run training
        result = train_model(
            bbox=[config.min_lon, config.min_lat, config.max_lon, config.max_lat],
            time_range=f"{config.start_date}/{config.end_date}",
            max_scenes=config.max_scenes,
            cloud_cover=config.cloud_cover,
            resolution=config.resolution,
            training_shapefile=config.training_shapefile,
            model_type=config.model_type,
            n_estimators=config.n_estimators,
            max_depth=config.max_depth,
            learning_rate=config.learning_rate,
            use_gpu=config.use_gpu,
            use_cache=config.use_cache,
            test_size=config.test_size,
            status_callback=lambda msg: update_progress(msg),
            cancel_check=should_cancel
        )
        
        if training_status.get("cancel_requested", False):
            training_status["is_training"] = False
            training_status["progress"] = "Đã hủy training"
            training_status["error"] = "Training cancelled by user"
        else:
            training_status["is_training"] = False
            training_status["progress"] = "Hoàn thành! Đang tạo báo cáo..."
            training_status["result"] = result
            
            # Auto generate report
            if result.get("success", False):
                try:
                    report_path, _ = generate_training_report(result)
                    training_status["result"]["report_path"] = report_path
                    training_status["result"]["report_filename"] = Path(report_path).name
                    training_status["progress"] = "Hoàn thành! Báo cáo đã được tạo."
                    print(f"[REPORT] Generated: {report_path}")
                except Exception as e:
                    print(f"[REPORT ERROR] Failed to generate report: {e}")
                    training_status["progress"] = "Hoàn thành! (Không thể tạo báo cáo)"
        
        training_status["end_time"] = datetime.now().isoformat()
        
    except Exception as e:
        training_status["is_training"] = False
        training_status["error"] = str(e)
        training_status["progress"] = f"Lỗi: {str(e)}"
        training_status["end_time"] = datetime.now().isoformat()
        import traceback
        print(traceback.format_exc())


def update_progress(message: str):
    """Cập nhật progress message"""
    global training_status
    training_status["progress"] = message
    print(f"[PROGRESS] {message}")


def rasterize_shapefile_overlay(shapefile_path, reference_raster, boundary_value=255):
    """
    Rasterize shapefile boundaries to overlay on prediction result.
    
    Args:
        shapefile_path: Path to shapefile
        reference_raster: xarray DataArray to match dimensions and CRS
        boundary_value: Value to use for boundaries (default 255 for white)
    
    Returns:
        numpy array with boundaries, same shape as reference_raster
    """
    try:
        import geopandas as gpd
        from rasterio.features import rasterize
        import numpy as np
        
        # Read shapefile
        gdf = gpd.read_file(shapefile_path)
        print(f"[OVERLAY] Loaded shapefile with {len(gdf)} features, CRS: {gdf.crs}")
        
        # Ensure CRS matches
        target_crs = reference_raster.rio.crs
        if gdf.crs != target_crs:
            print(f"[OVERLAY] Reprojecting from {gdf.crs} to {target_crs}")
            gdf = gdf.to_crs(target_crs)
        
        # Get raster dimensions and transform
        height, width = reference_raster.shape
        transform = reference_raster.rio.transform()
        print(f"[OVERLAY] Raster dimensions: {height}x{width}")
        print(f"[OVERLAY] Transform: {transform}")
        
        # Calculate appropriate buffer size based on pixel resolution
        # Get pixel size from transform (transform[0] is x resolution)
        pixel_size = abs(transform[0])  # in CRS units
        # Very thin boundary - only 0.2 pixels wide for 1px line
        buffer_distance = pixel_size * 0.2
        print(f"[OVERLAY] Pixel size: {pixel_size}, Buffer distance: {buffer_distance} (thin 1px line)")
        
        # Create boundary geometries with minimal buffering
        boundary_geoms = []
        for idx, geom in enumerate(gdf.geometry):
            if geom is not None and geom.is_valid:
                # Get boundary of each polygon
                boundary = geom.boundary
                if boundary is not None:
                    # Minimal buffer for 1-pixel thin line
                    buffered = boundary.buffer(buffer_distance)
                    boundary_geoms.append((buffered, boundary_value))
        
        if not boundary_geoms:
            print(f"[WARNING] No valid boundary geometries found in {shapefile_path}")
            return np.zeros((height, width), dtype=np.uint8)
        
        print(f"[OVERLAY] Rasterizing {len(boundary_geoms)} boundaries...")
        
        # Rasterize boundaries
        boundary_mask = rasterize(
            shapes=boundary_geoms,
            out_shape=(height, width),
            transform=transform,
            fill=0,  # Background
            dtype=np.uint8
        )
        
        boundary_count = np.count_nonzero(boundary_mask)
        print(f"[OVERLAY] Boundary pixels: {boundary_count} / {height*width} ({boundary_count/(height*width)*100:.2f}%)")
        
        if boundary_count == 0:
            print(f"[OVERLAY WARNING] No boundary pixels were rasterized! Check CRS and geometry overlap.")
        
        return boundary_mask
        
    except Exception as e:
        print(f"[ERROR] Failed to rasterize shapefile {shapefile_path}: {e}")
        return None


def update_prediction_progress(message: str):
    """Cập nhật prediction progress message"""
    global prediction_status
    prediction_status["progress"] = message
    print(f"[PREDICTION PROGRESS] {message}")


async def run_prediction(config: PredictionConfig):
    """Chạy prediction process - Sử dụng FeatureExtractor để đồng bộ với training"""
    global prediction_status
    
    try:
        prediction_status["progress"] = "Đang import thư viện..."
        
        # Import required libraries
        import numpy as np
        import xarray as xr
        import rioxarray
        from datetime import datetime as dt
        from feature_extractor import get_feature_extractor
        
        # Validate bbox
        if (config.min_lon < -180 or config.max_lon > 180 or 
            config.min_lat < -90 or config.max_lat > 90):
            raise ValueError(f"Bbox không hợp lệ: ({config.min_lon}, {config.min_lat}, {config.max_lon}, {config.max_lat}). "
                           f"Phải trong phạm vi (-180, -90, 180, 90)")
        
        prediction_status["progress"] = "Đang load model..."
        
        # Load model using ModelManager
        model_manager = get_model_manager()
        model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
        
        # Get feature_mode and features from metadata (default to 'simple' if not specified)
        feature_mode = model_metadata.get("feature_mode", "simple")
        required_features = model_metadata.get("features", [])
        n_features_expected = model_metadata.get("n_features", len(required_features))

        prediction_status["progress"] = f"Model: {model_metadata.get('model_type', 'unknown')}, mode={feature_mode}, features={n_features_expected}"

        # Initialize FeatureExtractor với đúng mode như lúc training
        extractor = get_feature_extractor(mode=feature_mode)
        
        # Check if it's a PyTorch model (CNN, Swin-UNet, MobileNet, etc.)
        is_pytorch_model = hasattr(model, '__class__') and any(
            name in model.__class__.__name__ for name in ['CNN', 'SwinUNet', 'MobileNet']
        )
        if is_pytorch_model:
            model_class_name = model.__class__.__name__
            prediction_status["progress"] = f"Phát hiện PyTorch {model_class_name} model..."
            try:
                import torch
            except ImportError:
                raise ImportError(f"PyTorch required for {model_class_name} models. Install: pip install torch")
        
        # Initialize common variables
        bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
        time_range = f"{config.start_date}/{config.end_date}"
        
        # ============ CHECK PREDICTION CACHE FIRST ============
        cached_s2_data = None
        cache_key = f"{config.min_lon:.2f}_{config.min_lat:.2f}_{config.max_lon:.2f}_{config.max_lat:.2f}"
        
        # Try to find matching cache
        cache_dir = Path("prediction_cache")
        if cache_dir.exists():
            for cache_file in cache_dir.glob(f"pred_{cache_key}_*.json"):
                try:
                    with open(cache_file, 'r') as f:
                        cache_data = json.load(f)
                    
                    # Check if cache matches current config
                    if (cache_data.get('start_date') == config.start_date and
                        cache_data.get('end_date') == config.end_date and
                        cache_data.get('resolution') == config.resolution and
                        cache_data.get('data_file')):
                        
                        data_file = cache_dir / cache_data['data_file']
                        if data_file.exists():
                            prediction_status["progress"] = "Đang load dữ liệu từ cache..."
                            import joblib
                            cached_s2_data = joblib.load(data_file)
                            print(f"[CACHE HIT] Using cached Sentinel-2 data from {cache_file.name}")
                            break
                except Exception as e:
                    print(f"[CACHE] Error loading cache {cache_file}: {e}")
        
        # ============ LOAD SENTINEL-2 DATA ============
        if cached_s2_data is not None:
            s2_data = cached_s2_data
            s2_items = []  # Empty list when using cache
            prediction_status["progress"] = "Đã load dữ liệu từ cache, đang xử lý..."
        else:
            prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
            import pystac_client
            import planetary_computer
            from odc.stac import load
            
            catalog = pystac_client.Client.open(
                "https://planetarycomputer.microsoft.com/api/stac/v1",
                modifier=planetary_computer.sign_inplace,
            )
            
            prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
            s2_items = fetch_sentinel_items_with_retry(
                catalog=catalog,
                bbox=bbox,
                time_range=time_range,
                cloud_cover=config.cloud_cover,
                max_scenes=config.max_scenes,
                max_retries=3
            )
        
        prediction_status["progress"] = f"Đang xử lý dữ liệu Sentinel-2..."
        
        # Load different bands based on feature mode (skip if using cache)
        if cached_s2_data is None:
            if feature_mode == 'simple':
                bands_to_load = ["B04", "B08", "SCL"]
            else:  # temporal or extended
                bands_to_load = ["B02", "B03", "B04", "B08", "B11", "SCL"]
            
            # Retry logic for data loading
            max_retries = 3
            s2_data = None
            for attempt in range(max_retries):
                try:
                    s2_data = load(
                        s2_items,
                        bbox=bbox,
                        bands=bands_to_load,
                        chunks={"time": 1, "x": 2048, "y": 2048},
                        groupby="solar_day",
                        resolution=config.resolution
                    ).compute()
                    break
                except Exception as e:
                    error_msg = str(e)
                    
                    is_auth_error = any(err in error_msg for err in [
                        "AuthenticationFailed", "Signature fields not well formed",
                        "Server failed to authenticate", "403", "401"
                    ])
                    
                    if is_auth_error:
                        print(f"⚠️  Authentication error. Re-signing items...")
                        try:
                            import planetary_computer
                            s2_items = [planetary_computer.sign(item) for item in s2_items]
                            if attempt < max_retries - 1:
                                import time
                                time.sleep(1)
                                continue
                        except Exception:
                            pass
                    
                    if "does not exist" in error_msg or "RasterioIOError" in error_msg:
                        if len(s2_items) > 3 and attempt < max_retries - 1:
                            remove_count = max(1, len(s2_items) // 5)
                            s2_items = s2_items[:-remove_count]
                            print(f"⚠️  Access error. Reduced to {len(s2_items)} scenes, retrying...")
                            continue
                    if attempt == max_retries - 1:
                        raise ValueError(f"Failed to load Sentinel-2 data: {error_msg[:200]}")
                    
            if s2_data is None:
                raise ValueError("Failed to load Sentinel-2 data after retries")
        
        prediction_status["progress"] = "Đã load Sentinel-2 data"
        
        # ============ LOAD SENTINEL-1 DATA (RADAR) ============
        prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
        use_radar = False
        vh_data = None
        vv_data = None
        
        try:
            s1_search = catalog.search(
                collections=["sentinel-1-rtc"],
                bbox=bbox,
                datetime=time_range,
            )
            s1_items = list(s1_search.items())
            
            if s1_items:
                s1_items = s1_items[:config.max_scenes]
                # Try to load Sentinel-1 with error handling
                try:
                    s1_data = load(
                        s1_items,
                        bbox=bbox,
                        bands=["vh", "vv"],
                        chunks={"time": 1, "x": 2048, "y": 2048},
                        groupby="solar_day",
                        resolution=config.resolution
                    ).compute()
                    
                    # Convert to dB
                    vh_data = 10 * np.log10(s1_data['vh'].where(s1_data['vh'] > 0))
                    vv_data = 10 * np.log10(s1_data['vv'].where(s1_data['vv'] > 0))
                    use_radar = True
                    prediction_status["progress"] = f"Đã load Sentinel-1 data ({len(s1_items)} scenes)"
                except Exception as s1_error:
                    print(f"⚠️  Failed to load Sentinel-1 data: {str(s1_error)[:100]}")
                    prediction_status["progress"] = "Lỗi Sentinel-1, bỏ qua radar features"
            else:
                prediction_status["progress"] = "Không có dữ liệu Sentinel-1, bỏ qua radar features"
        except Exception as e:
            prediction_status["progress"] = f"Lỗi load Sentinel-1: {str(e)}, bỏ qua radar features"
        
        # ============ ADVANCED CLOUD MASKING & REMOVAL ============
        prediction_status["progress"] = "Đang xử lý mây nâng cao..."
        
        # Use cloud removal module with user-selected method
        cloud_removal_method = config.cloud_removal_method if hasattr(config, 'cloud_removal_method') else "classic"
        cloud_removal_model = config.cloud_removal_model if hasattr(config, 'cloud_removal_model') else None
        
        print(f"[CLOUD REMOVAL] Using method: {cloud_removal_method}")
        if cloud_removal_model:
            print(f"[CLOUD REMOVAL] Using custom model: {cloud_removal_model}")
        
        s2_data, cloud_metadata = process_cloud_removal(
            s2_data=s2_data,
            method=cloud_removal_method,
            model_path=f"cloud_removal_model/{cloud_removal_model}" if cloud_removal_model else None,
            verbose=True
        )
        
        cloud_coverage_percent = cloud_metadata.get('cloud_coverage_percent', 0)
        
        # Quality warning if cloud coverage too high
        if cloud_coverage_percent > 30:
            print(f"[WARNING] High cloud coverage ({cloud_coverage_percent:.1f}%) - prediction quality may be affected")
            prediction_status["progress"] = f"⚠️ Cảnh báo: Độ phủ mây cao ({cloud_coverage_percent:.1f}%)"
        
        # ============ EXTRACT FEATURES ============
        prediction_status["progress"] = f"Đang trích xuất features (mode={feature_mode})..."
        
        print(f"[PREDICTION DEBUG] Feature mode: {feature_mode}")
        print(f"[PREDICTION DEBUG] S2 bands available: {list(s2_data.data_vars)}")
        print(f"[PREDICTION DEBUG] S2 dimensions: {dict(s2_data.dims)}")

        # Always fill NaN for all bands in s2_data if present
        for band in ["B02", "B03", "B04", "B08", "B11"]:
            if band in s2_data:
                s2_data[band] = s2_data[band].ffill(dim='time').bfill(dim='time')

        # Calculate NDVI if needed (for simple mode)
        ndvi_filled = None
        if feature_mode == 'simple' and 'B08' in s2_data and 'B04' in s2_data:
            nir = s2_data["B08"].astype('float32')
            red = s2_data["B04"].astype('float32')
            ndvi = (nir - red) / (nir + red + 1e-8)
            ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')

        # Extract features using FeatureExtractor, always pass all possible data
        features = extractor.extract(
            s2_data=s2_data,
            ndvi_data=ndvi_filled,
            vh_data=vh_data,
            vv_data=vv_data
        )

        # Handle NaN values
        features = np.nan_to_num(features, nan=0.0)
        
        print(f"[PREDICTION DEBUG] Features extracted: shape={features.shape}")
        print(f"[PREDICTION DEBUG] Features range: [{features.min():.3f}, {features.max():.3f}]")
        print(f"[PREDICTION DEBUG] Features mean: {features.mean():.3f}, std: {features.std():.3f}")
        print(f"[PREDICTION DEBUG] NaN count: {np.isnan(features).sum()}")
        print(f"[PREDICTION DEBUG] First pixel features: {features[0][:min(8, features.shape[1])]}")

        # Ensure features shape matches model expectation
        if features.shape[1] != n_features_expected:
            raise ValueError(f"Số lượng features ({features.shape[1]}) không khớp với model ({n_features_expected}). Hãy kiểm tra lại cấu hình trích xuất đặc trưng và metadata của model.")

        prediction_status["progress"] = f"Đã extract {features.shape[1]} features cho {features.shape[0]} pixels"
        
        # ============ PREDICT ============
        prediction_status["progress"] = "Đang dự đoán..."
        
        print(f"[PREDICTION DEBUG] Starting prediction with {features.shape[0]} pixels, {features.shape[1]} features")
        
        # Make prediction (all PyTorch models have the same predict interface)
        predictions = model.predict(features)
        
        print(f"[PREDICTION DEBUG] Predictions shape: {predictions.shape}")
        print(f"[PREDICTION DEBUG] Unique predicted classes: {np.unique(predictions)}")
        print(f"[PREDICTION DEBUG] Class distribution:")
        unique, counts = np.unique(predictions, return_counts=True)
        for cls, cnt in zip(unique, counts):
            print(f"  Class {cls}: {cnt} pixels ({cnt/len(predictions)*100:.1f}%)")
        
        # Decode labels if label_encoder exists
        if label_encoder is not None:
            try:
                predictions = label_encoder.inverse_transform(predictions.astype(int))
            except:
                pass
        
        # Reshape to original shape
        if feature_mode == 'simple' and 'B08' in s2_data:
            # Use B08 to get shape
            y_size = len(s2_data.y)
            x_size = len(s2_data.x)
        else:
            y_size = len(s2_data.y)
            x_size = len(s2_data.x)
        
        pred_shape = (y_size, x_size)
        predictions_2d = predictions.reshape(pred_shape)

        # Smooth classification map to reduce salt-and-pepper noise
        try:
            from scipy import ndimage
            smoothed = ndimage.median_filter(predictions_2d, size=3)
            # Keep invalid/nodata pixels (-1) untouched
            smoothed[predictions_2d < 0] = -1
            predictions_2d = smoothed
            print("[SMOOTH] Applied 3x3 median filter to classification map")
        except Exception as smooth_err:
            print(f"[SMOOTH WARNING] Failed to smooth classification map: {smooth_err}")
        
        # ============ CREATE OUTPUT ============
        prediction_status["progress"] = "Đang tạo bản đồ phân loại..."
        
        # Create output xarray
        prediction_da = xr.DataArray(
            predictions_2d,
            coords={
                "y": s2_data.y,
                "x": s2_data.x
            },
            dims=["y", "x"],
            name="classification"
        )
        
        # Save output
        output_dir = Path("predictions")
        output_dir.mkdir(exist_ok=True)
        
        timestamp = dt.now().strftime("%Y%m%d_%H%M%S")
        output_file = output_dir / f"prediction_{timestamp}.tif"
        
        prediction_status["progress"] = "Đang lưu kết quả GeoTIFF..."
        
        # Set CRS and save as GeoTIFF
        if hasattr(s2_data, 'rio') and s2_data.rio.crs is not None:
            prediction_da.rio.write_crs(s2_data.rio.crs, inplace=True)
        else:
            prediction_da.rio.write_crs("EPSG:4326", inplace=True)
        
        prediction_da.rio.to_raster(str(output_file), driver="GTiff")
        
        # ============ SHAPEFILE OVERLAY ============
        overlay_mask = None
        if config.shapefile_overlay:
            prediction_status["progress"] = f"Đang overlay shapefile: {config.shapefile_overlay}..."
            print(f"[OVERLAY] Shapefile overlay requested: {config.shapefile_overlay}")
            
            # Validate shapefile path exists
            shapefile_path = Path(config.shapefile_overlay)
            if not shapefile_path.exists():
                print(f"[OVERLAY ERROR] Shapefile not found: {shapefile_path}")
                print(f"[OVERLAY ERROR] Absolute path: {shapefile_path.absolute()}")
                print(f"[OVERLAY ERROR] Current working directory: {Path.cwd()}")
            else:
                print(f"[OVERLAY] Shapefile exists: {shapefile_path.absolute()}")
                try:
                    overlay_mask = rasterize_shapefile_overlay(str(shapefile_path), prediction_da)
                    if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
                        print(f"[OVERLAY] Successfully rasterized shapefile boundaries ({np.count_nonzero(overlay_mask)} pixels)")
                    else:
                        print(f"[OVERLAY WARNING] Shapefile rasterized but no boundary pixels found")
                except Exception as overlay_error:
                    print(f"[OVERLAY ERROR] Exception: {overlay_error}")
                    import traceback
                    traceback.print_exc()
        else:
            print(f"[OVERLAY] No shapefile overlay requested")
        
        # Generate PNG preview for web display
        prediction_status["progress"] = "Đang tạo PNG preview..."
        png_file = output_dir / f"prediction_{timestamp}.png"
        try:
            import matplotlib
            matplotlib.use('Agg')  # Non-interactive backend
            import matplotlib.pyplot as plt
            from matplotlib.patches import Patch
            
            # Create a figure with prediction result and legend
            fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
            
            # Plot prediction with colormap
            im = ax.imshow(predictions_2d, cmap='tab20', interpolation='nearest')
            
            # Overlay shapefile boundaries if available
            if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
                print(f"[PNG OVERLAY] Overlaying {np.count_nonzero(overlay_mask)} boundary pixels")
                
                # Create a mask for boundaries (where overlay_mask > 0)
                boundary_mask = overlay_mask > 0
                
                # Method: Direct overlay with high-contrast colors
                # Create RGBA overlay image
                overlay_rgba = np.zeros((*predictions_2d.shape, 4))
                overlay_rgba[boundary_mask, 0] = 1.0  # Red = 1.0 (white)
                overlay_rgba[boundary_mask, 1] = 1.0  # Green = 1.0 (white)
                overlay_rgba[boundary_mask, 2] = 1.0  # Blue = 1.0 (white)
                overlay_rgba[boundary_mask, 3] = 1.0  # Alpha = 1.0 (fully opaque)
                
                # Overlay on top of prediction
                ax.imshow(overlay_rgba, interpolation='nearest')
                
                # Also add a black outline for better contrast
                from scipy import ndimage
                boundary_dilated = ndimage.binary_dilation(boundary_mask, iterations=1)
                boundary_outline = boundary_dilated & ~boundary_mask
                
                outline_rgba = np.zeros((*predictions_2d.shape, 4))
                outline_rgba[boundary_outline, 0] = 0.0  # Black outline
                outline_rgba[boundary_outline, 1] = 0.0
                outline_rgba[boundary_outline, 2] = 0.0
                outline_rgba[boundary_outline, 3] = 0.8
                
                ax.imshow(outline_rgba, interpolation='nearest')
                
                print(f"[PNG OVERLAY] Added shapefile boundaries to visualization (direct overlay method)")
            else:
                print(f"[PNG OVERLAY] No overlay mask or empty mask (pixels: {np.count_nonzero(overlay_mask) if overlay_mask is not None else 0})")
            
            ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
            ax.set_xlabel('X (pixels)', fontsize=11)
            ax.set_ylabel('Y (pixels)', fontsize=11)

            # Build mapping from numeric class value -> display label
            # ALWAYS use DEFAULT_LABEL_NAMES - it has the correct Vietnamese names
            class_map = DEFAULT_LABEL_NAMES.copy()
            
            print(f"[LEGEND DEBUG] DEFAULT_LABEL_NAMES: {DEFAULT_LABEL_NAMES}")
            print(f"[LEGEND DEBUG] Model metadata: {model_metadata.get('label_mapping') if isinstance(model_metadata, dict) else 'No metadata'}")
            print(f"[LEGEND DEBUG] Final class_map: {class_map}")

            # Get unique classes in prediction to show only relevant legend items
            unique_pred_classes = np.unique(predictions_2d)
            unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
            
            # Create custom legend with color patches
            legend_elements = []
            cmap = plt.cm.get_cmap('tab20')
            
            for cls_val in sorted(unique_pred_classes):
                try:
                    cls_int = int(cls_val)
                    # Get color from colormap (normalize to 0-1 range)
                    color = cmap(cls_int / 20.0)  # tab20 has 20 colors
                    # Get label name
                    label_name = class_map.get(cls_int, f"Class {cls_int}")
                    # Create patch for legend
                    legend_elements.append(
                        Patch(facecolor=color, edgecolor='black', linewidth=0.5, 
                              label=f"{cls_int}: {label_name}")
                    )
                except:
                    pass
            
            # Add legend outside plot area
            if legend_elements:
                legend = ax.legend(
                    handles=legend_elements,
                    loc='center left',
                    bbox_to_anchor=(1.02, 0.5),
                    fontsize=10,
                    title='Land Classes',
                    title_fontsize=11,
                    framealpha=0.9,
                    edgecolor='black'
                )
                legend.get_title().set_fontweight('bold')
            
            # Add grid
            ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
            
            # Save PNG
            plt.tight_layout()
            plt.savefig(str(png_file), dpi=150, bbox_inches='tight')
            plt.close(fig)
            
            print(f"[PNG PREVIEW] Created: {png_file}")
        except Exception as e:
            print(f"[PNG PREVIEW ERROR] Failed to create PNG: {e}")
            png_file = None
        
        # Get unique classes for result
        unique_classes = np.unique(predictions_2d)
        unique_classes = unique_classes[~np.isnan(unique_classes)].tolist()
        
        prediction_status["is_predicting"] = False
        prediction_status["progress"] = "Hoàn thành! Đang tạo báo cáo..."
        prediction_status["output_file"] = str(output_file)
        prediction_status["result"] = {
            "output_file": str(output_file),
            "png_file": str(png_file) if png_file else None,
            "shape": list(pred_shape),
            "unique_classes": unique_classes,
            "bbox": bbox,
            "time_range": time_range,
            "n_features": features.shape[1],
            "feature_mode": feature_mode,
            "used_radar": use_radar,
            "model_used": config.model_filename,
            "shapefile_overlay": config.shapefile_overlay,
            "overlay_applied": overlay_mask is not None
        }
        
        # Auto generate prediction report
        try:
            report_path, _ = generate_prediction_report(prediction_status["result"])
            prediction_status["result"]["report_path"] = report_path
            prediction_status["result"]["report_filename"] = Path(report_path).name
            prediction_status["progress"] = "Hoàn thành! Báo cáo đã được tạo."
            print(f"[PREDICTION REPORT] Generated: {report_path}")
        except Exception as e:
            print(f"[PREDICTION REPORT ERROR] Failed to generate report: {e}")
            prediction_status["progress"] = "Hoàn thành! (Không thể tạo báo cáo)"
        
        # Auto-save prediction cache (including Sentinel-2 data)
        print(f"[DEBUG] Starting cache save process...")
        print(f"[DEBUG] cached_s2_data is None: {cached_s2_data is None}")
        print(f"[DEBUG] s2_data type: {type(s2_data)}")
        print(f"[DEBUG] s2_data is None: {s2_data is None}")
        
        try:
            cache_config = {
                "min_lon": config.min_lon,
                "min_lat": config.min_lat,
                "max_lon": config.max_lon,
                "max_lat": config.max_lat,
                "start_date": config.start_date,
                "end_date": config.end_date,
                "max_scenes": config.max_scenes,
                "cloud_cover": config.cloud_cover,
                "resolution": config.resolution,
                "model_filename": config.model_filename
            }
            
            print(f"[DEBUG] cache_config created: {cache_config}")
            
            # Save cache with Sentinel-2 data (only if not from cache)
            data_to_save = None if cached_s2_data is not None else s2_data
            print(f"[DEBUG] data_to_save is None: {data_to_save is None}")
            print(f"[DEBUG] Calling save_prediction_cache_sync...")
            
            result = save_prediction_cache_sync(cache_config, data_to_save)
            
            print(f"[CACHE] Prediction cache saved: {result.get('message')} (with data: {data_to_save is not None})")
            print(f"[CACHE] Result: {result}")
        except Exception as cache_error:
            print(f"[CACHE ERROR] Failed to auto-save cache: {cache_error}")
            import traceback
            traceback.print_exc()
        
        prediction_status["end_time"] = dt.now().isoformat()
        
    except Exception as e:
        prediction_status["is_predicting"] = False
        prediction_status["error"] = str(e)
        prediction_status["progress"] = f"Lỗi: {str(e)}"
        prediction_status["end_time"] = dt.now().isoformat()
        import traceback
        print(traceback.format_exc())


@app.get("/api/predictions/list")
async def list_predictions():
    """Lấy danh sách các file prediction đã tạo"""
    predictions_dir = Path("predictions")
    predictions_dir.mkdir(exist_ok=True)
    
    predictions = []
    for pred_file in predictions_dir.glob("*.tif"):
        pred_info = {
            "filename": pred_file.name,
            "created": datetime.fromtimestamp(pred_file.stat().st_mtime).isoformat(),
            "size_mb": round(pred_file.stat().st_size / 1024 / 1024, 2),
            "download_url": f"/api/predictions/download/{pred_file.name}",
            "is_batch_job": pred_file.name.startswith("batch_"),
            "batch_metadata": None
        }
        
        # Try to load batch metadata from JSON sidecar if exists
        json_file = pred_file.with_suffix('.json')
        if json_file.exists():
            try:
                import json
                with open(json_file, 'r') as f:
                    metadata = json.load(f)
                    pred_info["batch_metadata"] = {
                        "batch_job_id": metadata.get("batch_job_id"),
                        "batch_name": metadata.get("batch_name"),
                        "batch_timestamp": metadata.get("batch_timestamp")
                    }
            except Exception as e:
                print(f"[METADATA ERROR] Failed to load {json_file}: {e}")
        
        # Check PNG preview
        png_file = pred_file.with_suffix('.png')
        pred_info["has_preview"] = png_file.exists()
        if png_file.exists():
            pred_info["preview_url"] = f"/api/predictions/preview/{png_file.name}"
        
        predictions.append(pred_info)
    
    # Sort by creation time (newest first)
    predictions.sort(key=lambda x: x["created"], reverse=True)
    return {"predictions": predictions}


@app.get("/api/predictions/download/{filename}")
async def download_prediction(filename: str):
    """Download file prediction GeoTIFF"""
    predictions_dir = Path("predictions")
    file_path = predictions_dir / filename
    
    # Security check: ensure filename doesn't contain path traversal
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"File không tồn tại: {filename}")
    
    return FileResponse(
        path=str(file_path),
        filename=filename,
        media_type="image/tiff",
        headers={
            "Content-Disposition": f"attachment; filename={filename}"
        }
    )


@app.get("/api/predictions/preview/{filename}")
async def preview_prediction_png(filename: str):
    """Preview PNG image of prediction"""
    predictions_dir = Path("predictions")
    file_path = predictions_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"PNG preview không tồn tại: {filename}")
    
    return FileResponse(
        path=str(file_path),
        media_type="image/png"
    )


@app.get("/api/predictions/preview/{filename}")
async def preview_prediction_png(filename: str):
    """Preview PNG image of prediction"""
    predictions_dir = Path("predictions")
    file_path = predictions_dir / filename
    
    # Security check
    if ".." in filename or "/" in filename or "\\" in filename:
        raise HTTPException(status_code=400, detail="Invalid filename")
    
    if not file_path.exists():
        raise HTTPException(status_code=404, detail=f"PNG preview không tồn tại: {filename}")
    
    return FileResponse(
        path=str(file_path),
        media_type="image/png"
    )


# ============ PREDICTION CACHE API (Auto-save) ============

def save_prediction_cache_sync(config: dict, s2_data=None):
    """Tự động lưu prediction cache sau khi predict thành công (bao gồm cả dữ liệu Sentinel-2)"""
    print(f"[DEBUG save_prediction_cache_sync] Called with s2_data is None: {s2_data is None}")
    print(f"[DEBUG save_prediction_cache_sync] Config: {config}")
    
    try:
        cache_dir = Path("prediction_cache")
        print(f"[DEBUG save_prediction_cache_sync] Cache dir: {cache_dir.absolute()}")
        cache_dir.mkdir(exist_ok=True)
        print(f"[DEBUG save_prediction_cache_sync] Cache dir created/exists")
        
        # Create cache filename from bbox and timestamp
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        bbox_str = f"{config['min_lon']:.2f}_{config['min_lat']:.2f}_{config['max_lon']:.2f}_{config['max_lat']:.2f}"
        base_filename = f"pred_{bbox_str}_{timestamp}"
        config_file = cache_dir / f"{base_filename}.json"
        data_file = cache_dir / f"{base_filename}_data.joblib"
        
        print(f"[DEBUG save_prediction_cache_sync] Config file: {config_file}")
        print(f"[DEBUG save_prediction_cache_sync] Data file: {data_file}")
        
        # Prepare cache data with full metadata
        cache_data = {
            "bbox": [config['min_lon'], config['min_lat'], config['max_lon'], config['max_lat']],
            "min_lon": config['min_lon'],
            "min_lat": config['min_lat'],
            "max_lon": config['max_lon'],
            "max_lat": config['max_lat'],
            "start_date": config['start_date'],
            "end_date": config['end_date'],
            "max_scenes": config['max_scenes'],
            "cloud_cover": config['cloud_cover'],
            "resolution": config['resolution'],
            "model_filename": config.get('model_filename'),
            "created": timestamp,
            "type": "prediction_cache",
            "auto_saved": True,
            "has_data": s2_data is not None,
            "data_file": f"{base_filename}_data.joblib" if s2_data is not None else None
        }
        
        print(f"[DEBUG save_prediction_cache_sync] Saving config JSON...")
        # Save config to JSON file
        with open(config_file, 'w', encoding='utf-8') as f:
            json.dump(cache_data, f, indent=2, ensure_ascii=False)
        print(f"[DEBUG save_prediction_cache_sync] Config JSON saved to {config_file}")
        
        # Save Sentinel-2 data if provided
        if s2_data is not None:
            print(f"[DEBUG save_prediction_cache_sync] Saving Sentinel-2 data with joblib...")
            import joblib
            joblib.dump(s2_data, data_file)
            data_size_mb = data_file.stat().st_size / 1024 / 1024
            cache_data['data_size_mb'] = round(data_size_mb, 2)
            print(f"[CACHE] Saved Sentinel-2 data: {data_size_mb:.2f} MB to {data_file}")
        else:
            print(f"[DEBUG save_prediction_cache_sync] No s2_data to save")
        
        print(f"[CACHE SUCCESS] Cache saved successfully: {base_filename}.json")
        
        return {
            "success": True,
            "message": f"Đã lưu cache tự động" + (" (bao gồm dữ liệu Sentinel-2)" if s2_data is not None else ""),
            "filename": f"{base_filename}.json",
            "cache": cache_data
        }
        
    except Exception as e:
        print(f"[CACHE ERROR save_prediction_cache_sync] Error: {e}")
        import traceback
        traceback.print_exc()
        return {"success": False, "error": str(e)}

@app.get("/api/prediction/cache/list")
async def list_prediction_cache():
    """Liệt kê các prediction cache đã lưu (bao gồm thông tin về dữ liệu Sentinel-2)"""
    try:
        cache_dir = Path("prediction_cache")
        if not cache_dir.exists():
            return {"caches": [], "count": 0}
        
        caches = []
        for cache_file in cache_dir.glob("pred_*.json"):
            try:
                with open(cache_file, 'r', encoding='utf-8') as f:
                    cache_data = json.load(f)
                
                # Check if data file exists
                data_filename = cache_data.get("data_file")
                has_data = False
                data_size_mb = 0
                
                if data_filename:
                    data_file_path = cache_dir / data_filename
                    if data_file_path.exists():
                        has_data = True
                        data_size_mb = round(data_file_path.stat().st_size / 1024 / 1024, 2)
                
                # Create display name from metadata
                bbox = cache_data.get("bbox", [])
                time_range = f"{cache_data.get('start_date', 'N/A')} → {cache_data.get('end_date', 'N/A')}"
                data_badge = f" [💾 {data_size_mb}MB]" if has_data else " [⚙️ Config only]"
                display_name = f"[{bbox[0]:.2f},{bbox[1]:.2f}→{bbox[2]:.2f},{bbox[3]:.2f}] {time_range}{data_badge}"
                
                caches.append({
                    "filename": cache_file.name,
                    "display_name": display_name,
                    "bbox": bbox,
                    "created": cache_data.get("created"),
                    "has_data": has_data,
                    "data_size_mb": data_size_mb,
                    "data_file": data_filename,
                    "config": cache_data
                })
            except Exception as e:
                print(f"Error loading cache {cache_file}: {e}")
                continue
        
        # Sort by created time (newest first)
        caches.sort(key=lambda x: x.get("created", ""), reverse=True)
        
        return {
            "caches": caches,
            "count": len(caches)
        }
        
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi load cache: {str(e)}")

@app.get("/api/prediction/cache/load-data/{filename}")
async def load_cached_data(filename: str):
    """Load Sentinel-2 data từ cache"""
    try:
        cache_dir = Path("prediction_cache")
        
        # Security check
        if ".." in filename or "/" in filename or "\\" in filename:
            raise HTTPException(status_code=400, detail="Invalid filename")
        
        # Load config to get data filename
        config_file = cache_dir / filename
        if not config_file.exists():
            raise HTTPException(status_code=404, detail="Cache config not found")
        
        with open(config_file, 'r') as f:
            cache_data = json.load(f)
        
        data_filename = cache_data.get("data_file")
        if not data_filename:
            raise HTTPException(status_code=404, detail="No data file in cache")
        
        data_file = cache_dir / data_filename
        if not data_file.exists():
            raise HTTPException(status_code=404, detail="Data file not found")
        
        return {
            "success": True,
            "has_data": True,
            "data_file": data_filename,
            "data_size_mb": round(data_file.stat().st_size / 1024 / 1024, 2)
        }
        
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi: {str(e)}")

@app.delete("/api/prediction/cache/delete/{filename}")
async def delete_prediction_cache(filename: str):
    """Xóa một prediction cache (bao gồm cả file data nếu có)"""
    try:
        cache_dir = Path("prediction_cache")
        cache_file = cache_dir / filename
        
        # Security check
        if ".." in filename or "/" in filename or "\\" in filename:
            raise HTTPException(status_code=400, detail="Invalid filename")
        
        if not cache_file.exists():
            raise HTTPException(status_code=404, detail="Cache not found")
        
        # Load config to check for data file
        try:
            with open(cache_file, 'r') as f:
                cache_data = json.load(f)
            data_filename = cache_data.get("data_file")
            if data_filename:
                data_file = cache_dir / data_filename
                if data_file.exists():
                    data_file.unlink()
                    print(f"[CACHE] Deleted data file: {data_filename}")
        except Exception as e:
            print(f"[CACHE] Error deleting data file: {e}")
        
        # Delete config file
        cache_file.unlink()
        
        return {
            "success": True,
            "message": f"Đã xóa cache: {filename}"
        }
        
    except HTTPException:
        raise
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Lỗi khi xóa cache: {str(e)}")


# ============ DASHBOARD & VISUALIZATION API ============

@app.get("/api/dashboard/accuracy-trends")
async def get_accuracy_trends():
    """Lấy dữ liệu accuracy trends của các models theo thời gian"""
    model_dir = Path("model_train")
    if not model_dir.exists():
        return {"trends": [], "models": []}
    
    trends_data = []
    for info_file in sorted(model_dir.glob("*.json")):
        try:
            with open(info_file) as f:
                info = json.load(f)
            
            # Extract relevant data
            if "training_date" in info and "metrics" in info:
                trends_data.append({
                    "date": info["training_date"],
                    "model_name": info.get("model_type", "unknown"),
                    "accuracy": info["metrics"].get("accuracy", 0),
                    "f1_score": info["metrics"].get("macro avg", {}).get("f1-score", 0),
                    "precision": info["metrics"].get("macro avg", {}).get("precision", 0),
                    "recall": info["metrics"].get("macro avg", {}).get("recall", 0),
                    "filename": info_file.stem + ".joblib"
                })
        except Exception as e:
            print(f"Error loading {info_file}: {e}")
            continue
    
    # Sort by date
    trends_data.sort(key=lambda x: x["date"])
    
    return {
        "trends": trends_data,
        "models": list(set(d["model_name"] for d in trends_data))
    }


@app.get("/api/dashboard/statistics")
async def get_statistics():
    """Lấy thống kê tổng quan: số models, predictions, reports"""
    model_dir = Path("model_train")
    predictions_dir = Path("predictions")
    reports_dir = Path("reports")
    
    # Count items
    n_models = len(list(model_dir.glob("*.joblib"))) if model_dir.exists() else 0
    n_predictions = len(list(predictions_dir.glob("*.tif"))) if predictions_dir.exists() else 0
    n_reports = len(list(reports_dir.glob("*.html"))) if reports_dir.exists() else 0
    
    # Get latest model info
    latest_model = None
    if model_dir.exists():
        model_files = sorted(model_dir.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True)
        if model_files:
            try:
                with open(model_files[0]) as f:
                    latest_model = json.load(f)
            except:
                pass
    
    # Get latest prediction
    latest_prediction = None
    if predictions_dir.exists():
        pred_files = sorted(predictions_dir.glob("*.tif"), key=lambda x: x.stat().st_mtime, reverse=True)
        if pred_files:
            latest_prediction = {
                "filename": pred_files[0].name,
                "created": datetime.fromtimestamp(pred_files[0].stat().st_mtime).isoformat(),
                "size_mb": round(pred_files[0].stat().st_size / 1024 / 1024, 2)
            }
    
    return {
        "models": {
            "total": n_models,
            "latest": latest_model
        },
        "predictions": {
            "total": n_predictions,
            "latest": latest_prediction
        },
        "reports": {
            "total": n_reports
        },
        "training_status": training_status,
        "prediction_status": prediction_status
    }


@app.get("/api/dashboard/class-distribution/{model_filename}")
async def get_class_distribution(model_filename: str):
    """Lấy phân bố các lớp từ model info"""
    # Convert model filename to info filename
    # e.g., model_cnn_20251221_163841.joblib -> model_cnn_20251221_163841_info.json
    base_name = model_filename.replace(".joblib", "")
    info_file = Path("model_train") / f"{base_name}_info.json"
    
    if not info_file.exists():
        raise HTTPException(status_code=404, detail="Model info không tồn tại")
    
    with open(info_file) as f:
        info = json.load(f)
    
    # Extract class distribution from classification report
    class_dist = {}
    if "classification_report" in info:
        for class_name, metrics in info["classification_report"].items():
            if isinstance(metrics, dict) and "support" in metrics:
                class_dist[class_name] = int(metrics["support"])
    
    return {
        "model": model_filename,
        "class_distribution": class_dist,
        "total_samples": sum(class_dist.values()) if class_dist else 0
    }


# ============ BATCH PROCESSING API ============

class BatchPredictionItem(BaseModel):
    """Một item trong batch prediction"""
    name: str
    min_lon: float
    min_lat: float
    max_lon: float
    max_lat: float
    start_date: str = "2023-03-01"
    end_date: str = "2023-05-31"
    max_scenes: int = 12
    cloud_cover: int = 30
    resolution: int = 20


class BatchPredictionConfig(BaseModel):
    """Cấu hình cho batch prediction"""
    model_filename: str
    items: List[BatchPredictionItem]
    auto_retry: bool = True
    max_retries: int = 3


@app.post("/api/batch/start")
async def start_batch_prediction(config: BatchPredictionConfig, background_tasks: BackgroundTasks):
    """Bắt đầu batch prediction"""
    global batch_queue, batch_results
    
    # Create batch jobs
    batch_id = datetime.now().strftime("%Y%m%d_%H%M%S")
    
    for idx, item in enumerate(config.items):
        job = {
            "batch_id": batch_id,
            "job_id": f"{batch_id}_{idx}",
            "name": item.name,
            "status": "queued",
            "progress": 0,
            "error": None,
            "result": None,
            "retries": 0,
            "max_retries": config.max_retries if config.auto_retry else 0,
            "config": {
                "model_filename": config.model_filename,
                "min_lon": item.min_lon,
                "min_lat": item.min_lat,
                "max_lon": item.max_lon,
                "max_lat": item.max_lat,
                "start_date": item.start_date,
                "end_date": item.end_date,
                "max_scenes": item.max_scenes,
                "cloud_cover": item.cloud_cover,
                "resolution": item.resolution
            },
            "created_at": datetime.now().isoformat()
        }
        batch_queue.append(job)
    
    # Start processing in background
    background_tasks.add_task(process_batch_queue)
    
    return {
        "message": f"Đã tạo {len(config.items)} batch jobs",
        "batch_id": batch_id,
        "total_jobs": len(config.items)
    }


@app.get("/api/batch/status")
async def get_batch_status():
    """Lấy trạng thái của batch queue"""
    global batch_queue, batch_results
    
    queued = [j for j in batch_queue if j["status"] == "queued"]
    running = [j for j in batch_queue if j["status"] == "running"]
    completed = [j for j in batch_results if j["status"] == "completed"]
    failed = [j for j in batch_results if j["status"] == "failed"]
    
    return {
        "queue": {
            "queued": len(queued),
            "running": len(running),
            "completed": len(completed),
            "failed": len(failed),
            "total": len(batch_queue) + len(batch_results)
        },
        "jobs": {
            "queued": queued[:5],  # Show first 5
            "running": running,
            "recent_completed": completed[:10],  # Show last 10
            "recent_failed": failed[:10]
        }
    }


@app.get("/api/batch/results/{batch_id}")
async def get_batch_results(batch_id: str):
    """Lấy kết quả của một batch"""
    global batch_results
    
    results = [j for j in batch_results if j["batch_id"] == batch_id]
    
    if not results:
        # Check if still in queue
        queued = [j for j in batch_queue if j["batch_id"] == batch_id]
        if queued:
            return {
                "batch_id": batch_id,
                "status": "processing",
                "jobs": queued
            }
        else:
            raise HTTPException(status_code=404, detail="Batch không tồn tại")
    
    return {
        "batch_id": batch_id,
        "status": "completed",
        "jobs": results,
        "summary": {
            "total": len(results),
            "successful": len([j for j in results if j["status"] == "completed"]),
            "failed": len([j for j in results if j["status"] == "failed"])
        }
    }


@app.post("/api/batch/cancel/{batch_id}")
async def cancel_batch(batch_id: str):
    """Hủy một batch đang chạy"""
    global batch_queue
    
    # Remove from queue
    removed = 0
    batch_queue_copy = batch_queue.copy()
    for job in batch_queue_copy:
        if job["batch_id"] == batch_id and job["status"] == "queued":
            batch_queue.remove(job)
            removed += 1
    
    return {
        "message": f"Đã hủy {removed} jobs",
        "batch_id": batch_id
    }


async def process_batch_queue():
    """Process batch prediction queue"""
    global batch_queue, batch_results
    
    import asyncio
    
    while batch_queue:
        # Get next job
        job = None
        for j in batch_queue:
            if j["status"] == "queued":
                job = j
                break
        
        if not job:
            break
        
        # Mark as running
        job["status"] = "running"
        job["progress"] = 0
        job["started_at"] = datetime.now().isoformat()
        
        try:
            # Create PredictionConfig from job config
            pred_config = PredictionConfig(**job["config"])
            
            print(f"[BATCH] Processing job {job['job_id']}: {job['name']}")
            job["progress"] = 5
            
            # Run prediction synchronously (in the same thread to avoid conflicts)
            await asyncio.to_thread(run_batch_prediction, job, pred_config)
            
            # Check if prediction was successful
            if job.get("result") and not job.get("error"):
                job["status"] = "completed"
                job["progress"] = 100
                job["completed_at"] = datetime.now().isoformat()
                print(f"[BATCH] Job {job['job_id']} completed successfully")
            else:
                raise Exception(job.get("error", "Unknown error during prediction"))
            
        except Exception as e:
            job["error"] = str(e)
            
            # Retry logic
            if job["retries"] < job["max_retries"]:
                job["retries"] += 1
                job["status"] = "queued"  # Retry
                job["progress"] = 0
                print(f"[BATCH] Job {job['job_id']} ({job['name']}) failed, retrying ({job['retries']}/{job['max_retries']}): {e}")
                continue
            else:
                job["status"] = "failed"
                job["progress"] = 0
                job["completed_at"] = datetime.now().isoformat()
                print(f"[BATCH] Job {job['job_id']} ({job['name']}) failed permanently: {e}")
        
        # Move to results
        batch_queue.remove(job)
        batch_results.append(job)
        
        # Keep only last 100 results
        if len(batch_results) > 100:
            batch_results = batch_results[-100:]


def run_batch_prediction(job: dict, config: PredictionConfig):
    """Run prediction for a single batch job"""
    try:
        job["progress"] = 10
        
        # Import required libraries
        import xarray as xr
        import numpy as np
        from datetime import datetime as dt
        import rioxarray
        import dask.array as da
        
        job["progress"] = 15
        
        # Load model using ModelManager
        model_manager = get_model_manager()
        model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
        
        # Check if it's a PyTorch model (CNN, Swin-UNet, MobileNet, etc.)
        is_pytorch_model = hasattr(model, '__class__') and any(
            name in model.__class__.__name__ for name in ['CNN', 'SwinUNet', 'MobileNet']
        )
        
        job["progress"] = 20
        
        # Load data from Microsoft Planetary Computer
        import pystac_client
        import planetary_computer
        from odc.stac import load
        
        catalog = pystac_client.Client.open(
            "https://planetarycomputer.microsoft.com/api/stac/v1",
            modifier=planetary_computer.sign_inplace,
        )
        
        bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
        time_range = f"{config.start_date}/{config.end_date}"
        
        job["progress"] = 25
        
        # Search Sentinel-2 with retry
        s2_items = fetch_sentinel_items_with_retry(
            catalog=catalog,
            bbox=bbox,
            time_range=time_range,
            cloud_cover=config.cloud_cover,
            max_scenes=config.max_scenes,
            max_retries=3
        )
        
        job["progress"] = 35
        
        # Load Sentinel-2 data
        s2_data = load(
            s2_items,
            bbox=bbox,
            chunks={"time": 1, "x": 2048, "y": 2048},
            groupby="solar_day",
            resolution=config.resolution
        )
        
        job["progress"] = 50
        
        # Calculate NDVI
        nir = s2_data["B08"].astype('float32')
        red = s2_data["B04"].astype('float32')
        ndvi = (nir - red) / (nir + red + 1e-8)
        
        # Mask clouds if SCL available
        if "SCL" in s2_data:
            scl = s2_data["SCL"]
            cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10)
            ndvi = ndvi.where(~cloud_mask)
        
        # Fill NaN and resample
        ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')
        ndvi_monthly = ndvi_filled.resample(time="1ME").mean().compute()
        
        job["progress"] = 70
        
        # Prepare features
        n_times_ndvi = len(ndvi_monthly.time)
        y_size = len(ndvi_monthly.y)
        x_size = len(ndvi_monthly.x)
        n_pixels = y_size * x_size
        
        ndvi_features = []
        for t in range(n_times_ndvi):
            ndvi_t = ndvi_monthly.isel(time=t).values.flatten()
            ndvi_features.append(ndvi_t)
        
        features = np.column_stack(ndvi_features)
        features = np.nan_to_num(features, nan=0.0)
        
        job["progress"] = 80
        
        # Adjust features to match model expectations
        try:
            if is_pytorch_model:
                expected_features = model.n_features
            elif hasattr(model, 'n_features_in_'):
                expected_features = model.n_features_in_
            else:
                try:
                    expected_features = model.get_booster().num_features()
                except:
                    expected_features = features.shape[1]
            
            if features.shape[1] > expected_features:
                features = features[:, :expected_features]
            elif features.shape[1] < expected_features:
                n_missing = expected_features - features.shape[1]
                padding = np.tile(features[:, -1:], (1, n_missing))
                features = np.column_stack([features, padding])
        except:
            pass
        
        # Predict (all models have same predict interface)
        predictions = model.predict(features)
        
        # Decode labels
        if label_encoder is not None:
            try:
                predictions = label_encoder.inverse_transform(predictions)
            except:
                pass
        
        job["progress"] = 90
        
        # Reshape and create output
        pred_shape = (y_size, x_size)
        predictions_2d = predictions.reshape(pred_shape)
        
        prediction_da = xr.DataArray(
            predictions_2d,
            coords={"y": ndvi_monthly.y, "x": ndvi_monthly.x},
            dims=["y", "x"],
            name="classification"
        )
        
        # Save output
        output_dir = Path("predictions")
        output_dir.mkdir(exist_ok=True)
        
        output_file = output_dir / f"batch_{job['job_id']}_{job['name'].replace(' ', '_')}.tif"
        
        if hasattr(s2_data, 'rio') and s2_data.rio.crs is not None:
            prediction_da.rio.write_crs(s2_data.rio.crs, inplace=True)
        else:
            prediction_da.rio.write_crs("EPSG:4326", inplace=True)
        
        prediction_da.rio.to_raster(str(output_file), driver="GTiff")
        
        # Generate PNG preview
        png_file = output_dir / f"batch_{job['job_id']}_{job['name'].replace(' ', '_')}.png"
        try:
            import matplotlib
            matplotlib.use('Agg')
            import matplotlib.pyplot as plt
            from matplotlib.patches import Patch
            
            fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
            im = ax.imshow(predictions_2d, cmap='tab20', interpolation='nearest')
            ax.set_title(f"{job['name']} - Batch {job['job_id']}", fontsize=16, fontweight='bold', pad=20)
            ax.set_xlabel('X (pixels)', fontsize=11)
            ax.set_ylabel('Y (pixels)', fontsize=11)
            
            # Build mapping - ALWAYS use DEFAULT_LABEL_NAMES
            class_map = DEFAULT_LABEL_NAMES.copy()

            # Get unique classes in prediction to show only relevant legend items
            unique_pred_classes = np.unique(predictions_2d)
            unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
            
            # Create custom legend with color patches
            legend_elements = []
            cmap = plt.cm.get_cmap('tab20')
            
            for cls_val in sorted(unique_pred_classes):
                try:
                    cls_int = int(cls_val)
                    # Get color from colormap (normalize to 0-1 range)
                    color = cmap(cls_int / 20.0)  # tab20 has 20 colors
                    # Get label name
                    label_name = class_map.get(cls_int, f"Class {cls_int}")
                    # Create patch for legend
                    legend_elements.append(
                        Patch(facecolor=color, edgecolor='black', linewidth=0.5, 
                              label=f"{cls_int}: {label_name}")
                    )
                except:
                    pass
            
            # Add legend outside plot area
            if legend_elements:
                legend = ax.legend(
                    handles=legend_elements,
                    loc='center left',
                    bbox_to_anchor=(1.02, 0.5),
                    fontsize=10,
                    title='Land Classes',
                    title_fontsize=11,
                    framealpha=0.9,
                    edgecolor='black'
                )
                legend.get_title().set_fontweight('bold')
            
            ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
            
            plt.tight_layout()
            plt.savefig(str(png_file), dpi=150, bbox_inches='tight')
            plt.close(fig)
        except Exception as e:
            print(f"[BATCH PNG ERROR] {e}")
            png_file = None
        
        # Get unique classes
        unique_classes = np.unique(predictions_2d)
        unique_classes = unique_classes[~np.isnan(unique_classes)].tolist()
        
        # Store result in job with batch metadata
        job["result"] = {
            "output_file": str(output_file),
            "png_file": str(png_file) if png_file else None,
            "shape": list(pred_shape),
            "unique_classes": unique_classes,
            "bbox": bbox,
            "time_range": time_range,
            "n_features": features.shape[1],
            "n_times_ndvi": n_times_ndvi,
            "model_used": config.model_filename,
            "batch_job_id": job["job_id"],
            "batch_name": job["name"],
            "batch_timestamp": datetime.now().isoformat()
        }
        
        # Save batch metadata to JSON sidecar file for persistence
        metadata_file = output_file.with_suffix('.json')
        try:
            import json
            with open(metadata_file, 'w') as f:
                json.dump(job["result"], f, indent=2, default=str)
            print(f"[BATCH METADATA] Saved to {metadata_file}")
        except Exception as e:
            print(f"[BATCH METADATA ERROR] Failed to save metadata: {e}")
        
        # Auto generate prediction report for batch job
        try:
            from report_generator import generate_prediction_report
            report_path, _ = generate_prediction_report(job["result"])
            job["result"]["report_path"] = report_path
            job["result"]["report_filename"] = Path(report_path).name
            print(f"[BATCH REPORT] Generated prediction report: {report_path}")
        except Exception as e:
            print(f"[BATCH REPORT ERROR] Failed to generate report: {e}")
        
        job["progress"] = 100
        
    except Exception as e:
        job["error"] = str(e)
        import traceback
        print(f"[BATCH ERROR] Job {job['job_id']}: {traceback.format_exc()}")


# ============ CHANGE DETECTION API ============

def rasterize_ground_truth(shapefile_path, out_shape, bbox, class_column="class"):
    """Rasterize ground truth shapefile to match prediction raster shape."""
    try:
        import geopandas as gpd
        from rasterio import features as rio_features
        
        gdf = gpd.read_file(shapefile_path)
        
        # Convert to WGS84 if not already
        if gdf.crs and gdf.crs.to_epsg() != 4326:
            print(f"📍 Converting shapefile from {gdf.crs} to WGS84 for rasterization")
            gdf = gdf.to_crs("EPSG:4326")
        
        minx, miny, maxx, maxy = bbox
        
        # Crop to bbox
        gdf = gdf.cx[minx:maxx, miny:maxy]
        
        if class_column not in gdf.columns:
            raise ValueError(f"Shapefile missing '{class_column}' column. Available: {list(gdf.columns)}")
        
        # Create transform for rasterization
        transform = from_bounds(minx, miny, maxx, maxy, out_shape[1], out_shape[0])
        
        # Prepare geometries and values for rasterization
        shapes = zip(gdf.geometry, gdf[class_column])
        
        # Rasterize
        gt_raster = rio_features.rasterize(
            shapes,
            out_shape=out_shape,
            fill=-1,
            transform=transform,
            dtype="int16"
        )
        
        return gt_raster
    except Exception as e:
        print(f"[RASTERIZE ERROR] {e}")
        raise


@app.post("/api/change-detection/predict")
async def change_detection_predict_workflow(
    model_filename: str,
    min_lon: float,
    min_lat: float,
    max_lon: float,
    max_lat: float,
    start_date: str,
    end_date: str,
    max_scenes: int = 12,
    cloud_cover: int = 30,
    resolution: int = 20
):
    """
    Complete workflow: Predict + Compare with Ground Truth
    1. Load Sentinel-2 data for bbox and date range
    2. Run prediction using trained model
    3. Rasterize ground truth from training shapefile
    4. Compare and generate change detection results
    """
    try:
        # --- STEP 1: LOAD MODEL ---
        model_manager = get_model_manager()
        model, label_encoder, model_metadata = model_manager.load_model(model_filename)
        
        print(f"[CHANGE DETECTION] Loaded model: {model_filename}")
        print(f"  - Type: {model_metadata.get('model_type', 'unknown')}")
        print(f"  - Features: {model_metadata.get('features', [])}")
        
        # --- STEP 2: LOAD SENTINEL-2 DATA ---
        bbox = [min_lon, min_lat, max_lon, max_lat]
        time_range = f"{start_date}/{end_date}"
        
        catalog = Client.open(
            "https://planetarycomputer.microsoft.com/api/stac/v1",
            modifier=planetary_computer.sign_inplace
        )
        
        # Use retry logic for fetching items
        items = fetch_sentinel_items_with_retry(
            catalog=catalog,
            bbox=bbox,
            time_range=time_range,
            cloud_cover=cloud_cover,
            max_scenes=max_scenes,
            max_retries=3
        )
        print(f"[CHANGE DETECTION] Found {len(items)} Sentinel-2 scenes")
        
        if len(items) == 0:
            raise HTTPException(status_code=404, detail="No Sentinel-2 data found for the given area and date range")
        
        # Load data with error handling
        signed_items = [planetary_computer.sign(item) for item in items]
        
        max_retries = 3
        data = None
        for attempt in range(max_retries):
            try:
                data = odc.stac.load(
                    signed_items,
                    bbox=bbox,
                    bands=["B02", "B03", "B04", "B08"],
                    resolution=resolution,
                    chunks={"x": 2048, "y": 2048}
                ).compute()
                break
            except Exception as e:
                error_msg = str(e)
                
                is_auth_error = any(err in error_msg for err in [
                    "AuthenticationFailed", "Signature fields not well formed",
                    "Server failed to authenticate", "403", "401"
                ])
                
                if is_auth_error:
                    print(f"⚠️  Authentication error. Re-signing items...")
                    try:
                        signed_items = [planetary_computer.sign(item) for item in items]
                        if attempt < max_retries - 1:
                            import time
                            time.sleep(1)
                            continue
                    except Exception:
                        pass
                
                if ("does not exist" in error_msg or "RasterioIOError" in error_msg) and len(signed_items) > 3:
                    remove_count = max(1, len(signed_items) // 5)
                    signed_items = signed_items[:-remove_count]
                    items = items[:-remove_count]
                    print(f"⚠️  Access error. Reduced to {len(signed_items)} scenes, retrying...")
                    if attempt < max_retries - 1:
                        continue
                if attempt == max_retries - 1:
                    raise HTTPException(
                        status_code=500, 
                        detail=f"Failed to load Sentinel-2 data: {error_msg[:200]}"
                    )
        
        if data is None:
            raise HTTPException(status_code=500, detail="Failed to load Sentinel-2 data")
        
        # --- STEP 3: CALCULATE NDVI ---
        print("[CHANGE DETECTION] Calculating NDVI...")
        nir = data["B08"].astype('float32')
        red = data["B04"].astype('float32')
        ndvi = (nir - red) / (nir + red + 1e-8)
        
        # Handle clouds if SCL available
        if "SCL" in data:
            scl = data["SCL"]
            cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10)
            ndvi = ndvi.where(~cloud_mask)
        
        # --- STEP 4: PREPARE FEATURES ---
        ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')
        ndvi_mean = np.nanmean(ndvi_filled.values, axis=0)
        
        height, width = ndvi_mean.shape
        n_pixels = height * width
        
        # Prepare features
        features = ndvi_mean.flatten().reshape(-1, 1)
        valid_mask = ~np.isnan(features[:, 0])
        features_clean = features[valid_mask]
        
        # --- STEP 5: PREDICT ---
        print("[CHANGE DETECTION] Running prediction...")
        predictions = model.predict(features_clean)
        
        # Decode labels if needed
        if label_encoder is not None:
            try:
                predictions = label_encoder.inverse_transform(predictions)
            except:
                pass
        
        # Reshape to raster
        prediction_raster = np.full(n_pixels, -1, dtype=np.int16)
        prediction_raster[valid_mask] = predictions.astype(np.int16)
        prediction_raster = prediction_raster.reshape(height, width)
        
        # --- STEP 6: COMPARE WITH GROUND TRUTH ---
        print("[CHANGE DETECTION] Comparing with ground truth...")
        gt_shapefile = "train/ST_training data_updated_1130points_new.shp"
        gt_raster = rasterize_ground_truth(gt_shapefile, (height, width), bbox, class_column="class")
        
        # Calculate changes
        mask_valid = (gt_raster >= 0) & (prediction_raster >= 0)
        changes = gt_raster[mask_valid] != prediction_raster[mask_valid]
        n_total = np.count_nonzero(mask_valid)
        n_changed = np.count_nonzero(changes)
        
        # Create change matrix
        from collections import Counter
        change_pairs = list(zip(gt_raster[mask_valid][changes], prediction_raster[mask_valid][changes]))
        change_counter = Counter(change_pairs)
        change_matrix = {f"{int(gt)}->{int(pred)}": int(cnt) for (gt, pred), cnt in change_counter.items()}
        
        # Create change map
        change_map = np.full((height, width), -1, dtype=np.int8)
        change_map[mask_valid] = changes.astype(np.int8)
        
        # --- STEP 7: SAVE RESULTS ---
        output_dir = Path("predictions")
        output_dir.mkdir(exist_ok=True)
        
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        change_file = output_dir / f"change_map_{timestamp}.tif"
        
        transform = from_bounds(min_lon, min_lat, max_lon, max_lat, width, height)
        
        with rasterio.open(
            change_file, 'w',
            driver='GTiff',
            height=height,
            width=width,
            count=1,
            dtype=change_map.dtype,
            crs='EPSG:4326',
            transform=transform
        ) as dst:
            dst.write(change_map, 1)
        
        print(f"[CHANGE DETECTION] Saved change map to {change_file}")
        
        # --- RETURN RESULTS ---
        return {
            "success": True,
            "n_scenes": len(items),
            "ndvi_stats": {
                "mean": float(np.nanmean(ndvi_mean)),
                "min": float(np.nanmin(ndvi_mean)),
                "max": float(np.nanmax(ndvi_mean)),
                "std": float(np.nanstd(ndvi_mean))
            },
            "class_distribution": {
                int(cls): int(count) 
                for cls, count in zip(*np.unique(predictions, return_counts=True))
            },
            "change_detection": {
                "n_total_pixels": int(n_total),
                "n_changed_pixels": int(n_changed),
                "change_rate": float(n_changed) / n_total if n_total > 0 else 0.0,
                "change_matrix": change_matrix,
                "message": f"Detected {n_changed} changes out of {n_total} valid pixels ({(n_changed/n_total*100):.1f}%)" if n_total > 0 else "No valid pixels for comparison"
            },
            "change_map_file": str(change_file),
            "timestamp": timestamp
        }
        
    except HTTPException:
        raise
    except Exception as e:
        print(f"[CHANGE DETECTION ERROR] {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Change detection workflow failed: {str(e)}")


@app.post("/api/change-detection/workflow")
async def change_detection_workflow(request: ChangeDetectionWorkflowRequest):
    """Workflow: compare prediction with ground truth training data."""
    from collections import Counter
    
    try:
        prediction_result = request.prediction_result
        bbox = request.bbox
        
        if not prediction_result or "output_files" not in prediction_result:
            raise ValueError("Invalid prediction result")
        
        # Get classification raster from prediction
        class_file = None
        for f in prediction_result.get("output_files", []):
            if f.get("type") == "classification":
                class_file = f.get("path")
                break
        
        if not class_file:
            raise ValueError("No classification raster in prediction result")
        
        # Load prediction raster
        with rasterio.open(class_file) as pred_ds:
            pred_arr = pred_ds.read(1)
            pred_crs = pred_ds.crs
            pred_transform = pred_ds.transform
        
        # Rasterize ground truth training data
        gt_shapefile = "train/ST_training data_updated_1130points_new.shp"
        gt_raster = rasterize_ground_truth(gt_shapefile, pred_arr.shape, bbox, class_column="class")
        
        # Calculate change detection
        mask_valid = (gt_raster >= 0) & (pred_arr >= 0) & ~np.isnan(gt_raster) & ~np.isnan(pred_arr)
        changes = gt_raster[mask_valid] != pred_arr[mask_valid]
        n_total = np.count_nonzero(mask_valid)
        n_changed = np.count_nonzero(changes)
        
        # Create change pairs matrix
        change_pairs = list(zip(gt_raster[mask_valid][changes], pred_arr[mask_valid][changes]))
        change_counter = Counter(change_pairs)
        change_matrix = {f"{int(gt)}->{int(pred)}": int(cnt) for (gt, pred), cnt in change_counter.items()}
        
        # Create change map
        change_map = np.full(pred_arr.shape, -1, dtype=np.int8)
        change_map[mask_valid] = changes.astype(np.int8)
        
        # Change rate
        change_rate = float(n_changed) / n_total if n_total > 0 else 0.0
        
        # Save change map
        change_dir = Path("predictions")
        change_dir.mkdir(exist_ok=True)
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        change_file = change_dir / f"change_map_{timestamp}.tif"
        
        with rasterio.open(
            change_file, 'w',
            driver='GTiff',
            height=change_map.shape[0],
            width=change_map.shape[1],
            count=1,
            dtype=change_map.dtype,
            crs=pred_crs,
            transform=pred_transform
        ) as dst:
            dst.write(change_map, 1)
        
        return {
            "success": True,
            "change_detection": {
                "n_total_pixels": int(n_total),
                "n_changed_pixels": int(n_changed),
                "change_rate": change_rate,
                "change_matrix": change_matrix,
                "message": f"Detected {n_changed} changes out of {n_total} valid pixels ({change_rate*100:.1f}%)"
            },
            "change_map_file": str(change_file),
            "timestamp": timestamp
        }
    
    except Exception as e:
        print(f"[CHANGE DETECTION WORKFLOW ERROR] {str(e)}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Change detection workflow failed: {str(e)}")


@app.post("/api/change-detection/compare-periods")
async def compare_periods(request: ComparePeriodsPredictionConfig, background_tasks: BackgroundTasks):
    """Compare land use classification between two time periods."""
    from collections import Counter
    
    try:
        # Extract parameters
        model_filename = request.model_filename
        bbox = [request.min_lon, request.min_lat, request.max_lon, request.max_lat]
        
        current_start = request.current_period["start_date"]
        current_end = request.current_period["end_date"]
        pred_start = request.prediction_period["start_date"]
        pred_end = request.prediction_period["end_date"]
        
        max_scenes = request.max_scenes
        cloud_cover = request.cloud_cover
        resolution = request.resolution
        
        print(f"[COMPARE PERIODS] Current: {current_start} to {current_end} | Prediction: {pred_start} to {pred_end}")
        
        # Step 1: Predict on current period
        print(f"[COMPARE PERIODS] Step 1: Predicting current period...")
        current_result = await predict_with_ndvi(PredictionWithNDVIConfig(
            model_filename=model_filename,
            min_lon=request.min_lon,
            min_lat=request.min_lat,
            max_lon=request.max_lon,
            max_lat=request.max_lat,
            start_date=current_start,
            end_date=current_end,
            max_scenes=max_scenes,
            cloud_cover=cloud_cover,
            resolution=resolution,
            export_classification=True,
            export_ndvi=False
        ), background_tasks)
        
        # Get current classification raster
        current_class_file = None
        for f in current_result.get("output_files", []):
            if f.get("type") == "classification":
                current_class_file = f.get("path")
                break
        
        if not current_class_file:
            raise ValueError("No classification raster for current period")
        
        # Step 2: Predict on prediction period
        print(f"[COMPARE PERIODS] Step 2: Predicting future period...")
        pred_result = await predict_with_ndvi(PredictionWithNDVIConfig(
            model_filename=model_filename,
            min_lon=request.min_lon,
            min_lat=request.min_lat,
            max_lon=request.max_lon,
            max_lat=request.max_lat,
            start_date=pred_start,
            end_date=pred_end,
            max_scenes=max_scenes,
            cloud_cover=cloud_cover,
            resolution=resolution,
            export_classification=True,
            export_ndvi=False
        ), background_tasks)
        
        # Get prediction classification raster
        pred_class_file = None
        for f in pred_result.get("output_files", []):
            if f.get("type") == "classification":
                pred_class_file = f.get("path")
                break
        
        if not pred_class_file:
            raise ValueError("No classification raster for prediction period")
        
        # Step 3: Load both rasters
        print(f"[COMPARE PERIODS] Step 3: Comparing classifications...")
        with rasterio.open(current_class_file) as src:
            current_arr = src.read(1)
            crs = src.crs
            transform = src.transform
        
        with rasterio.open(pred_class_file) as src:
            pred_arr = src.read(1)
        
        # Ensure same shape
        if current_arr.shape != pred_arr.shape:
            raise ValueError(f"Shape mismatch: current {current_arr.shape} vs prediction {pred_arr.shape}")
        
        # Calculate changes
        mask_valid = ~np.isnan(current_arr) & ~np.isnan(pred_arr)
        changes = current_arr[mask_valid] != pred_arr[mask_valid]
        n_total = np.count_nonzero(mask_valid)
        n_changed = np.count_nonzero(changes)
        
        # Create change pairs
        change_pairs = list(zip(current_arr[mask_valid][changes], pred_arr[mask_valid][changes]))
        change_counter = Counter(change_pairs)
        change_matrix = {f"{int(curr)}->{int(pred)}": int(cnt) 
                        for (curr, pred), cnt in change_counter.items()}
        
        # Change rate
        change_rate = float(n_changed) / n_total if n_total > 0 else 0.0
        
        # Save change map
        change_dir = Path("predictions")
        change_dir.mkdir(exist_ok=True)
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        change_file = change_dir / f"change_map_{timestamp}.tif"
        
        change_map = np.zeros(current_arr.shape, dtype=np.int8)
        change_map[mask_valid] = changes.astype(np.int8)
        
        with rasterio.open(
            change_file, 'w',
            driver='GTiff',
            height=change_map.shape[0],
            width=change_map.shape[1],
            count=1,
            dtype=change_map.dtype,
            crs=crs,
            transform=transform
        ) as dst:
            dst.write(change_map, 1)
        
        # Extract class distributions
        current_classes = np.unique(current_arr[~np.isnan(current_arr)]).astype(int)
        current_dist = {int(c): int(np.count_nonzero(current_arr == c)) for c in current_classes}
        
        pred_classes = np.unique(pred_arr[~np.isnan(pred_arr)]).astype(int)
        pred_dist = {int(c): int(np.count_nonzero(pred_arr == c)) for c in pred_classes}
        
        return {
            "success": True,
            "current_classification": {
                "n_scenes": current_result.get("n_scenes"),
                "resolution": current_result.get("resolution"),
                "class_distribution": current_dist
            },
            "prediction_classification": {
                "n_scenes": pred_result.get("n_scenes"),
                "resolution": pred_result.get("resolution"),
                "class_distribution": pred_dist
            },
            "change_detection": {
                "n_total_pixels": int(n_total),
                "n_changed_pixels": int(n_changed),
                "change_rate": change_rate,
                "change_matrix": change_matrix,
                "message": f"Detected {n_changed} changes out of {n_total} valid pixels ({change_rate*100:.1f}%)"
            },
            "change_map_file": str(change_file),
            "timestamp": timestamp
        }
    
    except Exception as e:
        print(f"[COMPARE PERIODS ERROR] {str(e)}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Period comparison failed: {str(e)}")


@app.post("/api/change-detection")
async def change_detection_api(
    prediction_file: UploadFile = File(...),
    gt_file: UploadFile = File(...)
):
    """Detect changes between two raster files (prediction and ground truth)."""
    import tempfile
    from collections import Counter
    
    try:
        # Save uploaded files temporarily
        pred_tmp = tempfile.NamedTemporaryFile(suffix='.tif', delete=False)
        gt_tmp = tempfile.NamedTemporaryFile(suffix='.tif', delete=False)
        
        try:
            # Write uploaded files to temp
            pred_content = await prediction_file.read()
            gt_content = await gt_file.read()
            
            pred_tmp.write(pred_content)
            gt_tmp.write(gt_content)
            pred_tmp.close()
            gt_tmp.close()
            
            # Read prediction raster
            with rasterio.open(pred_tmp.name) as pred_ds:
                pred_arr = pred_ds.read(1)
                pred_crs = pred_ds.crs
                pred_transform = pred_ds.transform
            
            # Read ground truth raster
            with rasterio.open(gt_tmp.name) as gt_ds:
                gt_arr = gt_ds.read(1)
            
            # Ensure same shape
            if pred_arr.shape != gt_arr.shape:
                raise ValueError(f"Raster shapes don't match: prediction {pred_arr.shape} vs ground truth {gt_arr.shape}")
            
            # Calculate change detection
            mask_valid = (gt_arr >= 0) & (pred_arr >= 0) & ~np.isnan(gt_arr) & ~np.isnan(pred_arr)
            changes = gt_arr[mask_valid] != pred_arr[mask_valid]
            n_total = np.count_nonzero(mask_valid)
            n_changed = np.count_nonzero(changes)
            
            # Create change pairs matrix
            change_pairs = list(zip(gt_arr[mask_valid][changes], pred_arr[mask_valid][changes]))
            change_counter = Counter(change_pairs)
            change_matrix = {f"{int(gt)}->{int(pred)}": int(cnt) for (gt, pred), cnt in change_counter.items()}
            
            # Create change map (0=same, 1=changed, -1=invalid)
            change_map = np.full(pred_arr.shape, -1, dtype=np.int8)
            change_map[mask_valid] = changes.astype(np.int8)
            
            # Change rate
            change_rate = float(n_changed) / n_total if n_total > 0 else 0.0
            
            # Save change map as GeoTIFF
            change_dir = Path("predictions")
            change_dir.mkdir(exist_ok=True)
            
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            change_file = change_dir / f"change_map_{timestamp}.tif"
            
            with rasterio.open(
                change_file, 'w',
                driver='GTiff',
                height=change_map.shape[0],
                width=change_map.shape[1],
                count=1,
                dtype=change_map.dtype,
                crs=pred_crs,
                transform=pred_transform
            ) as dst:
                dst.write(change_map, 1)
            
            # Return results
            return {
                "success": True,
                "change_detection": {
                    "n_total_pixels": int(n_total),
                    "n_changed_pixels": int(n_changed),
                    "change_rate": change_rate,
                    "change_matrix": change_matrix,
                    "message": f"Detected {n_changed} changes out of {n_total} valid pixels ({change_rate*100:.1f}%)"
                },
                "change_map_file": str(change_file),
                "timestamp": timestamp
            }
            
        finally:
            # Cleanup temp files
            try:
                Path(pred_tmp.name).unlink()
                Path(gt_tmp.name).unlink()
            except:
                pass
    
    except Exception as e:
        print(f"[CHANGE DETECTION ERROR] {str(e)}")
        raise HTTPException(status_code=500, detail=f"Change detection failed: {str(e)}")
async def change_detection_api(
    prediction_file: UploadFile = File(...),
    gt_file: UploadFile = File(...)
):
    """Detect changes between prediction raster and ground truth raster."""
    import tempfile
    from collections import Counter
    
    try:
        # Save uploaded files temporarily
        pred_tmp = tempfile.NamedTemporaryFile(suffix='.tif', delete=False)
        gt_tmp = tempfile.NamedTemporaryFile(suffix='.tif', delete=False)
        
        try:
            # Write uploaded files to temp
            pred_content = await prediction_file.read()
            gt_content = await gt_file.read()
            
            pred_tmp.write(pred_content)
            gt_tmp.write(gt_content)
            pred_tmp.close()
            gt_tmp.close()
            
            # Read prediction raster
            import rasterio
            with rasterio.open(pred_tmp.name) as pred_ds:
                pred_arr = pred_ds.read(1)
                pred_crs = pred_ds.crs
                pred_transform = pred_ds.transform
            
            # Read ground truth raster
            with rasterio.open(gt_tmp.name) as gt_ds:
                gt_arr = gt_ds.read(1)
            
            # Ensure same shape
            if pred_arr.shape != gt_arr.shape:
                raise ValueError(f"Raster shapes don't match: prediction {pred_arr.shape} vs ground truth {gt_arr.shape}")
            
            # Calculate change detection
            mask_valid = (gt_arr >= 0) & (pred_arr >= 0) & ~np.isnan(gt_arr) & ~np.isnan(pred_arr)
            changes = gt_arr[mask_valid] != pred_arr[mask_valid]
            n_total = np.count_nonzero(mask_valid)
            n_changed = np.count_nonzero(changes)
            
            # Create change pairs matrix
            change_pairs = list(zip(gt_arr[mask_valid][changes], pred_arr[mask_valid][changes]))
            change_counter = Counter(change_pairs)
            change_matrix = {f"{int(gt)}->{int(pred)}": int(cnt) for (gt, pred), cnt in change_counter.items()}
            
            # Create change map (0=same, 1=changed, -1=invalid)
            change_map = np.full(pred_arr.shape, -1, dtype=np.int8)
            change_map[mask_valid] = changes.astype(np.int8)
            
            # Change rate
            change_rate = float(n_changed) / n_total if n_total > 0 else 0.0
            
            # Save change map as GeoTIFF
            change_dir = Path("predictions")
            change_dir.mkdir(exist_ok=True)
            
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            change_file = change_dir / f"change_map_{timestamp}.tif"
            
            with rasterio.open(
                change_file, 'w',
                driver='GTiff',
                height=change_map.shape[0],
                width=change_map.shape[1],
                count=1,
                dtype=change_map.dtype,
                crs=pred_crs,
                transform=pred_transform
            ) as dst:
                dst.write(change_map, 1)
            
            # Return results
            return {
                "success": True,
                "change_detection": {
                    "n_total_pixels": int(n_total),
                    "n_changed_pixels": int(n_changed),
                    "change_rate": change_rate,
                    "change_matrix": change_matrix,
                    "message": f"Detected {n_changed} changes out of {n_total} valid pixels ({change_rate*100:.1f}%)"
                },
                "change_map_file": str(change_file),
                "timestamp": timestamp
            }
            
        finally:
            # Cleanup temp files
            try:
                Path(pred_tmp.name).unlink()
                Path(gt_tmp.name).unlink()
            except:
                pass
    
    except Exception as e:
        print(f"[CHANGE DETECTION ERROR] {str(e)}")
        raise HTTPException(status_code=500, detail=f"Change detection failed: {str(e)}")


# ============ PREDICTION WITH NDVI API ============

def fetch_sentinel_items_with_retry(catalog, bbox, time_range, cloud_cover, max_scenes, max_retries=3):
    """
    Fetch Sentinel-2 items with retry logic and exponential backoff
    
    Optimizations:
    - Reduce page size on retry
    - Use shorter timeouts for each attempt
    - Fetch fewer items initially and expand if successful
    """
    import time
    
    for attempt in range(max_retries):
        try:
            # Reduce target items on each retry to minimize timeout risk
            target_items = max_scenes if attempt == 0 else min(max_scenes, 20 // (attempt + 1) * 10)
            page_limit = 50 if attempt == 0 else 20  # Smaller pages on retry
            
            print(f"[FETCH ATTEMPT {attempt + 1}/{max_retries}] Searching Sentinel-2...")
            print(f"  → Target items: {target_items}, Page limit: {page_limit}")
            
            # Search with reduced limit on retries
            search = catalog.search(
                collections=["sentinel-2-l2a"],
                bbox=bbox,
                datetime=time_range,
                query={"eo:cloud_cover": {"lt": cloud_cover}},
                limit=page_limit
            )
            
            # Try to get items with timeout protection
            items = []
            page_count = 0
            max_pages = 3 if attempt > 0 else 5  # Fewer pages on retry
            
            for item in search.items():
                items.append(item)
                if len(items) >= target_items:
                    print(f"[FETCH] Reached target ({target_items} items)")
                    break
                    
                # Track pagination to prevent hanging
                if len(items) % page_limit == 0:
                    page_count += 1
                    if page_count >= max_pages:
                        print(f"[FETCH] Max pages reached ({max_pages}), got {len(items)} items")
                        break
            
            if items:
                print(f"[FETCH SUCCESS] Retrieved {len(items)} items")
                # Return up to max_scenes, but accept fewer if that's all we got
                return items[:min(len(items), max_scenes)]
            else:
                raise ValueError("No Sentinel-2 scenes found for the specified criteria")
                
        except Exception as e:
            error_msg = str(e)
            print(f"[FETCH ERROR] Attempt {attempt + 1} failed: {error_msg}")
            
            if attempt < max_retries - 1:
                # Longer exponential backoff: 3, 6, 12 seconds
                wait_time = 3 * (2 ** attempt)
                print(f"[RETRY] Waiting {wait_time}s before retry...")
                time.sleep(wait_time)
            else:
                # Final attempt failed
                if "exceeded the maximum allowed time" in error_msg or "timeout" in error_msg.lower():
                    raise HTTPException(
                        status_code=504,
                        detail=f"Microsoft Planetary Computer request timed out after {max_retries} attempts. "
                               f"Please try: (1) Reduce date range (2) Reduce max_scenes to 5-10 (3) Use smaller bbox area"
                    )
                elif "no sentinel-2 scenes found" in error_msg.lower():
                    raise HTTPException(
                        status_code=404,
                        detail="No Sentinel-2 data found. Try: (1) Different date range (2) Higher cloud_cover threshold (3) Different location"
                    )
                else:
                    raise


@app.post("/api/predict/with-ndvi")
async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks: BackgroundTasks):
    """Predict land classification và NDVI cho một khu vực"""
    try:
        import numpy as np
        
        # Load model using ModelManager
        model_manager = get_model_manager()
        model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
        
        print(f"[PREDICT+NDVI] Loaded model: {config.model_filename}")
        print(f"  - Type: {model_metadata.get('model_type', 'unknown')}")
        print(f"  - Features: {model_metadata.get('features', [])}")
        
        # Check cache first
        cache_dir = Path("dataset_cache")
        cache_dir.mkdir(exist_ok=True)
        cache_key = f"pred_{config.min_lon}_{config.min_lat}_{config.max_lon}_{config.max_lat}_{config.start_date}_{config.end_date}_{config.max_scenes}_{config.cloud_cover}_{config.resolution}"
        cache_hash = hashlib.md5(cache_key.encode()).hexdigest()
        cache_file = cache_dir / f"prediction_input_{cache_hash}.joblib"
        
        bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
        
        # Load from cache or fetch from Microsoft
        if cache_file.exists():
            print(f"[PREDICT+NDVI] Loading from cache: {cache_file.name}")
            cached = joblib.load(cache_file)
            # Extract s2_data from cache (already computed in cache)
            s2_data = cached["s2_data"]
            # Check if needed bands are available in cache
            available_bands = list(s2_data.data_vars.keys())
            needed_bands = ["B02", "B03", "B04", "B08"]
            if all(band in available_bands for band in needed_bands):
                # Use cached data directly (no need to compute again)
                data = s2_data[needed_bands]
                print(f"[PREDICT+NDVI] Using cached bands: {needed_bands}")
                # Set items to match the number of time slices in the cached data
                items = [None] * s2_data.sizes.get("time", 1)
            else:
                raise HTTPException(status_code=400, 
                    detail=f"Cache thiếu bands cần thiết. Có: {available_bands}, Cần: {needed_bands}")
        else:
            print(f"[PREDICT+NDVI] No cache found, fetching from Microsoft Planetary Computer")
            # Connect to Microsoft Planetary Computer
            catalog = Client.open(
                "https://planetarycomputer.microsoft.com/api/stac/v1",
                modifier=planetary_computer.sign_inplace
            )
            
            time_range = f"{config.start_date}/{config.end_date}"
            
            # Search for Sentinel-2 data with retry logic
            items = fetch_sentinel_items_with_retry(
                catalog=catalog,
                bbox=bbox,
                time_range=time_range,
                cloud_cover=config.cloud_cover,
                max_scenes=config.max_scenes,
                max_retries=3
            )
            print(f"[PREDICT+NDVI] Found {len(items)} Sentinel-2 scenes")
            
            if len(items) == 0:
                raise HTTPException(status_code=404, detail="Không tìm thấy dữ liệu vệ tinh")
            
            # Sign items to refresh SAS tokens (keep as pystac.Item, not dict)
            signed_items = [planetary_computer.sign(item) for item in items]
            
            # Load all bands needed for features
            data = odc.stac.load(
                signed_items,
                bbox=bbox,
                bands=["B02", "B03", "B04", "B08"],  # Blue, Green, Red, NIR
                resolution=config.resolution,
                chunks={"x": 2048, "y": 2048}
            ).compute()
        
        print(f"[PREDICT+NDVI] Loaded data shape: {data.dims}")
        
        # Get expected number of features from model metadata
        expected_n_features = model_metadata.get("n_features", 3)
        print(f"[PREDICT+NDVI] Model expects {expected_n_features} features")
        
        # Calculate NDVI and other indices
        blue = data["B02"].values
        green = data["B03"].values
        red = data["B04"].values
        nir = data["B08"].values
        
        # Calculate indices for each time step
        # NDVI = (NIR - Red) / (NIR + Red)
        ndvi = (nir - red) / (nir + red + 1e-8)
        
        # NDWI = (Green - NIR) / (Green + NIR)
        ndwi = (green - nir) / (green + nir + 1e-8)
        
        # NDBI = (SWIR - NIR) / (SWIR + NIR) - we use Red as proxy
        ndbi = (red - nir) / (red + nir + 1e-8)
        
        # Prepare features for prediction
        height, width = ndvi.shape[1:3]  # Skip time dimension
        n_pixels = height * width
        n_times = ndvi.shape[0]
        
        print(f"[PREDICT+NDVI] Data has {n_times} time steps, spatial size: {height}x{width}")
        
        # Build features using FeatureExtractor to match training
        feature_mode = model_metadata.get("feature_mode", "simple")
        expected_n_features = model_metadata.get("n_features", 3)
        
        print(f"[PREDICT+NDVI] Model feature_mode: {feature_mode}")
        print(f"[PREDICT+NDVI] Model n_features: {expected_n_features}")
        
        # COMPATIBILITY FIX: If model was trained with buggy code (n_features=3 but feature_mode='odc'),
        # fallback to simple mode to match what model actually expects
        if feature_mode == 'odc' and expected_n_features == 3:
            print(f"[PREDICT+NDVI] ⚠️ WARNING: Model metadata shows odc mode but only 3 features")
            print(f"[PREDICT+NDVI] This model was trained with old buggy code - using simple mode for compatibility")
            feature_mode = 'simple'
        
        # Use FeatureExtractor for consistent feature building
        from feature_extractor import FeatureExtractor
        extractor = FeatureExtractor(mode=feature_mode)
        
        print(f"[PREDICT+NDVI] Using FeatureExtractor with mode='{feature_mode}'")
        print(f"[PREDICT+NDVI] Expected features: {extractor.get_info()}")
        
        # Extract features from data
        # data is already an xr.Dataset with B02, B03, B04, B08
        # Need to add B11 for ODC mode (NDBI calculation uses SWIR)
        if feature_mode == 'odc' and 'B11' not in data:
            # Load B11 if needed for ODC mode
            print(f"[PREDICT+NDVI] ODC mode requires B11 (SWIR), loading...")
            
            # Fetch B11 band (whether from cache or fresh fetch)
            try:
                # If we haven't fetched items yet (cache scenario), do it now
                if 'signed_items' not in locals():
                    catalog = Client.open(
                        "https://planetarycomputer.microsoft.com/api/stac/v1",
                        modifier=planetary_computer.sign_inplace
                    )
                    time_range = f"{config.start_date}/{config.end_date}"
                    
                    # Use retry logic for B11 band
                    b11_items = fetch_sentinel_items_with_retry(
                        catalog=catalog,
                        bbox=bbox,
                        time_range=time_range,
                        cloud_cover=config.cloud_cover,
                        max_scenes=config.max_scenes,
                        max_retries=3
                    )
                    signed_items = [planetary_computer.sign(item) for item in b11_items]
                    print(f"[PREDICT+NDVI] Fetched {len(signed_items)} scenes for B11")
                
                b11_data = odc.stac.load(
                    signed_items,
                    bbox=bbox,
                    bands=["B11"],
                    resolution=config.resolution,
                    chunks={"x": 2048, "y": 2048}
                ).compute()
                
                # Merge B11 into existing data
                data = xr.merge([data, b11_data])
                print(f"[PREDICT+NDVI] Added B11 to data")
            except Exception as e:
                print(f"[PREDICT+NDVI] Warning: Failed to load B11: {e}")
                print(f"[PREDICT+NDVI] Will proceed without B11 (may affect NDBI accuracy)")
        
        # Extract features using FeatureExtractor
        if feature_mode == 'simple':
            # Simple mode needs pre-calculated NDVI
            print(f"[PREDICT+NDVI] Calculating NDVI for simple mode...")
            red_band = data["B04"].values
            nir_band = data["B08"].values
            ndvi_array = (nir_band - red_band) / (nir_band + red_band + 1e-8)
            
            # Convert to xarray DataArray with proper dims
            ndvi_data = xr.DataArray(
                ndvi_array,
                dims=data["B04"].dims,
                coords=data["B04"].coords
            )
            
            # Simple mode also needs VH/VV radar data, but we don't have it for this endpoint
            # Pass None and let extractor handle it
            features = extractor.extract(s2_data=None, ndvi_data=ndvi_data, vh_data=None, vv_data=None)
        else:
            # ODC/extended modes use s2_data directly
            features = extractor.extract(s2_data=data, vh_data=None, vv_data=None)
        
        print(f"[PREDICT+NDVI] Built features shape: {features.shape}")
        print(f"[PREDICT+NDVI] Features per pixel: {features.shape[1] if len(features.shape) > 1 else 1}")
        
        # Handle NaN, inf, and extreme values
        # Replace inf with 0
        features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)
        
        # Clip extreme values to reasonable range
        features = np.clip(features, -1e6, 1e6)
        
        # Double-check no inf/nan remain
        valid_mask = np.isfinite(features).all(axis=1)
        features_clean = features[valid_mask]
        
        print(f"[PREDICT+NDVI] Predicting {features_clean.shape[0]} valid pixels...")
        print(f"[PREDICT+NDVI] Features range: [{features_clean.min():.3f}, {features_clean.max():.3f}]")
        
        # Check if model is PyTorch/deep learning model and use GPU if available
        is_pytorch_model = hasattr(model, '__class__') and ('CNN' in model.__class__.__name__ or 'Swin' in model.__class__.__name__ or 'UNet' in model.__class__.__name__ or 'MobileNet' in model.__class__.__name__)
        
        if is_pytorch_model and config.use_gpu:
            try:    
                import torch
                device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
                
                if torch.cuda.is_available():
                    print(f"[PREDICT+NDVI] Using GPU: {torch.cuda.get_device_name(0)}")
                    # Move model to GPU
                    model = model.to(device)
                    
                    # Predict in batches to avoid GPU memory overflow
                    batch_size = 8192  # Adjust based on GPU memory
                    predictions_list = []
                    
                    for i in range(0, len(features_clean), batch_size):
                        batch = features_clean[i:i+batch_size]
                        batch_tensor = torch.from_numpy(batch).float().to(device)
                        
                        with torch.no_grad():
                            batch_pred = model.predict(batch_tensor)
                        
                        # Move back to CPU if needed
                        if isinstance(batch_pred, torch.Tensor):
                            batch_pred = batch_pred.cpu().numpy()
                        
                        predictions_list.append(batch_pred)
                        
                        if (i // batch_size) % 10 == 0:
                            print(f"[PREDICT+NDVI] Processed {i + len(batch)}/{len(features_clean)} pixels on GPU")
                    
                    predictions = np.concatenate(predictions_list)
                    print(f"[PREDICT+NDVI] GPU prediction completed!")
                else:
                    print(f"[PREDICT+NDVI] GPU requested but not available, using CPU")
                    predictions = model.predict(features_clean)
            except Exception as gpu_error:
                print(f"[PREDICT+NDVI] GPU prediction failed: {gpu_error}, falling back to CPU")
                # Move model back to CPU before retrying
                try:
                    model = model.cpu()
                    print(f"[PREDICT+NDVI] Moved model to CPU")
                except:
                    pass
                predictions = model.predict(features_clean)
        else:
            # Use CPU for traditional ML models
            if is_pytorch_model and not config.use_gpu:
                print(f"[PREDICT+NDVI] GPU disabled by user, using CPU")
            predictions = model.predict(features_clean)
        
        # Reshape back to raster
        prediction_raster = np.full(n_pixels, -1, dtype=np.int16)
        prediction_raster[valid_mask] = predictions
        prediction_raster = prediction_raster.reshape(height, width)

        # Smooth classification map to make output cleaner (reduce speckle)
        try:
            from scipy import ndimage
            smoothed = ndimage.median_filter(prediction_raster, size=3)
            smoothed[prediction_raster < 0] = -1  # keep nodata
            prediction_raster = smoothed
            print("[PREDICT+NDVI][SMOOTH] Applied 3x3 median filter to classification")
        except Exception as smooth_err:
            print(f"[PREDICT+NDVI][SMOOTH WARNING] {smooth_err}")

        # --- CHANGE DETECTION ---
        change_summary = None
        change_map = None
        try:
            # Use training shapefile as ground truth
            gt_shapefile = "train/ST_training data_updated_1130points_new.shp"
            gt_raster = rasterize_ground_truth(gt_shapefile, (height, width), bbox, class_column="class")
            # Compare prediction and ground truth
            mask_valid = (gt_raster >= 0) & (prediction_raster >= 0)
            changes = gt_raster[mask_valid] != prediction_raster[mask_valid]
            n_total = np.count_nonzero(mask_valid)
            n_changed = np.count_nonzero(changes)
            # Per-class change matrix
            from collections import Counter
            change_pairs = list(zip(gt_raster[mask_valid][changes], prediction_raster[mask_valid][changes]))
            change_counter = Counter(change_pairs)
            change_matrix = {f"{int(gt)}->{int(pred)}": int(cnt) for (gt, pred), cnt in change_counter.items()}
            change_summary = {
                "n_total": int(n_total),
                "n_changed": int(n_changed),
                "change_rate": float(n_changed) / n_total if n_total > 0 else 0.0,
                "change_matrix": change_matrix
            }
            # Optionally, create a change map (1=changed, 0=same, -1=invalid)
            change_map = np.full((height, width), -1, dtype=np.int8)
            change_map[mask_valid] = changes.astype(np.int8)
            # Save change map as GeoTIFF
            change_file = output_dir / f"change_map_{timestamp}.tif"
            with rasterio.open(
                change_file, 'w',
                driver='GTiff',
                height=height,
                width=width,
                count=1,
                dtype=change_map.dtype,
                crs='EPSG:4326',
                transform=from_bounds(bbox[0], bbox[1], bbox[2], bbox[3], width, height)
            ) as dst:
                dst.write(change_map, 1)
            output_files.append({"type": "change_map", "path": str(change_file)})
            print(f"[CHANGE DETECTION] Saved change map to {change_file}")
        except Exception as change_exc:
            print(f"[CHANGE DETECTION] Warning: {change_exc}")
        
        # Prepare outputs
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        output_dir = Path("predictions")
        output_dir.mkdir(exist_ok=True)
        
        output_files = []
        
        # Export NDVI if requested
        if config.export_ndvi:
            # Calculate NDVI from data for export (data contains B04=red, B08=nir)
            red_band = data["B04"].values
            nir_band = data["B08"].values
            ndvi_array = (nir_band - red_band) / (nir_band + red_band + 1e-8)
            # Average over time dimension to get mean NDVI
            ndvi_mean = np.nanmean(ndvi_array, axis=0)
            
            ndvi_file = output_dir / f"ndvi_{timestamp}.tif"
            transform = from_bounds(bbox[0], bbox[1], bbox[2], bbox[3], width, height)
            
            with rasterio.open(
                ndvi_file, 'w',
                driver='GTiff',
                height=height,
                width=width,
                count=1,
                dtype=ndvi_mean.dtype,
                crs='EPSG:4326',
                transform=transform
            ) as dst:
                dst.write(ndvi_mean, 1)
            
            output_files.append({"type": "ndvi", "path": str(ndvi_file)})
            print(f"[PREDICT+NDVI] Saved NDVI to {ndvi_file}")
            
            # Create PNG preview for NDVI
            ndvi_png = output_dir / f"ndvi_{timestamp}.png"
            try:
                import matplotlib
                matplotlib.use('Agg')
                import matplotlib.pyplot as plt
                
                fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
                im = ax.imshow(ndvi_mean, cmap='RdYlGn', vmin=-1, vmax=1, interpolation='nearest')
                ax.set_title(f'NDVI - {timestamp}', fontsize=14, fontweight='bold')
                ax.set_xlabel('X (pixels)', fontsize=10)
                ax.set_ylabel('Y (pixels)', fontsize=10)
                
                cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
                cbar.set_label('NDVI', rotation=270, labelpad=15)
                ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
                
                plt.tight_layout()
                plt.savefig(str(ndvi_png), dpi=150, bbox_inches='tight')
                plt.close(fig)
                
                output_files.append({"type": "ndvi_png", "path": str(ndvi_png)})
                print(f"[PREDICT+NDVI] Created PNG: {ndvi_png}")
            except Exception as e:
                print(f"[PREDICT+NDVI] PNG creation failed: {e}")
        
        # Export classification if requested
        if config.export_classification:
            class_file = output_dir / f"classification_{timestamp}.tif"
            transform = from_bounds(bbox[0], bbox[1], bbox[2], bbox[3], width, height)
            
            with rasterio.open(
                class_file, 'w',
                driver='GTiff',
                height=height,
                width=width,
                count=1,
                dtype=prediction_raster.dtype,
                crs='EPSG:4326',
                transform=transform
            ) as dst:
                dst.write(prediction_raster, 1)
            
            output_files.append({"type": "classification", "path": str(class_file)})
            print(f"[PREDICT+NDVI] Saved classification to {class_file}")
            
            # Create PNG preview for classification
            class_png = output_dir / f"classification_{timestamp}.png"
            try:
                import matplotlib
                matplotlib.use('Agg')
                import matplotlib.pyplot as plt
                from matplotlib.patches import Patch
                
                # ============ SHAPEFILE OVERLAY ============
                overlay_mask = None
                if config.shapefile_overlay:
                    print(f"[OVERLAY] Shapefile overlay requested: {config.shapefile_overlay}")
                    
                    # Validate shapefile path exists
                    shapefile_path = Path(config.shapefile_overlay)
                    if not shapefile_path.exists():
                        print(f"[OVERLAY ERROR] Shapefile not found: {shapefile_path}")
                        print(f"[OVERLAY ERROR] Absolute path: {shapefile_path.absolute()}")
                    else:
                        print(f"[OVERLAY] Shapefile exists: {shapefile_path.absolute()}")
                        try:
                            # Import rioxarray for rio accessor
                            import rioxarray
                            
                            # Create temporary DataArray for rasterization
                            temp_da = xr.DataArray(
                                prediction_raster,
                                coords={
                                    "y": np.linspace(bbox[3], bbox[1], height),
                                    "x": np.linspace(bbox[0], bbox[2], width)
                                },
                                dims=["y", "x"]
                            )
                            temp_da.rio.write_crs("EPSG:4326", inplace=True)
                            temp_da.rio.write_transform(transform, inplace=True)
                            
                            overlay_mask = rasterize_shapefile_overlay(str(shapefile_path), temp_da)
                            if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
                                print(f"[OVERLAY] Successfully rasterized shapefile boundaries ({np.count_nonzero(overlay_mask)} pixels)")
                            else:
                                print(f"[OVERLAY WARNING] Shapefile rasterized but no boundary pixels found")
                        except Exception as overlay_error:
                            print(f"[OVERLAY ERROR] Exception: {overlay_error}")
                            import traceback
                            traceback.print_exc()
                else:
                    print(f"[OVERLAY] No shapefile overlay requested")
                
                fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
                im = ax.imshow(prediction_raster, cmap='tab20', interpolation='nearest')
                
                # Overlay shapefile boundaries if available
                if overlay_mask is not None and np.count_nonzero(overlay_mask) > 0:
                    print(f"[PNG OVERLAY] Overlaying {np.count_nonzero(overlay_mask)} boundary pixels (1px thin line)")
                    
                    # Create a mask for boundaries
                    boundary_mask = overlay_mask > 0
                    
                    # Create RGBA overlay image - thin 1px white line only
                    overlay_rgba = np.zeros((*prediction_raster.shape, 4))
                    overlay_rgba[boundary_mask, 0] = 1.0  # White (R=1)
                    overlay_rgba[boundary_mask, 1] = 1.0  # White (G=1)
                    overlay_rgba[boundary_mask, 2] = 1.0  # White (B=1)
                    overlay_rgba[boundary_mask, 3] = 1.0  # Fully opaque
                    
                    ax.imshow(overlay_rgba, interpolation='nearest')
                    
                    print(f"[PNG OVERLAY] Added thin 1px shapefile boundaries to visualization")
                else:
                    print(f"[PNG OVERLAY] No overlay mask or empty mask")
                
                ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
                ax.set_xlabel('X (pixels)', fontsize=11)
                ax.set_ylabel('Y (pixels)', fontsize=11)
                
                # Try to get class labels - ALWAYS use DEFAULT_LABEL_NAMES
                class_map = DEFAULT_LABEL_NAMES.copy()
                
                # Get unique classes in prediction
                unique_pred_classes = np.unique(prediction_raster)
                unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
                unique_pred_classes = unique_pred_classes[unique_pred_classes >= 0]  # Exclude -1
                
                # Create custom legend with color patches
                legend_elements = []
                cmap = plt.cm.get_cmap('tab20')
                
                for cls_val in sorted(unique_pred_classes):
                    try:
                        cls_int = int(cls_val)
                        color = cmap(cls_int / 20.0)
                        label_name = class_map.get(cls_int, f"Class {cls_int}")
                        legend_elements.append(
                            Patch(facecolor=color, edgecolor='black', linewidth=0.5,
                                  label=f"{cls_int}: {label_name}")
                        )
                    except:
                        pass
                
                # Add legend outside plot area
                if legend_elements:
                    legend = ax.legend(
                        handles=legend_elements,
                        loc='center left',
                        bbox_to_anchor=(1.02, 0.5),
                        fontsize=10,
                        title='Land Classes',
                        title_fontsize=11,
                        framealpha=0.9,
                        edgecolor='black'
                    )
                    legend.get_title().set_fontweight('bold')
                
                ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
                
                plt.tight_layout()
                plt.savefig(str(class_png), dpi=150, bbox_inches='tight')
                plt.close(fig)
                
                output_files.append({"type": "classification_png", "path": str(class_png)})
                print(f"[PREDICT+NDVI] Created PNG: {class_png}")
            except Exception as e:
                print(f"[PREDICT+NDVI] PNG creation failed: {e}")
        
        # Calculate statistics
        ndvi_stats = {
            "mean": float(np.nanmean(ndvi_mean)),
            "min": float(np.nanmin(ndvi_mean)),
            "max": float(np.nanmax(ndvi_mean)),
            "std": float(np.nanstd(ndvi_mean))
        }
        
        # Count classes
        unique_classes, counts = np.unique(predictions, return_counts=True)
        class_distribution = {
            int(cls): int(count) for cls, count in zip(unique_classes, counts)
        }

        # Auto-save prediction cache (Sentinel-2 data + metadata) for this NDVI workflow
        try:
            cache_config = {
                "min_lon": config.min_lon,
                "min_lat": config.min_lat,
                "max_lon": config.max_lon,
                "max_lat": config.max_lat,
                "start_date": config.start_date,
                "end_date": config.end_date,
                "max_scenes": config.max_scenes,
                "cloud_cover": config.cloud_cover,
                "resolution": config.resolution,
                "model_filename": config.model_filename
            }

            # Always save data for predict_with_ndvi (so lần sau không phải tải lại)
            data_to_save = data
            cache_result = save_prediction_cache_sync(cache_config, data_to_save)
            print(f"[PREDICT+NDVI][CACHE] Saved cache: {cache_result.get('filename')} (with data: {data_to_save is not None})")
        except Exception as cache_exc:
            print(f"[PREDICT+NDVI][CACHE ERROR] {cache_exc}")
        
        return {
            "success": True,
            "message": "Prediction with NDVI completed",
            "output_files": output_files,
            "ndvi_stats": ndvi_stats,
            "class_distribution": class_distribution,
            "n_scenes": len(items),
            "resolution": config.resolution,
            "bbox": bbox,
            "change_detection": change_summary
        }
    
    except HTTPException:
        # Re-raise HTTPException with original status code (e.g., 504 for timeout)
        raise
    except Exception as e:
        print(f"[PREDICT+NDVI ERROR] {str(e)}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=str(e))


# ============ NDVI TIME SERIES API ============

@app.post("/api/ndvi/timeseries")
async def calculate_ndvi_timeseries(config: NDVIConfig):
    """Tính NDVI time series cho một khu vực"""
    try:
        import numpy as np
        import xarray as xr
        from pystac_client import Client
        import planetary_computer
        import odc.stac
        
        print(f"[NDVI] Starting calculation for bbox: {config.bbox}, time: {config.start_date} to {config.end_date}")
        
        # Connect to Microsoft Planetary Computer STAC API
        catalog = Client.open(
            "https://planetarycomputer.microsoft.com/api/stac/v1",
            modifier=planetary_computer.sign_inplace
        )
        
        bbox = config.bbox
        time_range = f"{config.start_date}/{config.end_date}"
        
        # Search for Sentinel-2 data with retry logic
        items = fetch_sentinel_items_with_retry(
            catalog=catalog,
            bbox=bbox,
            time_range=time_range,
            cloud_cover=config.max_cloud_cover,
            max_scenes=1000,  # Get all available scenes for time series
            max_retries=3
        )
        print(f"[NDVI] Found {len(items)} Sentinel-2 scenes")
        
        if len(items) == 0:
            raise HTTPException(status_code=404, detail="Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
        
        # Load data for each time step
        ndvi_timeseries = []
        
        for item in items:
            try:
                # Load NIR (B08) and Red (B04) bands
                data = odc.stac.load(
                    [item],
                    bbox=bbox,
                    bands=["B04", "B08"],  # Red and NIR
                    resolution=config.resolution,
                    chunks={"x": 2048, "y": 2048}
                ).compute()
                
                if data is None or len(data.keys()) == 0:
                    continue
                
                # Calculate NDVI = (NIR - Red) / (NIR + Red)
                nir = data["B08"].values
                red = data["B04"].values
                
                # Avoid division by zero
                denominator = nir + red
                denominator = np.where(denominator == 0, np.nan, denominator)
                
                ndvi = (nir - red) / denominator
                
                # Calculate mean NDVI (ignore NaN values)
                mean_ndvi = float(np.nanmean(ndvi))
                
                # Get date from item
                date_str = item.datetime.strftime("%Y-%m-%d")
                
                ndvi_timeseries.append({
                    "date": date_str,
                    "ndvi": mean_ndvi
                })
                
                print(f"[NDVI] {date_str}: NDVI = {mean_ndvi:.3f}")
                
            except Exception as e:
                print(f"[NDVI WARNING] Failed to process item {item.id}: {e}")
                continue
        
        if len(ndvi_timeseries) == 0:
            raise HTTPException(status_code=500, detail="Không thể tính NDVI cho bất kỳ ảnh nào")
        
        # Sort by date
        ndvi_timeseries.sort(key=lambda x: x["date"])
        
        # Calculate statistics
        ndvi_values = [item["ndvi"] for item in ndvi_timeseries]
        mean_ndvi = float(np.mean(ndvi_values))
        min_ndvi = float(np.min(ndvi_values))
        max_ndvi = float(np.max(ndvi_values))
        
        result = {
            "timeseries": ndvi_timeseries,
            "n_images": len(ndvi_timeseries),
            "mean_ndvi": mean_ndvi,
            "min_ndvi": min_ndvi,
            "max_ndvi": max_ndvi,
            "bbox": bbox,
            "time_range": time_range
        }
        
        print(f"[NDVI] Calculation complete. Mean NDVI: {mean_ndvi:.3f}, Images: {len(ndvi_timeseries)}")
        
        return result
        
    except HTTPException:
        raise
    except Exception as e:
        print(f"[NDVI ERROR] {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Lỗi khi tính NDVI: {str(e)}")


# ===========================
# NDVI PREDICTION TIME SERIES
# ===========================

class NDVIPredictionConfig(BaseModel):
    """Cấu hình NDVI prediction time series"""
    model_filename: str
    bbox: List[float]  # [min_lon, min_lat, max_lon, max_lat]
    start_date: str
    end_date: str
    max_cloud_cover: int = 30
    max_scenes: int = 12  # Số lượng scenes tối đa
    resolution: int = 20
    sample_points: int = 1000  # Số điểm ngẫu nhiên để predict
    use_gpu: bool = False


@app.post("/api/ndvi/predict-timeseries")
async def ndvi_predict_timeseries(config: NDVIPredictionConfig):
    """
    Predict land classification tại các điểm ngẫu nhiên trong bbox theo time series
    và tính NDVI trung bình cho mỗi class theo thời gian
    """
    try:
        print(f"\n{'='*70}")
        print(f"[NDVI PREDICTION TIME SERIES] Starting...")
        print(f"  Model: {config.model_filename}")
        print(f"  Bbox: {config.bbox}")
        print(f"  Time: {config.start_date} → {config.end_date}")
        print(f"  Sample points: {config.sample_points}")
        print(f"  GPU: {config.use_gpu}")
        print(f"{'='*70}\n")
        
        # Load model
        model_path = Path("model_train") / config.model_filename
        if not model_path.exists():
            raise HTTPException(status_code=404, detail=f"Model not found: {config.model_filename}")
        
        print(f"📂 Loading model using ModelManager...")
        
        # Load model using ModelManager to get metadata
        from model_manager import get_model_manager
        model_manager = get_model_manager()
        model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
        
        # Get feature info from metadata
        feature_mode = model_metadata.get("feature_mode", "simple")
        n_features_expected = model_metadata.get("n_features", 0)
        
        print(f"  Model type: {model_metadata.get('model_type', 'unknown')}")
        print(f"  Feature mode: {feature_mode}")
        print(f"  Expected features: {n_features_expected}")
        
        # Check if PyTorch model for GPU
        is_pytorch_model = hasattr(model, 'forward') or str(type(model).__name__) in ['SwinUnet', 'CNN', 'MobileNetLRASPPClassifier']
        
        if is_pytorch_model and config.use_gpu:
            import torch
            if torch.cuda.is_available():
                print(f"🚀 Moving model to GPU...")
                model = model.to('cuda')
                model.eval()
            else:
                print(f"⚠️  GPU not available, using CPU")
        
        # Setup bbox
        min_lon, min_lat, max_lon, max_lat = config.bbox
        bbox = [min_lon, min_lat, max_lon, max_lat]
        time_range = f"{config.start_date}/{config.end_date}"
        
        # Check network connectivity first
        print(f"\n🌐 Checking network connectivity...")
        is_connected, conn_error = check_internet_connectivity()
        if not is_connected:
            raise HTTPException(
                status_code=503,
                detail=f"❌ {conn_error}\n\n"
                       f"Please check:\n"
                       f"1. Your internet connection is active\n"
                       f"2. DNS servers are configured correctly\n"
                       f"3. Firewall/proxy settings allow outbound connections"
            )
        
        is_accessible, access_error = check_planetary_computer_access()
        if not is_accessible:
            print(f"⚠️  {access_error}")
            print(f"   Will attempt to load data anyway (may fail)...")
        
        # Load Sentinel-2 time series using Planetary Computer
        print(f"\n📡 Loading Sentinel-2 data from Planetary Computer...")
        
        # Import required libraries
        import pystac_client
        import planetary_computer
        from odc.stac import load
        import pandas as pd
        
        catalog = pystac_client.Client.open(
            "https://planetarycomputer.microsoft.com/api/stac/v1",
            modifier=planetary_computer.sign_inplace,
        )
        
        # Search for Sentinel-2 data with retry
        s2_items = fetch_sentinel_items_with_retry(
            catalog=catalog,
            bbox=bbox,
            time_range=time_range,
            cloud_cover=config.max_cloud_cover,
            max_scenes=config.max_scenes,
            max_retries=3
        )
        
        print(f"✅ Found {len(s2_items)} Sentinel-2 scenes (limited to {config.max_scenes})")
        
        # Load data with retry logic for network errors
        max_retries = 3
        retry_delay = 2
        s2_data = None
        
        for attempt in range(max_retries):
            try:
                print(f"📥 Loading Sentinel-2 data (attempt {attempt + 1}/{max_retries})...")
                s2_data = load(
                    s2_items,
                    bbox=bbox,
                    bands=["B02", "B03", "B04", "B05", "B06", "B07", "B08", "B11", "B12", "SCL"],
                    chunks={"time": 1, "x": 2048, "y": 2048},
                    groupby="solar_day",
                    resolution=config.resolution
                ).compute()
                
                print(f"✅ Loaded Sentinel-2 data with {len(s2_data.time)} time steps")
                break
                
            except Exception as e:
                error_msg = str(e)
                
                # Check for various error types that might indicate problematic scenes
                is_network_error = any(err in error_msg for err in [
                    "Could not resolve host", "CURL error", "Connection", "Timeout"
                ])
                is_access_error = any(err in error_msg for err in [
                    "does not exist in the file system", "RasterioIOError", 
                    "not recognized as a supported dataset", "Aborting load"
                ])
                is_auth_error = any(err in error_msg for err in [
                    "AuthenticationFailed", "Signature fields not well formed",
                    "Server failed to authenticate", "403", "401"
                ])
                
                if is_auth_error:
                    print(f"⚠️  Authentication error detected. Re-signing items and retrying...")
                    # Re-sign all items with fresh tokens
                    try:
                        import planetary_computer
                        s2_items = [planetary_computer.sign(item) for item in s2_items]
                        print(f"   Re-signed {len(s2_items)} items")
                        
                        if attempt < max_retries - 1:
                            import time
                            time.sleep(1)  # Brief pause before retry
                            continue
                        else:
                            raise HTTPException(
                                status_code=500,
                                detail=f"❌ Authentication failed after {max_retries} attempts.\n\n"
                                       f"Azure Blob Storage authentication is failing. This may be due to:\n"
                                       f"1. Expired or invalid SAS tokens from Planetary Computer\n"
                                       f"2. Planetary Computer API service issues\n"
                                       f"3. System clock synchronization issues (check date/time)\n\n"
                                       f"Please try again in a few minutes. If the problem persists,\n"
                                       f"Microsoft Planetary Computer may be experiencing issues.\n\n"
                                       f"Error: {error_msg[:300]}"
                            )
                    except Exception as resign_error:
                        print(f"   Re-signing failed: {resign_error}")
                        if attempt == max_retries - 1:
                            raise HTTPException(
                                status_code=500,
                                detail=f"Failed to re-sign authentication tokens: {str(resign_error)}"
                            )
                
                elif is_access_error and len(s2_items) > 5:
                    # If we have access errors and multiple scenes, try with fewer scenes
                    print(f"⚠️  Data access error detected. Reducing scene count and retrying...")
                    # Remove last 20% of scenes and try again
                    remove_count = max(1, len(s2_items) // 5)
                    s2_items = s2_items[:-remove_count]
                    print(f"   Reduced to {len(s2_items)} scenes")
                    
                    if attempt < max_retries - 1:
                        continue
                    else:
                        # Final attempt with reduced scenes
                        print(f"   Final attempt with {len(s2_items)} scenes...")
                        
                elif is_network_error:
                    print(f"⚠️  Network error on attempt {attempt + 1}: {error_msg[:100]}")
                    
                    # Re-check connectivity on network errors
                    if "Could not resolve host" in error_msg:
                        is_conn, conn_msg = check_internet_connectivity()
                        if not is_conn:
                            raise HTTPException(
                                status_code=503,
                                detail=f"❌ Network connectivity lost!\n\n"
                                       f"{conn_msg}\n\n"
                                       f"Please:\n"
                                       f"1. Check your internet connection\n"
                                       f"2. Verify DNS settings (try 8.8.8.8 or 1.1.1.1)\n"
                                       f"3. Check firewall/proxy settings\n"
                                       f"4. Try again once connection is restored\n\n"
                                       f"Original error: {error_msg[:150]}"
                            )
                    
                    if attempt < max_retries - 1:
                        import time
                        print(f"   Retrying in {retry_delay} seconds...")
                        time.sleep(retry_delay)
                        retry_delay *= 2  # Exponential backoff
                    else:
                        raise HTTPException(
                            status_code=503,
                            detail=f"❌ Network error persists after {max_retries} attempts.\n\n"
                                   f"Unable to download Sentinel-2 data from Microsoft Planetary Computer.\n\n"
                                   f"Possible causes:\n"
                                   f"1. Unstable internet connection\n"
                                   f"2. DNS resolution issues\n"
                                   f"3. Planetary Computer service temporarily unavailable\n"
                                   f"4. Firewall blocking access to sentinel2l2a01.blob.core.windows.net\n\n"
                                   f"Please check your network and try again.\n\n"
                                   f"Error: {error_msg[:200]}"
                        )
                else:
                    # If still failing after retries, try with minimal data
                    if attempt == max_retries - 1 and len(s2_items) > 3:
                        print(f"⚠️  Multiple failures. Attempting with minimal scenes (3)...")
                        s2_items = s2_items[:3]
                        continue
                    else:
                        # Last resort: provide meaningful error
                        raise HTTPException(
                            status_code=500,
                            detail=f"Failed to load Sentinel-2 data. This may be due to expired access tokens "
                                   f"or unavailable data. Please try a different time range or area. "
                                   f"Error: {error_msg[:300]}"
                        )
        
        if s2_data is None:
            raise HTTPException(status_code=500, detail="Failed to load Sentinel-2 data")
        
        # Generate random sample points
        print(f"\n🎲 Generating {config.sample_points} random sample points...")
        np.random.seed(42)
        
        lats = np.random.uniform(min_lat, max_lat, config.sample_points)
        lons = np.random.uniform(min_lon, max_lon, config.sample_points)
        
        # EXTRACT AGGREGATE FEATURES (same as training in 01.train_ODC.ipynb)
        # Model expects: ndvi_mean, ndvi_min, ndvi_max, ndvi_std, ndvi_range, ndwi_mean, ndbi_mean, evi_mean
        
        print(f"\n🔍 Extracting aggregate features from time series for {config.sample_points} sample points...")
        print(f"   (Matching training methodology from 01.train_ODC.ipynb)")
        
        # Calculate spectral indices for all time steps
        print(f"\n📊 Calculating spectral indices...")
        
        # NDVI = (NIR - Red) / (NIR + Red)
        nir = s2_data['B08'].astype(float)
        red = s2_data['B04'].astype(float)
        ndvi = (nir - red) / (nir + red + 1e-8)
        
        # NDWI = (Green - NIR) / (Green + NIR)
        green = s2_data['B03'].astype(float)
        ndwi = (green - nir) / (green + nir + 1e-8)
        
        # NDBI = (SWIR - NIR) / (SWIR + NIR)
        swir = s2_data['B11'].astype(float)
        ndbi = (swir - nir) / (swir + nir + 1e-8)
        
        # EVI = 2.5 * (NIR - Red) / (NIR + 6*Red - 7.5*Blue + 1)
        blue = s2_data['B02'].astype(float)
        evi = 2.5 * (nir - red) / (nir + 6*red - 7.5*blue + 1)
        
        print(f"✅ Calculated NDVI, NDWI, NDBI, EVI for {len(s2_data.time)} time steps")
        
        # Extract features at sample points
        print(f"\n🎯 Extracting aggregate features at sample points...")
        all_point_features = []
        all_point_metadata = []
        
        for i, (lat, lon) in enumerate(zip(lats, lons)):
            try:
                # Extract time series for this point
                point_ndvi = ndvi.sel(y=lat, x=lon, method='nearest').values
                point_ndwi = ndwi.sel(y=lat, x=lon, method='nearest').values
                point_ndbi = ndbi.sel(y=lat, x=lon, method='nearest').values
                point_evi = evi.sel(y=lat, x=lon, method='nearest').values
                point_scl = s2_data['SCL'].sel(y=lat, x=lon, method='nearest').values
                
                # Mask out cloud/no data values
                valid_mask = ~np.isin(point_scl, [3, 8, 9, 10, 0, 1])
                
                if not valid_mask.any():
                    # All time steps are invalid
                    continue
                
                # Calculate aggregate features (same as training)
                features = [
                    float(np.nanmean(point_ndvi[valid_mask])),  # ndvi_mean
                    float(np.nanmin(point_ndvi[valid_mask])),   # ndvi_min
                    float(np.nanmax(point_ndvi[valid_mask])),   # ndvi_max
                    float(np.nanstd(point_ndvi[valid_mask])),   # ndvi_std
                    float(np.nanmax(point_ndvi[valid_mask]) - np.nanmin(point_ndvi[valid_mask])),  # ndvi_range
                    float(np.nanmean(point_ndwi[valid_mask])),  # ndwi_mean
                    float(np.nanmean(point_ndbi[valid_mask])),  # ndbi_mean
                    float(np.nanmean(point_evi[valid_mask]))    # evi_mean
                ]
                
                # Skip if any NaN values
                if not np.isnan(features).any():
                    all_point_features.append(features)
                    all_point_metadata.append({'lat': lat, 'lon': lon, 'idx': i})
                
            except Exception as e:
                # Skip problematic points
                continue
        
        if len(all_point_features) == 0:
            raise HTTPException(
                status_code=404,
                detail="❌ Không có điểm hợp lệ. Tất cả sample points bị mây che hoặc no data."
            )
        
        # Convert to array
        X = np.array(all_point_features)
        
        print(f"✅ Extracted aggregate features for {len(all_point_features)} valid points")
        print(f"   Feature shape: {X.shape} (points, features)")
        print(f"   Features: ndvi_mean, ndvi_min, ndvi_max, ndvi_std, ndvi_range, ndwi_mean, ndbi_mean, evi_mean")
        
        # Verify feature count
        if X.shape[1] != n_features_expected:
            print(f"⚠️  Feature mismatch: got {X.shape[1]}, expected {n_features_expected}")
            if X.shape[1] < n_features_expected:
                # Pad with zeros
                padding = np.zeros((X.shape[0], n_features_expected - X.shape[1]))
                X = np.hstack([X, padding])
                print(f"   → Padded to {X.shape[1]} features")
            else:
                # Truncate
                X = X[:, :n_features_expected]
                print(f"   → Truncated to {X.shape[1]} features")
        
        # Predict land use classification
        print(f"\n🤖 Predicting land use classification...")
        
        if is_pytorch_model and config.use_gpu:
            import torch
            with torch.no_grad():
                X_tensor = torch.FloatTensor(X).to('cuda')
                predictions = model.predict(X)
        else:
            predictions = model.predict(X)
        
        # Decode labels if needed
        if label_encoder is not None:
            try:
                predictions = label_encoder.inverse_transform(predictions.astype(int))
            except:
                pass
        
        print(f"✅ Predicted {len(predictions)} points")
        
        # Count class distribution
        unique_classes, class_counts = np.unique(predictions, return_counts=True)
        print(f"\n📊 Class distribution:")
        for cls, count in zip(unique_classes, class_counts):
            print(f"   Class {cls}: {count} points ({count/len(predictions)*100:.1f}%)")
        
        # Generate time series data by calculating NDVI at each time step
        print(f"\n⏱️  Generating time series data for {len(s2_data.time)} time steps...")
        timeseries_data = []
        
        for time_idx, time_val in enumerate(s2_data.time.values):
            # Calculate NDVI for this time step
            nir_t = s2_data['B08'].isel(time=time_idx)
            red_t = s2_data['B04'].isel(time=time_idx)
            ndvi_t = (nir_t - red_t) / (nir_t + red_t + 1e-8)
            
            # Extract NDVI values at valid points
            ndvi_values_at_points = []
            class_ndvi = {}
            
            for meta_idx, metadata in enumerate(all_point_metadata):
                lat, lon = metadata['lat'], metadata['lon']
                pred_class = predictions[meta_idx]
                
                try:
                    ndvi_val = float(ndvi_t.sel(y=lat, x=lon, method='nearest').values)
                    if not np.isnan(ndvi_val):
                        ndvi_values_at_points.append(ndvi_val)
                        
                        # Group by class
                        if pred_class not in class_ndvi:
                            class_ndvi[pred_class] = []
                        class_ndvi[pred_class].append(ndvi_val)
                except:
                    continue
            
            # Calculate class-wise NDVI statistics
            class_ndvi_stats = {}
            for cls, ndvi_vals in class_ndvi.items():
                if len(ndvi_vals) > 0:
                    class_ndvi_stats[int(cls)] = {
                        'mean_ndvi': float(np.mean(ndvi_vals)),
                        'min_ndvi': float(np.min(ndvi_vals)),
                        'max_ndvi': float(np.max(ndvi_vals)),
                        'std_ndvi': float(np.std(ndvi_vals)),
                        'count': len(ndvi_vals)
                    }
            
            # Overall statistics for this time step
            if len(ndvi_values_at_points) > 0:
                timeseries_data.append({
                    'date': pd.Timestamp(time_val).strftime('%Y-%m-%d'),
                    'mean_ndvi': float(np.mean(ndvi_values_at_points)),
                    'min_ndvi': float(np.min(ndvi_values_at_points)),
                    'max_ndvi': float(np.max(ndvi_values_at_points)),
                    'std_ndvi': float(np.std(ndvi_values_at_points)),
                    'class_distribution': {int(k): int(v) for k, v in zip(unique_classes, class_counts)},
                    'class_ndvi': class_ndvi_stats,
                    'n_valid_points': len(ndvi_values_at_points)
                })
                
                if time_idx % 5 == 0 or time_idx == len(s2_data.time) - 1:
                    print(f"   [{time_idx + 1:2d}/{len(s2_data.time)}] {pd.Timestamp(time_val).strftime('%Y-%m-%d')}: NDVI={np.mean(ndvi_values_at_points):.3f}")
        
        # Check if we have any valid data
        if not timeseries_data:
            # Provide detailed suggestions
            suggestions = [
                "📅 Chọn mùa khô (Tháng 1-4): ít mây hơn, dữ liệu tốt hơn",
                "🗺️ Thử khu vực khác: Đồng bằng sông Cửu Long [105.6, 9.3, 106.2, 9.8]",
                "☁️ Tăng max_cloud_cover lên 50-80% (hiện tại: {}%)".format(config.max_cloud_cover),
                "📸 Tăng max_scenes lên 20-30 (hiện tại: {})".format(config.max_scenes),
                "📍 Giảm số sample_points xuống 500 để test nhanh",
                "🌍 Khu vực đề xuất: Hà Nội [105.7, 20.9, 105.9, 21.1], Đà Nẵng [107.9, 15.9, 108.3, 16.2]"
            ]
            raise HTTPException(
                status_code=404, 
                detail=f"❌ Không có dữ liệu hợp lệ. Tất cả điểm đều bị che phủ bởi mây/NaN.\n\n💡 Gợi ý:\n" + "\n".join(f"  {i+1}. {s}" for i, s in enumerate(suggestions))
            )
        
        # Overall statistics
        all_ndvi_values = [item['mean_ndvi'] for item in timeseries_data]
        
        result = {
            'timeseries': timeseries_data,
            'n_images': len(timeseries_data),
            'mean_ndvi': float(np.mean(all_ndvi_values)),
            'min_ndvi': float(np.min(all_ndvi_values)),
            'max_ndvi': float(np.max(all_ndvi_values)),
            'std_ndvi': float(np.std(all_ndvi_values)),
            'bbox': config.bbox,
            'model_used': config.model_filename,
            'sample_points': config.sample_points,
            'date_range': f"{config.start_date} to {config.end_date}"
        }
        
        print(f"\n{'='*70}")
        print(f"✅ NDVI Prediction Time Series completed!")
        print(f"  Total scenes: {len(timeseries_data)}")
        print(f"  Mean NDVI: {result['mean_ndvi']:.3f}")
        print(f"  NDVI range: [{result['min_ndvi']:.3f}, {result['max_ndvi']:.3f}]")
        print(f"{'='*70}\n")

        # Auto-save prediction cache (Sentinel-2 data + metadata) similar to rice predict
        try:
            cache_config = {
                "min_lon": config.bbox[0],
                "min_lat": config.bbox[1],
                "max_lon": config.bbox[2],
                "max_lat": config.bbox[3],
                "start_date": config.start_date,
                "end_date": config.end_date,
                "max_scenes": config.max_scenes,
                "cloud_cover": config.max_cloud_cover,
                "resolution": config.resolution,
                "model_filename": config.model_filename
            }

            cache_result = save_prediction_cache_sync(cache_config, s2_data)
            print(f"[NDVI TIMESERIES][CACHE] Saved cache: {cache_result.get('filename')} (with data: {s2_data is not None})")
        except Exception as cache_exc:
            print(f"[NDVI TIMESERIES][CACHE ERROR] {cache_exc}")
        
        return result
        
    except HTTPException:
        raise
    except Exception as e:
        print(f"[NDVI PREDICTION ERROR] {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Lỗi khi predict NDVI time series: {str(e)}")


class NDVIForecastConfig(BaseModel):
    """Configuration for NDVI forecasting"""
    bbox: List[float]  # [min_lon, min_lat, max_lon, max_lat]
    forecast_start_date: str  # Start date for forecast (can be future)
    forecast_end_date: str  # End date for forecast
    historical_months: int = 12  # Number of historical months to use for pattern
    resolution: int = 20
    max_cloud_cover: int = 30
    max_scenes: int = 20
    model_filename: Optional[str] = None  # Optional: use ML model for land-type-specific forecasting
    sample_points: int = 1000  # Number of sample points for classification


@app.post("/api/ndvi/forecast")
async def ndvi_forecast(config: NDVIForecastConfig):
    """
    Dự đoán NDVI tương lai dựa trên land-type-specific seasonal patterns
    
    Method: Land-Type-Specific Forecasting
    1. Sử dụng ML model để classify land types từ dữ liệu lịch sử
    2. Tính seasonal pattern riêng cho từng loại đất
    3. Forecast dựa trên pattern của land type tương ứng
    
    Advantages:
    - Chính xác hơn seasonal averaging đơn thuần (75-85% vs 60-70%)
    - Tận dụng model classification đã được train
    - Phản ánh đúng đặc điểm của từng loại đất (lúa vs rừng vs đô thị)
    """
    try:
        print(f"\n{'='*70}")
        print(f"[NDVI FORECAST] Starting Land-Type-Specific Forecasting...")
        print(f"  Bbox: {config.bbox}")
        print(f"  Forecast period: {config.forecast_start_date} → {config.forecast_end_date}")
        print(f"  Historical lookback: {config.historical_months} months")
        print(f"  Model: {config.model_filename or 'None (simple seasonal)'}")
        print(f"{'='*70}\n")
        
        import pandas as pd
        from dateutil.relativedelta import relativedelta
        
        # Parse forecast dates
        forecast_start = pd.to_datetime(config.forecast_start_date)
        forecast_end = pd.to_datetime(config.forecast_end_date)
        
        # Calculate historical period
        historical_end = forecast_start - relativedelta(days=1)
        historical_start = historical_end - relativedelta(months=config.historical_months)
        
        # Validate historical period (Sentinel-2 available from 2015-06-23)
        sentinel2_start = pd.to_datetime("2015-06-23")
        if historical_start < sentinel2_start:
            print(f"⚠️ WARNING: Historical start {historical_start.date()} is before Sentinel-2 availability (2015-06-23)")
            print(f"   Adjusting to use data from 2015-06-23 onwards...")
            historical_start = sentinel2_start
        
        print(f"📅 Using historical data: {historical_start.date()} → {historical_end.date()}")
        print(f"   ({(historical_end - historical_start).days} days / {(historical_end - historical_start).days / 30:.1f} months)")
        
        # Setup bbox
        min_lon, min_lat, max_lon, max_lat = config.bbox
        bbox = [min_lon, min_lat, max_lon, max_lat]
        time_range = f"{historical_start.date()}/{historical_end.date()}"
        
        # Load historical Sentinel-2 data
        print(f"\n📡 Loading historical Sentinel-2 data...")
        
        import pystac_client
        import planetary_computer
        from odc.stac import load
        
        catalog = pystac_client.Client.open(
            "https://planetarycomputer.microsoft.com/api/stac/v1",
            modifier=planetary_computer.sign_inplace,
        )
        
        # Search for historical Sentinel-2 data with retry
        try:
            s2_items = fetch_sentinel_items_with_retry(
                catalog=catalog,
                bbox=bbox,
                time_range=time_range,
                cloud_cover=config.max_cloud_cover,
                max_scenes=config.max_scenes,
                max_retries=3
            )
        except ValueError:
            # No items found - provide helpful error message
            raise HTTPException(
                status_code=404, 
                detail=f"⚠️ Không tìm thấy dữ liệu Sentinel-2 cho khu vực này!\n\n"
                       f"📅 Đang tìm dữ liệu lịch sử: {historical_start.date()} → {historical_end.date()}\n"
                       f"🗺️ Bbox: [{bbox[0]:.4f}, {bbox[1]:.4f}, {bbox[2]:.4f}, {bbox[3]:.4f}]\n"
                       f"☁️ Max cloud cover: {config.max_cloud_cover}%\n\n"
                       f"💡 Giải pháp:\n"
                       f"1. Sentinel-2 chỉ có từ 2015 → nay. Khoảng thời gian lịch sử phải sau 2015.\n"
                       f"2. Tăng 'Số tháng lịch sử' (historical_months) lên 24-36 tháng\n"
                       f"3. Tăng max_cloud_cover lên 50-80% để lấy nhiều ảnh hơn\n"
                       f"4. Chọn khu vực có dữ liệu tốt hơn (tránh vùng biển/núi cao)\n"
                       f"5. Đảm bảo forecast_start_date không quá xa trong tương lai"
            )
        
        print(f"✅ Found {len(s2_items)} historical Sentinel-2 scenes")
        
        print(f"✅ Found {len(s2_items)} historical scenes")
        
        # Load data with retry logic for network and access errors
        max_retries = 3
        retry_delay = 2
        s2_data = None
        
        for attempt in range(max_retries):
            try:
                print(f"📥 Loading Sentinel-2 data (attempt {attempt + 1}/{max_retries})...")
                s2_data = load(
                    s2_items,
                    bbox=bbox,
                    bands=["B02", "B03", "B04", "B05", "B08", "B11", "SCL"],
                    chunks={"time": 1, "x": 2048, "y": 2048},
                    groupby="solar_day",
                    resolution=config.resolution
                ).compute()
                
                print(f"✅ Loaded {len(s2_data.time)} time steps")
                break
                
            except Exception as e:
                error_msg = str(e)
                
                # Check for various error types
                is_network_error = any(err in error_msg for err in [
                    "Could not resolve host", "CURL error", "Connection", "Timeout"
                ])
                is_access_error = any(err in error_msg for err in [
                    "does not exist in the file system", "RasterioIOError", 
                    "not recognized as a supported dataset", "Aborting load"
                ])
                is_auth_error = any(err in error_msg for err in [
                    "AuthenticationFailed", "Signature fields not well formed",
                    "Server failed to authenticate", "403", "401"
                ])
                
                if is_auth_error:
                    print(f"⚠️  Authentication error. Re-signing items...")
                    try:
                        import planetary_computer
                        s2_items = [planetary_computer.sign(item) for item in s2_items]
                        if attempt < max_retries - 1:
                            import time
                            time.sleep(1)
                            continue
                        else:
                            raise HTTPException(
                                status_code=500,
                                detail=f"Authentication failed: {error_msg[:200]}"
                            )
                    except Exception as resign_error:
                        if attempt == max_retries - 1:
                            raise HTTPException(
                                status_code=500,
                                detail=f"Failed to re-sign tokens: {str(resign_error)}"
                            )
                
                elif is_access_error and len(s2_items) > 5:
                    # If we have access errors and multiple scenes, try with fewer scenes
                    print(f"⚠️  Data access error detected. Reducing scene count and retrying...")
                    remove_count = max(1, len(s2_items) // 5)
                    s2_items = s2_items[:-remove_count]
                    print(f"   Reduced to {len(s2_items)} scenes")
                    
                    if attempt < max_retries - 1:
                        continue
                    else:
                        print(f"   Final attempt with {len(s2_items)} scenes...")
                        
                elif is_network_error:
                    print(f"⚠️  Network error on attempt {attempt + 1}: {error_msg[:100]}")
                    if attempt < max_retries - 1:
                        import time
                        print(f"   Retrying in {retry_delay} seconds...")
                        time.sleep(retry_delay)
                        retry_delay *= 2
                    else:
                        raise HTTPException(
                            status_code=503,
                            detail=f"Network error: Unable to download Sentinel-2 data after {max_retries} attempts. "
                                   f"Error: {error_msg[:200]}"
                        )
                else:
                    # Try with minimal scenes on final attempt
                    if attempt == max_retries - 1 and len(s2_items) > 3:
                        print(f"⚠️  Multiple failures. Attempting with minimal scenes (3)...")
                        s2_items = s2_items[:3]
                        continue
                    else:
                        raise HTTPException(
                            status_code=500,
                            detail=f"Failed to load Sentinel-2 data. This may be due to expired access tokens "
                                   f"or unavailable data. Please try a different time range or area. "
                                   f"Error: {error_msg[:300]}"
                        )
        
        if s2_data is None:
            raise HTTPException(status_code=500, detail="Failed to load Sentinel-2 data")
        
        # Calculate spectral indices
        print(f"\n📊 Calculating spectral indices...")
        
        nir = s2_data['B08'].astype(float)
        red = s2_data['B04'].astype(float)
        green = s2_data['B03'].astype(float)
        blue = s2_data['B02'].astype(float)
        swir = s2_data['B11'].astype(float)
        
        # NDVI
        ndvi = (nir - red) / (nir + red + 1e-8)
        
        # NDWI
        ndwi = (green - nir) / (green + nir + 1e-8)
        
        # NDBI
        ndbi = (swir - nir) / (swir + nir + 1e-8)
        
        # EVI
        evi = 2.5 * (nir - red) / (nir + 6*red - 7.5*blue + 1)
        
        # Mask cloud pixels
        scl = s2_data['SCL']
        # cloud_mask: shape (time, y, x) if scl is xarray.DataArray, else (time, y, x) ndarray
        # If scl is xarray.DataArray, cloud_mask will be xarray.DataArray, else numpy ndarray
        if hasattr(scl, 'isel'):
            cloud_mask = ~np.isin(scl, [3, 8, 9, 10, 0, 1])  # xarray.DataArray
        else:
            # scl is numpy ndarray, so cloud_mask is ndarray
            cloud_mask = ~np.isin(scl, [3, 8, 9, 10, 0, 1])
        
        # LAND-TYPE-SPECIFIC FORECASTING
        use_ml_classification = config.model_filename is not None
        force_simple_forecast = False
        
        if use_ml_classification:
            print(f"\n🤖 Using ML model for land-type-specific forecasting...")
            
            # Load model
            model_path = Path("model_train") / config.model_filename
            if not model_path.exists():
                raise HTTPException(status_code=404, detail=f"Model not found: {config.model_filename}")
            
            from model_manager import get_model_manager
            model_manager = get_model_manager()
            model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
            
            feature_mode = model_metadata.get("feature_mode", "odc")
            print(f"  Model type: {model_metadata.get('model_type', 'unknown')}")
            print(f"  Feature mode: {feature_mode}")
            
            # Generate sample points
            print(f"\n🎲 Generating {config.sample_points} sample points for classification...")
            np.random.seed(42)
            lats = np.random.uniform(min_lat, max_lat, config.sample_points)
            lons = np.random.uniform(min_lon, max_lon, config.sample_points)
            
            # Extract aggregate features for classification
            print(f"🔍 Extracting aggregate features for land classification...")
            point_features = []
            point_coords = []
            
            for i, (lat, lon) in enumerate(zip(lats, lons)):
                try:
                    # Extract time series
                    point_ndvi = ndvi.sel(y=lat, x=lon, method='nearest').values
                    point_ndwi = ndwi.sel(y=lat, x=lon, method='nearest').values
                    point_ndbi = ndbi.sel(y=lat, x=lon, method='nearest').values
                    point_evi = evi.sel(y=lat, x=lon, method='nearest').values
                    point_scl = s2_data['SCL'].sel(y=lat, x=lon, method='nearest').values
                    
                    # Mask valid values
                    valid_mask = ~np.isin(point_scl, [3, 8, 9, 10, 0, 1])
                    
                    if not valid_mask.any():
                        continue
                    
                    # Calculate aggregate features (matching training)
                    features = [
                        float(np.mean(point_ndvi[valid_mask])),  # ndvi_mean
                        float(np.min(point_ndvi[valid_mask])),   # ndvi_min
                        float(np.max(point_ndvi[valid_mask])),   # ndvi_max
                        float(np.std(point_ndvi[valid_mask])),   # ndvi_std
                        float(np.max(point_ndvi[valid_mask]) - np.min(point_ndvi[valid_mask])),  # ndvi_range
                        float(np.mean(point_ndwi[valid_mask])),  # ndwi_mean
                        float(np.mean(point_ndbi[valid_mask])),  # ndbi_mean
                        float(np.mean(point_evi[valid_mask]))    # evi_mean
                    ]
                    
                    if not any(np.isnan(features)):
                        point_features.append(features)
                        point_coords.append((lat, lon))
                        
                except:
                    continue
            
            if len(point_features) == 0:
                print("[NDVI FORECAST][FALLBACK] No valid classification points. Falling back to simple seasonal forecast.")
                force_simple_forecast = True
            else:
                print(f"✅ Extracted features for {len(point_features)} valid points")
                
                # Classify points
                X = np.array(point_features)
                predictions = model.predict(X)
                
                if label_encoder is not None:
                    try:
                        predictions = label_encoder.inverse_transform(predictions.astype(int))
                    except:
                        pass
                
                # Count land types
                unique_types, type_counts = np.unique(predictions, return_counts=True)
                print(f"\n📊 Detected land types:")
                for land_type, count in zip(unique_types, type_counts):
                    print(f"   Type {land_type}: {count} points ({count/len(predictions)*100:.1f}%)")
                
                # Calculate land-type-specific seasonal patterns
                print(f"\n📈 Calculating land-type-specific seasonal patterns...")
                
                land_type_patterns = {}
                
                for time_idx in range(len(s2_data.time)):
                    time_val = pd.Timestamp(s2_data.time.values[time_idx])
                    month = time_val.month
                    
                    # Get valid pixels for this time step
                    if hasattr(cloud_mask, 'isel'):
                        mask_t = cloud_mask.isel(time=time_idx)
                    else:
                        # Wrap numpy mask to xarray with same coords/dims as ndvi slice
                        ndvi_slice = ndvi.isel(time=time_idx)
                        mask_t = xr.DataArray(mask_t := cloud_mask[time_idx], coords=ndvi_slice.coords, dims=ndvi_slice.dims)

                    ndvi_t = ndvi.isel(time=time_idx).where(mask_t)
                    ndwi_t = ndwi.isel(time=time_idx).where(mask_t)
                    ndbi_t = ndbi.isel(time=time_idx).where(mask_t)
                    evi_t = evi.isel(time=time_idx).where(mask_t)
                    
                    # Extract values at classified points
                    for point_idx, (lat, lon) in enumerate(point_coords):
                        land_type = predictions[point_idx]
                        
                        try:
                            ndvi_val = float(ndvi_t.sel(y=lat, x=lon, method='nearest').values)
                            
                            if not np.isnan(ndvi_val):
                                ndwi_val = float(ndwi_t.sel(y=lat, x=lon, method='nearest').values)
                                ndbi_val = float(ndbi_t.sel(y=lat, x=lon, method='nearest').values)
                                evi_val = float(evi_t.sel(y=lat, x=lon, method='nearest').values)
                                
                                # Initialize land type if not exists
                                if land_type not in land_type_patterns:
                                    land_type_patterns[land_type] = {}
                                
                                if month not in land_type_patterns[land_type]:
                                    land_type_patterns[land_type][month] = {
                                        'ndvi': [], 'ndwi': [], 'ndbi': [], 'evi': []
                                    }
                                
                                # Append values
                                land_type_patterns[land_type][month]['ndvi'].append(ndvi_val)
                                land_type_patterns[land_type][month]['ndwi'].append(ndwi_val)
                                land_type_patterns[land_type][month]['ndbi'].append(ndbi_val)
                                land_type_patterns[land_type][month]['evi'].append(evi_val)
                        except:
                            continue
                
                # Calculate statistics for each land type and month
                land_type_seasonal_stats = {}
                
                for land_type, month_data in land_type_patterns.items():
                    land_type_seasonal_stats[land_type] = {}
                    
                    for month, values in month_data.items():
                        ndvi_vals = values['ndvi']
                        
                        if len(ndvi_vals) > 0:
                            land_type_seasonal_stats[land_type][month] = {
                                'ndvi_mean': float(np.mean(ndvi_vals)),
                                'ndvi_min': float(np.min(ndvi_vals)),
                                'ndvi_max': float(np.max(ndvi_vals)),
                                'ndvi_std': float(np.std(ndvi_vals)),
                                'ndvi_range': float(np.max(ndvi_vals) - np.min(ndvi_vals)),
                                'ndwi_mean': float(np.mean(values['ndwi'])),
                                'ndbi_mean': float(np.mean(values['ndbi'])),
                                'evi_mean': float(np.mean(values['evi'])),
                                'n_samples': len(ndvi_vals)
                            }
                
                print(f"✅ Calculated patterns for {len(land_type_seasonal_stats)} land types")

                if len(land_type_seasonal_stats) == 0:
                    print("[NDVI FORECAST][FALLBACK] No seasonal patterns. Falling back to simple seasonal forecast.")
                    force_simple_forecast = True
                
                # Generate forecast using land-type-weighted average
                if force_simple_forecast:
                    print("[NDVI FORECAST][FALLBACK] Skipping ML forecast, will use simple seasonal averaging.")
                else:
                    print(f"\n🔮 Generating land-type-specific forecast...")
                    
                    forecast_timeseries = []
                    current_date = forecast_start
                    
                    # Calculate land type weights
                    total_points = len(predictions)
                    if total_points == 0:
                        print("[NDVI FORECAST][FALLBACK] Zero valid classified points. Switching to simple seasonal forecast.")
                        force_simple_forecast = True
                    else:
                        land_type_weights = {lt: np.sum(predictions == lt) / total_points 
                                            for lt in unique_types}
                    
                    if not force_simple_forecast:
                        while current_date <= forecast_end:
                            month = current_date.month
                            
                            # Aggregate forecast across all land types (weighted)
                            weighted_forecast = {
                                'ndvi_mean': 0, 'ndvi_min': 0, 'ndvi_max': 0, 'ndvi_std': 0,
                                'ndvi_range': 0, 'ndwi_mean': 0, 'ndbi_mean': 0, 'evi_mean': 0
                            }
                            
                            land_type_contributions = {}
                            
                            for land_type, weight in land_type_weights.items():
                                if land_type in land_type_seasonal_stats and month in land_type_seasonal_stats[land_type]:
                                    stats = land_type_seasonal_stats[land_type][month]
                                    
                                    land_type_contributions[int(land_type)] = {
                                        **stats,
                                        'weight': float(weight)
                                    }
                                    
                                    for key in weighted_forecast:
                                        weighted_forecast[key] += stats[key] * weight
                            
                            if land_type_contributions:
                                forecast_data = {
                                    'date': current_date.strftime('%Y-%m-%d'),
                                    'is_forecast': True,
                                    'land_type_specific': land_type_contributions,
                                    **weighted_forecast
                                }
                                forecast_timeseries.append(forecast_data)
                            
                            current_date += relativedelta(months=1)
                        
                        method_used = "Land-Type-Specific Forecasting (ML-Enhanced)"

        if force_simple_forecast or not use_ml_classification:
            # Simple seasonal averaging (fallback)
            print(f"\n📈 Calculating simple seasonal patterns (no ML)...")
            
            historical_patterns = []
            
            for time_idx in range(len(s2_data.time)):
                time_val = pd.Timestamp(s2_data.time.values[time_idx])
                mask_t = cloud_mask.isel(time=time_idx)
                
                ndvi_valid = ndvi.isel(time=time_idx).where(mask_t)
                ndwi_valid = ndwi.isel(time=time_idx).where(mask_t)
                ndbi_valid = ndbi.isel(time=time_idx).where(mask_t)
                evi_valid = evi.isel(time=time_idx).where(mask_t)
                
                ndvi_vals = ndvi_valid.values.flatten()
                ndvi_vals = ndvi_vals[~np.isnan(ndvi_vals)]
                
                if len(ndvi_vals) > 0:
                    ndwi_vals = ndwi_valid.values.flatten()
                    ndwi_vals = ndwi_vals[~np.isnan(ndwi_vals)]
                    
                    ndbi_vals = ndbi_valid.values.flatten()
                    ndbi_vals = ndbi_vals[~np.isnan(ndbi_vals)]
                    
                    evi_vals = evi_valid.values.flatten()
                    evi_vals = evi_vals[~np.isnan(evi_vals)]
                    
                    historical_patterns.append({
                        'month': time_val.month,
                        'year': time_val.year,
                        'date': time_val,
                        'ndvi_mean': float(np.mean(ndvi_vals)),
                        'ndvi_min': float(np.min(ndvi_vals)),
                        'ndvi_max': float(np.max(ndvi_vals)),
                        'ndvi_std': float(np.std(ndvi_vals)),
                        'ndvi_range': float(np.max(ndvi_vals) - np.min(ndvi_vals)),
                        'ndwi_mean': float(np.mean(ndwi_vals)) if len(ndwi_vals) > 0 else 0.0,
                        'ndbi_mean': float(np.mean(ndbi_vals)) if len(ndbi_vals) > 0 else 0.0,
                        'evi_mean': float(np.mean(evi_vals)) if len(evi_vals) > 0 else 0.0
                    })
            
            if not historical_patterns:
                raise HTTPException(status_code=404, detail="Không có dữ liệu lịch sử hợp lệ")
            
            df_history = pd.DataFrame(historical_patterns)
            monthly_avg = df_history.groupby('month').agg({
                'ndvi_mean': 'mean', 'ndvi_min': 'mean', 'ndvi_max': 'mean', 'ndvi_std': 'mean',
                'ndvi_range': 'mean', 'ndwi_mean': 'mean', 'ndbi_mean': 'mean', 'evi_mean': 'mean'
            }).to_dict('index')
            
            forecast_timeseries = []
            current_date = forecast_start
            
            while current_date <= forecast_end:
                month = current_date.month
                
                if month in monthly_avg:
                    forecast_data = monthly_avg[month].copy()
                    forecast_data['date'] = current_date.strftime('%Y-%m-%d')
                    forecast_data['is_forecast'] = True
                    forecast_timeseries.append(forecast_data)
                
                current_date += relativedelta(months=1)
            
            method_used = "Simple Seasonal Averaging"
        
        if not forecast_timeseries:
            raise HTTPException(status_code=400, detail="Không thể tạo forecast")
        
        # Calculate overall statistics
        forecast_ndvi_means = [item['ndvi_mean'] for item in forecast_timeseries]
        
        result = {
            'timeseries': forecast_timeseries,
            'n_forecast_points': len(forecast_timeseries),
            'mean_ndvi': float(np.mean(forecast_ndvi_means)),
            'min_ndvi': float(np.min(forecast_ndvi_means)),
            'max_ndvi': float(np.max(forecast_ndvi_means)),
            'std_ndvi': float(np.std(forecast_ndvi_means)),
            'bbox': config.bbox,
            'forecast_period': f"{config.forecast_start_date} to {config.forecast_end_date}",
            'historical_period': f"{historical_start.date()} to {historical_end.date()}",
            'method': method_used,
            'model_used': config.model_filename,
            'land_types_detected': list(map(int, unique_types)) if use_ml_classification else None,
            'note': '🔮 Forecast using land-type-specific seasonal patterns for higher accuracy' if use_ml_classification else '⚠️ Simple seasonal forecast without ML classification'
        }
        
        print(f"\n{'='*70}")
        print(f"✅ NDVI Forecast completed!")
        print(f"  Method: {method_used}")
        print(f"  Forecast points: {len(forecast_timeseries)}")
        print(f"  Predicted mean NDVI: {result['mean_ndvi']:.3f}")
        print(f"{'='*70}\n")
        
        return result
        
    except HTTPException:
        raise
    except Exception as e:
        print(f"[NDVI FORECAST ERROR] {e}")
        import traceback
        traceback.print_exc()
        raise HTTPException(status_code=500, detail=f"Lỗi khi forecast NDVI: {str(e)}")


if __name__ == "__main__":
    print("=" * 70)
    print("🚀 LAND CLASSIFICATION TRAINING API SERVER")
    print("=" * 70)
    print("\n📍 Endpoints:")
    print("   - Web Interface: http://localhost:8000")
    print("   - API Docs: http://localhost:8000/docs")
    print("   - Start Training: POST http://localhost:8000/api/training/start")
    print("   - Check Status: GET http://localhost:8000/api/training/status")
    print("\n" + "=" * 70)
    
    uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info")
