hoàn thành chức năng tính ndvi analysys 2 màn hình
This commit is contained in:
Victor Phan
+397
-346
@@ -26,6 +26,9 @@ import traceback
|
||||
# Import report generator
|
||||
from report_generator import generate_training_report, generate_prediction_report
|
||||
|
||||
# Import Model Manager
|
||||
from model_manager import ModelManager, get_model_manager
|
||||
|
||||
# Import planetary computer libraries (conditional)
|
||||
try:
|
||||
from pystac_client import Client
|
||||
@@ -277,6 +280,84 @@ async def reports_page():
|
||||
raise HTTPException(status_code=404, detail="Reports interface không tồn tại")
|
||||
|
||||
|
||||
@app.get("/api/models/list")
|
||||
async def list_models():
|
||||
"""Liệt kê tất cả models có sẵn với metadata"""
|
||||
try:
|
||||
model_manager = get_model_manager()
|
||||
models = model_manager.list_models()
|
||||
return {
|
||||
"success": True,
|
||||
"models": models,
|
||||
"count": len(models)
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"models": []
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/models/{model_filename}/info")
|
||||
async def get_model_info(model_filename: str):
|
||||
"""Lấy thông tin chi tiết về model"""
|
||||
try:
|
||||
model_manager = get_model_manager()
|
||||
info = model_manager.get_model_info(model_filename)
|
||||
if info is None:
|
||||
raise HTTPException(status_code=404, detail=f"Model không tồn tại: {model_filename}")
|
||||
return {
|
||||
"success": True,
|
||||
"model": info
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/models/{model_filename}/validate")
|
||||
async def validate_model(model_filename: str):
|
||||
"""Validate model file"""
|
||||
try:
|
||||
model_manager = get_model_manager()
|
||||
validation = model_manager.validate_model(model_filename)
|
||||
return {
|
||||
"success": True,
|
||||
"validation": validation
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/models/{model_filename}")
|
||||
async def delete_model(model_filename: str):
|
||||
"""Xóa model"""
|
||||
try:
|
||||
model_manager = get_model_manager()
|
||||
success = model_manager.delete_model(model_filename)
|
||||
if not success:
|
||||
raise HTTPException(status_code=404, detail=f"Model không tồn tại: {model_filename}")
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Đã xóa model: {model_filename}"
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/config/presets")
|
||||
async def get_presets():
|
||||
"""Lấy các preset cấu hình sẵn"""
|
||||
@@ -751,18 +832,18 @@ def update_prediction_progress(message: str):
|
||||
|
||||
|
||||
async def run_prediction(config: PredictionConfig):
|
||||
"""Chạy prediction process - Áp dụng phương pháp từ 02.predict_ODC.ipynb"""
|
||||
"""Chạy prediction process - Sử dụng FeatureExtractor để đồng bộ với training"""
|
||||
global prediction_status
|
||||
|
||||
try:
|
||||
prediction_status["progress"] = "Đang import thư viện..."
|
||||
|
||||
# Import required libraries
|
||||
import xarray as xr
|
||||
import numpy as np
|
||||
from datetime import datetime as dt
|
||||
import xarray as xr
|
||||
import rioxarray
|
||||
import dask.array as da
|
||||
from datetime import datetime as dt
|
||||
from feature_extractor import get_feature_extractor
|
||||
|
||||
# Validate bbox
|
||||
if (config.min_lon < -180 or config.max_lon > 180 or
|
||||
@@ -772,331 +853,160 @@ async def run_prediction(config: PredictionConfig):
|
||||
|
||||
prediction_status["progress"] = "Đang load model..."
|
||||
|
||||
# Load model
|
||||
model_path = Path("model_train") / config.model_filename
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"Model không tồn tại: {config.model_filename}")
|
||||
# Load model using ModelManager
|
||||
model_manager = get_model_manager()
|
||||
model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
|
||||
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
# Extract model from dict (models are saved as {'model': xgb_model, 'label_encoder': encoder})
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
# Get feature_mode and features from metadata (default to 'simple' if not specified)
|
||||
feature_mode = model_metadata.get("feature_mode", "simple")
|
||||
required_features = model_metadata.get("features", [])
|
||||
n_features_expected = model_metadata.get("n_features", len(required_features))
|
||||
|
||||
prediction_status["progress"] = f"Model: {model_metadata.get('model_type', 'unknown')}, mode={feature_mode}, features={n_features_expected}"
|
||||
|
||||
# Initialize FeatureExtractor với đúng mode như lúc training
|
||||
extractor = get_feature_extractor(mode=feature_mode)
|
||||
|
||||
# Check if it's a CNN model (PyTorch)
|
||||
is_cnn_model = hasattr(model, '__class__') and 'CNN' in model.__class__.__name__
|
||||
if is_cnn_model:
|
||||
prediction_status["progress"] = "Phát hiện PyTorch CNN model..."
|
||||
# Import PyTorch if needed
|
||||
try:
|
||||
import torch
|
||||
except ImportError:
|
||||
raise ImportError("PyTorch is required for CNN prediction. Install: pip install torch")
|
||||
|
||||
prediction_status["progress"] = "Đang kiểm tra cache dữ liệu đầu vào..."
|
||||
import hashlib, os
|
||||
cache_dir = Path("dataset_cache")
|
||||
cache_dir.mkdir(exist_ok=True)
|
||||
# Tạo cache key từ bbox, time_range, max_scenes, cloud_cover, resolution
|
||||
cache_key = f"pred_{config.min_lon}_{config.min_lat}_{config.max_lon}_{config.max_lat}_{config.start_date}_{config.end_date}_{config.max_scenes}_{config.cloud_cover}_{config.resolution}"
|
||||
cache_hash = hashlib.md5(cache_key.encode()).hexdigest()
|
||||
cache_file = cache_dir / f"prediction_input_{cache_hash}.joblib"
|
||||
raise ImportError("PyTorch required for CNN models. Install: pip install torch")
|
||||
|
||||
# Initialize common variables
|
||||
bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# Try to load from cache first
|
||||
s2_data = None
|
||||
use_cache = False
|
||||
|
||||
if cache_file.exists():
|
||||
prediction_status["progress"] = "Đang load dữ liệu từ cache..."
|
||||
try:
|
||||
cached = joblib.load(cache_file)
|
||||
s2_data_temp = cached["s2_data"]
|
||||
|
||||
# Verify that cached data is not lazy (to avoid 403 errors from expired URLs)
|
||||
# If s2_data has chunks attribute, it's a dask array (lazy)
|
||||
is_lazy = False
|
||||
try:
|
||||
is_lazy = any(hasattr(s2_data_temp[var].data, 'chunks') for var in s2_data_temp.data_vars)
|
||||
except:
|
||||
pass
|
||||
|
||||
if is_lazy:
|
||||
print(f"[WARNING] Cache contains lazy data with potentially expired URLs. Deleting cache...")
|
||||
cache_file.unlink()
|
||||
raise ValueError("Cache invalid - contains lazy data")
|
||||
|
||||
# Cache is valid, use it
|
||||
s2_data = s2_data_temp
|
||||
s2_items = cached.get("s2_items", [])
|
||||
vh_monthly = cached.get("vh_monthly")
|
||||
vv_monthly = cached.get("vv_monthly")
|
||||
use_radar = cached.get("use_radar", False)
|
||||
use_cache = True
|
||||
print(f"[INFO] Loaded valid cache from {cache_file.name}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARNING] Failed to load cache: {e}. Fetching fresh data...")
|
||||
s2_data = None
|
||||
# ============ LOAD SENTINEL-2 DATA ============
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
|
||||
# If cache not available or invalid, fetch from Microsoft
|
||||
if s2_data is None:
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
# ============ BƯỚC 1: TẢI DỮ LIỆU SENTINEL-2 ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
|
||||
s2_search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
s2_items = list(s2_search.items())
|
||||
if not s2_items:
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s2_items)} scenes Sentinel-2..."
|
||||
s2_data_lazy = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
)
|
||||
# Compute s2_data to load into memory (avoid lazy loading from expired URLs)
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2 vào bộ nhớ..."
|
||||
s2_data = s2_data_lazy.compute()
|
||||
|
||||
# ============ BƯỚC 4: TẢI DỮ LIỆU SENTINEL-1 (Radar)... ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
|
||||
s2_search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
s2_items = list(s2_search.items())
|
||||
|
||||
if not s2_items:
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s2_items)} scenes Sentinel-2..."
|
||||
|
||||
# Load different bands based on feature mode
|
||||
if feature_mode == 'simple':
|
||||
bands_to_load = ["B04", "B08", "SCL"]
|
||||
else: # temporal or extended
|
||||
bands_to_load = ["B02", "B03", "B04", "B08", "B11", "SCL"]
|
||||
|
||||
s2_data = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
bands=bands_to_load,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
).compute()
|
||||
|
||||
prediction_status["progress"] = "Đã load Sentinel-2 data"
|
||||
|
||||
# ============ LOAD SENTINEL-1 DATA (RADAR) ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
use_radar = False
|
||||
vh_data = None
|
||||
vv_data = None
|
||||
|
||||
try:
|
||||
s1_search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
s1_items = list(s1_search.items())
|
||||
|
||||
if s1_items:
|
||||
s1_items = s1_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..."
|
||||
s1_data = load(
|
||||
s1_items,
|
||||
bbox=bbox,
|
||||
bands=["vh", "vv"],
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="sat:absolute_orbit",
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
)
|
||||
if "vh" in s1_data and "vv" in s1_data:
|
||||
vh = s1_data["vh"].astype('float32')
|
||||
vv = s1_data["vv"].astype('float32')
|
||||
vh_monthly = vh.resample(time="1ME").mean().compute()
|
||||
vv_monthly = vv.resample(time="1ME").mean().compute()
|
||||
use_radar = True
|
||||
else:
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
).compute()
|
||||
|
||||
# Convert to dB
|
||||
vh_data = 10 * np.log10(s1_data['vh'].where(s1_data['vh'] > 0))
|
||||
vv_data = 10 * np.log10(s1_data['vv'].where(s1_data['vv'] > 0))
|
||||
use_radar = True
|
||||
prediction_status["progress"] = f"Đã load Sentinel-1 data ({len(s1_items)} scenes)"
|
||||
else:
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
# Lưu cache
|
||||
joblib.dump({
|
||||
"s2_data": s2_data,
|
||||
"s2_items": s2_items,
|
||||
"vh_monthly": vh_monthly,
|
||||
"vv_monthly": vv_monthly,
|
||||
"use_radar": use_radar
|
||||
}, cache_file)
|
||||
prediction_status["progress"] = "Không có dữ liệu Sentinel-1, bỏ qua radar features"
|
||||
except Exception as e:
|
||||
prediction_status["progress"] = f"Lỗi load Sentinel-1: {str(e)}, bỏ qua radar features"
|
||||
|
||||
# ============ BƯỚC 2: TÍNH NDVI VÀ XỬ LÝ MÂY ============
|
||||
prediction_status["progress"] = "Đang tính toán NDVI và xử lý mây..."
|
||||
|
||||
# Calculate NDVI using Sentinel-2 band names (B08 = NIR, B04 = Red)
|
||||
nir = s2_data["B08"].astype('float32')
|
||||
red = s2_data["B04"].astype('float32')
|
||||
ndvi = (nir - red) / (nir + red + 1e-8)
|
||||
|
||||
# Mask clouds using SCL band if available
|
||||
# ============ APPLY CLOUD MASK ============
|
||||
prediction_status["progress"] = "Đang xử lý mây..."
|
||||
if "SCL" in s2_data:
|
||||
scl = s2_data["SCL"]
|
||||
# SCL values: 4=vegetation, 5=bare soil, 6=water - these are clear
|
||||
# 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus - mask these
|
||||
# SCL values: 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus
|
||||
cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10)
|
||||
ndvi = ndvi.where(~cloud_mask)
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
s2_data[band] = s2_data[band].where(~cloud_mask)
|
||||
|
||||
# ============ BƯỚC 3: ĐIỀN GIÁ TRỊ NAN (FILL NAN) ============
|
||||
prediction_status["progress"] = "Đang điền giá trị bị che mây..."
|
||||
# ============ EXTRACT FEATURES ============
|
||||
prediction_status["progress"] = f"Đang trích xuất features (mode={feature_mode})..."
|
||||
|
||||
# Fill NaN using forward fill and backward fill
|
||||
ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')
|
||||
|
||||
# Resample to monthly average
|
||||
prediction_status["progress"] = "Đang tính trung bình NDVI theo tháng..."
|
||||
ndvi_monthly = ndvi_filled.resample(time="1ME").mean()
|
||||
|
||||
# Compute NDVI (convert from dask to numpy)
|
||||
ndvi_monthly = ndvi_monthly.compute()
|
||||
|
||||
# ============ BƯỚC 4: TẢI DỮ LIỆU SENTINEL-1 (VH, VV) ============
|
||||
# Only load radar if not already in cache
|
||||
if not cache_file.exists() or (cache_file.exists() and not use_radar):
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
try:
|
||||
# Initialize catalog if not already done
|
||||
if not cache_file.exists():
|
||||
pass
|
||||
else:
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
# Search Sentinel-1 data
|
||||
s1_search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
s1_items = list(s1_search.items())
|
||||
if s1_items:
|
||||
s1_items = s1_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..."
|
||||
try:
|
||||
s1_data = load(
|
||||
s1_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="sat:absolute_orbit",
|
||||
resolution=config.resolution
|
||||
)
|
||||
if "vh" in s1_data and "vv" in s1_data:
|
||||
vh = s1_data["vh"].astype('float32')
|
||||
vv = s1_data["vv"].astype('float32')
|
||||
prediction_status["progress"] = "Đang tính trung bình VH/VV theo tháng..."
|
||||
try:
|
||||
vh_monthly = vh.resample(time="1ME").mean().compute()
|
||||
vv_monthly = vv.resample(time="1ME").mean().compute()
|
||||
use_radar = True
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể tính radar monthly: {radar_exc}")
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không tìm thấy bands VH/VV, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể tải dữ liệu Sentinel-1: {radar_exc}")
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không có dữ liệu Sentinel-1, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể truy cập Sentinel-1: {radar_exc}")
|
||||
prediction_status["progress"] = "Không thể truy cập Sentinel-1, tiếp tục với NDVI..."
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
|
||||
# ============ BƯỚC 5: CHUẨN BỊ FEATURES CHO DỰ ĐOÁN ============
|
||||
prediction_status["progress"] = "Đang chuẩn bị features cho dự đoán..."
|
||||
|
||||
# Get shape information
|
||||
n_times_ndvi = len(ndvi_monthly.time)
|
||||
y_size = len(ndvi_monthly.y)
|
||||
x_size = len(ndvi_monthly.x)
|
||||
n_pixels = y_size * x_size
|
||||
|
||||
# Prepare NDVI features (flatten each time step)
|
||||
ndvi_features = []
|
||||
for t in range(n_times_ndvi):
|
||||
ndvi_t = ndvi_monthly.isel(time=t).values.flatten()
|
||||
ndvi_features.append(ndvi_t)
|
||||
|
||||
# Stack NDVI features
|
||||
features = np.column_stack(ndvi_features)
|
||||
|
||||
# Add radar features if available
|
||||
if use_radar:
|
||||
n_times_vh = len(vh_monthly.time)
|
||||
n_times_vv = len(vv_monthly.time)
|
||||
|
||||
# Add VH features
|
||||
for t in range(min(n_times_vh, n_times_ndvi)):
|
||||
vh_t = vh_monthly.isel(time=t).values.flatten()
|
||||
# Resize if needed
|
||||
if len(vh_t) != n_pixels:
|
||||
vh_t = np.resize(vh_t, n_pixels)
|
||||
features = np.column_stack([features, vh_t])
|
||||
|
||||
# Add VV features
|
||||
for t in range(min(n_times_vv, n_times_ndvi)):
|
||||
vv_t = vv_monthly.isel(time=t).values.flatten()
|
||||
# Resize if needed
|
||||
if len(vv_t) != n_pixels:
|
||||
vv_t = np.resize(vv_t, n_pixels)
|
||||
features = np.column_stack([features, vv_t])
|
||||
|
||||
# Handle NaN values in features✓ CNN PyTorch: Mạnh nhất với ảnh vệ tinh, tự học features, tương thích GPU tốt, cần pip install torch
|
||||
|
||||
# Always fill NaN for all bands in s2_data if present
|
||||
for band in ["B02", "B03", "B04", "B08", "B11"]:
|
||||
if band in s2_data:
|
||||
s2_data[band] = s2_data[band].ffill(dim='time').bfill(dim='time')
|
||||
|
||||
# Calculate NDVI if needed (for simple mode)
|
||||
ndvi_filled = None
|
||||
if feature_mode == 'simple' and 'B08' in s2_data and 'B04' in s2_data:
|
||||
nir = s2_data["B08"].astype('float32')
|
||||
red = s2_data["B04"].astype('float32')
|
||||
ndvi = (nir - red) / (nir + red + 1e-8)
|
||||
ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')
|
||||
|
||||
# Extract features using FeatureExtractor, always pass all possible data
|
||||
features = extractor.extract(
|
||||
s2_data=s2_data,
|
||||
ndvi_data=ndvi_filled,
|
||||
vh_data=vh_data,
|
||||
vv_data=vv_data
|
||||
)
|
||||
|
||||
# Handle NaN values
|
||||
features = np.nan_to_num(features, nan=0.0)
|
||||
|
||||
# Ensure features shape matches model expectation
|
||||
if features.shape[1] != n_features_expected:
|
||||
raise ValueError(f"Số lượng features ({features.shape[1]}) không khớp với model ({n_features_expected}). Hãy kiểm tra lại cấu hình trích xuất đặc trưng và metadata của model.")
|
||||
|
||||
prediction_status["progress"] = f"Đã extract {features.shape[1]} features cho {features.shape[0]} pixels"
|
||||
|
||||
# ============ BƯỚC 6: DỰ ĐOÁN ============
|
||||
# Check model's expected feature count and adjust
|
||||
try:
|
||||
# Get expected number of features from model
|
||||
if is_cnn_model:
|
||||
# For PyTorch CNN, get n_features from model
|
||||
expected_features = model.n_features
|
||||
elif hasattr(model, 'n_features_in_'):
|
||||
expected_features = model.n_features_in_
|
||||
elif hasattr(model, 'feature_names_in_'):
|
||||
expected_features = len(model.feature_names_in_)
|
||||
else:
|
||||
# Try to get from booster for XGBoost
|
||||
try:
|
||||
expected_features = model.get_booster().num_features()
|
||||
except:
|
||||
expected_features = features.shape[1]
|
||||
|
||||
prediction_status["progress"] = f"Model cần {expected_features} features, đang có {features.shape[1]} features..."
|
||||
|
||||
# Adjust features to match model
|
||||
if features.shape[1] > expected_features:
|
||||
# Trim to expected number (use only first N features - NDVI only)
|
||||
prediction_status["progress"] = f"Cắt bớt features từ {features.shape[1]} xuống {expected_features}..."
|
||||
features = features[:, :expected_features]
|
||||
elif features.shape[1] < expected_features:
|
||||
# Pad with zeros or repeat last features
|
||||
prediction_status["progress"] = f"Thêm features từ {features.shape[1]} lên {expected_features}..."
|
||||
n_missing = expected_features - features.shape[1]
|
||||
# Repeat last feature column to fill
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
except Exception as e:
|
||||
prediction_status["progress"] = f"Không thể xác định số features của model, tiếp tục với {features.shape[1]} features..."
|
||||
|
||||
prediction_status["progress"] = f"Đang dự đoán với {features.shape[1]} features..."
|
||||
# ============ PREDICT ============
|
||||
prediction_status["progress"] = "Đang dự đoán..."
|
||||
|
||||
# Make prediction
|
||||
if is_cnn_model:
|
||||
# PyTorch CNN prediction
|
||||
predictions = model.predict(features)
|
||||
else:
|
||||
predictions = model.predict(features)
|
||||
@@ -1104,23 +1014,31 @@ async def run_prediction(config: PredictionConfig):
|
||||
# Decode labels if label_encoder exists
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
predictions = label_encoder.inverse_transform(predictions)
|
||||
predictions = label_encoder.inverse_transform(predictions.astype(int))
|
||||
except:
|
||||
pass # Keep numeric predictions if inverse_transform fails
|
||||
pass
|
||||
|
||||
# Reshape to original shape
|
||||
if feature_mode == 'simple' and 'B08' in s2_data:
|
||||
# Use B08 to get shape
|
||||
y_size = len(s2_data.y)
|
||||
x_size = len(s2_data.x)
|
||||
else:
|
||||
y_size = len(s2_data.y)
|
||||
x_size = len(s2_data.x)
|
||||
|
||||
pred_shape = (y_size, x_size)
|
||||
predictions_2d = predictions.reshape(pred_shape)
|
||||
|
||||
# ============ BƯỚC 7: TẠO OUTPUT VÀ LƯU KẾT QUẢ ============
|
||||
# ============ CREATE OUTPUT ============
|
||||
prediction_status["progress"] = "Đang tạo bản đồ phân loại..."
|
||||
|
||||
# Create output xarray
|
||||
prediction_da = xr.DataArray(
|
||||
predictions_2d,
|
||||
coords={
|
||||
"y": ndvi_monthly.y,
|
||||
"x": ndvi_monthly.x
|
||||
"y": s2_data.y,
|
||||
"x": s2_data.x
|
||||
},
|
||||
dims=["y", "x"],
|
||||
name="classification"
|
||||
@@ -1159,10 +1077,73 @@ async def run_prediction(config: PredictionConfig):
|
||||
ax.set_title(f'Prediction Result - {timestamp}', fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
|
||||
# Add colorbar
|
||||
|
||||
# Build mapping from numeric class value -> display label
|
||||
class_map = None
|
||||
try:
|
||||
# 1) Try label_encoder (preferred)
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
# label_encoder.classes_ may be strings or numbers
|
||||
le_classes = list(label_encoder.classes_)
|
||||
# If classes are strings like names, we'll map indices -> names
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
# If classes are numeric labels matching values, map value->str(value)
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
except Exception:
|
||||
class_map = None
|
||||
except Exception:
|
||||
class_map = None
|
||||
|
||||
# 2) Try model metadata 'class_names' (list ordered by class code)
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
try:
|
||||
cn = model_metadata.get('class_names')
|
||||
if isinstance(cn, list):
|
||||
class_map = {i: str(name) for i, name in enumerate(cn)}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 3) Try invert label_mapping in metadata if exists (name->code)
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
try:
|
||||
lm = model_metadata.get('label_mapping') or model_metadata.get('labels')
|
||||
if isinstance(lm, dict):
|
||||
# invert mapping: code -> name
|
||||
inv = {}
|
||||
for k, v in lm.items():
|
||||
try:
|
||||
key_int = int(v)
|
||||
except Exception:
|
||||
continue
|
||||
inv[key_int] = str(k)
|
||||
if inv:
|
||||
class_map = inv
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Create colorbar
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
|
||||
# If we have a class_map, set ticks and labels
|
||||
try:
|
||||
if class_map:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
else:
|
||||
# fallback: label numeric ticks from min..max
|
||||
if np.issubdtype(predictions_2d.dtype, np.number):
|
||||
minv = int(np.nanmin(predictions_2d))
|
||||
maxv = int(np.nanmax(predictions_2d))
|
||||
ticks = np.arange(minv, maxv + 1)
|
||||
cbar.set_ticks(ticks)
|
||||
cbar.set_ticklabels([str(t) for t in ticks])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Add grid
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
@@ -1192,7 +1173,7 @@ async def run_prediction(config: PredictionConfig):
|
||||
"bbox": bbox,
|
||||
"time_range": time_range,
|
||||
"n_features": features.shape[1],
|
||||
"n_times_ndvi": n_times_ndvi,
|
||||
"feature_mode": feature_mode,
|
||||
"used_radar": use_radar,
|
||||
"model_used": config.model_filename
|
||||
}
|
||||
@@ -1669,25 +1650,15 @@ def run_batch_prediction(job: dict, config: PredictionConfig):
|
||||
|
||||
job["progress"] = 15
|
||||
|
||||
# Load model
|
||||
model_path = Path("model_train") / config.model_filename
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"Model không tồn tại: {config.model_filename}")
|
||||
# Load model using ModelManager
|
||||
model_manager = get_model_manager()
|
||||
model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
|
||||
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
# Check if it's a CNN model
|
||||
is_cnn_model = hasattr(model, '__class__') and 'CNN' in model.__class__.__name__
|
||||
|
||||
job["progress"] = 20
|
||||
|
||||
# Check if CNN model
|
||||
is_cnn_model = hasattr(model, '__class__') and 'CNN' in model.__class__.__name__
|
||||
|
||||
# Load data from Microsoft Planetary Computer
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
@@ -1838,6 +1809,50 @@ def run_batch_prediction(job: dict, config: PredictionConfig):
|
||||
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
try:
|
||||
# Build mapping from numeric class value -> label (reuse logic from above)
|
||||
class_map = None
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
le_classes = list(label_encoder.classes_)
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
except Exception:
|
||||
class_map = None
|
||||
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
cn = model_metadata.get('class_names')
|
||||
if isinstance(cn, list):
|
||||
class_map = {i: str(name) for i, name in enumerate(cn)}
|
||||
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
lm = model_metadata.get('label_mapping') or model_metadata.get('labels')
|
||||
if isinstance(lm, dict):
|
||||
inv = {}
|
||||
for k, v in lm.items():
|
||||
try:
|
||||
key_int = int(v)
|
||||
except Exception:
|
||||
continue
|
||||
inv[key_int] = str(k)
|
||||
if inv:
|
||||
class_map = inv
|
||||
|
||||
if class_map:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
else:
|
||||
if np.issubdtype(predictions_2d.dtype, np.number):
|
||||
minv = int(np.nanmin(predictions_2d))
|
||||
maxv = int(np.nanmax(predictions_2d))
|
||||
ticks = np.arange(minv, maxv + 1)
|
||||
cbar.set_ticks(ticks)
|
||||
cbar.set_ticklabels([str(t) for t in ticks])
|
||||
except Exception:
|
||||
pass
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
|
||||
plt.tight_layout()
|
||||
@@ -1955,20 +1970,12 @@ async def change_detection_predict_workflow(
|
||||
"""
|
||||
try:
|
||||
# --- STEP 1: LOAD MODEL ---
|
||||
model_path = Path("model_train") / model_filename
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Model not found: {model_filename}")
|
||||
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
model_manager = get_model_manager()
|
||||
model, label_encoder, model_metadata = model_manager.load_model(model_filename)
|
||||
|
||||
print(f"[CHANGE DETECTION] Loaded model: {model_filename}")
|
||||
print(f" - Type: {model_metadata.get('model_type', 'unknown')}")
|
||||
print(f" - Features: {model_metadata.get('features', [])}")
|
||||
|
||||
# --- STEP 2: LOAD SENTINEL-2 DATA ---
|
||||
bbox = [min_lon, min_lat, max_lon, max_lat]
|
||||
@@ -2580,22 +2587,13 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
try:
|
||||
import numpy as np
|
||||
|
||||
# Load model
|
||||
model_path = Path(f"model_train/{config.model_filename}")
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Model {config.model_filename} không tồn tại")
|
||||
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
# Extract model from dict (models are saved as {'model': xgb_model, 'label_encoder': encoder})
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
# Load model using ModelManager
|
||||
model_manager = get_model_manager()
|
||||
model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
|
||||
|
||||
print(f"[PREDICT+NDVI] Loaded model: {config.model_filename}")
|
||||
print(f" - Type: {model_metadata.get('model_type', 'unknown')}")
|
||||
print(f" - Features: {model_metadata.get('features', [])}")
|
||||
|
||||
# Check cache first
|
||||
cache_dir = Path("dataset_cache")
|
||||
@@ -2662,13 +2660,17 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
|
||||
print(f"[PREDICT+NDVI] Loaded data shape: {data.dims}")
|
||||
|
||||
# Get expected number of features from model metadata
|
||||
expected_n_features = model_metadata.get("n_features", 3)
|
||||
print(f"[PREDICT+NDVI] Model expects {expected_n_features} features")
|
||||
|
||||
# Calculate NDVI and other indices
|
||||
blue = data["B02"].values
|
||||
green = data["B03"].values
|
||||
red = data["B04"].values
|
||||
nir = data["B08"].values
|
||||
|
||||
# Calculate indices
|
||||
# Calculate indices for each time step
|
||||
# NDVI = (NIR - Red) / (NIR + Red)
|
||||
ndvi = (nir - red) / (nir + red + 1e-8)
|
||||
|
||||
@@ -2679,17 +2681,66 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
ndbi = (red - nir) / (red + nir + 1e-8)
|
||||
|
||||
# Prepare features for prediction
|
||||
# Assuming model was trained with [NDVI, NDWI, NDBI] features
|
||||
height, width = ndvi.shape[1:3] # Skip time dimension
|
||||
n_pixels = height * width
|
||||
n_times = ndvi.shape[0]
|
||||
|
||||
# Average over time dimension
|
||||
ndvi_mean = np.nanmean(ndvi, axis=0)
|
||||
ndwi_mean = np.nanmean(ndwi, axis=0)
|
||||
ndbi_mean = np.nanmean(ndbi, axis=0)
|
||||
print(f"[PREDICT+NDVI] Data has {n_times} time steps, spatial size: {height}x{width}")
|
||||
|
||||
# Reshape for prediction
|
||||
features = np.stack([ndvi_mean.flatten(), ndwi_mean.flatten(), ndbi_mean.flatten()], axis=1)
|
||||
# Build features based on what model expects
|
||||
# Model metadata should tell us what features were used
|
||||
model_features = model_metadata.get("features", ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"])
|
||||
|
||||
# If model was trained with temporal features (multiple time steps)
|
||||
if expected_n_features > 10: # Likely temporal features
|
||||
print(f"[PREDICT+NDVI] Building temporal features (all time steps)")
|
||||
# Use all time steps for each index
|
||||
feature_list = []
|
||||
|
||||
# Add NDVI for each time step
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndvi[t].flatten())
|
||||
|
||||
# If model has more features, add NDWI and NDBI time series
|
||||
if expected_n_features >= n_times * 2:
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndwi[t].flatten())
|
||||
|
||||
if expected_n_features >= n_times * 3:
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndbi[t].flatten())
|
||||
|
||||
features = np.stack(feature_list, axis=1)
|
||||
|
||||
# Adjust to match expected features
|
||||
if features.shape[1] < expected_n_features:
|
||||
# Pad with mean values
|
||||
n_missing = expected_n_features - features.shape[1]
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
elif features.shape[1] > expected_n_features:
|
||||
# Trim to expected
|
||||
features = features[:, :expected_n_features]
|
||||
else:
|
||||
# Use mean values (aggregate features)
|
||||
print(f"[PREDICT+NDVI] Building aggregate features (mean values)")
|
||||
# Average over time dimension
|
||||
ndvi_mean = np.nanmean(ndvi, axis=0)
|
||||
ndwi_mean = np.nanmean(ndwi, axis=0)
|
||||
ndbi_mean = np.nanmean(ndbi, axis=0)
|
||||
|
||||
# Reshape for prediction
|
||||
features = np.stack([ndvi_mean.flatten(), ndwi_mean.flatten(), ndbi_mean.flatten()], axis=1)
|
||||
|
||||
# Adjust to match expected features if needed
|
||||
if features.shape[1] < expected_n_features:
|
||||
n_missing = expected_n_features - features.shape[1]
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
elif features.shape[1] > expected_n_features:
|
||||
features = features[:, :expected_n_features]
|
||||
|
||||
print(f"[PREDICT+NDVI] Built features shape: {features.shape}")
|
||||
|
||||
# Handle NaN values
|
||||
valid_mask = ~np.isnan(features).any(axis=1)
|
||||
|
||||
Reference in New Issue
Block a user