sửa các lỗi tại màn hình predict
This commit is contained in:
Victor Phan
+224
-100
@@ -218,6 +218,16 @@ async def ndvi_page():
|
||||
raise HTTPException(status_code=404, detail="NDVI interface không tồn tại")
|
||||
|
||||
|
||||
@app.get("/reports", response_class=HTMLResponse)
|
||||
async def reports_page():
|
||||
"""Serve reports management interface"""
|
||||
html_file = Path(__file__).parent / "reports_interface.html"
|
||||
if html_file.exists():
|
||||
return FileResponse(html_file)
|
||||
else:
|
||||
raise HTTPException(status_code=404, detail="Reports interface không tồn tại")
|
||||
|
||||
|
||||
@app.get("/api/config/presets")
|
||||
async def get_presets():
|
||||
"""Lấy các preset cấu hình sẵn"""
|
||||
@@ -324,7 +334,7 @@ async def clear_cache():
|
||||
|
||||
@app.get("/api/cache/info")
|
||||
async def get_cache_info():
|
||||
"""Lấy thông tin về cache với metadata đầy đủ"""
|
||||
"""Lấy thông tin về cache với metadata đầy đủ, tự động xóa cache cũ có lazy data"""
|
||||
cache_dir = Path("dataset_cache")
|
||||
|
||||
if not cache_dir.exists():
|
||||
@@ -332,16 +342,37 @@ async def get_cache_info():
|
||||
|
||||
cache_files = []
|
||||
total_size = 0
|
||||
deleted_count = 0
|
||||
|
||||
for cache_file in cache_dir.glob("*.joblib"):
|
||||
# Skip if file doesn't exist (race condition)
|
||||
if not cache_file.exists():
|
||||
continue
|
||||
|
||||
size = cache_file.stat().st_size
|
||||
total_size += size
|
||||
|
||||
# Try to load metadata from cache
|
||||
# Try to load metadata from cache and check if it's valid
|
||||
metadata = {}
|
||||
is_valid = True
|
||||
try:
|
||||
cached_data = joblib.load(cache_file)
|
||||
if isinstance(cached_data, dict):
|
||||
|
||||
# Check if cache contains lazy data (will cause 403 errors)
|
||||
if isinstance(cached_data, dict) and "s2_data" in cached_data:
|
||||
s2_data_temp = cached_data["s2_data"]
|
||||
is_lazy = False
|
||||
try:
|
||||
is_lazy = any(hasattr(s2_data_temp[var].data, 'chunks') for var in s2_data_temp.data_vars)
|
||||
except:
|
||||
pass
|
||||
|
||||
if is_lazy:
|
||||
print(f"[CLEANUP] Deleting cache with lazy data: {cache_file.name}")
|
||||
cache_file.unlink()
|
||||
deleted_count += 1
|
||||
is_valid = False
|
||||
|
||||
if is_valid and isinstance(cached_data, dict):
|
||||
metadata = {
|
||||
"bbox": cached_data.get("bbox", []),
|
||||
"time_range": cached_data.get("time_range", ""),
|
||||
@@ -362,23 +393,36 @@ async def get_cache_info():
|
||||
metadata["max_lon"] = metadata["bbox"][2]
|
||||
metadata["max_lat"] = metadata["bbox"][3]
|
||||
except Exception as e:
|
||||
print(f"Error loading cache metadata: {e}")
|
||||
print(f"[CLEANUP] Error loading cache {cache_file.name}: {e}. Deleting...")
|
||||
try:
|
||||
cache_file.unlink()
|
||||
deleted_count += 1
|
||||
is_valid = False
|
||||
except:
|
||||
pass
|
||||
|
||||
cache_files.append({
|
||||
"filename": cache_file.name,
|
||||
"size_mb": round(size / 1024 / 1024, 2),
|
||||
"modified": datetime.fromtimestamp(cache_file.stat().st_mtime).isoformat(),
|
||||
"metadata": metadata
|
||||
})
|
||||
# Only add valid cache files to the list
|
||||
if is_valid:
|
||||
total_size += size
|
||||
cache_files.append({
|
||||
"filename": cache_file.name,
|
||||
"size_mb": round(size / 1024 / 1024, 2),
|
||||
"modified": datetime.fromtimestamp(cache_file.stat().st_mtime).isoformat(),
|
||||
"metadata": metadata
|
||||
})
|
||||
|
||||
# Sort by modified time (newest first)
|
||||
cache_files.sort(key=lambda x: x["modified"], reverse=True)
|
||||
|
||||
if deleted_count > 0:
|
||||
print(f"[CLEANUP] Deleted {deleted_count} invalid cache files")
|
||||
|
||||
return {
|
||||
"exists": True,
|
||||
"files": cache_files,
|
||||
"count": len(cache_files),
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2)
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2),
|
||||
"deleted_invalid": deleted_count
|
||||
}
|
||||
|
||||
|
||||
@@ -717,15 +761,44 @@ async def run_prediction(config: PredictionConfig):
|
||||
bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# Try to load from cache first
|
||||
s2_data = None
|
||||
use_cache = False
|
||||
|
||||
if cache_file.exists():
|
||||
prediction_status["progress"] = "Đang load dữ liệu từ cache..."
|
||||
cached = joblib.load(cache_file)
|
||||
s2_data = cached["s2_data"]
|
||||
s2_items = cached["s2_items"]
|
||||
vh_monthly = cached.get("vh_monthly")
|
||||
vv_monthly = cached.get("vv_monthly")
|
||||
use_radar = cached.get("use_radar", False)
|
||||
else:
|
||||
try:
|
||||
cached = joblib.load(cache_file)
|
||||
s2_data_temp = cached["s2_data"]
|
||||
|
||||
# Verify that cached data is not lazy (to avoid 403 errors from expired URLs)
|
||||
# If s2_data has chunks attribute, it's a dask array (lazy)
|
||||
is_lazy = False
|
||||
try:
|
||||
is_lazy = any(hasattr(s2_data_temp[var].data, 'chunks') for var in s2_data_temp.data_vars)
|
||||
except:
|
||||
pass
|
||||
|
||||
if is_lazy:
|
||||
print(f"[WARNING] Cache contains lazy data with potentially expired URLs. Deleting cache...")
|
||||
cache_file.unlink()
|
||||
raise ValueError("Cache invalid - contains lazy data")
|
||||
|
||||
# Cache is valid, use it
|
||||
s2_data = s2_data_temp
|
||||
s2_items = cached.get("s2_items", [])
|
||||
vh_monthly = cached.get("vh_monthly")
|
||||
vv_monthly = cached.get("vv_monthly")
|
||||
use_radar = cached.get("use_radar", False)
|
||||
use_cache = True
|
||||
print(f"[INFO] Loaded valid cache from {cache_file.name}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[WARNING] Failed to load cache: {e}. Fetching fresh data...")
|
||||
s2_data = None
|
||||
|
||||
# If cache not available or invalid, fetch from Microsoft
|
||||
if s2_data is None:
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
@@ -747,13 +820,17 @@ async def run_prediction(config: PredictionConfig):
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s2_items)} scenes Sentinel-2..."
|
||||
s2_data = load(
|
||||
s2_data_lazy = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
)
|
||||
# Compute s2_data to load into memory (avoid lazy loading from expired URLs)
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2 vào bộ nhớ..."
|
||||
s2_data = s2_data_lazy.compute()
|
||||
|
||||
# ============ BƯỚC 4: TẢI DỮ LIỆU SENTINEL-1 (Radar)... ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
s1_search = catalog.search(
|
||||
@@ -828,59 +905,65 @@ async def run_prediction(config: PredictionConfig):
|
||||
# Only load radar if not already in cache
|
||||
if not cache_file.exists() or (cache_file.exists() and not use_radar):
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
|
||||
# Initialize catalog if not already done
|
||||
if not cache_file.exists():
|
||||
# catalog already initialized in the else block above
|
||||
pass
|
||||
else:
|
||||
# Need to initialize catalog for radar search
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
# Search Sentinel-1 data
|
||||
s1_search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
|
||||
s1_items = list(s1_search.items())
|
||||
|
||||
if s1_items:
|
||||
s1_items = s1_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..."
|
||||
|
||||
# Load Sentinel-1 data (without like= to avoid conflict with bbox/resolution)
|
||||
s1_data = load(
|
||||
s1_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="sat:absolute_orbit",
|
||||
resolution=config.resolution
|
||||
)
|
||||
|
||||
# Extract VH and VV bands
|
||||
if "vh" in s1_data and "vv" in s1_data:
|
||||
vh = s1_data["vh"].astype('float32')
|
||||
vv = s1_data["vv"].astype('float32')
|
||||
|
||||
# Resample to monthly average
|
||||
prediction_status["progress"] = "Đang tính trung bình VH/VV theo tháng..."
|
||||
vh_monthly = vh.resample(time="1ME").mean().compute()
|
||||
vv_monthly = vv.resample(time="1ME").mean().compute()
|
||||
|
||||
use_radar = True
|
||||
try:
|
||||
# Initialize catalog if not already done
|
||||
if not cache_file.exists():
|
||||
pass
|
||||
else:
|
||||
prediction_status["progress"] = "Không tìm thấy bands VH/VV, tiếp tục với NDVI..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
# Search Sentinel-1 data
|
||||
s1_search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
s1_items = list(s1_search.items())
|
||||
if s1_items:
|
||||
s1_items = s1_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..."
|
||||
try:
|
||||
s1_data = load(
|
||||
s1_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="sat:absolute_orbit",
|
||||
resolution=config.resolution
|
||||
)
|
||||
if "vh" in s1_data and "vv" in s1_data:
|
||||
vh = s1_data["vh"].astype('float32')
|
||||
vv = s1_data["vv"].astype('float32')
|
||||
prediction_status["progress"] = "Đang tính trung bình VH/VV theo tháng..."
|
||||
try:
|
||||
vh_monthly = vh.resample(time="1ME").mean().compute()
|
||||
vv_monthly = vv.resample(time="1ME").mean().compute()
|
||||
use_radar = True
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể tính radar monthly: {radar_exc}")
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không tìm thấy bands VH/VV, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể tải dữ liệu Sentinel-1: {radar_exc}")
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không có dữ liệu Sentinel-1, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không có dữ liệu Sentinel-1, tiếp tục với NDVI..."
|
||||
except Exception as radar_exc:
|
||||
print(f"[RADAR WARNING] Không thể truy cập Sentinel-1: {radar_exc}")
|
||||
prediction_status["progress"] = "Không thể truy cập Sentinel-1, tiếp tục với NDVI..."
|
||||
vh_monthly = None
|
||||
vv_monthly = None
|
||||
use_radar = False
|
||||
|
||||
# ============ BƯỚC 5: CHUẨN BỊ FEATURES CHO DỰ ĐOÁN ============
|
||||
@@ -1776,46 +1859,87 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
import odc.stac
|
||||
import rasterio
|
||||
from rasterio.transform import from_bounds
|
||||
import hashlib, os
|
||||
|
||||
# Load model
|
||||
model_path = Path(f"model_train/{config.model_filename}")
|
||||
if not model_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Model {config.model_filename} không tồn tại")
|
||||
|
||||
model = joblib.load(model_path)
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
# Extract model from dict (models are saved as {'model': xgb_model, 'label_encoder': encoder})
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
|
||||
print(f"[PREDICT+NDVI] Loaded model: {config.model_filename}")
|
||||
|
||||
# Connect to Microsoft Planetary Computer
|
||||
catalog = Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace
|
||||
)
|
||||
# Check cache first
|
||||
cache_dir = Path("dataset_cache")
|
||||
cache_dir.mkdir(exist_ok=True)
|
||||
cache_key = f"pred_{config.min_lon}_{config.min_lat}_{config.max_lon}_{config.max_lat}_{config.start_date}_{config.end_date}_{config.max_scenes}_{config.cloud_cover}_{config.resolution}"
|
||||
cache_hash = hashlib.md5(cache_key.encode()).hexdigest()
|
||||
cache_file = cache_dir / f"prediction_input_{cache_hash}.joblib"
|
||||
|
||||
bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# Search for Sentinel-2 data
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
|
||||
items = list(search.items())[:config.max_scenes]
|
||||
print(f"[PREDICT+NDVI] Found {len(items)} Sentinel-2 scenes")
|
||||
|
||||
if len(items) == 0:
|
||||
raise HTTPException(status_code=404, detail="Không tìm thấy dữ liệu vệ tinh")
|
||||
|
||||
# Load all bands needed for features
|
||||
data = odc.stac.load(
|
||||
items,
|
||||
bbox=bbox,
|
||||
bands=["B02", "B03", "B04", "B08"], # Blue, Green, Red, NIR
|
||||
resolution=config.resolution,
|
||||
chunks={"x": 2048, "y": 2048}
|
||||
).compute()
|
||||
# Load from cache or fetch from Microsoft
|
||||
if cache_file.exists():
|
||||
print(f"[PREDICT+NDVI] Loading from cache: {cache_file.name}")
|
||||
cached = joblib.load(cache_file)
|
||||
# Extract s2_data from cache (already computed in cache)
|
||||
s2_data = cached["s2_data"]
|
||||
# Check if needed bands are available in cache
|
||||
available_bands = list(s2_data.data_vars.keys())
|
||||
needed_bands = ["B02", "B03", "B04", "B08"]
|
||||
if all(band in available_bands for band in needed_bands):
|
||||
# Use cached data directly (no need to compute again)
|
||||
data = s2_data[needed_bands]
|
||||
print(f"[PREDICT+NDVI] Using cached bands: {needed_bands}")
|
||||
# Set items to match the number of time slices in the cached data
|
||||
items = [None] * s2_data.sizes.get("time", 1)
|
||||
else:
|
||||
raise HTTPException(status_code=400,
|
||||
detail=f"Cache thiếu bands cần thiết. Có: {available_bands}, Cần: {needed_bands}")
|
||||
else:
|
||||
print(f"[PREDICT+NDVI] No cache found, fetching from Microsoft Planetary Computer")
|
||||
# Connect to Microsoft Planetary Computer
|
||||
catalog = Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace
|
||||
)
|
||||
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# Search for Sentinel-2 data
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
|
||||
items = list(search.items())[:config.max_scenes]
|
||||
print(f"[PREDICT+NDVI] Found {len(items)} Sentinel-2 scenes")
|
||||
|
||||
if len(items) == 0:
|
||||
raise HTTPException(status_code=404, detail="Không tìm thấy dữ liệu vệ tinh")
|
||||
|
||||
# Sign items to refresh SAS tokens (keep as pystac.Item, not dict)
|
||||
signed_items = [planetary_computer.sign(item) for item in items]
|
||||
|
||||
# Load all bands needed for features
|
||||
data = odc.stac.load(
|
||||
signed_items,
|
||||
bbox=bbox,
|
||||
bands=["B02", "B03", "B04", "B08"], # Blue, Green, Red, NIR
|
||||
resolution=config.resolution,
|
||||
chunks={"x": 2048, "y": 2048}
|
||||
).compute()
|
||||
|
||||
print(f"[PREDICT+NDVI] Loaded data shape: {data.dims}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user