làm mịn các điểm ảnh
This commit is contained in:
Victor Phan
+581
-192
@@ -96,14 +96,14 @@ DEFAULT_LABEL_MAPPING = {
|
||||
}
|
||||
|
||||
DEFAULT_LABEL_NAMES = {
|
||||
"0": "Lua tom",
|
||||
"1": "Lua",
|
||||
"2": "CHN",
|
||||
"3": "CLN",
|
||||
"4": "TS",
|
||||
"5": "Song",
|
||||
"6": "Dat xay dung",
|
||||
"7": "Rung",
|
||||
0: "Lua tom",
|
||||
1: "Lua",
|
||||
2: "CHN",
|
||||
3: "CLN",
|
||||
4: "TS",
|
||||
5: "Song",
|
||||
6: "Dat xay dung",
|
||||
7: "Rung",
|
||||
}
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ class TrainingConfig(BaseModel):
|
||||
resolution: int # 10m hoặc 20m
|
||||
|
||||
# Model parameters
|
||||
model_type: str # xgboost, random_forest, decision_tree, svm, cnn, swin-unet
|
||||
model_type: str # xgboost, random_forest, decision_tree, svm, cnn, swin-unet, mobilenet-lraspp
|
||||
n_estimators: int
|
||||
max_depth: int
|
||||
learning_rate: float
|
||||
@@ -1151,9 +1151,9 @@ async def run_prediction(config: PredictionConfig):
|
||||
# Initialize FeatureExtractor với đúng mode như lúc training
|
||||
extractor = get_feature_extractor(mode=feature_mode)
|
||||
|
||||
# Check if it's a PyTorch model (CNN, Swin-UNet, etc.)
|
||||
# Check if it's a PyTorch model (CNN, Swin-UNet, MobileNet, etc.)
|
||||
is_pytorch_model = hasattr(model, '__class__') and any(
|
||||
name in model.__class__.__name__ for name in ['CNN', 'SwinUNet']
|
||||
name in model.__class__.__name__ for name in ['CNN', 'SwinUNet', 'MobileNet']
|
||||
)
|
||||
if is_pytorch_model:
|
||||
model_class_name = model.__class__.__name__
|
||||
@@ -1167,43 +1167,78 @@ async def run_prediction(config: PredictionConfig):
|
||||
bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# ============ CHECK PREDICTION CACHE FIRST ============
|
||||
cached_s2_data = None
|
||||
cache_key = f"{config.min_lon:.2f}_{config.min_lat:.2f}_{config.max_lon:.2f}_{config.max_lat:.2f}"
|
||||
|
||||
# Try to find matching cache
|
||||
cache_dir = Path("prediction_cache")
|
||||
if cache_dir.exists():
|
||||
for cache_file in cache_dir.glob(f"pred_{cache_key}_*.json"):
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
# Check if cache matches current config
|
||||
if (cache_data.get('start_date') == config.start_date and
|
||||
cache_data.get('end_date') == config.end_date and
|
||||
cache_data.get('resolution') == config.resolution and
|
||||
cache_data.get('data_file')):
|
||||
|
||||
data_file = cache_dir / cache_data['data_file']
|
||||
if data_file.exists():
|
||||
prediction_status["progress"] = "Đang load dữ liệu từ cache..."
|
||||
import joblib
|
||||
cached_s2_data = joblib.load(data_file)
|
||||
print(f"[CACHE HIT] Using cached Sentinel-2 data from {cache_file.name}")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"[CACHE] Error loading cache {cache_file}: {e}")
|
||||
|
||||
# ============ LOAD SENTINEL-2 DATA ============
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
if cached_s2_data is not None:
|
||||
s2_data = cached_s2_data
|
||||
s2_items = [] # Empty list when using cache
|
||||
prediction_status["progress"] = "Đã load dữ liệu từ cache, đang xử lý..."
|
||||
else:
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
|
||||
s2_search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
s2_items = list(s2_search.items())
|
||||
|
||||
if not s2_items:
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
prediction_status["progress"] = f"Đang xử lý dữ liệu Sentinel-2..."
|
||||
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
|
||||
s2_search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
s2_items = list(s2_search.items())
|
||||
|
||||
if not s2_items:
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s2_items)} scenes Sentinel-2..."
|
||||
|
||||
# Load different bands based on feature mode
|
||||
if feature_mode == 'simple':
|
||||
bands_to_load = ["B04", "B08", "SCL"]
|
||||
else: # temporal or extended
|
||||
bands_to_load = ["B02", "B03", "B04", "B08", "B11", "SCL"]
|
||||
|
||||
s2_data = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
bands=bands_to_load,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
# Load different bands based on feature mode (skip if using cache)
|
||||
if cached_s2_data is None:
|
||||
if feature_mode == 'simple':
|
||||
bands_to_load = ["B04", "B08", "SCL"]
|
||||
else: # temporal or extended
|
||||
bands_to_load = ["B02", "B03", "B04", "B08", "B11", "SCL"]
|
||||
|
||||
s2_data = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
bands=bands_to_load,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
).compute()
|
||||
@@ -1245,15 +1280,90 @@ async def run_prediction(config: PredictionConfig):
|
||||
except Exception as e:
|
||||
prediction_status["progress"] = f"Lỗi load Sentinel-1: {str(e)}, bỏ qua radar features"
|
||||
|
||||
# ============ APPLY CLOUD MASK ============
|
||||
prediction_status["progress"] = "Đang xử lý mây..."
|
||||
# ============ ADVANCED CLOUD MASKING & REMOVAL ============
|
||||
prediction_status["progress"] = "Đang xử lý mây nâng cao..."
|
||||
|
||||
cloud_coverage_percent = 0
|
||||
if "SCL" in s2_data:
|
||||
scl = s2_data["SCL"]
|
||||
# SCL values: 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus
|
||||
cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10)
|
||||
|
||||
# SCL classification values (Sentinel-2 Scene Classification):
|
||||
# 0: No data, 1: Saturated/Defective, 2: Dark Area Pixels
|
||||
# 3: Cloud shadows, 4: Vegetation, 5: Not vegetated, 6: Water
|
||||
# 7: Unclassified, 8: Cloud medium probability, 9: Cloud high probability
|
||||
# 10: Thin cirrus, 11: Snow/Ice
|
||||
|
||||
# Comprehensive cloud mask (clouds, shadows, cirrus, snow)
|
||||
cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10) | (scl == 11)
|
||||
|
||||
# Also mask no-data and saturated pixels
|
||||
invalid_mask = (scl == 0) | (scl == 1)
|
||||
full_mask = cloud_mask | invalid_mask
|
||||
|
||||
# Calculate cloud coverage percentage
|
||||
total_pixels = full_mask.size
|
||||
masked_pixels = int(full_mask.sum().values)
|
||||
cloud_coverage_percent = (masked_pixels / total_pixels * 100) if total_pixels > 0 else 0
|
||||
|
||||
print(f"[CLOUD MASK] Cloud coverage: {cloud_coverage_percent:.1f}%")
|
||||
print(f"[CLOUD MASK] Masked pixels: {masked_pixels}/{total_pixels}")
|
||||
|
||||
# Apply mask to all bands
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
s2_data[band] = s2_data[band].where(~cloud_mask)
|
||||
s2_data[band] = s2_data[band].where(~full_mask)
|
||||
|
||||
# ============ CLOUD REMOVAL STRATEGIES ============
|
||||
|
||||
# Strategy 1: Temporal Interpolation (fill gaps between time steps)
|
||||
prediction_status["progress"] = "Đang khử mây bằng temporal interpolation..."
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
# Forward fill then backward fill along time dimension
|
||||
s2_data[band] = s2_data[band].ffill(dim='time').bfill(dim='time')
|
||||
|
||||
print(f"[CLOUD REMOVAL] Applied temporal interpolation")
|
||||
|
||||
# Strategy 2: Median Compositing (if multiple time steps available)
|
||||
if len(s2_data.time) >= 3:
|
||||
prediction_status["progress"] = "Đang tạo median composite để giảm nhiễu mây..."
|
||||
|
||||
# Create median composite for each band
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
# Median reduces cloud noise better than mean
|
||||
median_composite = s2_data[band].median(dim='time', skipna=True)
|
||||
|
||||
# Fill remaining NaN with median
|
||||
s2_data[band] = s2_data[band].fillna(median_composite)
|
||||
|
||||
print(f"[CLOUD REMOVAL] Applied median compositing from {len(s2_data.time)} scenes")
|
||||
|
||||
# Strategy 3: Spatial Interpolation (fill small gaps)
|
||||
prediction_status["progress"] = "Đang khử mây bằng spatial interpolation..."
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
# Use nearest neighbor interpolation for remaining small gaps
|
||||
s2_data[band] = s2_data[band].interpolate_na(dim='x', method='nearest', fill_value='extrapolate')
|
||||
s2_data[band] = s2_data[band].interpolate_na(dim='y', method='nearest', fill_value='extrapolate')
|
||||
|
||||
print(f"[CLOUD REMOVAL] Applied spatial interpolation")
|
||||
|
||||
# Final check: replace any remaining NaN with 0
|
||||
for band in s2_data.data_vars:
|
||||
if band != "SCL":
|
||||
s2_data[band] = s2_data[band].fillna(0)
|
||||
|
||||
print(f"[CLOUD REMOVAL] Completed - all NaN values handled")
|
||||
|
||||
# Quality warning if cloud coverage too high
|
||||
if cloud_coverage_percent > 30:
|
||||
print(f"[WARNING] High cloud coverage ({cloud_coverage_percent:.1f}%) - prediction quality may be affected")
|
||||
prediction_status["progress"] = f"⚠️ Cảnh báo: Độ phủ mây cao ({cloud_coverage_percent:.1f}%)"
|
||||
|
||||
else:
|
||||
print("[WARNING] No SCL band available - skipping cloud masking")
|
||||
prediction_status["progress"] = "⚠️ Không có SCL band - bỏ qua khử mây"
|
||||
|
||||
# ============ EXTRACT FEATURES ============
|
||||
prediction_status["progress"] = f"Đang trích xuất features (mode={feature_mode})..."
|
||||
@@ -1331,6 +1441,17 @@ async def run_prediction(config: PredictionConfig):
|
||||
|
||||
pred_shape = (y_size, x_size)
|
||||
predictions_2d = predictions.reshape(pred_shape)
|
||||
|
||||
# Smooth classification map to reduce salt-and-pepper noise
|
||||
try:
|
||||
from scipy import ndimage
|
||||
smoothed = ndimage.median_filter(predictions_2d, size=3)
|
||||
# Keep invalid/nodata pixels (-1) untouched
|
||||
smoothed[predictions_2d < 0] = -1
|
||||
predictions_2d = smoothed
|
||||
print("[SMOOTH] Applied 3x3 median filter to classification map")
|
||||
except Exception as smooth_err:
|
||||
print(f"[SMOOTH WARNING] Failed to smooth classification map: {smooth_err}")
|
||||
|
||||
# ============ CREATE OUTPUT ============
|
||||
prediction_status["progress"] = "Đang tạo bản đồ phân loại..."
|
||||
@@ -1370,82 +1491,61 @@ async def run_prediction(config: PredictionConfig):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg') # Non-interactive backend
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
|
||||
# Create a figure with prediction result
|
||||
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
|
||||
# Create a figure with prediction result and legend
|
||||
fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
|
||||
|
||||
# Plot prediction with colormap
|
||||
im = ax.imshow(predictions_2d, cmap='tab20', interpolation='nearest')
|
||||
ax.set_title(f'Prediction Result - {timestamp}', fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
|
||||
ax.set_xlabel('X (pixels)', fontsize=11)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=11)
|
||||
|
||||
# Build mapping from numeric class value -> display label
|
||||
class_map = None
|
||||
try:
|
||||
# 1) Try label_encoder (preferred)
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
# label_encoder.classes_ may be strings or numbers
|
||||
le_classes = list(label_encoder.classes_)
|
||||
# If classes are strings like names, we'll map indices -> names
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
# If classes are numeric labels matching values, map value->str(value)
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
except Exception:
|
||||
class_map = None
|
||||
except Exception:
|
||||
class_map = None
|
||||
# ALWAYS use DEFAULT_LABEL_NAMES - it has the correct Vietnamese names
|
||||
class_map = DEFAULT_LABEL_NAMES.copy()
|
||||
|
||||
print(f"[LEGEND DEBUG] DEFAULT_LABEL_NAMES: {DEFAULT_LABEL_NAMES}")
|
||||
print(f"[LEGEND DEBUG] Model metadata: {model_metadata.get('label_mapping') if isinstance(model_metadata, dict) else 'No metadata'}")
|
||||
print(f"[LEGEND DEBUG] Final class_map: {class_map}")
|
||||
|
||||
# 2) Try model metadata 'class_names' (list ordered by class code)
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
# Get unique classes in prediction to show only relevant legend items
|
||||
unique_pred_classes = np.unique(predictions_2d)
|
||||
unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
|
||||
|
||||
# Create custom legend with color patches
|
||||
legend_elements = []
|
||||
cmap = plt.cm.get_cmap('tab20')
|
||||
|
||||
for cls_val in sorted(unique_pred_classes):
|
||||
try:
|
||||
cn = model_metadata.get('class_names')
|
||||
if isinstance(cn, list):
|
||||
class_map = {i: str(name) for i, name in enumerate(cn)}
|
||||
except Exception:
|
||||
cls_int = int(cls_val)
|
||||
# Get color from colormap (normalize to 0-1 range)
|
||||
color = cmap(cls_int / 20.0) # tab20 has 20 colors
|
||||
# Get label name
|
||||
label_name = class_map.get(cls_int, f"Class {cls_int}")
|
||||
# Create patch for legend
|
||||
legend_elements.append(
|
||||
Patch(facecolor=color, edgecolor='black', linewidth=0.5,
|
||||
label=f"{cls_int}: {label_name}")
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 3) Try invert label_mapping in metadata if exists (name->code)
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
try:
|
||||
lm = model_metadata.get('label_mapping') or model_metadata.get('labels')
|
||||
if isinstance(lm, dict):
|
||||
# invert mapping: code -> name
|
||||
inv = {}
|
||||
for k, v in lm.items():
|
||||
try:
|
||||
key_int = int(v)
|
||||
except Exception:
|
||||
continue
|
||||
inv[key_int] = str(k)
|
||||
if inv:
|
||||
class_map = inv
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Create colorbar
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
|
||||
# If we have a class_map, set ticks and labels
|
||||
try:
|
||||
if class_map:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
else:
|
||||
# fallback: label numeric ticks from min..max
|
||||
if np.issubdtype(predictions_2d.dtype, np.number):
|
||||
minv = int(np.nanmin(predictions_2d))
|
||||
maxv = int(np.nanmax(predictions_2d))
|
||||
ticks = np.arange(minv, maxv + 1)
|
||||
cbar.set_ticks(ticks)
|
||||
cbar.set_ticklabels([str(t) for t in ticks])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Add legend outside plot area
|
||||
if legend_elements:
|
||||
legend = ax.legend(
|
||||
handles=legend_elements,
|
||||
loc='center left',
|
||||
bbox_to_anchor=(1.02, 0.5),
|
||||
fontsize=10,
|
||||
title='Land Classes',
|
||||
title_fontsize=11,
|
||||
framealpha=0.9,
|
||||
edgecolor='black'
|
||||
)
|
||||
legend.get_title().set_fontweight('bold')
|
||||
|
||||
# Add grid
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
@@ -1491,6 +1591,42 @@ async def run_prediction(config: PredictionConfig):
|
||||
print(f"[PREDICTION REPORT ERROR] Failed to generate report: {e}")
|
||||
prediction_status["progress"] = "Hoàn thành! (Không thể tạo báo cáo)"
|
||||
|
||||
# Auto-save prediction cache (including Sentinel-2 data)
|
||||
print(f"[DEBUG] Starting cache save process...")
|
||||
print(f"[DEBUG] cached_s2_data is None: {cached_s2_data is None}")
|
||||
print(f"[DEBUG] s2_data type: {type(s2_data)}")
|
||||
print(f"[DEBUG] s2_data is None: {s2_data is None}")
|
||||
|
||||
try:
|
||||
cache_config = {
|
||||
"min_lon": config.min_lon,
|
||||
"min_lat": config.min_lat,
|
||||
"max_lon": config.max_lon,
|
||||
"max_lat": config.max_lat,
|
||||
"start_date": config.start_date,
|
||||
"end_date": config.end_date,
|
||||
"max_scenes": config.max_scenes,
|
||||
"cloud_cover": config.cloud_cover,
|
||||
"resolution": config.resolution,
|
||||
"model_filename": config.model_filename
|
||||
}
|
||||
|
||||
print(f"[DEBUG] cache_config created: {cache_config}")
|
||||
|
||||
# Save cache with Sentinel-2 data (only if not from cache)
|
||||
data_to_save = None if cached_s2_data is not None else s2_data
|
||||
print(f"[DEBUG] data_to_save is None: {data_to_save is None}")
|
||||
print(f"[DEBUG] Calling save_prediction_cache_sync...")
|
||||
|
||||
result = save_prediction_cache_sync(cache_config, data_to_save)
|
||||
|
||||
print(f"[CACHE] Prediction cache saved: {result.get('message')} (with data: {data_to_save is not None})")
|
||||
print(f"[CACHE] Result: {result}")
|
||||
except Exception as cache_error:
|
||||
print(f"[CACHE ERROR] Failed to auto-save cache: {cache_error}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
prediction_status["end_time"] = dt.now().isoformat()
|
||||
|
||||
except Exception as e:
|
||||
@@ -1608,6 +1744,216 @@ async def preview_prediction_png(filename: str):
|
||||
)
|
||||
|
||||
|
||||
# ============ PREDICTION CACHE API (Auto-save) ============
|
||||
|
||||
def save_prediction_cache_sync(config: dict, s2_data=None):
|
||||
"""Tự động lưu prediction cache sau khi predict thành công (bao gồm cả dữ liệu Sentinel-2)"""
|
||||
print(f"[DEBUG save_prediction_cache_sync] Called with s2_data is None: {s2_data is None}")
|
||||
print(f"[DEBUG save_prediction_cache_sync] Config: {config}")
|
||||
|
||||
try:
|
||||
cache_dir = Path("prediction_cache")
|
||||
print(f"[DEBUG save_prediction_cache_sync] Cache dir: {cache_dir.absolute()}")
|
||||
cache_dir.mkdir(exist_ok=True)
|
||||
print(f"[DEBUG save_prediction_cache_sync] Cache dir created/exists")
|
||||
|
||||
# Create cache filename from bbox and timestamp
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
bbox_str = f"{config['min_lon']:.2f}_{config['min_lat']:.2f}_{config['max_lon']:.2f}_{config['max_lat']:.2f}"
|
||||
base_filename = f"pred_{bbox_str}_{timestamp}"
|
||||
config_file = cache_dir / f"{base_filename}.json"
|
||||
data_file = cache_dir / f"{base_filename}_data.joblib"
|
||||
|
||||
print(f"[DEBUG save_prediction_cache_sync] Config file: {config_file}")
|
||||
print(f"[DEBUG save_prediction_cache_sync] Data file: {data_file}")
|
||||
|
||||
# Prepare cache data with full metadata
|
||||
cache_data = {
|
||||
"bbox": [config['min_lon'], config['min_lat'], config['max_lon'], config['max_lat']],
|
||||
"min_lon": config['min_lon'],
|
||||
"min_lat": config['min_lat'],
|
||||
"max_lon": config['max_lon'],
|
||||
"max_lat": config['max_lat'],
|
||||
"start_date": config['start_date'],
|
||||
"end_date": config['end_date'],
|
||||
"max_scenes": config['max_scenes'],
|
||||
"cloud_cover": config['cloud_cover'],
|
||||
"resolution": config['resolution'],
|
||||
"model_filename": config.get('model_filename'),
|
||||
"created": timestamp,
|
||||
"type": "prediction_cache",
|
||||
"auto_saved": True,
|
||||
"has_data": s2_data is not None,
|
||||
"data_file": f"{base_filename}_data.joblib" if s2_data is not None else None
|
||||
}
|
||||
|
||||
print(f"[DEBUG save_prediction_cache_sync] Saving config JSON...")
|
||||
# Save config to JSON file
|
||||
with open(config_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(cache_data, f, indent=2, ensure_ascii=False)
|
||||
print(f"[DEBUG save_prediction_cache_sync] Config JSON saved to {config_file}")
|
||||
|
||||
# Save Sentinel-2 data if provided
|
||||
if s2_data is not None:
|
||||
print(f"[DEBUG save_prediction_cache_sync] Saving Sentinel-2 data with joblib...")
|
||||
import joblib
|
||||
joblib.dump(s2_data, data_file)
|
||||
data_size_mb = data_file.stat().st_size / 1024 / 1024
|
||||
cache_data['data_size_mb'] = round(data_size_mb, 2)
|
||||
print(f"[CACHE] Saved Sentinel-2 data: {data_size_mb:.2f} MB to {data_file}")
|
||||
else:
|
||||
print(f"[DEBUG save_prediction_cache_sync] No s2_data to save")
|
||||
|
||||
print(f"[CACHE SUCCESS] Cache saved successfully: {base_filename}.json")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Đã lưu cache tự động" + (" (bao gồm dữ liệu Sentinel-2)" if s2_data is not None else ""),
|
||||
"filename": f"{base_filename}.json",
|
||||
"cache": cache_data
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
print(f"[CACHE ERROR save_prediction_cache_sync] Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
@app.get("/api/prediction/cache/list")
|
||||
async def list_prediction_cache():
|
||||
"""Liệt kê các prediction cache đã lưu (bao gồm thông tin về dữ liệu Sentinel-2)"""
|
||||
try:
|
||||
cache_dir = Path("prediction_cache")
|
||||
if not cache_dir.exists():
|
||||
return {"caches": [], "count": 0}
|
||||
|
||||
caches = []
|
||||
for cache_file in cache_dir.glob("pred_*.json"):
|
||||
try:
|
||||
with open(cache_file, 'r', encoding='utf-8') as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
# Check if data file exists
|
||||
data_filename = cache_data.get("data_file")
|
||||
has_data = False
|
||||
data_size_mb = 0
|
||||
|
||||
if data_filename:
|
||||
data_file_path = cache_dir / data_filename
|
||||
if data_file_path.exists():
|
||||
has_data = True
|
||||
data_size_mb = round(data_file_path.stat().st_size / 1024 / 1024, 2)
|
||||
|
||||
# Create display name from metadata
|
||||
bbox = cache_data.get("bbox", [])
|
||||
time_range = f"{cache_data.get('start_date', 'N/A')} → {cache_data.get('end_date', 'N/A')}"
|
||||
data_badge = f" [💾 {data_size_mb}MB]" if has_data else " [⚙️ Config only]"
|
||||
display_name = f"[{bbox[0]:.2f},{bbox[1]:.2f}→{bbox[2]:.2f},{bbox[3]:.2f}] {time_range}{data_badge}"
|
||||
|
||||
caches.append({
|
||||
"filename": cache_file.name,
|
||||
"display_name": display_name,
|
||||
"bbox": bbox,
|
||||
"created": cache_data.get("created"),
|
||||
"has_data": has_data,
|
||||
"data_size_mb": data_size_mb,
|
||||
"data_file": data_filename,
|
||||
"config": cache_data
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"Error loading cache {cache_file}: {e}")
|
||||
continue
|
||||
|
||||
# Sort by created time (newest first)
|
||||
caches.sort(key=lambda x: x.get("created", ""), reverse=True)
|
||||
|
||||
return {
|
||||
"caches": caches,
|
||||
"count": len(caches)
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Lỗi khi load cache: {str(e)}")
|
||||
|
||||
@app.get("/api/prediction/cache/load-data/{filename}")
|
||||
async def load_cached_data(filename: str):
|
||||
"""Load Sentinel-2 data từ cache"""
|
||||
try:
|
||||
cache_dir = Path("prediction_cache")
|
||||
|
||||
# Security check
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
# Load config to get data filename
|
||||
config_file = cache_dir / filename
|
||||
if not config_file.exists():
|
||||
raise HTTPException(status_code=404, detail="Cache config not found")
|
||||
|
||||
with open(config_file, 'r') as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
data_filename = cache_data.get("data_file")
|
||||
if not data_filename:
|
||||
raise HTTPException(status_code=404, detail="No data file in cache")
|
||||
|
||||
data_file = cache_dir / data_filename
|
||||
if not data_file.exists():
|
||||
raise HTTPException(status_code=404, detail="Data file not found")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"has_data": True,
|
||||
"data_file": data_filename,
|
||||
"data_size_mb": round(data_file.stat().st_size / 1024 / 1024, 2)
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Lỗi: {str(e)}")
|
||||
|
||||
@app.delete("/api/prediction/cache/delete/{filename}")
|
||||
async def delete_prediction_cache(filename: str):
|
||||
"""Xóa một prediction cache (bao gồm cả file data nếu có)"""
|
||||
try:
|
||||
cache_dir = Path("prediction_cache")
|
||||
cache_file = cache_dir / filename
|
||||
|
||||
# Security check
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
if not cache_file.exists():
|
||||
raise HTTPException(status_code=404, detail="Cache not found")
|
||||
|
||||
# Load config to check for data file
|
||||
try:
|
||||
with open(cache_file, 'r') as f:
|
||||
cache_data = json.load(f)
|
||||
data_filename = cache_data.get("data_file")
|
||||
if data_filename:
|
||||
data_file = cache_dir / data_filename
|
||||
if data_file.exists():
|
||||
data_file.unlink()
|
||||
print(f"[CACHE] Deleted data file: {data_filename}")
|
||||
except Exception as e:
|
||||
print(f"[CACHE] Error deleting data file: {e}")
|
||||
|
||||
# Delete config file
|
||||
cache_file.unlink()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Đã xóa cache: {filename}"
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Lỗi khi xóa cache: {str(e)}")
|
||||
|
||||
|
||||
# ============ DASHBOARD & VISUALIZATION API ============
|
||||
|
||||
@app.get("/api/dashboard/accuracy-trends")
|
||||
@@ -1956,9 +2302,9 @@ def run_batch_prediction(job: dict, config: PredictionConfig):
|
||||
model_manager = get_model_manager()
|
||||
model, label_encoder, model_metadata = model_manager.load_model(config.model_filename)
|
||||
|
||||
# Check if it's a PyTorch model (CNN, Swin-UNet, etc.)
|
||||
# Check if it's a PyTorch model (CNN, Swin-UNet, MobileNet, etc.)
|
||||
is_pytorch_model = hasattr(model, '__class__') and any(
|
||||
name in model.__class__.__name__ for name in ['CNN', 'SwinUNet']
|
||||
name in model.__class__.__name__ for name in ['CNN', 'SwinUNet', 'MobileNet']
|
||||
)
|
||||
|
||||
job["progress"] = 20
|
||||
@@ -2101,59 +2447,54 @@ def run_batch_prediction(job: dict, config: PredictionConfig):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
|
||||
fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
|
||||
im = ax.imshow(predictions_2d, cmap='tab20', interpolation='nearest')
|
||||
ax.set_title(f"{job['name']} - Batch {job['job_id']}", fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
ax.set_title(f"{job['name']} - Batch {job['job_id']}", fontsize=16, fontweight='bold', pad=20)
|
||||
ax.set_xlabel('X (pixels)', fontsize=11)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=11)
|
||||
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
try:
|
||||
# Build mapping from numeric class value -> label (reuse logic from above)
|
||||
class_map = None
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
le_classes = list(label_encoder.classes_)
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
except Exception:
|
||||
class_map = None
|
||||
# Build mapping - ALWAYS use DEFAULT_LABEL_NAMES
|
||||
class_map = DEFAULT_LABEL_NAMES.copy()
|
||||
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
cn = model_metadata.get('class_names')
|
||||
if isinstance(cn, list):
|
||||
class_map = {i: str(name) for i, name in enumerate(cn)}
|
||||
|
||||
if class_map is None and isinstance(model_metadata, dict):
|
||||
lm = model_metadata.get('label_mapping') or model_metadata.get('labels')
|
||||
if isinstance(lm, dict):
|
||||
inv = {}
|
||||
for k, v in lm.items():
|
||||
try:
|
||||
key_int = int(v)
|
||||
except Exception:
|
||||
continue
|
||||
inv[key_int] = str(k)
|
||||
if inv:
|
||||
class_map = inv
|
||||
|
||||
if class_map:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
else:
|
||||
if np.issubdtype(predictions_2d.dtype, np.number):
|
||||
minv = int(np.nanmin(predictions_2d))
|
||||
maxv = int(np.nanmax(predictions_2d))
|
||||
ticks = np.arange(minv, maxv + 1)
|
||||
cbar.set_ticks(ticks)
|
||||
cbar.set_ticklabels([str(t) for t in ticks])
|
||||
except Exception:
|
||||
pass
|
||||
# Get unique classes in prediction to show only relevant legend items
|
||||
unique_pred_classes = np.unique(predictions_2d)
|
||||
unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
|
||||
|
||||
# Create custom legend with color patches
|
||||
legend_elements = []
|
||||
cmap = plt.cm.get_cmap('tab20')
|
||||
|
||||
for cls_val in sorted(unique_pred_classes):
|
||||
try:
|
||||
cls_int = int(cls_val)
|
||||
# Get color from colormap (normalize to 0-1 range)
|
||||
color = cmap(cls_int / 20.0) # tab20 has 20 colors
|
||||
# Get label name
|
||||
label_name = class_map.get(cls_int, f"Class {cls_int}")
|
||||
# Create patch for legend
|
||||
legend_elements.append(
|
||||
Patch(facecolor=color, edgecolor='black', linewidth=0.5,
|
||||
label=f"{cls_int}: {label_name}")
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
# Add legend outside plot area
|
||||
if legend_elements:
|
||||
legend = ax.legend(
|
||||
handles=legend_elements,
|
||||
loc='center left',
|
||||
bbox_to_anchor=(1.02, 0.5),
|
||||
fontsize=10,
|
||||
title='Land Classes',
|
||||
title_fontsize=11,
|
||||
framealpha=0.9,
|
||||
edgecolor='black'
|
||||
)
|
||||
legend.get_title().set_fontweight('bold')
|
||||
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
|
||||
plt.tight_layout()
|
||||
@@ -3095,7 +3436,7 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
print(f"[PREDICT+NDVI] Features range: [{features_clean.min():.3f}, {features_clean.max():.3f}]")
|
||||
|
||||
# Check if model is PyTorch/deep learning model and use GPU if available
|
||||
is_pytorch_model = hasattr(model, '__class__') and ('CNN' in model.__class__.__name__ or 'Swin' in model.__class__.__name__ or 'UNet' in model.__class__.__name__)
|
||||
is_pytorch_model = hasattr(model, '__class__') and ('CNN' in model.__class__.__name__ or 'Swin' in model.__class__.__name__ or 'UNet' in model.__class__.__name__ or 'MobileNet' in model.__class__.__name__)
|
||||
|
||||
if is_pytorch_model and config.use_gpu:
|
||||
try:
|
||||
@@ -3152,6 +3493,16 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
prediction_raster[valid_mask] = predictions
|
||||
prediction_raster = prediction_raster.reshape(height, width)
|
||||
|
||||
# Smooth classification map to make output cleaner (reduce speckle)
|
||||
try:
|
||||
from scipy import ndimage
|
||||
smoothed = ndimage.median_filter(prediction_raster, size=3)
|
||||
smoothed[prediction_raster < 0] = -1 # keep nodata
|
||||
prediction_raster = smoothed
|
||||
print("[PREDICT+NDVI][SMOOTH] Applied 3x3 median filter to classification")
|
||||
except Exception as smooth_err:
|
||||
print(f"[PREDICT+NDVI][SMOOTH WARNING] {smooth_err}")
|
||||
|
||||
# --- CHANGE DETECTION ---
|
||||
change_summary = None
|
||||
change_map = None
|
||||
@@ -3282,35 +3633,51 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
|
||||
fig, ax = plt.subplots(figsize=(14, 10), dpi=150)
|
||||
im = ax.imshow(prediction_raster, cmap='tab20', interpolation='nearest')
|
||||
ax.set_title(f'Land Classification - {timestamp}', fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
ax.set_title(f'Land Classification - {timestamp}', fontsize=16, fontweight='bold', pad=20)
|
||||
ax.set_xlabel('X (pixels)', fontsize=11)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=11)
|
||||
|
||||
# Try to get class labels
|
||||
class_map = None
|
||||
if label_encoder is not None:
|
||||
# Try to get class labels - ALWAYS use DEFAULT_LABEL_NAMES
|
||||
class_map = DEFAULT_LABEL_NAMES.copy()
|
||||
|
||||
# Get unique classes in prediction
|
||||
unique_pred_classes = np.unique(prediction_raster)
|
||||
unique_pred_classes = unique_pred_classes[~np.isnan(unique_pred_classes)]
|
||||
unique_pred_classes = unique_pred_classes[unique_pred_classes >= 0] # Exclude -1
|
||||
|
||||
# Create custom legend with color patches
|
||||
legend_elements = []
|
||||
cmap = plt.cm.get_cmap('tab20')
|
||||
|
||||
for cls_val in sorted(unique_pred_classes):
|
||||
try:
|
||||
le_classes = list(label_encoder.classes_)
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
cls_int = int(cls_val)
|
||||
color = cmap(cls_int / 20.0)
|
||||
label_name = class_map.get(cls_int, f"Class {cls_int}")
|
||||
legend_elements.append(
|
||||
Patch(facecolor=color, edgecolor='black', linewidth=0.5,
|
||||
label=f"{cls_int}: {label_name}")
|
||||
)
|
||||
except:
|
||||
pass
|
||||
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
|
||||
if class_map:
|
||||
try:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
except:
|
||||
pass
|
||||
# Add legend outside plot area
|
||||
if legend_elements:
|
||||
legend = ax.legend(
|
||||
handles=legend_elements,
|
||||
loc='center left',
|
||||
bbox_to_anchor=(1.02, 0.5),
|
||||
fontsize=10,
|
||||
title='Land Classes',
|
||||
title_fontsize=11,
|
||||
framealpha=0.9,
|
||||
edgecolor='black'
|
||||
)
|
||||
legend.get_title().set_fontweight('bold')
|
||||
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
|
||||
@@ -3336,6 +3703,28 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
class_distribution = {
|
||||
int(cls): int(count) for cls, count in zip(unique_classes, counts)
|
||||
}
|
||||
|
||||
# Auto-save prediction cache (Sentinel-2 data + metadata) for this NDVI workflow
|
||||
try:
|
||||
cache_config = {
|
||||
"min_lon": config.min_lon,
|
||||
"min_lat": config.min_lat,
|
||||
"max_lon": config.max_lon,
|
||||
"max_lat": config.max_lat,
|
||||
"start_date": config.start_date,
|
||||
"end_date": config.end_date,
|
||||
"max_scenes": config.max_scenes,
|
||||
"cloud_cover": config.cloud_cover,
|
||||
"resolution": config.resolution,
|
||||
"model_filename": config.model_filename
|
||||
}
|
||||
|
||||
# Always save data for predict_with_ndvi (so lần sau không phải tải lại)
|
||||
data_to_save = data
|
||||
cache_result = save_prediction_cache_sync(cache_config, data_to_save)
|
||||
print(f"[PREDICT+NDVI][CACHE] Saved cache: {cache_result.get('filename')} (with data: {data_to_save is not None})")
|
||||
except Exception as cache_exc:
|
||||
print(f"[PREDICT+NDVI][CACHE ERROR] {cache_exc}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
@@ -3526,7 +3915,7 @@ async def ndvi_predict_timeseries(config: NDVIPredictionConfig):
|
||||
print(f" Expected features: {n_features_expected}")
|
||||
|
||||
# Check if PyTorch model for GPU
|
||||
is_pytorch_model = hasattr(model, 'forward') or str(type(model).__name__) in ['SwinUnet', 'CNN']
|
||||
is_pytorch_model = hasattr(model, 'forward') or str(type(model).__name__) in ['SwinUnet', 'CNN', 'MobileNetLRASPPClassifier']
|
||||
|
||||
if is_pytorch_model and config.use_gpu:
|
||||
import torch
|
||||
|
||||
Reference in New Issue
Block a user