hoàn thành model swing-unet
This commit is contained in:
Victor Phan
+226
-83
@@ -108,61 +108,61 @@ DEFAULT_LABEL_NAMES = {
|
||||
|
||||
|
||||
class TrainingConfig(BaseModel):
|
||||
"""Cấu hình training"""
|
||||
# Khu vực (bbox) - từ 01.train_ODC.ipynb
|
||||
min_lon: float = 105.5
|
||||
min_lat: float = 9.2
|
||||
max_lon: float = 106.4
|
||||
max_lat: float = 10.0
|
||||
"""Cấu hình training - Tất cả bắt buộc nhập từ giao diện"""
|
||||
# Khu vực (bbox)
|
||||
min_lon: float
|
||||
min_lat: float
|
||||
max_lon: float
|
||||
max_lat: float
|
||||
|
||||
# Thời gian - từ 01.train_ODC.ipynb
|
||||
start_date: str = "2023-03-01"
|
||||
end_date: str = "2023-12-31"
|
||||
# Thời gian
|
||||
start_date: str
|
||||
end_date: str
|
||||
|
||||
# Dữ liệu
|
||||
max_scenes: int = 12
|
||||
cloud_cover: int = 30
|
||||
resolution: int = 20 # 10m hoặc 20m
|
||||
max_scenes: int
|
||||
cloud_cover: int
|
||||
resolution: int # 10m hoặc 20m
|
||||
|
||||
# Model parameters
|
||||
model_type: str = "xgboost" # xgboost, random_forest, decision_tree, svm, cnn, swin-unet
|
||||
n_estimators: int = 100
|
||||
max_depth: int = 20
|
||||
learning_rate: float = 0.1
|
||||
use_gpu: bool = True
|
||||
model_type: str # xgboost, random_forest, decision_tree, svm, cnn, swin-unet
|
||||
n_estimators: int
|
||||
max_depth: int
|
||||
learning_rate: float
|
||||
use_gpu: bool
|
||||
|
||||
# Train/test split
|
||||
test_size: float = 0.2 # Tỷ lệ dữ liệu dùng làm test (0-1)
|
||||
test_size: float # Tỷ lệ dữ liệu dùng làm test (0-1)
|
||||
|
||||
# Cache
|
||||
use_cache: bool = True # Cache dataset để test nhanh hơn
|
||||
use_cache: bool # Cache dataset để test nhanh hơn
|
||||
|
||||
# Training data
|
||||
training_shapefile: str = "train/ST_training data_updated_1130points_new.shp"
|
||||
training_shapefile: str
|
||||
|
||||
|
||||
class PredictionConfig(BaseModel):
|
||||
"""Cấu hình dự đoán"""
|
||||
"""Cấu hình dự đoán - Tất cả bắt buộc nhập từ giao diện"""
|
||||
# Model to use
|
||||
model_filename: str
|
||||
|
||||
# Khu vực (bbox) - từ 01.train_ODC.ipynb
|
||||
min_lon: float = 105.5
|
||||
min_lat: float = 9.2
|
||||
max_lon: float = 106.4
|
||||
max_lat: float = 10.0
|
||||
# Khu vực (bbox)
|
||||
min_lon: float
|
||||
min_lat: float
|
||||
max_lon: float
|
||||
max_lat: float
|
||||
|
||||
# Thời gian - từ 01.train_ODC.ipynb
|
||||
start_date: str = "2023-03-01"
|
||||
end_date: str = "2023-12-31"
|
||||
# Thời gian
|
||||
start_date: str
|
||||
end_date: str
|
||||
|
||||
# Dữ liệu
|
||||
max_scenes: int = 12
|
||||
cloud_cover: int = 30
|
||||
resolution: int = 20
|
||||
max_scenes: int
|
||||
cloud_cover: int
|
||||
resolution: int
|
||||
|
||||
# GPU support for deep learning models
|
||||
use_gpu: bool = True
|
||||
use_gpu: bool
|
||||
|
||||
|
||||
class TrainingStatus(BaseModel):
|
||||
@@ -1258,6 +1258,9 @@ async def run_prediction(config: PredictionConfig):
|
||||
# ============ EXTRACT FEATURES ============
|
||||
prediction_status["progress"] = f"Đang trích xuất features (mode={feature_mode})..."
|
||||
|
||||
print(f"[PREDICTION DEBUG] Feature mode: {feature_mode}")
|
||||
print(f"[PREDICTION DEBUG] S2 bands available: {list(s2_data.data_vars)}")
|
||||
print(f"[PREDICTION DEBUG] S2 dimensions: {dict(s2_data.dims)}")
|
||||
|
||||
# Always fill NaN for all bands in s2_data if present
|
||||
for band in ["B02", "B03", "B04", "B08", "B11"]:
|
||||
@@ -1282,6 +1285,12 @@ async def run_prediction(config: PredictionConfig):
|
||||
|
||||
# Handle NaN values
|
||||
features = np.nan_to_num(features, nan=0.0)
|
||||
|
||||
print(f"[PREDICTION DEBUG] Features extracted: shape={features.shape}")
|
||||
print(f"[PREDICTION DEBUG] Features range: [{features.min():.3f}, {features.max():.3f}]")
|
||||
print(f"[PREDICTION DEBUG] Features mean: {features.mean():.3f}, std: {features.std():.3f}")
|
||||
print(f"[PREDICTION DEBUG] NaN count: {np.isnan(features).sum()}")
|
||||
print(f"[PREDICTION DEBUG] First pixel features: {features[0][:min(8, features.shape[1])]}")
|
||||
|
||||
# Ensure features shape matches model expectation
|
||||
if features.shape[1] != n_features_expected:
|
||||
@@ -1292,9 +1301,18 @@ async def run_prediction(config: PredictionConfig):
|
||||
# ============ PREDICT ============
|
||||
prediction_status["progress"] = "Đang dự đoán..."
|
||||
|
||||
print(f"[PREDICTION DEBUG] Starting prediction with {features.shape[0]} pixels, {features.shape[1]} features")
|
||||
|
||||
# Make prediction (all PyTorch models have the same predict interface)
|
||||
predictions = model.predict(features)
|
||||
|
||||
print(f"[PREDICTION DEBUG] Predictions shape: {predictions.shape}")
|
||||
print(f"[PREDICTION DEBUG] Unique predicted classes: {np.unique(predictions)}")
|
||||
print(f"[PREDICTION DEBUG] Class distribution:")
|
||||
unique, counts = np.unique(predictions, return_counts=True)
|
||||
for cls, cnt in zip(unique, counts):
|
||||
print(f" Class {cls}: {cnt} pixels ({cnt/len(predictions)*100:.1f}%)")
|
||||
|
||||
# Decode labels if label_encoder exists
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
@@ -2976,72 +2994,111 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
|
||||
print(f"[PREDICT+NDVI] Data has {n_times} time steps, spatial size: {height}x{width}")
|
||||
|
||||
# Build features based on what model expects
|
||||
# Model metadata should tell us what features were used
|
||||
model_features = model_metadata.get("features", ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"])
|
||||
# Build features using FeatureExtractor to match training
|
||||
feature_mode = model_metadata.get("feature_mode", "simple")
|
||||
expected_n_features = model_metadata.get("n_features", 3)
|
||||
|
||||
# If model was trained with temporal features (multiple time steps)
|
||||
if expected_n_features > 10: # Likely temporal features
|
||||
print(f"[PREDICT+NDVI] Building temporal features (all time steps)")
|
||||
# Use all time steps for each index
|
||||
feature_list = []
|
||||
print(f"[PREDICT+NDVI] Model feature_mode: {feature_mode}")
|
||||
print(f"[PREDICT+NDVI] Model n_features: {expected_n_features}")
|
||||
|
||||
# COMPATIBILITY FIX: If model was trained with buggy code (n_features=3 but feature_mode='odc'),
|
||||
# fallback to simple mode to match what model actually expects
|
||||
if feature_mode == 'odc' and expected_n_features == 3:
|
||||
print(f"[PREDICT+NDVI] ⚠️ WARNING: Model metadata shows odc mode but only 3 features")
|
||||
print(f"[PREDICT+NDVI] This model was trained with old buggy code - using simple mode for compatibility")
|
||||
feature_mode = 'simple'
|
||||
|
||||
# Use FeatureExtractor for consistent feature building
|
||||
from feature_extractor import FeatureExtractor
|
||||
extractor = FeatureExtractor(mode=feature_mode)
|
||||
|
||||
print(f"[PREDICT+NDVI] Using FeatureExtractor with mode='{feature_mode}'")
|
||||
print(f"[PREDICT+NDVI] Expected features: {extractor.get_info()}")
|
||||
|
||||
# Extract features from data
|
||||
# data is already an xr.Dataset with B02, B03, B04, B08
|
||||
# Need to add B11 for ODC mode (NDBI calculation uses SWIR)
|
||||
if feature_mode == 'odc' and 'B11' not in data:
|
||||
# Load B11 if needed for ODC mode
|
||||
print(f"[PREDICT+NDVI] ODC mode requires B11 (SWIR), loading...")
|
||||
|
||||
# Add NDVI for each time step
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndvi[t].flatten())
|
||||
# Fetch B11 band (whether from cache or fresh fetch)
|
||||
try:
|
||||
# If we haven't fetched items yet (cache scenario), do it now
|
||||
if 'signed_items' not in locals():
|
||||
catalog = Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace
|
||||
)
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
signed_items = [planetary_computer.sign(item) for item in list(search.items())[:config.max_scenes]]
|
||||
print(f"[PREDICT+NDVI] Fetched {len(signed_items)} scenes for B11")
|
||||
|
||||
b11_data = odc.stac.load(
|
||||
signed_items,
|
||||
bbox=bbox,
|
||||
bands=["B11"],
|
||||
resolution=config.resolution,
|
||||
chunks={"x": 2048, "y": 2048}
|
||||
).compute()
|
||||
|
||||
# Merge B11 into existing data
|
||||
data = xr.merge([data, b11_data])
|
||||
print(f"[PREDICT+NDVI] Added B11 to data")
|
||||
except Exception as e:
|
||||
print(f"[PREDICT+NDVI] Warning: Failed to load B11: {e}")
|
||||
print(f"[PREDICT+NDVI] Will proceed without B11 (may affect NDBI accuracy)")
|
||||
|
||||
# Extract features using FeatureExtractor
|
||||
if feature_mode == 'simple':
|
||||
# Simple mode needs pre-calculated NDVI
|
||||
print(f"[PREDICT+NDVI] Calculating NDVI for simple mode...")
|
||||
red_band = data["B04"].values
|
||||
nir_band = data["B08"].values
|
||||
ndvi_array = (nir_band - red_band) / (nir_band + red_band + 1e-8)
|
||||
|
||||
# If model has more features, add NDWI and NDBI time series
|
||||
if expected_n_features >= n_times * 2:
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndwi[t].flatten())
|
||||
# Convert to xarray DataArray with proper dims
|
||||
ndvi_data = xr.DataArray(
|
||||
ndvi_array,
|
||||
dims=data["B04"].dims,
|
||||
coords=data["B04"].coords
|
||||
)
|
||||
|
||||
if expected_n_features >= n_times * 3:
|
||||
for t in range(n_times):
|
||||
feature_list.append(ndbi[t].flatten())
|
||||
|
||||
features = np.stack(feature_list, axis=1)
|
||||
|
||||
# Adjust to match expected features
|
||||
if features.shape[1] < expected_n_features:
|
||||
# Pad with mean values
|
||||
n_missing = expected_n_features - features.shape[1]
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
elif features.shape[1] > expected_n_features:
|
||||
# Trim to expected
|
||||
features = features[:, :expected_n_features]
|
||||
# Simple mode also needs VH/VV radar data, but we don't have it for this endpoint
|
||||
# Pass None and let extractor handle it
|
||||
features = extractor.extract(s2_data=None, ndvi_data=ndvi_data, vh_data=None, vv_data=None)
|
||||
else:
|
||||
# Use mean values (aggregate features)
|
||||
print(f"[PREDICT+NDVI] Building aggregate features (mean values)")
|
||||
# Average over time dimension
|
||||
ndvi_mean = np.nanmean(ndvi, axis=0)
|
||||
ndwi_mean = np.nanmean(ndwi, axis=0)
|
||||
ndbi_mean = np.nanmean(ndbi, axis=0)
|
||||
|
||||
# Reshape for prediction
|
||||
features = np.stack([ndvi_mean.flatten(), ndwi_mean.flatten(), ndbi_mean.flatten()], axis=1)
|
||||
|
||||
# Adjust to match expected features if needed
|
||||
if features.shape[1] < expected_n_features:
|
||||
n_missing = expected_n_features - features.shape[1]
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
elif features.shape[1] > expected_n_features:
|
||||
features = features[:, :expected_n_features]
|
||||
# ODC/extended modes use s2_data directly
|
||||
features = extractor.extract(s2_data=data, vh_data=None, vv_data=None)
|
||||
|
||||
print(f"[PREDICT+NDVI] Built features shape: {features.shape}")
|
||||
print(f"[PREDICT+NDVI] Features per pixel: {features.shape[1] if len(features.shape) > 1 else 1}")
|
||||
|
||||
# Handle NaN values
|
||||
valid_mask = ~np.isnan(features).any(axis=1)
|
||||
# Handle NaN, inf, and extreme values
|
||||
# Replace inf with 0
|
||||
features = np.nan_to_num(features, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
# Clip extreme values to reasonable range
|
||||
features = np.clip(features, -1e6, 1e6)
|
||||
|
||||
# Double-check no inf/nan remain
|
||||
valid_mask = np.isfinite(features).all(axis=1)
|
||||
features_clean = features[valid_mask]
|
||||
|
||||
print(f"[PREDICT+NDVI] Predicting {features_clean.shape[0]} valid pixels...")
|
||||
print(f"[PREDICT+NDVI] Features range: [{features_clean.min():.3f}, {features_clean.max():.3f}]")
|
||||
|
||||
# Check if model is PyTorch/deep learning model and use GPU if available
|
||||
is_pytorch_model = hasattr(model, '__class__') and ('CNN' in model.__class__.__name__ or 'Swin' in model.__class__.__name__ or 'UNet' in model.__class__.__name__)
|
||||
|
||||
if is_pytorch_model and config.use_gpu:
|
||||
try:
|
||||
try:
|
||||
import torch
|
||||
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
|
||||
@@ -3077,6 +3134,12 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
predictions = model.predict(features_clean)
|
||||
except Exception as gpu_error:
|
||||
print(f"[PREDICT+NDVI] GPU prediction failed: {gpu_error}, falling back to CPU")
|
||||
# Move model back to CPU before retrying
|
||||
try:
|
||||
model = model.cpu()
|
||||
print(f"[PREDICT+NDVI] Moved model to CPU")
|
||||
except:
|
||||
pass
|
||||
predictions = model.predict(features_clean)
|
||||
else:
|
||||
# Use CPU for traditional ML models
|
||||
@@ -3142,6 +3205,13 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
|
||||
# Export NDVI if requested
|
||||
if config.export_ndvi:
|
||||
# Calculate NDVI from data for export (data contains B04=red, B08=nir)
|
||||
red_band = data["B04"].values
|
||||
nir_band = data["B08"].values
|
||||
ndvi_array = (nir_band - red_band) / (nir_band + red_band + 1e-8)
|
||||
# Average over time dimension to get mean NDVI
|
||||
ndvi_mean = np.nanmean(ndvi_array, axis=0)
|
||||
|
||||
ndvi_file = output_dir / f"ndvi_{timestamp}.tif"
|
||||
transform = from_bounds(bbox[0], bbox[1], bbox[2], bbox[3], width, height)
|
||||
|
||||
@@ -3159,6 +3229,32 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
|
||||
output_files.append({"type": "ndvi", "path": str(ndvi_file)})
|
||||
print(f"[PREDICT+NDVI] Saved NDVI to {ndvi_file}")
|
||||
|
||||
# Create PNG preview for NDVI
|
||||
ndvi_png = output_dir / f"ndvi_{timestamp}.png"
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
|
||||
im = ax.imshow(ndvi_mean, cmap='RdYlGn', vmin=-1, vmax=1, interpolation='nearest')
|
||||
ax.set_title(f'NDVI - {timestamp}', fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('NDVI', rotation=270, labelpad=15)
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
|
||||
plt.tight_layout()
|
||||
plt.savefig(str(ndvi_png), dpi=150, bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
output_files.append({"type": "ndvi_png", "path": str(ndvi_png)})
|
||||
print(f"[PREDICT+NDVI] Created PNG: {ndvi_png}")
|
||||
except Exception as e:
|
||||
print(f"[PREDICT+NDVI] PNG creation failed: {e}")
|
||||
|
||||
# Export classification if requested
|
||||
if config.export_classification:
|
||||
@@ -3179,6 +3275,53 @@ async def predict_with_ndvi(config: PredictionWithNDVIConfig, background_tasks:
|
||||
|
||||
output_files.append({"type": "classification", "path": str(class_file)})
|
||||
print(f"[PREDICT+NDVI] Saved classification to {class_file}")
|
||||
|
||||
# Create PNG preview for classification
|
||||
class_png = output_dir / f"classification_{timestamp}.png"
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 10), dpi=150)
|
||||
im = ax.imshow(prediction_raster, cmap='tab20', interpolation='nearest')
|
||||
ax.set_title(f'Land Classification - {timestamp}', fontsize=14, fontweight='bold')
|
||||
ax.set_xlabel('X (pixels)', fontsize=10)
|
||||
ax.set_ylabel('Y (pixels)', fontsize=10)
|
||||
|
||||
# Try to get class labels
|
||||
class_map = None
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
le_classes = list(label_encoder.classes_)
|
||||
if all(isinstance(x, str) for x in le_classes):
|
||||
class_map = {i: name for i, name in enumerate(le_classes)}
|
||||
else:
|
||||
class_map = {int(v): str(v) for v in le_classes}
|
||||
except:
|
||||
pass
|
||||
|
||||
cbar = plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04)
|
||||
cbar.set_label('Class', rotation=270, labelpad=15)
|
||||
|
||||
if class_map:
|
||||
try:
|
||||
vals = np.array(sorted(class_map.keys()))
|
||||
cbar.set_ticks(vals)
|
||||
cbar.set_ticklabels([class_map[int(v)] for v in vals])
|
||||
except:
|
||||
pass
|
||||
|
||||
ax.grid(True, alpha=0.3, linestyle='--', linewidth=0.5)
|
||||
|
||||
plt.tight_layout()
|
||||
plt.savefig(str(class_png), dpi=150, bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
output_files.append({"type": "classification_png", "path": str(class_png)})
|
||||
print(f"[PREDICT+NDVI] Created PNG: {class_png}")
|
||||
except Exception as e:
|
||||
print(f"[PREDICT+NDVI] PNG creation failed: {e}")
|
||||
|
||||
# Calculate statistics
|
||||
ndvi_stats = {
|
||||
|
||||
Reference in New Issue
Block a user