đã áp dụng file shapefile vào train và predict
This commit is contained in:
Victor Phan
+183
-6
@@ -277,7 +277,7 @@ def train_model(
|
||||
use_gpu=True,
|
||||
use_cache=True,
|
||||
test_size=0.2,
|
||||
feature_mode='simple',
|
||||
feature_mode='simple', # Changed from 'odc' - simple mode works with B04, B08, SCL only
|
||||
output_model_path=None,
|
||||
status_callback=None,
|
||||
cancel_check=None
|
||||
@@ -300,7 +300,7 @@ def train_model(
|
||||
status_callback: Optional callback function to report progress
|
||||
cancel_check: Optional function that returns True if training should be cancelled
|
||||
test_size: Fraction of data to use for test set (0-1)
|
||||
feature_mode: 'simple' (3 features), 'temporal' (39 features), or 'extended' (15 features)
|
||||
feature_mode: 'simple' (3 features), 'temporal' (39 features), 'extended' (15 features), or 'odc' (8 features)
|
||||
|
||||
Returns:
|
||||
Dictionary containing training results
|
||||
@@ -346,14 +346,29 @@ def train_model(
|
||||
|
||||
# Try to load from cache
|
||||
if use_cache and cache_file.exists():
|
||||
update_status(f"📦 Loading cached dataset from {cache_file.name}...", 5)
|
||||
update_status(f"📦 Đang load cache: {cache_file.name}...", 5)
|
||||
try:
|
||||
cached_data = joblib.load(cache_file)
|
||||
features = cached_data['features']
|
||||
labels = cached_data['labels']
|
||||
update_status(f"✅ Loaded {len(features)} samples from cache (skipped satellite download!)", 50)
|
||||
|
||||
# Validate cached data
|
||||
if len(features) == 0:
|
||||
update_status(
|
||||
f"❌ Cache rỗng (0 samples)! Đây là cache từ lần training thất bại trước.\n"
|
||||
f" Nguyên nhân: Bbox không overlap với shapefile HOẶC tất cả điểm bị NaN.\n"
|
||||
f" Đang xóa cache lỗi và tải lại dữ liệu...", 10
|
||||
)
|
||||
cache_file.unlink() # Delete empty cache
|
||||
features = None
|
||||
else:
|
||||
update_status(
|
||||
f"✅ Loaded {len(features)} samples từ cache!\n"
|
||||
f" ⚡ Đã bỏ qua download위성 data (tiết kiệm thời gian)", 50
|
||||
)
|
||||
print(f"[CACHE HIT] Using cached dataset with {len(features)} samples")
|
||||
except Exception as e:
|
||||
update_status(f"⚠️ Cache load failed: {str(e)}, downloading fresh data...", 10)
|
||||
update_status(f"⚠️ Cache bị lỗi: {str(e)}\n Đang tải lại dữ liệu mới...", 10)
|
||||
features = None
|
||||
|
||||
# If no cache or cache failed, download data
|
||||
@@ -404,6 +419,15 @@ def train_model(
|
||||
fail_on_error=False,
|
||||
)
|
||||
|
||||
# Debug: Print S2 data info
|
||||
print(f"[DEBUG S2] Loaded S2 data")
|
||||
print(f"[DEBUG S2] Dimensions: {dict(ds_s2.dims)}")
|
||||
print(f"[DEBUG S2] Bands: {list(ds_s2.data_vars)}")
|
||||
print(f"[DEBUG S2] CRS: {ds_s2.rio.crs if hasattr(ds_s2, 'rio') else 'No CRS'}")
|
||||
print(f"[DEBUG S2] Spatial bounds: x=[{float(ds_s2.x.min())}, {float(ds_s2.x.max())}], y=[{float(ds_s2.y.min())}, {float(ds_s2.y.max())}]")
|
||||
if 'time' in ds_s2.dims:
|
||||
print(f"[DEBUG S2] Time range: {ds_s2.time.min().values} to {ds_s2.time.max().values}")
|
||||
|
||||
# Rename for compatibility (simple mode)
|
||||
if "B04" in ds_s2 and "red" not in ds_s2:
|
||||
ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})
|
||||
@@ -443,14 +467,99 @@ def train_model(
|
||||
ds_s1['vv_db'] = 10 * np.log10(ds_s1['vv'].where(ds_s1['vv'] > 0))
|
||||
ds_s1['vh_db'] = 10 * np.log10(ds_s1['vh'].where(ds_s1['vh'] > 0))
|
||||
|
||||
# Debug: Print S1 data info
|
||||
print(f"[DEBUG S1] Loaded S1 data")
|
||||
print(f"[DEBUG S1] Dimensions: {dict(ds_s1.dims)}")
|
||||
print(f"[DEBUG S1] Bands: {list(ds_s1.data_vars)}")
|
||||
print(f"[DEBUG S1] Spatial bounds: x=[{float(ds_s1.x.min())}, {float(ds_s1.x.max())}], y=[{float(ds_s1.y.min())}, {float(ds_s1.y.max())}]")
|
||||
|
||||
check_cancellation()
|
||||
|
||||
# Load training data
|
||||
update_status("Loading training data...", 55)
|
||||
|
||||
# Normalize training shapefile path
|
||||
# If path doesn't start with 'train/', add it
|
||||
if not training_shapefile.startswith('train/'):
|
||||
training_shapefile = f'train/{training_shapefile}'
|
||||
|
||||
print(f"[DEBUG] Original training shapefile: {training_shapefile}")
|
||||
print(f"[DEBUG] Current working directory: {os.getcwd()}")
|
||||
|
||||
# Try to find the file with exact name first
|
||||
if not os.path.exists(training_shapefile):
|
||||
# File not found, try to find similar files in train directory
|
||||
train_dir = Path('train')
|
||||
if train_dir.exists():
|
||||
# List all .shp files
|
||||
shp_files = list(train_dir.glob('*.shp'))
|
||||
print(f"[DEBUG] Available shapefile files in train/:")
|
||||
for f in shp_files:
|
||||
print(f" - {f.name}")
|
||||
|
||||
# Try to find a matching file (case-insensitive, ignore underscores vs spaces)
|
||||
filename_normalized = os.path.basename(training_shapefile).lower().replace('_', ' ')
|
||||
for shp_file in shp_files:
|
||||
if shp_file.name.lower().replace('_', ' ') == filename_normalized:
|
||||
print(f"[DEBUG] Found matching file: {shp_file}")
|
||||
training_shapefile = str(shp_file)
|
||||
break
|
||||
|
||||
if not os.path.exists(training_shapefile):
|
||||
raise FileNotFoundError(
|
||||
f"Training shapefile not found: {training_shapefile}\n"
|
||||
f"Available files: {[f.name for f in shp_files]}"
|
||||
)
|
||||
else:
|
||||
raise FileNotFoundError(f"Train directory not found: {train_dir}")
|
||||
|
||||
print(f"[DEBUG] Final training shapefile path: {training_shapefile}")
|
||||
print(f"[DEBUG] File exists: {os.path.exists(training_shapefile)}")
|
||||
|
||||
train_gdf = gpd.read_file(training_shapefile)
|
||||
|
||||
if train_gdf.crs != 'EPSG:32648':
|
||||
# Print initial shapefile info
|
||||
update_status(f"📍 Loaded {len(train_gdf)} points from shapefile", 56)
|
||||
print(f"[DEBUG] Shapefile CRS: {train_gdf.crs}")
|
||||
print(f"[DEBUG] Shapefile bounds: {train_gdf.total_bounds}")
|
||||
|
||||
# Convert to WGS84 first (if not already) to match bbox coordinates
|
||||
original_crs = train_gdf.crs
|
||||
if train_gdf.crs and train_gdf.crs.to_epsg() != 4326:
|
||||
print(f"📍 Converting training shapefile from {train_gdf.crs} to WGS84")
|
||||
train_gdf = train_gdf.to_crs("EPSG:4326")
|
||||
print(f"[DEBUG] WGS84 bounds: {train_gdf.total_bounds}")
|
||||
|
||||
# Check bbox overlap in WGS84
|
||||
shp_bounds = train_gdf.total_bounds # [minx, miny, maxx, maxy]
|
||||
bbox_wgs84 = bbox # [min_lon, min_lat, max_lon, max_lat]
|
||||
|
||||
# Check if there's overlap
|
||||
overlap_x = not (shp_bounds[2] < bbox_wgs84[0] or shp_bounds[0] > bbox_wgs84[2])
|
||||
overlap_y = not (shp_bounds[3] < bbox_wgs84[1] or shp_bounds[1] > bbox_wgs84[3])
|
||||
|
||||
if not (overlap_x and overlap_y):
|
||||
update_status(f"⚠️ WARNING: Shapefile and bbox may not overlap!", 57)
|
||||
print(f"[WARNING] Shapefile bounds (WGS84): {shp_bounds}")
|
||||
print(f"[WARNING] Requested bbox (WGS84): {bbox_wgs84}")
|
||||
print(f"[WARNING] This may result in 0 training samples!")
|
||||
else:
|
||||
# Crop to bbox to see how many points are actually in the region
|
||||
train_gdf_cropped = train_gdf.cx[bbox_wgs84[0]:bbox_wgs84[2], bbox_wgs84[1]:bbox_wgs84[3]]
|
||||
update_status(f"📍 {len(train_gdf_cropped)} points within bbox", 57)
|
||||
if len(train_gdf_cropped) == 0:
|
||||
raise ValueError(
|
||||
f"No training points found within bbox!\n"
|
||||
f"Shapefile bounds: {shp_bounds}\n"
|
||||
f"Requested bbox: {bbox_wgs84}\n"
|
||||
f"Please adjust bbox to cover your training data."
|
||||
)
|
||||
|
||||
# Then convert to UTM Zone 48N (EPSG:32648) for extraction
|
||||
if train_gdf.crs.to_epsg() != 32648:
|
||||
print(f"📍 Converting training shapefile from WGS84 to UTM Zone 48N (EPSG:32648)")
|
||||
train_gdf = train_gdf.to_crs('EPSG:32648')
|
||||
print(f"[DEBUG] UTM bounds: {train_gdf.total_bounds}")
|
||||
|
||||
# Auto-detect label column
|
||||
label_column = None
|
||||
@@ -465,6 +574,12 @@ def train_model(
|
||||
# Extract features using FeatureExtractor
|
||||
update_status("Extracting features from satellite data...", 60)
|
||||
|
||||
print(f"[DEBUG] Starting feature extraction...")
|
||||
print(f"[DEBUG] Training GDF has {len(train_gdf)} points")
|
||||
print(f"[DEBUG] Training GDF CRS: {train_gdf.crs}")
|
||||
print(f"[DEBUG] Training GDF bounds (UTM): {train_gdf.total_bounds}")
|
||||
print(f"[DEBUG] Label column: {label_column}")
|
||||
|
||||
if feature_mode == 'simple':
|
||||
# For simple mode: calculate NDVI first
|
||||
ndvi = (ds_s2['nir'] - ds_s2['red']) / (ds_s2['nir'] + ds_s2['red'] + 1e-8)
|
||||
@@ -472,9 +587,19 @@ def train_model(
|
||||
cloud_mask = ds_s2['scl'].isin([1, 3, 8, 9, 10])
|
||||
ndvi_masked = ndvi.where(~cloud_mask)
|
||||
|
||||
print(f"[DEBUG] NDVI shape: {ndvi_masked.shape}")
|
||||
print(f"[DEBUG] NDVI range: [{float(ndvi_masked.min())}, {float(ndvi_masked.max())}]")
|
||||
|
||||
# Extract features at training points
|
||||
features = []
|
||||
labels = []
|
||||
failed_extractions = 0
|
||||
|
||||
# Test first point to see what's happening
|
||||
first_point = train_gdf.iloc[0]
|
||||
print(f"[DEBUG] Testing first point:")
|
||||
print(f" Coords: ({first_point.geometry.x}, {first_point.geometry.y})")
|
||||
print(f" Label: {first_point[label_column]}")
|
||||
|
||||
for idx, row in train_gdf.iterrows():
|
||||
point = row.geometry
|
||||
@@ -489,12 +614,26 @@ def train_model(
|
||||
|
||||
feature_vec = [float(ndvi_val), float(vh_val), float(vv_val)]
|
||||
|
||||
# Debug first few points
|
||||
if idx < 3:
|
||||
print(f"[DEBUG] Point {idx}: coords=({x_coord:.2f}, {y_coord:.2f}), ndvi={ndvi_val:.3f}, vh={vh_val:.3f}, vv={vv_val:.3f}")
|
||||
|
||||
if not np.isnan(feature_vec).any():
|
||||
features.append(feature_vec)
|
||||
labels.append(label)
|
||||
else:
|
||||
failed_extractions += 1
|
||||
if idx < 3:
|
||||
print(f"[DEBUG] Point {idx} has NaN: {feature_vec}")
|
||||
except Exception as e:
|
||||
failed_extractions += 1
|
||||
if idx < 3:
|
||||
print(f"[DEBUG] Point {idx} extraction failed: {e}")
|
||||
continue
|
||||
|
||||
if failed_extractions > 0:
|
||||
update_status(f"⚠️ {failed_extractions}/{len(train_gdf)} points had NaN/missing data", 65)
|
||||
|
||||
features = np.array(features)
|
||||
labels = np.array(labels)
|
||||
|
||||
@@ -510,6 +649,7 @@ def train_model(
|
||||
# Extract features at training points
|
||||
features = []
|
||||
labels = []
|
||||
failed_extractions = 0
|
||||
|
||||
for idx, row in train_gdf.iterrows():
|
||||
point = row.geometry
|
||||
@@ -557,9 +697,15 @@ def train_model(
|
||||
if not np.isnan(feature_vec).any():
|
||||
features.append(feature_vec)
|
||||
labels.append(label)
|
||||
else:
|
||||
failed_extractions += 1
|
||||
except Exception as e:
|
||||
failed_extractions += 1
|
||||
continue
|
||||
|
||||
if failed_extractions > 0:
|
||||
update_status(f"⚠️ {failed_extractions}/{len(train_gdf)} points had NaN/missing data", 65)
|
||||
|
||||
features = np.array(features)
|
||||
labels = np.array(labels)
|
||||
|
||||
@@ -567,6 +713,25 @@ def train_model(
|
||||
|
||||
update_status(f"Extracted {len(features)} valid training samples", 70)
|
||||
|
||||
# ============ VALIDATE SAMPLES ============
|
||||
if len(features) == 0:
|
||||
error_msg = (
|
||||
f"❌ No valid training samples extracted!\n"
|
||||
f"Possible reasons:\n"
|
||||
f"1. Training shapefile points don't overlap with bbox: {bbox}\n"
|
||||
f"2. All points have NaN values (cloud cover, missing data)\n"
|
||||
f"3. Coordinate system mismatch\n"
|
||||
f"Suggestions:\n"
|
||||
f"- Check if bbox matches your region\n"
|
||||
f"- Try a different time range with less cloud cover\n"
|
||||
f"- Verify training shapefile coordinates are correct"
|
||||
)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Warn if very few samples
|
||||
if len(features) < 20:
|
||||
update_status(f"⚠️ Warning: Only {len(features)} samples extracted. Results may be unreliable.", 70)
|
||||
|
||||
# ============ SAVE TO CACHE ============
|
||||
if use_cache:
|
||||
update_status(f"💾 Saving dataset to cache for future use...", 72)
|
||||
@@ -585,6 +750,18 @@ def train_model(
|
||||
except Exception as e:
|
||||
update_status(f"⚠️ Cache save failed: {str(e)}", 75)
|
||||
|
||||
# Validate samples after cache loading
|
||||
if len(features) == 0:
|
||||
error_msg = (
|
||||
f"❌ No training samples available!\n"
|
||||
f"The cached or loaded dataset is empty.\n"
|
||||
f"Please try:\n"
|
||||
f"1. Clear cache and reload data\n"
|
||||
f"2. Check training shapefile and bbox overlap\n"
|
||||
f"3. Adjust time range and cloud cover settings"
|
||||
)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
# Encode labels
|
||||
label_encoder = LabelEncoder()
|
||||
labels_encoded = label_encoder.fit_transform(labels)
|
||||
|
||||
Reference in New Issue
Block a user