diff --git a/__pycache__/api_server.cpython-310.pyc b/__pycache__/api_server.cpython-310.pyc index 4463aa3..6557ea6 100644 Binary files a/__pycache__/api_server.cpython-310.pyc and b/__pycache__/api_server.cpython-310.pyc differ diff --git a/__pycache__/train_module.cpython-310.pyc b/__pycache__/train_module.cpython-310.pyc index cfb31ff..47c9e56 100644 Binary files a/__pycache__/train_module.cpython-310.pyc and b/__pycache__/train_module.cpython-310.pyc differ diff --git a/api_server.py b/api_server.py index 9f1455e..b547c87 100644 --- a/api_server.py +++ b/api_server.py @@ -68,11 +68,15 @@ class TrainingConfig(BaseModel): resolution: int = 20 # 10m hoặc 20m # Model parameters + model_type: str = "xgboost" # xgboost, random_forest, decision_tree, svm, cnn n_estimators: int = 100 max_depth: int = 20 learning_rate: float = 0.1 use_gpu: bool = True + # Cache + use_cache: bool = True # Cache dataset để test nhanh hơn + # Training data training_shapefile: str = "train/ST_training data_updated_1130points_new.shp" @@ -205,6 +209,90 @@ async def stop_training(): return {"message": "Đang dừng training..."} +@app.post("/api/cache/clear") +async def clear_cache(): + """Xóa cache dataset""" + import shutil + cache_dir = Path("dataset_cache") + + if not cache_dir.exists(): + return {"message": "Không có cache để xóa", "deleted": 0} + + # Count files + cache_files = list(cache_dir.glob("*.joblib")) + count = len(cache_files) + + # Delete all cache files + for cache_file in cache_files: + try: + cache_file.unlink() + except: + pass + + return {"message": f"Đã xóa {count} file cache", "deleted": count} + + +@app.get("/api/cache/info") +async def get_cache_info(): + """Lấy thông tin về cache với metadata đầy đủ""" + cache_dir = Path("dataset_cache") + + if not cache_dir.exists(): + return {"exists": False, "files": [], "total_size_mb": 0} + + cache_files = [] + total_size = 0 + + for cache_file in cache_dir.glob("*.joblib"): + size = cache_file.stat().st_size + total_size += size + + # Try to load metadata from cache + metadata = {} + try: + cached_data = joblib.load(cache_file) + if isinstance(cached_data, dict): + metadata = { + "bbox": cached_data.get("bbox", []), + "time_range": cached_data.get("time_range", ""), + "resolution": cached_data.get("resolution", 20), + "n_samples": len(cached_data.get("features", [])), + "created": cached_data.get("timestamp", "") + } + # Parse time_range to get start/end dates + if metadata["time_range"]: + time_parts = metadata["time_range"].split("/") + if len(time_parts) == 2: + metadata["start_date"] = time_parts[0] + metadata["end_date"] = time_parts[1] + # Parse bbox to get min/max lon/lat + if metadata["bbox"] and len(metadata["bbox"]) == 4: + metadata["min_lon"] = metadata["bbox"][0] + metadata["min_lat"] = metadata["bbox"][1] + metadata["max_lon"] = metadata["bbox"][2] + metadata["max_lat"] = metadata["bbox"][3] + except Exception as e: + print(f"Error loading cache metadata: {e}") + + cache_files.append({ + "filename": cache_file.name, + "size_mb": round(size / 1024 / 1024, 2), + "modified": datetime.fromtimestamp(cache_file.stat().st_mtime).isoformat(), + "metadata": metadata + }) + + # Sort by modified time (newest first) + cache_files.sort(key=lambda x: x["modified"], reverse=True) + + return { + "exists": True, + "files": cache_files, + "count": len(cache_files), + "total_size_mb": round(total_size / 1024 / 1024, 2) + } + + + @app.get("/api/models/list") async def list_models(): """Liệt kê các model đã train""" @@ -288,10 +376,12 @@ async def run_training(config: TrainingConfig): cloud_cover=config.cloud_cover, resolution=config.resolution, training_shapefile=config.training_shapefile, + model_type=config.model_type, n_estimators=config.n_estimators, max_depth=config.max_depth, learning_rate=config.learning_rate, use_gpu=config.use_gpu, + use_cache=config.use_cache, status_callback=lambda msg: update_progress(msg), cancel_check=should_cancel ) @@ -360,6 +450,16 @@ async def run_prediction(config: PredictionConfig): model = model_data label_encoder = None + # Check if it's a CNN model (PyTorch) + is_cnn_model = hasattr(model, '__class__') and 'CNN' in model.__class__.__name__ + if is_cnn_model: + prediction_status["progress"] = "Phát hiện PyTorch CNN model..." + # Import PyTorch if needed + try: + import torch + except ImportError: + raise ImportError("PyTorch is required for CNN prediction. Install: pip install torch") + prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..." # Import and use Microsoft Planetary Computer STAC API @@ -447,14 +547,13 @@ async def run_prediction(config: PredictionConfig): s1_items = s1_items[:config.max_scenes] prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..." - # Load Sentinel-1 data + # Load Sentinel-1 data (without like= to avoid conflict with bbox/resolution) s1_data = load( s1_items, bbox=bbox, chunks={"time": 1, "x": 2048, "y": 2048}, groupby="sat:absolute_orbit", - resolution=config.resolution, - like=ndvi_monthly # Align with NDVI grid + resolution=config.resolution ) # Extract VH and VV bands @@ -514,14 +613,52 @@ async def run_prediction(config: PredictionConfig): vv_t = np.resize(vv_t, n_pixels) features = np.column_stack([features, vv_t]) - # Handle NaN values in features + # Handle NaN values in features✓ CNN PyTorch: Mạnh nhất với ảnh vệ tinh, tự học features, tương thích GPU tốt, cần pip install torch features = np.nan_to_num(features, nan=0.0) # ============ BƯỚC 6: DỰ ĐOÁN ============ + # Check model's expected feature count and adjust + try: + # Get expected number of features from model + if is_cnn_model: + # For PyTorch CNN, get n_features from model + expected_features = model.n_features + elif hasattr(model, 'n_features_in_'): + expected_features = model.n_features_in_ + elif hasattr(model, 'feature_names_in_'): + expected_features = len(model.feature_names_in_) + else: + # Try to get from booster for XGBoost + try: + expected_features = model.get_booster().num_features() + except: + expected_features = features.shape[1] + + prediction_status["progress"] = f"Model cần {expected_features} features, đang có {features.shape[1]} features..." + + # Adjust features to match model + if features.shape[1] > expected_features: + # Trim to expected number (use only first N features - NDVI only) + prediction_status["progress"] = f"Cắt bớt features từ {features.shape[1]} xuống {expected_features}..." + features = features[:, :expected_features] + elif features.shape[1] < expected_features: + # Pad with zeros or repeat last features + prediction_status["progress"] = f"Thêm features từ {features.shape[1]} lên {expected_features}..." + n_missing = expected_features - features.shape[1] + # Repeat last feature column to fill + padding = np.tile(features[:, -1:], (1, n_missing)) + features = np.column_stack([features, padding]) + except Exception as e: + prediction_status["progress"] = f"Không thể xác định số features của model, tiếp tục với {features.shape[1]} features..." + prediction_status["progress"] = f"Đang dự đoán với {features.shape[1]} features..." # Make prediction - predictions = model.predict(features) + if is_cnn_model: + # PyTorch CNN prediction + predictions = model.predict(features) + else: + predictions = model.predict(features) # Decode labels if label_encoder exists if label_encoder is not None: @@ -594,6 +731,49 @@ async def run_prediction(config: PredictionConfig): print(traceback.format_exc()) +@app.get("/api/predictions/list") +async def list_predictions(): + """Lấy danh sách các file prediction đã tạo""" + predictions_dir = Path("predictions") + predictions_dir.mkdir(exist_ok=True) + + predictions = [] + for pred_file in predictions_dir.glob("*.tif"): + predictions.append({ + "filename": pred_file.name, + "created": datetime.fromtimestamp(pred_file.stat().st_mtime).isoformat(), + "size_mb": round(pred_file.stat().st_size / 1024 / 1024, 2), + "download_url": f"/api/predictions/download/{pred_file.name}" + }) + + # Sort by creation time (newest first) + predictions.sort(key=lambda x: x["created"], reverse=True) + return {"predictions": predictions} + + +@app.get("/api/predictions/download/{filename}") +async def download_prediction(filename: str): + """Download file prediction GeoTIFF""" + predictions_dir = Path("predictions") + file_path = predictions_dir / filename + + # Security check: ensure filename doesn't contain path traversal + if ".." in filename or "/" in filename or "\\" in filename: + raise HTTPException(status_code=400, detail="Invalid filename") + + if not file_path.exists(): + raise HTTPException(status_code=404, detail=f"File không tồn tại: {filename}") + + return FileResponse( + path=str(file_path), + filename=filename, + media_type="image/tiff", + headers={ + "Content-Disposition": f"attachment; filename={filename}" + } + ) + + if __name__ == "__main__": print("=" * 70) print("🚀 LAND CLASSIFICATION TRAINING API SERVER") diff --git a/dataset_cache/training_data_59838d7be931abe93b5dd38e7cd89ad7.joblib b/dataset_cache/training_data_59838d7be931abe93b5dd38e7cd89ad7.joblib new file mode 100644 index 0000000..e3e6077 --- /dev/null +++ b/dataset_cache/training_data_59838d7be931abe93b5dd38e7cd89ad7.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7318566231680cb0c97883b7a5e4177aa1bfeec462a0f52ed34aa103ddec210b +size 20903 diff --git a/dataset_cache/training_data_73f65eba2eb052d78cdbf76250e1e68a.joblib b/dataset_cache/training_data_73f65eba2eb052d78cdbf76250e1e68a.joblib new file mode 100644 index 0000000..f03043e --- /dev/null +++ b/dataset_cache/training_data_73f65eba2eb052d78cdbf76250e1e68a.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cabf5e0241dcc3a73133ac8ae11171f34042c491f895b268c016407619bdfe1 +size 20903 diff --git a/model_train/model_cnn_20251214_180423.joblib b/model_train/model_cnn_20251214_180423.joblib new file mode 100644 index 0000000..746561c --- /dev/null +++ b/model_train/model_cnn_20251214_180423.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2f82f6c837396729cc63efa41ee3048d9a7de3197e28318dc846be830239b9 +size 41536 diff --git a/model_train/model_cnn_20251214_180423_info.json b/model_train/model_cnn_20251214_180423_info.json new file mode 100644 index 0000000..1f8c6ed --- /dev/null +++ b/model_train/model_cnn_20251214_180423_info.json @@ -0,0 +1,33 @@ +{ + "timestamp": "2025-12-14T18:04:27.406540", + "data_source": "Microsoft Planetary Computer STAC", + "collections": [ + "sentinel-2-l2a", + "sentinel-1-rtc" + ], + "features": [ + "NDVI_mean", + "VH_dB_mean", + "VV_dB_mean" + ], + "training_samples": 510, + "testing_samples": 128, + "train_accuracy": 0.515686274509804, + "test_accuracy": 0.5, + "model_type": "cnn", + "device": "cpu", + "n_estimators": 50, + "max_depth": null, + "learning_rate": null, + "cnn_epochs": 25, + "n_features": 3, + "n_classes": 7, + "bbox": [ + 105.6, + 9.3, + 106.2, + 9.8 + ], + "time_range": "2023-03-01/2023-05-31", + "resolution": 20 +} \ No newline at end of file diff --git a/model_train/model_cnn_20251214_181104.joblib b/model_train/model_cnn_20251214_181104.joblib new file mode 100644 index 0000000..d94bf3a --- /dev/null +++ b/model_train/model_cnn_20251214_181104.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abc03d62c7f620b88150a6481143026d516fd3a34bd5ab67c734adac4a8900f9 +size 41536 diff --git a/model_train/model_cnn_20251214_181104_info.json b/model_train/model_cnn_20251214_181104_info.json new file mode 100644 index 0000000..c198133 --- /dev/null +++ b/model_train/model_cnn_20251214_181104_info.json @@ -0,0 +1,33 @@ +{ + "timestamp": "2025-12-14T18:17:48.399298", + "data_source": "Microsoft Planetary Computer STAC", + "collections": [ + "sentinel-2-l2a", + "sentinel-1-rtc" + ], + "features": [ + "NDVI_mean", + "VH_dB_mean", + "VV_dB_mean" + ], + "training_samples": 510, + "testing_samples": 128, + "train_accuracy": 0.4803921568627451, + "test_accuracy": 0.484375, + "model_type": "cnn", + "device": "cpu", + "n_estimators": 50, + "max_depth": null, + "learning_rate": null, + "cnn_epochs": 25, + "n_features": 3, + "n_classes": 7, + "bbox": [ + 105.6, + 9.3, + 106.2, + 9.8 + ], + "time_range": "2023-03-01/2023-05-25", + "resolution": 20 +} \ No newline at end of file diff --git a/model_train/model_cnn_20251214_182307.joblib b/model_train/model_cnn_20251214_182307.joblib new file mode 100644 index 0000000..1ecbfc2 --- /dev/null +++ b/model_train/model_cnn_20251214_182307.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f906aa2a61d2e793a3463df95fe3e364134934af326771efae90d80caca419 +size 41536 diff --git a/model_train/model_cnn_20251214_182307_info.json b/model_train/model_cnn_20251214_182307_info.json new file mode 100644 index 0000000..71e0f37 --- /dev/null +++ b/model_train/model_cnn_20251214_182307_info.json @@ -0,0 +1,33 @@ +{ + "timestamp": "2025-12-14T18:23:10.713912", + "data_source": "Microsoft Planetary Computer STAC", + "collections": [ + "sentinel-2-l2a", + "sentinel-1-rtc" + ], + "features": [ + "NDVI_mean", + "VH_dB_mean", + "VV_dB_mean" + ], + "training_samples": 510, + "testing_samples": 128, + "train_accuracy": 0.46862745098039216, + "test_accuracy": 0.4609375, + "model_type": "cnn", + "device": "cpu", + "n_estimators": 50, + "max_depth": null, + "learning_rate": null, + "cnn_epochs": 25, + "n_features": 3, + "n_classes": 7, + "bbox": [ + 105.6, + 9.3, + 106.2, + 9.8 + ], + "time_range": "2023-03-01/2023-05-31", + "resolution": 20 +} \ No newline at end of file diff --git a/model_train/model_xgboost_gpu_20251214_133256.joblib b/model_train/model_xgboost_gpu_20251214_133256.joblib new file mode 100644 index 0000000..a8bca94 --- /dev/null +++ b/model_train/model_xgboost_gpu_20251214_133256.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b717f564f9413a6e5c9cd3f7011cbc18be02479691d01c554986defb400f0490 +size 1347520 diff --git a/model_train/model_xgboost_gpu_20251214_133256_info.json b/model_train/model_xgboost_gpu_20251214_133256_info.json new file mode 100644 index 0000000..790ed09 --- /dev/null +++ b/model_train/model_xgboost_gpu_20251214_133256_info.json @@ -0,0 +1,31 @@ +{ + "timestamp": "2025-12-14T13:40:03.325930", + "data_source": "Microsoft Planetary Computer STAC", + "collections": [ + "sentinel-2-l2a", + "sentinel-1-rtc" + ], + "features": [ + "NDVI_mean", + "VH_dB_mean", + "VV_dB_mean" + ], + "training_samples": 510, + "testing_samples": 128, + "train_accuracy": 1.0, + "test_accuracy": 0.578125, + "model_type": "XGBClassifier", + "device": "cuda:0", + "tree_method": "hist", + "n_estimators": 100, + "max_depth": 20, + "learning_rate": 0.1, + "bbox": [ + 105.6, + 9.3, + 106.2, + 9.8 + ], + "time_range": "2023-03-01/2023-05-31", + "resolution": 20 +} \ No newline at end of file diff --git a/model_train/model_xgboost_gpu_20251214_164426.joblib b/model_train/model_xgboost_gpu_20251214_164426.joblib new file mode 100644 index 0000000..2843e78 --- /dev/null +++ b/model_train/model_xgboost_gpu_20251214_164426.joblib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a44efc173619782c8add224e9024f33ef0150ab192b5435d009cb119c379f03c +size 2088632 diff --git a/model_train/model_xgboost_gpu_20251214_164426_info.json b/model_train/model_xgboost_gpu_20251214_164426_info.json new file mode 100644 index 0000000..a0ddcb0 --- /dev/null +++ b/model_train/model_xgboost_gpu_20251214_164426_info.json @@ -0,0 +1,31 @@ +{ + "timestamp": "2025-12-14T16:52:19.862770", + "data_source": "Microsoft Planetary Computer STAC", + "collections": [ + "sentinel-2-l2a", + "sentinel-1-rtc" + ], + "features": [ + "NDVI_mean", + "VH_dB_mean", + "VV_dB_mean" + ], + "training_samples": 859, + "testing_samples": 215, + "train_accuracy": 0.9976717112922002, + "test_accuracy": 0.6837209302325581, + "model_type": "XGBClassifier", + "device": "cuda:0", + "tree_method": "hist", + "n_estimators": 100, + "max_depth": 20, + "learning_rate": 0.1, + "bbox": [ + 105.6, + 9.3, + 106.2, + 9.8 + ], + "time_range": "2023-03-01/2023-12-31", + "resolution": 20 +} \ No newline at end of file diff --git a/predictions/prediction_20251214_100243.tif b/predictions/prediction_20251214_100243.tif new file mode 100644 index 0000000..84cafa6 --- /dev/null +++ b/predictions/prediction_20251214_100243.tif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179d3a61420c1552e9653de5c7657665c9880ad29d751e56bae646fb3b634687 +size 73272920 diff --git a/predictions/prediction_20251214_165405.tif b/predictions/prediction_20251214_165405.tif new file mode 100644 index 0000000..07cc4d8 --- /dev/null +++ b/predictions/prediction_20251214_165405.tif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8fb13c2e466b9811104cbd7747ea6ca8c14ddb44802d934aadfe52a4b5dd916 +size 73272920 diff --git a/predictions/prediction_20251214_182353.tif b/predictions/prediction_20251214_182353.tif new file mode 100644 index 0000000..4876e4b --- /dev/null +++ b/predictions/prediction_20251214_182353.tif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2fe092e1fd96e56da519acffe5ca5c9a8246e796bc5700cffb9694dc99f3aec +size 73272920 diff --git a/train_module.py b/train_module.py index e4b58e7..f741a41 100644 --- a/train_module.py +++ b/train_module.py @@ -9,11 +9,109 @@ import geopandas as gpd from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from sklearn.metrics import classification_report, confusion_matrix +from sklearn.ensemble import RandomForestClassifier +from sklearn.tree import DecisionTreeClassifier +from sklearn.svm import SVC from xgboost import XGBClassifier import joblib from datetime import datetime import json import os +import warnings +import hashlib +from pathlib import Path +warnings.filterwarnings('ignore') + +# PyTorch for CNN +try: + import torch + import torch.nn as nn + import torch.nn.functional as F + import torch.optim as optim + from torch.utils.data import TensorDataset, DataLoader + PYTORCH_AVAILABLE = True +except ImportError: + PYTORCH_AVAILABLE = False + print("Warning: PyTorch not available. CNN model will not work.") + +# Define CNN model class for PyTorch +class CNNClassifier(nn.Module): + def __init__(self, n_features, n_classes): + super(CNNClassifier, self).__init__() + self.n_features = n_features + self.n_classes = n_classes + + # For small feature sets (like 3 features), use simpler architecture + if n_features < 8: + # Simple fully connected network for small features + self.use_conv = False + self.fc1 = nn.Linear(n_features, 64) + self.dropout1 = nn.Dropout(0.3) + self.fc2 = nn.Linear(64, 128) + self.dropout2 = nn.Dropout(0.5) + self.fc3 = nn.Linear(128, n_classes) + else: + # CNN architecture for larger feature sets + self.use_conv = True + self.conv1 = nn.Conv1d(in_channels=1, out_channels=32, kernel_size=3, padding=1) + self.pool1 = nn.MaxPool1d(kernel_size=2) + self.conv2 = nn.Conv1d(in_channels=32, out_channels=64, kernel_size=3, padding=1) + self.pool2 = nn.MaxPool1d(kernel_size=2) + + # Calculate size after convolutions + conv_output_size = (n_features // 2 // 2) * 64 + + # Fully connected layers + self.fc1 = nn.Linear(conv_output_size, 128) + self.dropout = nn.Dropout(0.5) + self.fc2 = nn.Linear(128, n_classes) + + def forward(self, x): + # x shape: (batch, n_features) or (batch, 1, n_features) + if self.use_conv: + # CNN path for larger feature sets + if len(x.shape) == 2: + x = x.unsqueeze(1) # Add channel dimension + x = F.relu(self.conv1(x)) + x = self.pool1(x) + x = F.relu(self.conv2(x)) + x = self.pool2(x) + x = x.view(x.size(0), -1) # Flatten + x = F.relu(self.fc1(x)) + x = self.dropout(x) + x = self.fc2(x) + else: + # Fully connected path for small feature sets + if len(x.shape) == 3: + x = x.squeeze(1) # Remove channel dimension if present + x = F.relu(self.fc1(x)) + x = self.dropout1(x) + x = F.relu(self.fc2(x)) + x = self.dropout2(x) + x = self.fc3(x) + return x + + def predict(self, X): + """Scikit-learn style predict method""" + self.eval() + with torch.no_grad(): + if isinstance(X, np.ndarray): + X = torch.FloatTensor(X) + # Handle both 2D and 3D inputs + if not self.use_conv and len(X.shape) == 3: + X = X.squeeze(1) + elif self.use_conv and len(X.shape) == 2: + X = X.unsqueeze(1) + outputs = self(X) + _, predicted = torch.max(outputs, 1) + return predicted.cpu().numpy() + + def score(self, X, y): + """Scikit-learn style score method""" + predictions = self.predict(X) + if isinstance(y, torch.Tensor): + y = y.cpu().numpy() + return np.mean(predictions == y) # Microsoft Planetary Computer imports import planetary_computer @@ -28,10 +126,12 @@ def train_model( cloud_cover=30, resolution=20, training_shapefile='train/ST_training data_updated_1130points_new.shp', + model_type='xgboost', n_estimators=100, max_depth=20, learning_rate=0.1, use_gpu=True, + use_cache=True, output_model_path=None, status_callback=None, cancel_check=None @@ -77,139 +177,185 @@ def train_model( # Auto-generate output path if not provided if output_model_path is None: timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - output_model_path = f'model_train/model_xgboost_gpu_{timestamp}.joblib' + output_model_path = f'model_train/model_{model_type}_{timestamp}.joblib' - # Connect to Microsoft Planetary Computer - update_status("Connecting to Microsoft Planetary Computer...", 0) - catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1") - check_cancellation() + # ============ CACHE SYSTEM ============ + # Create cache directory + cache_dir = Path("dataset_cache") + cache_dir.mkdir(exist_ok=True) - # Search for Sentinel-2 scenes - update_status("Searching for Sentinel-2 scenes...", 10) - query_s2 = catalog.search( - collections=["sentinel-2-l2a"], - bbox=bbox, - datetime=time_range, - query={"eo:cloud_cover": {"lt": cloud_cover}} - ) - items_s2 = list(query_s2.item_collection()) + # Generate cache key from parameters + cache_params = f"{bbox}_{time_range}_{max_scenes}_{cloud_cover}_{resolution}" + cache_key = hashlib.md5(cache_params.encode()).hexdigest() + cache_file = cache_dir / f"training_data_{cache_key}.joblib" - check_cancellation() + features = None + labels = None - # Limit scenes - if len(items_s2) > max_scenes: - step = len(items_s2) // max_scenes - items_s2 = items_s2[::step][:max_scenes] - - update_status(f"Found {len(items_s2)} Sentinel-2 scenes", 20) - - # Sign and load Sentinel-2 data - update_status("Loading Sentinel-2 data...", 25) - items_s2 = [planetary_computer.sign(item) for item in items_s2] - ds_s2 = stac_load( - items_s2, - bands=["B04", "B08", "SCL"], - crs="EPSG:32648", - resolution=resolution, - bbox=bbox, - patch_url=planetary_computer.sign, - fail_on_error=False, - ) - ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"}) - - check_cancellation() - - # Search for Sentinel-1 scenes - update_status("Searching for Sentinel-1 scenes...", 35) - query_s1 = catalog.search( - collections=["sentinel-1-rtc"], - bbox=bbox, - datetime=time_range, - ) - items_s1 = list(query_s1.item_collection()) - - # Limit scenes - if len(items_s1) > max_scenes: - step = len(items_s1) // max_scenes - items_s1 = items_s1[::step][:max_scenes] - - update_status(f"Found {len(items_s1)} Sentinel-1 scenes", 40) - - # Sign and load Sentinel-1 data - update_status("Loading Sentinel-1 data...", 45) - items_s1 = [planetary_computer.sign(item) for item in items_s1] - ds_s1 = stac_load( - items_s1, - bands=["vv", "vh"], - crs="EPSG:32648", - resolution=resolution, - bbox=bbox, - patch_url=planetary_computer.sign, - fail_on_error=False, - ) - - # Convert to dB - ds_s1['vv_db'] = 10 * np.log10(ds_s1['vv'].where(ds_s1['vv'] > 0)) - ds_s1['vh_db'] = 10 * np.log10(ds_s1['vh'].where(ds_s1['vh'] > 0)) - - check_cancellation() - - # Calculate NDVI - update_status("Calculating NDVI...", 50) - ndvi = (ds_s2['nir'] - ds_s2['red']) / (ds_s2['nir'] + ds_s2['red'] + 1e-8) - - # Apply cloud mask - cloud_mask = ds_s2['scl'].isin([1, 3, 8, 9, 10]) - ndvi_masked = ndvi.where(~cloud_mask) - ndvi_mean = ndvi_masked.mean(dim='time') - - # Load training data - update_status("Loading training data...", 55) - train_gdf = gpd.read_file(training_shapefile) - - if train_gdf.crs != 'EPSG:32648': - train_gdf = train_gdf.to_crs('EPSG:32648') - - # Auto-detect label column - label_column = None - for col in ['HT_code', 'Ma_LU', 'LU2022', 'Hientrang', 'class', 'Class', 'CLASS']: - if col in train_gdf.columns: - label_column = col - break - - if label_column is None: - raise ValueError(f"Cannot find label column in shapefile. Available: {list(train_gdf.columns)}") - - # Extract features - update_status("Extracting features from training points...", 60) - features = [] - labels = [] - - for idx, row in train_gdf.iterrows(): - point = row.geometry - x_coord = point.x - y_coord = point.y - label = row[label_column] - + # Try to load from cache + if use_cache and cache_file.exists(): + update_status(f"📦 Loading cached dataset from {cache_file.name}...", 5) try: - ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method='nearest').values - vh_val = ds_s1['vh_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values - vv_val = ds_s1['vv_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values + cached_data = joblib.load(cache_file) + features = cached_data['features'] + labels = cached_data['labels'] + update_status(f"✅ Loaded {len(features)} samples from cache (skipped satellite download!)", 50) + except Exception as e: + update_status(f"⚠️ Cache load failed: {str(e)}, downloading fresh data...", 10) + features = None + + # If no cache or cache failed, download data + if features is None: + update_status("📡 Cache not found or disabled, downloading satellite data...", 10) + + # Connect to Microsoft Planetary Computer + update_status("Connecting to Microsoft Planetary Computer...", 12) + catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1") + check_cancellation() + + # Search for Sentinel-2 scenes + update_status("Searching for Sentinel-2 scenes...", 10) + query_s2 = catalog.search( + collections=["sentinel-2-l2a"], + bbox=bbox, + datetime=time_range, + query={"eo:cloud_cover": {"lt": cloud_cover}} + ) + items_s2 = list(query_s2.item_collection()) + + check_cancellation() + + # Limit scenes + if len(items_s2) > max_scenes: + step = len(items_s2) // max_scenes + items_s2 = items_s2[::step][:max_scenes] + + update_status(f"Found {len(items_s2)} Sentinel-2 scenes", 20) + + # Sign and load Sentinel-2 data + update_status("Loading Sentinel-2 data...", 25) + items_s2 = [planetary_computer.sign(item) for item in items_s2] + ds_s2 = stac_load( + items_s2, + bands=["B04", "B08", "SCL"], + crs="EPSG:32648", + resolution=resolution, + bbox=bbox, + patch_url=planetary_computer.sign, + fail_on_error=False, + ) + ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"}) + + check_cancellation() + + # Search for Sentinel-1 scenes + update_status("Searching for Sentinel-1 scenes...", 35) + query_s1 = catalog.search( + collections=["sentinel-1-rtc"], + bbox=bbox, + datetime=time_range, + ) + items_s1 = list(query_s1.item_collection()) + + # Limit scenes + if len(items_s1) > max_scenes: + step = len(items_s1) // max_scenes + items_s1 = items_s1[::step][:max_scenes] + + update_status(f"Found {len(items_s1)} Sentinel-1 scenes", 40) + + # Sign and load Sentinel-1 data + update_status("Loading Sentinel-1 data...", 45) + items_s1 = [planetary_computer.sign(item) for item in items_s1] + ds_s1 = stac_load( + items_s1, + bands=["vv", "vh"], + crs="EPSG:32648", + resolution=resolution, + bbox=bbox, + patch_url=planetary_computer.sign, + fail_on_error=False, + ) + + # Convert to dB + ds_s1['vv_db'] = 10 * np.log10(ds_s1['vv'].where(ds_s1['vv'] > 0)) + ds_s1['vh_db'] = 10 * np.log10(ds_s1['vh'].where(ds_s1['vh'] > 0)) + + check_cancellation() + + # Calculate NDVI + update_status("Calculating NDVI...", 50) + ndvi = (ds_s2['nir'] - ds_s2['red']) / (ds_s2['nir'] + ds_s2['red'] + 1e-8) + + # Apply cloud mask + cloud_mask = ds_s2['scl'].isin([1, 3, 8, 9, 10]) + ndvi_masked = ndvi.where(~cloud_mask) + ndvi_mean = ndvi_masked.mean(dim='time') + + # Load training data + update_status("Loading training data...", 55) + train_gdf = gpd.read_file(training_shapefile) + + if train_gdf.crs != 'EPSG:32648': + train_gdf = train_gdf.to_crs('EPSG:32648') + + # Auto-detect label column + label_column = None + for col in ['HT_code', 'Ma_LU', 'LU2022', 'Hientrang', 'class', 'Class', 'CLASS']: + if col in train_gdf.columns: + label_column = col + break + + if label_column is None: + raise ValueError(f"Cannot find label column in shapefile. Available: {list(train_gdf.columns)}") + + # Extract features + update_status("Extracting features from training points...", 60) + features = [] + labels = [] + + for idx, row in train_gdf.iterrows(): + point = row.geometry + x_coord = point.x + y_coord = point.y + label = row[label_column] - feature_vec = [ndvi_val, vh_val, vv_val] - - if not np.isnan(feature_vec).any(): - features.append(feature_vec) - labels.append(label) - except: - continue - - features = np.array(features) - labels = np.array(labels) - - check_cancellation() - - update_status(f"Extracted {len(features)} valid training samples", 70) + try: + ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method='nearest').values + vh_val = ds_s1['vh_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values + vv_val = ds_s1['vv_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values + + feature_vec = [ndvi_val, vh_val, vv_val] + + if not np.isnan(feature_vec).any(): + features.append(feature_vec) + labels.append(label) + except: + continue + + features = np.array(features) + labels = np.array(labels) + + check_cancellation() + + update_status(f"Extracted {len(features)} valid training samples", 70) + + # ============ SAVE TO CACHE ============ + if use_cache: + update_status(f"💾 Saving dataset to cache for future use...", 72) + try: + cache_data = { + 'features': features, + 'labels': labels, + 'bbox': bbox, + 'time_range': time_range, + 'resolution': resolution, + 'timestamp': datetime.now().isoformat() + } + joblib.dump(cache_data, cache_file) + update_status(f"✅ Cached to {cache_file.name}", 75) + except Exception as e: + update_status(f"⚠️ Cache save failed: {str(e)}", 75) # Encode labels label_encoder = LabelEncoder() @@ -220,33 +366,115 @@ def train_model( features, labels_encoded, test_size=0.2, random_state=42, stratify=labels_encoded ) - # Train XGBoost model - update_status("Training XGBoost model on GPU...", 75) + # Train model based on selected type + update_status(f"Training {model_type.upper()} model...", 75) device = 'cuda:0' if use_gpu else 'cpu' - xgb_model = XGBClassifier( - n_estimators=n_estimators, - max_depth=max_depth, - learning_rate=learning_rate, - device=device, - tree_method='hist', - random_state=42, - eval_metric='mlogloss', - verbosity=0 - ) + if model_type == 'xgboost': + model = XGBClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + learning_rate=learning_rate, + device=device if use_gpu else 'cpu', + tree_method='hist', + random_state=42, + eval_metric='mlogloss', + verbosity=0 + ) + elif model_type == 'random_forest': + model = RandomForestClassifier( + n_estimators=n_estimators, + max_depth=max_depth, + random_state=42, + n_jobs=-1, # Use all cores + verbose=0 + ) + elif model_type == 'decision_tree': + model = DecisionTreeClassifier( + max_depth=max_depth, + random_state=42 + ) + elif model_type == 'svm': + model = SVC( + kernel='rbf', + random_state=42, + verbose=False + ) + elif model_type == 'cnn': + if not PYTORCH_AVAILABLE: + raise ImportError("PyTorch is required for CNN. Install: pip install torch") + + # CNN requires reshaping data + n_features = X_train.shape[1] + n_classes = len(np.unique(y_train)) + + # Build PyTorch CNN model + device = torch.device('cuda' if torch.cuda.is_available() and use_gpu else 'cpu') + update_status(f"Building CNN model on {device}...", 75) + + model = CNNClassifier(n_features, n_classes).to(device) + + # Convert to PyTorch tensors + X_train_tensor = torch.FloatTensor(X_train).unsqueeze(1) # Add channel dim: (N, 1, features) + y_train_tensor = torch.LongTensor(y_train) + X_test_tensor = torch.FloatTensor(X_test).unsqueeze(1) + y_test_tensor = torch.LongTensor(y_test) + + # Create data loaders + train_dataset = TensorDataset(X_train_tensor, y_train_tensor) + train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True) + + # Loss and optimizer + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=0.001) + + # Train CNN + update_status("Training CNN model with PyTorch...", 80) + epochs = min(50, n_estimators // 2) # Use n_estimators as epochs + + model.train() + for epoch in range(epochs): + epoch_loss = 0.0 + for batch_X, batch_y in train_loader: + batch_X, batch_y = batch_X.to(device), batch_y.to(device) + + optimizer.zero_grad() + outputs = model(batch_X) + loss = criterion(outputs, batch_y) + loss.backward() + optimizer.step() + + epoch_loss += loss.item() + + if (epoch + 1) % 10 == 0: + avg_loss = epoch_loss / len(train_loader) + update_status(f"CNN Epoch {epoch+1}/{epochs}, Loss: {avg_loss:.4f}", 80 + (epoch / epochs) * 10) + + # Move model to CPU for saving (compatible with non-GPU systems) + model = model.cpu() + model.device_used = str(device) + else: + raise ValueError(f"Unknown model type: {model_type}. Choose: xgboost, random_forest, decision_tree, svm, cnn") - xgb_model.fit(X_train, y_train) + # Fit non-CNN models + if model_type != 'cnn': + model.fit(X_train, y_train) # Evaluate update_status("Evaluating model...", 90) - train_score = xgb_model.score(X_train, y_train) - test_score = xgb_model.score(X_test, y_test) + if model_type == 'cnn': + # PyTorch CNN evaluation + train_score = model.score(X_train, y_train) + test_score = model.score(X_test, y_test) + else: + train_score = model.score(X_train, y_train) + test_score = model.score(X_test, y_test) # Save model update_status("Saving model...", 95) os.makedirs(os.path.dirname(output_model_path), exist_ok=True) - joblib.dump({'model': xgb_model, 'label_encoder': label_encoder}, output_model_path) + joblib.dump({'model': model, 'label_encoder': label_encoder}, output_model_path) # Save model info info = { @@ -258,12 +486,14 @@ def train_model( "testing_samples": len(X_test), "train_accuracy": float(train_score), "test_accuracy": float(test_score), - "model_type": "XGBClassifier", - "device": device, - "tree_method": "hist", - "n_estimators": n_estimators, - "max_depth": max_depth, - "learning_rate": learning_rate, + "model_type": model_type, + "device": device if model_type == 'xgboost' else 'cpu', + "n_estimators": n_estimators if model_type in ['xgboost', 'random_forest', 'cnn'] else None, + "max_depth": max_depth if model_type != 'cnn' else None, + "learning_rate": learning_rate if model_type == 'xgboost' else None, + "cnn_epochs": min(50, n_estimators // 2) if model_type == 'cnn' else None, + "n_features": X_train.shape[1], + "n_classes": len(np.unique(y_train)), "bbox": bbox, "time_range": time_range, "resolution": resolution diff --git a/training_interface.html b/training_interface.html index 3899c0c..89332e9 100644 --- a/training_interface.html +++ b/training_interface.html @@ -258,6 +258,29 @@ margin: 5px 0; color: #666; } + + /* Animations for notifications */ + @keyframes slideIn { + from { + transform: translateX(400px); + opacity: 0; + } + to { + transform: translateX(0); + opacity: 1; + } + } + + @keyframes slideOut { + from { + transform: translateX(0); + opacity: 1; + } + to { + transform: translateX(400px); + opacity: 0; + } + }
@@ -369,7 +392,18 @@ -Đang tải...
Chưa có file dự đoán nào.
'; + } + } catch (error) { + console.error('Error loading predictions list:', error); + document.getElementById('predictionsList').innerHTML = + 'Lỗi tải danh sách: ' + error.message + '
'; + } + } + // View prediction result details function viewPredictionResult() { if (window.lastPredictionResult) {