Files
remote-sensing/01.train_ODC_MobileNet.ipynb
T
2026-03-07 17:14:01 +07:00

441 lines
20 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "17da4353",
"metadata": {},
"outputs": [],
"source": [
"import importlib\n",
"import new_import_ODC as odc_tools\n",
"importlib.reload(odc_tools)\n",
"from new_import_ODC import *\n",
"print(\"✅ Import thành công\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9c063be3",
"metadata": {},
"outputs": [],
"source": [
"# Khởi tạo Dask + Datacube + S3\n",
"cluster, client = initialize_dask(use_gateway=True)\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=True)\n",
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83784d01",
"metadata": {},
"outputs": [],
"source": [
"# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"\n",
"data_sen2 = load_data(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c3faed92",
"metadata": {},
"outputs": [],
"source": [
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
"data_clean = mask_clean(data_sen2)\n",
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
"data_fill = fill_nan(data_ndvi)\n",
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "569bfebb",
"metadata": {},
"outputs": [],
"source": [
"# Tải Sentinel-1 (SAR VV/VH)\n",
"data_sen1 = load_data_sen1(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ecc56c2f",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
"}\n",
"\n",
"train_data = load_train_data(label_mapping=label_mapping)\n",
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
"\n",
"X_train_np = np.array(X_train, dtype=np.float32)\n",
"X_val_np = np.array(X_val, dtype=np.float32)\n",
"X_test_np = np.array(X_test, dtype=np.float32)\n",
"y_train_np = np.array(y_train, dtype=np.int64)\n",
"y_val_np = np.array(y_val, dtype=np.int64)\n",
"y_test_np = np.array(y_test, dtype=np.int64)\n",
"\n",
"n_features = X_train_np.shape[1]\n",
"n_classes = len(np.unique(y_train_np))\n",
"\n",
"print(f\"✅ Train: {X_train_np.shape} Val: {X_val_np.shape} Test: {X_test_np.shape}\")\n",
"print(f\" n_features={n_features} n_classes={n_classes}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5492528b",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import TensorDataset, DataLoader\n",
"\n",
"# ── MobileNetV3 + LR-ASPP classifier ──────────────────────────────────────────\n",
"class MobileNetLRASPPClassifier(nn.Module):\n",
" \"\"\"\n",
" MobileNetV3-inspired backbone with LR-ASPP (Lite Reduced ASPP) head\n",
" for land-use classification on flat feature vectors.\n",
" \"\"\"\n",
" def __init__(self, n_features, n_classes):\n",
" super().__init__()\n",
" # Feature extraction backbone\n",
" self.feature_extractor = nn.Sequential(\n",
" nn.Linear(n_features, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True), nn.Dropout(0.2),\n",
" nn.Linear(128, 256), nn.BatchNorm1d(256), nn.ReLU(inplace=True), nn.Dropout(0.3),\n",
" nn.Linear(256, 512), nn.BatchNorm1d(512), nn.ReLU(inplace=True), nn.Dropout(0.3),\n",
" )\n",
" # LR-ASPP Branch 1: global pooling → 128\n",
" self.global_pool = nn.AdaptiveAvgPool1d(1)\n",
" self.global_conv = nn.Sequential(nn.Linear(512, 128), nn.ReLU(inplace=True))\n",
" # LR-ASPP Branch 2: direct 1×1 → 128\n",
" self.branch_conv = nn.Sequential(nn.Linear(512, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True))\n",
" # Fusion → n_classes\n",
" self.classifier = nn.Sequential(\n",
" nn.Linear(256, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True), nn.Dropout(0.4),\n",
" nn.Linear(128, n_classes),\n",
" )\n",
"\n",
" def forward(self, x):\n",
" feat = self.feature_extractor(x)\n",
" global_feat = self.global_pool(feat.unsqueeze(-1)).squeeze(-1)\n",
" global_feat = self.global_conv(global_feat)\n",
" branch_feat = self.branch_conv(feat)\n",
" fused = torch.cat([global_feat, branch_feat], dim=1)\n",
" return self.classifier(fused)\n",
"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"print(f\"✅ Device: {device}\")\n",
"model = MobileNetLRASPPClassifier(n_features, n_classes).to(device)\n",
"print(model)\n",
"total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
"print(f\" Trainable params: {total_params:,}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9da40f5a",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# ── Hyper-parameters ───────────────────────────────────────────────────────────\n",
"LEARNING_RATE = 1e-3\n",
"BATCH_SIZE = 64\n",
"N_EPOCHS = 60\n",
"PATIENCE = 10\n",
"CONV_THRESHOLD = 1e-4 # cải thiện val_loss < THRESHOLD → đánh dấu hội tụ\n",
"CONV_WINDOW = 3 # cần CONV_WINDOW bước liên tiếp thỏa mãn\n",
"\n",
"# ── Tensors & DataLoaders ──────────────────────────────────────────────────────\n",
"X_tr_t = torch.FloatTensor(X_train_np)\n",
"y_tr_t = torch.LongTensor(y_train_np)\n",
"X_va_t = torch.FloatTensor(X_val_np)\n",
"y_va_t = torch.LongTensor(y_val_np)\n",
"\n",
"train_loader = DataLoader(TensorDataset(X_tr_t, y_tr_t), batch_size=BATCH_SIZE, shuffle=True)\n",
"val_loader = DataLoader(TensorDataset(X_va_t, y_va_t), batch_size=BATCH_SIZE, shuffle=False)\n",
"\n",
"# ── Class-weighted loss ────────────────────────────────────────────────────────\n",
"class_counts = np.bincount(y_train_np)\n",
"class_weights = 1.0 / (class_counts + 1e-6)\n",
"class_weights = class_weights / class_weights.sum() * n_classes\n",
"criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weights).to(device))\n",
"\n",
"optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)\n",
"scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode=\"min\", factor=0.5, patience=5)\n",
"\n",
"# ── Training loop ──────────────────────────────────────────────────────────────\n",
"train_losses, val_losses, val_accs, lr_history = [], [], [], []\n",
"best_val_loss = float(\"inf\")\n",
"best_epoch = 1\n",
"best_state_dict = None\n",
"patience_counter = 0\n",
"convergence_epoch = None # ← điểm hội tụ sẽ được ghi lại ở đây\n",
"early_stop_epoch = None\n",
"\n",
"print(\"🚀 Training MobileNetV3 + LR-ASPP...\")\n",
"print(f\" conv_threshold={CONV_THRESHOLD} conv_window={CONV_WINDOW} patience={PATIENCE}\")\n",
"for epoch in range(1, N_EPOCHS + 1):\n",
" # --- Train ---\n",
" model.train()\n",
" epoch_loss = 0.0\n",
" for bx, by in train_loader:\n",
" bx, by = bx.to(device), by.to(device)\n",
" optimizer.zero_grad()\n",
" loss = criterion(model(bx), by)\n",
" loss.backward()\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
" optimizer.step()\n",
" epoch_loss += loss.item()\n",
" avg_train_loss = epoch_loss / len(train_loader)\n",
"\n",
" # --- Validate ---\n",
" model.eval()\n",
" val_loss = 0.0; correct = 0; total = 0\n",
" with torch.no_grad():\n",
" for bx, by in val_loader:\n",
" bx, by = bx.to(device), by.to(device)\n",
" out = model(bx)\n",
" val_loss += criterion(out, by).item()\n",
" pred = out.argmax(1)\n",
" correct += (pred == by).sum().item()\n",
" total += by.size(0)\n",
" avg_val_loss = val_loss / len(val_loader)\n",
" val_acc = correct / total\n",
"\n",
" scheduler.step(avg_val_loss)\n",
" lr = optimizer.param_groups[0][\"lr\"]\n",
" lr_history.append(lr)\n",
"\n",
" train_losses.append(avg_train_loss)\n",
" val_losses.append(avg_val_loss)\n",
" val_accs.append(val_acc)\n",
"\n",
" # ── Phát hiện điểm hội tụ (sliding window trên val_loss) ──────────────\n",
" if convergence_epoch is None and epoch >= CONV_WINDOW + 1:\n",
" window = val_losses[-(CONV_WINDOW + 1):]\n",
" improvements = [abs(window[i] - window[i - 1]) for i in range(1, len(window))]\n",
" if all(imp < CONV_THRESHOLD for imp in improvements):\n",
" convergence_epoch = epoch - CONV_WINDOW + 1\n",
" print(f\" 📍 Hội tụ phát hiện tại epoch {convergence_epoch} \"\n",
" f\"(val_loss={val_losses[convergence_epoch-1]:.4f})\")\n",
"\n",
" if epoch % 5 == 0 or epoch == 1:\n",
" print(f\" Epoch {epoch:3d}/{N_EPOCHS} train={avg_train_loss:.4f} \"\n",
" f\"val={avg_val_loss:.4f} acc={val_acc:.4f} lr={lr:.2e}\")\n",
"\n",
" # ── Early stopping ─────────────────────────────────────────────────────\n",
" if avg_val_loss < best_val_loss:\n",
" best_val_loss = avg_val_loss\n",
" best_epoch = epoch\n",
" best_state_dict = {k: v.clone() for k, v in model.state_dict().items()}\n",
" patience_counter = 0\n",
" else:\n",
" patience_counter += 1\n",
" if patience_counter >= PATIENCE:\n",
" early_stop_epoch = epoch\n",
" print(f\"⏹ Early stopping tại epoch {epoch} (patience={PATIENCE})\")\n",
" break\n",
"\n",
"# Restore best weights\n",
"model.load_state_dict(best_state_dict)\n",
"total_epochs = len(train_losses)\n",
"print(f\"\\n✅ Training hoàn tất! Best epoch={best_epoch} Best val_loss={best_val_loss:.4f}\")\n",
"if convergence_epoch:\n",
" print(f\" Điểm hội tụ : epoch {convergence_epoch}\")\n",
"\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"# PHÂN TÍCH ĐIỂM HỘI TỤ — MobileNetV3 + LR-ASPP\n",
"# EMA-smoothed curves + annotated convergence / best / early-stop markers\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"epochs_axis = list(range(1, total_epochs + 1))\n",
"\n",
"# Exponential Moving Average smoothing\n",
"def ema(values, alpha=0.2):\n",
" s = [values[0]]\n",
" for v in values[1:]:\n",
" s.append(alpha * v + (1 - alpha) * s[-1])\n",
" return s\n",
"\n",
"val_losses_ema = ema(val_losses, alpha=0.3)\n",
"train_losses_ema = ema(train_losses, alpha=0.3)\n",
"val_accs_ema = ema(val_accs, alpha=0.3)\n",
"\n",
"# ── Tính ΔVal-loss per epoch ──────────────────────────────────────────────────\n",
"delta_val = [abs(val_losses_ema[i] - val_losses_ema[i-1])\n",
" for i in range(1, len(val_losses_ema))]\n",
"\n",
"fig, axes = plt.subplots(2, 2, figsize=(16, 10))\n",
"\n",
"# --- Top-left: Loss curves ---\n",
"ax = axes[0, 0]\n",
"ax.plot(epochs_axis, train_losses, \"b-\", alpha=0.25, linewidth=0.8)\n",
"ax.plot(epochs_axis, train_losses_ema, \"b-\", linewidth=2, label=\"Train loss (EMA)\")\n",
"ax.plot(epochs_axis, val_losses, \"g-\", alpha=0.25, linewidth=0.8)\n",
"ax.plot(epochs_axis, val_losses_ema, \"g-\", linewidth=2, label=\"Val loss (EMA)\")\n",
"ax.axvline(x=best_epoch, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best epoch={best_epoch}\")\n",
"if convergence_epoch:\n",
" ax.axvline(x=convergence_epoch, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"HỘI TỤ epoch={convergence_epoch}\")\n",
"if early_stop_epoch:\n",
" ax.axvline(x=early_stop_epoch, color=\"gray\", linestyle=\"-.\", linewidth=1.5,\n",
" label=f\"Early stop epoch={early_stop_epoch}\")\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Loss\")\n",
"ax.set_title(\"Loss Curves (raw + EMA)\")\n",
"ax.legend(fontsize=8)\n",
"ax.grid(True, alpha=0.3)\n",
"\n",
"# --- Top-right: Val accuracy ---\n",
"ax = axes[0, 1]\n",
"ax.plot(epochs_axis, val_accs, \"g-\", alpha=0.3, linewidth=0.8)\n",
"ax.plot(epochs_axis, val_accs_ema, \"g-\", linewidth=2, label=\"Val acc (EMA)\")\n",
"ax.axvline(x=best_epoch, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best epoch={best_epoch} ({val_accs[best_epoch-1]*100:.2f}%)\")\n",
"if convergence_epoch:\n",
" ax.axvline(x=convergence_epoch, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"HỘI TỤ epoch={convergence_epoch} ({val_accs[convergence_epoch-1]*100:.2f}%)\")\n",
"if early_stop_epoch:\n",
" ax.axvline(x=early_stop_epoch, color=\"gray\", linestyle=\"-.\", linewidth=1.5,\n",
" label=f\"Early stop\")\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Accuracy\")\n",
"ax.set_title(\"Val Accuracy Curve\")\n",
"ax.legend(fontsize=8)\n",
"ax.grid(True, alpha=0.3)\n",
"\n",
"# --- Bottom-left: ΔVal-loss (marginal improvement) ---\n",
"ax = axes[1, 0]\n",
"ax.bar(epochs_axis[1:], delta_val,\n",
" color=[\"green\" if d > CONV_THRESHOLD else \"salmon\" for d in delta_val],\n",
" alpha=0.75)\n",
"ax.axhline(y=CONV_THRESHOLD, color=\"red\", linestyle=\"--\",\n",
" label=f\"Threshold = {CONV_THRESHOLD:.0e}\")\n",
"if convergence_epoch:\n",
" ax.axvline(x=convergence_epoch, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"HỘI TỤ epoch={convergence_epoch}\")\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"|ΔVal Loss|\")\n",
"ax.set_title(\"Marginal Val-Loss Improvement per Epoch\")\n",
"ax.legend(fontsize=8)\n",
"ax.grid(True, alpha=0.3)\n",
"\n",
"# --- Bottom-right: Learning rate schedule ---\n",
"ax = axes[1, 1]\n",
"ax.semilogy(epochs_axis, lr_history, \"purple\", linewidth=2)\n",
"if convergence_epoch:\n",
" ax.axvline(x=convergence_epoch, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"HỘI TỤ epoch={convergence_epoch}\")\n",
"ax.axvline(x=best_epoch, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best epoch={best_epoch}\")\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Learning Rate (log)\")\n",
"ax.set_title(\"Learning Rate Schedule (ReduceLROnPlateau)\")\n",
"ax.legend(fontsize=8)\n",
"ax.grid(True, alpha=0.3)\n",
"\n",
"plt.suptitle(\"MobileNetV3 + LR-ASPP — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
"print(f\"\\n{'═'*60}\")\n",
"print(f\" Tổng số epoch : {total_epochs}\")\n",
"print(f\" Best epoch : {best_epoch} (val_loss={best_val_loss:.4f})\")\n",
"print(f\" Best val accuracy : {val_accs[best_epoch-1]*100:.4f}%\")\n",
"if convergence_epoch:\n",
" print(f\" Điểm HỘI TỤ : epoch {convergence_epoch} \"\n",
" f\"(val_acc={val_accs[convergence_epoch-1]*100:.2f}%)\")\n",
" wasted = total_epochs - convergence_epoch\n",
" print(f\" Epochs sau hội tụ : {wasted} \"\n",
" f\"(có thể giảm N_EPOCHS không ảnh hưởng nhiều đến kết quả)\")\n",
"if early_stop_epoch:\n",
" print(f\" Early stop tại epoch: {early_stop_epoch}\")\n",
"print(f\"{'═'*60}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc690b92",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import seaborn as sns\n",
"\n",
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
"model.eval()\n",
"with torch.no_grad():\n",
" X_te_t = torch.FloatTensor(X_test_np).to(device)\n",
" logits = model(X_te_t)\n",
" y_pred = logits.argmax(1).cpu().numpy()\n",
"\n",
"acc = accuracy_score(y_test_np, y_pred)\n",
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"print(classification_report(y_test_np, y_pred, digits=4))\n",
"\n",
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred)\n",
"plt.figure(figsize=(9, 7))\n",
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Purples\",\n",
" xticklabels=class_names, yticklabels=class_names)\n",
"plt.xlabel(\"Predicted\")\n",
"plt.ylabel(\"Actual\")\n",
"plt.title(\"Confusion Matrix — MobileNetV3 + LR-ASPP\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}