basketballcantho/remote-sensing
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update toàn bộ các mô hình

Browse Source
This commit is contained in:
Victor Phan
2026-03-03 20:09:38 +07:00
parent 6d966c6dde
commit 6a08ae1613
6 changed files with 5238 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
01.train_ODC_CNN.ipynb
+3241
View File
File diff suppressed because one or more lines are too long
01.train_ODC_DecisionTree.ipynb
+360
View File
@@ -0,0 +1,360 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "b05aa740",
"metadata": {},
"outputs": [],
"source": [
"import importlib\n",
"import new_import_ODC as odc_tools\n",
"importlib.reload(odc_tools)\n",
"from new_import_ODC import *\n",
"print(\"✅ Import thành công\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b794d005",
"metadata": {},
"outputs": [],
"source": [
"# Khởi tạo Dask + Datacube + S3\n",
"cluster, client = initialize_dask(use_gateway=True)\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=True)\n",
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5bc42a3c",
"metadata": {},
"outputs": [],
"source": [
"# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"\n",
"# Tải dữ liệu Sentinel-2\n",
"data_sen2 = load_data(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0ef51e7d",
"metadata": {},
"outputs": [],
"source": [
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
"data_clean = mask_clean(data_sen2)\n",
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
"data_fill = fill_nan(data_ndvi)\n",
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0da4f86d",
"metadata": {},
"outputs": [],
"source": [
"# Tải Sentinel-1 (SAR VV/VH)\n",
"data_sen1 = load_data_sen1(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "412b3716",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
"}\n",
"\n",
"# Tải và ghép dữ liệu train từ S1 + S2\n",
"train_data = load_train_data(label_mapping=label_mapping)\n",
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"\n",
"# Chia tập train / val / test\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
"\n",
"X_train_np = np.array(X_train, dtype=np.float32)\n",
"X_val_np = np.array(X_val, dtype=np.float32)\n",
"X_test_np = np.array(X_test, dtype=np.float32)\n",
"y_train_np = np.array(y_train, dtype=np.int64)\n",
"y_val_np = np.array(y_val, dtype=np.int64)\n",
"y_test_np = np.array(y_test, dtype=np.int64)\n",
"\n",
"# Gộp train + val cho sklearn\n",
"X_fit = np.concatenate([X_train_np, X_val_np], axis=0)\n",
"y_fit = np.concatenate([y_train_np, y_val_np], axis=0)\n",
"\n",
"print(f\"✅ X_fit: {X_fit.shape} | X_test: {X_test_np.shape}\")\n",
"print(f\" Classes: {sorted(set(y_fit.tolist()))}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84a8a1d0",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"# ── Xây dựng và train mô hình Decision Tree ─────────────────────────────────\n",
"model = DecisionTreeClassifier(\n",
" max_depth=30,\n",
" min_samples_leaf=2,\n",
" min_samples_split=5,\n",
" class_weight=\"balanced\",\n",
" random_state=42,\n",
")\n",
"\n",
"print(\"🚀 Training Decision Tree...\")\n",
"model.fit(X_fit, y_fit)\n",
"\n",
"val_acc = model.score(X_val_np, y_val_np)\n",
"print(f\"✅ Training hoàn tất! Depth: {model.get_depth()} \"\n",
" f\"Leaves: {model.get_n_leaves()} Val accuracy: {val_acc:.4f}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8248d748",
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"# PHÂN TÍCH ĐIỂM HỘI TỤ — Decision Tree\n",
"# Phương pháp: quét max_depth từ 1→50 và theo dõi train/val accuracy\n",
"# Điểm hội tụ = độ sâu tại đó val_acc đạt cực đại rồi bắt đầu giảm (overfitting)\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"DEPTH_RANGE = list(range(1, 51))\n",
"THRESHOLD = 0.001 # cải thiện val_acc < 0.1% → coi là hội tụ\n",
"\n",
"train_accs_d, val_accs_d = [], []\n",
"print(\"🔍 Phân tích hội tụ theo max_depth ...\")\n",
"for d in DEPTH_RANGE:\n",
" m = DecisionTreeClassifier(\n",
" min_samples_leaf=2, min_samples_split=5,\n",
" class_weight=\"balanced\", random_state=42, max_depth=d,\n",
" )\n",
" m.fit(X_fit, y_fit)\n",
" train_accs_d.append(m.score(X_fit, y_fit))\n",
" val_accs_d.append( m.score(X_val_np, y_val_np))\n",
"\n",
"train_accs_d = np.array(train_accs_d)\n",
"val_accs_d = np.array(val_accs_d)\n",
"improvements = np.diff(val_accs_d)\n",
"\n",
"# ── Tìm điểm hội tụ ───────────────────────────────────────────────────────────\n",
"best_depth = DEPTH_RANGE[int(np.argmax(val_accs_d))]\n",
"best_val_acc = float(np.max(val_accs_d))\n",
"\n",
"# Điểm hội tụ sớm: lần đầu cải thiện < threshold\n",
"conv_depth = None\n",
"for i, imp in enumerate(improvements):\n",
" if abs(imp) < THRESHOLD:\n",
" conv_depth = DEPTH_RANGE[i + 1]\n",
" break\n",
"\n",
"# Điểm overfit: val_acc bắt đầu giảm so với peak\n",
"overfit_depth = None\n",
"peak_idx = int(np.argmax(val_accs_d))\n",
"for i in range(peak_idx + 1, len(val_accs_d)):\n",
" if val_accs_d[i] < best_val_acc - 0.005: # giảm > 0.5%\n",
" overfit_depth = DEPTH_RANGE[i]\n",
" break\n",
"\n",
"# Khoảng cách train-val (generalization gap)\n",
"gap = train_accs_d - val_accs_d\n",
"\n",
"# ── Vẽ đồ thị ─────────────────────────────────────────────────────────────────\n",
"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
"\n",
"# --- Trái: accuracy curves ---\n",
"axes[0].plot(DEPTH_RANGE, train_accs_d, \"b-o\", markersize=3, label=\"Train\")\n",
"axes[0].plot(DEPTH_RANGE, val_accs_d, \"g-o\", markersize=3, label=\"Val\")\n",
"axes[0].axvline(x=best_depth, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best depth={best_depth} ({best_val_acc*100:.2f}%)\")\n",
"if conv_depth:\n",
" axes[0].axvline(x=conv_depth, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"Hội tụ depth={conv_depth}\")\n",
"if overfit_depth:\n",
" axes[0].axvline(x=overfit_depth, color=\"purple\", linestyle=\"-.\", linewidth=1.5,\n",
" label=f\"Overfit depth={overfit_depth}\")\n",
"axes[0].set_xlabel(\"max_depth\")\n",
"axes[0].set_ylabel(\"Accuracy\")\n",
"axes[0].set_title(\"Train / Val Accuracy vs max_depth\")\n",
"axes[0].legend(fontsize=8)\n",
"axes[0].grid(True, alpha=0.3)\n",
"\n",
"# --- Giữa: marginal improvement ---\n",
"axes[1].bar(DEPTH_RANGE[1:], improvements * 100,\n",
" color=[\"green\" if v > 0 else \"red\" for v in improvements], alpha=0.7)\n",
"axes[1].axhline(y=0, color=\"black\", linewidth=0.8)\n",
"axes[1].axhline(y=THRESHOLD * 100, color=\"orange\", linestyle=\"--\",\n",
" label=f\"Threshold={THRESHOLD*100:.2f}%\")\n",
"if conv_depth:\n",
" axes[1].axvline(x=conv_depth, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"Hội tụ depth={conv_depth}\")\n",
"axes[1].set_xlabel(\"max_depth\")\n",
"axes[1].set_ylabel(\"ΔVal Accuracy (%)\")\n",
"axes[1].set_title(\"Marginal Val Improvement per Depth Step\")\n",
"axes[1].legend(fontsize=8)\n",
"axes[1].grid(True, alpha=0.3)\n",
"\n",
"# --- Phải: generalization gap ---\n",
"axes[2].fill_between(DEPTH_RANGE, gap * 100, alpha=0.5, color=\"tomato\", label=\"Gap = Train − Val\")\n",
"axes[2].plot(DEPTH_RANGE, gap * 100, \"r-o\", markersize=3)\n",
"if best_depth:\n",
" axes[2].axvline(x=best_depth, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best depth={best_depth}\")\n",
"axes[2].set_xlabel(\"max_depth\")\n",
"axes[2].set_ylabel(\"Gap (%)\")\n",
"axes[2].set_title(\"Generalization Gap (Overfitting Risk)\")\n",
"axes[2].legend(fontsize=8)\n",
"axes[2].grid(True, alpha=0.3)\n",
"\n",
"plt.suptitle(\"Decision Tree — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
"print(f\"\\n{'═'*58}\")\n",
"print(f\" Độ sâu TỐI ƯU (best val acc) : max_depth = {best_depth} ({best_val_acc*100:.4f}%)\")\n",
"if conv_depth:\n",
" print(f\" Điểm HỘI TỤ (Δacc < {THRESHOLD*100:.1f}%) : max_depth = {conv_depth}\")\n",
"if overfit_depth:\n",
" print(f\" Điểm OVERFIT bắt đầu : max_depth ≥ {overfit_depth}\")\n",
" print(f\" → Nên dùng max_depth ≤ {best_depth} để tránh overfit\")\n",
"print(f\"{'═'*58}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5660e2ec",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
"y_pred = model.predict(X_test_np)\n",
"\n",
"acc = accuracy_score(y_test_np, y_pred)\n",
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"print(classification_report(y_test_np, y_pred, digits=4))\n",
"\n",
"# ── Feature importance ──────────────────────────────────────────────────────────\n",
"feat_imp = model.feature_importances_\n",
"idx = feat_imp.argsort()[::-1][:20]\n",
"plt.figure(figsize=(12, 4))\n",
"plt.bar(range(len(idx)), feat_imp[idx])\n",
"plt.xticks(range(len(idx)), idx, rotation=45)\n",
"plt.title(\"Top-20 Feature Importances\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred)\n",
"plt.figure(figsize=(9, 7))\n",
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Greens\",\n",
" xticklabels=class_names, yticklabels=class_names)\n",
"plt.xlabel(\"Predicted\")\n",
"plt.ylabel(\"Actual\")\n",
"plt.title(\"Confusion Matrix — Decision Tree\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e416c2aa",
"metadata": {},
"outputs": [],
"source": [
"import joblib, json\n",
"from datetime import datetime\n",
"\n",
"# ── Lưu mô hình ────────────────────────────────────────────────────────────────\n",
"model_path = \"model_decision_tree_land_use.joblib\"\n",
"joblib.dump(model, model_path)\n",
"print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
"info = {\n",
" \"model_type\": \"DecisionTree\",\n",
" \"max_depth\": model.get_depth(),\n",
" \"n_leaves\": model.get_n_leaves(),\n",
" \"class_weight\": \"balanced\",\n",
" \"n_features\": int(X_fit.shape[1]),\n",
" \"label_mapping\": label_mapping,\n",
" \"test_accuracy\": float(acc),\n",
" \"train_samples\": int(len(X_fit)),\n",
" \"test_samples\": int(len(X_test_np)),\n",
" \"saved_at\": datetime.now().isoformat(),\n",
"}\n",
"info_path = \"model_decision_tree_land_use_info.json\"\n",
"with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
"print(f\"✅ Info saved → {info_path}\")\n",
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
"try:\n",
" client.close()\n",
" cluster.close()\n",
" print(\"✅ Dask cluster closed.\")\n",
"except Exception:\n",
" pass\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
01.train_ODC_MobileNet.ipynb
+369
View File
@@ -0,0 +1,369 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "17da4353",
"metadata": {},
"outputs": [],
"source": [
"import importlib\n",
"import new_import_ODC as odc_tools\n",
"importlib.reload(odc_tools)\n",
"from new_import_ODC import *\n",
"print(\"✅ Import thành công\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9c063be3",
"metadata": {},
"outputs": [],
"source": [
"# Khởi tạo Dask + Datacube + S3\n",
"cluster, client = initialize_dask(use_gateway=True)\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=True)\n",
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83784d01",
"metadata": {},
"outputs": [],
"source": [
"# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"\n",
"data_sen2 = load_data(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c3faed92",
"metadata": {},
"outputs": [],
"source": [
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
"data_clean = mask_clean(data_sen2)\n",
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
"data_fill = fill_nan(data_ndvi)\n",
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "569bfebb",
"metadata": {},
"outputs": [],
"source": [
"# Tải Sentinel-1 (SAR VV/VH)\n",
"data_sen1 = load_data_sen1(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ecc56c2f",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
"}\n",
"\n",
"train_data = load_train_data(label_mapping=label_mapping)\n",
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
"\n",
"X_train_np = np.array(X_train, dtype=np.float32)\n",
"X_val_np = np.array(X_val, dtype=np.float32)\n",
"X_test_np = np.array(X_test, dtype=np.float32)\n",
"y_train_np = np.array(y_train, dtype=np.int64)\n",
"y_val_np = np.array(y_val, dtype=np.int64)\n",
"y_test_np = np.array(y_test, dtype=np.int64)\n",
"\n",
"n_features = X_train_np.shape[1]\n",
"n_classes = len(np.unique(y_train_np))\n",
"\n",
"print(f\"✅ Train: {X_train_np.shape} Val: {X_val_np.shape} Test: {X_test_np.shape}\")\n",
"print(f\" n_features={n_features} n_classes={n_classes}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5492528b",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import TensorDataset, DataLoader\n",
"\n",
"# ── MobileNetV3 + LR-ASPP classifier ──────────────────────────────────────────\n",
"class MobileNetLRASPPClassifier(nn.Module):\n",
" \"\"\"\n",
" MobileNetV3-inspired backbone with LR-ASPP (Lite Reduced ASPP) head\n",
" for land-use classification on flat feature vectors.\n",
" \"\"\"\n",
" def __init__(self, n_features, n_classes):\n",
" super().__init__()\n",
" # Feature extraction backbone\n",
" self.feature_extractor = nn.Sequential(\n",
" nn.Linear(n_features, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True), nn.Dropout(0.2),\n",
" nn.Linear(128, 256), nn.BatchNorm1d(256), nn.ReLU(inplace=True), nn.Dropout(0.3),\n",
" nn.Linear(256, 512), nn.BatchNorm1d(512), nn.ReLU(inplace=True), nn.Dropout(0.3),\n",
" )\n",
" # LR-ASPP Branch 1: global pooling → 128\n",
" self.global_pool = nn.AdaptiveAvgPool1d(1)\n",
" self.global_conv = nn.Sequential(nn.Linear(512, 128), nn.ReLU(inplace=True))\n",
" # LR-ASPP Branch 2: direct 1×1 → 128\n",
" self.branch_conv = nn.Sequential(nn.Linear(512, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True))\n",
" # Fusion → n_classes\n",
" self.classifier = nn.Sequential(\n",
" nn.Linear(256, 128), nn.BatchNorm1d(128), nn.ReLU(inplace=True), nn.Dropout(0.4),\n",
" nn.Linear(128, n_classes),\n",
" )\n",
"\n",
" def forward(self, x):\n",
" feat = self.feature_extractor(x)\n",
" global_feat = self.global_pool(feat.unsqueeze(-1)).squeeze(-1)\n",
" global_feat = self.global_conv(global_feat)\n",
" branch_feat = self.branch_conv(feat)\n",
" fused = torch.cat([global_feat, branch_feat], dim=1)\n",
" return self.classifier(fused)\n",
"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"print(f\"✅ Device: {device}\")\n",
"model = MobileNetLRASPPClassifier(n_features, n_classes).to(device)\n",
"print(model)\n",
"total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
"print(f\" Trainable params: {total_params:,}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9da40f5a",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# ── Hyper-parameters ───────────────────────────────────────────────────────────\n",
"LEARNING_RATE = 1e-3\n",
"BATCH_SIZE = 64\n",
"N_EPOCHS = 60\n",
"PATIENCE = 10\n",
"\n",
"# ── Tensors & DataLoaders ──────────────────────────────────────────────────────\n",
"X_tr_t = torch.FloatTensor(X_train_np)\n",
"y_tr_t = torch.LongTensor(y_train_np)\n",
"X_va_t = torch.FloatTensor(X_val_np)\n",
"y_va_t = torch.LongTensor(y_val_np)\n",
"\n",
"train_loader = DataLoader(TensorDataset(X_tr_t, y_tr_t), batch_size=BATCH_SIZE, shuffle=True)\n",
"val_loader = DataLoader(TensorDataset(X_va_t, y_va_t), batch_size=BATCH_SIZE, shuffle=False)\n",
"\n",
"# ── Class-weighted loss ────────────────────────────────────────────────────────\n",
"class_counts = np.bincount(y_train_np)\n",
"class_weights = 1.0 / (class_counts + 1e-6)\n",
"class_weights = class_weights / class_weights.sum() * n_classes\n",
"criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weights).to(device))\n",
"\n",
"optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)\n",
"scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode=\"min\", factor=0.5, patience=5)\n",
"\n",
"# ── Training loop ──────────────────────────────────────────────────────────────\n",
"train_losses, val_losses, val_accs = [], [], []\n",
"best_val_loss = float(\"inf\")\n",
"best_state_dict = None\n",
"patience_counter = 0\n",
"\n",
"print(\"🚀 Training MobileNetV3 + LR-ASPP...\")\n",
"for epoch in range(1, N_EPOCHS + 1):\n",
" # --- Train ---\n",
" model.train()\n",
" epoch_loss = 0.0\n",
" for bx, by in train_loader:\n",
" bx, by = bx.to(device), by.to(device)\n",
" optimizer.zero_grad()\n",
" loss = criterion(model(bx), by)\n",
" loss.backward()\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
" optimizer.step()\n",
" epoch_loss += loss.item()\n",
" avg_train_loss = epoch_loss / len(train_loader)\n",
"\n",
" # --- Validate ---\n",
" model.eval()\n",
" val_loss = 0.0; correct = 0; total = 0\n",
" with torch.no_grad():\n",
" for bx, by in val_loader:\n",
" bx, by = bx.to(device), by.to(device)\n",
" out = model(bx)\n",
" val_loss += criterion(out, by).item()\n",
" pred = out.argmax(1)\n",
" correct += (pred == by).sum().item()\n",
" total += by.size(0)\n",
" avg_val_loss = val_loss / len(val_loader)\n",
" val_acc = correct / total\n",
"\n",
" scheduler.step(avg_val_loss)\n",
" lr = optimizer.param_groups[0][\"lr\"]\n",
"\n",
" train_losses.append(avg_train_loss)\n",
" val_losses.append(avg_val_loss)\n",
" val_accs.append(val_acc)\n",
"\n",
" if epoch % 5 == 0 or epoch == 1:\n",
" print(f\"Epoch {epoch:3d}/{N_EPOCHS} train_loss={avg_train_loss:.4f} \"\n",
" f\"val_loss={avg_val_loss:.4f} val_acc={val_acc:.4f} lr={lr:.2e}\")\n",
"\n",
" # Early stopping\n",
" if avg_val_loss < best_val_loss:\n",
" best_val_loss = avg_val_loss\n",
" best_state_dict = {k: v.clone() for k, v in model.state_dict().items()}\n",
" patience_counter = 0\n",
" else:\n",
" patience_counter += 1\n",
" if patience_counter >= PATIENCE:\n",
" print(f\"⏹ Early stopping at epoch {epoch}.\")\n",
" break\n",
"\n",
"# Restore best weights\n",
"model.load_state_dict(best_state_dict)\n",
"print(f\"\\n✅ Training hoàn tất! Best val_loss={best_val_loss:.4f}\")\n",
"\n",
"# ── Learning curves ──────────────────────────────────────────────────────────\n",
"fig, axes = plt.subplots(1, 2, figsize=(13, 4))\n",
"axes[0].plot(train_losses, label=\"Train\")\n",
"axes[0].plot(val_losses, label=\"Val\")\n",
"axes[0].set_title(\"Loss\")\n",
"axes[0].legend()\n",
"axes[1].plot(val_accs)\n",
"axes[1].set_title(\"Val Accuracy\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc690b92",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import seaborn as sns\n",
"\n",
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
"model.eval()\n",
"with torch.no_grad():\n",
" X_te_t = torch.FloatTensor(X_test_np).to(device)\n",
" logits = model(X_te_t)\n",
" y_pred = logits.argmax(1).cpu().numpy()\n",
"\n",
"acc = accuracy_score(y_test_np, y_pred)\n",
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"print(classification_report(y_test_np, y_pred, digits=4))\n",
"\n",
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred)\n",
"plt.figure(figsize=(9, 7))\n",
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Purples\",\n",
" xticklabels=class_names, yticklabels=class_names)\n",
"plt.xlabel(\"Predicted\")\n",
"plt.ylabel(\"Actual\")\n",
"plt.title(\"Confusion Matrix — MobileNetV3 + LR-ASPP\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28f87d0e",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from datetime import datetime\n",
"\n",
"# ── Lưu mô hình PyTorch ────────────────────────────────────────────────────────\n",
"model_path = \"model_mobilenet_land_use.pth\"\n",
"torch.save(model.state_dict(), model_path)\n",
"print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
"info = {\n",
" \"model_type\": \"MobileNetV3_LR-ASPP\",\n",
" \"n_features\": n_features,\n",
" \"n_classes\": n_classes,\n",
" \"learning_rate\": LEARNING_RATE,\n",
" \"batch_size\": BATCH_SIZE,\n",
" \"max_epochs\": N_EPOCHS,\n",
" \"early_stopping_patience\": PATIENCE,\n",
" \"optimizer\": \"Adam\",\n",
" \"scheduler\": \"ReduceLROnPlateau(factor=0.5, patience=5)\",\n",
" \"class_weight\": \"balanced\",\n",
" \"label_mapping\": label_mapping,\n",
" \"test_accuracy\": float(acc),\n",
" \"train_samples\": len(X_train_np),\n",
" \"val_samples\": len(X_val_np),\n",
" \"test_samples\": len(X_test_np),\n",
" \"saved_at\": datetime.now().isoformat(),\n",
"}\n",
"info_path = \"model_mobilenet_land_use_info.json\"\n",
"with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
"print(f\"✅ Info saved → {info_path}\")\n",
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
"try:\n",
" client.close()\n",
" cluster.close()\n",
" print(\"✅ Dask cluster closed.\")\n",
"except Exception:\n",
" pass\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
01.train_ODC_RandomForest.ipynb
+353
View File
@@ -0,0 +1,353 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "3ab233c4",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"%matplotlib inline\n",
"\n",
"import importlib\n",
"import new_import_ODC\n",
"\n",
"importlib.reload(new_import_ODC)\n",
"from new_import_ODC import *\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0af1f969",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# Cấu hình Dask + ODC + S3\n",
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
"client\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b0809939",
"metadata": {},
"outputs": [],
"source": [
"## cấu hình thời gian và tọa độ\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"coordinates = (longtitude_range, latitude_range)\n",
"\n",
"## truy vấn ảnh Sentinel-2\n",
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
"display(data)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b74030e3",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# Cloud masking + NDVI + fill nan + resample\n",
"result = mask_clean(data)\n",
"progress(result)\n",
"\n",
"ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
"ndvi = ds1[\"NDVI\"]\n",
"\n",
"time_split = [\n",
" slice(\"2022-09-01\", \"2023-01-01\"),\n",
" slice(\"2023-01-01\", \"2023-05-01\"),\n",
" slice(\"2023-05-01\", \"2023-07-01\"),\n",
" slice(\"2023-07-01\", \"2023-10-01\"),\n",
"]\n",
"fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
"plt.imshow(fill_nan_ndvi.isel(time=6)); plt.title(\"NDVI (after fill)\"); plt.colorbar(); plt.show()\n",
"\n",
"average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
"progress(average_ndvi)\n",
"average_ndvi = average_ndvi.compute()\n",
"print(f\"NDVI monthly: {average_ndvi.shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df61871b",
"metadata": {},
"outputs": [],
"source": [
"# Load Sentinel-1 (VH, VV)\n",
"dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
"average_vv = calculate_average(dsvv, time_pattern=\"1M\")\n",
"average_vh = calculate_average(dsvh, time_pattern=\"1M\")\n",
"print(f\"VV: {average_vv.shape} VH: {average_vh.shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0def6c0",
"metadata": {},
"outputs": [],
"source": [
"## Chuẩn bị dữ liệu train\n",
"train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
"train = load_train_data(train_path)\n",
"train.head()\n",
"\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
"}\n",
"\n",
"datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)\n",
"\n",
"import numpy as np\n",
"X_train_np = np.asarray(X_train, dtype=np.float32)\n",
"X_val_np = np.asarray(X_val, dtype=np.float32)\n",
"X_test_np = np.asarray(X_test, dtype=np.float32)\n",
"y_train_np = np.asarray(y_train, dtype=np.int32)\n",
"y_val_np = np.asarray(y_val, dtype=np.int32)\n",
"y_test_np = np.asarray(y_test, dtype=np.int32)\n",
"\n",
"# Gộp train + val để tận dụng toàn bộ dữ liệu train\n",
"X_fit = np.concatenate([X_train_np, X_val_np], axis=0)\n",
"y_fit = np.concatenate([y_train_np, y_val_np], axis=0)\n",
"\n",
"print(f\"Train (fit): {X_fit.shape} Test: {X_test_np.shape} Classes: {len(label_mapping)}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2dc75f84",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"\n",
"# ── Xây dựng và train mô hình Random Forest ───────────────────────────────────\n",
"model = RandomForestClassifier(\n",
" n_estimators=200,\n",
" max_depth=30,\n",
" min_samples_leaf=2,\n",
" n_jobs=-1,\n",
" class_weight=\"balanced\", # xử lý mất cân bằng nhãn\n",
" random_state=42,\n",
")\n",
"\n",
"print(\"🚀 Training Random Forest...\")\n",
"print(f\" n_estimators = {model.n_estimators}\")\n",
"print(f\" max_depth = {model.max_depth}\")\n",
"print(f\" Train samples: {len(X_fit)}\")\n",
"\n",
"model.fit(X_fit, y_fit)\n",
"\n",
"val_acc = model.score(X_val_np, y_val_np)\n",
"print(f\"\\n✅ Training hoàn tất! Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "90588a5d",
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.ensemble import RandomForestClassifier\n",
"\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"# PHÂN TÍCH ĐIỂM HỘI TỤ — Random Forest\n",
"# Phương pháp: tăng dần n_estimators (warm_start) và theo dõi val accuracy\n",
"# Điểm hội tụ = lần đầu cải thiện val_acc < THRESHOLD trong WINDOW bước liên tiếp\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"N_RANGE = list(range(5, 205, 5)) # 5, 10, 15, ... 200\n",
"THRESHOLD = 0.0005 # cải thiện < 0.05% → coi là hội tụ\n",
"WINDOW = 3 # cần WINDOW bước liên tiếp dưới threshold\n",
"\n",
"conv_model = RandomForestClassifier(\n",
" max_depth=30, min_samples_leaf=2, n_jobs=-1,\n",
" class_weight=\"balanced\", random_state=42,\n",
" warm_start=True, # ← cho phép thêm cây mà không retrain lại\n",
")\n",
"\n",
"train_accs_c, val_accs_c = [], []\n",
"print(\"🔍 Phân tích hội tụ (warm_start)...\")\n",
"for n in N_RANGE:\n",
" conv_model.n_estimators = n\n",
" conv_model.fit(X_fit, y_fit)\n",
" train_accs_c.append(conv_model.score(X_fit, y_fit))\n",
" val_accs_c.append(conv_model.score(X_val_np, y_val_np))\n",
"\n",
"val_accs_c = np.array(val_accs_c)\n",
"train_accs_c = np.array(train_accs_c)\n",
"\n",
"# ── Tìm điểm hội tụ ───────────────────────────────────────────────────────────\n",
"improvements = np.abs(np.diff(val_accs_c))\n",
"convergence_idx = None\n",
"for i in range(len(improvements) - WINDOW + 1):\n",
" if all(improvements[i : i + WINDOW] < THRESHOLD):\n",
" convergence_idx = i + 1 # chỉ số của điểm đầu tiên trong cửa sổ\n",
" break\n",
"\n",
"best_idx = int(np.argmax(val_accs_c))\n",
"best_n = N_RANGE[best_idx]\n",
"conv_n = N_RANGE[convergence_idx] if convergence_idx is not None else None\n",
"\n",
"# ── Vẽ đồ thị ─────────────────────────────────────────────────────────────────\n",
"fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
"\n",
"# --- Trái: accuracy curve ---\n",
"axes[0].plot(N_RANGE, train_accs_c, \"b-o\", markersize=3, label=\"Train\")\n",
"axes[0].plot(N_RANGE, val_accs_c, \"g-o\", markersize=3, label=\"Val\")\n",
"axes[0].axvline(x=best_n, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best val acc n={best_n} ({max(val_accs_c)*100:.2f}%)\")\n",
"if conv_n:\n",
" axes[0].axvline(x=conv_n, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"Hội tụ n={conv_n} ({val_accs_c[convergence_idx]*100:.2f}%)\")\n",
"axes[0].set_xlabel(\"n_estimators\")\n",
"axes[0].set_ylabel(\"Accuracy\")\n",
"axes[0].set_title(\"Convergence — Val Accuracy vs n_estimators\")\n",
"axes[0].legend(fontsize=8)\n",
"axes[0].grid(True, alpha=0.3)\n",
"\n",
"# --- Phải: cải thiện biên (marginal improvement) ---\n",
"axes[1].bar(N_RANGE[1:], improvements * 100, color=\"steelblue\", alpha=0.7)\n",
"axes[1].axhline(y=THRESHOLD * 100, color=\"red\", linestyle=\"--\",\n",
" label=f\"Threshold = {THRESHOLD*100:.3f}%\")\n",
"if conv_n:\n",
" axes[1].axvline(x=conv_n, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
" label=f\"Hội tụ n={conv_n}\")\n",
"axes[1].set_xlabel(\"n_estimators\")\n",
"axes[1].set_ylabel(\"ΔVal Accuracy (%)\")\n",
"axes[1].set_title(\"Marginal Improvement per Step\")\n",
"axes[1].legend(fontsize=8)\n",
"axes[1].grid(True, alpha=0.3)\n",
"\n",
"plt.suptitle(\"Random Forest — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
"print(f\"\\n{'═'*55}\")\n",
"print(f\" Best val accuracy : {max(val_accs_c)*100:.4f}% (n_estimators={best_n})\")\n",
"if conv_n:\n",
" print(f\" Điểm HỘI TỤ : n_estimators = {conv_n}\")\n",
" print(f\" → Có thể dùng n_estimators={conv_n} thay vì 200 để tiết kiệm thời gian\")\n",
" saved_pct = (1 - conv_n / 200) * 100\n",
" print(f\" → Tiết kiệm ~{saved_pct:.0f}% thời gian train\")\n",
"else:\n",
" print(\" → Mô hình chưa hội tụ trong phạm vi [5, 200]. Thử tăng n_estimators.\")\n",
"print(f\"{'═'*55}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc57f244",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
"y_pred = model.predict(X_test_np)\n",
"\n",
"acc = accuracy_score(y_test_np, y_pred)\n",
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"print(classification_report(y_test_np, y_pred, digits=4))\n",
"\n",
"# ── Feature importance ──────────────────────────────────────────────────────────\n",
"feat_imp = model.feature_importances_\n",
"idx = feat_imp.argsort()[::-1][:20]\n",
"plt.figure(figsize=(12, 4))\n",
"plt.bar(range(len(idx)), feat_imp[idx])\n",
"plt.xticks(range(len(idx)), idx, rotation=45)\n",
"plt.title(\"Top-20 Feature Importances\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred)\n",
"plt.figure(figsize=(9, 7))\n",
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\",\n",
" xticklabels=class_names, yticklabels=class_names)\n",
"plt.xlabel(\"Predicted\")\n",
"plt.ylabel(\"Actual\")\n",
"plt.title(\"Confusion Matrix — Random Forest\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "efc23f2c",
"metadata": {},
"outputs": [],
"source": [
"import joblib, json, os\n",
"from datetime import datetime\n",
"\n",
"# ── Lưu mô hình ────────────────────────────────────────────────────────────────\n",
"model_path = \"model_random_forest_land_use.joblib\"\n",
"joblib.dump(model, model_path)\n",
"print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
"info = {\n",
" \"model_type\": \"RandomForest\",\n",
" \"n_estimators\": model.n_estimators,\n",
" \"max_depth\": model.max_depth,\n",
" \"min_samples_leaf\": model.min_samples_leaf,\n",
" \"class_weight\": \"balanced\",\n",
" \"n_features\": int(X_fit.shape[1]),\n",
" \"label_mapping\": label_mapping,\n",
" \"test_accuracy\": float(acc),\n",
" \"train_samples\": int(len(X_fit)),\n",
" \"test_samples\": int(len(X_test_np)),\n",
" \"saved_at\": datetime.now().isoformat(),\n",
"}\n",
"info_path = \"model_random_forest_land_use_info.json\"\n",
"with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
"print(f\"✅ Info saved → {info_path}\")\n",
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
"try:\n",
" client.close()\n",
" cluster.close()\n",
" print(\"✅ Dask cluster closed.\")\n",
"except Exception:\n",
" pass\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
01.train_ODC_SVM.ipynb
+372
View File
@@ -0,0 +1,372 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "f8e59602",
"metadata": {},
"outputs": [],
"source": [
"import importlib\n",
"import new_import_ODC as odc_tools\n",
"importlib.reload(odc_tools)\n",
"from new_import_ODC import *\n",
"print(\"✅ Import thành công\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8101f17e",
"metadata": {},
"outputs": [],
"source": [
"# Khởi tạo Dask + Datacube + S3\n",
"cluster, client = initialize_dask(use_gateway=True)\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=True)\n",
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cba4d661",
"metadata": {},
"outputs": [],
"source": [
"# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"\n",
"# Tải dữ liệu Sentinel-2\n",
"data_sen2 = load_data(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28b9d92d",
"metadata": {},
"outputs": [],
"source": [
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
"data_clean = mask_clean(data_sen2)\n",
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
"data_fill = fill_nan(data_ndvi)\n",
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e4efc29",
"metadata": {},
"outputs": [],
"source": [
"# Tải Sentinel-1 (SAR VV/VH)\n",
"data_sen1 = load_data_sen1(\n",
" dc=dc,\n",
" date_range=date_range,\n",
" longtitude_range=longtitude_range,\n",
" latitude_range=latitude_range,\n",
")\n",
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb119111",
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"\n",
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
"}\n",
"\n",
"train_data = load_train_data(label_mapping=label_mapping)\n",
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
"\n",
"X_train_np = np.array(X_train, dtype=np.float32)\n",
"X_val_np = np.array(X_val, dtype=np.float32)\n",
"X_test_np = np.array(X_test, dtype=np.float32)\n",
"y_train_np = np.array(y_train, dtype=np.int64)\n",
"y_val_np = np.array(y_val, dtype=np.int64)\n",
"y_test_np = np.array(y_test, dtype=np.int64)\n",
"\n",
"X_fit = np.concatenate([X_train_np, X_val_np], axis=0)\n",
"y_fit = np.concatenate([y_train_np, y_val_np], axis=0)\n",
"\n",
"print(f\"✅ X_fit: {X_fit.shape} | X_test: {X_test_np.shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "69492c1f",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"from sklearn.svm import SVC\n",
"from sklearn.preprocessing import StandardScaler\n",
"\n",
"# ── Chuẩn hoá đặc trưng (quan trọng với SVM) ──────────────────────────────────\n",
"scaler = StandardScaler()\n",
"X_fit_scaled = scaler.fit_transform(X_fit)\n",
"X_test_scaled = scaler.transform(X_test_np)\n",
"X_val_scaled = scaler.transform(X_val_np)\n",
"\n",
"# ── Xây dựng và train mô hình SVM ─────────────────────────────────────────────\n",
"model = SVC(\n",
" kernel=\"rbf\",\n",
" C=10,\n",
" gamma=\"scale\",\n",
" probability=True,\n",
" class_weight=\"balanced\",\n",
" random_state=42,\n",
" verbose=True,\n",
")\n",
"\n",
"print(\"🚀 Training SVM (RBF kernel)...\")\n",
"print(f\" Train samples: {len(X_fit_scaled)}\")\n",
"model.fit(X_fit_scaled, y_fit)\n",
"\n",
"val_acc = model.score(X_val_scaled, y_val_np)\n",
"print(f\"✅ Training hoàn tất! Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
"print(f\" n_support_vectors: {model.n_support_}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ac391274",
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.model_selection import learning_curve\n",
"from sklearn.svm import SVC\n",
"\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"# PHÂN TÍCH ĐIỂM HỘI TỤ — SVM\n",
"# Phương pháp 1: Learning Curve (accuracy vs training set size)\n",
"# Phương pháp 2: C-sensitivity (val accuracy vs regularization C)\n",
"# Điểm hội tụ = training size tại đó cải thiện val_score < threshold\n",
"# ═══════════════════════════════════════════════════════════════════════════════\n",
"THRESHOLD = 0.002 # cải thiện val_score < 0.2% → hội tụ\n",
"N_CV = 3 # số fold cho cross-validation (tăng để chính xác hơn)\n",
"\n",
"# ── 1. Learning Curve ──────────────────────────────────────────────────────────\n",
"print(\"🔍 Phân tích learning curve (train size)... [có thể mất vài phút]\")\n",
"svc_for_lc = SVC(kernel=\"rbf\", C=10, gamma=\"scale\",\n",
" class_weight=\"balanced\", random_state=42)\n",
"\n",
"train_sizes_pct = np.linspace(0.1, 1.0, 10)\n",
"train_sizes, train_scores, val_scores = learning_curve(\n",
" svc_for_lc, X_fit_scaled, y_fit,\n",
" train_sizes=train_sizes_pct,\n",
" cv=N_CV, scoring=\"accuracy\", n_jobs=-1, verbose=0,\n",
")\n",
"\n",
"train_mean = train_scores.mean(axis=1)\n",
"train_std = train_scores.std(axis=1)\n",
"val_mean = val_scores.mean(axis=1)\n",
"val_std = val_scores.std(axis=1)\n",
"\n",
"# Tìm điểm hội tụ\n",
"val_improvements = np.abs(np.diff(val_mean))\n",
"convergence_size_idx = None\n",
"for i in range(len(val_improvements) - 1):\n",
" if val_improvements[i] < THRESHOLD and val_improvements[i + 1] < THRESHOLD:\n",
" convergence_size_idx = i + 1\n",
" break\n",
"\n",
"# ── 2. C-Sensitivity ──────────────────────────────────────────────────────────\n",
"print(\"🔍 Phân tích C-sensitivity...\")\n",
"C_range = [0.01, 0.1, 1, 5, 10, 50, 100, 500]\n",
"val_accs_c = []\n",
"for c_val in C_range:\n",
" m = SVC(kernel=\"rbf\", C=c_val, gamma=\"scale\",\n",
" class_weight=\"balanced\", random_state=42)\n",
" m.fit(X_fit_scaled, y_fit)\n",
" val_accs_c.append(m.score(X_val_scaled, y_val_np))\n",
"\n",
"val_accs_c = np.array(val_accs_c)\n",
"best_C = C_range[int(np.argmax(val_accs_c))]\n",
"\n",
"# ── Vẽ đồ thị ─────────────────────────────────────────────────────────────────\n",
"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
"\n",
"# --- Trái: Learning Curve ---\n",
"axes[0].plot(train_sizes, train_mean, \"b-o\", markersize=4, label=\"Train\")\n",
"axes[0].fill_between(train_sizes, train_mean - train_std, train_mean + train_std,\n",
" alpha=0.2, color=\"blue\")\n",
"axes[0].plot(train_sizes, val_mean, \"g-o\", markersize=4, label=\"Val (CV)\")\n",
"axes[0].fill_between(train_sizes, val_mean - val_std, val_mean + val_std,\n",
" alpha=0.2, color=\"green\")\n",
"if convergence_size_idx is not None:\n",
" csize = train_sizes[convergence_size_idx]\n",
" axes[0].axvline(x=csize, color=\"orange\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Hội tụ ~{int(csize):,} mẫu\")\n",
"axes[0].set_xlabel(\"Training samples\")\n",
"axes[0].set_ylabel(\"Accuracy\")\n",
"axes[0].set_title(\"Learning Curve — Score vs Train Size\")\n",
"axes[0].legend(fontsize=8)\n",
"axes[0].grid(True, alpha=0.3)\n",
"\n",
"# --- Giữa: Marginal improvement of val score ---\n",
"axes[1].bar(range(len(val_improvements)), val_improvements * 100,\n",
" color=[\"green\" if v > THRESHOLD else \"salmon\" for v in val_improvements],\n",
" alpha=0.8)\n",
"axes[1].axhline(y=THRESHOLD * 100, color=\"red\", linestyle=\"--\",\n",
" label=f\"Threshold={THRESHOLD*100:.2f}%\")\n",
"if convergence_size_idx is not None:\n",
" axes[1].axvline(x=convergence_size_idx - 0.5, color=\"orange\", linestyle=\":\",\n",
" linewidth=1.5, label=f\"Hội tụ tại step {convergence_size_idx}\")\n",
"step_labels = [f\"{int(train_sizes[i])}\" for i in range(1, len(train_sizes))]\n",
"axes[1].set_xticks(range(len(val_improvements)))\n",
"axes[1].set_xticklabels(step_labels, rotation=45, fontsize=7)\n",
"axes[1].set_xlabel(\"Training size step\")\n",
"axes[1].set_ylabel(\"ΔVal Accuracy (%)\")\n",
"axes[1].set_title(\"Marginal Val Improvement per Add. Samples\")\n",
"axes[1].legend(fontsize=8)\n",
"axes[1].grid(True, alpha=0.3)\n",
"\n",
"# --- Phải: C sensitivity ---\n",
"axes[2].semilogx(C_range, val_accs_c * 100, \"m-o\", markersize=6)\n",
"axes[2].axvline(x=best_C, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
" label=f\"Best C={best_C} ({max(val_accs_c)*100:.2f}%)\")\n",
"axes[2].set_xlabel(\"C (regularization)\")\n",
"axes[2].set_ylabel(\"Val Accuracy (%)\")\n",
"axes[2].set_title(\"C-Sensitivity (Regularization)\")\n",
"axes[2].legend(fontsize=8)\n",
"axes[2].grid(True, alpha=0.3)\n",
"\n",
"plt.suptitle(\"SVM — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
"print(f\"\\n{'═'*60}\")\n",
"if convergence_size_idx is not None:\n",
" print(f\" Điểm HỘI TỤ (Δval < {THRESHOLD*100:.1f}%) : \"\n",
" f\"~{int(train_sizes[convergence_size_idx]):,} mẫu \"\n",
" f\"(val_acc={val_mean[convergence_size_idx]*100:.2f}%)\")\n",
" pct_data = train_sizes[convergence_size_idx] / len(X_fit_scaled) * 100\n",
" print(f\" → Chỉ cần ~{pct_data:.0f}% dữ liệu để mô hình hội tụ\")\n",
"else:\n",
" print(\" → Cần thêm dữ liệu: val score vẫn đang cải thiện ở toàn bộ tập train\")\n",
"print(f\" C tối ưu : {best_C} (val_acc={max(val_accs_c)*100:.2f}%)\")\n",
"print(f\" C hiện tại dùng : 10 {'✅' if best_C == 10 else '⚠️ Thử dùng C=' + str(best_C)}\")\n",
"print(f\"{'═'*60}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "940f640d",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
"y_pred = model.predict(X_test_scaled)\n",
"\n",
"acc = accuracy_score(y_test_np, y_pred)\n",
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"print(classification_report(y_test_np, y_pred, digits=4))\n",
"\n",
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred)\n",
"plt.figure(figsize=(9, 7))\n",
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Oranges\",\n",
" xticklabels=class_names, yticklabels=class_names)\n",
"plt.xlabel(\"Predicted\")\n",
"plt.ylabel(\"Actual\")\n",
"plt.title(\"Confusion Matrix — SVM (RBF)\")\n",
"plt.tight_layout()\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "182c64ab",
"metadata": {},
"outputs": [],
"source": [
"import joblib, json\n",
"from datetime import datetime\n",
"\n",
"# ── Lưu scaler (cần thiết khi inference) ──────────────────────────────────────\n",
"scaler_path = \"model_svm_land_use_scaler.joblib\"\n",
"joblib.dump(scaler, scaler_path)\n",
"print(f\"✅ Scaler saved → {scaler_path}\")\n",
"\n",
"# ── Lưu mô hình SVM ────────────────────────────────────────────────────────────\n",
"model_path = \"model_svm_land_use.joblib\"\n",
"joblib.dump(model, model_path)\n",
"print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
"info = {\n",
" \"model_type\": \"SVM\",\n",
" \"kernel\": model.kernel,\n",
" \"C\": model.C,\n",
" \"gamma\": model.gamma,\n",
" \"probability\": model.probability,\n",
" \"class_weight\": \"balanced\",\n",
" \"n_features\": int(X_fit.shape[1]),\n",
" \"label_mapping\": label_mapping,\n",
" \"scaler\": scaler_path,\n",
" \"test_accuracy\": float(acc),\n",
" \"train_samples\": int(len(X_fit_scaled)),\n",
" \"test_samples\": int(len(X_test_scaled)),\n",
" \"saved_at\": datetime.now().isoformat(),\n",
"}\n",
"info_path = \"model_svm_land_use_info.json\"\n",
"with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
"print(f\"✅ Info saved → {info_path}\")\n",
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
"try:\n",
" client.close()\n",
" cluster.close()\n",
" print(\"✅ Dask cluster closed.\")\n",
"except Exception:\n",
" pass\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
01.train_ODC_SwinUNet.ipynb
+543
View File
@@ -0,0 +1,543 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0cb933a1",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"%matplotlib inline\n",
"\n",
"import importlib\n",
"import new_import_ODC\n",
"\n",
"importlib.reload(new_import_ODC)\n",
"from new_import_ODC import *\n",
"\n",
"import numpy as np\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
"\n",
"# ── Hyperparameters & constants ───────────────────────────────────────────────\n",
"N_VARS = 3 # số kênh mỗi bước thời gian (ndvi, vh, vv)\n",
"EMBED_DIM = 128 # kích thước embedding Swin blocks\n",
"NUM_HEADS = 4 # heads cho MultiheadAttention\n",
"NUM_CLASSES = 8 # số lớp phân loại\n",
"EPOCHS = 150 # số epoch tối đa\n",
"PATIENCE = 20 # early stopping patience\n",
"BATCH_SIZE = 32\n",
"LR = 1e-3\n",
"WEIGHT_DECAY = 0.01\n",
"\n",
"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"print(f\"Device: {DEVICE}\")\n",
"print(f\"PyTorch version: {torch.__version__}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "46b8f479",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# ── Kết nối Dask + ODC + S3 ──────────────────────────────────────────────────\n",
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
"dc = datacube.Datacube()\n",
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
"client\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cb9a8d0c",
"metadata": {},
"outputs": [],
"source": [
"## cấu hình thời gian và tọa độ\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n",
"coordinates = (longtitude_range, latitude_range)\n",
"\n",
"## truy vấn ảnh Sentinel-2\n",
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
"display(data)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf2647d3",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# ── Loại bỏ mây + tính NDVI ───────────────────────────────────────────────────\n",
"result = mask_clean(data)\n",
"progress(result)\n",
"\n",
"ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
"ndvi = ds1[\"NDVI\"]\n",
"display(ndvi)\n",
"\n",
"## Hiển thị ảnh NDVI trước khi fill mây\n",
"plt.imshow(ndvi.isel(time=6))\n",
"plt.title(\"NDVI (before cloud fill)\")\n",
"plt.colorbar()\n",
"plt.show()\n",
"\n",
"# ── Fill nan theo mùa vụ ──────────────────────────────────────────────────────\n",
"time_split = [\n",
" slice(\"2022-09-01\", \"2023-01-01\"),\n",
" slice(\"2023-01-01\", \"2023-05-01\"),\n",
" slice(\"2023-05-01\", \"2023-07-01\"),\n",
" slice(\"2023-07-01\", \"2023-10-01\"),\n",
"]\n",
"fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
"\n",
"plt.imshow(fill_nan_ndvi.isel(time=6))\n",
"plt.title(\"NDVI (after cloud fill)\")\n",
"plt.colorbar()\n",
"plt.show()\n",
"\n",
"# ── Resample về trung bình tháng ─────────────────────────────────────────────\n",
"average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
"progress(average_ndvi)\n",
"average_ndvi = average_ndvi.compute()\n",
"print(f\"NDVI monthly shape: {average_ndvi.shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7cae5302",
"metadata": {},
"outputs": [],
"source": [
"# ── Load Sentinel-1 (VH, VV) và tính trung bình tháng ────────────────────────\n",
"dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
"average_vv = calculate_average(dsvv, time_pattern=\"1M\")\n",
"average_vh = calculate_average(dsvh, time_pattern=\"1M\")\n",
"\n",
"print(f\"VV monthly shape: {average_vv.shape}\")\n",
"print(f\"VH monthly shape: {average_vh.shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8901a611",
"metadata": {},
"outputs": [],
"source": [
"## ── Chuẩn bị dữ liệu train ───────────────────────────────────────────────────\n",
"train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
"\n",
"train = load_train_data(train_path)\n",
"train.head()\n",
"\n",
"label_mapping = {\n",
" \"Lua tom\": \"0\",\n",
" \"Lua\": \"1\",\n",
" \"CHN\": \"2\",\n",
" \"CLN\": \"3\",\n",
" \"TS\": \"4\",\n",
" \"Song\": \"5\",\n",
" \"Dat xay dung\": \"6\",\n",
" \"Rung\": \"7\",\n",
"}\n",
"\n",
"# Xây dựng dataset gồm VH, VV, NDVI\n",
"datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)\n",
"\n",
"# Chia 80-20-20\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n",
" train, label_mapping, datasets\n",
")\n",
"\n",
"print(f\"X_train: {np.asarray(X_train).shape} y_train: {np.asarray(y_train).shape}\")\n",
"print(f\"X_val : {np.asarray(X_val).shape} y_val : {np.asarray(y_val).shape}\")\n",
"print(f\"X_test : {np.asarray(X_test).shape} y_test : {np.asarray(y_test).shape}\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "64492335",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# ── Kiến trúc Swin-UNet (giống train_module.py / api_server) ─────────────────\n",
"import numpy as np\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torch.utils.data import TensorDataset, DataLoader\n",
"\n",
"class SwinUNetClassifier(nn.Module):\n",
" \"\"\"\n",
" Swin Transformer U-Net style architecture adapted for feature vector classification.\n",
" Combines hierarchical Swin Transformer blocks with skip connections.\n",
" Identical to SwinUNetClassifier in train_module.py.\n",
" \"\"\"\n",
" def __init__(self, n_features, n_classes, embed_dim=128):\n",
" super().__init__()\n",
" self.n_features = n_features\n",
" self.n_classes = n_classes\n",
" self.embed_dim = embed_dim\n",
"\n",
" # Feature adapter\n",
" self.adapter = nn.Sequential(\n",
" nn.Linear(n_features, embed_dim * 2),\n",
" nn.ReLU(),\n",
" nn.Dropout(0.1),\n",
" nn.Linear(embed_dim * 2, embed_dim),\n",
" )\n",
"\n",
" # Encoder\n",
" self.encoder1 = nn.Sequential(\n",
" nn.Linear(embed_dim, embed_dim), nn.LayerNorm(embed_dim), nn.GELU(), nn.Dropout(0.1)\n",
" )\n",
" self.down1 = nn.Linear(embed_dim, embed_dim * 2)\n",
"\n",
" self.encoder2 = nn.Sequential(\n",
" nn.Linear(embed_dim * 2, embed_dim * 2), nn.LayerNorm(embed_dim * 2), nn.GELU(), nn.Dropout(0.1)\n",
" )\n",
" self.down2 = nn.Linear(embed_dim * 2, embed_dim * 4)\n",
"\n",
" self.encoder3 = nn.Sequential(\n",
" nn.Linear(embed_dim * 4, embed_dim * 4), nn.LayerNorm(embed_dim * 4), nn.GELU(), nn.Dropout(0.1)\n",
" )\n",
"\n",
" # Decoder with skip connections\n",
" self.up2 = nn.Linear(embed_dim * 4, embed_dim * 2)\n",
" self.decoder2 = nn.Sequential(\n",
" nn.Linear(embed_dim * 4, embed_dim * 2), nn.LayerNorm(embed_dim * 2), nn.GELU(), nn.Dropout(0.1)\n",
" )\n",
"\n",
" self.up1 = nn.Linear(embed_dim * 2, embed_dim)\n",
" self.decoder1 = nn.Sequential(\n",
" nn.Linear(embed_dim * 2, embed_dim), nn.LayerNorm(embed_dim), nn.GELU(), nn.Dropout(0.1)\n",
" )\n",
"\n",
" # Attention for better aggregation\n",
" self.attention = nn.MultiheadAttention(embed_dim, num_heads=4, batch_first=True)\n",
"\n",
" # Classification head\n",
" self.classifier = nn.Sequential(\n",
" nn.Linear(embed_dim, embed_dim // 2),\n",
" nn.GELU(),\n",
" nn.Dropout(0.3),\n",
" nn.Linear(embed_dim // 2, n_classes),\n",
" )\n",
"\n",
" def forward(self, x):\n",
" if len(x.shape) == 3:\n",
" x = x.squeeze(1)\n",
"\n",
" # Adapter\n",
" x = self.adapter(x) # (B, embed_dim)\n",
" x_seq = x.unsqueeze(1) # (B, 1, embed_dim)\n",
"\n",
" # Encoder\n",
" x1 = self.encoder1(x_seq) # (B, 1, embed_dim)\n",
" x_d1 = self.down1(x1.squeeze(1)) # (B, embed_dim*2)\n",
"\n",
" x2 = self.encoder2(x_d1.unsqueeze(1)) # (B, 1, embed_dim*2)\n",
" x_d2 = self.down2(x2.squeeze(1)) # (B, embed_dim*4)\n",
"\n",
" x3 = self.encoder3(x_d2.unsqueeze(1)) # (B, 1, embed_dim*4)\n",
"\n",
" # Decoder\n",
" x_u2 = self.up2(x3.squeeze(1)) # (B, embed_dim*2)\n",
" x_cat2 = torch.cat([x_u2, x_d1], dim=1) # (B, embed_dim*4)\n",
" x_dec2 = self.decoder2(x_cat2) # (B, embed_dim*2)\n",
"\n",
" x_u1 = self.up1(x_dec2) # (B, embed_dim)\n",
" x_cat1 = torch.cat([x_u1, x.squeeze(1) if len(x.shape)==3 else x], dim=1) # (B, embed_dim*2)\n",
" x_dec1 = self.decoder1(x_cat1) # (B, embed_dim)\n",
"\n",
" # Attention\n",
" x_seq2 = x_dec1.unsqueeze(1)\n",
" attn, _ = self.attention(x_seq2, x_seq2, x_seq2)\n",
"\n",
" return self.classifier(attn.squeeze(1))\n",
"\n",
" def predict(self, X):\n",
" \"\"\"Scikit-learn style predict.\"\"\"\n",
" self.eval()\n",
" with torch.no_grad():\n",
" if isinstance(X, np.ndarray):\n",
" X = torch.FloatTensor(X)\n",
" outputs = self(X)\n",
" return outputs.argmax(1).cpu().numpy()\n",
"\n",
" def score(self, X, y):\n",
" preds = self.predict(X)\n",
" if isinstance(y, torch.Tensor):\n",
" y = y.cpu().numpy()\n",
" return float(np.mean(preds == y))\n",
"\n",
"\n",
"# ── Chuẩn bị tensor & DataLoader ─────────────────────────────────────────────\n",
"X_train_np = np.asarray(X_train, dtype=np.float32)\n",
"X_val_np = np.asarray(X_val, dtype=np.float32)\n",
"y_train_np = np.asarray(y_train, dtype=np.int64)\n",
"y_val_np = np.asarray(y_val, dtype=np.int64)\n",
"\n",
"n_features = X_train_np.shape[1]\n",
"NUM_CLASSES = len(label_mapping)\n",
"\n",
"X_train_t = torch.from_numpy(X_train_np)\n",
"X_val_t = torch.from_numpy(X_val_np)\n",
"y_train_t = torch.from_numpy(y_train_np)\n",
"y_val_t = torch.from_numpy(y_val_np)\n",
"\n",
"train_loader = DataLoader(TensorDataset(X_train_t, y_train_t), batch_size=BATCH_SIZE, shuffle=True)\n",
"val_loader = DataLoader(TensorDataset(X_val_t, y_val_t), batch_size=64, shuffle=False)\n",
"\n",
"# ── Khởi tạo mô hình ─────────────────────────────────────────────────────────\n",
"model = SwinUNetClassifier(n_features, NUM_CLASSES, embed_dim=EMBED_DIM).to(DEVICE)\n",
"print(model)\n",
"total_params = sum(p.numel() for p in model.parameters())\n",
"print(f\"\\nTotal parameters: {total_params:,}\")\n",
"print(f\"n_features={n_features} n_classes={NUM_CLASSES} embed_dim={EMBED_DIM}\\n\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d26a3308",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"# ── Train Swin-UNet ───────────────────────────────────────────────────────────\n",
"\n",
"# Class weights để xử lý mất cân bằng dữ liệu (giống api_server)\n",
"class_counts = np.bincount(y_train_np)\n",
"class_weights = 1.0 / (class_counts + 1e-6)\n",
"class_weights = class_weights / class_weights.sum() * len(class_counts)\n",
"class_weights_t = torch.FloatTensor(class_weights).to(DEVICE)\n",
"\n",
"print(f\"Class distribution : {class_counts}\")\n",
"print(f\"Class weights : {np.round(class_weights, 3)}\\n\")\n",
"\n",
"criterion = nn.CrossEntropyLoss(weight=class_weights_t)\n",
"optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n",
"scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=50)\n",
"\n",
"best_val_acc = 0.0\n",
"best_state = None\n",
"no_improve = 0\n",
"history = {\"train_loss\": [], \"train_acc\": [], \"val_loss\": [], \"val_acc\": []}\n",
"\n",
"def evaluate(loader):\n",
" model.eval()\n",
" total_loss, correct, n = 0.0, 0, 0\n",
" with torch.no_grad():\n",
" for xb, yb in loader:\n",
" xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n",
" logits = model(xb)\n",
" total_loss += criterion(logits, yb).item() * len(yb)\n",
" correct += (logits.argmax(1) == yb).sum().item()\n",
" n += len(yb)\n",
" return total_loss / n, correct / n\n",
"\n",
"print(\"🚀 Training Swin-UNet model (PyTorch)...\")\n",
"for epoch in range(1, EPOCHS + 1):\n",
" model.train()\n",
" t_loss, t_correct, t_n = 0.0, 0, 0\n",
" for xb, yb in train_loader:\n",
" xb, yb = xb.to(DEVICE), yb.to(DEVICE)\n",
" optimizer.zero_grad()\n",
" logits = model(xb)\n",
" loss = criterion(logits, yb)\n",
" loss.backward()\n",
" # Gradient clipping — quan trọng cho Swin blocks\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
" optimizer.step()\n",
" t_loss += loss.item() * len(yb)\n",
" t_correct += (logits.argmax(1) == yb).sum().item()\n",
" t_n += len(yb)\n",
"\n",
" scheduler.step()\n",
" train_loss, train_acc = t_loss / t_n, t_correct / t_n\n",
" val_loss, val_acc = evaluate(val_loader)\n",
" lr_now = optimizer.param_groups[0][\"lr\"]\n",
"\n",
" history[\"train_loss\"].append(train_loss)\n",
" history[\"train_acc\"].append(train_acc)\n",
" history[\"val_loss\"].append(val_loss)\n",
" history[\"val_acc\"].append(val_acc)\n",
"\n",
" if val_acc > best_val_acc:\n",
" best_val_acc = val_acc\n",
" best_state = {k: v.cpu().clone() for k, v in model.state_dict().items()}\n",
" no_improve = 0\n",
" else:\n",
" no_improve += 1\n",
"\n",
" if epoch % 10 == 0 or epoch == 1:\n",
" print(f\"Epoch {epoch:3d}/{EPOCHS} \"\n",
" f\"train_loss={train_loss:.4f} train_acc={train_acc:.4f} \"\n",
" f\"val_loss={val_loss:.4f} val_acc={val_acc:.4f} lr={lr_now:.6f}\")\n",
"\n",
" if no_improve >= PATIENCE:\n",
" print(f\"\\nEarly stopping tại epoch {epoch} (không cải thiện {PATIENCE} epochs liên tiếp)\")\n",
" break\n",
"\n",
"model.load_state_dict(best_state)\n",
"print(f\"\\n✅ Training hoàn tất! Best val accuracy: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a806eaf",
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.metrics import (\n",
" accuracy_score, precision_score, recall_score, f1_score,\n",
" confusion_matrix, ConfusionMatrixDisplay,\n",
")\n",
"\n",
"# ── 1. Đánh giá trên tập test ────────────────────────────────────────────────\n",
"X_test_np = np.asarray(X_test, dtype=np.float32)\n",
"y_test_np = np.asarray(y_test, dtype=np.int64)\n",
"X_test_t = torch.from_numpy(X_test_np)\n",
"\n",
"print(\"📊 Evaluating Swin-UNet on test set...\\n\")\n",
"\n",
"model.eval()\n",
"all_preds = []\n",
"with torch.no_grad():\n",
" for i in range(0, len(X_test_t), 64):\n",
" xb = X_test_t[i:i+64].to(DEVICE)\n",
" preds = model(xb).argmax(1).cpu().numpy()\n",
" all_preds.append(preds)\n",
"\n",
"y_pred_test = np.concatenate(all_preds)\n",
"\n",
"test_accuracy = accuracy_score(y_test_np, y_pred_test)\n",
"precision = precision_score(y_test_np, y_pred_test, average=\"weighted\", zero_division=0)\n",
"recall = recall_score(y_test_np, y_pred_test, average=\"weighted\", zero_division=0)\n",
"f1 = f1_score(y_test_np, y_pred_test, average=\"weighted\", zero_division=0)\n",
"\n",
"print(f\"📈 Test Results:\")\n",
"print(f\" Accuracy : {test_accuracy:.4f} ({test_accuracy*100:.2f}%)\")\n",
"print(f\" Precision: {precision:.4f}\")\n",
"print(f\" Recall : {recall:.4f}\")\n",
"print(f\" F1-Score : {f1:.4f}\\n\")\n",
"\n",
"# ── 2. Learning curves ───────────────────────────────────────────────────────\n",
"fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n",
"\n",
"axes[0].plot(history[\"train_loss\"], label=\"Train loss\")\n",
"axes[0].plot(history[\"val_loss\"], label=\"Val loss\")\n",
"axes[0].set_title(\"Loss over epochs\"); axes[0].set_xlabel(\"Epoch\"); axes[0].legend()\n",
"\n",
"axes[1].plot(history[\"train_acc\"], label=\"Train accuracy\")\n",
"axes[1].plot(history[\"val_acc\"], label=\"Val accuracy\")\n",
"axes[1].set_title(\"Accuracy over epochs\"); axes[1].set_xlabel(\"Epoch\"); axes[1].legend()\n",
"\n",
"plt.tight_layout(); plt.show()\n",
"\n",
"# ── 3. Confusion matrix ──────────────────────────────────────────────────────\n",
"class_names = list(label_mapping.keys())\n",
"cm = confusion_matrix(y_test_np, y_pred_test)\n",
"disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\n",
"\n",
"fig, ax = plt.subplots(figsize=(10, 8))\n",
"disp.plot(cmap=\"Blues\", ax=ax)\n",
"plt.xticks(rotation=45, ha=\"right\")\n",
"plt.title(\"Swin-UNet Confusion Matrix — Test Set\")\n",
"plt.tight_layout(); plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf1b5923",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"\n",
"# ── Lưu weights mô hình Swin-UNet ────────────────────────────────────────────\n",
"model_path = \"model_swinunet_land_use.pth\"\n",
"torch.save({\n",
" \"model_state_dict\": model.state_dict(),\n",
" \"n_features\": n_features,\n",
" \"num_classes\": NUM_CLASSES,\n",
" \"embed_dim\": EMBED_DIM,\n",
" \"label_mapping\": label_mapping,\n",
"}, model_path)\n",
"print(f\"✅ Model saved to {model_path}\")\n",
"\n",
"# ── Lưu metadata ─────────────────────────────────────────────────────────────\n",
"info = {\n",
" \"model_type\": \"Swin-UNet (PyTorch)\",\n",
" \"input_shape\": [n_features],\n",
" \"embed_dim\": EMBED_DIM,\n",
" \"num_classes\": NUM_CLASSES,\n",
" \"classes\": list(label_mapping.keys()),\n",
" \"label_mapping\": label_mapping,\n",
" \"num_parameters\": sum(p.numel() for p in model.parameters()),\n",
" \"accuracy\": float(test_accuracy),\n",
" \"precision\": float(precision),\n",
" \"recall\": float(recall),\n",
" \"f1_score\": float(f1),\n",
"}\n",
"\n",
"info_path = \"model_swinunet_land_use_info.json\"\n",
"with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
"\n",
"print(f\"✅ Metadata saved to {info_path}\")\n",
"print(f\"\\nSummary:\")\n",
"print(f\" n_features : {n_features}\")\n",
"print(f\" Parameters : {info['num_parameters']:,}\")\n",
"print(f\" Test accuracy: {test_accuracy*100:.2f}%\")\n",
"\n",
"# ── Ví dụ load lại mô hình ──────────────────────────────────────────────────\n",
"# ck = torch.load(\"model_swinunet_land_use.pth\")\n",
"# model_loaded = SwinUNetClassifier(ck[\"n_features\"], ck[\"num_classes\"], ck[\"embed_dim\"])\n",
"# model_loaded.load_state_dict(ck[\"model_state_dict\"])\n",
"# model_loaded.eval()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "93c9d96e",
"metadata": {},
"outputs": [],
"source": [
"# đóng client, cluster\n",
"client.close()\n",
"cluster.close()\n"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}