Victor Phan
373 lines
16 KiB
Plaintext
373 lines
16 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f8e59602",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import importlib\n",
|
|
"import new_import_ODC as odc_tools\n",
|
|
"importlib.reload(odc_tools)\n",
|
|
"from new_import_ODC import *\n",
|
|
"print(\"✅ Import thành công\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8101f17e",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Khởi tạo Dask + Datacube + S3\n",
|
|
"cluster, client = initialize_dask(use_gateway=True)\n",
|
|
"dc = datacube.Datacube()\n",
|
|
"configure_s3_access(aws_unsigned=True)\n",
|
|
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cba4d661",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Cấu hình vùng và thời gian\n",
|
|
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
|
"longtitude_range = (105.5, 106.4)\n",
|
|
"latitude_range = (9.2, 10.0)\n",
|
|
"\n",
|
|
"# Tải dữ liệu Sentinel-2\n",
|
|
"data_sen2 = load_data(\n",
|
|
" dc=dc,\n",
|
|
" date_range=date_range,\n",
|
|
" longtitude_range=longtitude_range,\n",
|
|
" latitude_range=latitude_range,\n",
|
|
")\n",
|
|
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "28b9d92d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
|
|
"data_clean = mask_clean(data_sen2)\n",
|
|
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
|
|
"data_fill = fill_nan(data_ndvi)\n",
|
|
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
|
|
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0e4efc29",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Tải Sentinel-1 (SAR VV/VH)\n",
|
|
"data_sen1 = load_data_sen1(\n",
|
|
" dc=dc,\n",
|
|
" date_range=date_range,\n",
|
|
" longtitude_range=longtitude_range,\n",
|
|
" latitude_range=latitude_range,\n",
|
|
")\n",
|
|
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
|
|
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "cb119111",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import numpy as np\n",
|
|
"\n",
|
|
"# Ánh xạ nhãn lớp đất\n",
|
|
"label_mapping = {\n",
|
|
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
|
|
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
|
|
"}\n",
|
|
"\n",
|
|
"train_data = load_train_data(label_mapping=label_mapping)\n",
|
|
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
|
|
"\n",
|
|
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
|
|
"\n",
|
|
"X_train_np = np.array(X_train, dtype=np.float32)\n",
|
|
"X_val_np = np.array(X_val, dtype=np.float32)\n",
|
|
"X_test_np = np.array(X_test, dtype=np.float32)\n",
|
|
"y_train_np = np.array(y_train, dtype=np.int64)\n",
|
|
"y_val_np = np.array(y_val, dtype=np.int64)\n",
|
|
"y_test_np = np.array(y_test, dtype=np.int64)\n",
|
|
"\n",
|
|
"X_fit = np.concatenate([X_train_np, X_val_np], axis=0)\n",
|
|
"y_fit = np.concatenate([y_train_np, y_val_np], axis=0)\n",
|
|
"\n",
|
|
"print(f\"✅ X_fit: {X_fit.shape} | X_test: {X_test_np.shape}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "69492c1f",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%time\n",
|
|
"from sklearn.svm import SVC\n",
|
|
"from sklearn.preprocessing import StandardScaler\n",
|
|
"\n",
|
|
"# ── Chuẩn hoá đặc trưng (quan trọng với SVM) ──────────────────────────────────\n",
|
|
"scaler = StandardScaler()\n",
|
|
"X_fit_scaled = scaler.fit_transform(X_fit)\n",
|
|
"X_test_scaled = scaler.transform(X_test_np)\n",
|
|
"X_val_scaled = scaler.transform(X_val_np)\n",
|
|
"\n",
|
|
"# ── Xây dựng và train mô hình SVM ─────────────────────────────────────────────\n",
|
|
"model = SVC(\n",
|
|
" kernel=\"rbf\",\n",
|
|
" C=10,\n",
|
|
" gamma=\"scale\",\n",
|
|
" probability=True,\n",
|
|
" class_weight=\"balanced\",\n",
|
|
" random_state=42,\n",
|
|
" verbose=True,\n",
|
|
")\n",
|
|
"\n",
|
|
"print(\"🚀 Training SVM (RBF kernel)...\")\n",
|
|
"print(f\" Train samples: {len(X_fit_scaled)}\")\n",
|
|
"model.fit(X_fit_scaled, y_fit)\n",
|
|
"\n",
|
|
"val_acc = model.score(X_val_scaled, y_val_np)\n",
|
|
"print(f\"✅ Training hoàn tất! Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
|
|
"print(f\" n_support_vectors: {model.n_support_}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "ac391274",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import numpy as np\n",
|
|
"from sklearn.model_selection import learning_curve\n",
|
|
"from sklearn.svm import SVC\n",
|
|
"\n",
|
|
"# ═══════════════════════════════════════════════════════════════════════════════\n",
|
|
"# PHÂN TÍCH ĐIỂM HỘI TỤ — SVM\n",
|
|
"# Phương pháp 1: Learning Curve (accuracy vs training set size)\n",
|
|
"# Phương pháp 2: C-sensitivity (val accuracy vs regularization C)\n",
|
|
"# Điểm hội tụ = training size tại đó cải thiện val_score < threshold\n",
|
|
"# ═══════════════════════════════════════════════════════════════════════════════\n",
|
|
"THRESHOLD = 0.002 # cải thiện val_score < 0.2% → hội tụ\n",
|
|
"N_CV = 3 # số fold cho cross-validation (tăng để chính xác hơn)\n",
|
|
"\n",
|
|
"# ── 1. Learning Curve ──────────────────────────────────────────────────────────\n",
|
|
"print(\"🔍 Phân tích learning curve (train size)... [có thể mất vài phút]\")\n",
|
|
"svc_for_lc = SVC(kernel=\"rbf\", C=10, gamma=\"scale\",\n",
|
|
" class_weight=\"balanced\", random_state=42)\n",
|
|
"\n",
|
|
"train_sizes_pct = np.linspace(0.1, 1.0, 10)\n",
|
|
"train_sizes, train_scores, val_scores = learning_curve(\n",
|
|
" svc_for_lc, X_fit_scaled, y_fit,\n",
|
|
" train_sizes=train_sizes_pct,\n",
|
|
" cv=N_CV, scoring=\"accuracy\", n_jobs=-1, verbose=0,\n",
|
|
")\n",
|
|
"\n",
|
|
"train_mean = train_scores.mean(axis=1)\n",
|
|
"train_std = train_scores.std(axis=1)\n",
|
|
"val_mean = val_scores.mean(axis=1)\n",
|
|
"val_std = val_scores.std(axis=1)\n",
|
|
"\n",
|
|
"# Tìm điểm hội tụ\n",
|
|
"val_improvements = np.abs(np.diff(val_mean))\n",
|
|
"convergence_size_idx = None\n",
|
|
"for i in range(len(val_improvements) - 1):\n",
|
|
" if val_improvements[i] < THRESHOLD and val_improvements[i + 1] < THRESHOLD:\n",
|
|
" convergence_size_idx = i + 1\n",
|
|
" break\n",
|
|
"\n",
|
|
"# ── 2. C-Sensitivity ──────────────────────────────────────────────────────────\n",
|
|
"print(\"🔍 Phân tích C-sensitivity...\")\n",
|
|
"C_range = [0.01, 0.1, 1, 5, 10, 50, 100, 500]\n",
|
|
"val_accs_c = []\n",
|
|
"for c_val in C_range:\n",
|
|
" m = SVC(kernel=\"rbf\", C=c_val, gamma=\"scale\",\n",
|
|
" class_weight=\"balanced\", random_state=42)\n",
|
|
" m.fit(X_fit_scaled, y_fit)\n",
|
|
" val_accs_c.append(m.score(X_val_scaled, y_val_np))\n",
|
|
"\n",
|
|
"val_accs_c = np.array(val_accs_c)\n",
|
|
"best_C = C_range[int(np.argmax(val_accs_c))]\n",
|
|
"\n",
|
|
"# ── Vẽ đồ thị ─────────────────────────────────────────────────────────────────\n",
|
|
"fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
|
|
"\n",
|
|
"# --- Trái: Learning Curve ---\n",
|
|
"axes[0].plot(train_sizes, train_mean, \"b-o\", markersize=4, label=\"Train\")\n",
|
|
"axes[0].fill_between(train_sizes, train_mean - train_std, train_mean + train_std,\n",
|
|
" alpha=0.2, color=\"blue\")\n",
|
|
"axes[0].plot(train_sizes, val_mean, \"g-o\", markersize=4, label=\"Val (CV)\")\n",
|
|
"axes[0].fill_between(train_sizes, val_mean - val_std, val_mean + val_std,\n",
|
|
" alpha=0.2, color=\"green\")\n",
|
|
"if convergence_size_idx is not None:\n",
|
|
" csize = train_sizes[convergence_size_idx]\n",
|
|
" axes[0].axvline(x=csize, color=\"orange\", linestyle=\"--\", linewidth=1.5,\n",
|
|
" label=f\"Hội tụ ~{int(csize):,} mẫu\")\n",
|
|
"axes[0].set_xlabel(\"Training samples\")\n",
|
|
"axes[0].set_ylabel(\"Accuracy\")\n",
|
|
"axes[0].set_title(\"Learning Curve — Score vs Train Size\")\n",
|
|
"axes[0].legend(fontsize=8)\n",
|
|
"axes[0].grid(True, alpha=0.3)\n",
|
|
"\n",
|
|
"# --- Giữa: Marginal improvement of val score ---\n",
|
|
"axes[1].bar(range(len(val_improvements)), val_improvements * 100,\n",
|
|
" color=[\"green\" if v > THRESHOLD else \"salmon\" for v in val_improvements],\n",
|
|
" alpha=0.8)\n",
|
|
"axes[1].axhline(y=THRESHOLD * 100, color=\"red\", linestyle=\"--\",\n",
|
|
" label=f\"Threshold={THRESHOLD*100:.2f}%\")\n",
|
|
"if convergence_size_idx is not None:\n",
|
|
" axes[1].axvline(x=convergence_size_idx - 0.5, color=\"orange\", linestyle=\":\",\n",
|
|
" linewidth=1.5, label=f\"Hội tụ tại step {convergence_size_idx}\")\n",
|
|
"step_labels = [f\"{int(train_sizes[i])}\" for i in range(1, len(train_sizes))]\n",
|
|
"axes[1].set_xticks(range(len(val_improvements)))\n",
|
|
"axes[1].set_xticklabels(step_labels, rotation=45, fontsize=7)\n",
|
|
"axes[1].set_xlabel(\"Training size step\")\n",
|
|
"axes[1].set_ylabel(\"ΔVal Accuracy (%)\")\n",
|
|
"axes[1].set_title(\"Marginal Val Improvement per Add. Samples\")\n",
|
|
"axes[1].legend(fontsize=8)\n",
|
|
"axes[1].grid(True, alpha=0.3)\n",
|
|
"\n",
|
|
"# --- Phải: C sensitivity ---\n",
|
|
"axes[2].semilogx(C_range, val_accs_c * 100, \"m-o\", markersize=6)\n",
|
|
"axes[2].axvline(x=best_C, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
|
|
" label=f\"Best C={best_C} ({max(val_accs_c)*100:.2f}%)\")\n",
|
|
"axes[2].set_xlabel(\"C (regularization)\")\n",
|
|
"axes[2].set_ylabel(\"Val Accuracy (%)\")\n",
|
|
"axes[2].set_title(\"C-Sensitivity (Regularization)\")\n",
|
|
"axes[2].legend(fontsize=8)\n",
|
|
"axes[2].grid(True, alpha=0.3)\n",
|
|
"\n",
|
|
"plt.suptitle(\"SVM — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
|
|
"print(f\"\\n{'═'*60}\")\n",
|
|
"if convergence_size_idx is not None:\n",
|
|
" print(f\" Điểm HỘI TỤ (Δval < {THRESHOLD*100:.1f}%) : \"\n",
|
|
" f\"~{int(train_sizes[convergence_size_idx]):,} mẫu \"\n",
|
|
" f\"(val_acc={val_mean[convergence_size_idx]*100:.2f}%)\")\n",
|
|
" pct_data = train_sizes[convergence_size_idx] / len(X_fit_scaled) * 100\n",
|
|
" print(f\" → Chỉ cần ~{pct_data:.0f}% dữ liệu để mô hình hội tụ\")\n",
|
|
"else:\n",
|
|
" print(\" → Cần thêm dữ liệu: val score vẫn đang cải thiện ở toàn bộ tập train\")\n",
|
|
"print(f\" C tối ưu : {best_C} (val_acc={max(val_accs_c)*100:.2f}%)\")\n",
|
|
"print(f\" C hiện tại dùng : 10 {'✅' if best_C == 10 else '⚠️ Thử dùng C=' + str(best_C)}\")\n",
|
|
"print(f\"{'═'*60}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "940f640d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import seaborn as sns\n",
|
|
"\n",
|
|
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
|
|
"y_pred = model.predict(X_test_scaled)\n",
|
|
"\n",
|
|
"acc = accuracy_score(y_test_np, y_pred)\n",
|
|
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
|
|
"print(classification_report(y_test_np, y_pred, digits=4))\n",
|
|
"\n",
|
|
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
|
|
"class_names = list(label_mapping.keys())\n",
|
|
"cm = confusion_matrix(y_test_np, y_pred)\n",
|
|
"plt.figure(figsize=(9, 7))\n",
|
|
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Oranges\",\n",
|
|
" xticklabels=class_names, yticklabels=class_names)\n",
|
|
"plt.xlabel(\"Predicted\")\n",
|
|
"plt.ylabel(\"Actual\")\n",
|
|
"plt.title(\"Confusion Matrix — SVM (RBF)\")\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.show()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "182c64ab",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import joblib, json\n",
|
|
"from datetime import datetime\n",
|
|
"\n",
|
|
"# ── Lưu scaler (cần thiết khi inference) ──────────────────────────────────────\n",
|
|
"scaler_path = \"model_svm_land_use_scaler.joblib\"\n",
|
|
"joblib.dump(scaler, scaler_path)\n",
|
|
"print(f\"✅ Scaler saved → {scaler_path}\")\n",
|
|
"\n",
|
|
"# ── Lưu mô hình SVM ────────────────────────────────────────────────────────────\n",
|
|
"model_path = \"model_svm_land_use.joblib\"\n",
|
|
"joblib.dump(model, model_path)\n",
|
|
"print(f\"✅ Model saved → {model_path}\")\n",
|
|
"\n",
|
|
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
|
|
"info = {\n",
|
|
" \"model_type\": \"SVM\",\n",
|
|
" \"kernel\": model.kernel,\n",
|
|
" \"C\": model.C,\n",
|
|
" \"gamma\": model.gamma,\n",
|
|
" \"probability\": model.probability,\n",
|
|
" \"class_weight\": \"balanced\",\n",
|
|
" \"n_features\": int(X_fit.shape[1]),\n",
|
|
" \"label_mapping\": label_mapping,\n",
|
|
" \"scaler\": scaler_path,\n",
|
|
" \"test_accuracy\": float(acc),\n",
|
|
" \"train_samples\": int(len(X_fit_scaled)),\n",
|
|
" \"test_samples\": int(len(X_test_scaled)),\n",
|
|
" \"saved_at\": datetime.now().isoformat(),\n",
|
|
"}\n",
|
|
"info_path = \"model_svm_land_use_info.json\"\n",
|
|
"with open(info_path, \"w\") as f:\n",
|
|
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
|
|
"print(f\"✅ Info saved → {info_path}\")\n",
|
|
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
|
|
"\n",
|
|
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
|
|
"try:\n",
|
|
" client.close()\n",
|
|
" cluster.close()\n",
|
|
" print(\"✅ Dask cluster closed.\")\n",
|
|
"except Exception:\n",
|
|
" pass\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|