Victor Phan
354 lines
15 KiB
Plaintext
354 lines
15 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3ab233c4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%time\n",
|
|
"%matplotlib inline\n",
|
|
"\n",
|
|
"import importlib\n",
|
|
"import new_import_ODC\n",
|
|
"\n",
|
|
"importlib.reload(new_import_ODC)\n",
|
|
"from new_import_ODC import *\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0af1f969",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%time\n",
|
|
"# Cấu hình Dask + ODC + S3\n",
|
|
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
|
|
"dc = datacube.Datacube()\n",
|
|
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
|
"client\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b0809939",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"## cấu hình thời gian và tọa độ\n",
|
|
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
|
"longtitude_range = (105.5, 106.4)\n",
|
|
"latitude_range = (9.2, 10.0)\n",
|
|
"coordinates = (longtitude_range, latitude_range)\n",
|
|
"\n",
|
|
"## truy vấn ảnh Sentinel-2\n",
|
|
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
|
|
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
|
"display(data)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "b74030e3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%time\n",
|
|
"# Cloud masking + NDVI + fill nan + resample\n",
|
|
"result = mask_clean(data)\n",
|
|
"progress(result)\n",
|
|
"\n",
|
|
"ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
|
|
"ndvi = ds1[\"NDVI\"]\n",
|
|
"\n",
|
|
"time_split = [\n",
|
|
" slice(\"2022-09-01\", \"2023-01-01\"),\n",
|
|
" slice(\"2023-01-01\", \"2023-05-01\"),\n",
|
|
" slice(\"2023-05-01\", \"2023-07-01\"),\n",
|
|
" slice(\"2023-07-01\", \"2023-10-01\"),\n",
|
|
"]\n",
|
|
"fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
|
|
"plt.imshow(fill_nan_ndvi.isel(time=6)); plt.title(\"NDVI (after fill)\"); plt.colorbar(); plt.show()\n",
|
|
"\n",
|
|
"average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
|
|
"progress(average_ndvi)\n",
|
|
"average_ndvi = average_ndvi.compute()\n",
|
|
"print(f\"NDVI monthly: {average_ndvi.shape}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "df61871b",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load Sentinel-1 (VH, VV)\n",
|
|
"dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
|
|
"average_vv = calculate_average(dsvv, time_pattern=\"1M\")\n",
|
|
"average_vh = calculate_average(dsvh, time_pattern=\"1M\")\n",
|
|
"print(f\"VV: {average_vv.shape} VH: {average_vh.shape}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "d0def6c0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"## Chuẩn bị dữ liệu train\n",
|
|
"train_path = \"train/ST_training_data_updated_1130points_new.shp\"\n",
|
|
"train = load_train_data(train_path)\n",
|
|
"train.head()\n",
|
|
"\n",
|
|
"label_mapping = {\n",
|
|
" \"Lua tom\": \"0\", \"Lua\": \"1\", \"CHN\": \"2\", \"CLN\": \"3\",\n",
|
|
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
|
|
"}\n",
|
|
"\n",
|
|
"datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)\n",
|
|
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(train, label_mapping, datasets)\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"X_train_np = np.asarray(X_train, dtype=np.float32)\n",
|
|
"X_val_np = np.asarray(X_val, dtype=np.float32)\n",
|
|
"X_test_np = np.asarray(X_test, dtype=np.float32)\n",
|
|
"y_train_np = np.asarray(y_train, dtype=np.int32)\n",
|
|
"y_val_np = np.asarray(y_val, dtype=np.int32)\n",
|
|
"y_test_np = np.asarray(y_test, dtype=np.int32)\n",
|
|
"\n",
|
|
"# Gộp train + val để tận dụng toàn bộ dữ liệu train\n",
|
|
"X_fit = np.concatenate([X_train_np, X_val_np], axis=0)\n",
|
|
"y_fit = np.concatenate([y_train_np, y_val_np], axis=0)\n",
|
|
"\n",
|
|
"print(f\"Train (fit): {X_fit.shape} Test: {X_test_np.shape} Classes: {len(label_mapping)}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2dc75f84",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%%time\n",
|
|
"from sklearn.ensemble import RandomForestClassifier\n",
|
|
"\n",
|
|
"# ── Xây dựng và train mô hình Random Forest ───────────────────────────────────\n",
|
|
"model = RandomForestClassifier(\n",
|
|
" n_estimators=200,\n",
|
|
" max_depth=30,\n",
|
|
" min_samples_leaf=2,\n",
|
|
" n_jobs=-1,\n",
|
|
" class_weight=\"balanced\", # xử lý mất cân bằng nhãn\n",
|
|
" random_state=42,\n",
|
|
")\n",
|
|
"\n",
|
|
"print(\"🚀 Training Random Forest...\")\n",
|
|
"print(f\" n_estimators = {model.n_estimators}\")\n",
|
|
"print(f\" max_depth = {model.max_depth}\")\n",
|
|
"print(f\" Train samples: {len(X_fit)}\")\n",
|
|
"\n",
|
|
"model.fit(X_fit, y_fit)\n",
|
|
"\n",
|
|
"val_acc = model.score(X_val_np, y_val_np)\n",
|
|
"print(f\"\\n✅ Training hoàn tất! Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "90588a5d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import numpy as np\n",
|
|
"from sklearn.ensemble import RandomForestClassifier\n",
|
|
"\n",
|
|
"# ═══════════════════════════════════════════════════════════════════════════════\n",
|
|
"# PHÂN TÍCH ĐIỂM HỘI TỤ — Random Forest\n",
|
|
"# Phương pháp: tăng dần n_estimators (warm_start) và theo dõi val accuracy\n",
|
|
"# Điểm hội tụ = lần đầu cải thiện val_acc < THRESHOLD trong WINDOW bước liên tiếp\n",
|
|
"# ═══════════════════════════════════════════════════════════════════════════════\n",
|
|
"N_RANGE = list(range(5, 205, 5)) # 5, 10, 15, ... 200\n",
|
|
"THRESHOLD = 0.0005 # cải thiện < 0.05% → coi là hội tụ\n",
|
|
"WINDOW = 3 # cần WINDOW bước liên tiếp dưới threshold\n",
|
|
"\n",
|
|
"conv_model = RandomForestClassifier(\n",
|
|
" max_depth=30, min_samples_leaf=2, n_jobs=-1,\n",
|
|
" class_weight=\"balanced\", random_state=42,\n",
|
|
" warm_start=True, # ← cho phép thêm cây mà không retrain lại\n",
|
|
")\n",
|
|
"\n",
|
|
"train_accs_c, val_accs_c = [], []\n",
|
|
"print(\"🔍 Phân tích hội tụ (warm_start)...\")\n",
|
|
"for n in N_RANGE:\n",
|
|
" conv_model.n_estimators = n\n",
|
|
" conv_model.fit(X_fit, y_fit)\n",
|
|
" train_accs_c.append(conv_model.score(X_fit, y_fit))\n",
|
|
" val_accs_c.append(conv_model.score(X_val_np, y_val_np))\n",
|
|
"\n",
|
|
"val_accs_c = np.array(val_accs_c)\n",
|
|
"train_accs_c = np.array(train_accs_c)\n",
|
|
"\n",
|
|
"# ── Tìm điểm hội tụ ───────────────────────────────────────────────────────────\n",
|
|
"improvements = np.abs(np.diff(val_accs_c))\n",
|
|
"convergence_idx = None\n",
|
|
"for i in range(len(improvements) - WINDOW + 1):\n",
|
|
" if all(improvements[i : i + WINDOW] < THRESHOLD):\n",
|
|
" convergence_idx = i + 1 # chỉ số của điểm đầu tiên trong cửa sổ\n",
|
|
" break\n",
|
|
"\n",
|
|
"best_idx = int(np.argmax(val_accs_c))\n",
|
|
"best_n = N_RANGE[best_idx]\n",
|
|
"conv_n = N_RANGE[convergence_idx] if convergence_idx is not None else None\n",
|
|
"\n",
|
|
"# ── Vẽ đồ thị ─────────────────────────────────────────────────────────────────\n",
|
|
"fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
|
|
"\n",
|
|
"# --- Trái: accuracy curve ---\n",
|
|
"axes[0].plot(N_RANGE, train_accs_c, \"b-o\", markersize=3, label=\"Train\")\n",
|
|
"axes[0].plot(N_RANGE, val_accs_c, \"g-o\", markersize=3, label=\"Val\")\n",
|
|
"axes[0].axvline(x=best_n, color=\"red\", linestyle=\"--\", linewidth=1.5,\n",
|
|
" label=f\"Best val acc n={best_n} ({max(val_accs_c)*100:.2f}%)\")\n",
|
|
"if conv_n:\n",
|
|
" axes[0].axvline(x=conv_n, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
|
|
" label=f\"Hội tụ n={conv_n} ({val_accs_c[convergence_idx]*100:.2f}%)\")\n",
|
|
"axes[0].set_xlabel(\"n_estimators\")\n",
|
|
"axes[0].set_ylabel(\"Accuracy\")\n",
|
|
"axes[0].set_title(\"Convergence — Val Accuracy vs n_estimators\")\n",
|
|
"axes[0].legend(fontsize=8)\n",
|
|
"axes[0].grid(True, alpha=0.3)\n",
|
|
"\n",
|
|
"# --- Phải: cải thiện biên (marginal improvement) ---\n",
|
|
"axes[1].bar(N_RANGE[1:], improvements * 100, color=\"steelblue\", alpha=0.7)\n",
|
|
"axes[1].axhline(y=THRESHOLD * 100, color=\"red\", linestyle=\"--\",\n",
|
|
" label=f\"Threshold = {THRESHOLD*100:.3f}%\")\n",
|
|
"if conv_n:\n",
|
|
" axes[1].axvline(x=conv_n, color=\"orange\", linestyle=\":\", linewidth=1.5,\n",
|
|
" label=f\"Hội tụ n={conv_n}\")\n",
|
|
"axes[1].set_xlabel(\"n_estimators\")\n",
|
|
"axes[1].set_ylabel(\"ΔVal Accuracy (%)\")\n",
|
|
"axes[1].set_title(\"Marginal Improvement per Step\")\n",
|
|
"axes[1].legend(fontsize=8)\n",
|
|
"axes[1].grid(True, alpha=0.3)\n",
|
|
"\n",
|
|
"plt.suptitle(\"Random Forest — Convergence Analysis\", fontsize=13, fontweight=\"bold\")\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"# ── Tổng kết ──────────────────────────────────────────────────────────────────\n",
|
|
"print(f\"\\n{'═'*55}\")\n",
|
|
"print(f\" Best val accuracy : {max(val_accs_c)*100:.4f}% (n_estimators={best_n})\")\n",
|
|
"if conv_n:\n",
|
|
" print(f\" Điểm HỘI TỤ : n_estimators = {conv_n}\")\n",
|
|
" print(f\" → Có thể dùng n_estimators={conv_n} thay vì 200 để tiết kiệm thời gian\")\n",
|
|
" saved_pct = (1 - conv_n / 200) * 100\n",
|
|
" print(f\" → Tiết kiệm ~{saved_pct:.0f}% thời gian train\")\n",
|
|
"else:\n",
|
|
" print(\" → Mô hình chưa hội tụ trong phạm vi [5, 200]. Thử tăng n_estimators.\")\n",
|
|
"print(f\"{'═'*55}\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "bc57f244",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import seaborn as sns\n",
|
|
"\n",
|
|
"# ── Đánh giá trên tập test ─────────────────────────────────────────────────────\n",
|
|
"y_pred = model.predict(X_test_np)\n",
|
|
"\n",
|
|
"acc = accuracy_score(y_test_np, y_pred)\n",
|
|
"print(f\"Test Accuracy : {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
|
|
"print(classification_report(y_test_np, y_pred, digits=4))\n",
|
|
"\n",
|
|
"# ── Feature importance ──────────────────────────────────────────────────────────\n",
|
|
"feat_imp = model.feature_importances_\n",
|
|
"idx = feat_imp.argsort()[::-1][:20]\n",
|
|
"plt.figure(figsize=(12, 4))\n",
|
|
"plt.bar(range(len(idx)), feat_imp[idx])\n",
|
|
"plt.xticks(range(len(idx)), idx, rotation=45)\n",
|
|
"plt.title(\"Top-20 Feature Importances\")\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"# ── Confusion matrix ────────────────────────────────────────────────────────────\n",
|
|
"class_names = list(label_mapping.keys())\n",
|
|
"cm = confusion_matrix(y_test_np, y_pred)\n",
|
|
"plt.figure(figsize=(9, 7))\n",
|
|
"sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\",\n",
|
|
" xticklabels=class_names, yticklabels=class_names)\n",
|
|
"plt.xlabel(\"Predicted\")\n",
|
|
"plt.ylabel(\"Actual\")\n",
|
|
"plt.title(\"Confusion Matrix — Random Forest\")\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.show()\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "efc23f2c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import joblib, json, os\n",
|
|
"from datetime import datetime\n",
|
|
"\n",
|
|
"# ── Lưu mô hình ────────────────────────────────────────────────────────────────\n",
|
|
"model_path = \"model_random_forest_land_use.joblib\"\n",
|
|
"joblib.dump(model, model_path)\n",
|
|
"print(f\"✅ Model saved → {model_path}\")\n",
|
|
"\n",
|
|
"# ── Lưu thông tin mô hình ──────────────────────────────────────────────────────\n",
|
|
"info = {\n",
|
|
" \"model_type\": \"RandomForest\",\n",
|
|
" \"n_estimators\": model.n_estimators,\n",
|
|
" \"max_depth\": model.max_depth,\n",
|
|
" \"min_samples_leaf\": model.min_samples_leaf,\n",
|
|
" \"class_weight\": \"balanced\",\n",
|
|
" \"n_features\": int(X_fit.shape[1]),\n",
|
|
" \"label_mapping\": label_mapping,\n",
|
|
" \"test_accuracy\": float(acc),\n",
|
|
" \"train_samples\": int(len(X_fit)),\n",
|
|
" \"test_samples\": int(len(X_test_np)),\n",
|
|
" \"saved_at\": datetime.now().isoformat(),\n",
|
|
"}\n",
|
|
"info_path = \"model_random_forest_land_use_info.json\"\n",
|
|
"with open(info_path, \"w\") as f:\n",
|
|
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
|
|
"print(f\"✅ Info saved → {info_path}\")\n",
|
|
"print(json.dumps(info, indent=2, ensure_ascii=False))\n",
|
|
"\n",
|
|
"# ── Đóng kết nối Dask ──────────────────────────────────────────────────────────\n",
|
|
"try:\n",
|
|
" client.close()\n",
|
|
" cluster.close()\n",
|
|
" print(\"✅ Dask cluster closed.\")\n",
|
|
"except Exception:\n",
|
|
" pass\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|