thêm chức năng train trên odc predict trên planetary
This commit is contained in:
Victor Phan
@@ -0,0 +1,632 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b1dab7f7",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🌍 Decision Tree Land Classification - Planetary Computer\n",
|
||||
"\n",
|
||||
"## 📌 Notebook này có thể chạy trên:\n",
|
||||
"- ✅ **Local machine** (không cần ODC database)\n",
|
||||
"- ✅ **Server ODC/JupyterHub** (có ODC database)\n",
|
||||
"\n",
|
||||
"## 🎯 Nguồn dữ liệu:\n",
|
||||
"**Microsoft Planetary Computer STAC API**\n",
|
||||
"- Sentinel-2 L2A (optical)\n",
|
||||
"- Sentinel-1 RTC (SAR)\n",
|
||||
"\n",
|
||||
"## 🔄 Workflow:\n",
|
||||
"1. Load data từ Planetary Computer (STAC)\n",
|
||||
"2. Preprocessing (cloud mask, NDVI, resampling)\n",
|
||||
"3. Train Decision Tree model\n",
|
||||
"4. Evaluate & save model\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "df9820b8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')\n",
|
||||
"\n",
|
||||
"# Import module load dữ liệu không cần ODC database\n",
|
||||
"import importlib\n",
|
||||
"import load_data_no_odc\n",
|
||||
"importlib.reload(load_data_no_odc)\n",
|
||||
"\n",
|
||||
"from load_data_no_odc import (\n",
|
||||
" load_and_process_s2,\n",
|
||||
" load_and_process_s1,\n",
|
||||
" load_sentinel2_stac,\n",
|
||||
" load_sentinel1_stac,\n",
|
||||
" mask_clean_s2,\n",
|
||||
" calculate_ndvi,\n",
|
||||
" fill_nan_temporal,\n",
|
||||
" resample_monthly\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Standard imports\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import xarray as xr\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import seaborn as sns\n",
|
||||
"sns.set_style('whitegrid')\n",
|
||||
"\n",
|
||||
"# ML imports\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
|
||||
"import joblib\n",
|
||||
"import json\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"# Dask for parallel processing\n",
|
||||
"from dask.distributed import Client, LocalCluster\n",
|
||||
"\n",
|
||||
"print(\"✅ All modules loaded successfully!\")\n",
|
||||
"print(\"📡 Data source: Microsoft Planetary Computer\")\n",
|
||||
"print(\"💻 Environment: Local or Remote compatible\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "53397b2f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🚀 Step 1: Initialize Dask Cluster\n",
|
||||
"\n",
|
||||
"Khởi tạo Dask local cluster để xử lý song song"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3c4d6779",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Khởi tạo Dask LocalCluster\n",
|
||||
"print(\"🚀 Initializing Dask LocalCluster...\")\n",
|
||||
"\n",
|
||||
"cluster = LocalCluster(\n",
|
||||
" n_workers=4,\n",
|
||||
" threads_per_worker=1,\n",
|
||||
" memory_limit='4GB'\n",
|
||||
")\n",
|
||||
"client = Client(cluster)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Dask cluster ready!\")\n",
|
||||
"print(f\" Workers: {len(cluster.workers)}\")\n",
|
||||
"print(f\" Dashboard: {client.dashboard_link}\")\n",
|
||||
"print(\"=\" * 70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "28bc5035",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📍 Step 2: Define Area of Interest (AOI)\n",
|
||||
"\n",
|
||||
"Định nghĩa vùng nghiên cứu và khoảng thời gian"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "15a5291c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cấu hình vùng và thời gian\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"\n",
|
||||
"# Bounding box: (lon_min, lat_min, lon_max, lat_max)\n",
|
||||
"bbox = (105.5, 9.2, 106.4, 10.0) # Khu vực Mekong Delta\n",
|
||||
"\n",
|
||||
"print(\"📍 Area of Interest:\")\n",
|
||||
"print(f\" Bbox: {bbox}\")\n",
|
||||
"print(f\" Lon range: {bbox[0]} to {bbox[2]}\")\n",
|
||||
"print(f\" Lat range: {bbox[1]} to {bbox[3]}\")\n",
|
||||
"print(f\" Date range: {date_range[0]} to {date_range[1]}\")\n",
|
||||
"print(\"=\" * 70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "422e498e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📥 Step 3: Load Sentinel-2 Data\n",
|
||||
"\n",
|
||||
"Load dữ liệu Sentinel-2 L2A từ Planetary Computer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "611cd1c2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"\n",
|
||||
"print(\"📥 Loading Sentinel-2 data from Planetary Computer...\")\n",
|
||||
"print(\"-\" * 70)\n",
|
||||
"\n",
|
||||
"# Load và xử lý Sentinel-2: cloud mask + NDVI + monthly resampling\n",
|
||||
"data_sen2_monthly = load_and_process_s2(\n",
|
||||
" bbox=bbox,\n",
|
||||
" date_range=date_range,\n",
|
||||
" apply_cloud_mask=True,\n",
|
||||
" calculate_indices=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if data_sen2_monthly is not None:\n",
|
||||
" print(f\"\\n✅ Sentinel-2 monthly data loaded!\")\n",
|
||||
" print(f\" Dimensions: {dict(data_sen2_monthly.dims)}\")\n",
|
||||
" print(f\" Variables: {list(data_sen2_monthly.data_vars)}\")\n",
|
||||
" print(f\" Time steps: {len(data_sen2_monthly.time)}\")\n",
|
||||
" \n",
|
||||
" # Compute to load into memory\n",
|
||||
" data_sen2_monthly = data_sen2_monthly.compute()\n",
|
||||
" print(f\" ✓ Data computed and loaded into memory\")\n",
|
||||
"else:\n",
|
||||
" print(\"❌ Failed to load Sentinel-2 data\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "79ef9736",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📥 Step 4: Load Sentinel-1 Data\n",
|
||||
"\n",
|
||||
"Load dữ liệu Sentinel-1 RTC (SAR) từ Planetary Computer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3bda3dc0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"\n",
|
||||
"print(\"📥 Loading Sentinel-1 data from Planetary Computer...\")\n",
|
||||
"print(\"-\" * 70)\n",
|
||||
"\n",
|
||||
"# Load và xử lý Sentinel-1: monthly resampling\n",
|
||||
"data_sen1_monthly = load_and_process_s1(\n",
|
||||
" bbox=bbox,\n",
|
||||
" date_range=date_range\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if data_sen1_monthly is not None:\n",
|
||||
" print(f\"\\n✅ Sentinel-1 monthly data loaded!\")\n",
|
||||
" print(f\" Dimensions: {dict(data_sen1_monthly.dims)}\")\n",
|
||||
" print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")\n",
|
||||
" print(f\" Time steps: {len(data_sen1_monthly.time)}\")\n",
|
||||
" \n",
|
||||
" # Compute to load into memory\n",
|
||||
" data_sen1_monthly = data_sen1_monthly.compute()\n",
|
||||
" print(f\" ✓ Data computed and loaded into memory\")\n",
|
||||
"else:\n",
|
||||
" print(\"❌ Failed to load Sentinel-1 data\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "de80d48d",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🎯 Step 5: Load Training Data\n",
|
||||
"\n",
|
||||
"Load dữ liệu mẫu huấn luyện (training samples)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70925d09",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Ánh xạ nhãn lớp đất\n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": 0,\n",
|
||||
" \"Lua\": 1,\n",
|
||||
" \"CHN\": 2,\n",
|
||||
" \"CLN\": 3,\n",
|
||||
" \"TS\": 4,\n",
|
||||
" \"Song\": 5,\n",
|
||||
" \"Dat xay dung\": 6,\n",
|
||||
" \"Rung\": 7,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"🎯 Label mapping:\")\n",
|
||||
"for label, idx in label_mapping.items():\n",
|
||||
" print(f\" {idx}: {label}\")\n",
|
||||
"\n",
|
||||
"# TODO: Load training data from shapefile/GeoJSON\n",
|
||||
"# Bạn cần cung cấp đường dẫn đến file training data\n",
|
||||
"train_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/train_data.geojson\" # Thay đổi path này\n",
|
||||
"\n",
|
||||
"print(f\"\\n📂 Loading training data from: {train_path}\")\n",
|
||||
"print(\"⚠ Note: Bạn cần cập nhật train_path với đường dẫn thực tế\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fed9a8f1",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🔧 Step 6: Extract Features\n",
|
||||
"\n",
|
||||
"Trích xuất features từ Sentinel-1 và Sentinel-2 tại các điểm mẫu"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "62c41cd0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Placeholder for feature extraction\n",
|
||||
"# Bạn cần implement hàm extract features từ train points\n",
|
||||
"\n",
|
||||
"def extract_features(train_data, data_s2, data_s1):\n",
|
||||
" \"\"\"\n",
|
||||
" Extract features from S2 and S1 data at training point locations\n",
|
||||
" \"\"\"\n",
|
||||
" X_list = []\n",
|
||||
" y_list = []\n",
|
||||
" \n",
|
||||
" for idx, point in train_data.iterrows():\n",
|
||||
" try:\n",
|
||||
" lon, lat = point.geometry.x, point.geometry.y\n",
|
||||
" \n",
|
||||
" # Extract S2 data\n",
|
||||
" s2_values = data_s2.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
|
||||
" \n",
|
||||
" # Extract S1 data\n",
|
||||
" s1_values = data_s1.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
|
||||
" \n",
|
||||
" # Combine features\n",
|
||||
" features = np.concatenate([s2_values, s1_values])\n",
|
||||
" \n",
|
||||
" # Skip if contains NaN\n",
|
||||
" if not np.isnan(features).any():\n",
|
||||
" X_list.append(features)\n",
|
||||
" y_list.append(point['label_id'])\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" return np.array(X_list), np.array(y_list)\n",
|
||||
"\n",
|
||||
"# TODO: Uncomment when training data is available\n",
|
||||
"# X, y = extract_features(train_data, data_sen2_monthly, data_sen1_monthly)\n",
|
||||
"# print(f\"✅ Extracted {len(X)} training samples\")\n",
|
||||
"# print(f\" Features: {X.shape[1]}\")\n",
|
||||
"# print(f\" Classes: {sorted(set(y.tolist()))}\")\n",
|
||||
"\n",
|
||||
"print(\"⚠ Feature extraction step - waiting for training data\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f7cd0ca2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📊 Step 7: Split Data\n",
|
||||
"\n",
|
||||
"Chia dữ liệu thành train/val/test sets"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "523e1249",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TODO: Uncomment when features are extracted\n",
|
||||
"\n",
|
||||
"# # Split train/val/test\n",
|
||||
"# X_temp, X_test, y_temp, y_test = train_test_split(\n",
|
||||
"# X, y, test_size=0.2, random_state=42, stratify=y\n",
|
||||
"# )\n",
|
||||
"\n",
|
||||
"# X_train, X_val, y_train, y_val = train_test_split(\n",
|
||||
"# X_temp, y_temp, test_size=0.125, random_state=42, stratify=y_temp\n",
|
||||
"# )\n",
|
||||
"\n",
|
||||
"# # Combine train + val for final training\n",
|
||||
"# X_fit = np.concatenate([X_train, X_val], axis=0)\n",
|
||||
"# y_fit = np.concatenate([y_train, y_val], axis=0)\n",
|
||||
"\n",
|
||||
"# print(f\"✅ Data split:\")\n",
|
||||
"# print(f\" Train: {len(X_train)} samples\")\n",
|
||||
"# print(f\" Val: {len(X_val)} samples\")\n",
|
||||
"# print(f\" Test: {len(X_test)} samples\")\n",
|
||||
"# print(f\" Total: {len(X)} samples\")\n",
|
||||
"\n",
|
||||
"print(\"⚠ Data split step - waiting for features\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a469500c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🌲 Step 8: Train Decision Tree Model\n",
|
||||
"\n",
|
||||
"Huấn luyện mô hình Decision Tree"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "445c2d92",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"\n",
|
||||
"# TODO: Uncomment when data is ready\n",
|
||||
"\n",
|
||||
"# # Train Decision Tree\n",
|
||||
"# model = DecisionTreeClassifier(\n",
|
||||
"# max_depth=30,\n",
|
||||
"# min_samples_leaf=2,\n",
|
||||
"# min_samples_split=5,\n",
|
||||
"# class_weight=\"balanced\",\n",
|
||||
"# random_state=42,\n",
|
||||
"# )\n",
|
||||
"\n",
|
||||
"# print(\"🚀 Training Decision Tree...\")\n",
|
||||
"# model.fit(X_fit, y_fit)\n",
|
||||
"\n",
|
||||
"# val_acc = model.score(X_val, y_val)\n",
|
||||
"# print(f\"✅ Training complete!\")\n",
|
||||
"# print(f\" Tree depth: {model.get_depth()}\")\n",
|
||||
"# print(f\" Leaves: {model.get_n_leaves()}\")\n",
|
||||
"# print(f\" Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
|
||||
"\n",
|
||||
"print(\"⚠ Training step - waiting for data\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "7c1e681e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📈 Step 9: Hyperparameter Analysis\n",
|
||||
"\n",
|
||||
"Phân tích độ sâu tối ưu (max_depth) cho Decision Tree"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d13274e4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"\n",
|
||||
"# TODO: Uncomment when model is trained\n",
|
||||
"\n",
|
||||
"# DEPTH_RANGE = list(range(1, 51))\n",
|
||||
"# THRESHOLD = 0.001\n",
|
||||
"\n",
|
||||
"# train_accs = []\n",
|
||||
"# val_accs = []\n",
|
||||
"\n",
|
||||
"# print(\"🔍 Analyzing convergence for max_depth...\")\n",
|
||||
"# for d in DEPTH_RANGE:\n",
|
||||
"# m = DecisionTreeClassifier(\n",
|
||||
"# min_samples_leaf=2,\n",
|
||||
"# min_samples_split=5,\n",
|
||||
"# class_weight=\"balanced\",\n",
|
||||
"# random_state=42,\n",
|
||||
"# max_depth=d,\n",
|
||||
"# )\n",
|
||||
"# m.fit(X_fit, y_fit)\n",
|
||||
"# train_accs.append(m.score(X_fit, y_fit))\n",
|
||||
"# val_accs.append(m.score(X_val, y_val))\n",
|
||||
"\n",
|
||||
"# train_accs = np.array(train_accs)\n",
|
||||
"# val_accs = np.array(val_accs)\n",
|
||||
"\n",
|
||||
"# best_depth = DEPTH_RANGE[np.argmax(val_accs)]\n",
|
||||
"# best_val_acc = np.max(val_accs)\n",
|
||||
"\n",
|
||||
"# print(f\"✅ Best max_depth: {best_depth} (val_acc: {best_val_acc:.4f})\")\n",
|
||||
"\n",
|
||||
"# # Plot\n",
|
||||
"# fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n",
|
||||
"# ax.plot(DEPTH_RANGE, train_accs, 'b-o', markersize=3, label='Train')\n",
|
||||
"# ax.plot(DEPTH_RANGE, val_accs, 'g-o', markersize=3, label='Val')\n",
|
||||
"# ax.axvline(x=best_depth, color='red', linestyle='--', label=f'Best={best_depth}')\n",
|
||||
"# ax.set_xlabel('max_depth')\n",
|
||||
"# ax.set_ylabel('Accuracy')\n",
|
||||
"# ax.set_title('Train/Val Accuracy vs max_depth')\n",
|
||||
"# ax.legend()\n",
|
||||
"# ax.grid(True, alpha=0.3)\n",
|
||||
"# plt.tight_layout()\n",
|
||||
"# plt.show()\n",
|
||||
"\n",
|
||||
"print(\"⚠ Hyperparameter analysis - waiting for model\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b551cdb4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📊 Step 10: Evaluate Model\n",
|
||||
"\n",
|
||||
"Đánh giá mô hình trên tập test"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bc3d6b9a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TODO: Uncomment when model is trained\n",
|
||||
"\n",
|
||||
"# # Predictions\n",
|
||||
"# y_pred = model.predict(X_test)\n",
|
||||
"# acc = accuracy_score(y_test, y_pred)\n",
|
||||
"\n",
|
||||
"# print(f\"📊 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
|
||||
"# print(classification_report(y_test, y_pred, digits=4))\n",
|
||||
"\n",
|
||||
"# # Confusion Matrix\n",
|
||||
"# class_names = list(label_mapping.keys())\n",
|
||||
"# cm = confusion_matrix(y_test, y_pred)\n",
|
||||
"\n",
|
||||
"# plt.figure(figsize=(9, 7))\n",
|
||||
"# sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',\n",
|
||||
"# xticklabels=class_names, yticklabels=class_names)\n",
|
||||
"# plt.xlabel('Predicted')\n",
|
||||
"# plt.ylabel('Actual')\n",
|
||||
"# plt.title('Confusion Matrix — Decision Tree')\n",
|
||||
"# plt.tight_layout()\n",
|
||||
"# plt.show()\n",
|
||||
"\n",
|
||||
"print(\"⚠ Evaluation step - waiting for model\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "44d226fc",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 💾 Step 11: Save Model\n",
|
||||
"\n",
|
||||
"Lưu mô hình và metadata"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "044eefc4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# TODO: Uncomment when model is trained\n",
|
||||
"\n",
|
||||
"# # Save model\n",
|
||||
"# model_path = \"model_decision_tree_planetary_computer.joblib\"\n",
|
||||
"# joblib.dump(model, model_path)\n",
|
||||
"# print(f\"✅ Model saved → {model_path}\")\n",
|
||||
"\n",
|
||||
"# # Save metadata\n",
|
||||
"# info = {\n",
|
||||
"# \"model_type\": \"DecisionTree\",\n",
|
||||
"# \"data_source\": \"Microsoft Planetary Computer\",\n",
|
||||
"# \"max_depth\": model.get_depth(),\n",
|
||||
"# \"n_leaves\": model.get_n_leaves(),\n",
|
||||
"# \"n_features\": int(X_fit.shape[1]),\n",
|
||||
"# \"label_mapping\": label_mapping,\n",
|
||||
"# \"test_accuracy\": float(acc),\n",
|
||||
"# \"train_samples\": int(len(X_fit)),\n",
|
||||
"# \"test_samples\": int(len(X_test)),\n",
|
||||
"# \"bbox\": bbox,\n",
|
||||
"# \"date_range\": date_range,\n",
|
||||
"# \"saved_at\": datetime.now().isoformat(),\n",
|
||||
"# }\n",
|
||||
"\n",
|
||||
"# info_path = \"model_decision_tree_planetary_computer_info.json\"\n",
|
||||
"# with open(info_path, \"w\") as f:\n",
|
||||
"# json.dump(info, f, indent=2, ensure_ascii=False)\n",
|
||||
"\n",
|
||||
"# print(f\"✅ Metadata saved → {info_path}\")\n",
|
||||
"# print(json.dumps(info, indent=2, ensure_ascii=False))\n",
|
||||
"\n",
|
||||
"print(\"⚠ Save model step - waiting for trained model\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "18c95152",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 🧹 Step 12: Cleanup\n",
|
||||
"\n",
|
||||
"Đóng Dask cluster"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "20445068",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Close Dask cluster\n",
|
||||
"try:\n",
|
||||
" client.close()\n",
|
||||
" cluster.close()\n",
|
||||
" print(\"✅ Dask cluster closed.\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"⚠ Error closing cluster: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "046bf0f9",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## ✅ Summary\n",
|
||||
"\n",
|
||||
"### Notebook này:\n",
|
||||
"- ✅ Load dữ liệu từ **Microsoft Planetary Computer**\n",
|
||||
"- ✅ Không cần **ODC Database**\n",
|
||||
"- ✅ Có thể chạy trên **local machine** hoặc **server ODC**\n",
|
||||
"- ✅ Tương thích 100% với dữ liệu ODC\n",
|
||||
"\n",
|
||||
"### Workflow train/predict:\n",
|
||||
"1. **Train trên server ODC**: Chạy notebook này với training data đầy đủ\n",
|
||||
"2. **Save model**: Export `.joblib` file\n",
|
||||
"3. **Predict trên local**: Load model và sử dụng cùng `load_data_no_odc.py` để load dữ liệu mới\n",
|
||||
"\n",
|
||||
"### Next steps:\n",
|
||||
"1. Cập nhật `train_path` với đường dẫn training data thực tế\n",
|
||||
"2. Uncomment các TODO cells\n",
|
||||
"3. Run notebook để train model\n",
|
||||
"4. Test prediction trên local machine\n",
|
||||
"\n",
|
||||
"---"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -192,7 +192,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"id": "c3faed92",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@@ -243,7 +243,9 @@
|
||||
"\n",
|
||||
"def fill_nan(ds):\n",
|
||||
" \"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\n",
|
||||
" return ds.interpolate_na(dim=\"time\", method=\"linear\", fill_value=\"extrapolate\")\n",
|
||||
" # Rechunk time dimension thành 1 chunk để tránh lỗi với interpolate_na\n",
|
||||
" ds_rechunked = ds.chunk({\"time\": -1})\n",
|
||||
" return ds_rechunked.interpolate_na(dim=\"time\", method=\"linear\", fill_value=\"extrapolate\")\n",
|
||||
"\n",
|
||||
"# Áp dụng tiền xử lý\n",
|
||||
"data_clean = mask_clean(data_sen2)\n",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
export AWS_ACCESS_KEY_ID="ASIA4YF43ZWIXQ6HJIAY"
|
||||
export AWS_SECRET_ACCESS_KEY="3N8KoV2ZBqQcFqRUVxQXW8K9sm90CNDV9aHUkNw0"
|
||||
export AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEO7//////////wEaDmFwLXNvdXRoZWFzdC0xIkcwRQIgR/6jnB1QIlWdCryDauBSFI34+KF256iV4X1Lyz4bP6YCIQCKE6A1q3jrlX9RFZ2hrFSOhdHKq187xD6yCUjnXOBkACrgBAi3//////////8BEAAaDDg3NjU2OTQxNTA1NyIMWcHZX71GMI7BdhDIKrQED9cJwWh4/DI3QG+W0jXsvCgvKnFTb+Yd690xdVxIrZuYsrmA+ncbp9Xj0GQSSay28XTBpG6Wgx8b2RSh3+ClJWvRJmvgxvOHpRAZqzgDB+0SJTWS1syeyGK9dlvL6VjI0LMw2r37m3mJzcUKvrVD8Hsg6iUT3FpPp0FKCbjpaSO9rZDXgxIGBwN/QbTNnD6wBwndYB0SI5QGHIIr2uW7IznKgXXY6sCq4uMjQDRULn6OO+ADSDBBahI/ZyhiTbUzL6LNjgvv0i9+Lots3xVg3Mp3yo8iKK5ILIVhmd/2QAX2uRQoLd9QRlChSQ4fwBKXEWN+A1UaPUfHRV0zudVmICBu+fk4lF3EWMkD/TRrkDCDRSBtMMa85m9yyxE7O0bSE/9gErDd/1zNNV1d64MRNrVhG4KWa6VnoOvvcupmgxmDV17db5o0tfFxa1TxHpkHsaTuCY0U8Q+Ep0da/LNdRkbiSoAjPNoAOoE+K5c7Z8g06Q13tTbvYQlDVIH1Kl3KtLlSDa6JUgWg0AMAFLJJCEEgBtLOL0nBuBujDOETnSXoIQjEJQkOvgWaiFP1A74hvXmpR39yOUzWCLRYhwXGt4xQW2NOYuHVFUdhu0rZ7XqHwzV6rYoBSP2uAZPw9diEZXYItgp2hHTAVEc0xEJp5hmBL94pPkpaMEwgOONW3QpqOj+Wg6yv7VPMY8FEEVEexHL1W134htZTgi8SMvx6M4M0oiL7caMy2tPkKF4o87ei6b3OMKnyoM0GOoMCneMKuFPe1C2TkcJ2qQ878F7c4ov/MtFZB3MsCMFXxm9Bx2Nj+v+nB9wQIxEUPA4/aS/zwYMTaYpHdNWJlQrMQllCfEIM9AmjX6GtP46bCPzzcawPlWYa88rGzQNKEEty1qJLJFIk8dK9ulImwIoOJ0aoRbaENYGUzWYJo6YM/yewVp6fm0MuULvlaOR9+tMxXfSc4leiXN6gk90fAW8SDUuaSHkCVphiH0dLnBf0cuzFyxFBMb3i8qNCFM6Mrnj1xPISYp/2PeGnAzEY/n56Cbaq/anT5BjwehAyUV1wPXW5Sdvgg1fygPbluQwtS1h7GxYFt72+GcusVabuXEmu9ab+AQ=="
|
||||
|
||||
Cognito: eyJraWQiOiIzejR4V0txYmd5Mlo4NXR3TFVvRGFSNmp4WVNSZUdKNHdLeEM4K0phbUM0PSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiJiMmM2ZmU5Ny02MGQ5LTQ1YmMtYTRlNy02ZTk1MDkwMmZlYjMiLCJjb2duaXRvOmdyb3VwcyI6WyJkZWZhdWx0LWdyb3VwIiwiYWxsb2NhdGlvbjpSLTE5MjQ0OkNTSVJPIGFuZCBWaWV0bmFtIHBhcnRuZXJzIl0sImlzcyI6Imh0dHBzOlwvXC9jb2duaXRvLWlkcC5hcC1zb3V0aGVhc3QtMS5hbWF6b25hd3MuY29tXC9hcC1zb3V0aGVhc3QtMV9DNEdDYllhT2EiLCJjbGllbnRfaWQiOiI2YTczMzJyOXRybGhxNmkyZmZwbDVma3JnNSIsIm9yaWdpbl9qdGkiOiJmNjczYjFjOS01NGQ3LTQ5MzYtODg4Yi1mMjJjZDU1YTBkNDYiLCJldmVudF9pZCI6IjRlZTUwZGNjLWEzYWMtNGMxYy04MjFhLTU2MGIwOGI0ZGFlYyIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE3NzI2MzIzNTgsImV4cCI6MTc3MjY2MTE1OCwiaWF0IjoxNzcyNjMyMzU5LCJqdGkiOiJhYWMxMzFmMS1jZTc3LTRlMmMtOGQ2Zi1hNDc3MWM0ZTY3ZjUiLCJ1c2VybmFtZSI6ImhpZW5tMjUyMzAwMSJ9.N15e6a0MWQtdZWBIHpDC3rKcwjn9ATEo-WY7oaB1SV4m2u1j17ld_AlnkiplAFWq52viKjWEO8ArY4Okb9xMUraK_nV-YxkAuTv15_hG32Q-qbFWsholcJzc4jObCKc3NXPVolx8zFZn0r9nQoakCqGaQWCfshT3_ZPAMdhIisOn7Jq6jDWyTfivMQlbmCPNCxSR3Yqt6_UTs2pvLdTuyP7hEGyuk8u4F_FyMxZcmGRKwPeKeepjJ22HoDkcvL9rpWIKoMZ-SQABLGqXPAFUEOPuvfCYh6bcivcztwaLszW70zHUbJzJZmyY8wi0PViplg-CK31yTkBej3-ARaC9Zg
|
||||
ID: eyJraWQiOiJOMmdRc1c0S3o1YUltR3hGZEVJVmUxOUIxTWpZSmJPcG5kYUxKQUpNakxJPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiJiMmM2ZmU5Ny02MGQ5LTQ1YmMtYTRlNy02ZTk1MDkwMmZlYjMiLCJjb2duaXRvOmdyb3VwcyI6WyJkZWZhdWx0LWdyb3VwIiwiYWxsb2NhdGlvbjpSLTE5MjQ0OkNTSVJPIGFuZCBWaWV0bmFtIHBhcnRuZXJzIl0sImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAuYXAtc291dGhlYXN0LTEuYW1hem9uYXdzLmNvbVwvYXAtc291dGhlYXN0LTFfQzRHQ2JZYU9hIiwiY29nbml0bzp1c2VybmFtZSI6ImhpZW5tMjUyMzAwMSIsIm9yaWdpbl9qdGkiOiJmNjczYjFjOS01NGQ3LTQ5MzYtODg4Yi1mMjJjZDU1YTBkNDYiLCJhdWQiOiI2YTczMzJyOXRybGhxNmkyZmZwbDVma3JnNSIsImV2ZW50X2lkIjoiNGVlNTBkY2MtYTNhYy00YzFjLTgyMWEtNTYwYjA4YjRkYWVjIiwidG9rZW5fdXNlIjoiaWQiLCJhdXRoX3RpbWUiOjE3NzI2MzIzNTgsIm5hbWUiOiJIaWVuIFBoYW4iLCJleHAiOjE3NzI2NjExNTgsImlhdCI6MTc3MjYzMjM1OSwianRpIjoiMGViODczOTItNWZhMS00OTAwLWFiZmUtNzFmMWEyYjI1YzdiIiwiZW1haWwiOiJoaWVubTI1MjMwMDFAZ3N0dWRlbnQuY3R1LmVkdS52biJ9.QEoa4uJmmSk0qpOBi_a3RrCoqRu-oASbAHc24tgIuSeM_wcQsyTbKNbYdDS9P0n6JZf-wCiCknpsHl6TKiGDL3xDVguesm8ZPdyYkdgpCvE7EnyrCOSa01hcubAL-Z3_TMyUVs0WyDrJ3HS0YT-0Gig7m2y17oL44zwtrWwXcrTJMW94QimW5OMsDKZMn1oQKKGkBhC18FB4lNcerAh9tLknGfJQEseH6_5rJAeLJSwzXJBCfZjM_Yt-4ZmmB1ruNfiHUXfT2QRbBFRDcWcpA0gYJoROrm16tByyivwGcaV2BIRIiSKqi1NSRPdTNqOWmmYh-yufWFG8CdumGpX5ow
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,155 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "30681e56",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🧪 Test Planetary Computer Connection\n",
|
||||
"\n",
|
||||
"Notebook này test kết nối và load dữ liệu từ Microsoft Planetary Computer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e32c8eca",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"sys.path.insert(0, '/media/x79/2A7D-FAA0/remote-sensing')\n",
|
||||
"\n",
|
||||
"from load_data_no_odc import load_sentinel2_stac, load_sentinel1_stac\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"print(\"✅ Module imported successfully\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "501dd8ef",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Small test area (1 month, small bbox)\n",
|
||||
"bbox = (105.8, 9.5, 106.0, 9.7) # Small area in Mekong Delta\n",
|
||||
"date_range = (\"2023-01-01\", \"2023-01-31\") # 1 month only\n",
|
||||
"\n",
|
||||
"print(f\"Test parameters:\")\n",
|
||||
"print(f\" Bbox: {bbox}\")\n",
|
||||
"print(f\" Date: {date_range}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "187e2c1a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test Sentinel-2\n",
|
||||
"print(\"\\n\" + \"=\" * 70)\n",
|
||||
"print(\"Testing Sentinel-2 L2A\")\n",
|
||||
"print(\"=\" * 70)\n",
|
||||
"\n",
|
||||
"data_s2 = load_sentinel2_stac(\n",
|
||||
" bbox=bbox,\n",
|
||||
" date_range=date_range,\n",
|
||||
" bands=['red', 'green', 'blue', 'nir08', 'SCL'],\n",
|
||||
" resolution=60 # Lower resolution for faster test\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if data_s2 is not None:\n",
|
||||
" print(f\"\\n✅ SUCCESS! Sentinel-2 loaded\")\n",
|
||||
" print(f\" Dims: {dict(data_s2.dims)}\")\n",
|
||||
" print(f\" Vars: {list(data_s2.data_vars)}\")\n",
|
||||
"else:\n",
|
||||
" print(\"\\n❌ Failed to load Sentinel-2\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e688f3b9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test Sentinel-1\n",
|
||||
"print(\"\\n\" + \"=\" * 70)\n",
|
||||
"print(\"Testing Sentinel-1 RTC\")\n",
|
||||
"print(\"=\" * 70)\n",
|
||||
"\n",
|
||||
"data_s1 = load_sentinel1_stac(\n",
|
||||
" bbox=bbox,\n",
|
||||
" date_range=date_range,\n",
|
||||
" bands=['vv', 'vh'],\n",
|
||||
" resolution=60 # Lower resolution for faster test\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if data_s1 is not None:\n",
|
||||
" print(f\"\\n✅ SUCCESS! Sentinel-1 loaded\")\n",
|
||||
" print(f\" Dims: {dict(data_s1.dims)}\")\n",
|
||||
" print(f\" Vars: {list(data_s1.data_vars)}\")\n",
|
||||
"else:\n",
|
||||
" print(\"\\n❌ Failed to load Sentinel-1\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "211796a7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Visualize if data loaded successfully\n",
|
||||
"if data_s2 is not None:\n",
|
||||
" print(\"\\n📊 Visualizing Sentinel-2 RGB composite...\")\n",
|
||||
" \n",
|
||||
" # Select first timestep\n",
|
||||
" rgb = data_s2[['red', 'green', 'blue']].isel(time=0)\n",
|
||||
" \n",
|
||||
" # Plot\n",
|
||||
" fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n",
|
||||
" \n",
|
||||
" rgb['red'].plot(ax=axes[0], cmap='Reds')\n",
|
||||
" axes[0].set_title('Red band')\n",
|
||||
" \n",
|
||||
" rgb['green'].plot(ax=axes[1], cmap='Greens')\n",
|
||||
" axes[1].set_title('Green band')\n",
|
||||
" \n",
|
||||
" rgb['blue'].plot(ax=axes[2], cmap='Blues')\n",
|
||||
" axes[2].set_title('Blue band')\n",
|
||||
" \n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
" print(\"✅ Visualization complete!\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4e29a09",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## ✅ Results\n",
|
||||
"\n",
|
||||
"Nếu cả 2 tests đều pass:\n",
|
||||
"- ✅ Kết nối Planetary Computer OK\n",
|
||||
"- ✅ Load Sentinel-2 OK\n",
|
||||
"- ✅ Load Sentinel-1 OK\n",
|
||||
"- ✅ Sẵn sàng sử dụng cho training!\n",
|
||||
"\n",
|
||||
"Next step: Sử dụng `01.train_DecisionTree_PlanetaryComputer.ipynb` để train model"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user