diff --git a/train_files/01.train_DecisionTree_ODC.ipynb b/train_files/01.train_DecisionTree_ODC.ipynb new file mode 100644 index 0000000..6048a7c --- /dev/null +++ b/train_files/01.train_DecisionTree_ODC.ipynb @@ -0,0 +1,654 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bb54fa69", + "metadata": {}, + "source": [ + "# 🌍 Decision Tree Land Classification - ODC Database\n", + "\n", + "## 📌 Notebook này chạy trên:\n", + "- ✅ **Server ODC/JupyterHub** với Dask Gateway\n", + "- ✅ Load dữ liệu từ **ODC Database** (access nhanh trên server)\n", + "\n", + "## 🎯 Nguồn dữ liệu:\n", + "**ODC Database**\n", + "- Sentinel-2 L2A (optical) từ ODC\n", + "- Sentinel-1 RTC (SAR) từ ODC\n", + "\n", + "## 🚀 Infrastructure:\n", + "- **Dask Gateway**: Adaptive scaling (1-10 workers)\n", + "- **Datacube**: Load data từ ODC database\n", + "- **S3 Access**: Configured với requester_pays\n", + "\n", + "## 🔄 Workflow:\n", + "1. Load data từ ODC Database (nhanh trên server)\n", + "2. Preprocessing (cloud mask, NDVI, resampling)\n", + "3. Train Decision Tree model\n", + "4. Save model → dùng cho prediction trên Planetary Computer\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10e7e875", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "%matplotlib inline\n", + "\n", + "import sys\n", + "import os\n", + "sys.path.insert(0, '/home/jovyan/remote-sensing')\n", + "\n", + "# Import ODC modules\n", + "import datacube\n", + "from easi_tools import notebook_utils\n", + "from datacube.utils.rio import configure_s3_access\n", + "\n", + "# Import ODC data loading functions\n", + "import importlib\n", + "import new_import_ODC\n", + "importlib.reload(new_import_ODC)\n", + "from new_import_ODC import *\n", + "\n", + "# Standard imports\n", + "import numpy as np\n", + "import pandas as pd\n", + "import xarray as xr\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "sns.set_style('whitegrid')\n", + "\n", + "# ML imports\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n", + "import joblib\n", + "import json\n", + "from datetime import datetime\n", + "\n", + "print(\"✅ All modules loaded successfully!\")\n", + "print(\"📡 Data source: ODC Database\")\n", + "print(\"🚀 Infrastructure: Dask Gateway + ODC\")" + ] + }, + { + "cell_type": "markdown", + "id": "0ad19235", + "metadata": {}, + "source": [ + "## 🚀 Step 1: Initialize Dask Gateway + Datacube\n", + "\n", + "Khởi tạo Dask Gateway với adaptive scaling và kết nối tới ODC Database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d894cbd", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"🚀 Step 1: Dask Gateway + Datacube Initialization\")\n", + "print(\"=\" * 70)\n", + "\n", + "# Cấu hình Dask Gateway\n", + "cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n", + "\n", + "# Khai báo Datacube - kết nối tới ODC Database\n", + "dc = datacube.Datacube()\n", + "\n", + "# Cấu hình truy cập dịch vụ S3\n", + "configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n", + "\n", + "print(f\"\\n✅ Dask Gateway + Datacube + S3 ready!\")\n", + "print(f\" Dask dashboard: {client.dashboard_link}\")\n", + "print(f\" Workers: Adaptive scaling (1-10)\")\n", + "print(f\" ODC products available: {len(dc.list_products())}\")\n", + "print(\"=\" * 70)" + ] + }, + { + "cell_type": "markdown", + "id": "d96f19e5", + "metadata": {}, + "source": [ + "## 📍 Step 2: Define Area of Interest (AOI)\n", + "\n", + "Định nghĩa vùng nghiên cứu và khoảng thời gian" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98046ea8", + "metadata": {}, + "outputs": [], + "source": [ + "# Cấu hình vùng và thời gian\n", + "date_range = (\"2022-09-01\", \"2023-10-01\")\n", + "\n", + "# Coordinates: (lon_min, lon_max), (lat_min, lat_max)\n", + "longitude_range = (105.5, 106.4) # Khu vực Mekong Delta\n", + "latitude_range = (9.2, 10.0)\n", + "\n", + "print(\"📍 Area of Interest:\")\n", + "print(f\" Longitude range: {longitude_range[0]} to {longitude_range[1]}\")\n", + "print(f\" Latitude range: {latitude_range[0]} to {latitude_range[1]}\")\n", + "print(f\" Date range: {date_range[0]} to {date_range[1]}\")\n", + "print(\"=\" * 70)" + ] + }, + { + "cell_type": "markdown", + "id": "79d37cff", + "metadata": {}, + "source": [ + "## 📥 Step 3: Load Sentinel-2 Data from ODC\n", + "\n", + "Load dữ liệu Sentinel-2 L2A từ ODC Database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e8b535d", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"📥 Loading Sentinel-2 data from ODC Database...\")\n", + "print(\"-\" * 70)\n", + "\n", + "# Load Sentinel-2 từ ODC\n", + "data_sen2 = load_data(dc, date_range, longitude_range, latitude_range)\n", + "\n", + "if data_sen2 is not None:\n", + " print(f\"\\n✅ Sentinel-2 data loaded from ODC!\")\n", + " print(f\" Dimensions: {dict(data_sen2.dims)}\")\n", + " print(f\" Variables: {list(data_sen2.data_vars)}\")\n", + " print(f\" Time steps: {len(data_sen2.time)}\")\n", + " \n", + " # Apply cloud masking\n", + " print(\"\\n🔧 Applying cloud mask...\")\n", + " data_sen2_clean = mask_clean(data_sen2)\n", + " \n", + " # Calculate NDVI\n", + " print(\"📊 Calculating NDVI...\")\n", + " ndvi = calculate_indices(\n", + " data_sen2_clean,\n", + " index='NDVI',\n", + " red='red',\n", + " nir='nir',\n", + " collection='s2'\n", + " )\n", + " \n", + " # Resample to monthly\n", + " print(\"📅 Resampling to monthly averages...\")\n", + " data_sen2_monthly = calculate_average(ndvi, time_pattern='1M')\n", + " \n", + " # Compute to load into memory\n", + " data_sen2_monthly = data_sen2_monthly.compute()\n", + " print(f\"\\n✅ Sentinel-2 monthly data ready!\")\n", + " print(f\" ✓ Cloud masked\")\n", + " print(f\" ✓ NDVI calculated\")\n", + " print(f\" ✓ Monthly resampled\")\n", + " print(f\" ✓ Loaded into memory\")\n", + "else:\n", + " print(\"❌ Failed to load Sentinel-2 data\")" + ] + }, + { + "cell_type": "markdown", + "id": "d0c0f672", + "metadata": {}, + "source": [ + "## 📥 Step 4: Load Sentinel-1 Data from ODC\n", + "\n", + "Load dữ liệu Sentinel-1 RTC (SAR) từ ODC Database" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e629839", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"📥 Loading Sentinel-1 data from ODC Database...\")\n", + "print(\"-\" * 70)\n", + "\n", + "# Load Sentinel-1 từ ODC\n", + "# Note: load_data_sen1 returns (dsvh, dsvv) - 2 separate DataArrays\n", + "coordinates = (longitude_range, latitude_range)\n", + "\n", + "dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n", + "\n", + "if dsvh is not None and dsvv is not None:\n", + " print(f\"\\n✅ Sentinel-1 data loaded from ODC!\")\n", + " print(f\" VH dimensions: {dict(dsvh.dims)}\")\n", + " print(f\" VV dimensions: {dict(dsvv.dims)}\")\n", + " print(f\" Time steps: {len(dsvh.time)}\")\n", + " \n", + " # Resample to monthly\n", + " print(\"\\n📅 Resampling to monthly averages...\")\n", + " dsvh_monthly = calculate_average(dsvh, time_pattern='1M')\n", + " dsvv_monthly = calculate_average(dsvv, time_pattern='1M')\n", + " \n", + " # Compute to load into memory\n", + " dsvh_monthly = dsvh_monthly.compute()\n", + " dsvv_monthly = dsvv_monthly.compute()\n", + " print(f\"\\n✅ Sentinel-1 monthly data ready!\")\n", + " print(f\" ✓ Monthly resampled\")\n", + " print(f\" ✓ Loaded into memory\")\n", + "else:\n", + " print(\"❌ Failed to load Sentinel-1 data\")\n", + " print(\"⚠ Note: Nếu không có S1 trong ODC, có thể load từ file local:\")\n", + " print(\" dsvh, dsvv = load_sen1('path/to/vh.tif', 'path/to/vv.tif')\")" + ] + }, + { + "cell_type": "markdown", + "id": "40a2ffe6", + "metadata": {}, + "source": [ + "## 🎯 Step 5: Load Training Data\n", + "\n", + "Load dữ liệu mẫu huấn luyện (training samples)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9be070b8", + "metadata": {}, + "outputs": [], + "source": [ + "# Ánh xạ nhãn lớp đất\n", + "label_mapping = {\n", + " \"Lua tom\": 0,\n", + " \"Lua\": 1,\n", + " \"CHN\": 2,\n", + " \"CLN\": 3,\n", + " \"TS\": 4,\n", + " \"Song\": 5,\n", + " \"Dat xay dung\": 6,\n", + " \"Rung\": 7,\n", + "}\n", + "\n", + "print(\"🎯 Label mapping:\")\n", + "for label, idx in label_mapping.items():\n", + " print(f\" {idx}: {label}\")\n", + "\n", + "# Load training data\n", + "train_path = \"train/ST_training_data_updated_1130points_new.shp\"\n", + "\n", + "print(f\"\\n📂 Loading training data from: {train_path}\")\n", + "train_data = load_train_data(train_path)\n", + "\n", + "if train_data is not None:\n", + " print(f\"\\n✅ Training data loaded successfully!\")\n", + " print(f\" Total points: {len(train_data)}\")\n", + " print(f\" CRS: {train_data.crs}\")\n", + " print(f\" Columns: {list(train_data.columns)}\")\n", + "else:\n", + " print(\"❌ Failed to load training data\")" + ] + }, + { + "cell_type": "markdown", + "id": "b371c97f", + "metadata": {}, + "source": [ + "## 🔧 Step 6: Extract Features\n", + "\n", + "Trích xuất features từ Sentinel-1 và Sentinel-2 tại các điểm mẫu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d6c117d", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "print(\"🔧 Extracting features from satellite data...\")\n", + "print(\"-\" * 70)\n", + "\n", + "if train_data is not None and \\\n", + " 'data_sen2_monthly' in locals() and \\\n", + " 'dsvh_monthly' in locals() and \\\n", + " 'dsvv_monthly' in locals():\n", + " \n", + " # Extract features using ODC function\n", + " # Note: get_data_sen1_and_sen2 expects average_ndvi, dsvh, dsvv\n", + " datasets = get_data_sen1_and_sen2(\n", + " train_data, \n", + " data_sen2_monthly, # NDVI monthly average\n", + " dsvh_monthly, # VH monthly average\n", + " dsvv_monthly # VV monthly average\n", + " )\n", + " \n", + " print(f\"\\n✅ Feature extraction complete!\")\n", + " print(f\" Total datasets: {len(datasets)}\")\n", + " \n", + " # Display statistics\n", + " valid_datasets = [v for v in datasets.values() if v is not None]\n", + " if valid_datasets:\n", + " sample_data = valid_datasets[0]['data']\n", + " print(f\" Feature dimension: {len(sample_data)}\")\n", + " print(f\" Valid samples: {len(valid_datasets)}\")\n", + " \n", + " # Count labels\n", + " labels = [d['label'] for d in valid_datasets]\n", + " label_counts = pd.Series(labels).value_counts().sort_index()\n", + " print(f\"\\n Label distribution:\")\n", + " for label_id, count in label_counts.items():\n", + " print(f\" {label_id}: {count} samples ({count/len(labels)*100:.1f}%)\")\n", + " \n", + "else:\n", + " print(\"❌ Missing data: Please check training data or satellite data\")\n", + " print(f\" train_data: {'✓' if train_data is not None else '✗'}\")\n", + " print(f\" data_sen2_monthly: {'✓' if 'data_sen2_monthly' in locals() else '✗'}\")\n", + " print(f\" dsvh_monthly: {'✓' if 'dsvh_monthly' in locals() else '✗'}\")\n", + " print(f\" dsvv_monthly: {'✓' if 'dsvv_monthly' in locals() else '✗'}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b485ba1b", + "metadata": {}, + "source": [ + "## 📊 Step 7: Split Data\n", + "\n", + "Chia dữ liệu thành train/val/test sets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29726670", + "metadata": {}, + "outputs": [], + "source": [ + "if 'datasets' in locals():\n", + " # Split data using ODC function\n", + " X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n", + " train_data, \n", + " label_mapping, \n", + " datasets\n", + " )\n", + " \n", + " # Combine train + val for final model training\n", + " X_fit = X_train + X_val\n", + " y_fit = y_train + y_val\n", + " \n", + " print(f\"\\n✅ Data split complete:\")\n", + " print(f\" Training samples: {len(X_train)}\")\n", + " print(f\" Validation samples: {len(X_val)}\")\n", + " print(f\" Test samples: {len(X_test)}\")\n", + " print(f\"\\n Combined train+val: {len(X_fit)} samples\")\n", + " \n", + "else:\n", + " print(\"❌ Features not extracted yet. Please run Step 6 first.\")" + ] + }, + { + "cell_type": "markdown", + "id": "3966ba86", + "metadata": {}, + "source": [ + "## 🌲 Step 8: Train Decision Tree Model\n", + "\n", + "Huấn luyện mô hình Decision Tree" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2fc1c3da", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "if 'X_fit' in locals() and 'y_fit' in locals():\n", + " \n", + " print(\"🚀 Training Decision Tree model...\")\n", + " print(\"-\" * 70)\n", + " \n", + " # Train Decision Tree with tuned hyperparameters\n", + " model = DecisionTreeClassifier(\n", + " max_depth=30,\n", + " min_samples_leaf=2,\n", + " min_samples_split=5,\n", + " class_weight=\"balanced\",\n", + " random_state=42,\n", + " )\n", + " \n", + " model.fit(X_fit, y_fit)\n", + " \n", + " # Evaluate on validation set\n", + " val_acc = model.score(X_val, y_val)\n", + " \n", + " print(f\"\\n✅ Training complete!\")\n", + " print(f\" Model: Decision Tree\")\n", + " print(f\" Tree depth: {model.get_depth()}\")\n", + " print(f\" Number of leaves: {model.get_n_leaves()}\")\n", + " print(f\" Training samples: {len(X_fit)}\")\n", + " print(f\" Validation accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n", + " \n", + "else:\n", + " print(\"❌ Data not ready. Please run Step 7 first.\")" + ] + }, + { + "cell_type": "markdown", + "id": "b244cc21", + "metadata": {}, + "source": [ + "## 📊 Step 9: Evaluate Model\n", + "\n", + "Đánh giá mô hình trên tập test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c50781e", + "metadata": {}, + "outputs": [], + "source": [ + "if 'model' in locals() and 'X_test' in locals():\n", + " \n", + " print(\"📊 Evaluating model on test set...\")\n", + " print(\"-\" * 70)\n", + " \n", + " # Make predictions\n", + " y_pred = model.predict(X_test)\n", + " acc = accuracy_score(y_test, y_pred)\n", + " \n", + " print(f\"\\n🎯 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n", + " print(\"📋 Classification Report:\")\n", + " print(classification_report(y_test, y_pred, digits=4))\n", + " \n", + " # Confusion Matrix\n", + " class_names = list(label_mapping.keys())\n", + " cm = confusion_matrix(y_test, y_pred)\n", + " \n", + " plt.figure(figsize=(10, 8))\n", + " sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',\n", + " xticklabels=class_names, yticklabels=class_names,\n", + " cbar_kws={'label': 'Count'})\n", + " plt.xlabel('Predicted Label', fontsize=12)\n", + " plt.ylabel('True Label', fontsize=12)\n", + " plt.title('Confusion Matrix — Decision Tree (Test Set)', fontsize=14, fontweight='bold')\n", + " plt.tight_layout()\n", + " plt.show()\n", + " \n", + "else:\n", + " print(\"❌ Model not trained yet. Please run Step 8 first.\")" + ] + }, + { + "cell_type": "markdown", + "id": "3db1ed05", + "metadata": {}, + "source": [ + "## 💾 Step 10: Save Model\n", + "\n", + "Lưu mô hình để dùng cho prediction trên Planetary Computer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11e9ac2d", + "metadata": {}, + "outputs": [], + "source": [ + "if 'model' in locals() and 'acc' in locals():\n", + " \n", + " print(\"💾 Saving model and metadata...\")\n", + " print(\"-\" * 70)\n", + " \n", + " # Save model\n", + " model_path = \"model_decision_tree_odc.joblib\"\n", + " joblib.dump(model, model_path)\n", + " print(f\"✅ Model saved → {model_path}\")\n", + " \n", + " # Create metadata\n", + " info = {\n", + " \"model_type\": \"DecisionTree\",\n", + " \"data_source\": \"ODC Database\",\n", + " \"training_data\": train_path,\n", + " \"max_depth\": int(model.get_depth()),\n", + " \"n_leaves\": int(model.get_n_leaves()),\n", + " \"n_features\": len(X_fit[0]) if X_fit else 0,\n", + " \"label_mapping\": label_mapping,\n", + " \"test_accuracy\": float(acc),\n", + " \"train_samples\": int(len(X_fit)),\n", + " \"val_samples\": int(len(X_val)),\n", + " \"test_samples\": int(len(X_test)),\n", + " \"longitude_range\": list(longitude_range),\n", + " \"latitude_range\": list(latitude_range),\n", + " \"date_range\": list(date_range),\n", + " \"saved_at\": datetime.now().isoformat(),\n", + " \"hyperparameters\": {\n", + " \"max_depth\": 30,\n", + " \"min_samples_leaf\": 2,\n", + " \"min_samples_split\": 5,\n", + " \"class_weight\": \"balanced\",\n", + " \"random_state\": 42\n", + " },\n", + " \"note\": \"Model trained on ODC data, can be used for prediction on Planetary Computer\"\n", + " }\n", + " \n", + " # Save metadata\n", + " info_path = \"model_decision_tree_odc_info.json\"\n", + " with open(info_path, \"w\") as f:\n", + " json.dump(info, f, indent=2, ensure_ascii=False)\n", + " \n", + " print(f\"✅ Metadata saved → {info_path}\")\n", + " print(\"\\n📋 Model Info:\")\n", + " print(json.dumps(info, indent=2, ensure_ascii=False))\n", + " print(\"\\n💡 Model này có thể dùng cho prediction trên Planetary Computer!\")\n", + " \n", + "else:\n", + " print(\"❌ Model not trained yet or evaluation not complete.\")" + ] + }, + { + "cell_type": "markdown", + "id": "8280de27", + "metadata": {}, + "source": [ + "## 🧹 Step 11: Cleanup\n", + "\n", + "Đóng Dask cluster" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ffe4f8a", + "metadata": {}, + "outputs": [], + "source": [ + "# Close Dask Gateway cluster\n", + "try:\n", + " client.close()\n", + " cluster.close()\n", + " print(\"✅ Dask Gateway cluster closed.\")\n", + "except Exception as e:\n", + " print(f\"⚠ Error closing cluster: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "382428e1", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## ✅ Summary\n", + "\n", + "### Notebook này:\n", + "- ✅ Chạy trên **Server ODC/JupyterHub** với **Dask Gateway**\n", + "- ✅ Load dữ liệu từ **ODC Database** (rất nhanh trên server)\n", + "- ✅ Train model với dữ liệu từ ODC\n", + "- ✅ Save model để dùng cho prediction\n", + "\n", + "### Ưu điểm của approach này:\n", + "1. **Speed**: ODC Database truy cập cực nhanh trên server\n", + "2. **Scalability**: Dask Gateway adaptive scaling (1-10 workers)\n", + "3. **Compatibility**: Model có thể dùng trên Planetary Computer\n", + "\n", + "### Workflow hoàn chỉnh:\n", + "```\n", + "┌─────────────────────────────────────┐\n", + "│ TRAINING (ODC Server) │\n", + "│ ✅ Load từ ODC Database (FAST) │\n", + "│ ✅ Dask Gateway (1-10 workers) │\n", + "│ ✅ Train Decision Tree │\n", + "│ → Model: .joblib │\n", + "└─────────────────────────────────────┘\n", + " ↓\n", + "┌─────────────────────────────────────┐\n", + "│ PREDICTION (Local Machine) │\n", + "│ ✅ Load từ Planetary Computer │\n", + "│ ✅ Use trained model │\n", + "│ ✅ No VPN/ODC access needed │\n", + "│ → GeoTIFF output │\n", + "└─────────────────────────────────────┘\n", + "```\n", + "\n", + "### Prediction API:\n", + "- Máy local không cần truy cập ODC\n", + "- Load data từ Planetary Computer (public)\n", + "- Sử dụng model đã train từ ODC\n", + "- Kết quả prediction giống nhau!\n", + "\n", + "---" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}