thêm chức năng train trên odc predict trên planetary
This commit is contained in:
Victor Phan
File diff suppressed because one or more lines are too long
+5
-3
@@ -30,7 +30,7 @@ def load_sentinel2_stac(
|
|||||||
Args:
|
Args:
|
||||||
bbox: Bounding box (lon_min, lat_min, lon_max, lat_max)
|
bbox: Bounding box (lon_min, lat_min, lon_max, lat_max)
|
||||||
date_range: Tuple of start and end dates
|
date_range: Tuple of start and end dates
|
||||||
bands: List of bands to load (default: ['red', 'nir', 'scl'])
|
bands: List of bands to load (default: ['B02', 'B03', 'B04', 'B08', 'B8A', 'B11', 'B12', 'SCL'])
|
||||||
resolution: Spatial resolution in meters
|
resolution: Spatial resolution in meters
|
||||||
chunks: Dask chunk sizes
|
chunks: Dask chunk sizes
|
||||||
|
|
||||||
@@ -38,7 +38,8 @@ def load_sentinel2_stac(
|
|||||||
xarray.Dataset with Sentinel-2 data
|
xarray.Dataset with Sentinel-2 data
|
||||||
"""
|
"""
|
||||||
if bands is None:
|
if bands is None:
|
||||||
bands = ['red', 'nir', 'blue', 'green', 'nir08', 'swir16', 'swir22', 'SCL']
|
# Use correct Planetary Computer band names for Sentinel-2 L2A
|
||||||
|
bands = ['B02', 'B03', 'B04', 'B08', 'B8A', 'B11', 'B12', 'SCL']
|
||||||
|
|
||||||
print(f"🔍 Searching Sentinel-2 data...")
|
print(f"🔍 Searching Sentinel-2 data...")
|
||||||
print(f" Bbox: {bbox}")
|
print(f" Bbox: {bbox}")
|
||||||
@@ -194,12 +195,13 @@ def mask_clean_s2(data, scl_band='SCL'):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def calculate_ndvi(data, nir_band='nir08', red_band='red'):
|
def calculate_ndvi(data, nir_band='B08', red_band='B04'):
|
||||||
"""
|
"""
|
||||||
Calculate NDVI from Sentinel-2 data
|
Calculate NDVI from Sentinel-2 data
|
||||||
"""
|
"""
|
||||||
print(f"📊 Calculating NDVI using {nir_band} and {red_band}...")
|
print(f"📊 Calculating NDVI using {nir_band} and {red_band}...")
|
||||||
|
|
||||||
|
|
||||||
if nir_band not in data.data_vars or red_band not in data.data_vars:
|
if nir_band not in data.data_vars or red_band not in data.data_vars:
|
||||||
print(f"⚠ Warning: Required bands not found")
|
print(f"⚠ Warning: Required bands not found")
|
||||||
print(f" Available bands: {list(data.data_vars)}")
|
print(f" Available bands: {list(data.data_vars)}")
|
||||||
|
|||||||
@@ -1,922 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "b1dab7f7",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"# 🌍 Decision Tree Land Classification - Planetary Computer\n",
|
|
||||||
"\n",
|
|
||||||
"## 📌 Notebook này chạy trên:\n",
|
|
||||||
"- ✅ **Server ODC/JupyterHub** với Dask Gateway\n",
|
|
||||||
"- ✅ Load dữ liệu từ **Microsoft Planetary Computer** (không cần ODC database)\n",
|
|
||||||
"\n",
|
|
||||||
"## 🎯 Nguồn dữ liệu:\n",
|
|
||||||
"**Microsoft Planetary Computer STAC API**\n",
|
|
||||||
"- Sentinel-2 L2A (optical)\n",
|
|
||||||
"- Sentinel-1 RTC (SAR)\n",
|
|
||||||
"\n",
|
|
||||||
"## 🚀 Infrastructure:\n",
|
|
||||||
"- **Dask Gateway**: Adaptive scaling (1-10 workers)\n",
|
|
||||||
"- **Datacube**: Initialized nhưng không dùng để load data\n",
|
|
||||||
"- **S3 Access**: Configured với requester_pays\n",
|
|
||||||
"\n",
|
|
||||||
"## 🔄 Workflow:\n",
|
|
||||||
"1. Load data từ Planetary Computer (STAC API)\n",
|
|
||||||
"2. Preprocessing (cloud mask, NDVI, resampling)\n",
|
|
||||||
"3. Train Decision Tree model\n",
|
|
||||||
"4. Evaluate & save model\n",
|
|
||||||
"\n",
|
|
||||||
"---\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 1,
|
|
||||||
"id": "df9820b8",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"ename": "ImportError",
|
|
||||||
"evalue": "cannot import name 'notebook_utils' from 'deafrica_tools' (/home/jovyan/remote-sensing/deafrica_tools/__init__.py)",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m<timed exec>:9\u001b[0m\n",
|
|
||||||
"\u001b[0;31mImportError\u001b[0m: cannot import name 'notebook_utils' from 'deafrica_tools' (/home/jovyan/remote-sensing/deafrica_tools/__init__.py)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"%matplotlib inline\n",
|
|
||||||
"\n",
|
|
||||||
"import sys\n",
|
|
||||||
"import os\n",
|
|
||||||
"sys.path.insert(0, '/home/jovyan/remote-sensing')\n",
|
|
||||||
"\n",
|
|
||||||
"# Import ODC modules for Dask Gateway and Datacube\n",
|
|
||||||
"import datacube\n",
|
|
||||||
"from deafrica_tools import notebook_utils\n",
|
|
||||||
"\n",
|
|
||||||
"# Import module load dữ liệu không cần ODC database\n",
|
|
||||||
"import importlib\n",
|
|
||||||
"import load_data_no_odc\n",
|
|
||||||
"importlib.reload(load_data_no_odc)\n",
|
|
||||||
"\n",
|
|
||||||
"from load_data_no_odc import (\n",
|
|
||||||
" load_and_process_s2,\n",
|
|
||||||
" load_and_process_s1,\n",
|
|
||||||
" load_sentinel2_stac,\n",
|
|
||||||
" load_sentinel1_stac,\n",
|
|
||||||
" mask_clean_s2,\n",
|
|
||||||
" calculate_ndvi,\n",
|
|
||||||
" fill_nan_temporal,\n",
|
|
||||||
" resample_monthly\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"# Standard imports\n",
|
|
||||||
"import numpy as np\n",
|
|
||||||
"import pandas as pd\n",
|
|
||||||
"import xarray as xr\n",
|
|
||||||
"import matplotlib.pyplot as plt\n",
|
|
||||||
"import seaborn as sns\n",
|
|
||||||
"sns.set_style('whitegrid')\n",
|
|
||||||
"\n",
|
|
||||||
"# ML imports\n",
|
|
||||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
|
||||||
"from sklearn.model_selection import train_test_split\n",
|
|
||||||
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
|
|
||||||
"import joblib\n",
|
|
||||||
"import json\n",
|
|
||||||
"from datetime import datetime\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"✅ All modules loaded successfully!\")\n",
|
|
||||||
"print(\"📡 Data source: Microsoft Planetary Computer\")\n",
|
|
||||||
"print(\"🚀 Infrastructure: Dask Gateway + ODC\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "53397b2f",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 🚀 Step 1: Initialize Dask Gateway + Datacube\n",
|
|
||||||
"\n",
|
|
||||||
"Khởi tạo Dask Gateway với adaptive scaling và cấu hình S3 access\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 2,
|
|
||||||
"id": "3c4d6779",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"🚀 Step 1: Dask Gateway + Datacube Initialization\n",
|
|
||||||
"======================================================================\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "NameError",
|
|
||||||
"evalue": "name 'notebook_utils' is not defined",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m<timed exec>:5\u001b[0m\n",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m: name 'notebook_utils' is not defined"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"🚀 Step 1: Dask Gateway + Datacube Initialization\")\n",
|
|
||||||
"print(\"=\" * 70)\n",
|
|
||||||
"\n",
|
|
||||||
"# Cấu hình Dask Gateway\n",
|
|
||||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
|
|
||||||
"\n",
|
|
||||||
"# Khai báo Datacube\n",
|
|
||||||
"dc = datacube.Datacube()\n",
|
|
||||||
"\n",
|
|
||||||
"# Cấu hình truy cập dịch vụ S3\n",
|
|
||||||
"notebook_utils.configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
|
||||||
"\n",
|
|
||||||
"print(f\"\\n✅ Dask Gateway + Datacube + S3 ready!\")\n",
|
|
||||||
"print(f\" Dask dashboard: {client.dashboard_link}\")\n",
|
|
||||||
"print(f\" Workers: Adaptive scaling (1-10)\")\n",
|
|
||||||
"print(\"=\" * 70)\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "28bc5035",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📍 Step 2: Define Area of Interest (AOI)\n",
|
|
||||||
"\n",
|
|
||||||
"Định nghĩa vùng nghiên cứu và khoảng thời gian"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 3,
|
|
||||||
"id": "15a5291c",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"📍 Area of Interest:\n",
|
|
||||||
" Bbox: (105.5, 9.2, 106.4, 10.0)\n",
|
|
||||||
" Lon range: 105.5 to 106.4\n",
|
|
||||||
" Lat range: 9.2 to 10.0\n",
|
|
||||||
" Date range: 2022-09-01 to 2023-10-01\n",
|
|
||||||
"======================================================================\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"# Cấu hình vùng và thời gian\n",
|
|
||||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
|
||||||
"\n",
|
|
||||||
"# Bounding box: (lon_min, lat_min, lon_max, lat_max)\n",
|
|
||||||
"bbox = (105.5, 9.2, 106.4, 10.0) # Khu vực Mekong Delta\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"📍 Area of Interest:\")\n",
|
|
||||||
"print(f\" Bbox: {bbox}\")\n",
|
|
||||||
"print(f\" Lon range: {bbox[0]} to {bbox[2]}\")\n",
|
|
||||||
"print(f\" Lat range: {bbox[1]} to {bbox[3]}\")\n",
|
|
||||||
"print(f\" Date range: {date_range[0]} to {date_range[1]}\")\n",
|
|
||||||
"print(\"=\" * 70)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "422e498e",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📥 Step 3: Load Sentinel-2 Data\n",
|
|
||||||
"\n",
|
|
||||||
"Load dữ liệu Sentinel-2 L2A từ Planetary Computer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 4,
|
|
||||||
"id": "611cd1c2",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"📥 Loading Sentinel-2 data from Planetary Computer...\n",
|
|
||||||
"----------------------------------------------------------------------\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "NameError",
|
|
||||||
"evalue": "name 'load_and_process_s2' is not defined",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m<timed exec>:5\u001b[0m\n",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m: name 'load_and_process_s2' is not defined"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"📥 Loading Sentinel-2 data from Planetary Computer...\")\n",
|
|
||||||
"print(\"-\" * 70)\n",
|
|
||||||
"\n",
|
|
||||||
"# Load và xử lý Sentinel-2: cloud mask + NDVI + monthly resampling\n",
|
|
||||||
"data_sen2_monthly = load_and_process_s2(\n",
|
|
||||||
" bbox=bbox,\n",
|
|
||||||
" date_range=date_range,\n",
|
|
||||||
" apply_cloud_mask=True,\n",
|
|
||||||
" calculate_indices=True\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"if data_sen2_monthly is not None:\n",
|
|
||||||
" print(f\"\\n✅ Sentinel-2 monthly data loaded!\")\n",
|
|
||||||
" print(f\" Dimensions: {dict(data_sen2_monthly.dims)}\")\n",
|
|
||||||
" print(f\" Variables: {list(data_sen2_monthly.data_vars)}\")\n",
|
|
||||||
" print(f\" Time steps: {len(data_sen2_monthly.time)}\")\n",
|
|
||||||
" \n",
|
|
||||||
" # Compute to load into memory\n",
|
|
||||||
" data_sen2_monthly = data_sen2_monthly.compute()\n",
|
|
||||||
" print(f\" ✓ Data computed and loaded into memory\")\n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Failed to load Sentinel-2 data\")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "79ef9736",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📥 Step 4: Load Sentinel-1 Data\n",
|
|
||||||
"\n",
|
|
||||||
"Load dữ liệu Sentinel-1 RTC (SAR) từ Planetary Computer"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 5,
|
|
||||||
"id": "3bda3dc0",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"📥 Loading Sentinel-1 data from Planetary Computer...\n",
|
|
||||||
"----------------------------------------------------------------------\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "NameError",
|
|
||||||
"evalue": "name 'load_and_process_s1' is not defined",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m<timed exec>:5\u001b[0m\n",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m: name 'load_and_process_s1' is not defined"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"📥 Loading Sentinel-1 data from Planetary Computer...\")\n",
|
|
||||||
"print(\"-\" * 70)\n",
|
|
||||||
"\n",
|
|
||||||
"# Load và xử lý Sentinel-1: monthly resampling\n",
|
|
||||||
"data_sen1_monthly = load_and_process_s1(\n",
|
|
||||||
" bbox=bbox,\n",
|
|
||||||
" date_range=date_range\n",
|
|
||||||
")\n",
|
|
||||||
"\n",
|
|
||||||
"if data_sen1_monthly is not None:\n",
|
|
||||||
" print(f\"\\n✅ Sentinel-1 monthly data loaded!\")\n",
|
|
||||||
" print(f\" Dimensions: {dict(data_sen1_monthly.dims)}\")\n",
|
|
||||||
" print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")\n",
|
|
||||||
" print(f\" Time steps: {len(data_sen1_monthly.time)}\")\n",
|
|
||||||
" \n",
|
|
||||||
" # Compute to load into memory\n",
|
|
||||||
" data_sen1_monthly = data_sen1_monthly.compute()\n",
|
|
||||||
" print(f\" ✓ Data computed and loaded into memory\")\n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Failed to load Sentinel-1 data\")"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "de80d48d",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 🎯 Step 5: Load Training Data\n",
|
|
||||||
"\n",
|
|
||||||
"Load dữ liệu mẫu huấn luyện (training samples)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 6,
|
|
||||||
"id": "70925d09",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"======================================================================\n",
|
|
||||||
"📦 Data Loading Module (No ODC Database Required)\n",
|
|
||||||
"======================================================================\n",
|
|
||||||
"\n",
|
|
||||||
"💡 Usage:\n",
|
|
||||||
" from load_data_no_odc import load_and_process_s2, load_and_process_s1\n",
|
|
||||||
"\n",
|
|
||||||
" bbox = (lon_min, lat_min, lon_max, lat_max)\n",
|
|
||||||
" date_range = ('2022-09-01', '2023-10-01')\n",
|
|
||||||
"\n",
|
|
||||||
" data_s2 = load_and_process_s2(bbox, date_range)\n",
|
|
||||||
" data_s1 = load_and_process_s1(bbox, date_range)\n",
|
|
||||||
"\n",
|
|
||||||
" # Load training data\n",
|
|
||||||
" train = load_train_data('train/data.shp', label_mapping)\n",
|
|
||||||
" X, y = extract_features_at_points(train, data_s2, data_s1)\n",
|
|
||||||
" X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y)\n",
|
|
||||||
"======================================================================\n",
|
|
||||||
"\n",
|
|
||||||
"🎯 Label mapping:\n",
|
|
||||||
" 0: Lua tom\n",
|
|
||||||
" 1: Lua\n",
|
|
||||||
" 2: CHN\n",
|
|
||||||
" 3: CLN\n",
|
|
||||||
" 4: TS\n",
|
|
||||||
" 5: Song\n",
|
|
||||||
" 6: Dat xay dung\n",
|
|
||||||
" 7: Rung\n",
|
|
||||||
"\n",
|
|
||||||
"📂 Loading training data from: train/ST_training_data_updated_1130points_new.shp\n",
|
|
||||||
"📂 Loading training data from: train/ST_training_data_updated_1130points_new.shp\n",
|
|
||||||
"❌ Error loading training data: train/ST_training_data_updated_1130points_new.shp: No such file or directory\n",
|
|
||||||
"❌ Failed to load training data\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from load_data_no_odc import load_train_data\n",
|
|
||||||
"\n",
|
|
||||||
"# Ánh xạ nhãn lớp đất\n",
|
|
||||||
"label_mapping = {\n",
|
|
||||||
" \"Lua tom\": 0,\n",
|
|
||||||
" \"Lua\": 1,\n",
|
|
||||||
" \"CHN\": 2,\n",
|
|
||||||
" \"CLN\": 3,\n",
|
|
||||||
" \"TS\": 4,\n",
|
|
||||||
" \"Song\": 5,\n",
|
|
||||||
" \"Dat xay dung\": 6,\n",
|
|
||||||
" \"Rung\": 7,\n",
|
|
||||||
"}\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"🎯 Label mapping:\")\n",
|
|
||||||
"for label, idx in label_mapping.items():\n",
|
|
||||||
" print(f\" {idx}: {label}\")\n",
|
|
||||||
"\n",
|
|
||||||
"# Load training data from AWS shapefile\n",
|
|
||||||
"train_path = \"train/ST_training_data_updated_1130points_new.shp\"\n",
|
|
||||||
"\n",
|
|
||||||
"print(f\"\\n📂 Loading training data from: {train_path}\")\n",
|
|
||||||
"train_data = load_train_data(train_path, label_mapping)\n",
|
|
||||||
"\n",
|
|
||||||
"if train_data is not None:\n",
|
|
||||||
" print(f\"\\n✅ Training data loaded successfully!\")\n",
|
|
||||||
" print(f\" Total points: {len(train_data)}\")\n",
|
|
||||||
" print(f\" CRS: {train_data.crs}\")\n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Failed to load training data\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "fed9a8f1",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 🔧 Step 6: Extract Features\n",
|
|
||||||
"\n",
|
|
||||||
"Trích xuất features từ Sentinel-1 và Sentinel-2 tại các điểm mẫu"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 7,
|
|
||||||
"id": "62c41cd0",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"🔧 Extracting features from satellite data...\n",
|
|
||||||
"----------------------------------------------------------------------\n",
|
|
||||||
"❌ Missing data: Please check training data or satellite data\n",
|
|
||||||
" train_data: ✗\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "NameError",
|
|
||||||
"evalue": "name 'data_sen2_monthly' is not defined",
|
|
||||||
"output_type": "error",
|
|
||||||
"traceback": [
|
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
|
|
||||||
"File \u001b[0;32m<timed exec>:32\u001b[0m\n",
|
|
||||||
"\u001b[0;31mNameError\u001b[0m: name 'data_sen2_monthly' is not defined"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"from load_data_no_odc import extract_features_at_points\n",
|
|
||||||
"\n",
|
|
||||||
"print(\"🔧 Extracting features from satellite data...\")\n",
|
|
||||||
"print(\"-\" * 70)\n",
|
|
||||||
"\n",
|
|
||||||
"if train_data is not None and \\\n",
|
|
||||||
" data_sen2_monthly is not None and \\\n",
|
|
||||||
" data_sen1_monthly is not None:\n",
|
|
||||||
" \n",
|
|
||||||
" # Extract features at training point locations\n",
|
|
||||||
" X, y = extract_features_at_points(\n",
|
|
||||||
" train_data, \n",
|
|
||||||
" data_sen2_monthly, \n",
|
|
||||||
" data_sen1_monthly\n",
|
|
||||||
" )\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"\\n✅ Feature extraction complete!\")\n",
|
|
||||||
" print(f\" Total samples: {len(X)}\")\n",
|
|
||||||
" print(f\" Feature dimension: {X.shape[1]}\")\n",
|
|
||||||
" print(f\" Classes: {sorted(set(y.tolist()))}\")\n",
|
|
||||||
" print(f\" Class distribution:\")\n",
|
|
||||||
" \n",
|
|
||||||
" import pandas as pd\n",
|
|
||||||
" class_counts = pd.Series(y).value_counts().sort_index()\n",
|
|
||||||
" for class_id, count in class_counts.items():\n",
|
|
||||||
" class_name = [k for k, v in label_mapping.items() if v == class_id][0]\n",
|
|
||||||
" print(f\" {class_id} ({class_name}): {count} samples ({count/len(y)*100:.1f}%)\")\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Missing data: Please check training data or satellite data\")\n",
|
|
||||||
" print(f\" train_data: {'✓' if train_data is not None else '✗'}\")\n",
|
|
||||||
" print(f\" data_sen2_monthly: {'✓' if data_sen2_monthly is not None else '✗'}\")\n",
|
|
||||||
" print(f\" data_sen1_monthly: {'✓' if data_sen1_monthly is not None else '✗'}\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "f7cd0ca2",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📊 Step 7: Split Data\n",
|
|
||||||
"\n",
|
|
||||||
"Chia dữ liệu thành train/val/test sets"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 8,
|
|
||||||
"id": "523e1249",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"❌ Features not extracted yet. Please run Step 6 first.\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from load_data_no_odc import split_train_data\n",
|
|
||||||
"\n",
|
|
||||||
"if 'X' in locals() and 'y' in locals():\n",
|
|
||||||
" # Split data into train/val/test sets\n",
|
|
||||||
" X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n",
|
|
||||||
" X, y, \n",
|
|
||||||
" test_size=0.2, \n",
|
|
||||||
" val_size=0.1, \n",
|
|
||||||
" random_state=42\n",
|
|
||||||
" )\n",
|
|
||||||
" \n",
|
|
||||||
" # Combine train + val for final model training\n",
|
|
||||||
" X_fit = np.concatenate([X_train, X_val], axis=0)\n",
|
|
||||||
" y_fit = np.concatenate([y_train, y_val], axis=0)\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"\\n✅ Final training set:\")\n",
|
|
||||||
" print(f\" X_fit shape: {X_fit.shape}\")\n",
|
|
||||||
" print(f\" y_fit shape: {y_fit.shape}\")\n",
|
|
||||||
" print(f\"\\n X_test shape: {X_test.shape}\")\n",
|
|
||||||
" print(f\" y_test shape: {y_test.shape}\")\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Features not extracted yet. Please run Step 6 first.\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "a469500c",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 🌲 Step 8: Train Decision Tree Model\n",
|
|
||||||
"\n",
|
|
||||||
"Huấn luyện mô hình Decision Tree"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 9,
|
|
||||||
"id": "445c2d92",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"❌ Data not ready. Please run Step 7 first.\n",
|
|
||||||
"CPU times: user 187 ms, sys: 0 ns, total: 187 ms\n",
|
|
||||||
"Wall time: 187 ms\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
|
||||||
"\n",
|
|
||||||
"if 'X_fit' in locals() and 'y_fit' in locals():\n",
|
|
||||||
" \n",
|
|
||||||
" print(\"🚀 Training Decision Tree model...\")\n",
|
|
||||||
" print(\"-\" * 70)\n",
|
|
||||||
" \n",
|
|
||||||
" # Train Decision Tree with tuned hyperparameters\n",
|
|
||||||
" model = DecisionTreeClassifier(\n",
|
|
||||||
" max_depth=30,\n",
|
|
||||||
" min_samples_leaf=2,\n",
|
|
||||||
" min_samples_split=5,\n",
|
|
||||||
" class_weight=\"balanced\",\n",
|
|
||||||
" random_state=42,\n",
|
|
||||||
" )\n",
|
|
||||||
" \n",
|
|
||||||
" model.fit(X_fit, y_fit)\n",
|
|
||||||
" \n",
|
|
||||||
" # Evaluate on validation set\n",
|
|
||||||
" val_acc = model.score(X_val, y_val)\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"\\n✅ Training complete!\")\n",
|
|
||||||
" print(f\" Model: Decision Tree\")\n",
|
|
||||||
" print(f\" Tree depth: {model.get_depth()}\")\n",
|
|
||||||
" print(f\" Number of leaves: {model.get_n_leaves()}\")\n",
|
|
||||||
" print(f\" Training samples: {len(X_fit)}\")\n",
|
|
||||||
" print(f\" Validation accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Data not ready. Please run Step 7 first.\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "7c1e681e",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📈 Step 9: Hyperparameter Analysis\n",
|
|
||||||
"\n",
|
|
||||||
"Phân tích độ sâu tối ưu (max_depth) cho Decision Tree"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 10,
|
|
||||||
"id": "d13274e4",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"❌ Data not ready. Please run Step 7 first.\n",
|
|
||||||
"CPU times: user 46 μs, sys: 0 ns, total: 46 μs\n",
|
|
||||||
"Wall time: 50.8 μs\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%time\n",
|
|
||||||
"import matplotlib.pyplot as plt\n",
|
|
||||||
"\n",
|
|
||||||
"if 'X_fit' in locals() and 'X_val' in locals():\n",
|
|
||||||
" \n",
|
|
||||||
" DEPTH_RANGE = list(range(1, 51))\n",
|
|
||||||
" \n",
|
|
||||||
" train_accs = []\n",
|
|
||||||
" val_accs = []\n",
|
|
||||||
" \n",
|
|
||||||
" print(\"🔍 Analyzing optimal max_depth for Decision Tree...\")\n",
|
|
||||||
" print(\"-\" * 70)\n",
|
|
||||||
" print(\"Testing depths from 1 to 50...\")\n",
|
|
||||||
" \n",
|
|
||||||
" for d in DEPTH_RANGE:\n",
|
|
||||||
" m = DecisionTreeClassifier(\n",
|
|
||||||
" min_samples_leaf=2,\n",
|
|
||||||
" min_samples_split=5,\n",
|
|
||||||
" class_weight=\"balanced\",\n",
|
|
||||||
" random_state=42,\n",
|
|
||||||
" max_depth=d,\n",
|
|
||||||
" )\n",
|
|
||||||
" m.fit(X_fit, y_fit)\n",
|
|
||||||
" train_accs.append(m.score(X_fit, y_fit))\n",
|
|
||||||
" val_accs.append(m.score(X_val, y_val))\n",
|
|
||||||
" \n",
|
|
||||||
" if d % 10 == 0:\n",
|
|
||||||
" print(f\" Depth {d:2d}: train={train_accs[-1]:.4f}, val={val_accs[-1]:.4f}\")\n",
|
|
||||||
" \n",
|
|
||||||
" train_accs = np.array(train_accs)\n",
|
|
||||||
" val_accs = np.array(val_accs)\n",
|
|
||||||
" \n",
|
|
||||||
" best_depth = DEPTH_RANGE[np.argmax(val_accs)]\n",
|
|
||||||
" best_val_acc = np.max(val_accs)\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"\\n✅ Optimal max_depth: {best_depth}\")\n",
|
|
||||||
" print(f\" Best validation accuracy: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\")\n",
|
|
||||||
" \n",
|
|
||||||
" # Plot convergence analysis\n",
|
|
||||||
" fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n",
|
|
||||||
" ax.plot(DEPTH_RANGE, train_accs, 'b-o', markersize=3, label='Train Accuracy')\n",
|
|
||||||
" ax.plot(DEPTH_RANGE, val_accs, 'g-o', markersize=3, label='Validation Accuracy')\n",
|
|
||||||
" ax.axvline(x=best_depth, color='red', linestyle='--', linewidth=2, \n",
|
|
||||||
" label=f'Best Depth = {best_depth}')\n",
|
|
||||||
" ax.set_xlabel('max_depth', fontsize=12)\n",
|
|
||||||
" ax.set_ylabel('Accuracy', fontsize=12)\n",
|
|
||||||
" ax.set_title('Decision Tree: Training vs Validation Accuracy', fontsize=14, fontweight='bold')\n",
|
|
||||||
" ax.legend(fontsize=10)\n",
|
|
||||||
" ax.grid(True, alpha=0.3)\n",
|
|
||||||
" plt.tight_layout()\n",
|
|
||||||
" plt.show()\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Data not ready. Please run Step 7 first.\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "b551cdb4",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 📊 Step 10: Evaluate Model\n",
|
|
||||||
"\n",
|
|
||||||
"Đánh giá mô hình trên tập test"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 11,
|
|
||||||
"id": "bc3d6b9a",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"❌ Model not trained yet. Please run Step 8 first.\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
|
|
||||||
"import seaborn as sns\n",
|
|
||||||
"\n",
|
|
||||||
"if 'model' in locals() and 'X_test' in locals():\n",
|
|
||||||
" \n",
|
|
||||||
" print(\"📊 Evaluating model on test set...\")\n",
|
|
||||||
" print(\"-\" * 70)\n",
|
|
||||||
" \n",
|
|
||||||
" # Make predictions\n",
|
|
||||||
" y_pred = model.predict(X_test)\n",
|
|
||||||
" acc = accuracy_score(y_test, y_pred)\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"\\n🎯 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
|
|
||||||
" print(\"📋 Classification Report:\")\n",
|
|
||||||
" print(classification_report(y_test, y_pred, digits=4))\n",
|
|
||||||
" \n",
|
|
||||||
" # Confusion Matrix\n",
|
|
||||||
" class_names = list(label_mapping.keys())\n",
|
|
||||||
" cm = confusion_matrix(y_test, y_pred)\n",
|
|
||||||
" \n",
|
|
||||||
" plt.figure(figsize=(10, 8))\n",
|
|
||||||
" sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',\n",
|
|
||||||
" xticklabels=class_names, yticklabels=class_names,\n",
|
|
||||||
" cbar_kws={'label': 'Count'})\n",
|
|
||||||
" plt.xlabel('Predicted Label', fontsize=12)\n",
|
|
||||||
" plt.ylabel('True Label', fontsize=12)\n",
|
|
||||||
" plt.title('Confusion Matrix — Decision Tree (Test Set)', fontsize=14, fontweight='bold')\n",
|
|
||||||
" plt.tight_layout()\n",
|
|
||||||
" plt.show()\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Model not trained yet. Please run Step 8 first.\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "44d226fc",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 💾 Step 11: Save Model\n",
|
|
||||||
"\n",
|
|
||||||
"Lưu mô hình và metadata"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 12,
|
|
||||||
"id": "044eefc4",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"❌ Model not trained yet or evaluation not complete.\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import joblib\n",
|
|
||||||
"import json\n",
|
|
||||||
"from datetime import datetime\n",
|
|
||||||
"\n",
|
|
||||||
"if 'model' in locals() and 'acc' in locals():\n",
|
|
||||||
" \n",
|
|
||||||
" print(\"💾 Saving model and metadata...\")\n",
|
|
||||||
" print(\"-\" * 70)\n",
|
|
||||||
" \n",
|
|
||||||
" # Save model\n",
|
|
||||||
" model_path = \"model_decision_tree_planetary_computer.joblib\"\n",
|
|
||||||
" joblib.dump(model, model_path)\n",
|
|
||||||
" print(f\"✅ Model saved → {model_path}\")\n",
|
|
||||||
" \n",
|
|
||||||
" # Create metadata\n",
|
|
||||||
" info = {\n",
|
|
||||||
" \"model_type\": \"DecisionTree\",\n",
|
|
||||||
" \"data_source\": \"Microsoft Planetary Computer\",\n",
|
|
||||||
" \"training_data\": train_path,\n",
|
|
||||||
" \"max_depth\": int(model.get_depth()),\n",
|
|
||||||
" \"n_leaves\": int(model.get_n_leaves()),\n",
|
|
||||||
" \"n_features\": int(X_fit.shape[1]),\n",
|
|
||||||
" \"label_mapping\": label_mapping,\n",
|
|
||||||
" \"test_accuracy\": float(acc),\n",
|
|
||||||
" \"train_samples\": int(len(X_fit)),\n",
|
|
||||||
" \"val_samples\": int(len(X_val)),\n",
|
|
||||||
" \"test_samples\": int(len(X_test)),\n",
|
|
||||||
" \"bbox\": bbox,\n",
|
|
||||||
" \"date_range\": list(date_range),\n",
|
|
||||||
" \"saved_at\": datetime.now().isoformat(),\n",
|
|
||||||
" \"hyperparameters\": {\n",
|
|
||||||
" \"max_depth\": 30,\n",
|
|
||||||
" \"min_samples_leaf\": 2,\n",
|
|
||||||
" \"min_samples_split\": 5,\n",
|
|
||||||
" \"class_weight\": \"balanced\",\n",
|
|
||||||
" \"random_state\": 42\n",
|
|
||||||
" }\n",
|
|
||||||
" }\n",
|
|
||||||
" \n",
|
|
||||||
" # Save metadata\n",
|
|
||||||
" info_path = \"model_decision_tree_planetary_computer_info.json\"\n",
|
|
||||||
" with open(info_path, \"w\") as f:\n",
|
|
||||||
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
|
|
||||||
" \n",
|
|
||||||
" print(f\"✅ Metadata saved → {info_path}\")\n",
|
|
||||||
" print(\"\\n📋 Model Info:\")\n",
|
|
||||||
" print(json.dumps(info, indent=2, ensure_ascii=False))\n",
|
|
||||||
" \n",
|
|
||||||
"else:\n",
|
|
||||||
" print(\"❌ Model not trained yet or evaluation not complete.\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "18c95152",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"## 🧹 Step 12: Cleanup\n",
|
|
||||||
"\n",
|
|
||||||
"Đóng Dask cluster"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 13,
|
|
||||||
"id": "20445068",
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"⚠ Error closing cluster: name 'client' is not defined\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"# Close Dask Gateway cluster\n",
|
|
||||||
"try:\n",
|
|
||||||
" client.close()\n",
|
|
||||||
" cluster.close()\n",
|
|
||||||
" print(\"✅ Dask Gateway cluster closed.\")\n",
|
|
||||||
"except Exception as e:\n",
|
|
||||||
" print(f\"⚠ Error closing cluster: {e}\")\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"id": "046bf0f9",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"---\n",
|
|
||||||
"\n",
|
|
||||||
"## ✅ Summary\n",
|
|
||||||
"\n",
|
|
||||||
"### Notebook này:\n",
|
|
||||||
"- ✅ Chạy trên **Server ODC/JupyterHub** với **Dask Gateway**\n",
|
|
||||||
"- ✅ Load dữ liệu từ **Microsoft Planetary Computer** qua STAC API\n",
|
|
||||||
"- ✅ Không cần **ODC Database** để load data (dùng Planetary Computer)\n",
|
|
||||||
"- ✅ Sử dụng infrastructure của ODC (Dask Gateway, S3 access)\n",
|
|
||||||
"- ✅ Tương thích 100% với dữ liệu ODC\n",
|
|
||||||
"\n",
|
|
||||||
"### Ưu điểm của approach này:\n",
|
|
||||||
"1. **Scalability**: Dask Gateway adaptive scaling (1-10 workers)\n",
|
|
||||||
"2. **Public data**: Planetary Computer không cần VPN/private network\n",
|
|
||||||
"3. **Consistent**: Cùng infrastructure với ODC training pipeline\n",
|
|
||||||
"4. **Flexible**: Có thể train trên Planetary Computer, predict trên ODC hoặc ngược lại\n",
|
|
||||||
"\n",
|
|
||||||
"### Workflow train/predict:\n",
|
|
||||||
"1. **Train trên server ODC**: Chạy notebook này với Dask Gateway\n",
|
|
||||||
"2. **Save model**: Export `.joblib` file \n",
|
|
||||||
"3. **Predict qua API**: api_server.py tự động dùng Planetary Computer\n",
|
|
||||||
"\n",
|
|
||||||
"### Architecture:\n",
|
|
||||||
"```\n",
|
|
||||||
"┌─────────────────────────────────────┐\n",
|
|
||||||
"│ TRAINING (ODC Infrastructure) │\n",
|
|
||||||
"│ ✅ Dask Gateway (1-10 workers) │\n",
|
|
||||||
"│ ✅ Planetary Computer STAC API │\n",
|
|
||||||
"│ ✅ S3 access configured │\n",
|
|
||||||
"│ → Model: .joblib │\n",
|
|
||||||
"└─────────────────────────────────────┘\n",
|
|
||||||
" ↓\n",
|
|
||||||
"┌─────────────────────────────────────┐\n",
|
|
||||||
"│ PREDICTION (API Server) │\n",
|
|
||||||
"│ ✅ Planetary Computer (public) │\n",
|
|
||||||
"│ ✅ Same preprocessing pipeline │\n",
|
|
||||||
"│ → GeoTIFF output │\n",
|
|
||||||
"└─────────────────────────────────────┘\n",
|
|
||||||
"```\n",
|
|
||||||
"\n",
|
|
||||||
"---\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3 (ipykernel)",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.12.3"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 5
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user