update 01
This commit is contained in:
Victor Phan
+5
-4
@@ -1,4 +1,5 @@
|
||||
*.tif filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.nc filter=lfs diff=lfs merge=lfs -text
|
||||
*.ipynb filter=lfs diff=lfs merge=lfs -text
|
||||
# Disabled LFS tracking to avoid pushing large files
|
||||
#*.tif filter=lfs diff=lfs merge=lfs -text
|
||||
#*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
#*.nc filter=lfs diff=lfs merge=lfs -text
|
||||
#*.ipynb filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
|
||||
# Ignore all model weights and large data
|
||||
*.joblib
|
||||
*.nc
|
||||
*.tif
|
||||
*.tiff
|
||||
*.png
|
||||
*.jpg
|
||||
*.jpeg
|
||||
*.h5
|
||||
*.pt
|
||||
*.ckpt
|
||||
*.pb
|
||||
*.npz
|
||||
*.npy
|
||||
|
||||
# Ignore model info/metadata if không cần backup
|
||||
# *.json
|
||||
|
||||
# Ignore cache, prediction, backup folders
|
||||
dataset_cache/
|
||||
predictions/
|
||||
backup_model_train/
|
||||
backup_ketquaphanloai/
|
||||
backup_S3_download_Amazon/
|
||||
model_train/
|
||||
__pycache__/
|
||||
# Ignore large data files
|
||||
ndvi_results/
|
||||
ndvi_cache/
|
||||
prediction_cache/
|
||||
dataset_cache/
|
||||
bfg.jar
|
||||
..bfg-report/
|
||||
.dvc/
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"python-envs.defaultEnvManager": "ms-python.python:conda",
|
||||
"python-envs.defaultPackageManager": "ms-python.python:conda",
|
||||
"python-envs.pythonProjects": []
|
||||
}
|
||||
@@ -0,0 +1,947 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "912ed572-1658-406b-976c-cd6de2d4e89e",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ModuleNotFoundError",
|
||||
"evalue": "No module named 'easi_tools'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m<timed exec>:4\u001b[0m\n",
|
||||
"File \u001b[0;32m~/CSIROBoeingPhase5-Vietnam/new_import_ODC.py:23\u001b[0m\n\u001b[1;32m 21\u001b[0m easinotebooksrepo \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/home/jovyan/easi-notebooks\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m easinotebooksrepo \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m sys\u001b[38;5;241m.\u001b[39mpath: sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(easinotebooksrepo)\n\u001b[0;32m---> 23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01measi_tools\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m EasiDefaults, xarray_object_size, notebook_utils, unset_cachingproxy\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01measi_tools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mload_s2l2a\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m load_s2l2a_with_offset\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mdask\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdistributed\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m progress\n",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'easi_tools'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import new_import_ODC \n",
|
||||
"\n",
|
||||
"importlib.reload(new_import_ODC)\n",
|
||||
"\n",
|
||||
"from new_import_ODC import *\n",
|
||||
"\n",
|
||||
"print(\"✅ All modules loaded successfully\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "d824dc4f-994b-4d1c-8d24-ce6674da141c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ AWS credentials loaded from environment variables\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/x79/miniconda/envs/env_01/lib/python3.10/site-packages/distributed/node.py:187: UserWarning: Port 8787 is already in use.\n",
|
||||
"Perhaps you already have a cluster running?\n",
|
||||
"Hosting the HTTP server on port 41709 instead\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Dask cluster initialized\n",
|
||||
" Cluster: LocalCluster(9f2167a3, 'tcp://127.0.0.1:41233', workers=4, threads=24, memory=31.26 GiB)\n",
|
||||
"✅ Datacube connected (metadata only)\n",
|
||||
"\n",
|
||||
"======================================================================\n",
|
||||
"CPU times: user 4.98 s, sys: 831 ms, total: 5.81 s\n",
|
||||
"Wall time: 7.69 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"print(\"✅ AWS credentials loaded from environment variables\")\n",
|
||||
"\n",
|
||||
"# Cấu hình Dask local\n",
|
||||
"from dask.distributed import Client, LocalCluster\n",
|
||||
"\n",
|
||||
"cluster = LocalCluster(n_workers=4)\n",
|
||||
"client = Client(cluster)\n",
|
||||
"print(\"✅ Dask cluster initialized\")\n",
|
||||
"print(f\" Cluster: {cluster}\")\n",
|
||||
"\n",
|
||||
"# Khai báo Datacube (chỉ để lấy metadata, không dùng load())\n",
|
||||
"import datacube\n",
|
||||
"try:\n",
|
||||
" dc = datacube.Datacube()\n",
|
||||
" print(\"✅ Datacube connected (metadata only)\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"⚠️ Datacube connection not critical: {e}\")\n",
|
||||
" dc = None\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "1e113730",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"GETTING SENTINEL-2 SCENE METADATA\n",
|
||||
"======================================================================\n",
|
||||
"\n",
|
||||
"[1] Loading metadata from datacube...\n",
|
||||
" ✅ Found 40 scenes\n",
|
||||
"\n",
|
||||
"[2] Selected scene: S2A_48PWR_20231226_0_L2A\n",
|
||||
" Date: 2023-12-26 03:35:26.919000+00:00\n",
|
||||
"\n",
|
||||
"[3] Available bands:\n",
|
||||
" - nir: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - red: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - scl: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - blue: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - green: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - nir08: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - nir09: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - swir16: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - swir22: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - coastal: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - rededge1: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - rededge2: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
" - rededge3: https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/48/P/WR/20\n",
|
||||
"======================================================================\n",
|
||||
"CPU times: user 3.34 s, sys: 76.3 ms, total: 3.41 s\n",
|
||||
"Wall time: 3.23 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# 🔧 Get Sentinel-2 scene metadata from datacube\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"GETTING SENTINEL-2 SCENE METADATA\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"date_range = (\"2023-03-01\", \"2023-12-31\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" print(f\"\\n[1] Loading metadata from datacube...\")\n",
|
||||
" datasets = list(dc.find_datasets(product='s2_l2a', time=date_range))\n",
|
||||
" print(f\" ✅ Found {len(datasets)} scenes\")\n",
|
||||
" \n",
|
||||
" if datasets:\n",
|
||||
" selected = datasets[0]\n",
|
||||
" print(f\"\\n[2] Selected scene: {selected.metadata.label}\")\n",
|
||||
" scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time\n",
|
||||
" print(f\" Date: {scene_datetime}\")\n",
|
||||
" \n",
|
||||
" # Display measurement paths\n",
|
||||
" print(f\"\\n[3] Available bands:\")\n",
|
||||
" for name, measurement in selected.measurements.items():\n",
|
||||
" print(f\" - {name}: {measurement['path'][:80]}\")\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"❌ Error: {e}\")\n",
|
||||
" import traceback\n",
|
||||
" traceback.print_exc()\n",
|
||||
"\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "3cd69645",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"CHECKING FOR CACHED DATASET\n",
|
||||
"======================================================================\n",
|
||||
"\n",
|
||||
"⏳ Cache file not found: dataset_cache/sentinel2_timeseries_40scenes.nc\n",
|
||||
" Will download from S3 and save cache\n",
|
||||
" (Next run will use cache automatically)\n",
|
||||
"======================================================================\n",
|
||||
"CPU times: user 4.36 ms, sys: 3.7 ms, total: 8.06 ms\n",
|
||||
"Wall time: 7.22 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# 🔍 CHECK IF DATASET CACHE EXISTS (Skip download if available)\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"CHECKING FOR CACHED DATASET\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import xarray as xr\n",
|
||||
"\n",
|
||||
"cache_dir = \"dataset_cache\"\n",
|
||||
"cache_file = f\"{cache_dir}/sentinel2_timeseries_40scenes.nc\"\n",
|
||||
"\n",
|
||||
"use_cache = False\n",
|
||||
"\n",
|
||||
"if os.path.exists(cache_file):\n",
|
||||
" print(f\"\\n✅ Cache file found: {cache_file}\")\n",
|
||||
" \n",
|
||||
" # Get file info\n",
|
||||
" file_size_gb = os.path.getsize(cache_file) / (1024**3)\n",
|
||||
" print(f\" File size: {file_size_gb:.2f} GB\")\n",
|
||||
" \n",
|
||||
" # Try to load\n",
|
||||
" try:\n",
|
||||
" print(f\"\\n🔄 Loading dataset from cache...\")\n",
|
||||
" data = xr.open_dataset(cache_file)\n",
|
||||
" \n",
|
||||
" print(f\"✅ Dataset loaded from cache!\")\n",
|
||||
" print(f\" Total scenes: {len(data['time'])}\")\n",
|
||||
" print(f\" Variables: {len(data.data_vars)}\")\n",
|
||||
" print(f\" Dimensions: {dict(data.dims)}\")\n",
|
||||
" print(f\"\\n ⏭️ Skipping S3 download (using cached data)\")\n",
|
||||
" \n",
|
||||
" use_cache = True\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"❌ Error loading cache: {e}\")\n",
|
||||
" print(f\" Will download fresh data from S3\")\n",
|
||||
" use_cache = False\n",
|
||||
"else:\n",
|
||||
" print(f\"\\n⏳ Cache file not found: {cache_file}\")\n",
|
||||
" print(f\" Will download from S3 and save cache\")\n",
|
||||
" print(f\" (Next run will use cache automatically)\")\n",
|
||||
"\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "435f9f78-a9a4-4226-86ca-d4bec42d454e",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"LOADING SENTINEL-2 DATA FROM S3 COGs (RASTERIO) - OPTIMAL ACCURACY\n",
|
||||
"======================================================================\n",
|
||||
"\n",
|
||||
"📥 Downloading from S3...\n",
|
||||
"\n",
|
||||
"📦 Found 40 available scenes\n",
|
||||
" Date range: 2023-03-01 to 2023-12-31\n",
|
||||
"\n",
|
||||
"[LOADING] Loading ALL 40 scenes with ALL available bands...\n",
|
||||
" (Keeping NATIVE resolution - NO upsampling/magnification)\n",
|
||||
" Available bands: ['nir', 'red', 'scl', 'blue', 'green', 'nir08', 'nir09', 'swir16', 'swir22', 'coastal', 'rededge1', 'rededge2', 'rededge3']\n",
|
||||
"\n",
|
||||
" [ 1/1] S2A_48PWR_20231226_0_L2A (2023-12-26)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" ✅ 13 bands loaded\n",
|
||||
"\n",
|
||||
"✅ Successfully loaded 1 scenes!\n",
|
||||
"\n",
|
||||
"[RESOLUTION NORMALIZATION] Aligning all bands to native resolution (NO magnification)...\n",
|
||||
" Reference resolution: 10980×10980 pixels (native nir)\n",
|
||||
" Resampling scl: 5490×5490 → 10980×10980\n",
|
||||
" Resampling nir08: 5490×5490 → 10980×10980\n",
|
||||
" Resampling nir09: 1830×1830 → 10980×10980\n",
|
||||
" Resampling swir16: 5490×5490 → 10980×10980\n",
|
||||
" Resampling swir22: 5490×5490 → 10980×10980\n",
|
||||
" Resampling coastal: 1830×1830 → 10980×10980\n",
|
||||
" Resampling rededge1: 5490×5490 → 10980×10980\n",
|
||||
" Resampling rededge2: 5490×5490 → 10980×10980\n",
|
||||
" Resampling rededge3: 5490×5490 → 10980×10980\n",
|
||||
"✅ Resolution normalization complete! (9 bands resampled)\n",
|
||||
"\n",
|
||||
"[SPECTRAL INDICES] Calculating spectral indices for each scene...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"<timed exec>:169: RuntimeWarning: divide by zero encountered in divide\n",
|
||||
"<timed exec>:169: RuntimeWarning: invalid value encountered in divide\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Calculated 2 spectral indices per scene\n",
|
||||
"\n",
|
||||
"[STACKING] Stacking all 1 scenes to create time-series...\n",
|
||||
"\n",
|
||||
"[TEMPORAL FEATURES] Computing temporal features from time-series...\n",
|
||||
"✅ Added 6 temporal/aggregate features\n",
|
||||
"\n",
|
||||
"[CACHE] Saving dataset to cache...\n",
|
||||
"✅ Dataset saved to cache: dataset_cache/sentinel2_timeseries_40scenes.nc\n",
|
||||
" Cache size: 6.40 GB\n",
|
||||
"\n",
|
||||
"✅ OPTIMAL Dataset with native resolution + temporal features created!\n",
|
||||
" ======================================================================\n",
|
||||
" 🎬 Total scenes (time steps): 1\n",
|
||||
" 📊 Total bands/variables: 21\n",
|
||||
" 🖼️ Spatial size: 10980 × 10980 pixels (NATIVE resolution)\n",
|
||||
" 📏 Native resolution: 10m (Sentinel-2 L2A)\n",
|
||||
" ⏰ Temporal range: 2023-12-26 to 2023-12-26\n",
|
||||
"❌ Error: name 'notebook_utils' is not defined\n",
|
||||
"======================================================================\n",
|
||||
"CPU times: user 9min, sys: 4min 16s, total: 13min 16s\n",
|
||||
"Wall time: 16min 42s\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Traceback (most recent call last):\n",
|
||||
" File \"<timed exec>\", line 262, in <module>\n",
|
||||
"NameError: name 'notebook_utils' is not defined\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# 💾 LOAD SENTINEL-2 DATA DIRECTLY FROM S3 COGS USING RASTERIO - WITH TEMPORAL FEATURES\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"LOADING SENTINEL-2 DATA FROM S3 COGs (RASTERIO) - OPTIMAL ACCURACY\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" import rasterio\n",
|
||||
" import xarray as xr\n",
|
||||
" import numpy as np\n",
|
||||
" from scipy import ndimage\n",
|
||||
" \n",
|
||||
" # ===== CHECK IF SHOULD SKIP DOWNLOAD =====\n",
|
||||
" if use_cache and data is not None:\n",
|
||||
" print(f\"\\n✅ Using cached dataset - skipping download!\")\n",
|
||||
" print(f\" Variables: {len(data.data_vars)}\")\n",
|
||||
" print(f\" Shape: {data.dims}\")\n",
|
||||
" display(data)\n",
|
||||
" \n",
|
||||
" else:\n",
|
||||
" # ===== DOWNLOAD FROM S3 =====\n",
|
||||
" print(f\"\\n📥 Downloading from S3...\")\n",
|
||||
" \n",
|
||||
" # Get all scenes from datacube metadata\n",
|
||||
" datasets = list(dc.find_datasets(\n",
|
||||
" product='s2_l2a',\n",
|
||||
" time=date_range\n",
|
||||
" ))\n",
|
||||
" \n",
|
||||
" if not datasets:\n",
|
||||
" raise ValueError(\"No datasets found for date range\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n📦 Found {len(datasets)} available scenes\")\n",
|
||||
" print(f\" Date range: {date_range[0]} to {date_range[1]}\")\n",
|
||||
" \n",
|
||||
" # ===== LOAD ALL SCENES WITH ALL AVAILABLE BANDS (NO MAGNIFICATION) =====\n",
|
||||
" print(f\"\\n[LOADING] Loading ALL {len(datasets)} scenes with ALL available bands...\")\n",
|
||||
" print(f\" (Keeping NATIVE resolution - NO upsampling/magnification)\")\n",
|
||||
" \n",
|
||||
" # num_scenes = len(datasets) # Load ALL scenes\n",
|
||||
" num_scenes = 1 # Load ALL scenes\n",
|
||||
" all_data_dict = {}\n",
|
||||
" failed_scenes = []\n",
|
||||
" scene_dates = []\n",
|
||||
" \n",
|
||||
" # Discover all available bands from first scene\n",
|
||||
" first_scene = datasets[0]\n",
|
||||
" all_available_bands = list(first_scene.measurements.keys())\n",
|
||||
" print(f\" Available bands: {all_available_bands}\")\n",
|
||||
" \n",
|
||||
" for scene_idx in range(num_scenes):\n",
|
||||
" selected = datasets[scene_idx]\n",
|
||||
" scene_label = selected.metadata.label\n",
|
||||
" scene_datetime = selected.time.begin if hasattr(selected.time, 'begin') else selected.time\n",
|
||||
" scene_dates.append(scene_datetime)\n",
|
||||
" \n",
|
||||
" # Print progress every 5 scenes\n",
|
||||
" if scene_idx % 5 == 0 or scene_idx == 0 or scene_idx == num_scenes - 1:\n",
|
||||
" print(f\"\\n [{scene_idx + 1:2d}/{num_scenes}] {scene_label} ({scene_datetime.date()})\")\n",
|
||||
" \n",
|
||||
" # Load ALL available bands from S3 COGs\n",
|
||||
" scene_data_dict = {}\n",
|
||||
" \n",
|
||||
" for band_name in all_available_bands:\n",
|
||||
" if band_name in selected.measurements:\n",
|
||||
" band_path = selected.measurements[band_name]['path']\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" with rasterio.open(band_path) as src:\n",
|
||||
" data_band = src.read(1)\n",
|
||||
" scene_data_dict[band_name] = data_band\n",
|
||||
" except Exception as e:\n",
|
||||
" if scene_idx % 5 == 0:\n",
|
||||
" print(f\" ⚠️ Error loading {band_name}: {str(e)[:30]}\")\n",
|
||||
" failed_scenes.append((scene_idx, scene_label, band_name, str(e)))\n",
|
||||
" \n",
|
||||
" if scene_data_dict:\n",
|
||||
" all_data_dict[scene_idx] = scene_data_dict\n",
|
||||
" if scene_idx % 5 == 0 or scene_idx == num_scenes - 1:\n",
|
||||
" print(f\" ✅ {len(scene_data_dict)} bands loaded\")\n",
|
||||
" else:\n",
|
||||
" failed_scenes.append((scene_idx, scene_label, \"all\", \"No bands loaded\"))\n",
|
||||
" \n",
|
||||
" if not all_data_dict:\n",
|
||||
" raise ValueError(\"Could not load any bands from any scene\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n✅ Successfully loaded {len(all_data_dict)} scenes!\")\n",
|
||||
" if failed_scenes:\n",
|
||||
" print(f\"⚠️ Failed to load {len(failed_scenes)} band instances (will be skipped)\")\n",
|
||||
" \n",
|
||||
" # ===== NORMALIZE RESOLUTION (No upsampling - just match to highest) =====\n",
|
||||
" print(f\"\\n[RESOLUTION NORMALIZATION] Aligning all bands to native resolution (NO magnification)...\")\n",
|
||||
" \n",
|
||||
" # Find max resolution\n",
|
||||
" ref_resolution = None\n",
|
||||
" max_size = 0\n",
|
||||
" max_band = None\n",
|
||||
" \n",
|
||||
" for scene_idx in all_data_dict.keys():\n",
|
||||
" for band_name, data_band in all_data_dict[scene_idx].items():\n",
|
||||
" size = data_band.shape[0]\n",
|
||||
" if size > max_size:\n",
|
||||
" max_size = size\n",
|
||||
" ref_resolution = size\n",
|
||||
" max_band = band_name\n",
|
||||
" \n",
|
||||
" print(f\" Reference resolution: {max_size}×{max_size} pixels (native {max_band})\")\n",
|
||||
" \n",
|
||||
" # Resample all bands to match reference resolution (both up and down)\n",
|
||||
" resampled_count = 0\n",
|
||||
" for scene_idx in all_data_dict.keys():\n",
|
||||
" for band_name in list(all_data_dict[scene_idx].keys()):\n",
|
||||
" band_data_arr = all_data_dict[scene_idx][band_name]\n",
|
||||
" current_size = band_data_arr.shape[0]\n",
|
||||
" \n",
|
||||
" if current_size != ref_resolution:\n",
|
||||
" scale_factor = ref_resolution / current_size\n",
|
||||
" \n",
|
||||
" # Resample to match reference resolution (both up and down)\n",
|
||||
" if band_name == 'scl':\n",
|
||||
" resampled_data = ndimage.zoom(band_data_arr, scale_factor, order=0)\n",
|
||||
" else:\n",
|
||||
" resampled_data = ndimage.zoom(band_data_arr, scale_factor, order=1)\n",
|
||||
" \n",
|
||||
" all_data_dict[scene_idx][band_name] = resampled_data\n",
|
||||
" new_size = resampled_data.shape[0]\n",
|
||||
" if scene_idx == 0: # Print for first scene only to reduce clutter\n",
|
||||
" print(f\" Resampling {band_name}: {current_size}×{current_size} → {new_size}×{new_size}\")\n",
|
||||
" resampled_count += 1\n",
|
||||
" \n",
|
||||
" print(f\"✅ Resolution normalization complete! ({resampled_count} bands resampled)\")\n",
|
||||
" \n",
|
||||
" # ===== CALCULATE SPECTRAL INDICES FOR EACH SCENE =====\n",
|
||||
" print(f\"\\n[SPECTRAL INDICES] Calculating spectral indices for each scene...\")\n",
|
||||
" \n",
|
||||
" indices_count = 0\n",
|
||||
" for scene_idx in all_data_dict.keys():\n",
|
||||
" scene_data = all_data_dict[scene_idx]\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" # NDVI: (NIR - Red) / (NIR + Red)\n",
|
||||
" if 'nir' in scene_data and 'red' in scene_data:\n",
|
||||
" nir = scene_data['nir'].astype(float)\n",
|
||||
" red = scene_data['red'].astype(float)\n",
|
||||
" ndvi = (nir - red) / (nir + red + 1e-8)\n",
|
||||
" scene_data['ndvi'] = ndvi.astype(np.float32)\n",
|
||||
" indices_count += 1\n",
|
||||
" \n",
|
||||
" # NDBI: (SWIR - NIR) / (SWIR + NIR)\n",
|
||||
" if 'b11' in scene_data and 'nir' in scene_data:\n",
|
||||
" swir = scene_data['b11'].astype(float)\n",
|
||||
" nir = scene_data['nir'].astype(float)\n",
|
||||
" ndbi = (swir - nir) / (swir + nir + 1e-8)\n",
|
||||
" scene_data['ndbi'] = ndbi.astype(np.float32)\n",
|
||||
" indices_count += 1\n",
|
||||
" \n",
|
||||
" # NDWI: (NIR - SWIR) / (NIR + SWIR)\n",
|
||||
" if 'nir' in scene_data and 'b11' in scene_data:\n",
|
||||
" nir = scene_data['nir'].astype(float)\n",
|
||||
" swir = scene_data['b11'].astype(float)\n",
|
||||
" ndwi = (nir - swir) / (nir + swir + 1e-8)\n",
|
||||
" scene_data['ndwi'] = ndwi.astype(np.float32)\n",
|
||||
" indices_count += 1\n",
|
||||
" \n",
|
||||
" # EVI: Enhanced Vegetation Index\n",
|
||||
" if 'nir' in scene_data and 'red' in scene_data and 'blue' in scene_data:\n",
|
||||
" nir = scene_data['nir'].astype(float)\n",
|
||||
" red = scene_data['red'].astype(float)\n",
|
||||
" blue = scene_data['blue'].astype(float)\n",
|
||||
" evi = 2.5 * (nir - red) / (nir + 6*red - 7.5*blue + 1)\n",
|
||||
" scene_data['evi'] = evi.astype(np.float32)\n",
|
||||
" indices_count += 1\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" pass\n",
|
||||
" \n",
|
||||
" print(f\"✅ Calculated {indices_count} spectral indices per scene\")\n",
|
||||
" \n",
|
||||
" # ===== STACK SCENES ALONG TIME DIMENSION =====\n",
|
||||
" print(f\"\\n[STACKING] Stacking all {len(all_data_dict)} scenes to create time-series...\")\n",
|
||||
" \n",
|
||||
" data_vars = {}\n",
|
||||
" band_names = list(all_data_dict[0].keys())\n",
|
||||
" \n",
|
||||
" for band_name in band_names:\n",
|
||||
" band_data_list = []\n",
|
||||
" for scene_idx in sorted(all_data_dict.keys()):\n",
|
||||
" if band_name in all_data_dict[scene_idx]:\n",
|
||||
" band_data_list.append(all_data_dict[scene_idx][band_name])\n",
|
||||
" \n",
|
||||
" if band_data_list:\n",
|
||||
" stacked = np.stack(band_data_list, axis=0)\n",
|
||||
" data_vars[band_name] = (['time', 'y', 'x'], stacked)\n",
|
||||
" \n",
|
||||
" # Create xarray Dataset with time dimension\n",
|
||||
" first_band_data = list(all_data_dict[0].values())[0]\n",
|
||||
" y_size, x_size = first_band_data.shape\n",
|
||||
" \n",
|
||||
" data = xr.Dataset(\n",
|
||||
" data_vars,\n",
|
||||
" coords={\n",
|
||||
" 'time': np.arange(len(all_data_dict)),\n",
|
||||
" 'x': np.arange(x_size),\n",
|
||||
" 'y': np.arange(y_size)\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # ===== CALCULATE TEMPORAL FEATURES FOR ACCURACY =====\n",
|
||||
" print(f\"\\n[TEMPORAL FEATURES] Computing temporal features from time-series...\")\n",
|
||||
" \n",
|
||||
" temporal_features_added = 0\n",
|
||||
" \n",
|
||||
" # For NDVI: temporal statistics\n",
|
||||
" if 'ndvi' in data.data_vars:\n",
|
||||
" ndvi_ts = data['ndvi']\n",
|
||||
" \n",
|
||||
" # Min NDVI (vegetation stress indicator)\n",
|
||||
" data['ndvi_min'] = ndvi_ts.min(dim='time')\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" # Max NDVI (peak vegetation)\n",
|
||||
" data['ndvi_max'] = ndvi_ts.max(dim='time')\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" # Mean NDVI\n",
|
||||
" data['ndvi_mean'] = ndvi_ts.mean(dim='time')\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" # NDVI range (variability)\n",
|
||||
" data['ndvi_range'] = data['ndvi_max'] - data['ndvi_min']\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" # NDVI std (temporal consistency)\n",
|
||||
" data['ndvi_std'] = ndvi_ts.std(dim='time')\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" # For all indices: mean values (aggregate features)\n",
|
||||
" for band_name in ['ndbi', 'ndwi', 'evi']:\n",
|
||||
" if band_name in data.data_vars:\n",
|
||||
" band_ts = data[band_name]\n",
|
||||
" data[f'{band_name}_mean'] = band_ts.mean(dim='time')\n",
|
||||
" temporal_features_added += 1\n",
|
||||
" \n",
|
||||
" print(f\"✅ Added {temporal_features_added} temporal/aggregate features\")\n",
|
||||
" \n",
|
||||
" # ===== SAVE TO CACHE =====\n",
|
||||
" print(f\"\\n[CACHE] Saving dataset to cache...\")\n",
|
||||
" try:\n",
|
||||
" data.to_netcdf(cache_file, engine='netcdf4')\n",
|
||||
" cache_size = os.path.getsize(cache_file) / (1024**3)\n",
|
||||
" print(f\"✅ Dataset saved to cache: {cache_file}\")\n",
|
||||
" print(f\" Cache size: {cache_size:.2f} GB\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"⚠️ Error saving cache: {e}\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n✅ OPTIMAL Dataset with native resolution + temporal features created!\")\n",
|
||||
" print(f\" {'='*70}\")\n",
|
||||
" print(f\" 🎬 Total scenes (time steps): {len(all_data_dict)}\")\n",
|
||||
" print(f\" 📊 Total bands/variables: {len(data.data_vars)}\")\n",
|
||||
" print(f\" 🖼️ Spatial size: {x_size} × {y_size} pixels (NATIVE resolution)\")\n",
|
||||
" print(f\" 📏 Native resolution: 10m (Sentinel-2 L2A)\")\n",
|
||||
" print(f\" ⏰ Temporal range: {scene_dates[0].date()} to {scene_dates[-1].date()}\")\n",
|
||||
" print(f\" 💾 Total dataset size: {notebook_utils.xarray_object_size(data)}\")\n",
|
||||
" print(f\" 💿 Cached at: {cache_file}\")\n",
|
||||
" print(f\" {'='*70}\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n Dataset dimensions:\")\n",
|
||||
" for dim, size in data.dims.items():\n",
|
||||
" print(f\" {dim}: {size}\")\n",
|
||||
" \n",
|
||||
" print(f\"\\n Variables ({len(data.data_vars)}):\")\n",
|
||||
" spatial_vars = []\n",
|
||||
" temporal_vars = []\n",
|
||||
" for var_name in sorted(data.data_vars):\n",
|
||||
" if len(data[var_name].shape) == 3:\n",
|
||||
" spatial_vars.append(f\"{var_name} {data[var_name].shape}\")\n",
|
||||
" else:\n",
|
||||
" temporal_vars.append(f\"{var_name} {data[var_name].shape}\")\n",
|
||||
" \n",
|
||||
" print(f\" Spatial time-series ({len(spatial_vars)}):\")\n",
|
||||
" for v in spatial_vars:\n",
|
||||
" print(f\" - {v}\")\n",
|
||||
" print(f\" Temporal aggregates ({len(temporal_vars)}):\")\n",
|
||||
" for v in temporal_vars:\n",
|
||||
" print(f\" - {v}\")\n",
|
||||
" \n",
|
||||
" print(f\" {'='*70}\")\n",
|
||||
" \n",
|
||||
" display(data)\n",
|
||||
" \n",
|
||||
" # ===== EXTRACT NDVI FOR TRAINING =====\n",
|
||||
" print(f\"\\n[NDVI EXTRACTION] Extracting NDVI for model training...\")\n",
|
||||
" if 'ndvi_mean' in data.data_vars:\n",
|
||||
" # Use mean NDVI across time\n",
|
||||
" ndvi = data['ndvi_mean']\n",
|
||||
" print(f\"✅ NDVI extracted (mean across time)\")\n",
|
||||
" print(f\" Shape: {ndvi.shape}\")\n",
|
||||
" elif 'ndvi' in data.data_vars:\n",
|
||||
" # Use first time step if mean not available\n",
|
||||
" ndvi = data['ndvi'].isel(time=0)\n",
|
||||
" print(f\"✅ NDVI extracted (first time step)\")\n",
|
||||
" print(f\" Shape: {ndvi.shape}\")\n",
|
||||
" else:\n",
|
||||
" print(f\"❌ NDVI not found in dataset\")\n",
|
||||
" ndvi = None\n",
|
||||
" \n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"❌ Error: {e}\")\n",
|
||||
" import traceback\n",
|
||||
" traceback.print_exc()\n",
|
||||
" data = None\n",
|
||||
" ndvi = None\n",
|
||||
"\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "d2585562-88aa-4c7d-bf70-1f6affcf65d4",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"TRAINING DATA SETUP\n",
|
||||
"======================================================================\n",
|
||||
"\n",
|
||||
"[1] Loading training data: train/ST_training data_updated_1130points_new.shp\n",
|
||||
" ❌ Error: name 'load_train_data' is not defined\n",
|
||||
"\n",
|
||||
"[2] Label mapping:\n",
|
||||
" 0: Lua tom\n",
|
||||
" 1: Lua\n",
|
||||
" 2: CHN\n",
|
||||
" 3: CLN\n",
|
||||
" 4: TS\n",
|
||||
" 5: Song\n",
|
||||
" 6: Dat xay dung\n",
|
||||
" 7: Rung\n",
|
||||
"\n",
|
||||
"======================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 🎯 LOAD TRAINING DATA & EXTRACT FEATURES\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"TRAINING DATA SETUP\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"# Load training points\n",
|
||||
"train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
|
||||
"print(f\"\\n[1] Loading training data: {train_path}\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" train = load_train_data(train_path)\n",
|
||||
" print(f\" ✅ Loaded {len(train)} training points\")\n",
|
||||
" print(f\" Columns: {list(train.columns)}\")\n",
|
||||
" train.head()\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\" ❌ Error: {e}\")\n",
|
||||
" train = None\n",
|
||||
"\n",
|
||||
"# Label mapping\n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": \"0\",\n",
|
||||
" \"Lua\": \"1\",\n",
|
||||
" \"CHN\": \"2\",\n",
|
||||
" \"CLN\": \"3\",\n",
|
||||
" \"TS\": \"4\",\n",
|
||||
" \"Song\": \"5\",\n",
|
||||
" \"Dat xay dung\": \"6\",\n",
|
||||
" \"Rung\": \"7\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"\\n[2] Label mapping:\")\n",
|
||||
"for label, code in label_mapping.items():\n",
|
||||
" print(f\" {code}: {label}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "2e955884-d4af-422d-a8e6-d436199540e0",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"MODEL TRAINING\n",
|
||||
"======================================================================\n",
|
||||
"❌ Missing training data or NDVI\n",
|
||||
"======================================================================\n",
|
||||
"CPU times: user 700 μs, sys: 0 ns, total: 700 μs\n",
|
||||
"Wall time: 638 μs\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# 🤖 RANDOM FOREST MODEL TRAINING\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"MODEL TRAINING\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"if train is not None and ndvi is not None:\n",
|
||||
" print(\"\\n[1] Extracting features from NDVI...\")\n",
|
||||
" try:\n",
|
||||
" # Extract NDVI values at training point locations\n",
|
||||
" X = []\n",
|
||||
" y = []\n",
|
||||
" \n",
|
||||
" for idx, point in train.iterrows():\n",
|
||||
" try:\n",
|
||||
" # Get NDVI value at point location (nearest neighbor)\n",
|
||||
" ndvi_val = float(ndvi.sel(x=point.geometry.x, y=point.geometry.y, method='nearest').values)\n",
|
||||
" label = label_mapping[point.Hientrang]\n",
|
||||
" \n",
|
||||
" X.append([ndvi_val])\n",
|
||||
" y.append(int(label))\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" ⚠️ Point {idx}: {e}\")\n",
|
||||
" \n",
|
||||
" if len(X) > 0:\n",
|
||||
" X = np.array(X)\n",
|
||||
" y = np.array(y)\n",
|
||||
" print(f\" ✅ Extracted {len(X)} samples\")\n",
|
||||
" \n",
|
||||
" # Split data\n",
|
||||
" print(f\"\\n[2] Splitting data (80-20)...\")\n",
|
||||
" from sklearn.model_selection import train_test_split\n",
|
||||
" X_train, X_test, y_train, y_test = train_test_split(\n",
|
||||
" X, y, test_size=0.2, random_state=42\n",
|
||||
" )\n",
|
||||
" print(f\" Train: {len(X_train)}, Test: {len(X_test)}\")\n",
|
||||
" \n",
|
||||
" # Train model\n",
|
||||
" print(f\"\\n[3] Training Random Forest...\")\n",
|
||||
" from sklearn.ensemble import RandomForestClassifier\n",
|
||||
" from sklearn.metrics import accuracy_score\n",
|
||||
" \n",
|
||||
" model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)\n",
|
||||
" model.fit(X_train, y_train)\n",
|
||||
" \n",
|
||||
" # Evaluate\n",
|
||||
" y_pred = model.predict(X_test)\n",
|
||||
" accuracy = accuracy_score(y_test, y_pred)\n",
|
||||
" print(f\" ✅ Model trained!\")\n",
|
||||
" print(f\" Accuracy: {accuracy*100:.2f}%\")\n",
|
||||
" \n",
|
||||
" else:\n",
|
||||
" print(f\" ❌ No samples extracted\")\n",
|
||||
" model = None\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" ❌ Error: {e}\")\n",
|
||||
" import traceback\n",
|
||||
" traceback.print_exc()\n",
|
||||
" model = None\n",
|
||||
"else:\n",
|
||||
" print(\"❌ Missing training data or NDVI\")\n",
|
||||
" model = None\n",
|
||||
"\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "f1a14379-ed6e-4897-9ca4-2669743fab40",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"MODEL SAVING\n",
|
||||
"======================================================================\n",
|
||||
"❌ No model to save\n",
|
||||
"======================================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 💾 SAVE MODEL\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"MODEL SAVING\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"if model is not None:\n",
|
||||
" print(\"\\n🔄 Saving trained model...\")\n",
|
||||
" try:\n",
|
||||
" save_model(\"model_rasterio.joblib\", model)\n",
|
||||
" print(\"✅ Model saved to model_train/model_rasterio.joblib\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"❌ Error saving model: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"❌ No model to save\")\n",
|
||||
"\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "33dd516d-9824-499e-96b9-5cd9224c194c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"======================================================================\n",
|
||||
"CLEANUP\n",
|
||||
"======================================================================\n",
|
||||
"\n",
|
||||
"🔄 Closing Dask client and cluster...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Cleanup complete\n",
|
||||
"\n",
|
||||
"======================================================================\n",
|
||||
"✅ PIPELINE COMPLETE\n",
|
||||
"======================================================================\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
|
||||
"\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
|
||||
"\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
|
||||
"\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# 🛑 CLEANUP\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"print(\"CLEANUP\")\n",
|
||||
"print(\"=\"*70)\n",
|
||||
"\n",
|
||||
"print(\"\\n🔄 Closing Dask client and cluster...\")\n",
|
||||
"try:\n",
|
||||
" client.close()\n",
|
||||
" cluster.close()\n",
|
||||
" print(\"✅ Cleanup complete\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"⚠️ Error during cleanup: {e}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\"*70)\n",
|
||||
"print(\"✅ PIPELINE COMPLETE\")\n",
|
||||
"print(\"=\"*70)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "env_01",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.18"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"cells": [],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
+917
@@ -0,0 +1,917 @@
|
||||
"""
|
||||
API Server for Land Classification Model Training
|
||||
Cho phép chọn dữ liệu và cấu hình training qua giao diện web
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, BackgroundTasks, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import HTMLResponse, FileResponse
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional, List
|
||||
import uvicorn
|
||||
import joblib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# Import report generator
|
||||
from report_generator import generate_training_report, generate_prediction_report
|
||||
|
||||
app = FastAPI(title="Land Classification Training API", version="1.0.0")
|
||||
|
||||
# Enable CORS
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Global training status``
|
||||
training_status = {
|
||||
"is_training": False,
|
||||
"progress": "",
|
||||
"error": None,
|
||||
"result": None,
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"cancel_requested": False
|
||||
}
|
||||
|
||||
# Global prediction status
|
||||
prediction_status = {
|
||||
"is_predicting": False,
|
||||
"progress": "",
|
||||
"error": None,
|
||||
"result": None,
|
||||
"output_file": None,
|
||||
"start_time": None,
|
||||
"end_time": None
|
||||
}
|
||||
|
||||
|
||||
class TrainingConfig(BaseModel):
|
||||
"""Cấu hình training"""
|
||||
# Khu vực (bbox)
|
||||
min_lon: float = 105.6
|
||||
min_lat: float = 9.3
|
||||
max_lon: float = 106.2
|
||||
max_lat: float = 9.8
|
||||
|
||||
# Thời gian
|
||||
start_date: str = "2023-03-01"
|
||||
end_date: str = "2023-05-31"
|
||||
|
||||
# Dữ liệu
|
||||
max_scenes: int = 12
|
||||
cloud_cover: int = 30
|
||||
resolution: int = 20 # 10m hoặc 20m
|
||||
|
||||
# Model parameters
|
||||
model_type: str = "xgboost" # xgboost, random_forest, decision_tree, svm, cnn
|
||||
n_estimators: int = 100
|
||||
max_depth: int = 20
|
||||
learning_rate: float = 0.1
|
||||
use_gpu: bool = True
|
||||
|
||||
# Train/test split
|
||||
test_size: float = 0.2 # Tỷ lệ dữ liệu dùng làm test (0-1)
|
||||
|
||||
# Cache
|
||||
use_cache: bool = True # Cache dataset để test nhanh hơn
|
||||
|
||||
# Training data
|
||||
training_shapefile: str = "train/ST_training data_updated_1130points_new.shp"
|
||||
|
||||
|
||||
class PredictionConfig(BaseModel):
|
||||
"""Cấu hình dự đoán"""
|
||||
# Model to use
|
||||
model_filename: str
|
||||
|
||||
# Khu vực (bbox)
|
||||
min_lon: float = 105.6
|
||||
min_lat: float = 9.3
|
||||
max_lon: float = 106.2
|
||||
max_lat: float = 9.8
|
||||
|
||||
# Thời gian
|
||||
start_date: str = "2023-03-01"
|
||||
end_date: str = "2023-05-31"
|
||||
|
||||
# Dữ liệu
|
||||
max_scenes: int = 12
|
||||
cloud_cover: int = 30
|
||||
resolution: int = 20
|
||||
|
||||
|
||||
class TrainingStatus(BaseModel):
|
||||
"""Trạng thái training"""
|
||||
is_training: bool
|
||||
progress: str
|
||||
error: Optional[str]
|
||||
result: Optional[dict]
|
||||
start_time: Optional[str]
|
||||
end_time: Optional[str]
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def root():
|
||||
"""Serve giao diện web"""
|
||||
html_file = Path(__file__).parent / "training_interface.html"
|
||||
if html_file.exists():
|
||||
return FileResponse(html_file)
|
||||
else:
|
||||
return HTMLResponse("""
|
||||
<html>
|
||||
<head><title>Training Interface</title></head>
|
||||
<body>
|
||||
<h1>Land Classification Training API</h1>
|
||||
<p>API Documentation: <a href="/docs">/docs</a></p>
|
||||
<p>Training Interface: Tạo file training_interface.html</p>
|
||||
</body>
|
||||
</html>
|
||||
""")
|
||||
|
||||
|
||||
@app.get("/api/config/presets")
|
||||
async def get_presets():
|
||||
"""Lấy các preset cấu hình sẵn"""
|
||||
return {
|
||||
"presets": [
|
||||
{
|
||||
"name": "PC - Nhỏ (3 tháng, 20m, 12 scenes)",
|
||||
"config": {
|
||||
"min_lon": 105.6, "min_lat": 9.3, "max_lon": 106.2, "max_lat": 9.8,
|
||||
"start_date": "2023-03-01", "end_date": "2023-05-31",
|
||||
"max_scenes": 12, "cloud_cover": 30, "resolution": 20,
|
||||
"test_size": 0.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Server - Trung bình (6 tháng, 10m, 30 scenes)",
|
||||
"config": {
|
||||
"min_lon": 105.5, "min_lat": 9.2, "max_lon": 106.4, "max_lat": 10.0,
|
||||
"start_date": "2023-01-01", "end_date": "2023-06-30",
|
||||
"max_scenes": 30, "cloud_cover": 30, "resolution": 10,
|
||||
"test_size": 0.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Full - Lớn (1 năm, 10m, 60 scenes)",
|
||||
"config": {
|
||||
"min_lon": 105.5, "min_lat": 9.2, "max_lon": 106.4, "max_lat": 10.0,
|
||||
"start_date": "2022-09-01", "end_date": "2023-10-01",
|
||||
"max_scenes": 60, "cloud_cover": 50, "resolution": 10,
|
||||
"test_size": 0.2
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/training/status", response_model=TrainingStatus)
|
||||
async def get_training_status():
|
||||
"""Kiểm tra trạng thái training"""
|
||||
return training_status
|
||||
|
||||
|
||||
@app.post("/api/training/start")
|
||||
async def start_training(config: TrainingConfig, background_tasks: BackgroundTasks):
|
||||
"""Bắt đầu training với config đã chọn"""
|
||||
global training_status
|
||||
|
||||
if training_status["is_training"]:
|
||||
raise HTTPException(status_code=400, detail="Training đang chạy, vui lòng đợi")
|
||||
|
||||
# Reset status
|
||||
training_status = {
|
||||
"is_training": True,
|
||||
"progress": "Đang khởi tạo...",
|
||||
"error": None,
|
||||
"result": None,
|
||||
"start_time": datetime.now().isoformat(),
|
||||
"end_time": None
|
||||
}
|
||||
|
||||
# Run training in background
|
||||
background_tasks.add_task(run_training, config)
|
||||
|
||||
return {"message": "Training đã bắt đầu", "status": training_status}
|
||||
|
||||
|
||||
@app.post("/api/training/stop")
|
||||
async def stop_training():
|
||||
"""Dừng training (nếu đang chạy)"""
|
||||
global training_status
|
||||
|
||||
if not training_status["is_training"]:
|
||||
return {"message": "Không có training nào đang chạy"}
|
||||
|
||||
# Set cancel flag - the training will check this and stop
|
||||
training_status["cancel_requested"] = True
|
||||
training_status["progress"] = "Đang hủy training..."
|
||||
|
||||
return {"message": "Đang dừng training..."}
|
||||
|
||||
|
||||
@app.post("/api/cache/clear")
|
||||
async def clear_cache():
|
||||
"""Xóa cache dataset"""
|
||||
import shutil
|
||||
cache_dir = Path("dataset_cache")
|
||||
|
||||
if not cache_dir.exists():
|
||||
return {"message": "Không có cache để xóa", "deleted": 0}
|
||||
|
||||
# Count files
|
||||
cache_files = list(cache_dir.glob("*.joblib"))
|
||||
count = len(cache_files)
|
||||
|
||||
# Delete all cache files
|
||||
for cache_file in cache_files:
|
||||
try:
|
||||
cache_file.unlink()
|
||||
except:
|
||||
pass
|
||||
|
||||
return {"message": f"Đã xóa {count} file cache", "deleted": count}
|
||||
|
||||
|
||||
@app.get("/api/cache/info")
|
||||
async def get_cache_info():
|
||||
"""Lấy thông tin về cache với metadata đầy đủ"""
|
||||
cache_dir = Path("dataset_cache")
|
||||
|
||||
if not cache_dir.exists():
|
||||
return {"exists": False, "files": [], "total_size_mb": 0}
|
||||
|
||||
cache_files = []
|
||||
total_size = 0
|
||||
|
||||
for cache_file in cache_dir.glob("*.joblib"):
|
||||
size = cache_file.stat().st_size
|
||||
total_size += size
|
||||
|
||||
# Try to load metadata from cache
|
||||
metadata = {}
|
||||
try:
|
||||
cached_data = joblib.load(cache_file)
|
||||
if isinstance(cached_data, dict):
|
||||
metadata = {
|
||||
"bbox": cached_data.get("bbox", []),
|
||||
"time_range": cached_data.get("time_range", ""),
|
||||
"resolution": cached_data.get("resolution", 20),
|
||||
"n_samples": len(cached_data.get("features", [])),
|
||||
"created": cached_data.get("timestamp", "")
|
||||
}
|
||||
# Parse time_range to get start/end dates
|
||||
if metadata["time_range"]:
|
||||
time_parts = metadata["time_range"].split("/")
|
||||
if len(time_parts) == 2:
|
||||
metadata["start_date"] = time_parts[0]
|
||||
metadata["end_date"] = time_parts[1]
|
||||
# Parse bbox to get min/max lon/lat
|
||||
if metadata["bbox"] and len(metadata["bbox"]) == 4:
|
||||
metadata["min_lon"] = metadata["bbox"][0]
|
||||
metadata["min_lat"] = metadata["bbox"][1]
|
||||
metadata["max_lon"] = metadata["bbox"][2]
|
||||
metadata["max_lat"] = metadata["bbox"][3]
|
||||
except Exception as e:
|
||||
print(f"Error loading cache metadata: {e}")
|
||||
|
||||
cache_files.append({
|
||||
"filename": cache_file.name,
|
||||
"size_mb": round(size / 1024 / 1024, 2),
|
||||
"modified": datetime.fromtimestamp(cache_file.stat().st_mtime).isoformat(),
|
||||
"metadata": metadata
|
||||
})
|
||||
|
||||
# Sort by modified time (newest first)
|
||||
cache_files.sort(key=lambda x: x["modified"], reverse=True)
|
||||
|
||||
return {
|
||||
"exists": True,
|
||||
"files": cache_files,
|
||||
"count": len(cache_files),
|
||||
"total_size_mb": round(total_size / 1024 / 1024, 2)
|
||||
}
|
||||
|
||||
|
||||
|
||||
@app.get("/api/models/list")
|
||||
async def list_models():
|
||||
"""Liệt kê các model đã train"""
|
||||
model_dir = Path("model_train")
|
||||
if not model_dir.exists():
|
||||
return {"models": []}
|
||||
|
||||
models = []
|
||||
for model_file in model_dir.glob("*.joblib"):
|
||||
info_file = model_file.with_suffix('.json')
|
||||
info = {}
|
||||
if info_file.exists():
|
||||
with open(info_file) as f:
|
||||
info = json.load(f)
|
||||
|
||||
models.append({
|
||||
"filename": model_file.name,
|
||||
"created": datetime.fromtimestamp(model_file.stat().st_mtime).isoformat(),
|
||||
"size_mb": round(model_file.stat().st_size / 1024 / 1024, 2),
|
||||
"info": info
|
||||
})
|
||||
|
||||
# Sort by creation time (newest first)
|
||||
models.sort(key=lambda x: x["created"], reverse=True)
|
||||
return {"models": models}
|
||||
|
||||
|
||||
# ============ REPORTS API ============
|
||||
|
||||
@app.get("/api/reports/list")
|
||||
async def list_reports():
|
||||
"""Liệt kê các báo cáo đã tạo"""
|
||||
reports_dir = Path("reports")
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
|
||||
reports = []
|
||||
for report_file in reports_dir.glob("*.html"):
|
||||
# Determine report type from filename
|
||||
if "training" in report_file.name:
|
||||
report_type = "training"
|
||||
elif "prediction" in report_file.name:
|
||||
report_type = "prediction"
|
||||
else:
|
||||
report_type = "unknown"
|
||||
|
||||
reports.append({
|
||||
"filename": report_file.name,
|
||||
"type": report_type,
|
||||
"created": datetime.fromtimestamp(report_file.stat().st_mtime).isoformat(),
|
||||
"size_kb": round(report_file.stat().st_size / 1024, 2),
|
||||
"view_url": f"/api/reports/view/{report_file.name}",
|
||||
"download_url": f"/api/reports/download/{report_file.name}"
|
||||
})
|
||||
|
||||
# Sort by creation time (newest first)
|
||||
reports.sort(key=lambda x: x["created"], reverse=True)
|
||||
return {"reports": reports, "count": len(reports)}
|
||||
|
||||
|
||||
@app.get("/api/reports/view/{filename}", response_class=HTMLResponse)
|
||||
async def view_report(filename: str):
|
||||
"""Xem báo cáo HTML trực tiếp"""
|
||||
reports_dir = Path("reports")
|
||||
file_path = reports_dir / filename
|
||||
|
||||
# Security check
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
|
||||
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
html_content = f.read()
|
||||
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
|
||||
@app.get("/api/reports/download/{filename}")
|
||||
async def download_report(filename: str):
|
||||
"""Download báo cáo HTML"""
|
||||
reports_dir = Path("reports")
|
||||
file_path = reports_dir / filename
|
||||
|
||||
# Security check
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
|
||||
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=filename,
|
||||
media_type="text/html",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={filename}"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.delete("/api/reports/delete/{filename}")
|
||||
async def delete_report(filename: str):
|
||||
"""Xóa một báo cáo"""
|
||||
reports_dir = Path("reports")
|
||||
file_path = reports_dir / filename
|
||||
|
||||
# Security check
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"Report không tồn tại: {filename}")
|
||||
|
||||
try:
|
||||
file_path.unlink()
|
||||
return {"message": f"Đã xóa báo cáo: {filename}", "success": True}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Không thể xóa: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/api/prediction/start")
|
||||
async def start_prediction(config: PredictionConfig, background_tasks: BackgroundTasks):
|
||||
"""Bắt đầu dự đoán"""
|
||||
global prediction_status
|
||||
|
||||
if prediction_status["is_predicting"]:
|
||||
raise HTTPException(status_code=400, detail="Đang có dự đoán khác đang chạy")
|
||||
|
||||
# Reset status
|
||||
prediction_status = {
|
||||
"is_predicting": True,
|
||||
"progress": "Đang khởi động...",
|
||||
"error": None,
|
||||
"result": None,
|
||||
"output_file": None,
|
||||
"start_time": datetime.now().isoformat(),
|
||||
"end_time": None
|
||||
}
|
||||
|
||||
# Run prediction in background
|
||||
background_tasks.add_task(run_prediction, config)
|
||||
|
||||
return {"message": "Đã bắt đầu dự đoán", "status": prediction_status}
|
||||
|
||||
|
||||
@app.get("/api/prediction/status")
|
||||
async def get_prediction_status():
|
||||
"""Kiểm tra trạng thái dự đoán"""
|
||||
return prediction_status
|
||||
|
||||
|
||||
async def run_training(config: TrainingConfig):
|
||||
"""Chạy training process"""
|
||||
global training_status
|
||||
|
||||
try:
|
||||
training_status["cancel_requested"] = False
|
||||
training_status["progress"] = "Đang import thư viện..."
|
||||
|
||||
# Import training module
|
||||
from train_module import train_model
|
||||
|
||||
training_status["progress"] = "Đang load dữ liệu Sentinel-2..."
|
||||
|
||||
# Function to check if training should be cancelled
|
||||
def should_cancel():
|
||||
return training_status.get("cancel_requested", False)
|
||||
|
||||
# Run training
|
||||
result = train_model(
|
||||
bbox=[config.min_lon, config.min_lat, config.max_lon, config.max_lat],
|
||||
time_range=f"{config.start_date}/{config.end_date}",
|
||||
max_scenes=config.max_scenes,
|
||||
cloud_cover=config.cloud_cover,
|
||||
resolution=config.resolution,
|
||||
training_shapefile=config.training_shapefile,
|
||||
model_type=config.model_type,
|
||||
n_estimators=config.n_estimators,
|
||||
max_depth=config.max_depth,
|
||||
learning_rate=config.learning_rate,
|
||||
use_gpu=config.use_gpu,
|
||||
use_cache=config.use_cache,
|
||||
test_size=config.test_size,
|
||||
status_callback=lambda msg: update_progress(msg),
|
||||
cancel_check=should_cancel
|
||||
)
|
||||
|
||||
if training_status.get("cancel_requested", False):
|
||||
training_status["is_training"] = False
|
||||
training_status["progress"] = "Đã hủy training"
|
||||
training_status["error"] = "Training cancelled by user"
|
||||
else:
|
||||
training_status["is_training"] = False
|
||||
training_status["progress"] = "Hoàn thành! Đang tạo báo cáo..."
|
||||
training_status["result"] = result
|
||||
|
||||
# Auto generate report
|
||||
if result.get("success", False):
|
||||
try:
|
||||
report_path, _ = generate_training_report(result)
|
||||
training_status["result"]["report_path"] = report_path
|
||||
training_status["result"]["report_filename"] = Path(report_path).name
|
||||
training_status["progress"] = "Hoàn thành! Báo cáo đã được tạo."
|
||||
print(f"[REPORT] Generated: {report_path}")
|
||||
except Exception as e:
|
||||
print(f"[REPORT ERROR] Failed to generate report: {e}")
|
||||
training_status["progress"] = "Hoàn thành! (Không thể tạo báo cáo)"
|
||||
|
||||
training_status["end_time"] = datetime.now().isoformat()
|
||||
|
||||
except Exception as e:
|
||||
training_status["is_training"] = False
|
||||
training_status["error"] = str(e)
|
||||
training_status["progress"] = f"Lỗi: {str(e)}"
|
||||
training_status["end_time"] = datetime.now().isoformat()
|
||||
import traceback
|
||||
print(traceback.format_exc())
|
||||
|
||||
|
||||
def update_progress(message: str):
|
||||
"""Cập nhật progress message"""
|
||||
global training_status
|
||||
training_status["progress"] = message
|
||||
print(f"[PROGRESS] {message}")
|
||||
|
||||
|
||||
def update_prediction_progress(message: str):
|
||||
"""Cập nhật prediction progress message"""
|
||||
global prediction_status
|
||||
prediction_status["progress"] = message
|
||||
print(f"[PREDICTION PROGRESS] {message}")
|
||||
|
||||
|
||||
async def run_prediction(config: PredictionConfig):
|
||||
"""Chạy prediction process - Áp dụng phương pháp từ 02.predict_ODC.ipynb"""
|
||||
global prediction_status
|
||||
|
||||
try:
|
||||
prediction_status["progress"] = "Đang import thư viện..."
|
||||
|
||||
# Import required libraries
|
||||
import xarray as xr
|
||||
import numpy as np
|
||||
from datetime import datetime as dt
|
||||
import rioxarray
|
||||
import dask.array as da
|
||||
|
||||
prediction_status["progress"] = "Đang load model..."
|
||||
|
||||
# Load model
|
||||
model_path = Path("model_train") / config.model_filename
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"Model không tồn tại: {config.model_filename}")
|
||||
|
||||
model_data = joblib.load(model_path)
|
||||
|
||||
# Extract model from dict (models are saved as {'model': xgb_model, 'label_encoder': encoder})
|
||||
if isinstance(model_data, dict):
|
||||
model = model_data.get('model')
|
||||
label_encoder = model_data.get('label_encoder')
|
||||
else:
|
||||
model = model_data
|
||||
label_encoder = None
|
||||
|
||||
# Check if it's a CNN model (PyTorch)
|
||||
is_cnn_model = hasattr(model, '__class__') and 'CNN' in model.__class__.__name__
|
||||
if is_cnn_model:
|
||||
prediction_status["progress"] = "Phát hiện PyTorch CNN model..."
|
||||
# Import PyTorch if needed
|
||||
try:
|
||||
import torch
|
||||
except ImportError:
|
||||
raise ImportError("PyTorch is required for CNN prediction. Install: pip install torch")
|
||||
|
||||
prediction_status["progress"] = "Đang kết nối Microsoft Planetary Computer..."
|
||||
|
||||
# Import and use Microsoft Planetary Computer STAC API
|
||||
import pystac_client
|
||||
import planetary_computer
|
||||
from odc.stac import load
|
||||
|
||||
catalog = pystac_client.Client.open(
|
||||
"https://planetarycomputer.microsoft.com/api/stac/v1",
|
||||
modifier=planetary_computer.sign_inplace,
|
||||
)
|
||||
|
||||
bbox = [config.min_lon, config.min_lat, config.max_lon, config.max_lat]
|
||||
time_range = f"{config.start_date}/{config.end_date}"
|
||||
|
||||
# ============ BƯỚC 1: TẢI DỮ LIỆU SENTINEL-2 ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-2..."
|
||||
|
||||
# Search Sentinel-2 data
|
||||
s2_search = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": config.cloud_cover}}
|
||||
)
|
||||
|
||||
s2_items = list(s2_search.items())
|
||||
if not s2_items:
|
||||
raise ValueError("Không tìm thấy dữ liệu Sentinel-2 cho khu vực và thời gian này")
|
||||
|
||||
s2_items = s2_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s2_items)} scenes Sentinel-2..."
|
||||
|
||||
# Load Sentinel-2 data
|
||||
s2_data = load(
|
||||
s2_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="solar_day",
|
||||
resolution=config.resolution
|
||||
)
|
||||
|
||||
# ============ BƯỚC 2: TÍNH NDVI VÀ XỬ LÝ MÂY ============
|
||||
prediction_status["progress"] = "Đang tính toán NDVI và xử lý mây..."
|
||||
|
||||
# Calculate NDVI using Sentinel-2 band names (B08 = NIR, B04 = Red)
|
||||
nir = s2_data["B08"].astype('float32')
|
||||
red = s2_data["B04"].astype('float32')
|
||||
ndvi = (nir - red) / (nir + red + 1e-8)
|
||||
|
||||
# Mask clouds using SCL band if available
|
||||
if "SCL" in s2_data:
|
||||
scl = s2_data["SCL"]
|
||||
# SCL values: 4=vegetation, 5=bare soil, 6=water - these are clear
|
||||
# 3=cloud shadow, 8=cloud medium, 9=cloud high, 10=cirrus - mask these
|
||||
cloud_mask = (scl == 3) | (scl == 8) | (scl == 9) | (scl == 10)
|
||||
ndvi = ndvi.where(~cloud_mask)
|
||||
|
||||
# ============ BƯỚC 3: ĐIỀN GIÁ TRỊ NAN (FILL NAN) ============
|
||||
prediction_status["progress"] = "Đang điền giá trị bị che mây..."
|
||||
|
||||
# Fill NaN using forward fill and backward fill
|
||||
ndvi_filled = ndvi.ffill(dim='time').bfill(dim='time')
|
||||
|
||||
# Resample to monthly average
|
||||
prediction_status["progress"] = "Đang tính trung bình NDVI theo tháng..."
|
||||
ndvi_monthly = ndvi_filled.resample(time="1ME").mean()
|
||||
|
||||
# Compute NDVI (convert from dask to numpy)
|
||||
ndvi_monthly = ndvi_monthly.compute()
|
||||
|
||||
# ============ BƯỚC 4: TẢI DỮ LIỆU SENTINEL-1 (VH, VV) ============
|
||||
prediction_status["progress"] = "Đang tải dữ liệu Sentinel-1 (Radar)..."
|
||||
|
||||
# Search Sentinel-1 data
|
||||
s1_search = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
|
||||
s1_items = list(s1_search.items())
|
||||
|
||||
if s1_items:
|
||||
s1_items = s1_items[:config.max_scenes]
|
||||
prediction_status["progress"] = f"Đang xử lý {len(s1_items)} scenes Sentinel-1..."
|
||||
|
||||
# Load Sentinel-1 data (without like= to avoid conflict with bbox/resolution)
|
||||
s1_data = load(
|
||||
s1_items,
|
||||
bbox=bbox,
|
||||
chunks={"time": 1, "x": 2048, "y": 2048},
|
||||
groupby="sat:absolute_orbit",
|
||||
resolution=config.resolution
|
||||
)
|
||||
|
||||
# Extract VH and VV bands
|
||||
if "vh" in s1_data and "vv" in s1_data:
|
||||
vh = s1_data["vh"].astype('float32')
|
||||
vv = s1_data["vv"].astype('float32')
|
||||
|
||||
# Resample to monthly average
|
||||
prediction_status["progress"] = "Đang tính trung bình VH/VV theo tháng..."
|
||||
vh_monthly = vh.resample(time="1ME").mean().compute()
|
||||
vv_monthly = vv.resample(time="1ME").mean().compute()
|
||||
|
||||
use_radar = True
|
||||
else:
|
||||
prediction_status["progress"] = "Không tìm thấy bands VH/VV, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
else:
|
||||
prediction_status["progress"] = "Không có dữ liệu Sentinel-1, tiếp tục với NDVI..."
|
||||
use_radar = False
|
||||
|
||||
# ============ BƯỚC 5: CHUẨN BỊ FEATURES CHO DỰ ĐOÁN ============
|
||||
prediction_status["progress"] = "Đang chuẩn bị features cho dự đoán..."
|
||||
|
||||
# Get shape information
|
||||
n_times_ndvi = len(ndvi_monthly.time)
|
||||
y_size = len(ndvi_monthly.y)
|
||||
x_size = len(ndvi_monthly.x)
|
||||
n_pixels = y_size * x_size
|
||||
|
||||
# Prepare NDVI features (flatten each time step)
|
||||
ndvi_features = []
|
||||
for t in range(n_times_ndvi):
|
||||
ndvi_t = ndvi_monthly.isel(time=t).values.flatten()
|
||||
ndvi_features.append(ndvi_t)
|
||||
|
||||
# Stack NDVI features
|
||||
features = np.column_stack(ndvi_features)
|
||||
|
||||
# Add radar features if available
|
||||
if use_radar:
|
||||
n_times_vh = len(vh_monthly.time)
|
||||
n_times_vv = len(vv_monthly.time)
|
||||
|
||||
# Add VH features
|
||||
for t in range(min(n_times_vh, n_times_ndvi)):
|
||||
vh_t = vh_monthly.isel(time=t).values.flatten()
|
||||
# Resize if needed
|
||||
if len(vh_t) != n_pixels:
|
||||
vh_t = np.resize(vh_t, n_pixels)
|
||||
features = np.column_stack([features, vh_t])
|
||||
|
||||
# Add VV features
|
||||
for t in range(min(n_times_vv, n_times_ndvi)):
|
||||
vv_t = vv_monthly.isel(time=t).values.flatten()
|
||||
# Resize if needed
|
||||
if len(vv_t) != n_pixels:
|
||||
vv_t = np.resize(vv_t, n_pixels)
|
||||
features = np.column_stack([features, vv_t])
|
||||
|
||||
# Handle NaN values in features✓ CNN PyTorch: Mạnh nhất với ảnh vệ tinh, tự học features, tương thích GPU tốt, cần pip install torch
|
||||
features = np.nan_to_num(features, nan=0.0)
|
||||
|
||||
# ============ BƯỚC 6: DỰ ĐOÁN ============
|
||||
# Check model's expected feature count and adjust
|
||||
try:
|
||||
# Get expected number of features from model
|
||||
if is_cnn_model:
|
||||
# For PyTorch CNN, get n_features from model
|
||||
expected_features = model.n_features
|
||||
elif hasattr(model, 'n_features_in_'):
|
||||
expected_features = model.n_features_in_
|
||||
elif hasattr(model, 'feature_names_in_'):
|
||||
expected_features = len(model.feature_names_in_)
|
||||
else:
|
||||
# Try to get from booster for XGBoost
|
||||
try:
|
||||
expected_features = model.get_booster().num_features()
|
||||
except:
|
||||
expected_features = features.shape[1]
|
||||
|
||||
prediction_status["progress"] = f"Model cần {expected_features} features, đang có {features.shape[1]} features..."
|
||||
|
||||
# Adjust features to match model
|
||||
if features.shape[1] > expected_features:
|
||||
# Trim to expected number (use only first N features - NDVI only)
|
||||
prediction_status["progress"] = f"Cắt bớt features từ {features.shape[1]} xuống {expected_features}..."
|
||||
features = features[:, :expected_features]
|
||||
elif features.shape[1] < expected_features:
|
||||
# Pad with zeros or repeat last features
|
||||
prediction_status["progress"] = f"Thêm features từ {features.shape[1]} lên {expected_features}..."
|
||||
n_missing = expected_features - features.shape[1]
|
||||
# Repeat last feature column to fill
|
||||
padding = np.tile(features[:, -1:], (1, n_missing))
|
||||
features = np.column_stack([features, padding])
|
||||
except Exception as e:
|
||||
prediction_status["progress"] = f"Không thể xác định số features của model, tiếp tục với {features.shape[1]} features..."
|
||||
|
||||
prediction_status["progress"] = f"Đang dự đoán với {features.shape[1]} features..."
|
||||
|
||||
# Make prediction
|
||||
if is_cnn_model:
|
||||
# PyTorch CNN prediction
|
||||
predictions = model.predict(features)
|
||||
else:
|
||||
predictions = model.predict(features)
|
||||
|
||||
# Decode labels if label_encoder exists
|
||||
if label_encoder is not None:
|
||||
try:
|
||||
predictions = label_encoder.inverse_transform(predictions)
|
||||
except:
|
||||
pass # Keep numeric predictions if inverse_transform fails
|
||||
|
||||
# Reshape to original shape
|
||||
pred_shape = (y_size, x_size)
|
||||
predictions_2d = predictions.reshape(pred_shape)
|
||||
|
||||
# ============ BƯỚC 7: TẠO OUTPUT VÀ LƯU KẾT QUẢ ============
|
||||
prediction_status["progress"] = "Đang tạo bản đồ phân loại..."
|
||||
|
||||
# Create output xarray
|
||||
prediction_da = xr.DataArray(
|
||||
predictions_2d,
|
||||
coords={
|
||||
"y": ndvi_monthly.y,
|
||||
"x": ndvi_monthly.x
|
||||
},
|
||||
dims=["y", "x"],
|
||||
name="classification"
|
||||
)
|
||||
|
||||
# Save output
|
||||
output_dir = Path("predictions")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
timestamp = dt.now().strftime("%Y%m%d_%H%M%S")
|
||||
output_file = output_dir / f"prediction_{timestamp}.tif"
|
||||
|
||||
prediction_status["progress"] = "Đang lưu kết quả GeoTIFF..."
|
||||
|
||||
# Set CRS and save as GeoTIFF
|
||||
if hasattr(s2_data, 'rio') and s2_data.rio.crs is not None:
|
||||
prediction_da.rio.write_crs(s2_data.rio.crs, inplace=True)
|
||||
else:
|
||||
prediction_da.rio.write_crs("EPSG:4326", inplace=True)
|
||||
|
||||
prediction_da.rio.to_raster(str(output_file), driver="GTiff")
|
||||
|
||||
# Get unique classes for result
|
||||
unique_classes = np.unique(predictions_2d)
|
||||
unique_classes = unique_classes[~np.isnan(unique_classes)].tolist()
|
||||
|
||||
prediction_status["is_predicting"] = False
|
||||
prediction_status["progress"] = "Hoàn thành! Đang tạo báo cáo..."
|
||||
prediction_status["output_file"] = str(output_file)
|
||||
prediction_status["result"] = {
|
||||
"output_file": str(output_file),
|
||||
"shape": list(pred_shape),
|
||||
"unique_classes": unique_classes,
|
||||
"bbox": bbox,
|
||||
"time_range": time_range,
|
||||
"n_features": features.shape[1],
|
||||
"n_times_ndvi": n_times_ndvi,
|
||||
"used_radar": use_radar,
|
||||
"model_used": config.model_filename
|
||||
}
|
||||
|
||||
# Auto generate prediction report
|
||||
try:
|
||||
report_path, _ = generate_prediction_report(prediction_status["result"])
|
||||
prediction_status["result"]["report_path"] = report_path
|
||||
prediction_status["result"]["report_filename"] = Path(report_path).name
|
||||
prediction_status["progress"] = "Hoàn thành! Báo cáo đã được tạo."
|
||||
print(f"[PREDICTION REPORT] Generated: {report_path}")
|
||||
except Exception as e:
|
||||
print(f"[PREDICTION REPORT ERROR] Failed to generate report: {e}")
|
||||
prediction_status["progress"] = "Hoàn thành! (Không thể tạo báo cáo)"
|
||||
|
||||
prediction_status["end_time"] = dt.now().isoformat()
|
||||
|
||||
except Exception as e:
|
||||
prediction_status["is_predicting"] = False
|
||||
prediction_status["error"] = str(e)
|
||||
prediction_status["progress"] = f"Lỗi: {str(e)}"
|
||||
prediction_status["end_time"] = dt.now().isoformat()
|
||||
import traceback
|
||||
print(traceback.format_exc())
|
||||
|
||||
|
||||
@app.get("/api/predictions/list")
|
||||
async def list_predictions():
|
||||
"""Lấy danh sách các file prediction đã tạo"""
|
||||
predictions_dir = Path("predictions")
|
||||
predictions_dir.mkdir(exist_ok=True)
|
||||
|
||||
predictions = []
|
||||
for pred_file in predictions_dir.glob("*.tif"):
|
||||
predictions.append({
|
||||
"filename": pred_file.name,
|
||||
"created": datetime.fromtimestamp(pred_file.stat().st_mtime).isoformat(),
|
||||
"size_mb": round(pred_file.stat().st_size / 1024 / 1024, 2),
|
||||
"download_url": f"/api/predictions/download/{pred_file.name}"
|
||||
})
|
||||
|
||||
# Sort by creation time (newest first)
|
||||
predictions.sort(key=lambda x: x["created"], reverse=True)
|
||||
return {"predictions": predictions}
|
||||
|
||||
|
||||
@app.get("/api/predictions/download/{filename}")
|
||||
async def download_prediction(filename: str):
|
||||
"""Download file prediction GeoTIFF"""
|
||||
predictions_dir = Path("predictions")
|
||||
file_path = predictions_dir / filename
|
||||
|
||||
# Security check: ensure filename doesn't contain path traversal
|
||||
if ".." in filename or "/" in filename or "\\" in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
|
||||
if not file_path.exists():
|
||||
raise HTTPException(status_code=404, detail=f"File không tồn tại: {filename}")
|
||||
|
||||
return FileResponse(
|
||||
path=str(file_path),
|
||||
filename=filename,
|
||||
media_type="image/tiff",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={filename}"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 70)
|
||||
print("🚀 LAND CLASSIFICATION TRAINING API SERVER")
|
||||
print("=" * 70)
|
||||
print("\n📍 Endpoints:")
|
||||
print(" - Web Interface: http://localhost:8000")
|
||||
print(" - API Docs: http://localhost:8000/docs")
|
||||
print(" - Start Training: POST http://localhost:8000/api/training/start")
|
||||
print(" - Check Status: GET http://localhost:8000/api/training/status")
|
||||
print("\n" + "=" * 70)
|
||||
|
||||
uvicorn.run(app, host="0.0.0.0", port=8000, log_level="info")
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,164 @@
|
||||
# 🌾 Giải Thích Quy Trình Phân Loại Đất Trồng Cây
|
||||
|
||||
File notebook `02.predict_ODC.ipynb` sử dụng **Machine Learning** kết hợp với **dữ liệu vệ tinh** để phân loại các loại đất/cây trồng. Dưới đây là quy trình chi tiết:
|
||||
|
||||
---
|
||||
|
||||
## **Bước 1: Thu thập dữ liệu vệ tinh** (Cell 3-4)
|
||||
|
||||
```python
|
||||
date_range = ('2022-09-01', '2023-10-01')
|
||||
longtitude_range = (105.86575, 105.94120)
|
||||
latitude_range = (9.65070, 9.69850)
|
||||
data = load_data(dc, date_range, longtitude_range, latitude_range)
|
||||
```
|
||||
|
||||
- Lấy ảnh **Sentinel-2** (ảnh quang học) từ kho dữ liệu trong khoảng thời gian và vị trí cụ thể
|
||||
|
||||
---
|
||||
|
||||
## **Bước 2: Xử lý mây** (Cell 5)
|
||||
|
||||
```python
|
||||
result = mask_clean(data)
|
||||
```
|
||||
|
||||
- Loại bỏ các pixel bị mây che phủ để đảm bảo dữ liệu chính xác
|
||||
|
||||
---
|
||||
|
||||
## **Bước 3: Tính chỉ số NDVI** (Cell 6-10)
|
||||
|
||||
```python
|
||||
ndvi = calculate_indices(result, index='NDVI', satellite_mission='s2')
|
||||
fill_nan_ndvi = fill_nan(ndvi, time_split)
|
||||
average_ndvi = fill_nan_ndvi.resample(time='1M').mean()
|
||||
```
|
||||
|
||||
- **NDVI** (Normalized Difference Vegetation Index) = (NIR - Red) / (NIR + Red)
|
||||
- Giá trị từ **-1 đến 1**: cao = thực vật xanh tốt, thấp = đất trống/nước
|
||||
- Điền giá trị nan (mây) và tính trung bình theo tháng
|
||||
|
||||
---
|
||||
|
||||
## **Bước 4: Lấy dữ liệu Radar Sentinel-1** (Cell 11)
|
||||
|
||||
```python
|
||||
dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)
|
||||
average_vv = calculate_average(dsvv, time_pattern='1M')
|
||||
average_vh = calculate_average(dsvh, time_pattern='1M')
|
||||
```
|
||||
|
||||
- **VH, VV**: Dữ liệu radar (xuyên mây), cho biết cấu trúc bề mặt
|
||||
- Giúp phân biệt lúa ngập nước, cây trồng cạn, mặt nước...
|
||||
|
||||
---
|
||||
|
||||
## **Bước 5: Dự đoán bằng Model ML** (Cell 12) ⭐ **QUAN TRỌNG NHẤT**
|
||||
|
||||
```python
|
||||
loaded_model = joblib.load("model_train/model_odc.joblib")
|
||||
data_array = predict(loaded_model, data.rio.crs, average_ndvi, average_vh, average_vv)
|
||||
```
|
||||
|
||||
**Model đã được train trước** với dữ liệu mẫu (training data) gồm:
|
||||
- **Đầu vào (Features)**: NDVI theo tháng + VH + VV (chuỗi thời gian)
|
||||
- **Đầu ra (Labels)**: Loại đất đã được gắn nhãn thủ công
|
||||
|
||||
### Cách model phân loại:
|
||||
|
||||
| Đặc điểm | Loại đất |
|
||||
|----------|----------|
|
||||
| NDVI cao đều, VV thấp | Rừng |
|
||||
| NDVI biến đổi theo mùa vụ, VH cao (nước) | Lúa |
|
||||
| NDVI thấp, VV rất thấp | Sông/nước |
|
||||
| NDVI trung bình ổn định | Cây lâu năm (CLN) |
|
||||
|
||||
---
|
||||
|
||||
## **Bước 6: Hiển thị kết quả** (Cell 13-15)
|
||||
|
||||
```python
|
||||
colors = ["#abcee9", "#ffef44", "#c4ff9e", "#ffd6a8", "#93ddda", "#1aeef7", "#ffa7f2", "#33ee33"]
|
||||
labels = ["Lúa tôm", "Lúa", "CHN", "CLN", "TS", "Sông", "Đất xây dựng", "Rừng"]
|
||||
```
|
||||
|
||||
### 8 lớp phân loại:
|
||||
|
||||
| Mã | Tên | Màu | Ý nghĩa |
|
||||
|----|-----|-----|---------|
|
||||
| 0 | Lúa tôm | 🔵 Xanh nhạt | Luân canh lúa-tôm |
|
||||
| 1 | Lúa | 🟡 Vàng | Đất trồng lúa |
|
||||
| 2 | CHN | 🟢 Xanh lá nhạt | Cây hàng năm |
|
||||
| 3 | CLN | 🟠 Cam nhạt | Cây lâu năm (cây ăn trái) |
|
||||
| 4 | TS | 🩵 Xanh ngọc | Thủy sản |
|
||||
| 5 | Sông | 🔷 Cyan | Mặt nước sông |
|
||||
| 6 | Đất XD | 💗 Hồng | Đất xây dựng |
|
||||
| 7 | Rừng | 💚 Xanh đậm | Rừng |
|
||||
|
||||
---
|
||||
|
||||
## **Bước 7: Lưu kết quả** (Cell 16)
|
||||
|
||||
```python
|
||||
region_result.rio.to_raster("KetQuaPhanLoaiDatODC.tif")
|
||||
```
|
||||
|
||||
- Xuất file GeoTIFF chứa mã phân loại (0-7) cho từng pixel
|
||||
|
||||
---
|
||||
|
||||
## 📊 **Tóm tắt quy trình:**
|
||||
|
||||
```
|
||||
Ảnh vệ tinh (Sentinel-1 + Sentinel-2)
|
||||
↓
|
||||
Xử lý (loại mây, tính NDVI, VH, VV)
|
||||
↓
|
||||
Kết hợp features theo thời gian (13 tháng)
|
||||
↓
|
||||
Model ML (Random Forest/XGBoost) dự đoán
|
||||
↓
|
||||
Bản đồ phân loại 8 lớp đất
|
||||
↓
|
||||
File .tif (mỗi pixel = 1 mã loại đất)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📁 Cấu trúc dữ liệu đầu vào cho Model
|
||||
|
||||
### Features (Đặc trưng):
|
||||
- **NDVI theo 13 tháng**: 13 bands
|
||||
- **VH (radar) theo 13 tháng**: 13 bands
|
||||
- **VV (radar) theo 13 tháng**: 13 bands
|
||||
- **Tổng cộng**: ~39 features cho mỗi pixel
|
||||
|
||||
### Labels (Nhãn):
|
||||
- Được lấy từ shapefile training: `train/ST_training data_updated_1130points_new.shp`
|
||||
- 1130 điểm mẫu đã được gắn nhãn thủ công bởi chuyên gia
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Các thư viện sử dụng
|
||||
|
||||
| Thư viện | Mục đích |
|
||||
|----------|----------|
|
||||
| `datacube` | Truy vấn dữ liệu vệ tinh |
|
||||
| `xarray` | Xử lý dữ liệu đa chiều |
|
||||
| `rioxarray` | Đọc/ghi GeoTIFF |
|
||||
| `joblib` | Load/save model ML |
|
||||
| `sklearn` / `xgboost` | Training model |
|
||||
| `matplotlib` / `hvplot` | Trực quan hóa |
|
||||
|
||||
---
|
||||
|
||||
## 📝 Ghi chú
|
||||
|
||||
- **Độ phân giải**: 10-20m (tùy cấu hình)
|
||||
- **Thời gian xử lý**: Phụ thuộc vào kích thước vùng và số scenes
|
||||
- **Yêu cầu**: Cần kết nối internet để tải dữ liệu vệ tinh từ Planetary Computer hoặc ODC
|
||||
|
||||
---
|
||||
|
||||
*Tài liệu được tạo ngày 14/12/2025*
|
||||
+3728
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+3300
File diff suppressed because one or more lines are too long
@@ -0,0 +1,786 @@
|
||||
"""
|
||||
Auto Report Generator for Land Classification
|
||||
Tự động tạo báo cáo HTML chi tiết sau training/prediction
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import base64
|
||||
import io
|
||||
|
||||
# Optional: for generating charts
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg') # Non-interactive backend
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
MATPLOTLIB_AVAILABLE = True
|
||||
except ImportError:
|
||||
MATPLOTLIB_AVAILABLE = False
|
||||
|
||||
|
||||
def generate_confusion_matrix_image(conf_matrix, class_names):
|
||||
"""Tạo hình ảnh confusion matrix dạng base64"""
|
||||
if not MATPLOTLIB_AVAILABLE:
|
||||
return None
|
||||
|
||||
try:
|
||||
fig, ax = plt.subplots(figsize=(10, 8))
|
||||
conf_matrix = np.array(conf_matrix)
|
||||
|
||||
im = ax.imshow(conf_matrix, interpolation='nearest', cmap=plt.cm.Blues)
|
||||
ax.figure.colorbar(im, ax=ax)
|
||||
|
||||
ax.set(xticks=np.arange(len(class_names)),
|
||||
yticks=np.arange(len(class_names)),
|
||||
xticklabels=class_names, yticklabels=class_names,
|
||||
title='Confusion Matrix',
|
||||
ylabel='Thực tế (True)',
|
||||
xlabel='Dự đoán (Predicted)')
|
||||
|
||||
plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
|
||||
|
||||
# Add text annotations
|
||||
thresh = conf_matrix.max() / 2.
|
||||
for i in range(len(class_names)):
|
||||
for j in range(len(class_names)):
|
||||
ax.text(j, i, format(conf_matrix[i, j], 'd'),
|
||||
ha="center", va="center",
|
||||
color="white" if conf_matrix[i, j] > thresh else "black")
|
||||
|
||||
fig.tight_layout()
|
||||
|
||||
# Convert to base64
|
||||
buf = io.BytesIO()
|
||||
plt.savefig(buf, format='png', dpi=100, bbox_inches='tight')
|
||||
buf.seek(0)
|
||||
img_base64 = base64.b64encode(buf.read()).decode('utf-8')
|
||||
plt.close(fig)
|
||||
|
||||
return img_base64
|
||||
except Exception as e:
|
||||
print(f"Error generating confusion matrix image: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def generate_class_distribution_chart(class_names, classification_report):
|
||||
"""Tạo biểu đồ phân bố các class dạng base64"""
|
||||
if not MATPLOTLIB_AVAILABLE:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Extract support (number of samples) for each class
|
||||
supports = []
|
||||
for cls in class_names:
|
||||
if cls in classification_report:
|
||||
supports.append(classification_report[cls].get('support', 0))
|
||||
else:
|
||||
supports.append(0)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 6))
|
||||
colors = plt.cm.Set3(np.linspace(0, 1, len(class_names)))
|
||||
|
||||
bars = ax.bar(class_names, supports, color=colors)
|
||||
ax.set_xlabel('Loại đất')
|
||||
ax.set_ylabel('Số mẫu')
|
||||
ax.set_title('Phân bố số mẫu theo loại đất')
|
||||
plt.xticks(rotation=45, ha='right')
|
||||
|
||||
# Add value labels on bars
|
||||
for bar, val in zip(bars, supports):
|
||||
ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5,
|
||||
str(int(val)), ha='center', va='bottom', fontsize=9)
|
||||
|
||||
fig.tight_layout()
|
||||
|
||||
# Convert to base64
|
||||
buf = io.BytesIO()
|
||||
plt.savefig(buf, format='png', dpi=100, bbox_inches='tight')
|
||||
buf.seek(0)
|
||||
img_base64 = base64.b64encode(buf.read()).decode('utf-8')
|
||||
plt.close(fig)
|
||||
|
||||
return img_base64
|
||||
except Exception as e:
|
||||
print(f"Error generating class distribution chart: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def generate_metrics_chart(class_names, classification_report):
|
||||
"""Tạo biểu đồ precision/recall/f1 cho từng class"""
|
||||
if not MATPLOTLIB_AVAILABLE:
|
||||
return None
|
||||
|
||||
try:
|
||||
precisions = []
|
||||
recalls = []
|
||||
f1_scores = []
|
||||
|
||||
for cls in class_names:
|
||||
if cls in classification_report:
|
||||
precisions.append(classification_report[cls].get('precision', 0))
|
||||
recalls.append(classification_report[cls].get('recall', 0))
|
||||
f1_scores.append(classification_report[cls].get('f1-score', 0))
|
||||
else:
|
||||
precisions.append(0)
|
||||
recalls.append(0)
|
||||
f1_scores.append(0)
|
||||
|
||||
x = np.arange(len(class_names))
|
||||
width = 0.25
|
||||
|
||||
fig, ax = plt.subplots(figsize=(12, 6))
|
||||
|
||||
bars1 = ax.bar(x - width, precisions, width, label='Precision', color='#3498db')
|
||||
bars2 = ax.bar(x, recalls, width, label='Recall', color='#2ecc71')
|
||||
bars3 = ax.bar(x + width, f1_scores, width, label='F1-Score', color='#e74c3c')
|
||||
|
||||
ax.set_xlabel('Loại đất')
|
||||
ax.set_ylabel('Score')
|
||||
ax.set_title('Precision / Recall / F1-Score theo loại đất')
|
||||
ax.set_xticks(x)
|
||||
ax.set_xticklabels(class_names, rotation=45, ha='right')
|
||||
ax.legend()
|
||||
ax.set_ylim(0, 1.1)
|
||||
|
||||
# Add grid
|
||||
ax.yaxis.grid(True, linestyle='--', alpha=0.7)
|
||||
|
||||
fig.tight_layout()
|
||||
|
||||
# Convert to base64
|
||||
buf = io.BytesIO()
|
||||
plt.savefig(buf, format='png', dpi=100, bbox_inches='tight')
|
||||
buf.seek(0)
|
||||
img_base64 = base64.b64encode(buf.read()).decode('utf-8')
|
||||
plt.close(fig)
|
||||
|
||||
return img_base64
|
||||
except Exception as e:
|
||||
print(f"Error generating metrics chart: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def generate_training_report(training_result, config=None):
|
||||
"""
|
||||
Tạo báo cáo HTML cho kết quả training
|
||||
|
||||
Args:
|
||||
training_result: Dict chứa kết quả từ train_model()
|
||||
config: Dict chứa cấu hình training (optional)
|
||||
|
||||
Returns:
|
||||
Tuple (report_path, report_html)
|
||||
"""
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
# Extract data from result
|
||||
train_acc = training_result.get('train_accuracy', 0) * 100
|
||||
test_acc = training_result.get('test_accuracy', 0) * 100
|
||||
train_samples = training_result.get('training_samples', 0)
|
||||
test_samples = training_result.get('testing_samples', 0)
|
||||
test_size = training_result.get('test_size', 0.2)
|
||||
classes = training_result.get('classes', [])
|
||||
cls_report = training_result.get('classification_report', {})
|
||||
conf_matrix = training_result.get('confusion_matrix', [])
|
||||
model_type = training_result.get('model_type', 'unknown')
|
||||
model_path = training_result.get('model_path', '')
|
||||
bbox = training_result.get('bbox', [])
|
||||
time_range = training_result.get('time_range', '')
|
||||
resolution = training_result.get('resolution', 20)
|
||||
|
||||
# Generate charts
|
||||
conf_matrix_img = generate_confusion_matrix_image(conf_matrix, classes) if conf_matrix else None
|
||||
class_dist_img = generate_class_distribution_chart(classes, cls_report) if cls_report else None
|
||||
metrics_img = generate_metrics_chart(classes, cls_report) if cls_report else None
|
||||
|
||||
# Build classification report table
|
||||
cls_report_rows = ""
|
||||
for cls in classes:
|
||||
if cls in cls_report:
|
||||
metrics = cls_report[cls]
|
||||
cls_report_rows += f"""
|
||||
<tr>
|
||||
<td><strong>{cls}</strong></td>
|
||||
<td>{metrics.get('precision', 0):.3f}</td>
|
||||
<td>{metrics.get('recall', 0):.3f}</td>
|
||||
<td>{metrics.get('f1-score', 0):.3f}</td>
|
||||
<td>{int(metrics.get('support', 0))}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
# Add averages
|
||||
for avg_type in ['macro avg', 'weighted avg']:
|
||||
if avg_type in cls_report:
|
||||
metrics = cls_report[avg_type]
|
||||
cls_report_rows += f"""
|
||||
<tr style="background-color: #f0f0f0; font-weight: bold;">
|
||||
<td>{avg_type}</td>
|
||||
<td>{metrics.get('precision', 0):.3f}</td>
|
||||
<td>{metrics.get('recall', 0):.3f}</td>
|
||||
<td>{metrics.get('f1-score', 0):.3f}</td>
|
||||
<td>{int(metrics.get('support', 0))}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
# Build confusion matrix table (fallback if no image)
|
||||
conf_matrix_table = ""
|
||||
if conf_matrix:
|
||||
conf_matrix_table = "<table class='conf-matrix'><tr><th></th>"
|
||||
for cls in classes:
|
||||
conf_matrix_table += f"<th>{cls}</th>"
|
||||
conf_matrix_table += "</tr>"
|
||||
for i, row in enumerate(conf_matrix):
|
||||
conf_matrix_table += f"<tr><th>{classes[i]}</th>"
|
||||
for val in row:
|
||||
conf_matrix_table += f"<td>{val}</td>"
|
||||
conf_matrix_table += "</tr>"
|
||||
conf_matrix_table += "</table>"
|
||||
|
||||
# HTML Template
|
||||
html = f"""
|
||||
<!DOCTYPE html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Training Report - {timestamp}</title>
|
||||
<style>
|
||||
* {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
body {{
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: #f5f5f5;
|
||||
padding: 20px;
|
||||
line-height: 1.6;
|
||||
}}
|
||||
.container {{
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
background: white;
|
||||
border-radius: 15px;
|
||||
box-shadow: 0 10px 40px rgba(0,0,0,0.1);
|
||||
overflow: hidden;
|
||||
}}
|
||||
.header {{
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}}
|
||||
.header h1 {{
|
||||
font-size: 2.5em;
|
||||
margin-bottom: 10px;
|
||||
}}
|
||||
.header .subtitle {{
|
||||
opacity: 0.9;
|
||||
font-size: 1.1em;
|
||||
}}
|
||||
.content {{
|
||||
padding: 40px;
|
||||
}}
|
||||
.section {{
|
||||
margin-bottom: 40px;
|
||||
}}
|
||||
.section h2 {{
|
||||
color: #667eea;
|
||||
border-bottom: 3px solid #667eea;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 20px;
|
||||
font-size: 1.5em;
|
||||
}}
|
||||
.stats-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}}
|
||||
.stat-card {{
|
||||
background: linear-gradient(135deg, #667eea15 0%, #764ba215 100%);
|
||||
padding: 25px;
|
||||
border-radius: 10px;
|
||||
text-align: center;
|
||||
border: 1px solid #667eea30;
|
||||
}}
|
||||
.stat-card .value {{
|
||||
font-size: 2.5em;
|
||||
font-weight: bold;
|
||||
color: #667eea;
|
||||
}}
|
||||
.stat-card .label {{
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}}
|
||||
.stat-card.success .value {{
|
||||
color: #28a745;
|
||||
}}
|
||||
.stat-card.warning .value {{
|
||||
color: #ffc107;
|
||||
}}
|
||||
table {{
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
th, td {{
|
||||
padding: 12px 15px;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid #ddd;
|
||||
}}
|
||||
th {{
|
||||
background: #667eea;
|
||||
color: white;
|
||||
}}
|
||||
tr:hover {{
|
||||
background-color: #f5f5f5;
|
||||
}}
|
||||
.conf-matrix {{
|
||||
font-size: 14px;
|
||||
}}
|
||||
.conf-matrix th, .conf-matrix td {{
|
||||
text-align: center;
|
||||
padding: 8px;
|
||||
}}
|
||||
.chart-container {{
|
||||
text-align: center;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
.chart-container img {{
|
||||
max-width: 100%;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 4px 15px rgba(0,0,0,0.1);
|
||||
}}
|
||||
.info-box {{
|
||||
background: #e3f2fd;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
border-left: 5px solid #2196f3;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
.info-row {{
|
||||
display: flex;
|
||||
margin: 10px 0;
|
||||
}}
|
||||
.info-label {{
|
||||
font-weight: bold;
|
||||
width: 200px;
|
||||
color: #555;
|
||||
}}
|
||||
.info-value {{
|
||||
color: #333;
|
||||
}}
|
||||
.footer {{
|
||||
background: #f8f9fa;
|
||||
padding: 20px;
|
||||
text-align: center;
|
||||
color: #666;
|
||||
font-size: 14px;
|
||||
}}
|
||||
@media print {{
|
||||
body {{
|
||||
background: white;
|
||||
padding: 0;
|
||||
}}
|
||||
.container {{
|
||||
box-shadow: none;
|
||||
}}
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>📊 Báo Cáo Training Model</h1>
|
||||
<p class="subtitle">Land Classification - {datetime.now().strftime("%d/%m/%Y %H:%M:%S")}</p>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<!-- Summary Stats -->
|
||||
<div class="section">
|
||||
<h2>📈 Tóm Tắt Kết Quả</h2>
|
||||
<div class="stats-grid">
|
||||
<div class="stat-card success">
|
||||
<div class="value">{train_acc:.1f}%</div>
|
||||
<div class="label">Train Accuracy</div>
|
||||
</div>
|
||||
<div class="stat-card {'success' if test_acc >= 80 else 'warning'}">
|
||||
<div class="value">{test_acc:.1f}%</div>
|
||||
<div class="label">Test Accuracy</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{train_samples}</div>
|
||||
<div class="label">Training Samples</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{test_samples}</div>
|
||||
<div class="label">Testing Samples</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{len(classes)}</div>
|
||||
<div class="label">Số Classes</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{test_size*100:.0f}%</div>
|
||||
<div class="label">Test Size</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Configuration Info -->
|
||||
<div class="section">
|
||||
<h2>⚙️ Cấu Hình Training</h2>
|
||||
<div class="info-box">
|
||||
<div class="info-row">
|
||||
<span class="info-label">🤖 Model Type:</span>
|
||||
<span class="info-value">{model_type.upper()}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📍 Khu vực (bbox):</span>
|
||||
<span class="info-value">{bbox}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📅 Thời gian:</span>
|
||||
<span class="info-value">{time_range}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📐 Độ phân giải:</span>
|
||||
<span class="info-value">{resolution}m</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">💾 Model Path:</span>
|
||||
<span class="info-value">{model_path}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Classification Report -->
|
||||
<div class="section">
|
||||
<h2>📋 Classification Report</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Loại đất</th>
|
||||
<th>Precision</th>
|
||||
<th>Recall</th>
|
||||
<th>F1-Score</th>
|
||||
<th>Support</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{cls_report_rows}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Metrics Chart -->
|
||||
{'<div class="section"><h2>📊 Biểu Đồ Metrics</h2><div class="chart-container"><img src="data:image/png;base64,' + metrics_img + '" alt="Metrics Chart"></div></div>' if metrics_img else ''}
|
||||
|
||||
<!-- Class Distribution -->
|
||||
{'<div class="section"><h2>📊 Phân Bố Số Mẫu</h2><div class="chart-container"><img src="data:image/png;base64,' + class_dist_img + '" alt="Class Distribution"></div></div>' if class_dist_img else ''}
|
||||
|
||||
<!-- Confusion Matrix -->
|
||||
<div class="section">
|
||||
<h2>🔢 Confusion Matrix</h2>
|
||||
{'<div class="chart-container"><img src="data:image/png;base64,' + conf_matrix_img + '" alt="Confusion Matrix"></div>' if conf_matrix_img else conf_matrix_table}
|
||||
</div>
|
||||
|
||||
<!-- Classes List -->
|
||||
<div class="section">
|
||||
<h2>🏷️ Danh Sách Các Loại Đất</h2>
|
||||
<div class="info-box">
|
||||
<ul style="list-style: none; display: flex; flex-wrap: wrap; gap: 10px;">
|
||||
{''.join([f'<li style="background: #667eea; color: white; padding: 8px 15px; border-radius: 20px;">{cls}</li>' for cls in classes])}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>🌍 Land Classification Training System | Generated: {datetime.now().strftime("%d/%m/%Y %H:%M:%S")}</p>
|
||||
<p>Data Source: Microsoft Planetary Computer (Sentinel-2 L2A, Sentinel-1 RTC)</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Save report
|
||||
reports_dir = Path("reports")
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
|
||||
report_filename = f"training_report_{timestamp}.html"
|
||||
report_path = reports_dir / report_filename
|
||||
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
f.write(html)
|
||||
|
||||
return str(report_path), html
|
||||
|
||||
|
||||
def generate_prediction_report(prediction_result, config=None):
|
||||
"""
|
||||
Tạo báo cáo HTML cho kết quả prediction
|
||||
|
||||
Args:
|
||||
prediction_result: Dict chứa kết quả prediction
|
||||
config: Dict chứa cấu hình prediction (optional)
|
||||
|
||||
Returns:
|
||||
Tuple (report_path, report_html)
|
||||
"""
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
# Extract data
|
||||
output_file = prediction_result.get('output_file', '')
|
||||
shape = prediction_result.get('shape', [0, 0])
|
||||
unique_classes = prediction_result.get('unique_classes', [])
|
||||
bbox = prediction_result.get('bbox', [])
|
||||
time_range = prediction_result.get('time_range', '')
|
||||
n_features = prediction_result.get('n_features', 0)
|
||||
used_radar = prediction_result.get('used_radar', False)
|
||||
model_used = prediction_result.get('model_used', '')
|
||||
|
||||
# Calculate area (approximate)
|
||||
if len(bbox) == 4:
|
||||
# Approximate calculation (1 degree ≈ 111km at equator)
|
||||
width_km = (bbox[2] - bbox[0]) * 111 * 0.85 # cos adjustment for Vietnam
|
||||
height_km = (bbox[3] - bbox[1]) * 111
|
||||
area_km2 = width_km * height_km
|
||||
else:
|
||||
area_km2 = 0
|
||||
|
||||
total_pixels = shape[0] * shape[1] if len(shape) == 2 else 0
|
||||
|
||||
# HTML Template
|
||||
html = f"""
|
||||
<!DOCTYPE html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Prediction Report - {timestamp}</title>
|
||||
<style>
|
||||
* {{
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}}
|
||||
body {{
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: #f5f5f5;
|
||||
padding: 20px;
|
||||
line-height: 1.6;
|
||||
}}
|
||||
.container {{
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
background: white;
|
||||
border-radius: 15px;
|
||||
box-shadow: 0 10px 40px rgba(0,0,0,0.1);
|
||||
overflow: hidden;
|
||||
}}
|
||||
.header {{
|
||||
background: linear-gradient(135deg, #ff6b6b 0%, #ee5a6f 100%);
|
||||
color: white;
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}}
|
||||
.header h1 {{
|
||||
font-size: 2.5em;
|
||||
margin-bottom: 10px;
|
||||
}}
|
||||
.content {{
|
||||
padding: 40px;
|
||||
}}
|
||||
.section {{
|
||||
margin-bottom: 40px;
|
||||
}}
|
||||
.section h2 {{
|
||||
color: #ff6b6b;
|
||||
border-bottom: 3px solid #ff6b6b;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 20px;
|
||||
}}
|
||||
.stats-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 20px;
|
||||
}}
|
||||
.stat-card {{
|
||||
background: linear-gradient(135deg, #ff6b6b15 0%, #ee5a6f15 100%);
|
||||
padding: 25px;
|
||||
border-radius: 10px;
|
||||
text-align: center;
|
||||
border: 1px solid #ff6b6b30;
|
||||
}}
|
||||
.stat-card .value {{
|
||||
font-size: 2em;
|
||||
font-weight: bold;
|
||||
color: #ff6b6b;
|
||||
}}
|
||||
.stat-card .label {{
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}}
|
||||
.info-box {{
|
||||
background: #fff3cd;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
border-left: 5px solid #ff6b6b;
|
||||
margin: 20px 0;
|
||||
}}
|
||||
.info-row {{
|
||||
display: flex;
|
||||
margin: 10px 0;
|
||||
}}
|
||||
.info-label {{
|
||||
font-weight: bold;
|
||||
width: 200px;
|
||||
color: #555;
|
||||
}}
|
||||
.class-badge {{
|
||||
display: inline-block;
|
||||
background: #ff6b6b;
|
||||
color: white;
|
||||
padding: 8px 15px;
|
||||
border-radius: 20px;
|
||||
margin: 5px;
|
||||
}}
|
||||
.footer {{
|
||||
background: #f8f9fa;
|
||||
padding: 20px;
|
||||
text-align: center;
|
||||
color: #666;
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>🗺️ Báo Cáo Dự Đoán</h1>
|
||||
<p>Land Classification Prediction - {datetime.now().strftime("%d/%m/%Y %H:%M:%S")}</p>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<div class="section">
|
||||
<h2>📈 Tóm Tắt Kết Quả</h2>
|
||||
<div class="stats-grid">
|
||||
<div class="stat-card">
|
||||
<div class="value">{total_pixels:,}</div>
|
||||
<div class="label">Tổng số Pixels</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{shape[0]}x{shape[1]}</div>
|
||||
<div class="label">Kích thước (px)</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{area_km2:.1f}</div>
|
||||
<div class="label">Diện tích (km²)</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{len(unique_classes)}</div>
|
||||
<div class="label">Số Classes</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{n_features}</div>
|
||||
<div class="label">Số Features</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">{'✅' if used_radar else '❌'}</div>
|
||||
<div class="label">Sử dụng Radar</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>⚙️ Thông Tin Chi Tiết</h2>
|
||||
<div class="info-box">
|
||||
<div class="info-row">
|
||||
<span class="info-label">🤖 Model sử dụng:</span>
|
||||
<span>{model_used}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📍 Khu vực (bbox):</span>
|
||||
<span>{bbox}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📅 Thời gian:</span>
|
||||
<span>{time_range}</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">💾 Output file:</span>
|
||||
<span>{output_file}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>🏷️ Các Classes Phát Hiện</h2>
|
||||
<div>
|
||||
{''.join([f'<span class="class-badge">{cls}</span>' for cls in unique_classes])}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>🌍 Land Classification System | Generated: {datetime.now().strftime("%d/%m/%Y %H:%M:%S")}</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Save report
|
||||
reports_dir = Path("reports")
|
||||
reports_dir.mkdir(exist_ok=True)
|
||||
|
||||
report_filename = f"prediction_report_{timestamp}.html"
|
||||
report_path = reports_dir / report_filename
|
||||
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
f.write(html)
|
||||
|
||||
return str(report_path), html
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test report generation
|
||||
test_result = {
|
||||
"success": True,
|
||||
"train_accuracy": 0.95,
|
||||
"test_accuracy": 0.87,
|
||||
"training_samples": 800,
|
||||
"testing_samples": 200,
|
||||
"test_size": 0.2,
|
||||
"classes": ["Lua", "Rung", "Nuoc", "Dan_cu", "Cay_lau_nam"],
|
||||
"model_type": "xgboost",
|
||||
"model_path": "model_train/model_xgboost_20251221.joblib",
|
||||
"bbox": [105.6, 9.3, 106.2, 9.8],
|
||||
"time_range": "2023-03-01/2023-05-31",
|
||||
"resolution": 20,
|
||||
"classification_report": {
|
||||
"Lua": {"precision": 0.92, "recall": 0.89, "f1-score": 0.90, "support": 50},
|
||||
"Rung": {"precision": 0.88, "recall": 0.91, "f1-score": 0.89, "support": 45},
|
||||
"Nuoc": {"precision": 0.95, "recall": 0.93, "f1-score": 0.94, "support": 40},
|
||||
"Dan_cu": {"precision": 0.85, "recall": 0.82, "f1-score": 0.83, "support": 35},
|
||||
"Cay_lau_nam": {"precision": 0.80, "recall": 0.85, "f1-score": 0.82, "support": 30},
|
||||
"macro avg": {"precision": 0.88, "recall": 0.88, "f1-score": 0.88, "support": 200},
|
||||
"weighted avg": {"precision": 0.88, "recall": 0.87, "f1-score": 0.87, "support": 200}
|
||||
},
|
||||
"confusion_matrix": [
|
||||
[45, 2, 1, 1, 1],
|
||||
[3, 41, 0, 1, 0],
|
||||
[1, 0, 37, 1, 1],
|
||||
[2, 1, 1, 29, 2],
|
||||
[1, 1, 1, 2, 26]
|
||||
]
|
||||
}
|
||||
|
||||
path, html = generate_training_report(test_result)
|
||||
print(f"Report generated: {path}")
|
||||
@@ -0,0 +1,176 @@
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html lang="vi">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Prediction Report - 20251221_122210</title>
|
||||
<style>
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
background: #f5f5f5;
|
||||
padding: 20px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
.container {
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
background: white;
|
||||
border-radius: 15px;
|
||||
box-shadow: 0 10px 40px rgba(0,0,0,0.1);
|
||||
overflow: hidden;
|
||||
}
|
||||
.header {
|
||||
background: linear-gradient(135deg, #ff6b6b 0%, #ee5a6f 100%);
|
||||
color: white;
|
||||
padding: 40px;
|
||||
text-align: center;
|
||||
}
|
||||
.header h1 {
|
||||
font-size: 2.5em;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.content {
|
||||
padding: 40px;
|
||||
}
|
||||
.section {
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
.section h2 {
|
||||
color: #ff6b6b;
|
||||
border-bottom: 3px solid #ff6b6b;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.stats-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 20px;
|
||||
}
|
||||
.stat-card {
|
||||
background: linear-gradient(135deg, #ff6b6b15 0%, #ee5a6f15 100%);
|
||||
padding: 25px;
|
||||
border-radius: 10px;
|
||||
text-align: center;
|
||||
border: 1px solid #ff6b6b30;
|
||||
}
|
||||
.stat-card .value {
|
||||
font-size: 2em;
|
||||
font-weight: bold;
|
||||
color: #ff6b6b;
|
||||
}
|
||||
.stat-card .label {
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}
|
||||
.info-box {
|
||||
background: #fff3cd;
|
||||
padding: 20px;
|
||||
border-radius: 10px;
|
||||
border-left: 5px solid #ff6b6b;
|
||||
margin: 20px 0;
|
||||
}
|
||||
.info-row {
|
||||
display: flex;
|
||||
margin: 10px 0;
|
||||
}
|
||||
.info-label {
|
||||
font-weight: bold;
|
||||
width: 200px;
|
||||
color: #555;
|
||||
}
|
||||
.class-badge {
|
||||
display: inline-block;
|
||||
background: #ff6b6b;
|
||||
color: white;
|
||||
padding: 8px 15px;
|
||||
border-radius: 20px;
|
||||
margin: 5px;
|
||||
}
|
||||
.footer {
|
||||
background: #f8f9fa;
|
||||
padding: 20px;
|
||||
text-align: center;
|
||||
color: #666;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>🗺️ Báo Cáo Dự Đoán</h1>
|
||||
<p>Land Classification Prediction - 21/12/2025 12:22:10</p>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<div class="section">
|
||||
<h2>📈 Tóm Tắt Kết Quả</h2>
|
||||
<div class="stats-grid">
|
||||
<div class="stat-card">
|
||||
<div class="value">9,156,974</div>
|
||||
<div class="label">Tổng số Pixels</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">2774x3301</div>
|
||||
<div class="label">Kích thước (px)</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">3141.9</div>
|
||||
<div class="label">Diện tích (km²)</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">3</div>
|
||||
<div class="label">Số Classes</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">3</div>
|
||||
<div class="label">Số Features</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">✅</div>
|
||||
<div class="label">Sử dụng Radar</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>⚙️ Thông Tin Chi Tiết</h2>
|
||||
<div class="info-box">
|
||||
<div class="info-row">
|
||||
<span class="info-label">🤖 Model sử dụng:</span>
|
||||
<span>model_xgboost_20251221_122105.joblib</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📍 Khu vực (bbox):</span>
|
||||
<span>[105.6, 9.3, 106.2, 9.8]</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">📅 Thời gian:</span>
|
||||
<span>2023-03-01/2023-05-31</span>
|
||||
</div>
|
||||
<div class="info-row">
|
||||
<span class="info-label">💾 Output file:</span>
|
||||
<span>predictions/prediction_20251221_122209.tif</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>🏷️ Các Classes Phát Hiện</h2>
|
||||
<div>
|
||||
<span class="class-badge">3</span><span class="class-badge">5</span><span class="class-badge">6</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>🌍 Land Classification System | Generated: 21/12/2025 12:22:10</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+309
@@ -0,0 +1,309 @@
|
||||
affine @ file:///home/conda/feedstock_root/build_artifacts/affine_1733762038348/work
|
||||
aiobotocore==2.25.0
|
||||
aiohappyeyeballs==2.6.1
|
||||
aiohttp==3.12.15
|
||||
aioitertools==0.12.0
|
||||
aiosignal==1.4.0
|
||||
alembic==1.16.5
|
||||
annotated-doc==0.0.4
|
||||
annotated-types==0.7.0
|
||||
antimeridian @ file:///home/conda/feedstock_root/build_artifacts/antimeridian_1753706324394/work
|
||||
anyio @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_anyio_1758634638/work
|
||||
argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1749017159514/work
|
||||
argon2-cffi-bindings @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi-bindings_1649500328244/work
|
||||
arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1733584251875/work
|
||||
asciitree==0.3.3
|
||||
asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1733250440834/work
|
||||
async-lru @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_async-lru_1742153708/work
|
||||
async-timeout==3.0.1
|
||||
attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1741918516150/work
|
||||
babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1738490167835/work
|
||||
beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1759146011391/work
|
||||
bleach @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_bleach_1737382993/work
|
||||
blinker==1.9.0
|
||||
bokeh==3.7.3
|
||||
boto3==1.40.18
|
||||
botocore==1.40.49
|
||||
Bottleneck @ file:///croot/bottleneck_1731058641041/work
|
||||
branca @ file:///croot/branca_1675157607453/work
|
||||
Brotli @ file:///croot/brotli-split_1736182456865/work
|
||||
brotlicffi @ file:///croot/brotlicffi_1736182461069/work
|
||||
cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work
|
||||
cachetools==6.2.0
|
||||
Cartopy==0.25.0
|
||||
certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1759648874697/work/certifi
|
||||
cffi @ file:///croot/cffi_1736182485317/work
|
||||
cftime @ file:///home/conda/feedstock_root/build_artifacts/cftime_1649636873066/work
|
||||
chardet @ file:///home/conda/feedstock_root/build_artifacts/chardet_1649184137891/work
|
||||
charset-normalizer @ file:///croot/charset-normalizer_1721748349566/work
|
||||
ciso8601==2.3.3
|
||||
click @ file:///home/conda/feedstock_root/build_artifacts/click_1747811314515/work
|
||||
click-plugins @ file:///home/conda/feedstock_root/build_artifacts/click-plugins_1750848229740/work
|
||||
cligj @ file:///home/conda/feedstock_root/build_artifacts/cligj_1733749956636/work
|
||||
cloudpickle @ file:///home/conda/feedstock_root/build_artifacts/cloudpickle_1736947526808/work
|
||||
colorama==0.4.6
|
||||
colorcet==3.1.0
|
||||
comm @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_comm_1753453984/work
|
||||
contourpy @ file:///croot/contourpy_1732540045555/work
|
||||
cycler @ file:///tmp/build/80754af9/cycler_1637851556182/work
|
||||
cytoolz==0.11.2
|
||||
dask @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_dask-core_1760473436/work
|
||||
dask-gateway @ file:///Users/runner/miniforge3/conda-bld/bld/rattler-build_dask-gateway_1744370153/work/dask-gateway
|
||||
dask-glm @ file:///home/conda/feedstock_root/build_artifacts/dask-glm_1701346265909/work
|
||||
dask-image==2024.5.3
|
||||
dask-ml @ file:///home/conda/feedstock_root/build_artifacts/dask-ml_1679705292494/work
|
||||
datacube==1.8.15
|
||||
datacube_ows==1.9.4
|
||||
datashader==0.18.2
|
||||
dea-tools==0.3.0
|
||||
debugpy @ file:///home/task_175706711740264/conda-bld/debugpy_1757067131873/work
|
||||
decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1740384970518/work
|
||||
deepdiff==8.6.1
|
||||
defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work
|
||||
deprecat @ file:///home/conda/feedstock_root/build_artifacts/deprecat_1734684036993/work
|
||||
distributed @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_distributed_1760476147/work
|
||||
eo-tides==0.8.2
|
||||
exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1746947292760/work
|
||||
executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1756729339227/work
|
||||
fastapi==0.124.3
|
||||
fasteners @ file:///home/conda/feedstock_root/build_artifacts/fasteners_1734943108928/work
|
||||
fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_python-fastjsonschema_1755304154/work/dist
|
||||
filelock==3.19.1
|
||||
fiona==1.10.1
|
||||
Flask==3.1.2
|
||||
flask-babel==4.0.0
|
||||
flatbuffers==25.2.10
|
||||
folium==0.20.0
|
||||
fonttools @ file:///croot/fonttools_1737039080035/work
|
||||
fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1733327382592/work/dist
|
||||
frozenlist==1.7.0
|
||||
fsspec @ file:///home/conda/feedstock_root/build_artifacts/fsspec_1756908513222/work
|
||||
GDAL @ file:///croot/gdal-split_1734448174900/work/build/swig/python
|
||||
GeoAlchemy2 @ file:///home/conda/feedstock_root/build_artifacts/geoalchemy2_1753372953474/work
|
||||
geographiclib==2.1
|
||||
geojson==3.2.0
|
||||
geomad==1.0.0
|
||||
geopandas @ file:///croot/geopandas-split_1755761494241/work
|
||||
geopy==2.4.1
|
||||
greenlet @ file:///home/conda/feedstock_root/build_artifacts/greenlet_1648882383677/work
|
||||
h11 @ file:///home/conda/feedstock_root/build_artifacts/h11_1745526374115/work
|
||||
h2 @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_h2_1756364871/work
|
||||
h3==4.3.1
|
||||
hdstats==0.2.1
|
||||
holoviews==1.21.0
|
||||
hpack @ file:///home/conda/feedstock_root/build_artifacts/hpack_1737618293087/work
|
||||
httpcore @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_httpcore_1745602916/work
|
||||
httpx @ file:///home/conda/feedstock_root/build_artifacts/httpx_1733663348460/work
|
||||
hvplot==0.12.1
|
||||
hyperframe @ file:///home/conda/feedstock_root/build_artifacts/hyperframe_1737618333194/work
|
||||
idna==3.10
|
||||
imagecodecs==2025.3.30
|
||||
imageio==2.37.0
|
||||
importlib_metadata @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_importlib-metadata_1747934053/work
|
||||
ipykernel @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipykernel_1760459840/work
|
||||
ipyleaflet==0.20.0
|
||||
ipython @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_ipython_1748711175/work
|
||||
ipywidgets==8.1.7
|
||||
iso8601==2.1.0
|
||||
isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1733493628631/work/dist
|
||||
itsdangerous==2.2.0
|
||||
jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1733300866624/work
|
||||
Jinja2 @ file:///croot/jinja2_1741710844255/work
|
||||
jmespath @ file:///home/conda/feedstock_root/build_artifacts/jmespath_1733229141657/work
|
||||
joblib @ file:///home/conda/feedstock_root/build_artifacts/joblib_1756321760188/work
|
||||
json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1755034879854/work
|
||||
jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1756754132747/work
|
||||
jsonschema @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jsonschema_1755595646/work
|
||||
jsonschema-specifications==2025.4.1
|
||||
jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_events_1738765986/work
|
||||
jupyter-leaflet==0.20.0
|
||||
jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter-lsp_1756388269/work/jupyter-lsp
|
||||
jupyter-ui-poll==1.0.0
|
||||
jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1733440914442/work
|
||||
jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1748333051527/work
|
||||
jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_jupyter_server_1755870522/work
|
||||
jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1733427956852/work
|
||||
jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1758913905644/work
|
||||
jupyterlab_pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1733328101776/work
|
||||
jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server_1733599573484/work
|
||||
jupyterlab_widgets==3.0.15
|
||||
kiwisolver @ file:///croot/kiwisolver_1737039087198/work
|
||||
lark==1.2.2
|
||||
lark-parser==0.12.0
|
||||
lazy_loader==0.4
|
||||
linkify-it-py==2.0.3
|
||||
llvmlite @ file:///croot/llvmlite_1741209858218/work
|
||||
locket @ file:///home/conda/feedstock_root/build_artifacts/locket_1650660393415/work
|
||||
lxml==5.4.0
|
||||
lz4 @ file:///croot/lz4_1736366683208/work
|
||||
Mako @ file:///home/conda/feedstock_root/build_artifacts/mako_1744317760971/work
|
||||
mapclassify @ file:///croot/mapclassify_1675157730177/work
|
||||
Markdown==3.9
|
||||
markdown-it-py==4.0.0
|
||||
MarkupSafe @ file:///croot/markupsafe_1738584038848/work
|
||||
matplotlib==3.10.5
|
||||
matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1733416936468/work
|
||||
mdit-py-plugins==0.5.0
|
||||
mdurl==0.1.2
|
||||
mistune @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_mistune_1756495311/work
|
||||
mpmath==1.3.0
|
||||
msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1648745999384/work
|
||||
multidict @ file:///home/conda/feedstock_root/build_artifacts/multidict_1648882415384/work
|
||||
multipledispatch @ file:///home/conda/feedstock_root/build_artifacts/multipledispatch_1721907546485/work
|
||||
narwhals==2.3.0
|
||||
nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1734628800805/work
|
||||
nbconvert @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_nbconvert-core_1738067871/work
|
||||
nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1733402752141/work
|
||||
nest_asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1733325553580/work
|
||||
netCDF4 @ file:///croot/netcdf4_1743512888672/work
|
||||
networkx @ file:///croot/networkx_1737039604450/work
|
||||
notebook @ file:///home/conda/feedstock_root/build_artifacts/notebook_1759152069573/work
|
||||
notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1733408315203/work
|
||||
numba @ file:///croot/numba_1750798165355/work
|
||||
numcodecs @ file:///croot/numcodecs_1707513121886/work
|
||||
numexpr @ file:///croot/numexpr_1755766469354/work
|
||||
numpy @ file:///croot/numpy_and_numpy_base_1755590845055/work/dist/numpy-1.26.4-cp310-cp310-linux_x86_64.whl#sha256=1096d33ad9a9757a1b4b46634d809e894263fc8b78780bff36801684b6e8cc88
|
||||
nvidia-cublas-cu12==12.8.4.1
|
||||
nvidia-cuda-cupti-cu12==12.8.90
|
||||
nvidia-cuda-nvrtc-cu12==12.8.93
|
||||
nvidia-cuda-runtime-cu12==12.8.90
|
||||
nvidia-cudnn-cu12==9.10.2.21
|
||||
nvidia-cufft-cu12==11.3.3.83
|
||||
nvidia-cufile-cu12==1.13.1.3
|
||||
nvidia-curand-cu12==10.3.9.90
|
||||
nvidia-cusolver-cu12==11.7.3.90
|
||||
nvidia-cusparse-cu12==12.5.8.93
|
||||
nvidia-cusparselt-cu12==0.7.1
|
||||
nvidia-nccl-cu12==2.27.3
|
||||
nvidia-nvjitlink-cu12==12.8.93
|
||||
nvidia-nvtx-cu12==12.8.90
|
||||
odc-algo==0.2.3
|
||||
odc-geo==0.4.10
|
||||
odc-io==0.2.2
|
||||
odc-loader @ file:///home/conda/feedstock_root/build_artifacts/odc-loader_1743656085024/work
|
||||
odc-stac @ file:///home/conda/feedstock_root/build_artifacts/odc-stac_1746136311934/work
|
||||
odc-ui==0.2.1
|
||||
orderly-set==5.5.0
|
||||
overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1734587627321/work
|
||||
OWSLib==0.34.1
|
||||
packaging @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_packaging_1745345660/work
|
||||
pandas @ file:///home/task_175982153789305/conda-bld/pandas_1759822248912/work/dist/pandas-2.3.3-cp310-cp310-linux_x86_64.whl#sha256=0de7c83109c411cc2a74419a396c92f65e3d1e457fb4d835e5f100cfb04393a7
|
||||
pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work
|
||||
panel==1.7.5
|
||||
param==2.2.1
|
||||
parso @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_parso_1755974222/work
|
||||
partd @ file:///home/conda/feedstock_root/build_artifacts/partd_1715026491486/work
|
||||
pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1733301927746/work
|
||||
pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1733327343728/work
|
||||
pillow @ file:///croot/pillow_1738010226202/work
|
||||
PIMS==0.7
|
||||
planetary-computer==1.0.0
|
||||
platformdirs @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_platformdirs_1756227402/work
|
||||
prometheus_client==0.22.1
|
||||
prometheus_flask_exporter==0.23.2
|
||||
prompt_toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1756321756983/work
|
||||
propcache==0.3.2
|
||||
psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1653089181607/work
|
||||
psycopg2 @ file:///croot/psycopg2_1744919787325/work
|
||||
ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1733302279685/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl#sha256=92c32ff62b5fd8cf325bec5ab90d7be3d2a8ca8c8a3813ff487a8d2002630d1f
|
||||
pure_eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1733569405015/work
|
||||
pyarrow @ file:///home/task_175983338836370/conda-bld/pyarrow_1759833584228/work/python
|
||||
pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
|
||||
pyct==0.5.0
|
||||
pydantic==2.11.7
|
||||
pydantic_core==2.33.2
|
||||
Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1750615794071/work
|
||||
pyogrio @ file:///croot/pyogrio_1741107161422/work
|
||||
pyows==0.3.1
|
||||
pyparsing @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_pyparsing_1753873557/work
|
||||
pyproj @ file:///croot/pyproj_1739284761968/work
|
||||
PyQt6==6.7.1
|
||||
PyQt6_sip @ file:///croot/pyqt-split_1753427276959/work/pyqt_sip
|
||||
pyshp==2.3.1
|
||||
PySocks @ file:///home/builder/ci_310/pysocks_1640793678128/work
|
||||
pystac @ file:///home/conda/feedstock_root/build_artifacts/pystac_1758218055393/work
|
||||
pystac-client==0.9.0
|
||||
python-dateutil==2.9.0.post0
|
||||
python-dotenv==1.1.1
|
||||
python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work
|
||||
python-slugify==8.0.4
|
||||
pyTMD==2.2.8
|
||||
pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1742920838005/work
|
||||
pyviz_comms==3.0.6
|
||||
PyYAML==6.0.2
|
||||
pyzmq @ file:///croot/pyzmq_1734687138743/work
|
||||
rasterio @ file:///croot/rasterio_1740069178893/work
|
||||
rasterstats==0.20.0
|
||||
referencing==0.36.2
|
||||
regex==2025.9.1
|
||||
requests @ file:///croot/requests_1756709366904/work
|
||||
rfc3339_validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1733599910982/work
|
||||
rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work
|
||||
rfc3987==1.3.8
|
||||
rfc3987-syntax @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_rfc3987-syntax_1752876729/work
|
||||
rioxarray @ file:///home/conda/feedstock_root/build_artifacts/rioxarray_1737140588464/work
|
||||
rpds-py @ file:///croot/rpds-py_1736541261634/work
|
||||
ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1649033201098/work
|
||||
ruamel.yaml.clib==0.2.12
|
||||
s3fs==2025.9.0
|
||||
s3transfer==0.13.1
|
||||
scikit-image==0.25.2
|
||||
scikit-learn==1.7.1
|
||||
scipy @ file:///croot/scipy_1747238027288/work/dist/scipy-1.15.3-cp310-cp310-linux_x86_64.whl#sha256=2a791554880ad4f358fcc4cd2a982ffe1e9d472e9241011216b2be797457f1f9
|
||||
seaborn==0.13.2
|
||||
Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1733322040660/work
|
||||
setuptools-scm==9.2.0
|
||||
shapely @ file:///croot/shapely_1754380812723/work
|
||||
simplejson==3.20.1
|
||||
sip @ file:///croot/sip_1738856193618/work
|
||||
six==1.17.0
|
||||
slicerator==1.1.0
|
||||
sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1733244044561/work
|
||||
snuggs @ file:///home/conda/feedstock_root/build_artifacts/snuggs_1733818638588/work
|
||||
sortedcontainers @ file:///home/conda/feedstock_root/build_artifacts/sortedcontainers_1738440353519/work
|
||||
soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1756330469801/work
|
||||
sparse @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_sparse_1747799051/work
|
||||
SQLAlchemy==1.4.54
|
||||
stack_data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1733569443808/work
|
||||
starlette==0.50.0
|
||||
sympy==1.14.0
|
||||
tblib @ file:///home/conda/feedstock_root/build_artifacts/tblib_1743515515538/work
|
||||
terminado @ file:///home/conda/feedstock_root/build_artifacts/terminado_1710262609923/work
|
||||
text-unidecode==1.3
|
||||
threadpoolctl @ file:///home/conda/feedstock_root/build_artifacts/threadpoolctl_1741878222898/work
|
||||
tifffile==2025.5.10
|
||||
timescale==0.0.9
|
||||
timezonefinder==8.0.0
|
||||
tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1729802851396/work
|
||||
tomli @ file:///croot/tomli_1753774587605/work
|
||||
toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1733736030883/work
|
||||
torch==2.8.0
|
||||
tornado @ file:///croot/tornado_1748956929273/work
|
||||
tqdm==4.67.1
|
||||
traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1733367359838/work
|
||||
traittypes==0.2.1
|
||||
triton==3.4.0
|
||||
types-python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/types-python-dateutil_1759899809376/work
|
||||
typing-inspection==0.4.1
|
||||
typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/bld/rattler-build_typing_extensions_1756220668/work
|
||||
typing_utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1733331286120/work
|
||||
tzdata @ file:///croot/python-tzdata_1746123641790/work
|
||||
uc-micro-py==1.0.3
|
||||
unicodedata2 @ file:///croot/unicodedata2_1736541023050/work
|
||||
uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1733323593477/work/dist
|
||||
urllib3 @ file:///croot/urllib3_1750775463400/work
|
||||
uvicorn==0.38.0
|
||||
wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1733231326287/work
|
||||
webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1733359735138/work
|
||||
webencodings @ file:///home/conda/feedstock_root/build_artifacts/webencodings_1733236011802/work
|
||||
websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1759928050786/work
|
||||
Werkzeug==3.1.3
|
||||
widgetsnbextension==4.0.14
|
||||
wrapt @ file:///home/conda/feedstock_root/build_artifacts/wrapt_1651495243689/work
|
||||
xarray @ file:///home/conda/feedstock_root/build_artifacts/xarray_1749743207754/work
|
||||
xgboost==3.1.2
|
||||
xyzservices @ file:///croot/xyzservices_1675159059961/work
|
||||
yarl==1.20.1
|
||||
zarr @ file:///home/conda/feedstock_root/build_artifacts/zarr_1733237197728/work
|
||||
zict @ file:///home/conda/feedstock_root/build_artifacts/zict_1733261551178/work
|
||||
zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1749421620841/work
|
||||
@@ -0,0 +1,3 @@
|
||||
fastapi
|
||||
uvicorn
|
||||
pydantic
|
||||
@@ -0,0 +1 @@
|
||||
uvicorn api_server:app --reload --host 0.0.0.0 --port 8000
|
||||
+2368
File diff suppressed because one or more lines are too long
+558
@@ -0,0 +1,558 @@
|
||||
"""
|
||||
Training module for land classification using Sentinel-2 and Sentinel-1 data
|
||||
from Microsoft Planetary Computer STAC API
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import xarray as xr
|
||||
import geopandas as gpd
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from sklearn.metrics import classification_report, confusion_matrix
|
||||
from sklearn.ensemble import RandomForestClassifier
|
||||
from sklearn.tree import DecisionTreeClassifier
|
||||
from sklearn.svm import SVC
|
||||
from xgboost import XGBClassifier
|
||||
import joblib
|
||||
from datetime import datetime
|
||||
import json
|
||||
import os
|
||||
import warnings
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
# PyTorch for CNN
|
||||
try:
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
from torch.utils.data import TensorDataset, DataLoader
|
||||
PYTORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
PYTORCH_AVAILABLE = False
|
||||
print("Warning: PyTorch not available. CNN model will not work.")
|
||||
|
||||
# Define CNN model class for PyTorch
|
||||
class CNNClassifier(nn.Module):
|
||||
def __init__(self, n_features, n_classes):
|
||||
super(CNNClassifier, self).__init__()
|
||||
self.n_features = n_features
|
||||
self.n_classes = n_classes
|
||||
|
||||
# For small feature sets (like 3 features), use simpler architecture
|
||||
if n_features < 8:
|
||||
# Simple fully connected network for small features
|
||||
self.use_conv = False
|
||||
self.fc1 = nn.Linear(n_features, 64)
|
||||
self.dropout1 = nn.Dropout(0.3)
|
||||
self.fc2 = nn.Linear(64, 128)
|
||||
self.dropout2 = nn.Dropout(0.5)
|
||||
self.fc3 = nn.Linear(128, n_classes)
|
||||
else:
|
||||
# CNN architecture for larger feature sets
|
||||
self.use_conv = True
|
||||
self.conv1 = nn.Conv1d(in_channels=1, out_channels=32, kernel_size=3, padding=1)
|
||||
self.pool1 = nn.MaxPool1d(kernel_size=2)
|
||||
self.conv2 = nn.Conv1d(in_channels=32, out_channels=64, kernel_size=3, padding=1)
|
||||
self.pool2 = nn.MaxPool1d(kernel_size=2)
|
||||
|
||||
# Calculate size after convolutions
|
||||
conv_output_size = (n_features // 2 // 2) * 64
|
||||
|
||||
# Fully connected layers
|
||||
self.fc1 = nn.Linear(conv_output_size, 128)
|
||||
self.dropout = nn.Dropout(0.5)
|
||||
self.fc2 = nn.Linear(128, n_classes)
|
||||
|
||||
def forward(self, x):
|
||||
# x shape: (batch, n_features) or (batch, 1, n_features)
|
||||
if self.use_conv:
|
||||
# CNN path for larger feature sets
|
||||
if len(x.shape) == 2:
|
||||
x = x.unsqueeze(1) # Add channel dimension
|
||||
x = F.relu(self.conv1(x))
|
||||
x = self.pool1(x)
|
||||
x = F.relu(self.conv2(x))
|
||||
x = self.pool2(x)
|
||||
x = x.view(x.size(0), -1) # Flatten
|
||||
x = F.relu(self.fc1(x))
|
||||
x = self.dropout(x)
|
||||
x = self.fc2(x)
|
||||
else:
|
||||
# Fully connected path for small feature sets
|
||||
if len(x.shape) == 3:
|
||||
x = x.squeeze(1) # Remove channel dimension if present
|
||||
x = F.relu(self.fc1(x))
|
||||
x = self.dropout1(x)
|
||||
x = F.relu(self.fc2(x))
|
||||
x = self.dropout2(x)
|
||||
x = self.fc3(x)
|
||||
return x
|
||||
|
||||
def predict(self, X):
|
||||
"""Scikit-learn style predict method"""
|
||||
self.eval()
|
||||
with torch.no_grad():
|
||||
if isinstance(X, np.ndarray):
|
||||
X = torch.FloatTensor(X)
|
||||
# Handle both 2D and 3D inputs
|
||||
if not self.use_conv and len(X.shape) == 3:
|
||||
X = X.squeeze(1)
|
||||
elif self.use_conv and len(X.shape) == 2:
|
||||
X = X.unsqueeze(1)
|
||||
outputs = self(X)
|
||||
_, predicted = torch.max(outputs, 1)
|
||||
return predicted.cpu().numpy()
|
||||
|
||||
def score(self, X, y):
|
||||
"""Scikit-learn style score method"""
|
||||
predictions = self.predict(X)
|
||||
if isinstance(y, torch.Tensor):
|
||||
y = y.cpu().numpy()
|
||||
return np.mean(predictions == y)
|
||||
|
||||
# Microsoft Planetary Computer imports
|
||||
import planetary_computer
|
||||
from pystac_client import Client
|
||||
from odc.stac import load as stac_load
|
||||
|
||||
|
||||
def train_model(
|
||||
bbox=[105.6, 9.3, 106.2, 9.8],
|
||||
time_range='2023-03-01/2023-05-31',
|
||||
max_scenes=12,
|
||||
cloud_cover=30,
|
||||
resolution=20,
|
||||
training_shapefile='train/ST_training data_updated_1130points_new.shp',
|
||||
model_type='xgboost',
|
||||
n_estimators=100,
|
||||
max_depth=20,
|
||||
learning_rate=0.1,
|
||||
use_gpu=True,
|
||||
use_cache=True,
|
||||
test_size=0.2,
|
||||
output_model_path=None,
|
||||
status_callback=None,
|
||||
cancel_check=None
|
||||
):
|
||||
"""
|
||||
Train a land classification model using Sentinel-2 and Sentinel-1 data
|
||||
|
||||
Args:
|
||||
bbox: [min_lon, min_lat, max_lon, max_lat]
|
||||
time_range: "YYYY-MM-DD/YYYY-MM-DD"
|
||||
max_scenes: maximum number of scenes to load
|
||||
cloud_cover: maximum cloud cover percentage
|
||||
resolution: resolution in meters (e.g., 20)
|
||||
training_shapefile: path to training shapefile
|
||||
n_estimators: number of trees for XGBoost
|
||||
max_depth: maximum tree depth
|
||||
learning_rate: learning rate for XGBoost
|
||||
use_gpu: whether to use GPU for training
|
||||
output_model_path: path to save trained model (auto-generated if None)
|
||||
status_callback: Optional callback function to report progress
|
||||
cancel_check: Optional function that returns True if training should be cancelled
|
||||
test_size: Fraction of data to use for test set (0-1)
|
||||
|
||||
Returns:
|
||||
Dictionary containing training results
|
||||
"""
|
||||
|
||||
def update_status(message, progress=None):
|
||||
"""Helper to update status"""
|
||||
if status_callback:
|
||||
# Try calling with both arguments, fallback to just message
|
||||
try:
|
||||
status_callback(message, progress)
|
||||
except TypeError:
|
||||
status_callback(message)
|
||||
print(message)
|
||||
|
||||
def check_cancellation():
|
||||
"""Check if training should be cancelled"""
|
||||
if cancel_check and cancel_check():
|
||||
raise InterruptedError("Training cancelled by user")
|
||||
|
||||
try:
|
||||
# Auto-generate output path if not provided
|
||||
if output_model_path is None:
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
output_model_path = f'model_train/model_{model_type}_{timestamp}.joblib'
|
||||
|
||||
# ============ CACHE SYSTEM ============
|
||||
# Create cache directory
|
||||
cache_dir = Path("dataset_cache")
|
||||
cache_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Generate cache key from parameters
|
||||
cache_params = f"{bbox}_{time_range}_{max_scenes}_{cloud_cover}_{resolution}"
|
||||
cache_key = hashlib.md5(cache_params.encode()).hexdigest()
|
||||
cache_file = cache_dir / f"training_data_{cache_key}.joblib"
|
||||
|
||||
features = None
|
||||
labels = None
|
||||
|
||||
# Try to load from cache
|
||||
if use_cache and cache_file.exists():
|
||||
update_status(f"📦 Loading cached dataset from {cache_file.name}...", 5)
|
||||
try:
|
||||
cached_data = joblib.load(cache_file)
|
||||
features = cached_data['features']
|
||||
labels = cached_data['labels']
|
||||
update_status(f"✅ Loaded {len(features)} samples from cache (skipped satellite download!)", 50)
|
||||
except Exception as e:
|
||||
update_status(f"⚠️ Cache load failed: {str(e)}, downloading fresh data...", 10)
|
||||
features = None
|
||||
|
||||
# If no cache or cache failed, download data
|
||||
if features is None:
|
||||
update_status("📡 Cache not found or disabled, downloading satellite data...", 10)
|
||||
|
||||
# Connect to Microsoft Planetary Computer
|
||||
update_status("Connecting to Microsoft Planetary Computer...", 12)
|
||||
catalog = Client.open("https://planetarycomputer.microsoft.com/api/stac/v1")
|
||||
check_cancellation()
|
||||
|
||||
# Search for Sentinel-2 scenes
|
||||
update_status("Searching for Sentinel-2 scenes...", 10)
|
||||
query_s2 = catalog.search(
|
||||
collections=["sentinel-2-l2a"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
query={"eo:cloud_cover": {"lt": cloud_cover}}
|
||||
)
|
||||
items_s2 = list(query_s2.item_collection())
|
||||
|
||||
check_cancellation()
|
||||
|
||||
# Limit scenes
|
||||
if len(items_s2) > max_scenes:
|
||||
step = len(items_s2) // max_scenes
|
||||
items_s2 = items_s2[::step][:max_scenes]
|
||||
|
||||
update_status(f"Found {len(items_s2)} Sentinel-2 scenes", 20)
|
||||
|
||||
# Sign and load Sentinel-2 data
|
||||
update_status("Loading Sentinel-2 data...", 25)
|
||||
items_s2 = [planetary_computer.sign(item) for item in items_s2]
|
||||
ds_s2 = stac_load(
|
||||
items_s2,
|
||||
bands=["B04", "B08", "SCL"],
|
||||
crs="EPSG:32648",
|
||||
resolution=resolution,
|
||||
bbox=bbox,
|
||||
patch_url=planetary_computer.sign,
|
||||
fail_on_error=False,
|
||||
)
|
||||
ds_s2 = ds_s2.rename({"B04": "red", "B08": "nir", "SCL": "scl"})
|
||||
|
||||
check_cancellation()
|
||||
|
||||
# Search for Sentinel-1 scenes
|
||||
update_status("Searching for Sentinel-1 scenes...", 35)
|
||||
query_s1 = catalog.search(
|
||||
collections=["sentinel-1-rtc"],
|
||||
bbox=bbox,
|
||||
datetime=time_range,
|
||||
)
|
||||
items_s1 = list(query_s1.item_collection())
|
||||
|
||||
# Limit scenes
|
||||
if len(items_s1) > max_scenes:
|
||||
step = len(items_s1) // max_scenes
|
||||
items_s1 = items_s1[::step][:max_scenes]
|
||||
|
||||
update_status(f"Found {len(items_s1)} Sentinel-1 scenes", 40)
|
||||
|
||||
# Sign and load Sentinel-1 data
|
||||
update_status("Loading Sentinel-1 data...", 45)
|
||||
items_s1 = [planetary_computer.sign(item) for item in items_s1]
|
||||
ds_s1 = stac_load(
|
||||
items_s1,
|
||||
bands=["vv", "vh"],
|
||||
crs="EPSG:32648",
|
||||
resolution=resolution,
|
||||
bbox=bbox,
|
||||
patch_url=planetary_computer.sign,
|
||||
fail_on_error=False,
|
||||
)
|
||||
|
||||
# Convert to dB
|
||||
ds_s1['vv_db'] = 10 * np.log10(ds_s1['vv'].where(ds_s1['vv'] > 0))
|
||||
ds_s1['vh_db'] = 10 * np.log10(ds_s1['vh'].where(ds_s1['vh'] > 0))
|
||||
|
||||
check_cancellation()
|
||||
|
||||
# Calculate NDVI
|
||||
update_status("Calculating NDVI...", 50)
|
||||
ndvi = (ds_s2['nir'] - ds_s2['red']) / (ds_s2['nir'] + ds_s2['red'] + 1e-8)
|
||||
|
||||
# Apply cloud mask
|
||||
cloud_mask = ds_s2['scl'].isin([1, 3, 8, 9, 10])
|
||||
ndvi_masked = ndvi.where(~cloud_mask)
|
||||
ndvi_mean = ndvi_masked.mean(dim='time')
|
||||
|
||||
# Load training data
|
||||
update_status("Loading training data...", 55)
|
||||
train_gdf = gpd.read_file(training_shapefile)
|
||||
|
||||
if train_gdf.crs != 'EPSG:32648':
|
||||
train_gdf = train_gdf.to_crs('EPSG:32648')
|
||||
|
||||
# Auto-detect label column
|
||||
label_column = None
|
||||
for col in ['HT_code', 'Ma_LU', 'LU2022', 'Hientrang', 'class', 'Class', 'CLASS']:
|
||||
if col in train_gdf.columns:
|
||||
label_column = col
|
||||
break
|
||||
|
||||
if label_column is None:
|
||||
raise ValueError(f"Cannot find label column in shapefile. Available: {list(train_gdf.columns)}")
|
||||
|
||||
# Extract features
|
||||
update_status("Extracting features from training points...", 60)
|
||||
features = []
|
||||
labels = []
|
||||
|
||||
for idx, row in train_gdf.iterrows():
|
||||
point = row.geometry
|
||||
x_coord = point.x
|
||||
y_coord = point.y
|
||||
label = row[label_column]
|
||||
|
||||
try:
|
||||
ndvi_val = ndvi_mean.sel(x=x_coord, y=y_coord, method='nearest').values
|
||||
vh_val = ds_s1['vh_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values
|
||||
vv_val = ds_s1['vv_db'].sel(x=x_coord, y=y_coord, method='nearest').mean(dim='time').values
|
||||
|
||||
feature_vec = [ndvi_val, vh_val, vv_val]
|
||||
|
||||
if not np.isnan(feature_vec).any():
|
||||
features.append(feature_vec)
|
||||
labels.append(label)
|
||||
except:
|
||||
continue
|
||||
|
||||
features = np.array(features)
|
||||
labels = np.array(labels)
|
||||
|
||||
check_cancellation()
|
||||
|
||||
update_status(f"Extracted {len(features)} valid training samples", 70)
|
||||
|
||||
# ============ SAVE TO CACHE ============
|
||||
if use_cache:
|
||||
update_status(f"💾 Saving dataset to cache for future use...", 72)
|
||||
try:
|
||||
cache_data = {
|
||||
'features': features,
|
||||
'labels': labels,
|
||||
'bbox': bbox,
|
||||
'time_range': time_range,
|
||||
'resolution': resolution,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
}
|
||||
joblib.dump(cache_data, cache_file)
|
||||
update_status(f"✅ Cached to {cache_file.name}", 75)
|
||||
except Exception as e:
|
||||
update_status(f"⚠️ Cache save failed: {str(e)}", 75)
|
||||
|
||||
# Encode labels
|
||||
label_encoder = LabelEncoder()
|
||||
labels_encoded = label_encoder.fit_transform(labels)
|
||||
|
||||
# Split data
|
||||
X_train, X_test, y_train, y_test = train_test_split(
|
||||
features, labels_encoded, test_size=test_size, random_state=42, stratify=labels_encoded
|
||||
)
|
||||
|
||||
# Train model based on selected type
|
||||
update_status(f"Training {model_type.upper()} model...", 75)
|
||||
|
||||
device = 'cuda:0' if use_gpu else 'cpu'
|
||||
|
||||
if model_type == 'xgboost':
|
||||
model = XGBClassifier(
|
||||
n_estimators=n_estimators,
|
||||
max_depth=max_depth,
|
||||
learning_rate=learning_rate,
|
||||
device=device if use_gpu else 'cpu',
|
||||
tree_method='hist',
|
||||
random_state=42,
|
||||
eval_metric='mlogloss',
|
||||
verbosity=0
|
||||
)
|
||||
elif model_type == 'random_forest':
|
||||
model = RandomForestClassifier(
|
||||
n_estimators=n_estimators,
|
||||
max_depth=max_depth,
|
||||
random_state=42,
|
||||
n_jobs=-1, # Use all cores
|
||||
verbose=0
|
||||
)
|
||||
elif model_type == 'decision_tree':
|
||||
model = DecisionTreeClassifier(
|
||||
max_depth=max_depth,
|
||||
random_state=42
|
||||
)
|
||||
elif model_type == 'svm':
|
||||
model = SVC(
|
||||
kernel='rbf',
|
||||
random_state=42,
|
||||
verbose=False
|
||||
)
|
||||
elif model_type == 'cnn':
|
||||
if not PYTORCH_AVAILABLE:
|
||||
raise ImportError("PyTorch is required for CNN. Install: pip install torch")
|
||||
|
||||
# CNN requires reshaping data
|
||||
n_features = X_train.shape[1]
|
||||
n_classes = len(np.unique(y_train))
|
||||
|
||||
# Build PyTorch CNN model
|
||||
device = torch.device('cuda' if torch.cuda.is_available() and use_gpu else 'cpu')
|
||||
update_status(f"Building CNN model on {device}...", 75)
|
||||
|
||||
model = CNNClassifier(n_features, n_classes).to(device)
|
||||
|
||||
# Convert to PyTorch tensors
|
||||
X_train_tensor = torch.FloatTensor(X_train).unsqueeze(1) # Add channel dim: (N, 1, features)
|
||||
y_train_tensor = torch.LongTensor(y_train)
|
||||
X_test_tensor = torch.FloatTensor(X_test).unsqueeze(1)
|
||||
y_test_tensor = torch.LongTensor(y_test)
|
||||
|
||||
# Create data loaders
|
||||
train_dataset = TensorDataset(X_train_tensor, y_train_tensor)
|
||||
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
|
||||
|
||||
# Loss and optimizer
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
||||
|
||||
# Train CNN
|
||||
update_status("Training CNN model with PyTorch...", 80)
|
||||
epochs = min(50, n_estimators // 2) # Use n_estimators as epochs
|
||||
|
||||
model.train()
|
||||
for epoch in range(epochs):
|
||||
epoch_loss = 0.0
|
||||
for batch_X, batch_y in train_loader:
|
||||
batch_X, batch_y = batch_X.to(device), batch_y.to(device)
|
||||
|
||||
optimizer.zero_grad()
|
||||
outputs = model(batch_X)
|
||||
loss = criterion(outputs, batch_y)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
|
||||
if (epoch + 1) % 10 == 0:
|
||||
avg_loss = epoch_loss / len(train_loader)
|
||||
update_status(f"CNN Epoch {epoch+1}/{epochs}, Loss: {avg_loss:.4f}", 80 + (epoch / epochs) * 10)
|
||||
|
||||
# Move model to CPU for saving (compatible with non-GPU systems)
|
||||
model = model.cpu()
|
||||
model.device_used = str(device)
|
||||
else:
|
||||
raise ValueError(f"Unknown model type: {model_type}. Choose: xgboost, random_forest, decision_tree, svm, cnn")
|
||||
|
||||
# Fit non-CNN models
|
||||
if model_type != 'cnn':
|
||||
model.fit(X_train, y_train)
|
||||
|
||||
# Evaluate
|
||||
update_status("Evaluating model...", 90)
|
||||
if model_type == 'cnn':
|
||||
# PyTorch CNN evaluation
|
||||
train_score = model.score(X_train, y_train)
|
||||
test_score = model.score(X_test, y_test)
|
||||
y_pred = model.predict(X_test)
|
||||
else:
|
||||
train_score = model.score(X_train, y_train)
|
||||
test_score = model.score(X_test, y_test)
|
||||
y_pred = model.predict(X_test)
|
||||
|
||||
# Generate classification report and confusion matrix
|
||||
update_status("Generating classification report...", 92)
|
||||
class_names = label_encoder.classes_.tolist()
|
||||
|
||||
# Classification report as dict
|
||||
from sklearn.metrics import classification_report, confusion_matrix
|
||||
cls_report = classification_report(y_test, y_pred, target_names=class_names, output_dict=True, zero_division=0)
|
||||
|
||||
# Confusion matrix
|
||||
conf_matrix = confusion_matrix(y_test, y_pred).tolist()
|
||||
|
||||
# Save model
|
||||
update_status("Saving model...", 95)
|
||||
os.makedirs(os.path.dirname(output_model_path), exist_ok=True)
|
||||
joblib.dump({'model': model, 'label_encoder': label_encoder}, output_model_path)
|
||||
|
||||
# Save model info
|
||||
info = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"data_source": "Microsoft Planetary Computer STAC",
|
||||
"collections": ["sentinel-2-l2a", "sentinel-1-rtc"],
|
||||
"features": ["NDVI_mean", "VH_dB_mean", "VV_dB_mean"],
|
||||
"training_samples": len(X_train),
|
||||
"testing_samples": len(X_test),
|
||||
"test_size": test_size,
|
||||
"train_accuracy": float(train_score),
|
||||
"test_accuracy": float(test_score),
|
||||
"model_type": model_type,
|
||||
"device": device if model_type == 'xgboost' else 'cpu',
|
||||
"n_estimators": n_estimators if model_type in ['xgboost', 'random_forest', 'cnn'] else None,
|
||||
"max_depth": max_depth if model_type != 'cnn' else None,
|
||||
"learning_rate": learning_rate if model_type == 'xgboost' else None,
|
||||
"cnn_epochs": min(50, n_estimators // 2) if model_type == 'cnn' else None,
|
||||
"n_features": X_train.shape[1],
|
||||
"n_classes": len(np.unique(y_train)),
|
||||
"class_names": class_names,
|
||||
"classification_report": cls_report,
|
||||
"confusion_matrix": conf_matrix,
|
||||
"bbox": bbox,
|
||||
"time_range": time_range,
|
||||
"resolution": resolution
|
||||
}
|
||||
|
||||
info_path = output_model_path.replace('.joblib', '_info.json')
|
||||
with open(info_path, 'w') as f:
|
||||
json.dump(info, f, indent=2)
|
||||
|
||||
update_status("Training complete!", 100)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"model_path": output_model_path,
|
||||
"info_path": info_path,
|
||||
"train_accuracy": train_score,
|
||||
"test_accuracy": test_score,
|
||||
"training_samples": len(X_train),
|
||||
"testing_samples": len(X_test),
|
||||
"test_size": test_size,
|
||||
"classes": class_names,
|
||||
"classification_report": cls_report,
|
||||
"confusion_matrix": conf_matrix,
|
||||
"model_type": model_type,
|
||||
"bbox": bbox,
|
||||
"time_range": time_range,
|
||||
"resolution": resolution
|
||||
}
|
||||
|
||||
except InterruptedError as e:
|
||||
update_status(f"Cancelled: {str(e)}", -1)
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"cancelled": True
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
update_status(f"Error: {str(e)}", -1)
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user