thêm chức năng train trên odc predict trên planetary

This commit is contained in:
Victor Phan
2026-03-04 23:14:48 +07:00
parent 8a1e7bb22e
commit add7d16ecf
2 changed files with 353 additions and 178 deletions
+127
View File
@@ -9,10 +9,12 @@ Usage:
import xarray as xr
import numpy as np
import pandas as pd
import geopandas as gpd
from pystac_client import Client
import planetary_computer
import odc.stac
from datetime import datetime
from sklearn.model_selection import train_test_split
def load_sentinel2_stac(
@@ -292,6 +294,127 @@ def load_and_process_s1(bbox, date_range):
return data_monthly
# ══════════════════════════════════════════════════════════════════════════════
# TRAINING DATA FUNCTIONS
# ══════════════════════════════════════════════════════════════════════════════
def load_train_data(train_path, label_mapping=None):
"""
Load training data from shapefile or GeoJSON
Args:
train_path: Path to training data file (.shp, .geojson, etc.)
label_mapping: Optional dict to map labels to numeric IDs
Returns:
GeoDataFrame with training points
"""
print(f"📂 Loading training data from: {train_path}")
try:
train = gpd.read_file(train_path)
print(f"✅ Loaded {len(train)} training points")
print(f" Columns: {list(train.columns)}")
if label_mapping is not None and 'label' in train.columns:
train['label_id'] = train['label'].map(label_mapping).astype(int)
print(f" Labels mapped: {sorted(train['label_id'].unique())}")
return train
except Exception as e:
print(f"❌ Error loading training data: {e}")
return None
def extract_features_at_points(train_data, data_s2, data_s1):
"""
Extract Sentinel-1 and Sentinel-2 features at training point locations
Args:
train_data: GeoDataFrame with training points
data_s2: xarray Dataset with Sentinel-2 data
data_s1: xarray Dataset with Sentinel-1 data
Returns:
X (features), y (labels)
"""
print(f"🔧 Extracting features from satellite data at {len(train_data)} points...")
X_list = []
y_list = []
for idx, point in train_data.iterrows():
try:
lon, lat = point.geometry.x, point.geometry.y
# Extract S2 data at point location
s2_point = data_s2.sel(x=lon, y=lat, method='nearest')
s2_values = s2_point.to_array().values.flatten()
# Extract S1 data at point location
s1_point = data_s1.sel(x=lon, y=lat, method='nearest')
s1_values = s1_point.to_array().values.flatten()
# Combine features
features = np.concatenate([s2_values, s1_values])
# Skip if contains NaN
if not np.isnan(features).any():
X_list.append(features)
y_list.append(point['label_id'])
else:
print(f" ⚠ Skip point {idx}: contains NaN")
except Exception as e:
print(f" ⚠ Skip point {idx}: {e}")
continue
X = np.array(X_list)
y = np.array(y_list)
print(f"✅ Extracted features from {len(X)} points")
print(f" Feature dimension: {X.shape[1]}")
print(f" Classes: {sorted(set(y.tolist()))}")
return X, y
def split_train_data(X, y, test_size=0.2, val_size=0.1, random_state=42):
"""
Split data into train/val/test sets
Args:
X: Features array
y: Labels array
test_size: Proportion for test set
val_size: Proportion for validation set (from train+val)
random_state: Random seed
Returns:
X_train, X_val, X_test, y_train, y_val, y_test
"""
print(f"📊 Splitting data...")
# First split: train+val vs test
X_temp, X_test, y_temp, y_test = train_test_split(
X, y, test_size=test_size, random_state=random_state, stratify=y
)
# Second split: train vs val
val_ratio = val_size / (1 - test_size)
X_train, X_val, y_train, y_val = train_test_split(
X_temp, y_temp, test_size=val_ratio, random_state=random_state, stratify=y_temp
)
print(f"✅ Data split complete:")
print(f" Train: {len(X_train)} samples ({len(X_train)/len(X)*100:.1f}%)")
print(f" Val: {len(X_val)} samples ({len(X_val)/len(X)*100:.1f}%)")
print(f" Test: {len(X_test)} samples ({len(X_test)/len(X)*100:.1f}%)")
return X_train, X_val, X_test, y_train, y_val, y_test
# Print module info
print("=" * 70)
print("📦 Data Loading Module (No ODC Database Required)")
@@ -302,5 +425,9 @@ print("\n bbox = (lon_min, lat_min, lon_max, lat_max)")
print(" date_range = ('2022-09-01', '2023-10-01')")
print("\n data_s2 = load_and_process_s2(bbox, date_range)")
print(" data_s1 = load_and_process_s1(bbox, date_range)")
print("\n # Load training data")
print(" train = load_train_data('train/data.shp', label_mapping)")
print(" X, y = extract_features_at_points(train, data_s2, data_s1)")
print(" X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y)")
print("=" * 70)
print()
@@ -244,6 +244,8 @@
"metadata": {},
"outputs": [],
"source": [
"from load_data_no_odc import load_train_data\n",
"\n",
"# Ánh xạ nhãn lớp đất\n",
"label_mapping = {\n",
" \"Lua tom\": 0,\n",
@@ -260,12 +262,18 @@
"for label, idx in label_mapping.items():\n",
" print(f\" {idx}: {label}\")\n",
"\n",
"# TODO: Load training data from shapefile/GeoJSON\n",
"# Bạn cần cung cấp đường dẫn đến file training data\n",
"train_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/train_data.geojson\" # Thay đổi path này\n",
"# Load training data from AWS shapefile\n",
"train_path = \"train/ST_training_data_updated_1130points_new.shp\"\n",
"\n",
"print(f\"\\n📂 Loading training data from: {train_path}\")\n",
"print(\"⚠ Note: Bạn cần cập nhật train_path với đường dẫn thực tế\")"
"train_data = load_train_data(train_path, label_mapping)\n",
"\n",
"if train_data is not None:\n",
" print(f\"\\n✅ Training data loaded successfully!\")\n",
" print(f\" Total points: {len(train_data)}\")\n",
" print(f\" CRS: {train_data.crs}\")\n",
"else:\n",
" print(\"❌ Failed to load training data\")\n"
]
},
{
@@ -285,46 +293,40 @@
"metadata": {},
"outputs": [],
"source": [
"# Placeholder for feature extraction\n",
"# Bạn cần implement hàm extract features từ train points\n",
"%%time\n",
"from load_data_no_odc import extract_features_at_points\n",
"\n",
"def extract_features(train_data, data_s2, data_s1):\n",
" \"\"\"\n",
" Extract features from S2 and S1 data at training point locations\n",
" \"\"\"\n",
" X_list = []\n",
" y_list = []\n",
"print(\"🔧 Extracting features from satellite data...\")\n",
"print(\"-\" * 70)\n",
"\n",
"if train_data is not None and \\\n",
" data_sen2_monthly is not None and \\\n",
" data_sen1_monthly is not None:\n",
" \n",
" for idx, point in train_data.iterrows():\n",
" try:\n",
" lon, lat = point.geometry.x, point.geometry.y\n",
" \n",
" # Extract S2 data\n",
" s2_values = data_s2.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
" \n",
" # Extract S1 data\n",
" s1_values = data_s1.sel(x=lon, y=lat, method='nearest').to_array().values.flatten()\n",
" \n",
" # Combine features\n",
" features = np.concatenate([s2_values, s1_values])\n",
" \n",
" # Skip if contains NaN\n",
" if not np.isnan(features).any():\n",
" X_list.append(features)\n",
" y_list.append(point['label_id'])\n",
" \n",
" except Exception as e:\n",
" continue\n",
" # Extract features at training point locations\n",
" X, y = extract_features_at_points(\n",
" train_data, \n",
" data_sen2_monthly, \n",
" data_sen1_monthly\n",
" )\n",
" \n",
" return np.array(X_list), np.array(y_list)\n",
"\n",
"# TODO: Uncomment when training data is available\n",
"# X, y = extract_features(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"# print(f\"✅ Extracted {len(X)} training samples\")\n",
"# print(f\" Features: {X.shape[1]}\")\n",
"# print(f\" Classes: {sorted(set(y.tolist()))}\")\n",
"\n",
"print(\"⚠ Feature extraction step - waiting for training data\")"
" print(f\"\\n✅ Feature extraction complete!\")\n",
" print(f\" Total samples: {len(X)}\")\n",
" print(f\" Feature dimension: {X.shape[1]}\")\n",
" print(f\" Classes: {sorted(set(y.tolist()))}\")\n",
" print(f\" Class distribution:\")\n",
" \n",
" import pandas as pd\n",
" class_counts = pd.Series(y).value_counts().sort_index()\n",
" for class_id, count in class_counts.items():\n",
" class_name = [k for k, v in label_mapping.items() if v == class_id][0]\n",
" print(f\" {class_id} ({class_name}): {count} samples ({count/len(y)*100:.1f}%)\")\n",
" \n",
"else:\n",
" print(\"❌ Missing data: Please check training data or satellite data\")\n",
" print(f\" train_data: {'✓' if train_data is not None else '✗'}\")\n",
" print(f\" data_sen2_monthly: {'✓' if data_sen2_monthly is not None else '✗'}\")\n",
" print(f\" data_sen1_monthly: {'✓' if data_sen1_monthly is not None else '✗'}\")\n"
]
},
{
@@ -344,28 +346,29 @@
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when features are extracted\n",
"from load_data_no_odc import split_train_data\n",
"\n",
"# # Split train/val/test\n",
"# X_temp, X_test, y_temp, y_test = train_test_split(\n",
"# X, y, test_size=0.2, random_state=42, stratify=y\n",
"# )\n",
"\n",
"# X_train, X_val, y_train, y_val = train_test_split(\n",
"# X_temp, y_temp, test_size=0.125, random_state=42, stratify=y_temp\n",
"# )\n",
"\n",
"# # Combine train + val for final training\n",
"# X_fit = np.concatenate([X_train, X_val], axis=0)\n",
"# y_fit = np.concatenate([y_train, y_val], axis=0)\n",
"\n",
"# print(f\"✅ Data split:\")\n",
"# print(f\" Train: {len(X_train)} samples\")\n",
"# print(f\" Val: {len(X_val)} samples\")\n",
"# print(f\" Test: {len(X_test)} samples\")\n",
"# print(f\" Total: {len(X)} samples\")\n",
"\n",
"print(\"⚠ Data split step - waiting for features\")"
"if 'X' in locals() and 'y' in locals():\n",
" # Split data into train/val/test sets\n",
" X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n",
" X, y, \n",
" test_size=0.2, \n",
" val_size=0.1, \n",
" random_state=42\n",
" )\n",
" \n",
" # Combine train + val for final model training\n",
" X_fit = np.concatenate([X_train, X_val], axis=0)\n",
" y_fit = np.concatenate([y_train, y_val], axis=0)\n",
" \n",
" print(f\"\\n✅ Final training set:\")\n",
" print(f\" X_fit shape: {X_fit.shape}\")\n",
" print(f\" y_fit shape: {y_fit.shape}\")\n",
" print(f\"\\n X_test shape: {X_test.shape}\")\n",
" print(f\" y_test shape: {y_test.shape}\")\n",
" \n",
"else:\n",
" print(\"❌ Features not extracted yet. Please run Step 6 first.\")\n"
]
},
{
@@ -386,28 +389,36 @@
"outputs": [],
"source": [
"%%time\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"\n",
"# TODO: Uncomment when data is ready\n",
"\n",
"# # Train Decision Tree\n",
"# model = DecisionTreeClassifier(\n",
"# max_depth=30,\n",
"# min_samples_leaf=2,\n",
"# min_samples_split=5,\n",
"# class_weight=\"balanced\",\n",
"# random_state=42,\n",
"# )\n",
"\n",
"# print(\"🚀 Training Decision Tree...\")\n",
"# model.fit(X_fit, y_fit)\n",
"\n",
"# val_acc = model.score(X_val, y_val)\n",
"# print(f\"✅ Training complete!\")\n",
"# print(f\" Tree depth: {model.get_depth()}\")\n",
"# print(f\" Leaves: {model.get_n_leaves()}\")\n",
"# print(f\" Val accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
"\n",
"print(\"⚠ Training step - waiting for data\")"
"if 'X_fit' in locals() and 'y_fit' in locals():\n",
" \n",
" print(\"🚀 Training Decision Tree model...\")\n",
" print(\"-\" * 70)\n",
" \n",
" # Train Decision Tree with tuned hyperparameters\n",
" model = DecisionTreeClassifier(\n",
" max_depth=30,\n",
" min_samples_leaf=2,\n",
" min_samples_split=5,\n",
" class_weight=\"balanced\",\n",
" random_state=42,\n",
" )\n",
" \n",
" model.fit(X_fit, y_fit)\n",
" \n",
" # Evaluate on validation set\n",
" val_acc = model.score(X_val, y_val)\n",
" \n",
" print(f\"\\n✅ Training complete!\")\n",
" print(f\" Model: Decision Tree\")\n",
" print(f\" Tree depth: {model.get_depth()}\")\n",
" print(f\" Number of leaves: {model.get_n_leaves()}\")\n",
" print(f\" Training samples: {len(X_fit)}\")\n",
" print(f\" Validation accuracy: {val_acc:.4f} ({val_acc*100:.2f}%)\")\n",
" \n",
"else:\n",
" print(\"❌ Data not ready. Please run Step 7 first.\")\n"
]
},
{
@@ -428,50 +439,59 @@
"outputs": [],
"source": [
"%%time\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# TODO: Uncomment when model is trained\n",
"\n",
"# DEPTH_RANGE = list(range(1, 51))\n",
"# THRESHOLD = 0.001\n",
"\n",
"# train_accs = []\n",
"# val_accs = []\n",
"\n",
"# print(\"🔍 Analyzing convergence for max_depth...\")\n",
"# for d in DEPTH_RANGE:\n",
"# m = DecisionTreeClassifier(\n",
"# min_samples_leaf=2,\n",
"# min_samples_split=5,\n",
"# class_weight=\"balanced\",\n",
"# random_state=42,\n",
"# max_depth=d,\n",
"# )\n",
"# m.fit(X_fit, y_fit)\n",
"# train_accs.append(m.score(X_fit, y_fit))\n",
"# val_accs.append(m.score(X_val, y_val))\n",
"\n",
"# train_accs = np.array(train_accs)\n",
"# val_accs = np.array(val_accs)\n",
"\n",
"# best_depth = DEPTH_RANGE[np.argmax(val_accs)]\n",
"# best_val_acc = np.max(val_accs)\n",
"\n",
"# print(f\"✅ Best max_depth: {best_depth} (val_acc: {best_val_acc:.4f})\")\n",
"\n",
"# # Plot\n",
"# fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n",
"# ax.plot(DEPTH_RANGE, train_accs, 'b-o', markersize=3, label='Train')\n",
"# ax.plot(DEPTH_RANGE, val_accs, 'g-o', markersize=3, label='Val')\n",
"# ax.axvline(x=best_depth, color='red', linestyle='--', label=f'Best={best_depth}')\n",
"# ax.set_xlabel('max_depth')\n",
"# ax.set_ylabel('Accuracy')\n",
"# ax.set_title('Train/Val Accuracy vs max_depth')\n",
"# ax.legend()\n",
"# ax.grid(True, alpha=0.3)\n",
"# plt.tight_layout()\n",
"# plt.show()\n",
"\n",
"print(\"⚠ Hyperparameter analysis - waiting for model\")"
"if 'X_fit' in locals() and 'X_val' in locals():\n",
" \n",
" DEPTH_RANGE = list(range(1, 51))\n",
" \n",
" train_accs = []\n",
" val_accs = []\n",
" \n",
" print(\"🔍 Analyzing optimal max_depth for Decision Tree...\")\n",
" print(\"-\" * 70)\n",
" print(\"Testing depths from 1 to 50...\")\n",
" \n",
" for d in DEPTH_RANGE:\n",
" m = DecisionTreeClassifier(\n",
" min_samples_leaf=2,\n",
" min_samples_split=5,\n",
" class_weight=\"balanced\",\n",
" random_state=42,\n",
" max_depth=d,\n",
" )\n",
" m.fit(X_fit, y_fit)\n",
" train_accs.append(m.score(X_fit, y_fit))\n",
" val_accs.append(m.score(X_val, y_val))\n",
" \n",
" if d % 10 == 0:\n",
" print(f\" Depth {d:2d}: train={train_accs[-1]:.4f}, val={val_accs[-1]:.4f}\")\n",
" \n",
" train_accs = np.array(train_accs)\n",
" val_accs = np.array(val_accs)\n",
" \n",
" best_depth = DEPTH_RANGE[np.argmax(val_accs)]\n",
" best_val_acc = np.max(val_accs)\n",
" \n",
" print(f\"\\n✅ Optimal max_depth: {best_depth}\")\n",
" print(f\" Best validation accuracy: {best_val_acc:.4f} ({best_val_acc*100:.2f}%)\")\n",
" \n",
" # Plot convergence analysis\n",
" fig, ax = plt.subplots(1, 1, figsize=(12, 5))\n",
" ax.plot(DEPTH_RANGE, train_accs, 'b-o', markersize=3, label='Train Accuracy')\n",
" ax.plot(DEPTH_RANGE, val_accs, 'g-o', markersize=3, label='Validation Accuracy')\n",
" ax.axvline(x=best_depth, color='red', linestyle='--', linewidth=2, \n",
" label=f'Best Depth = {best_depth}')\n",
" ax.set_xlabel('max_depth', fontsize=12)\n",
" ax.set_ylabel('Accuracy', fontsize=12)\n",
" ax.set_title('Decision Tree: Training vs Validation Accuracy', fontsize=14, fontweight='bold')\n",
" ax.legend(fontsize=10)\n",
" ax.grid(True, alpha=0.3)\n",
" plt.tight_layout()\n",
" plt.show()\n",
" \n",
"else:\n",
" print(\"❌ Data not ready. Please run Step 7 first.\")\n"
]
},
{
@@ -491,29 +511,38 @@
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when model is trained\n",
"from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
"import seaborn as sns\n",
"\n",
"# # Predictions\n",
"# y_pred = model.predict(X_test)\n",
"# acc = accuracy_score(y_test, y_pred)\n",
"\n",
"# print(f\"📊 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
"# print(classification_report(y_test, y_pred, digits=4))\n",
"\n",
"# # Confusion Matrix\n",
"# class_names = list(label_mapping.keys())\n",
"# cm = confusion_matrix(y_test, y_pred)\n",
"\n",
"# plt.figure(figsize=(9, 7))\n",
"# sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',\n",
"# xticklabels=class_names, yticklabels=class_names)\n",
"# plt.xlabel('Predicted')\n",
"# plt.ylabel('Actual')\n",
"# plt.title('Confusion Matrix — Decision Tree')\n",
"# plt.tight_layout()\n",
"# plt.show()\n",
"\n",
"print(\"⚠ Evaluation step - waiting for model\")"
"if 'model' in locals() and 'X_test' in locals():\n",
" \n",
" print(\"📊 Evaluating model on test set...\")\n",
" print(\"-\" * 70)\n",
" \n",
" # Make predictions\n",
" y_pred = model.predict(X_test)\n",
" acc = accuracy_score(y_test, y_pred)\n",
" \n",
" print(f\"\\n🎯 Test Accuracy: {acc:.4f} ({acc*100:.2f}%)\\n\")\n",
" print(\"📋 Classification Report:\")\n",
" print(classification_report(y_test, y_pred, digits=4))\n",
" \n",
" # Confusion Matrix\n",
" class_names = list(label_mapping.keys())\n",
" cm = confusion_matrix(y_test, y_pred)\n",
" \n",
" plt.figure(figsize=(10, 8))\n",
" sns.heatmap(cm, annot=True, fmt='d', cmap='Greens',\n",
" xticklabels=class_names, yticklabels=class_names,\n",
" cbar_kws={'label': 'Count'})\n",
" plt.xlabel('Predicted Label', fontsize=12)\n",
" plt.ylabel('True Label', fontsize=12)\n",
" plt.title('Confusion Matrix — Decision Tree (Test Set)', fontsize=14, fontweight='bold')\n",
" plt.tight_layout()\n",
" plt.show()\n",
" \n",
"else:\n",
" print(\"❌ Model not trained yet. Please run Step 8 first.\")\n"
]
},
{
@@ -533,37 +562,56 @@
"metadata": {},
"outputs": [],
"source": [
"# TODO: Uncomment when model is trained\n",
"import joblib\n",
"import json\n",
"from datetime import datetime\n",
"\n",
"# # Save model\n",
"# model_path = \"model_decision_tree_planetary_computer.joblib\"\n",
"# joblib.dump(model, model_path)\n",
"# print(f\"✅ Model saved → {model_path}\")\n",
"\n",
"# # Save metadata\n",
"# info = {\n",
"# \"model_type\": \"DecisionTree\",\n",
"# \"data_source\": \"Microsoft Planetary Computer\",\n",
"# \"max_depth\": model.get_depth(),\n",
"# \"n_leaves\": model.get_n_leaves(),\n",
"# \"n_features\": int(X_fit.shape[1]),\n",
"# \"label_mapping\": label_mapping,\n",
"# \"test_accuracy\": float(acc),\n",
"# \"train_samples\": int(len(X_fit)),\n",
"# \"test_samples\": int(len(X_test)),\n",
"# \"bbox\": bbox,\n",
"# \"date_range\": date_range,\n",
"# \"saved_at\": datetime.now().isoformat(),\n",
"# }\n",
"\n",
"# info_path = \"model_decision_tree_planetary_computer_info.json\"\n",
"# with open(info_path, \"w\") as f:\n",
"# json.dump(info, f, indent=2, ensure_ascii=False)\n",
"\n",
"# print(f\"✅ Metadata saved → {info_path}\")\n",
"# print(json.dumps(info, indent=2, ensure_ascii=False))\n",
"\n",
"print(\"⚠ Save model step - waiting for trained model\")"
"if 'model' in locals() and 'acc' in locals():\n",
" \n",
" print(\"💾 Saving model and metadata...\")\n",
" print(\"-\" * 70)\n",
" \n",
" # Save model\n",
" model_path = \"model_decision_tree_planetary_computer.joblib\"\n",
" joblib.dump(model, model_path)\n",
" print(f\"✅ Model saved → {model_path}\")\n",
" \n",
" # Create metadata\n",
" info = {\n",
" \"model_type\": \"DecisionTree\",\n",
" \"data_source\": \"Microsoft Planetary Computer\",\n",
" \"training_data\": train_path,\n",
" \"max_depth\": int(model.get_depth()),\n",
" \"n_leaves\": int(model.get_n_leaves()),\n",
" \"n_features\": int(X_fit.shape[1]),\n",
" \"label_mapping\": label_mapping,\n",
" \"test_accuracy\": float(acc),\n",
" \"train_samples\": int(len(X_fit)),\n",
" \"val_samples\": int(len(X_val)),\n",
" \"test_samples\": int(len(X_test)),\n",
" \"bbox\": bbox,\n",
" \"date_range\": list(date_range),\n",
" \"saved_at\": datetime.now().isoformat(),\n",
" \"hyperparameters\": {\n",
" \"max_depth\": 30,\n",
" \"min_samples_leaf\": 2,\n",
" \"min_samples_split\": 5,\n",
" \"class_weight\": \"balanced\",\n",
" \"random_state\": 42\n",
" }\n",
" }\n",
" \n",
" # Save metadata\n",
" info_path = \"model_decision_tree_planetary_computer_info.json\"\n",
" with open(info_path, \"w\") as f:\n",
" json.dump(info, f, indent=2, ensure_ascii=False)\n",
" \n",
" print(f\"✅ Metadata saved → {info_path}\")\n",
" print(\"\\n📋 Model Info:\")\n",
" print(json.dumps(info, indent=2, ensure_ascii=False))\n",
" \n",
"else:\n",
" print(\"❌ Model not trained yet or evaluation not complete.\")\n"
]
},
{