mirror of
https://git.victorphan.net/basketballcantho/CSIROBoeingPhase5-Vietnam.git
synced 2026-08-05 21:53:12 +07:00
Victor Phan
469 lines
15 KiB
Plaintext
469 lines
15 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "302d4929",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Predict on Local Machine\n",
|
|
"Sử dụng model đã train để dự đoán trên toàn bộ dataset\n",
|
|
"\n",
|
|
"**Yêu cầu**: \n",
|
|
"- Đã huấn luyện model (chạy notebook 02.train_CNN_PyTorch_local.ipynb)\n",
|
|
"- Có file `model_cnn_pytorch_full.pt`"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5247872a",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import os\n",
|
|
"import numpy as np\n",
|
|
"import xarray as xr\n",
|
|
"import torch\n",
|
|
"import torch.nn as nn\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"from torch.utils.data import DataLoader, TensorDataset\n",
|
|
"import json\n",
|
|
"\n",
|
|
"print(f\"PyTorch version: {torch.__version__}\")\n",
|
|
"print(f\"GPU available: {torch.cuda.is_available()}\")\n",
|
|
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
|
"print(f\"Device: {device}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "7fb6ebae",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Load Model"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "316ac485",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Định nghĩa CNN model (giống như khi training)\n",
|
|
"class CNNClassifier(nn.Module):\n",
|
|
" def __init__(self, input_size, num_classes=8):\n",
|
|
" super(CNNClassifier, self).__init__()\n",
|
|
" \n",
|
|
" # Conv blocks\n",
|
|
" self.conv1 = nn.Conv1d(1, 64, kernel_size=3, padding=1)\n",
|
|
" self.bn1 = nn.BatchNorm1d(64)\n",
|
|
" self.conv2 = nn.Conv1d(64, 64, kernel_size=3, padding=1)\n",
|
|
" self.bn2 = nn.BatchNorm1d(64)\n",
|
|
" self.pool1 = nn.MaxPool1d(2)\n",
|
|
" self.drop1 = nn.Dropout(0.25)\n",
|
|
" \n",
|
|
" self.conv3 = nn.Conv1d(64, 128, kernel_size=3, padding=1)\n",
|
|
" self.bn3 = nn.BatchNorm1d(128)\n",
|
|
" self.conv4 = nn.Conv1d(128, 128, kernel_size=3, padding=1)\n",
|
|
" self.bn4 = nn.BatchNorm1d(128)\n",
|
|
" self.pool2 = nn.MaxPool1d(2)\n",
|
|
" self.drop2 = nn.Dropout(0.25)\n",
|
|
" \n",
|
|
" self.conv5 = nn.Conv1d(128, 256, kernel_size=3, padding=1)\n",
|
|
" self.bn5 = nn.BatchNorm1d(256)\n",
|
|
" self.conv6 = nn.Conv1d(256, 256, kernel_size=3, padding=1)\n",
|
|
" self.bn6 = nn.BatchNorm1d(256)\n",
|
|
" self.global_avg_pool = nn.AdaptiveAvgPool1d(1)\n",
|
|
" self.drop3 = nn.Dropout(0.25)\n",
|
|
" \n",
|
|
" # FC layers\n",
|
|
" self.fc1 = nn.Linear(256, 128)\n",
|
|
" self.bn_fc1 = nn.BatchNorm1d(128)\n",
|
|
" self.drop_fc1 = nn.Dropout(0.5)\n",
|
|
" \n",
|
|
" self.fc2 = nn.Linear(128, 64)\n",
|
|
" self.bn_fc2 = nn.BatchNorm1d(64)\n",
|
|
" self.drop_fc2 = nn.Dropout(0.5)\n",
|
|
" \n",
|
|
" self.fc3 = nn.Linear(64, num_classes)\n",
|
|
" \n",
|
|
" self.relu = nn.ReLU()\n",
|
|
" \n",
|
|
" def forward(self, x):\n",
|
|
" # Block 1\n",
|
|
" x = self.relu(self.bn1(self.conv1(x)))\n",
|
|
" x = self.relu(self.bn2(self.conv2(x)))\n",
|
|
" x = self.pool1(x)\n",
|
|
" x = self.drop1(x)\n",
|
|
" \n",
|
|
" # Block 2\n",
|
|
" x = self.relu(self.bn3(self.conv3(x)))\n",
|
|
" x = self.relu(self.bn4(self.conv4(x)))\n",
|
|
" x = self.pool2(x)\n",
|
|
" x = self.drop2(x)\n",
|
|
" \n",
|
|
" # Block 3\n",
|
|
" x = self.relu(self.bn5(self.conv5(x)))\n",
|
|
" x = self.relu(self.bn6(self.conv6(x)))\n",
|
|
" x = self.global_avg_pool(x)\n",
|
|
" x = x.squeeze(-1)\n",
|
|
" x = self.drop3(x)\n",
|
|
" \n",
|
|
" # FC layers\n",
|
|
" x = self.relu(self.bn_fc1(self.fc1(x)))\n",
|
|
" x = self.drop_fc1(x)\n",
|
|
" x = self.relu(self.bn_fc2(self.fc2(x)))\n",
|
|
" x = self.drop_fc2(x)\n",
|
|
" x = self.fc3(x)\n",
|
|
" \n",
|
|
" return x\n",
|
|
"\n",
|
|
"print(\"✅ Model class defined\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "eb73cfbd",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load model\n",
|
|
"print(\"📥 Load model...\")\n",
|
|
"model_path = 'model_cnn_pytorch_full.pt'\n",
|
|
"\n",
|
|
"if not os.path.exists(model_path):\n",
|
|
" raise FileNotFoundError(f\"❌ Model file '{model_path}' not found!\")\n",
|
|
"\n",
|
|
"model_info = torch.load(model_path, map_location=device)\n",
|
|
"num_classes = model_info['num_classes']\n",
|
|
"input_size = model_info['input_size']\n",
|
|
"label_mapping = model_info['label_mapping']\n",
|
|
"mean = model_info['mean']\n",
|
|
"std = model_info['std']\n",
|
|
"test_accuracy = model_info['test_accuracy']\n",
|
|
"test_loss = model_info['test_loss']\n",
|
|
"\n",
|
|
"# Khởi tạo model\n",
|
|
"model = CNNClassifier(input_size=input_size, num_classes=num_classes)\n",
|
|
"model.load_state_dict(model_info['state_dict'])\n",
|
|
"model = model.to(device)\n",
|
|
"model.eval()\n",
|
|
"\n",
|
|
"print(f\"✅ Model loaded\")\n",
|
|
"print(f\" Num classes: {num_classes}\")\n",
|
|
"print(f\" Input size: {input_size}\")\n",
|
|
"print(f\" Test Accuracy: {test_accuracy:.2f}%\")\n",
|
|
"print(f\"\\n📋 Label mapping:\")\n",
|
|
"for label, idx in sorted(label_mapping.items(), key=lambda x: x[1]):\n",
|
|
" print(f\" {idx}: {label}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "52872fca",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Load Data từ Server"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "72411a5c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load dữ liệu\n",
|
|
"data_dir = \"data_for_training\"\n",
|
|
"\n",
|
|
"print(\"📥 Load dữ liệu...\")\n",
|
|
"average_ndvi = xr.open_dataarray(os.path.join(data_dir, \"average_ndvi.nc\"))\n",
|
|
"average_vv = xr.open_dataarray(os.path.join(data_dir, \"average_vv.nc\"))\n",
|
|
"average_vh = xr.open_dataarray(os.path.join(data_dir, \"average_vh.nc\"))\n",
|
|
"\n",
|
|
"print(f\"✅ NDVI shape: {average_ndvi.shape}\")\n",
|
|
"print(f\"✅ VV shape: {average_vv.shape}\")\n",
|
|
"print(f\"✅ VH shape: {average_vh.shape}\")\n",
|
|
"\n",
|
|
"# Hiển thị thông tin\n",
|
|
"print(f\"\\n📊 Spatial coordinates:\")\n",
|
|
"print(f\" X: {average_ndvi.x.values.min():.4f} to {average_ndvi.x.values.max():.4f}\")\n",
|
|
"print(f\" Y: {average_ndvi.y.values.min():.4f} to {average_ndvi.y.values.max():.4f}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "cc3da280",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Predict trên toàn bộ Dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "17d7f8f1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Chuẩn bị dữ liệu prediction\n",
|
|
"print(\"🔧 Chuẩn bị dữ liệu prediction...\")\n",
|
|
"\n",
|
|
"# Reshape dữ liệu thành grid\n",
|
|
"height = average_ndvi.shape[1]\n",
|
|
"width = average_ndvi.shape[2]\n",
|
|
"\n",
|
|
"X_pred = []\n",
|
|
"for i in range(height):\n",
|
|
" for j in range(width):\n",
|
|
" # Lấy giá trị từ mỗi band tại vị trí (i, j)\n",
|
|
" ndvi_val = average_ndvi.values[:, i, j]\n",
|
|
" vv_val = average_vv.values[:, i, j]\n",
|
|
" vh_val = average_vh.values[:, i, j]\n",
|
|
" \n",
|
|
" # Kết hợp các band\n",
|
|
" pixel_data = np.concatenate((ndvi_val, vv_val, vh_val))\n",
|
|
" X_pred.append(pixel_data)\n",
|
|
"\n",
|
|
"X_pred = np.array(X_pred)\n",
|
|
"print(f\"✅ Prediction data shape: {X_pred.shape}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "45a1d60c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Normalize dữ liệu\n",
|
|
"print(\"🔧 Normalize dữ liệu...\")\n",
|
|
"X_pred_normalized = (X_pred - mean) / (std + 1e-8)\n",
|
|
"print(f\"✅ Data normalized\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1f2e5743",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Convert to tensor và predict\n",
|
|
"print(\"🚀 Predict...\")\n",
|
|
"X_pred_tensor = torch.FloatTensor(X_pred_normalized).unsqueeze(1)\n",
|
|
"\n",
|
|
"pred_dataset = TensorDataset(X_pred_tensor)\n",
|
|
"pred_loader = DataLoader(pred_dataset, batch_size=128, shuffle=False)\n",
|
|
"\n",
|
|
"predictions = []\n",
|
|
"with torch.no_grad():\n",
|
|
" for batch_idx, (X_batch,) in enumerate(pred_loader):\n",
|
|
" X_batch = X_batch.to(device)\n",
|
|
" outputs = model(X_batch)\n",
|
|
" _, predicted = torch.max(outputs, 1)\n",
|
|
" predictions.extend(predicted.cpu().numpy())\n",
|
|
" \n",
|
|
" if (batch_idx + 1) % 10 == 0:\n",
|
|
" print(f\" Processed {min((batch_idx + 1) * 128, len(X_pred))} / {len(X_pred)} pixels\")\n",
|
|
"\n",
|
|
"predictions = np.array(predictions)\n",
|
|
"print(f\"\\n✅ Prediction completed!\")\n",
|
|
"print(f\" Total predictions: {len(predictions)}\")\n",
|
|
"print(f\" Class distribution: {np.bincount(predictions)}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "62b43164",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Reshape predictions thành spatial grid\n",
|
|
"print(\"🔧 Reshape predictions...\")\n",
|
|
"pred_map = predictions.reshape(height, width)\n",
|
|
"print(f\"✅ Prediction map shape: {pred_map.shape}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6c159982",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Tạo xarray DataArray để dễ lưu\n",
|
|
"print(\"🔧 Tạo xarray DataArray...\")\n",
|
|
"pred_xarray = xr.DataArray(\n",
|
|
" pred_map,\n",
|
|
" coords={'y': average_ndvi.y.values, 'x': average_ndvi.x.values},\n",
|
|
" dims=['y', 'x'],\n",
|
|
" name='land_use_class'\n",
|
|
")\n",
|
|
"\n",
|
|
"# Copy CRS từ original data\n",
|
|
"if hasattr(average_ndvi, 'rio'):\n",
|
|
" pred_xarray = pred_xarray.rio.write_crs(average_ndvi.rio.crs)\n",
|
|
"\n",
|
|
"print(f\"✅ DataArray created\")\n",
|
|
"print(f\" Shape: {pred_xarray.shape}\")\n",
|
|
"if hasattr(pred_xarray, 'rio') and pred_xarray.rio.crs:\n",
|
|
" print(f\" CRS: {pred_xarray.rio.crs}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"id": "04600814",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Visualize và Save Results"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "e8ae5031",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Visualize prediction map\n",
|
|
"print(\"📊 Visualize prediction map...\")\n",
|
|
"fig, ax = plt.subplots(figsize=(12, 10))\n",
|
|
"\n",
|
|
"# Reverse label mapping để display\n",
|
|
"idx_to_label = {v: k for k, v in label_mapping.items()}\n",
|
|
"\n",
|
|
"# Vẽ\n",
|
|
"im = ax.imshow(pred_map, cmap='tab10', interpolation='nearest')\n",
|
|
"ax.set_title('Predicted Land Use Classification', fontsize=14, fontweight='bold')\n",
|
|
"ax.set_xlabel('X')\n",
|
|
"ax.set_ylabel('Y')\n",
|
|
"\n",
|
|
"# Colorbar\n",
|
|
"cbar = plt.colorbar(im, ax=ax)\n",
|
|
"cbar.set_label('Class Index')\n",
|
|
"\n",
|
|
"plt.tight_layout()\n",
|
|
"plt.savefig('prediction_map.png', dpi=150, bbox_inches='tight')\n",
|
|
"plt.show()\n",
|
|
"\n",
|
|
"print(\"✅ Prediction map saved: prediction_map.png\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "f88b461c",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Save prediction as NetCDF\n",
|
|
"print(\"💾 Save prediction as NetCDF...\")\n",
|
|
"output_file = 'land_use_prediction.nc'\n",
|
|
"pred_xarray.to_netcdf(output_file)\n",
|
|
"print(f\"✅ Saved: {output_file}\")\n",
|
|
"print(f\" Size: {os.path.getsize(output_file) / 1024**2:.2f} MB\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "efffef9d",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Save prediction as GeoTIFF (nếu có rasterio)\n",
|
|
"try:\n",
|
|
" import rasterio\n",
|
|
" from rasterio.transform import Affine\n",
|
|
" \n",
|
|
" print(\"💾 Save prediction as GeoTIFF...\")\n",
|
|
" output_tiff = 'land_use_prediction.tif'\n",
|
|
" \n",
|
|
" # Calculate transform\n",
|
|
" x_res = (average_ndvi.x.values[1] - average_ndvi.x.values[0])\n",
|
|
" y_res = (average_ndvi.y.values[1] - average_ndvi.y.values[0])\n",
|
|
" x_min = average_ndvi.x.values[0] - x_res / 2\n",
|
|
" y_max = average_ndvi.y.values[0] - y_res / 2\n",
|
|
" \n",
|
|
" transform = Affine.translation(x_min, y_max) * Affine.scale(x_res, y_res)\n",
|
|
" \n",
|
|
" # Write to GeoTIFF\n",
|
|
" with rasterio.open(\n",
|
|
" output_tiff, 'w',\n",
|
|
" driver='GTiff',\n",
|
|
" height=pred_map.shape[0],\n",
|
|
" width=pred_map.shape[1],\n",
|
|
" count=1,\n",
|
|
" dtype=pred_map.dtype,\n",
|
|
" transform=transform,\n",
|
|
" crs='EPSG:32648' # Thay đổi CRS nếu cần\n",
|
|
" ) as dst:\n",
|
|
" dst.write(pred_map, 1)\n",
|
|
" \n",
|
|
" print(f\"✅ Saved: {output_tiff}\")\n",
|
|
" print(f\" Size: {os.path.getsize(output_tiff) / 1024**2:.2f} MB\")\n",
|
|
"except ImportError:\n",
|
|
" print(\"⚠️ rasterio not installed, skipping GeoTIFF export\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "724a41a0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Lưu metadata\n",
|
|
"print(\"💾 Save metadata...\")\n",
|
|
"metadata = {\n",
|
|
" 'model_type': 'CNN PyTorch',\n",
|
|
" 'num_classes': num_classes,\n",
|
|
" 'label_mapping': label_mapping,\n",
|
|
" 'test_accuracy': float(test_accuracy),\n",
|
|
" 'test_loss': float(test_loss),\n",
|
|
" 'prediction_map_shape': pred_map.shape,\n",
|
|
" 'class_distribution': {int(k): int(v) for k, v in zip(*np.unique(pred_map, return_counts=True))},\n",
|
|
" 'x_range': [float(average_ndvi.x.values.min()), float(average_ndvi.x.values.max())],\n",
|
|
" 'y_range': [float(average_ndvi.y.values.min()), float(average_ndvi.y.values.max())]\n",
|
|
"}\n",
|
|
"\n",
|
|
"import json\n",
|
|
"with open('prediction_metadata.json', 'w') as f:\n",
|
|
" json.dump(metadata, f, indent=2)\n",
|
|
"\n",
|
|
"print(\"✅ Metadata saved: prediction_metadata.json\")\n",
|
|
"print(json.dumps(metadata, indent=2))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "33c761ae",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"print(\"\\n\" + \"=\"*50)\n",
|
|
"print(\"✅ PREDICTION COMPLETED!\")\n",
|
|
"print(\"=\"*50)\n",
|
|
"print(f\"\\n📁 Output files:\")\n",
|
|
"print(f\" 1. land_use_prediction.nc (NetCDF)\")\n",
|
|
"print(f\" 2. land_use_prediction.tif (GeoTIFF) - if rasterio available\")\n",
|
|
"print(f\" 3. prediction_map.png (visualization)\")\n",
|
|
"print(f\" 4. prediction_metadata.json (metadata)\")\n",
|
|
"print(f\"\\n🚀 Next steps: Upload these files back to server if needed\")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|