update toàn bộ các mô hình
This commit is contained in:
Victor Phan
File diff suppressed because one or more lines are too long
@@ -7,11 +7,26 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import importlib\n",
|
||||
"import new_import_ODC as odc_tools\n",
|
||||
"importlib.reload(odc_tools)\n",
|
||||
"from new_import_ODC import *\n",
|
||||
"print(\"✅ Import thành công\")\n"
|
||||
"# Import libraries for Element84 Earth Search\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import xarray as xr\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import seaborn as sns\n",
|
||||
"\n",
|
||||
"# Element84 Earth Search STAC\n",
|
||||
"import pystac_client\n",
|
||||
"from odc.stac import load\n",
|
||||
"\n",
|
||||
"# Machine Learning\n",
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch.optim as optim\n",
|
||||
"from torch.utils.data import TensorDataset, DataLoader\n",
|
||||
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"\n",
|
||||
"print(\"✅ Import thành công\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -21,11 +36,17 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Khởi tạo Dask + Datacube + S3\n",
|
||||
"cluster, client = initialize_dask(use_gateway=True)\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"configure_s3_access(aws_unsigned=True)\n",
|
||||
"print(\"✅ Dask + Datacube + S3 sẵn sàng\")\n"
|
||||
"# Kết nối Element84 Earth Search (hosted trên AWS)\n",
|
||||
"def connect_earth_search():\n",
|
||||
" \"\"\"Kết nối đến Element84 Earth Search STAC API\"\"\"\n",
|
||||
" catalog = pystac_client.Client.open(\n",
|
||||
" \"https://earth-search.aws.element84.com/v1\"\n",
|
||||
" )\n",
|
||||
" return catalog\n",
|
||||
"\n",
|
||||
"catalog = connect_earth_search()\n",
|
||||
"print(\"✅ Element84 Earth Search kết nối thành công\")\n",
|
||||
"print(f\" API: earth-search.aws.element84.com\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -36,17 +57,36 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cấu hình vùng và thời gian\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"data_sen2 = load_data(\n",
|
||||
" dc=dc,\n",
|
||||
" date_range=date_range,\n",
|
||||
" longtitude_range=longtitude_range,\n",
|
||||
" latitude_range=latitude_range,\n",
|
||||
"# Tạo bounding box\n",
|
||||
"bbox = (longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1])\n",
|
||||
"\n",
|
||||
"# Query Sentinel-2 từ Element84\n",
|
||||
"search = catalog.search(\n",
|
||||
" collections=[\"sentinel-2-l2a\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
" datetime=f\"{date_range[0]}/{date_range[1]}\",\n",
|
||||
" query={\"eo:cloud_cover\": {\"lt\": 30}}\n",
|
||||
")\n",
|
||||
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n"
|
||||
"\n",
|
||||
"items = search.item_collection()\n",
|
||||
"print(f\"✅ Tìm thấy {len(items)} scenes Sentinel-2\")\n",
|
||||
"\n",
|
||||
"# Load data với odc-stac\n",
|
||||
"data_sen2 = load(\n",
|
||||
" items,\n",
|
||||
" bands=[\"red\", \"green\", \"blue\", \"nir\", \"swir16\", \"swir22\", \"scl\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
" resolution=10,\n",
|
||||
" chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n",
|
||||
" groupby=\"solar_day\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"✅ Sentinel-2 raw: {data_sen2.dims}\")\n",
|
||||
"print(f\" Variables: {list(data_sen2.data_vars)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -57,11 +97,41 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
|
||||
"def mask_clean(ds):\n",
|
||||
" \"\"\"Cloud masking sử dụng SCL band (Scene Classification Layer)\"\"\"\n",
|
||||
" if \"scl\" not in ds:\n",
|
||||
" print(\"⚠️ Không có SCL band, bỏ qua cloud masking\")\n",
|
||||
" return ds\n",
|
||||
" \n",
|
||||
" cloud_mask = (\n",
|
||||
" (ds[\"scl\"] == 3) | # cloud shadow\n",
|
||||
" (ds[\"scl\"] == 8) | # cloud medium probability\n",
|
||||
" (ds[\"scl\"] == 9) | # cloud high probability\n",
|
||||
" (ds[\"scl\"] == 10) # thin cirrus\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" ds_masked = ds.where(~cloud_mask)\n",
|
||||
" return ds_masked.drop_vars(\"scl\", errors=\"ignore\")\n",
|
||||
"\n",
|
||||
"def calculate_indices(ds, index=\"NDVI\", satellite_mission=\"s2\"):\n",
|
||||
" \"\"\"Tính chỉ số NDVI cho Sentinel-2\"\"\"\n",
|
||||
" if index == \"NDVI\":\n",
|
||||
" ndvi = (ds[\"nir\"] - ds[\"red\"]) / (ds[\"nir\"] + ds[\"red\"] + 1e-8)\n",
|
||||
" ds[\"NDVI\"] = ndvi\n",
|
||||
" return ds\n",
|
||||
"\n",
|
||||
"def fill_nan(ds):\n",
|
||||
" \"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\n",
|
||||
" return ds.interpolate_na(dim=\"time\", method=\"linear\", fill_value=\"extrapolate\")\n",
|
||||
"\n",
|
||||
"# Áp dụng tiền xử lý\n",
|
||||
"data_clean = mask_clean(data_sen2)\n",
|
||||
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\")\n",
|
||||
"data_fill = fill_nan(data_ndvi)\n",
|
||||
"data_ndvi = calculate_indices(data_clean, index=\"NDVI\", satellite_mission=\"s2\")\n",
|
||||
"data_fill = fill_nan(data_ndvi)\n",
|
||||
"data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n",
|
||||
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n"
|
||||
"\n",
|
||||
"print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n",
|
||||
"print(f\" Variables: {list(data_sen2_monthly.data_vars)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -71,15 +141,48 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tải Sentinel-1 (SAR VV/VH)\n",
|
||||
"data_sen1 = load_data_sen1(\n",
|
||||
" dc=dc,\n",
|
||||
" date_range=date_range,\n",
|
||||
" longtitude_range=longtitude_range,\n",
|
||||
" latitude_range=latitude_range,\n",
|
||||
"# Tải Sentinel-1 (SAR VV/VH) từ Element84\n",
|
||||
"search_s1 = catalog.search(\n",
|
||||
" collections=[\"sentinel-1-grd\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
" datetime=f\"{date_range[0]}/{date_range[1]}\",\n",
|
||||
" query={\n",
|
||||
" \"sat:orbit_state\": {\"eq\": \"descending\"},\n",
|
||||
" \"sar:product_type\": {\"eq\": \"GRD\"}\n",
|
||||
" }\n",
|
||||
")\n",
|
||||
"data_sen1_monthly = calculate_average(data_sen1, [\"VV\", \"VH\"], resample=\"1MS\").compute()\n",
|
||||
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n"
|
||||
"\n",
|
||||
"items_s1 = search_s1.item_collection()\n",
|
||||
"print(f\"✅ Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n",
|
||||
"\n",
|
||||
"if len(items_s1) > 0:\n",
|
||||
" data_sen1 = load(\n",
|
||||
" items_s1,\n",
|
||||
" bands=[\"vv\", \"vh\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
" resolution=10,\n",
|
||||
" chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n",
|
||||
" groupby=\"solar_day\"\n",
|
||||
" )\n",
|
||||
" data_sen1_monthly = data_sen1.resample(time=\"1MS\").mean().compute()\n",
|
||||
"else:\n",
|
||||
" # Tạo dummy data nếu không có S1\n",
|
||||
" print(\"⚠️ Không có Sentinel-1, tạo dummy data\")\n",
|
||||
" data_sen1_monthly = xr.Dataset({\n",
|
||||
" \"vv\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" ),\n",
|
||||
" \"vh\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" )\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n",
|
||||
"print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -97,6 +200,76 @@
|
||||
" \"TS\": \"4\", \"Song\": \"5\", \"Dat xay dung\": \"6\", \"Rung\": \"7\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"def load_train_data(csv_path=\"/media/x79/2A7D-FAA0/remote-sensing/train_data.csv\", label_mapping=None):\n",
|
||||
" \"\"\"Load training points từ CSV\"\"\"\n",
|
||||
" df = pd.read_csv(csv_path)\n",
|
||||
" if label_mapping:\n",
|
||||
" df[\"label\"] = df[\"class_name\"].map(label_mapping).astype(str)\n",
|
||||
" return df\n",
|
||||
"\n",
|
||||
"def get_data_sen1_and_sen2(train_df, sen2_data, sen1_data):\n",
|
||||
" \"\"\"Extract features từ S2 và S1 tại các điểm training\"\"\"\n",
|
||||
" X_list = []\n",
|
||||
" y_list = []\n",
|
||||
" \n",
|
||||
" for idx, row in train_df.iterrows():\n",
|
||||
" try:\n",
|
||||
" lon, lat = float(row[\"longitude\"]), float(row[\"latitude\"])\n",
|
||||
" label = int(row[\"label\"])\n",
|
||||
" \n",
|
||||
" # Extract S2 features\n",
|
||||
" s2_point = sen2_data.sel(x=lon, y=lat, method=\"nearest\")\n",
|
||||
" s2_features = []\n",
|
||||
" \n",
|
||||
" for var in [\"red\", \"green\", \"blue\", \"nir\", \"swir16\", \"swir22\", \"NDVI\"]:\n",
|
||||
" if var in s2_point:\n",
|
||||
" vals = s2_point[var].values\n",
|
||||
" if vals.size > 0:\n",
|
||||
" s2_features.extend([\n",
|
||||
" np.nanmean(vals), np.nanstd(vals),\n",
|
||||
" np.nanmin(vals), np.nanmax(vals)\n",
|
||||
" ])\n",
|
||||
" else:\n",
|
||||
" s2_features.extend([0, 0, 0, 0])\n",
|
||||
" \n",
|
||||
" # Extract S1 features\n",
|
||||
" s1_point = sen1_data.sel(x=lon, y=lat, method=\"nearest\")\n",
|
||||
" s1_features = []\n",
|
||||
" \n",
|
||||
" for var in [\"vv\", \"vh\"]:\n",
|
||||
" if var in s1_point:\n",
|
||||
" vals = s1_point[var].values\n",
|
||||
" if vals.size > 0:\n",
|
||||
" s1_features.extend([np.nanmean(vals), np.nanstd(vals)])\n",
|
||||
" else:\n",
|
||||
" s1_features.extend([0, 0])\n",
|
||||
" \n",
|
||||
" features = s2_features + s1_features\n",
|
||||
" \n",
|
||||
" if not np.isnan(features).any() and not np.isinf(features).any():\n",
|
||||
" X_list.append(features)\n",
|
||||
" y_list.append(label)\n",
|
||||
" except Exception as e:\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" return X_list, y_list\n",
|
||||
"\n",
|
||||
"def split_train_data(X, y, test_size=0.2, val_size=0.1, random_state=42):\n",
|
||||
" \"\"\"Split data thành train/val/test\"\"\"\n",
|
||||
" X = np.array(X, dtype=np.float32)\n",
|
||||
" y = np.array(y, dtype=np.int64)\n",
|
||||
" \n",
|
||||
" X_train, X_temp, y_train, y_temp = train_test_split(\n",
|
||||
" X, y, test_size=test_size + val_size, random_state=random_state, stratify=y\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" val_ratio = val_size / (test_size + val_size)\n",
|
||||
" X_val, X_test, y_val, y_test = train_test_split(\n",
|
||||
" X_temp, y_temp, test_size=(1 - val_ratio), random_state=random_state, stratify=y_temp\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" return X_train, X_val, X_test, y_train, y_val, y_test\n",
|
||||
"\n",
|
||||
"train_data = load_train_data(label_mapping=label_mapping)\n",
|
||||
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
|
||||
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
|
||||
@@ -112,7 +285,7 @@
|
||||
"n_classes = len(np.unique(y_train_np))\n",
|
||||
"\n",
|
||||
"print(f\"✅ Train: {X_train_np.shape} Val: {X_val_np.shape} Test: {X_test_np.shape}\")\n",
|
||||
"print(f\" n_features={n_features} n_classes={n_classes}\")\n"
|
||||
"print(f\" n_features={n_features} n_classes={n_classes}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
Executable → Regular
Reference in New Issue
Block a user