basketballcantho/remote-sensing
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

lupdate MObileNet

Browse Source
This commit is contained in:
Victor Phan
2026-03-04 20:42:08 +07:00
parent 5310ae3f44
commit 4000a2c3b3
1 changed files with 110 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
train_files/01.train_ODC_MobileNet.ipynb
+103 -101
View File
@@ -2,25 +2,18 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": null,
"id": "17da4353", "id": "17da4353",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Import thành công\n"
]
}
],
"source": [ "source": [
"# Import libraries for Element84 Earth Search\n", "# Import libraries\n",
"import numpy as np\n", "import numpy as np\n",
"import pandas as pd\n", "import pandas as pd\n",
"import xarray as xr\n", "import xarray as xr\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"import seaborn as sns\n", "import seaborn as sns\n",
"from datetime import datetime\n",
"\n", "\n",
"# Element84 Earth Search STAC\n", "# Element84 Earth Search STAC\n",
"import pystac_client\n", "import pystac_client\n",
@@ -39,59 +32,87 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": null,
"id": "9c063be3", "id": "9c063be3",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Element84 Earth Search kết nối thành công\n",
" API: earth-search.aws.element84.com\n"
]
}
],
"source": [ "source": [
"# Kết nối Element84 Earth Search (hosted trên AWS)\n", "# Kết nối Element84 Earth Search (AWS-hosted STAC API)\n",
"def connect_earth_search():\n",
" \"\"\"Kết nối đến Element84 Earth Search STAC API\"\"\"\n",
"catalog = pystac_client.Client.open(\n", "catalog = pystac_client.Client.open(\n",
" \"https://earth-search.aws.element84.com/v1\"\n", " \"https://earth-search.aws.element84.com/v1\"\n",
")\n", ")\n",
" return catalog\n", "print(\"✅ Kết nối Element84 Earth Search thành công\")"
"\n",
"catalog = connect_earth_search()\n",
"print(\"✅ Element84 Earth Search kết nối thành công\")\n",
"print(f\" API: earth-search.aws.element84.com\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 3, "execution_count": null,
"id": "d87beed2",
"metadata": {},
"outputs": [],
"source": [
"# Tạo file CSV training data từ shapefile\n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"\n",
"# Đọc shapefile training data\n",
"shapefile_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/ST_training_data_updated_1130points_new.shp\"\n",
"gdf = gpd.read_file(shapefile_path)\n",
"\n",
"print(f\"✅ Đã đọc shapefile: {len(gdf)} points\")\n",
"print(f\" Columns: {list(gdf.columns)}\")\n",
"print(f\" CRS: {gdf.crs}\")\n",
"\n",
"# Extract longitude, latitude từ geometry\n",
"gdf['longitude'] = gdf.geometry.x\n",
"gdf['latitude'] = gdf.geometry.y\n",
"\n",
"# Tìm column chứa class name (có thể là 'LULC', 'class', 'label', etc.)\n",
"class_column = None\n",
"for col in gdf.columns:\n",
" if col.lower() in ['lulc', 'class', 'label', 'class_name', 'type', 'landuse']:\n",
" class_column = col\n",
" break\n",
"\n",
"if class_column is None:\n",
" print(\"⚠️ Không tìm thấy column class, hiển thị 5 dòng đầu:\")\n",
" print(gdf.head())\n",
"else:\n",
" # Tạo DataFrame với các cột cần thiết\n",
" train_df = pd.DataFrame({\n",
" 'longitude': gdf['longitude'],\n",
" 'latitude': gdf['latitude'],\n",
" 'class_name': gdf[class_column]\n",
" })\n",
" \n",
" # Export ra CSV\n",
" csv_path = \"/media/x79/2A7D-FAA0/remote-sensing/train_data.csv\"\n",
" train_df.to_csv(csv_path, index=False)\n",
" \n",
" print(f\"\\n✅ Đã tạo file CSV: {csv_path}\")\n",
" print(f\" Số lượng points: {len(train_df)}\")\n",
" print(f\" Classes: {train_df['class_name'].unique()}\")\n",
" print(f\" Class distribution:\")\n",
" print(train_df['class_name'].value_counts())\n",
" print(f\"\\n📋 Preview 5 dòng đầu:\")\n",
" print(train_df.head())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83784d01", "id": "83784d01",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Tìm thấy 45 scenes Sentinel-2\n",
"✅ Sentinel-2 raw: FrozenMappingWarningOnValuesAccess({'y': 8874, 'x': 9902, 'time': 28})\n",
" Variables: ['red', 'green', 'blue', 'nir', 'swir16', 'swir22', 'scl']\n"
]
}
],
"source": [ "source": [
"# Cấu hình vùng và thời gian\n", "# Cấu hình vùng và thời gian\n",
"date_range = (\"2022-09-01\", \"2023-10-01\")\n", "date_range = (\"2022-09-01\", \"2023-10-01\")\n",
"longtitude_range = (105.5, 106.4)\n", "longtitude_range = (105.5, 106.4)\n",
"latitude_range = (9.2, 10.0)\n", "latitude_range = (9.2, 10.0)\n",
"\n", "\n",
"# Tạo bounding box\n",
"bbox = (longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1])\n", "bbox = (longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1])\n",
"\n", "\n",
"# Query Sentinel-2 từ Element84\n", "# Tìm kiếm Sentinel-2 L2A\n",
"search = catalog.search(\n", "search = catalog.search(\n",
" collections=[\"sentinel-2-l2a\"],\n", " collections=[\"sentinel-2-l2a\"],\n",
" bbox=bbox,\n", " bbox=bbox,\n",
@@ -100,7 +121,10 @@
")\n", ")\n",
"\n", "\n",
"items = search.item_collection()\n", "items = search.item_collection()\n",
"print(f\"✅ Tìm thấy {len(items)} scenes Sentinel-2\")\n", "print(f\" Tìm thấy {len(items)} scenes Sentinel-2\")\n",
"\n",
"if len(items) == 0:\n",
" raise ValueError(\"Không tìm thấy dữ liệu Sentinel-2 cho vùng và thời gian này\")\n",
"\n", "\n",
"# Load data với odc-stac\n", "# Load data với odc-stac\n",
"data_sen2 = load(\n", "data_sen2 = load(\n",
@@ -118,31 +142,13 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": null,
"id": "c3faed92", "id": "c3faed92",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"ename": "ValueError",
"evalue": "dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage.",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[4], line 32\u001b[0m\n\u001b[1;32m 30\u001b[0m data_clean \u001b[38;5;241m=\u001b[39m mask_clean(data_sen2)\n\u001b[1;32m 31\u001b[0m data_ndvi \u001b[38;5;241m=\u001b[39m calculate_indices(data_clean, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNDVI\u001b[39m\u001b[38;5;124m\"\u001b[39m, satellite_mission\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ms2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 32\u001b[0m data_fill \u001b[38;5;241m=\u001b[39m \u001b[43mfill_nan\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_ndvi\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 33\u001b[0m data_sen2_monthly \u001b[38;5;241m=\u001b[39m data_fill\u001b[38;5;241m.\u001b[39mresample(time\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1MS\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mmean()\u001b[38;5;241m.\u001b[39mcompute()\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m✅ S2 monthly shape: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdata_sen2_monthly\u001b[38;5;241m.\u001b[39mdims\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"Cell \u001b[0;32mIn[4], line 27\u001b[0m, in \u001b[0;36mfill_nan\u001b[0;34m(ds)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfill_nan\u001b[39m(ds):\n\u001b[1;32m 26\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minterpolate_na\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtime\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinear\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mextrapolate\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/dataset.py:6773\u001b[0m, in \u001b[0;36mDataset.interpolate_na\u001b[0;34m(self, dim, method, limit, use_coordinate, max_gap, **kwargs)\u001b[0m\n\u001b[1;32m 6655\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Fill in NaNs by interpolating according to different methods.\u001b[39;00m\n\u001b[1;32m 6656\u001b[0m \n\u001b[1;32m 6657\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 6769\u001b[0m \u001b[38;5;124;03m D (x) float64 40B 5.0 3.0 1.0 -1.0 4.0\u001b[39;00m\n\u001b[1;32m 6770\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 6771\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxarray\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmissing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _apply_over_vars_with_dim, interp_na\n\u001b[0;32m-> 6773\u001b[0m new \u001b[38;5;241m=\u001b[39m \u001b[43m_apply_over_vars_with_dim\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 6774\u001b[0m \u001b[43m \u001b[49m\u001b[43minterp_na\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6775\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6776\u001b[0m \u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6777\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6778\u001b[0m \u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlimit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6779\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_coordinate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_coordinate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6780\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_gap\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_gap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6781\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6782\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6783\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m new\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:222\u001b[0m, in \u001b[0;36m_apply_over_vars_with_dim\u001b[0;34m(func, self, dim, **kwargs)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name, var \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_vars\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 221\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dim \u001b[38;5;129;01min\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdims:\n\u001b[0;32m--> 222\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 223\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 224\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m var\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:367\u001b[0m, in \u001b[0;36minterp_na\u001b[0;34m(self, dim, use_coordinate, method, limit, max_gap, keep_attrs, **kwargs)\u001b[0m\n\u001b[1;32m 365\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverflow\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[1;32m 366\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minvalid value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[0;32m--> 367\u001b[0m arr \u001b[38;5;241m=\u001b[39m \u001b[43mapply_ufunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 368\u001b[0m \u001b[43m \u001b[49m\u001b[43minterpolator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 369\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 370\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 371\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 372\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 373\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_dtypes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 374\u001b[0m \u001b[43m \u001b[49m\u001b[43mdask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mparallelized\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 375\u001b[0m \u001b[43m \u001b[49m\u001b[43mvectorize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 376\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 377\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdims)\n\u001b[1;32m 379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m limit \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 380\u001b[0m arr \u001b[38;5;241m=\u001b[39m arr\u001b[38;5;241m.\u001b[39mwhere(valids)\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:1265\u001b[0m, in \u001b[0;36mapply_ufunc\u001b[0;34m(func, input_core_dims, output_core_dims, exclude_dims, vectorize, join, dataset_join, dataset_fill_value, keep_attrs, kwargs, dask, output_dtypes, output_sizes, meta, dask_gufunc_kwargs, on_missing_core_dim, *args)\u001b[0m\n\u001b[1;32m 1263\u001b[0m \u001b[38;5;66;03m# feed DataArray apply_variable_ufunc through apply_dataarray_vfunc\u001b[39;00m\n\u001b[1;32m 1264\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, DataArray) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n\u001b[0;32m-> 1265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapply_dataarray_vfunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1266\u001b[0m \u001b[43m \u001b[49m\u001b[43mvariables_vfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1267\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1268\u001b[0m \u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1269\u001b[0m \u001b[43m \u001b[49m\u001b[43mjoin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1270\u001b[0m \u001b[43m \u001b[49m\u001b[43mexclude_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude_dims\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1271\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1272\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1273\u001b[0m \u001b[38;5;66;03m# feed Variables directly through apply_variable_ufunc\u001b[39;00m\n\u001b[1;32m 1274\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, Variable) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:307\u001b[0m, in \u001b[0;36mapply_dataarray_vfunc\u001b[0;34m(func, signature, join, exclude_dims, keep_attrs, *args)\u001b[0m\n\u001b[1;32m 302\u001b[0m result_coords, result_indexes \u001b[38;5;241m=\u001b[39m build_output_coords_and_indexes(\n\u001b[1;32m 303\u001b[0m args, signature, exclude_dims, combine_attrs\u001b[38;5;241m=\u001b[39mkeep_attrs\n\u001b[1;32m 304\u001b[0m )\n\u001b[1;32m 306\u001b[0m data_vars \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mgetattr\u001b[39m(a, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvariable\u001b[39m\u001b[38;5;124m\"\u001b[39m, a) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args]\n\u001b[0;32m--> 307\u001b[0m result_var \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdata_vars\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 309\u001b[0m out: \u001b[38;5;28mtuple\u001b[39m[DataArray, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m|\u001b[39m DataArray\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m signature\u001b[38;5;241m.\u001b[39mnum_outputs \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n",
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:764\u001b[0m, in \u001b[0;36mapply_variable_ufunc\u001b[0;34m(func, signature, exclude_dims, dask, output_dtypes, vectorize, keep_attrs, dask_gufunc_kwargs, *args)\u001b[0m\n\u001b[1;32m 762\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m axis, dim \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(core_dims, start\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;28mlen\u001b[39m(core_dims)):\n\u001b[1;32m 763\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(data\u001b[38;5;241m.\u001b[39mchunks[axis]) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m--> 764\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 765\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdimension \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mth function argument to \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 766\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapply_ufunc with dask=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparallelized\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m consists of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 767\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmultiple chunks, but is also a core dimension. To \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 768\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfix, either rechunk into a single array chunk along \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 769\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthis dimension, i.e., ``.chunk(dict(\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m=-1))``, or \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbut beware that this may significantly increase memory usage.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 772\u001b[0m )\n\u001b[1;32m 773\u001b[0m dask_gufunc_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_rechunk\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 775\u001b[0m output_sizes \u001b[38;5;241m=\u001b[39m dask_gufunc_kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput_sizes\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\n",
"\u001b[0;31mValueError\u001b[0m: dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage."
]
}
],
"source": [ "source": [
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n", "# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
"\n",
"def mask_clean(ds):\n", "def mask_clean(ds):\n",
" \"\"\"Cloud masking sử dụng SCL band (Scene Classification Layer)\"\"\"\n", " \"\"\"Cloud masking sử dụng SCL band (Scene Classification Layer)\"\"\"\n",
" if \"scl\" not in ds:\n", " if \"scl\" not in ds:\n",
@@ -160,7 +166,7 @@
" return ds_masked.drop_vars(\"scl\", errors=\"ignore\")\n", " return ds_masked.drop_vars(\"scl\", errors=\"ignore\")\n",
"\n", "\n",
"def calculate_indices(ds, index=\"NDVI\", satellite_mission=\"s2\"):\n", "def calculate_indices(ds, index=\"NDVI\", satellite_mission=\"s2\"):\n",
" \"\"\"Tính chỉ số NDVI cho Sentinel-2\"\"\"\n", " \"\"\"Tính chỉ số NDVI\"\"\"\n",
" if index == \"NDVI\":\n", " if index == \"NDVI\":\n",
" ndvi = (ds[\"nir\"] - ds[\"red\"]) / (ds[\"nir\"] + ds[\"red\"] + 1e-8)\n", " ndvi = (ds[\"nir\"] - ds[\"red\"]) / (ds[\"nir\"] + ds[\"red\"] + 1e-8)\n",
" ds[\"NDVI\"] = ndvi\n", " ds[\"NDVI\"] = ndvi\n",
@@ -188,6 +194,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Tải Sentinel-1 (SAR VV/VH) từ Element84\n", "# Tải Sentinel-1 (SAR VV/VH) từ Element84\n",
"\n",
"search_s1 = catalog.search(\n", "search_s1 = catalog.search(\n",
" collections=[\"sentinel-1-grd\"],\n", " collections=[\"sentinel-1-grd\"],\n",
" bbox=bbox,\n", " bbox=bbox,\n",
@@ -199,9 +206,25 @@
")\n", ")\n",
"\n", "\n",
"items_s1 = search_s1.item_collection()\n", "items_s1 = search_s1.item_collection()\n",
"print(f\"✅ Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n", "print(f\" Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n",
"\n", "\n",
"if len(items_s1) > 0:\n", "if len(items_s1) == 0:\n",
" print(\"⚠️ Không tìm thấy Sentinel-1, tạo dummy data\")\n",
" # Tạo dummy data với cùng kích thước như S2\n",
" data_sen1_monthly = xr.Dataset({\n",
" \"VV\": xr.DataArray(\n",
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
" coords=data_sen2_monthly[\"red\"].coords,\n",
" dims=data_sen2_monthly[\"red\"].dims\n",
" ),\n",
" \"VH\": xr.DataArray(\n",
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
" coords=data_sen2_monthly[\"red\"].coords,\n",
" dims=data_sen2_monthly[\"red\"].dims\n",
" )\n",
" })\n",
"else:\n",
" # Load Sentinel-1 data\n",
" data_sen1 = load(\n", " data_sen1 = load(\n",
" items_s1,\n", " items_s1,\n",
" bands=[\"vv\", \"vh\"],\n", " bands=[\"vv\", \"vh\"],\n",
@@ -210,22 +233,10 @@
" chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n", " chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n",
" groupby=\"solar_day\"\n", " groupby=\"solar_day\"\n",
" )\n", " )\n",
" \n",
" # Rename bands to uppercase (VV, VH)\n",
" data_sen1 = data_sen1.rename({\"vv\": \"VV\", \"vh\": \"VH\"})\n",
" data_sen1_monthly = data_sen1.resample(time=\"1MS\").mean().compute()\n", " data_sen1_monthly = data_sen1.resample(time=\"1MS\").mean().compute()\n",
"else:\n",
" # Tạo dummy data nếu không có S1\n",
" print(\"⚠️ Không có Sentinel-1, tạo dummy data\")\n",
" data_sen1_monthly = xr.Dataset({\n",
" \"vv\": xr.DataArray(\n",
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
" coords=data_sen2_monthly[\"red\"].coords,\n",
" dims=data_sen2_monthly[\"red\"].dims\n",
" ),\n",
" \"vh\": xr.DataArray(\n",
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
" coords=data_sen2_monthly[\"red\"].coords,\n",
" dims=data_sen2_monthly[\"red\"].dims\n",
" )\n",
" })\n",
"\n", "\n",
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n", "print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n",
"print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")" "print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")"
@@ -263,7 +274,7 @@
" lon, lat = float(row[\"longitude\"]), float(row[\"latitude\"])\n", " lon, lat = float(row[\"longitude\"]), float(row[\"latitude\"])\n",
" label = int(row[\"label\"])\n", " label = int(row[\"label\"])\n",
" \n", " \n",
" # Extract S2 features\n", " # Extract S2 features (mean, std, min, max theo time)\n",
" s2_point = sen2_data.sel(x=lon, y=lat, method=\"nearest\")\n", " s2_point = sen2_data.sel(x=lon, y=lat, method=\"nearest\")\n",
" s2_features = []\n", " s2_features = []\n",
" \n", " \n",
@@ -282,7 +293,7 @@
" s1_point = sen1_data.sel(x=lon, y=lat, method=\"nearest\")\n", " s1_point = sen1_data.sel(x=lon, y=lat, method=\"nearest\")\n",
" s1_features = []\n", " s1_features = []\n",
" \n", " \n",
" for var in [\"vv\", \"vh\"]:\n", " for var in [\"VV\", \"VH\"]:\n",
" if var in s1_point:\n", " if var in s1_point:\n",
" vals = s1_point[var].values\n", " vals = s1_point[var].values\n",
" if vals.size > 0:\n", " if vals.size > 0:\n",
@@ -292,10 +303,12 @@
" \n", " \n",
" features = s2_features + s1_features\n", " features = s2_features + s1_features\n",
" \n", " \n",
" # Bỏ qua nếu có NaN\n",
" if not np.isnan(features).any() and not np.isinf(features).any():\n", " if not np.isnan(features).any() and not np.isinf(features).any():\n",
" X_list.append(features)\n", " X_list.append(features)\n",
" y_list.append(label)\n", " y_list.append(label)\n",
" except Exception as e:\n", " except Exception as e:\n",
" print(f\"⚠️ Lỗi tại row {idx}: {e}\")\n",
" continue\n", " continue\n",
" \n", " \n",
" return X_list, y_list\n", " return X_list, y_list\n",
@@ -305,10 +318,12 @@
" X = np.array(X, dtype=np.float32)\n", " X = np.array(X, dtype=np.float32)\n",
" y = np.array(y, dtype=np.int64)\n", " y = np.array(y, dtype=np.int64)\n",
" \n", " \n",
" # Train + temp\n",
" X_train, X_temp, y_train, y_temp = train_test_split(\n", " X_train, X_temp, y_train, y_temp = train_test_split(\n",
" X, y, test_size=test_size + val_size, random_state=random_state, stratify=y\n", " X, y, test_size=test_size + val_size, random_state=random_state, stratify=y\n",
" )\n", " )\n",
" \n", " \n",
" # Val + test\n",
" val_ratio = val_size / (test_size + val_size)\n", " val_ratio = val_size / (test_size + val_size)\n",
" X_val, X_test, y_val, y_test = train_test_split(\n", " X_val, X_test, y_val, y_test = train_test_split(\n",
" X_temp, y_temp, test_size=(1 - val_ratio), random_state=random_state, stratify=y_temp\n", " X_temp, y_temp, test_size=(1 - val_ratio), random_state=random_state, stratify=y_temp\n",
@@ -316,6 +331,7 @@
" \n", " \n",
" return X_train, X_val, X_test, y_train, y_val, y_test\n", " return X_train, X_val, X_test, y_train, y_val, y_test\n",
"\n", "\n",
"# Load và extract features\n",
"train_data = load_train_data(label_mapping=label_mapping)\n", "train_data = load_train_data(label_mapping=label_mapping)\n",
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n", "X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n", "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
@@ -650,22 +666,8 @@
} }
], ],
"metadata": { "metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": { "language_info": {
"codemirror_mode": { "name": "python"
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
} }
}, },
"nbformat": 4, "nbformat": 4,