From 4000a2c3b34025d0ade7b6d249b7d0468c97ce42 Mon Sep 17 00:00:00 2001 From: Victor Phan Date: Wed, 4 Mar 2026 20:42:08 +0700 Subject: [PATCH] lupdate MObileNet --- train_files/01.train_ODC_MobileNet.ipynb | 218 ++++++++++++----------- 1 file changed, 110 insertions(+), 108 deletions(-) diff --git a/train_files/01.train_ODC_MobileNet.ipynb b/train_files/01.train_ODC_MobileNet.ipynb index b2ee14a..9c41244 100644 --- a/train_files/01.train_ODC_MobileNet.ipynb +++ b/train_files/01.train_ODC_MobileNet.ipynb @@ -2,25 +2,18 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "17da4353", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Import thành công\n" - ] - } - ], + "outputs": [], "source": [ - "# Import libraries for Element84 Earth Search\n", + "# Import libraries\n", "import numpy as np\n", "import pandas as pd\n", "import xarray as xr\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", + "from datetime import datetime\n", "\n", "# Element84 Earth Search STAC\n", "import pystac_client\n", @@ -39,59 +32,87 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "9c063be3", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Element84 Earth Search kết nối thành công\n", - " API: earth-search.aws.element84.com\n" - ] - } - ], + "outputs": [], "source": [ - "# Kết nối Element84 Earth Search (hosted trên AWS)\n", - "def connect_earth_search():\n", - " \"\"\"Kết nối đến Element84 Earth Search STAC API\"\"\"\n", - " catalog = pystac_client.Client.open(\n", - " \"https://earth-search.aws.element84.com/v1\"\n", - " )\n", - " return catalog\n", - "\n", - "catalog = connect_earth_search()\n", - "print(\"✅ Element84 Earth Search kết nối thành công\")\n", - "print(f\" API: earth-search.aws.element84.com\")" + "# Kết nối Element84 Earth Search (AWS-hosted STAC API)\n", + "catalog = pystac_client.Client.open(\n", + " \"https://earth-search.aws.element84.com/v1\"\n", + ")\n", + "print(\"✅ Kết nối Element84 Earth Search thành công\")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, + "id": "d87beed2", + "metadata": {}, + "outputs": [], + "source": [ + "# Tạo file CSV training data từ shapefile\n", + "import geopandas as gpd\n", + "import pandas as pd\n", + "\n", + "# Đọc shapefile training data\n", + "shapefile_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/ST_training_data_updated_1130points_new.shp\"\n", + "gdf = gpd.read_file(shapefile_path)\n", + "\n", + "print(f\"✅ Đã đọc shapefile: {len(gdf)} points\")\n", + "print(f\" Columns: {list(gdf.columns)}\")\n", + "print(f\" CRS: {gdf.crs}\")\n", + "\n", + "# Extract longitude, latitude từ geometry\n", + "gdf['longitude'] = gdf.geometry.x\n", + "gdf['latitude'] = gdf.geometry.y\n", + "\n", + "# Tìm column chứa class name (có thể là 'LULC', 'class', 'label', etc.)\n", + "class_column = None\n", + "for col in gdf.columns:\n", + " if col.lower() in ['lulc', 'class', 'label', 'class_name', 'type', 'landuse']:\n", + " class_column = col\n", + " break\n", + "\n", + "if class_column is None:\n", + " print(\"⚠️ Không tìm thấy column class, hiển thị 5 dòng đầu:\")\n", + " print(gdf.head())\n", + "else:\n", + " # Tạo DataFrame với các cột cần thiết\n", + " train_df = pd.DataFrame({\n", + " 'longitude': gdf['longitude'],\n", + " 'latitude': gdf['latitude'],\n", + " 'class_name': gdf[class_column]\n", + " })\n", + " \n", + " # Export ra CSV\n", + " csv_path = \"/media/x79/2A7D-FAA0/remote-sensing/train_data.csv\"\n", + " train_df.to_csv(csv_path, index=False)\n", + " \n", + " print(f\"\\n✅ Đã tạo file CSV: {csv_path}\")\n", + " print(f\" Số lượng points: {len(train_df)}\")\n", + " print(f\" Classes: {train_df['class_name'].unique()}\")\n", + " print(f\" Class distribution:\")\n", + " print(train_df['class_name'].value_counts())\n", + " print(f\"\\n📋 Preview 5 dòng đầu:\")\n", + " print(train_df.head())" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "83784d01", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Tìm thấy 45 scenes Sentinel-2\n", - "✅ Sentinel-2 raw: FrozenMappingWarningOnValuesAccess({'y': 8874, 'x': 9902, 'time': 28})\n", - " Variables: ['red', 'green', 'blue', 'nir', 'swir16', 'swir22', 'scl']\n" - ] - } - ], + "outputs": [], "source": [ "# Cấu hình vùng và thời gian\n", - "date_range = (\"2022-09-01\", \"2023-10-01\")\n", + "date_range = (\"2022-09-01\", \"2023-10-01\")\n", "longtitude_range = (105.5, 106.4)\n", - "latitude_range = (9.2, 10.0)\n", + "latitude_range = (9.2, 10.0)\n", "\n", - "# Tạo bounding box\n", "bbox = (longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1])\n", "\n", - "# Query Sentinel-2 từ Element84\n", + "# Tìm kiếm Sentinel-2 L2A\n", "search = catalog.search(\n", " collections=[\"sentinel-2-l2a\"],\n", " bbox=bbox,\n", @@ -100,7 +121,10 @@ ")\n", "\n", "items = search.item_collection()\n", - "print(f\"✅ Tìm thấy {len(items)} scenes Sentinel-2\")\n", + "print(f\" Tìm thấy {len(items)} scenes Sentinel-2\")\n", + "\n", + "if len(items) == 0:\n", + " raise ValueError(\"Không tìm thấy dữ liệu Sentinel-2 cho vùng và thời gian này\")\n", "\n", "# Load data với odc-stac\n", "data_sen2 = load(\n", @@ -118,31 +142,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "c3faed92", "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage.", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 32\u001b[0m\n\u001b[1;32m 30\u001b[0m data_clean \u001b[38;5;241m=\u001b[39m mask_clean(data_sen2)\n\u001b[1;32m 31\u001b[0m data_ndvi \u001b[38;5;241m=\u001b[39m calculate_indices(data_clean, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNDVI\u001b[39m\u001b[38;5;124m\"\u001b[39m, satellite_mission\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ms2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 32\u001b[0m data_fill \u001b[38;5;241m=\u001b[39m \u001b[43mfill_nan\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_ndvi\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 33\u001b[0m data_sen2_monthly \u001b[38;5;241m=\u001b[39m data_fill\u001b[38;5;241m.\u001b[39mresample(time\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1MS\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mmean()\u001b[38;5;241m.\u001b[39mcompute()\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m✅ S2 monthly shape: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdata_sen2_monthly\u001b[38;5;241m.\u001b[39mdims\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n", - "Cell \u001b[0;32mIn[4], line 27\u001b[0m, in \u001b[0;36mfill_nan\u001b[0;34m(ds)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfill_nan\u001b[39m(ds):\n\u001b[1;32m 26\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minterpolate_na\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtime\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinear\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mextrapolate\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/dataset.py:6773\u001b[0m, in \u001b[0;36mDataset.interpolate_na\u001b[0;34m(self, dim, method, limit, use_coordinate, max_gap, **kwargs)\u001b[0m\n\u001b[1;32m 6655\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Fill in NaNs by interpolating according to different methods.\u001b[39;00m\n\u001b[1;32m 6656\u001b[0m \n\u001b[1;32m 6657\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 6769\u001b[0m \u001b[38;5;124;03m D (x) float64 40B 5.0 3.0 1.0 -1.0 4.0\u001b[39;00m\n\u001b[1;32m 6770\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 6771\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxarray\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmissing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _apply_over_vars_with_dim, interp_na\n\u001b[0;32m-> 6773\u001b[0m new \u001b[38;5;241m=\u001b[39m \u001b[43m_apply_over_vars_with_dim\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 6774\u001b[0m \u001b[43m \u001b[49m\u001b[43minterp_na\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6775\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6776\u001b[0m \u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6777\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6778\u001b[0m \u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlimit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6779\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_coordinate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_coordinate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6780\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_gap\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_gap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6781\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6782\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6783\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m new\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:222\u001b[0m, in \u001b[0;36m_apply_over_vars_with_dim\u001b[0;34m(func, self, dim, **kwargs)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name, var \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_vars\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 221\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dim \u001b[38;5;129;01min\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdims:\n\u001b[0;32m--> 222\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 223\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 224\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m var\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:367\u001b[0m, in \u001b[0;36minterp_na\u001b[0;34m(self, dim, use_coordinate, method, limit, max_gap, keep_attrs, **kwargs)\u001b[0m\n\u001b[1;32m 365\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverflow\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[1;32m 366\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minvalid value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[0;32m--> 367\u001b[0m arr \u001b[38;5;241m=\u001b[39m \u001b[43mapply_ufunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 368\u001b[0m \u001b[43m \u001b[49m\u001b[43minterpolator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 369\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 370\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 371\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 372\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 373\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_dtypes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 374\u001b[0m \u001b[43m \u001b[49m\u001b[43mdask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mparallelized\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 375\u001b[0m \u001b[43m \u001b[49m\u001b[43mvectorize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 376\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 377\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdims)\n\u001b[1;32m 379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m limit \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 380\u001b[0m arr \u001b[38;5;241m=\u001b[39m arr\u001b[38;5;241m.\u001b[39mwhere(valids)\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:1265\u001b[0m, in \u001b[0;36mapply_ufunc\u001b[0;34m(func, input_core_dims, output_core_dims, exclude_dims, vectorize, join, dataset_join, dataset_fill_value, keep_attrs, kwargs, dask, output_dtypes, output_sizes, meta, dask_gufunc_kwargs, on_missing_core_dim, *args)\u001b[0m\n\u001b[1;32m 1263\u001b[0m \u001b[38;5;66;03m# feed DataArray apply_variable_ufunc through apply_dataarray_vfunc\u001b[39;00m\n\u001b[1;32m 1264\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, DataArray) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n\u001b[0;32m-> 1265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapply_dataarray_vfunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1266\u001b[0m \u001b[43m \u001b[49m\u001b[43mvariables_vfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1267\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1268\u001b[0m \u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1269\u001b[0m \u001b[43m \u001b[49m\u001b[43mjoin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1270\u001b[0m \u001b[43m \u001b[49m\u001b[43mexclude_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude_dims\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1271\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1272\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1273\u001b[0m \u001b[38;5;66;03m# feed Variables directly through apply_variable_ufunc\u001b[39;00m\n\u001b[1;32m 1274\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, Variable) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:307\u001b[0m, in \u001b[0;36mapply_dataarray_vfunc\u001b[0;34m(func, signature, join, exclude_dims, keep_attrs, *args)\u001b[0m\n\u001b[1;32m 302\u001b[0m result_coords, result_indexes \u001b[38;5;241m=\u001b[39m build_output_coords_and_indexes(\n\u001b[1;32m 303\u001b[0m args, signature, exclude_dims, combine_attrs\u001b[38;5;241m=\u001b[39mkeep_attrs\n\u001b[1;32m 304\u001b[0m )\n\u001b[1;32m 306\u001b[0m data_vars \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mgetattr\u001b[39m(a, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvariable\u001b[39m\u001b[38;5;124m\"\u001b[39m, a) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args]\n\u001b[0;32m--> 307\u001b[0m result_var \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdata_vars\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 309\u001b[0m out: \u001b[38;5;28mtuple\u001b[39m[DataArray, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m|\u001b[39m DataArray\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m signature\u001b[38;5;241m.\u001b[39mnum_outputs \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n", - "File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:764\u001b[0m, in \u001b[0;36mapply_variable_ufunc\u001b[0;34m(func, signature, exclude_dims, dask, output_dtypes, vectorize, keep_attrs, dask_gufunc_kwargs, *args)\u001b[0m\n\u001b[1;32m 762\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m axis, dim \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(core_dims, start\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;28mlen\u001b[39m(core_dims)):\n\u001b[1;32m 763\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(data\u001b[38;5;241m.\u001b[39mchunks[axis]) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m--> 764\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 765\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdimension \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mth function argument to \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 766\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapply_ufunc with dask=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparallelized\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m consists of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 767\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmultiple chunks, but is also a core dimension. To \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 768\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfix, either rechunk into a single array chunk along \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 769\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthis dimension, i.e., ``.chunk(dict(\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m=-1))``, or \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbut beware that this may significantly increase memory usage.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 772\u001b[0m )\n\u001b[1;32m 773\u001b[0m dask_gufunc_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_rechunk\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 775\u001b[0m output_sizes \u001b[38;5;241m=\u001b[39m dask_gufunc_kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput_sizes\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\n", - "\u001b[0;31mValueError\u001b[0m: dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage." - ] - } - ], + "outputs": [], "source": [ "# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n", + "\n", "def mask_clean(ds):\n", " \"\"\"Cloud masking sử dụng SCL band (Scene Classification Layer)\"\"\"\n", " if \"scl\" not in ds:\n", @@ -160,7 +166,7 @@ " return ds_masked.drop_vars(\"scl\", errors=\"ignore\")\n", "\n", "def calculate_indices(ds, index=\"NDVI\", satellite_mission=\"s2\"):\n", - " \"\"\"Tính chỉ số NDVI cho Sentinel-2\"\"\"\n", + " \"\"\"Tính chỉ số NDVI\"\"\"\n", " if index == \"NDVI\":\n", " ndvi = (ds[\"nir\"] - ds[\"red\"]) / (ds[\"nir\"] + ds[\"red\"] + 1e-8)\n", " ds[\"NDVI\"] = ndvi\n", @@ -172,8 +178,8 @@ "\n", "# Áp dụng tiền xử lý\n", "data_clean = mask_clean(data_sen2)\n", - "data_ndvi = calculate_indices(data_clean, index=\"NDVI\", satellite_mission=\"s2\")\n", - "data_fill = fill_nan(data_ndvi)\n", + "data_ndvi = calculate_indices(data_clean, index=\"NDVI\", satellite_mission=\"s2\")\n", + "data_fill = fill_nan(data_ndvi)\n", "data_sen2_monthly = data_fill.resample(time=\"1MS\").mean().compute()\n", "\n", "print(f\"✅ S2 monthly shape: {data_sen2_monthly.dims}\")\n", @@ -188,6 +194,7 @@ "outputs": [], "source": [ "# Tải Sentinel-1 (SAR VV/VH) từ Element84\n", + "\n", "search_s1 = catalog.search(\n", " collections=[\"sentinel-1-grd\"],\n", " bbox=bbox,\n", @@ -199,9 +206,25 @@ ")\n", "\n", "items_s1 = search_s1.item_collection()\n", - "print(f\"✅ Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n", + "print(f\" Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n", "\n", - "if len(items_s1) > 0:\n", + "if len(items_s1) == 0:\n", + " print(\"⚠️ Không tìm thấy Sentinel-1, tạo dummy data\")\n", + " # Tạo dummy data với cùng kích thước như S2\n", + " data_sen1_monthly = xr.Dataset({\n", + " \"VV\": xr.DataArray(\n", + " np.zeros_like(data_sen2_monthly[\"red\"].values),\n", + " coords=data_sen2_monthly[\"red\"].coords,\n", + " dims=data_sen2_monthly[\"red\"].dims\n", + " ),\n", + " \"VH\": xr.DataArray(\n", + " np.zeros_like(data_sen2_monthly[\"red\"].values),\n", + " coords=data_sen2_monthly[\"red\"].coords,\n", + " dims=data_sen2_monthly[\"red\"].dims\n", + " )\n", + " })\n", + "else:\n", + " # Load Sentinel-1 data\n", " data_sen1 = load(\n", " items_s1,\n", " bands=[\"vv\", \"vh\"],\n", @@ -210,22 +233,10 @@ " chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n", " groupby=\"solar_day\"\n", " )\n", + " \n", + " # Rename bands to uppercase (VV, VH)\n", + " data_sen1 = data_sen1.rename({\"vv\": \"VV\", \"vh\": \"VH\"})\n", " data_sen1_monthly = data_sen1.resample(time=\"1MS\").mean().compute()\n", - "else:\n", - " # Tạo dummy data nếu không có S1\n", - " print(\"⚠️ Không có Sentinel-1, tạo dummy data\")\n", - " data_sen1_monthly = xr.Dataset({\n", - " \"vv\": xr.DataArray(\n", - " np.zeros_like(data_sen2_monthly[\"red\"].values),\n", - " coords=data_sen2_monthly[\"red\"].coords,\n", - " dims=data_sen2_monthly[\"red\"].dims\n", - " ),\n", - " \"vh\": xr.DataArray(\n", - " np.zeros_like(data_sen2_monthly[\"red\"].values),\n", - " coords=data_sen2_monthly[\"red\"].coords,\n", - " dims=data_sen2_monthly[\"red\"].dims\n", - " )\n", - " })\n", "\n", "print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n", "print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")" @@ -263,7 +274,7 @@ " lon, lat = float(row[\"longitude\"]), float(row[\"latitude\"])\n", " label = int(row[\"label\"])\n", " \n", - " # Extract S2 features\n", + " # Extract S2 features (mean, std, min, max theo time)\n", " s2_point = sen2_data.sel(x=lon, y=lat, method=\"nearest\")\n", " s2_features = []\n", " \n", @@ -282,7 +293,7 @@ " s1_point = sen1_data.sel(x=lon, y=lat, method=\"nearest\")\n", " s1_features = []\n", " \n", - " for var in [\"vv\", \"vh\"]:\n", + " for var in [\"VV\", \"VH\"]:\n", " if var in s1_point:\n", " vals = s1_point[var].values\n", " if vals.size > 0:\n", @@ -292,10 +303,12 @@ " \n", " features = s2_features + s1_features\n", " \n", + " # Bỏ qua nếu có NaN\n", " if not np.isnan(features).any() and not np.isinf(features).any():\n", " X_list.append(features)\n", " y_list.append(label)\n", " except Exception as e:\n", + " print(f\"⚠️ Lỗi tại row {idx}: {e}\")\n", " continue\n", " \n", " return X_list, y_list\n", @@ -305,10 +318,12 @@ " X = np.array(X, dtype=np.float32)\n", " y = np.array(y, dtype=np.int64)\n", " \n", + " # Train + temp\n", " X_train, X_temp, y_train, y_temp = train_test_split(\n", " X, y, test_size=test_size + val_size, random_state=random_state, stratify=y\n", " )\n", " \n", + " # Val + test\n", " val_ratio = val_size / (test_size + val_size)\n", " X_val, X_test, y_val, y_test = train_test_split(\n", " X_temp, y_temp, test_size=(1 - val_ratio), random_state=random_state, stratify=y_temp\n", @@ -316,6 +331,7 @@ " \n", " return X_train, X_val, X_test, y_train, y_val, y_test\n", "\n", + "# Load và extract features\n", "train_data = load_train_data(label_mapping=label_mapping)\n", "X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n", "X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n", @@ -650,22 +666,8 @@ } ], "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" + "name": "python" } }, "nbformat": 4,