lupdate MObileNet
This commit is contained in:
Victor Phan
@@ -2,25 +2,18 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "17da4353",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Import thành công\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import libraries for Element84 Earth Search\n",
|
||||
"# Import libraries\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import xarray as xr\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import seaborn as sns\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"# Element84 Earth Search STAC\n",
|
||||
"import pystac_client\n",
|
||||
@@ -39,59 +32,87 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"id": "9c063be3",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Element84 Earth Search kết nối thành công\n",
|
||||
" API: earth-search.aws.element84.com\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Kết nối Element84 Earth Search (hosted trên AWS)\n",
|
||||
"def connect_earth_search():\n",
|
||||
" \"\"\"Kết nối đến Element84 Earth Search STAC API\"\"\"\n",
|
||||
" catalog = pystac_client.Client.open(\n",
|
||||
"# Kết nối Element84 Earth Search (AWS-hosted STAC API)\n",
|
||||
"catalog = pystac_client.Client.open(\n",
|
||||
" \"https://earth-search.aws.element84.com/v1\"\n",
|
||||
" )\n",
|
||||
" return catalog\n",
|
||||
"\n",
|
||||
"catalog = connect_earth_search()\n",
|
||||
"print(\"✅ Element84 Earth Search kết nối thành công\")\n",
|
||||
"print(f\" API: earth-search.aws.element84.com\")"
|
||||
")\n",
|
||||
"print(\"✅ Kết nối Element84 Earth Search thành công\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"id": "d87beed2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tạo file CSV training data từ shapefile\n",
|
||||
"import geopandas as gpd\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# Đọc shapefile training data\n",
|
||||
"shapefile_path = \"/media/x79/2A7D-FAA0/remote-sensing/train/ST_training_data_updated_1130points_new.shp\"\n",
|
||||
"gdf = gpd.read_file(shapefile_path)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Đã đọc shapefile: {len(gdf)} points\")\n",
|
||||
"print(f\" Columns: {list(gdf.columns)}\")\n",
|
||||
"print(f\" CRS: {gdf.crs}\")\n",
|
||||
"\n",
|
||||
"# Extract longitude, latitude từ geometry\n",
|
||||
"gdf['longitude'] = gdf.geometry.x\n",
|
||||
"gdf['latitude'] = gdf.geometry.y\n",
|
||||
"\n",
|
||||
"# Tìm column chứa class name (có thể là 'LULC', 'class', 'label', etc.)\n",
|
||||
"class_column = None\n",
|
||||
"for col in gdf.columns:\n",
|
||||
" if col.lower() in ['lulc', 'class', 'label', 'class_name', 'type', 'landuse']:\n",
|
||||
" class_column = col\n",
|
||||
" break\n",
|
||||
"\n",
|
||||
"if class_column is None:\n",
|
||||
" print(\"⚠️ Không tìm thấy column class, hiển thị 5 dòng đầu:\")\n",
|
||||
" print(gdf.head())\n",
|
||||
"else:\n",
|
||||
" # Tạo DataFrame với các cột cần thiết\n",
|
||||
" train_df = pd.DataFrame({\n",
|
||||
" 'longitude': gdf['longitude'],\n",
|
||||
" 'latitude': gdf['latitude'],\n",
|
||||
" 'class_name': gdf[class_column]\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" # Export ra CSV\n",
|
||||
" csv_path = \"/media/x79/2A7D-FAA0/remote-sensing/train_data.csv\"\n",
|
||||
" train_df.to_csv(csv_path, index=False)\n",
|
||||
" \n",
|
||||
" print(f\"\\n✅ Đã tạo file CSV: {csv_path}\")\n",
|
||||
" print(f\" Số lượng points: {len(train_df)}\")\n",
|
||||
" print(f\" Classes: {train_df['class_name'].unique()}\")\n",
|
||||
" print(f\" Class distribution:\")\n",
|
||||
" print(train_df['class_name'].value_counts())\n",
|
||||
" print(f\"\\n📋 Preview 5 dòng đầu:\")\n",
|
||||
" print(train_df.head())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "83784d01",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ Tìm thấy 45 scenes Sentinel-2\n",
|
||||
"✅ Sentinel-2 raw: FrozenMappingWarningOnValuesAccess({'y': 8874, 'x': 9902, 'time': 28})\n",
|
||||
" Variables: ['red', 'green', 'blue', 'nir', 'swir16', 'swir22', 'scl']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cấu hình vùng và thời gian\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"# Tạo bounding box\n",
|
||||
"bbox = (longtitude_range[0], latitude_range[0], longtitude_range[1], latitude_range[1])\n",
|
||||
"\n",
|
||||
"# Query Sentinel-2 từ Element84\n",
|
||||
"# Tìm kiếm Sentinel-2 L2A\n",
|
||||
"search = catalog.search(\n",
|
||||
" collections=[\"sentinel-2-l2a\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
@@ -100,7 +121,10 @@
|
||||
")\n",
|
||||
"\n",
|
||||
"items = search.item_collection()\n",
|
||||
"print(f\"✅ Tìm thấy {len(items)} scenes Sentinel-2\")\n",
|
||||
"print(f\" Tìm thấy {len(items)} scenes Sentinel-2\")\n",
|
||||
"\n",
|
||||
"if len(items) == 0:\n",
|
||||
" raise ValueError(\"Không tìm thấy dữ liệu Sentinel-2 cho vùng và thời gian này\")\n",
|
||||
"\n",
|
||||
"# Load data với odc-stac\n",
|
||||
"data_sen2 = load(\n",
|
||||
@@ -118,31 +142,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"id": "c3faed92",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ValueError",
|
||||
"evalue": "dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage.",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[4], line 32\u001b[0m\n\u001b[1;32m 30\u001b[0m data_clean \u001b[38;5;241m=\u001b[39m mask_clean(data_sen2)\n\u001b[1;32m 31\u001b[0m data_ndvi \u001b[38;5;241m=\u001b[39m calculate_indices(data_clean, index\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNDVI\u001b[39m\u001b[38;5;124m\"\u001b[39m, satellite_mission\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124ms2\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 32\u001b[0m data_fill \u001b[38;5;241m=\u001b[39m \u001b[43mfill_nan\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata_ndvi\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 33\u001b[0m data_sen2_monthly \u001b[38;5;241m=\u001b[39m data_fill\u001b[38;5;241m.\u001b[39mresample(time\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m1MS\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mmean()\u001b[38;5;241m.\u001b[39mcompute()\n\u001b[1;32m 35\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m✅ S2 monthly shape: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdata_sen2_monthly\u001b[38;5;241m.\u001b[39mdims\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"Cell \u001b[0;32mIn[4], line 27\u001b[0m, in \u001b[0;36mfill_nan\u001b[0;34m(ds)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfill_nan\u001b[39m(ds):\n\u001b[1;32m 26\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Fill NaN bằng interpolation theo thời gian\"\"\"\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minterpolate_na\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mtime\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinear\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfill_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mextrapolate\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/dataset.py:6773\u001b[0m, in \u001b[0;36mDataset.interpolate_na\u001b[0;34m(self, dim, method, limit, use_coordinate, max_gap, **kwargs)\u001b[0m\n\u001b[1;32m 6655\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Fill in NaNs by interpolating according to different methods.\u001b[39;00m\n\u001b[1;32m 6656\u001b[0m \n\u001b[1;32m 6657\u001b[0m \u001b[38;5;124;03mParameters\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 6769\u001b[0m \u001b[38;5;124;03m D (x) float64 40B 5.0 3.0 1.0 -1.0 4.0\u001b[39;00m\n\u001b[1;32m 6770\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 6771\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mxarray\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmissing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _apply_over_vars_with_dim, interp_na\n\u001b[0;32m-> 6773\u001b[0m new \u001b[38;5;241m=\u001b[39m \u001b[43m_apply_over_vars_with_dim\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 6774\u001b[0m \u001b[43m \u001b[49m\u001b[43minterp_na\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6775\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6776\u001b[0m \u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6777\u001b[0m \u001b[43m \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6778\u001b[0m \u001b[43m \u001b[49m\u001b[43mlimit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlimit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6779\u001b[0m \u001b[43m \u001b[49m\u001b[43muse_coordinate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_coordinate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6780\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_gap\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_gap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6781\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6782\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6783\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m new\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:222\u001b[0m, in \u001b[0;36m_apply_over_vars_with_dim\u001b[0;34m(func, self, dim, **kwargs)\u001b[0m\n\u001b[1;32m 220\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m name, var \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_vars\u001b[38;5;241m.\u001b[39mitems():\n\u001b[1;32m 221\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m dim \u001b[38;5;129;01min\u001b[39;00m var\u001b[38;5;241m.\u001b[39mdims:\n\u001b[0;32m--> 222\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvar\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdim\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 223\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 224\u001b[0m ds[name] \u001b[38;5;241m=\u001b[39m var\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/missing.py:367\u001b[0m, in \u001b[0;36minterp_na\u001b[0;34m(self, dim, use_coordinate, method, limit, max_gap, keep_attrs, **kwargs)\u001b[0m\n\u001b[1;32m 365\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moverflow\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[1;32m 366\u001b[0m warnings\u001b[38;5;241m.\u001b[39mfilterwarnings(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minvalid value\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;167;01mRuntimeWarning\u001b[39;00m)\n\u001b[0;32m--> 367\u001b[0m arr \u001b[38;5;241m=\u001b[39m \u001b[43mapply_ufunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 368\u001b[0m \u001b[43m \u001b[49m\u001b[43minterpolator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 369\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 370\u001b[0m \u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 371\u001b[0m \u001b[43m \u001b[49m\u001b[43minput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 372\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_core_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdim\u001b[49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 373\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_dtypes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 374\u001b[0m \u001b[43m \u001b[49m\u001b[43mdask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mparallelized\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 375\u001b[0m \u001b[43m \u001b[49m\u001b[43mvectorize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 376\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 377\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdims)\n\u001b[1;32m 379\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m limit \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 380\u001b[0m arr \u001b[38;5;241m=\u001b[39m arr\u001b[38;5;241m.\u001b[39mwhere(valids)\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:1265\u001b[0m, in \u001b[0;36mapply_ufunc\u001b[0;34m(func, input_core_dims, output_core_dims, exclude_dims, vectorize, join, dataset_join, dataset_fill_value, keep_attrs, kwargs, dask, output_dtypes, output_sizes, meta, dask_gufunc_kwargs, on_missing_core_dim, *args)\u001b[0m\n\u001b[1;32m 1263\u001b[0m \u001b[38;5;66;03m# feed DataArray apply_variable_ufunc through apply_dataarray_vfunc\u001b[39;00m\n\u001b[1;32m 1264\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, DataArray) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n\u001b[0;32m-> 1265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mapply_dataarray_vfunc\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1266\u001b[0m \u001b[43m \u001b[49m\u001b[43mvariables_vfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1267\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1268\u001b[0m \u001b[43m \u001b[49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1269\u001b[0m \u001b[43m \u001b[49m\u001b[43mjoin\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1270\u001b[0m \u001b[43m \u001b[49m\u001b[43mexclude_dims\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexclude_dims\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1271\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeep_attrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeep_attrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1272\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1273\u001b[0m \u001b[38;5;66;03m# feed Variables directly through apply_variable_ufunc\u001b[39;00m\n\u001b[1;32m 1274\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\u001b[38;5;28misinstance\u001b[39m(a, Variable) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args):\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:307\u001b[0m, in \u001b[0;36mapply_dataarray_vfunc\u001b[0;34m(func, signature, join, exclude_dims, keep_attrs, *args)\u001b[0m\n\u001b[1;32m 302\u001b[0m result_coords, result_indexes \u001b[38;5;241m=\u001b[39m build_output_coords_and_indexes(\n\u001b[1;32m 303\u001b[0m args, signature, exclude_dims, combine_attrs\u001b[38;5;241m=\u001b[39mkeep_attrs\n\u001b[1;32m 304\u001b[0m )\n\u001b[1;32m 306\u001b[0m data_vars \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mgetattr\u001b[39m(a, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvariable\u001b[39m\u001b[38;5;124m\"\u001b[39m, a) \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m args]\n\u001b[0;32m--> 307\u001b[0m result_var \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mdata_vars\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 309\u001b[0m out: \u001b[38;5;28mtuple\u001b[39m[DataArray, \u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m] \u001b[38;5;241m|\u001b[39m DataArray\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m signature\u001b[38;5;241m.\u001b[39mnum_outputs \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n",
|
||||
"File \u001b[0;32m/env/lib/python3.12/site-packages/xarray/core/computation.py:764\u001b[0m, in \u001b[0;36mapply_variable_ufunc\u001b[0;34m(func, signature, exclude_dims, dask, output_dtypes, vectorize, keep_attrs, dask_gufunc_kwargs, *args)\u001b[0m\n\u001b[1;32m 762\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m axis, dim \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(core_dims, start\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;28mlen\u001b[39m(core_dims)):\n\u001b[1;32m 763\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(data\u001b[38;5;241m.\u001b[39mchunks[axis]) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m--> 764\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 765\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdimension \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mn\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124mth function argument to \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 766\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapply_ufunc with dask=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mparallelized\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m consists of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 767\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmultiple chunks, but is also a core dimension. To \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 768\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfix, either rechunk into a single array chunk along \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 769\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthis dimension, i.e., ``.chunk(dict(\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdim\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m=-1))``, or \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 770\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 771\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbut beware that this may significantly increase memory usage.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 772\u001b[0m )\n\u001b[1;32m 773\u001b[0m dask_gufunc_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_rechunk\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m 775\u001b[0m output_sizes \u001b[38;5;241m=\u001b[39m dask_gufunc_kwargs\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput_sizes\u001b[39m\u001b[38;5;124m\"\u001b[39m, {})\n",
|
||||
"\u001b[0;31mValueError\u001b[0m: dimension time on 0th function argument to apply_ufunc with dask='parallelized' consists of multiple chunks, but is also a core dimension. To fix, either rechunk into a single array chunk along this dimension, i.e., ``.chunk(dict(time=-1))``, or pass ``allow_rechunk=True`` in ``dask_gufunc_kwargs`` but beware that this may significantly increase memory usage."
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tiền xử lý Sentinel-2: cloud mask + NDVI + resampling\n",
|
||||
"\n",
|
||||
"def mask_clean(ds):\n",
|
||||
" \"\"\"Cloud masking sử dụng SCL band (Scene Classification Layer)\"\"\"\n",
|
||||
" if \"scl\" not in ds:\n",
|
||||
@@ -160,7 +166,7 @@
|
||||
" return ds_masked.drop_vars(\"scl\", errors=\"ignore\")\n",
|
||||
"\n",
|
||||
"def calculate_indices(ds, index=\"NDVI\", satellite_mission=\"s2\"):\n",
|
||||
" \"\"\"Tính chỉ số NDVI cho Sentinel-2\"\"\"\n",
|
||||
" \"\"\"Tính chỉ số NDVI\"\"\"\n",
|
||||
" if index == \"NDVI\":\n",
|
||||
" ndvi = (ds[\"nir\"] - ds[\"red\"]) / (ds[\"nir\"] + ds[\"red\"] + 1e-8)\n",
|
||||
" ds[\"NDVI\"] = ndvi\n",
|
||||
@@ -188,6 +194,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tải Sentinel-1 (SAR VV/VH) từ Element84\n",
|
||||
"\n",
|
||||
"search_s1 = catalog.search(\n",
|
||||
" collections=[\"sentinel-1-grd\"],\n",
|
||||
" bbox=bbox,\n",
|
||||
@@ -199,9 +206,25 @@
|
||||
")\n",
|
||||
"\n",
|
||||
"items_s1 = search_s1.item_collection()\n",
|
||||
"print(f\"✅ Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n",
|
||||
"print(f\" Tìm thấy {len(items_s1)} scenes Sentinel-1\")\n",
|
||||
"\n",
|
||||
"if len(items_s1) > 0:\n",
|
||||
"if len(items_s1) == 0:\n",
|
||||
" print(\"⚠️ Không tìm thấy Sentinel-1, tạo dummy data\")\n",
|
||||
" # Tạo dummy data với cùng kích thước như S2\n",
|
||||
" data_sen1_monthly = xr.Dataset({\n",
|
||||
" \"VV\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" ),\n",
|
||||
" \"VH\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" )\n",
|
||||
" })\n",
|
||||
"else:\n",
|
||||
" # Load Sentinel-1 data\n",
|
||||
" data_sen1 = load(\n",
|
||||
" items_s1,\n",
|
||||
" bands=[\"vv\", \"vh\"],\n",
|
||||
@@ -210,22 +233,10 @@
|
||||
" chunks={\"time\": 1, \"x\": 2048, \"y\": 2048},\n",
|
||||
" groupby=\"solar_day\"\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # Rename bands to uppercase (VV, VH)\n",
|
||||
" data_sen1 = data_sen1.rename({\"vv\": \"VV\", \"vh\": \"VH\"})\n",
|
||||
" data_sen1_monthly = data_sen1.resample(time=\"1MS\").mean().compute()\n",
|
||||
"else:\n",
|
||||
" # Tạo dummy data nếu không có S1\n",
|
||||
" print(\"⚠️ Không có Sentinel-1, tạo dummy data\")\n",
|
||||
" data_sen1_monthly = xr.Dataset({\n",
|
||||
" \"vv\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" ),\n",
|
||||
" \"vh\": xr.DataArray(\n",
|
||||
" np.zeros_like(data_sen2_monthly[\"red\"].values),\n",
|
||||
" coords=data_sen2_monthly[\"red\"].coords,\n",
|
||||
" dims=data_sen2_monthly[\"red\"].dims\n",
|
||||
" )\n",
|
||||
" })\n",
|
||||
"\n",
|
||||
"print(f\"✅ S1 monthly shape: {data_sen1_monthly.dims}\")\n",
|
||||
"print(f\" Variables: {list(data_sen1_monthly.data_vars)}\")"
|
||||
@@ -263,7 +274,7 @@
|
||||
" lon, lat = float(row[\"longitude\"]), float(row[\"latitude\"])\n",
|
||||
" label = int(row[\"label\"])\n",
|
||||
" \n",
|
||||
" # Extract S2 features\n",
|
||||
" # Extract S2 features (mean, std, min, max theo time)\n",
|
||||
" s2_point = sen2_data.sel(x=lon, y=lat, method=\"nearest\")\n",
|
||||
" s2_features = []\n",
|
||||
" \n",
|
||||
@@ -282,7 +293,7 @@
|
||||
" s1_point = sen1_data.sel(x=lon, y=lat, method=\"nearest\")\n",
|
||||
" s1_features = []\n",
|
||||
" \n",
|
||||
" for var in [\"vv\", \"vh\"]:\n",
|
||||
" for var in [\"VV\", \"VH\"]:\n",
|
||||
" if var in s1_point:\n",
|
||||
" vals = s1_point[var].values\n",
|
||||
" if vals.size > 0:\n",
|
||||
@@ -292,10 +303,12 @@
|
||||
" \n",
|
||||
" features = s2_features + s1_features\n",
|
||||
" \n",
|
||||
" # Bỏ qua nếu có NaN\n",
|
||||
" if not np.isnan(features).any() and not np.isinf(features).any():\n",
|
||||
" X_list.append(features)\n",
|
||||
" y_list.append(label)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"⚠️ Lỗi tại row {idx}: {e}\")\n",
|
||||
" continue\n",
|
||||
" \n",
|
||||
" return X_list, y_list\n",
|
||||
@@ -305,10 +318,12 @@
|
||||
" X = np.array(X, dtype=np.float32)\n",
|
||||
" y = np.array(y, dtype=np.int64)\n",
|
||||
" \n",
|
||||
" # Train + temp\n",
|
||||
" X_train, X_temp, y_train, y_temp = train_test_split(\n",
|
||||
" X, y, test_size=test_size + val_size, random_state=random_state, stratify=y\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # Val + test\n",
|
||||
" val_ratio = val_size / (test_size + val_size)\n",
|
||||
" X_val, X_test, y_val, y_test = train_test_split(\n",
|
||||
" X_temp, y_temp, test_size=(1 - val_ratio), random_state=random_state, stratify=y_temp\n",
|
||||
@@ -316,6 +331,7 @@
|
||||
" \n",
|
||||
" return X_train, X_val, X_test, y_train, y_val, y_test\n",
|
||||
"\n",
|
||||
"# Load và extract features\n",
|
||||
"train_data = load_train_data(label_mapping=label_mapping)\n",
|
||||
"X, y = get_data_sen1_and_sen2(train_data, data_sen2_monthly, data_sen1_monthly)\n",
|
||||
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(X, y, test_size=0.2, val_size=0.1)\n",
|
||||
@@ -650,22 +666,8 @@
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
Reference in New Issue
Block a user