mirror of
https://git.victorphan.net/basketballcantho/CSIROBoeingPhase5-Vietnam.git
synced 2026-08-05 05:43:10 +07:00
train CNN thành công
This commit is contained in:
Victor Phan
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "72723821",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Prepare Data on Server - Load Raw Data Only\n",
|
||||
"Chuẩn bị dữ liệu trên server: **Chỉ tải dữ liệu thô từ S3, không xử lý**\n",
|
||||
"\n",
|
||||
"**Workflow:**\n",
|
||||
"1. Server: Tải S2 (red, nir, scl) + S1 (VH, VV) thô → Lưu NetCDF\n",
|
||||
"2. Local: Tải xuống → Tính NDVI → Cloud mask → Fill NaN → Aggregation → Train model\n",
|
||||
"\n",
|
||||
"**Sau khi chạy xong, tải các file data xuống máy cá nhân để xử lý + train model**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f502b085",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import new_import_ODC \n",
|
||||
"\n",
|
||||
"importlib.reload(new_import_ODC)\n",
|
||||
"\n",
|
||||
"from new_import_ODC import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "379dd847",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Cấu hình Daskgateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
|
||||
"# Khai báo 1 Datacube là dc\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"\n",
|
||||
"# Cấu hình truy cập dịch vụ S3\n",
|
||||
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
||||
"\n",
|
||||
"client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "da3e2180",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## Cấu hình thời gian lấy ảnh và tọa độ\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"coordinates = (longtitude_range, latitude_range)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "3f521c79",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## DEBUG: Inspect what datacube wants to load\n",
|
||||
"print(\"🔍 DIAGNOSTIC: Checking datacube metadata...\\n\")\n",
|
||||
"\n",
|
||||
"# Check available products\n",
|
||||
"available_products = dc.list_products()\n",
|
||||
"s2_products = available_products[available_products['name'].str.contains('s2', case=False)]\n",
|
||||
"print(f\"Available S2 products:\\n{s2_products[['name', 'description']].to_string()}\\n\")\n",
|
||||
"\n",
|
||||
"# Query to check what would be loaded\n",
|
||||
"test_query = {\n",
|
||||
" 'product': 's2_l2a',\n",
|
||||
" 'x': longtitude_range,\n",
|
||||
" 'y': latitude_range,\n",
|
||||
" 'time': (\"2023-01-01\", \"2023-02-01\"), # Just 1 month for testing\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"Test query: {test_query}\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" # This queries metadata only, doesn't load data\n",
|
||||
" test_datasets = dc.find_datasets(**test_query)\n",
|
||||
" print(f\"\\n📊 Metadata check for Jan 2023:\")\n",
|
||||
" print(f\" Found {len(test_datasets)} scenes\")\n",
|
||||
" if test_datasets:\n",
|
||||
" first_ds = test_datasets[0]\n",
|
||||
" print(f\" First scene: {first_ds.center_time}\")\n",
|
||||
" print(f\" Bounds: {first_ds.bounds}\")\n",
|
||||
" print(f\" CRS: {first_ds.crs}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"❌ Error: {e}\")\n",
|
||||
"\n",
|
||||
"print(\"\\n\" + \"=\"*60 + \"\\n\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8cf560b0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## SENTINEL-2 LOADING: Monthly chunks to prevent OOM\n",
|
||||
"print(\"📡 Tải dữ liệu Sentinel-2 L2A từ S3...\")\n",
|
||||
"print(f\" AOI: {longtitude_range}, {latitude_range}\")\n",
|
||||
"print(f\" Time range: {date_range}\\n\")\n",
|
||||
"\n",
|
||||
"data = None\n",
|
||||
"\n",
|
||||
"# Strategy: Load 13 monthly chunks instead of 396 scenes at once\n",
|
||||
"# This keeps memory usage manageable (~5-15 GB per month)\n",
|
||||
"\n",
|
||||
"date_ranges = [\n",
|
||||
" (\"2022-09-01\", \"2022-10-01\"),\n",
|
||||
" (\"2022-10-01\", \"2022-11-01\"),\n",
|
||||
" (\"2022-11-01\", \"2022-12-01\"),\n",
|
||||
" (\"2022-12-01\", \"2023-01-01\"),\n",
|
||||
" (\"2023-01-01\", \"2023-02-01\"),\n",
|
||||
" (\"2023-02-01\", \"2023-03-01\"),\n",
|
||||
" (\"2023-03-01\", \"2023-04-01\"),\n",
|
||||
" (\"2023-04-01\", \"2023-05-01\"),\n",
|
||||
" (\"2023-05-01\", \"2023-06-01\"),\n",
|
||||
" (\"2023-06-01\", \"2023-07-01\"),\n",
|
||||
" (\"2023-07-01\", \"2023-08-01\"),\n",
|
||||
" (\"2023-08-01\", \"2023-09-01\"),\n",
|
||||
" (\"2023-09-01\", \"2023-10-01\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"product = 's2_l2a'\n",
|
||||
"measurements = ['red', 'nir', 'scl']\n",
|
||||
"\n",
|
||||
"# Get native CRS once\n",
|
||||
"try:\n",
|
||||
" query_crs = {\n",
|
||||
" 'product': product,\n",
|
||||
" 'x': longtitude_range,\n",
|
||||
" 'y': latitude_range,\n",
|
||||
" 'time': date_range,\n",
|
||||
" }\n",
|
||||
" native_crs = notebook_utils.mostcommon_crs(dc, query_crs)\n",
|
||||
" print(f\"✅ Native CRS: {native_crs}\\n\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"⚠️ Could not determine CRS: {e}\")\n",
|
||||
" native_crs = 'EPSG:32648' # Fallback for UTM Zone 48N\n",
|
||||
"\n",
|
||||
"data_list = []\n",
|
||||
"\n",
|
||||
"for i, (start_date, end_date) in enumerate(date_ranges):\n",
|
||||
" print(f\"[{i+1:2d}/13] {start_date} → {end_date} \", end=\"\", flush=True)\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" monthly_query = {\n",
|
||||
" 'product': product,\n",
|
||||
" 'x': longtitude_range,\n",
|
||||
" 'y': latitude_range,\n",
|
||||
" 'time': (start_date, end_date),\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" load_params = {\n",
|
||||
" 'measurements': measurements,\n",
|
||||
" 'output_crs': native_crs,\n",
|
||||
" 'resolution': (-10, 10),\n",
|
||||
" 'group_by': 'solar_day',\n",
|
||||
" 'dask_chunks': {'x': 512, 'y': 512, 'time': 1},\n",
|
||||
" 'skip_broken_datasets': True,\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" monthly_data = load_s2l2a_with_offset(dc, monthly_query | load_params)\n",
|
||||
" \n",
|
||||
" n_scenes = monthly_data.sizes['time']\n",
|
||||
" if n_scenes > 0:\n",
|
||||
" data_list.append(monthly_data)\n",
|
||||
" print(f\"✓ {n_scenes} scenes\")\n",
|
||||
" else:\n",
|
||||
" print(\"⚠️ 0 scenes\")\n",
|
||||
" \n",
|
||||
" except MemoryError as e:\n",
|
||||
" print(f\"❌ OOM: {str(e)[:60]}\")\n",
|
||||
" break\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"❌ {str(e)[:60]}\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
"# Combine all monthly chunks\n",
|
||||
"if data_list:\n",
|
||||
" print(f\"\\n🔗 Combining {len(data_list)} monthly chunks...\")\n",
|
||||
" data = xr.concat(data_list, dim='time')\n",
|
||||
" print(f\"✅ Success! Shape: {dict(data.dims)}\")\n",
|
||||
" print(f\" Memory: {notebook_utils.xarray_object_size(data)}\")\n",
|
||||
" display(data)\n",
|
||||
"else:\n",
|
||||
" print(\"\\n❌ Failed to load any scenes\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cef198a0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## Tải dữ liệu Sentinel-1 (VH, VV) - Raw data, không xử lý\n",
|
||||
"print(\"📡 Tải dữ liệu Sentinel-1 từ S3...\")\n",
|
||||
"try:\n",
|
||||
" dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
|
||||
" print(f\"✅ VH shape: {dsvh.shape}\")\n",
|
||||
" print(f\"✅ VV shape: {dsvv.shape}\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"⚠️ Error loading S1: {e}\")\n",
|
||||
" dsvh = None\n",
|
||||
" dsvv = None"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5ba52958",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Lưu dữ liệu RAW thành file NetCDF (chưa xử lý)\n",
|
||||
"Dữ liệu này sẽ được tải xuống máy local để tiếp tục xử lý và training"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "06bf3f40",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Close Dask client FIRST to avoid permission issues\n",
|
||||
"print(\"🔌 Closing Dask client to free resources...\")\n",
|
||||
"client.close()\n",
|
||||
"cluster.close()\n",
|
||||
"print(\"✅ Dask closed\\n\")\n",
|
||||
"\n",
|
||||
"# Now save data locally without Dask interference\n",
|
||||
"data_dir = \"data_for_training\"\n",
|
||||
"if not os.path.exists(data_dir):\n",
|
||||
" os.makedirs(data_dir)\n",
|
||||
" print(f\"✅ Tạo thư mục {data_dir}\\n\")\n",
|
||||
"\n",
|
||||
"print(\"💾 Lưu dữ liệu RAW từ S2 (red, nir, scl)...\")\n",
|
||||
"\n",
|
||||
"# FIX: Loại bỏ metadata problematic trước khi save\n",
|
||||
"data_clean = data.copy()\n",
|
||||
"\n",
|
||||
"# 1. Loại bỏ dict attributes từ variables\n",
|
||||
"for var_name in data_clean.data_vars:\n",
|
||||
" attrs_to_remove = []\n",
|
||||
" for attr_name, attr_value in data_clean[var_name].attrs.items():\n",
|
||||
" if isinstance(attr_value, dict):\n",
|
||||
" attrs_to_remove.append(attr_name)\n",
|
||||
" \n",
|
||||
" for attr_name in attrs_to_remove:\n",
|
||||
" print(f\" ⚠️ Removing problematic attr: {var_name}.{attr_name}\")\n",
|
||||
" del data_clean[var_name].attrs[attr_name]\n",
|
||||
"\n",
|
||||
"# 2. Xóa attrs từ coordinates\n",
|
||||
"for coord_name in data_clean.coords:\n",
|
||||
" attrs_to_remove = []\n",
|
||||
" for attr_name, attr_value in data_clean[coord_name].attrs.items():\n",
|
||||
" if attr_name in ['units', 'calendar', 'long_name', 'standard_name']:\n",
|
||||
" attrs_to_remove.append(attr_name)\n",
|
||||
" elif isinstance(attr_value, dict):\n",
|
||||
" attrs_to_remove.append(attr_name)\n",
|
||||
" \n",
|
||||
" for attr_name in attrs_to_remove:\n",
|
||||
" print(f\" ⚠️ Removing coord attr: {coord_name}.{attr_name}\")\n",
|
||||
" del data_clean[coord_name].attrs[attr_name]\n",
|
||||
"\n",
|
||||
"# Compute to memory BEFORE closing Dask\n",
|
||||
"print(\"\\n ⏳ Loading data into memory...\")\n",
|
||||
"try:\n",
|
||||
" data_computed = data_clean.compute()\n",
|
||||
" print(\" ✅ Data loaded to memory\\n\")\n",
|
||||
" \n",
|
||||
" # Save from memory (no Dask involved)\n",
|
||||
" s2_path = os.path.join(data_dir, \"sentinel2_raw.nc\")\n",
|
||||
" print(f\" - Saving to {s2_path}...\")\n",
|
||||
" data_computed.to_netcdf(s2_path, engine='netcdf4')\n",
|
||||
" s2_size = os.path.getsize(s2_path) / 1024**3\n",
|
||||
" print(f\" ✅ Sentinel-2 saved ({s2_size:.2f} GB)\")\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\" ❌ Error: {e}\")\n",
|
||||
" print(f\" ⚠️ Data may still be in memory, proceed to next step\")\n",
|
||||
"\n",
|
||||
"# Lưu raw Sentinel-1 (nếu thành công)\n",
|
||||
"if dsvh is not None and dsvv is not None:\n",
|
||||
" print(f\"\\n💾 Lưu dữ liệu RAW từ S1 (VH, VV)...\")\n",
|
||||
" \n",
|
||||
" # Remove problematic attrs từ S1 data\n",
|
||||
" if hasattr(dsvh, 'attrs'):\n",
|
||||
" for attr in list(dsvh.attrs.keys()):\n",
|
||||
" if attr in ['units', 'calendar'] or isinstance(dsvh.attrs[attr], dict):\n",
|
||||
" del dsvh.attrs[attr]\n",
|
||||
" \n",
|
||||
" if hasattr(dsvv, 'attrs'):\n",
|
||||
" for attr in list(dsvv.attrs.keys()):\n",
|
||||
" if attr in ['units', 'calendar'] or isinstance(dsvv.attrs[attr], dict):\n",
|
||||
" del dsvv.attrs[attr]\n",
|
||||
" \n",
|
||||
" s1_data = xr.Dataset({\n",
|
||||
" 'VH': dsvh,\n",
|
||||
" 'VV': dsvv,\n",
|
||||
" })\n",
|
||||
" \n",
|
||||
" s1_path = os.path.join(data_dir, \"sentinel1_raw.nc\")\n",
|
||||
" print(f\" - Saving to {s1_path}...\")\n",
|
||||
" \n",
|
||||
" try:\n",
|
||||
" s1_data.to_netcdf(s1_path, engine='netcdf4')\n",
|
||||
" s1_size = os.path.getsize(s1_path) / 1024**3\n",
|
||||
" print(f\" ✅ Sentinel-1 saved ({s1_size:.2f} GB)\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" ❌ Error saving S1: {e}\")\n",
|
||||
"else:\n",
|
||||
" print(\"⚠️ Sentinel-1 not saved\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n✅ Data ready in '{data_dir}'\")\n",
|
||||
"print(f\"\\n📥 **Download to local machine using rsync:**\")\n",
|
||||
"print(f\" rsync -avz --progress user@server:~/CSIROBoeingPhase5-Vietnam/{data_dir}/ ./{data_dir}/\")\n",
|
||||
"print(f\"\\n🔧 Next: Process data locally (NDVI, cloud mask, aggregation, training)\")\n",
|
||||
"\n",
|
||||
"# Hiển thị tổng kích thước\n",
|
||||
"import subprocess\n",
|
||||
"try:\n",
|
||||
" result = subprocess.run(['du', '-sh', data_dir], capture_output=True, text=True)\n",
|
||||
" print(f\"\\n📊 Data size on server: {result.stdout.strip()}\")\n",
|
||||
"except:\n",
|
||||
" pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "93df03e7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Lưu training data\n",
|
||||
"train_path = \"train/ST_training data_updated_1130points_new.shp\"\n",
|
||||
"import shutil\n",
|
||||
"\n",
|
||||
"print(f\"📋 Copy training data...\")\n",
|
||||
"# Copy toàn bộ các file liên quan đến shapefile\n",
|
||||
"train_dir = \"train\"\n",
|
||||
"train_output = os.path.join(data_dir, \"train_data\")\n",
|
||||
"if not os.path.exists(train_output):\n",
|
||||
" os.makedirs(train_output)\n",
|
||||
"\n",
|
||||
"for file in os.listdir(train_dir):\n",
|
||||
" if \"1130points_new\" in file:\n",
|
||||
" src = os.path.join(train_dir, file)\n",
|
||||
" dst = os.path.join(train_output, file)\n",
|
||||
" shutil.copy2(src, dst)\n",
|
||||
" print(f\"✅ {file}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n✅ Training data đã copy vào '{train_output}'\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8bb935d9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Đóng client, cluster\n",
|
||||
"print(\"\\n🔌 Đóng kết nối...\")\n",
|
||||
"client.close()\n",
|
||||
"cluster.close()\n",
|
||||
"print(\"✅ Xong!\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,405 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "912ed572-1658-406b-976c-cd6de2d4e89e",
|
||||
"metadata": {
|
||||
"editable": true,
|
||||
"slideshow": {
|
||||
"slide_type": ""
|
||||
},
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import new_import_ODC \n",
|
||||
"\n",
|
||||
"importlib.reload(new_import_ODC)\n",
|
||||
"\n",
|
||||
"from new_import_ODC import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d824dc4f-994b-4d1c-8d24-ce6674da141c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Cấu hình Daskgateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
|
||||
"# Khai báo 1 Datacube là dc\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"\n",
|
||||
"# Cấu hình truy cập dịch vụ S3\n",
|
||||
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
||||
"\n",
|
||||
"client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fbed4c80-bbf8-4ea8-aa45-2460b2ba04c7",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình thời gian lấy ảnh và tọa độ\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"coordinates = (longtitude_range, latitude_range)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6b90c49b-0665-4478-a23b-d111ef88eb79",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## truy vấn ảnh vệ tinh sen2\n",
|
||||
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"display(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2f6938b6-82e2-4916-bc1d-719c169e25e4",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
|
||||
"result = mask_clean(data)\n",
|
||||
"progress(result)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "435f9f78-a9a4-4226-86ca-d4bec42d454e",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tiến hành tính toán NDVI\n",
|
||||
"ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
|
||||
"ndvi = ds1[\"NDVI\"]\n",
|
||||
"display(ndvi)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "84992d28-8e3f-468e-be08-ded511f2c662",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)\n",
|
||||
"plt.imshow(ndvi.isel(time=6))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "72318b60-532a-4f08-a5f7-94762d08a42c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa\n",
|
||||
"time_split = [\n",
|
||||
" slice(\"2022-09-01\", \"2023-01-01\"),\n",
|
||||
" slice(\"2023-01-01\", \"2023-05-01\"),\n",
|
||||
" slice(\"2023-05-01\", \"2023-07-01\"),\n",
|
||||
" slice(\"2023-07-01\", \"2023-10-01\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Điền mây ở các vị trí mang giá trị nan (fill nan)\n",
|
||||
"fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
|
||||
"\n",
|
||||
"# In kết quả ảnh NDVI đã điền mây (đã fill nan)\n",
|
||||
"plt.imshow(fill_nan_ndvi.isel(time=6))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "375b1cfb-37f5-49fe-8061-ea32eb47f9f6",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"## tính ndvi theo tháng\n",
|
||||
"average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
|
||||
"progress(average_ndvi)\n",
|
||||
"\n",
|
||||
"# compute average_ndvi\n",
|
||||
"average_ndvi = average_ndvi.compute()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "187cc640-aef9-476b-91fc-b63f4d3ff2e3",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#Load dữ liệu ảnh Sentinel 1\n",
|
||||
"dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
|
||||
"average_vv = calculate_average(dsvv, time_pattern='1M')\n",
|
||||
"average_vh = calculate_average(dsvh, time_pattern='1M')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d2585562-88aa-4c7d-bf70-1f6affcf65d4",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)\n",
|
||||
"train_path = \"train/ST_training data_updated_1130points_new.shp\" # đường dẫn shp file train\n",
|
||||
"\n",
|
||||
"## load dữ liệu điểm huấn luyện mô hình (train file)\n",
|
||||
"train = load_train_data(train_path)\n",
|
||||
"train.head()\n",
|
||||
"\n",
|
||||
"# cấu hình nhãn dữ liệu \n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": \"0\",\n",
|
||||
" \"Lua\": \"1\",\n",
|
||||
" \"CHN\": \"2\",\n",
|
||||
" \"CLN\": \"3\",\n",
|
||||
" \"TS\": \"4\",\n",
|
||||
" \"Song\": \"5\",\n",
|
||||
" \"Dat xay dung\": \"6\",\n",
|
||||
" \"Rung\": \"7\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI\n",
|
||||
"datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)\n",
|
||||
"\n",
|
||||
"# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test\n",
|
||||
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n",
|
||||
" train, label_mapping, datasets\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2e955884-d4af-422d-a8e6-d436199540e0",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Import XGBoost\n",
|
||||
"import xgboost as xgb\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Convert to numpy arrays\n",
|
||||
"X_train_np = np.asarray(X_train, dtype=np.float32)\n",
|
||||
"X_val_np = np.asarray(X_val, dtype=np.float32)\n",
|
||||
"y_train_np = np.asarray(y_train, dtype=np.int32)\n",
|
||||
"y_val_np = np.asarray(y_val, dtype=np.int32)\n",
|
||||
"\n",
|
||||
"print(\"🚀 Training XGBoost model...\")\n",
|
||||
"print(f\" Train samples: {len(X_train_np)}\")\n",
|
||||
"print(f\" Val samples: {len(X_val_np)}\")\n",
|
||||
"print(f\" Features: {X_train_np.shape[1]}\")\n",
|
||||
"print(f\" Classes: 8\\n\")\n",
|
||||
"\n",
|
||||
"# XGBoost parameters\n",
|
||||
"params = {\n",
|
||||
" 'objective': 'multi:softmax', # Multi-class classification\n",
|
||||
" 'num_class': 8, # 8 land use classes\n",
|
||||
" 'max_depth': 6, # Maximum tree depth\n",
|
||||
" 'learning_rate': 0.1, # Learning rate\n",
|
||||
" 'n_estimators': 200, # Number of trees\n",
|
||||
" 'subsample': 0.8, # Subsample ratio\n",
|
||||
" 'colsample_bytree': 0.8, # Feature sampling ratio\n",
|
||||
" 'random_state': 42,\n",
|
||||
" 'n_jobs': -1, # Use all CPU cores\n",
|
||||
" 'eval_metric': 'mlogloss' # Multi-class log loss\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Train XGBoost model\n",
|
||||
"model = xgb.XGBClassifier(**params)\n",
|
||||
"\n",
|
||||
"model.fit(\n",
|
||||
" X_train_np, y_train_np,\n",
|
||||
" eval_set=[(X_train_np, y_train_np), (X_val_np, y_val_np)],\n",
|
||||
" verbose=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Validation accuracy\n",
|
||||
"y_val_pred = model.predict(X_val_np)\n",
|
||||
"val_accuracy = accuracy_score(y_val_np, y_val_pred)\n",
|
||||
"print(f\"\\n✅ Training completed!\")\n",
|
||||
"print(f\" Validation Accuracy: {val_accuracy:.4f} ({val_accuracy*100:.2f}%)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b2a1e42c-cf1b-4d82-a6af-b06e3496918f",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Evaluate on test set\n",
|
||||
"X_test_np = np.asarray(X_test, dtype=np.float32)\n",
|
||||
"y_test_np = np.asarray(y_test, dtype=np.int32)\n",
|
||||
"\n",
|
||||
"print(\"📊 Evaluating XGBoost model on test set...\\n\")\n",
|
||||
"\n",
|
||||
"# Predictions\n",
|
||||
"y_pred_test = model.predict(X_test_np)\n",
|
||||
"\n",
|
||||
"# Metrics\n",
|
||||
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\n",
|
||||
"\n",
|
||||
"test_accuracy = accuracy_score(y_test_np, y_pred_test)\n",
|
||||
"precision = precision_score(y_test_np, y_pred_test, average='weighted', zero_division=0)\n",
|
||||
"recall = recall_score(y_test_np, y_pred_test, average='weighted', zero_division=0)\n",
|
||||
"f1 = f1_score(y_test_np, y_pred_test, average='weighted', zero_division=0)\n",
|
||||
"\n",
|
||||
"print(f\"📈 Test Results:\")\n",
|
||||
"print(f\" Accuracy: {test_accuracy:.4f} ({test_accuracy*100:.2f}%)\")\n",
|
||||
"print(f\" Precision: {precision:.4f}\")\n",
|
||||
"print(f\" Recall: {recall:.4f}\")\n",
|
||||
"print(f\" F1-Score: {f1:.4f}\\n\")\n",
|
||||
"\n",
|
||||
"# Confusion Matrix\n",
|
||||
"from sklearn.metrics import ConfusionMatrixDisplay\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"\n",
|
||||
"# Create figure first\n",
|
||||
"fig, ax = plt.subplots(figsize=(10, 8))\n",
|
||||
"\n",
|
||||
"class_names = list(label_mapping.keys())\n",
|
||||
"cm = confusion_matrix(y_test_np, y_pred_test)\n",
|
||||
"disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=class_names)\n",
|
||||
"disp.plot(cmap='Blues', ax=ax)\n",
|
||||
"plt.xticks(rotation=45, ha='right')\n",
|
||||
"plt.title('XGBoost Confusion Matrix')\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f1a14379-ed6e-4897-9ca4-2669743fab40",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Lưu mô hình huấn luyện\n",
|
||||
"import json\n",
|
||||
"import joblib\n",
|
||||
"\n",
|
||||
"# Save XGBoost model\n",
|
||||
"model_path = \"model_xgboost.joblib\"\n",
|
||||
"joblib.dump(model, model_path)\n",
|
||||
"print(f\"✅ Model saved to {model_path}\")\n",
|
||||
"\n",
|
||||
"# Save model info\n",
|
||||
"info = {\n",
|
||||
" \"model_type\": \"XGBoost\",\n",
|
||||
" \"num_classes\": 8,\n",
|
||||
" \"classes\": list(label_mapping.keys()),\n",
|
||||
" \"num_features\": X_train_np.shape[1],\n",
|
||||
" \"params\": params,\n",
|
||||
" \"accuracy\": float(test_accuracy),\n",
|
||||
" \"precision\": float(precision),\n",
|
||||
" \"recall\": float(recall),\n",
|
||||
" \"f1_score\": float(f1),\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"with open(\"model_xgboost_info.json\", \"w\") as f:\n",
|
||||
" json.dump(info, f, indent=2)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Model info saved to model_xgboost_info.json\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "33dd516d-9824-499e-96b9-5cd9224c194c",
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# đóng client, cluster\n",
|
||||
"client.close()\n",
|
||||
"cluster.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ee0ecd6a-733b-4655-8864-bc037a539ce2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,571 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "a58466bf",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Train CNN Model on Local Machine\n",
|
||||
"Huấn luyện mô hình CNN với PyTorch trên máy cá nhân\n",
|
||||
"\n",
|
||||
"**Yêu cầu**: Đã tải dữ liệu từ server vào thư mục `data_for_training/`"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0e5a5a39",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import numpy as np\n",
|
||||
"import xarray as xr\n",
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch.optim as optim\n",
|
||||
"from torch.utils.data import DataLoader, TensorDataset\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.preprocessing import LabelEncoder\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"import geopandas as gpd\n",
|
||||
"import joblib\n",
|
||||
"from utils import load_data_geo\n",
|
||||
"\n",
|
||||
"print(f\"PyTorch version: {torch.__version__}\")\n",
|
||||
"print(f\"GPU available: {torch.cuda.is_available()}\")\n",
|
||||
"if torch.cuda.is_available():\n",
|
||||
" print(f\"GPU device: {torch.cuda.get_device_name(0)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ff3f92e3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Load dữ liệu từ server"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "880dba25",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Đường dẫn dữ liệu\n",
|
||||
"data_dir = \"data_for_training\"\n",
|
||||
"\n",
|
||||
"print(\"📂 Kiểm tra các file dữ liệu...\")\n",
|
||||
"if not os.path.exists(data_dir):\n",
|
||||
" raise FileNotFoundError(f\"❌ Thư mục '{data_dir}' không tồn tại. Hãy tải dữ liệu từ server trước.\")\n",
|
||||
"\n",
|
||||
"# Load dữ liệu\n",
|
||||
"print(\"\\n📥 Load dữ liệu...\")\n",
|
||||
"average_ndvi = xr.open_dataarray(os.path.join(data_dir, \"average_ndvi.nc\"))\n",
|
||||
"average_vv = xr.open_dataarray(os.path.join(data_dir, \"average_vv.nc\"))\n",
|
||||
"average_vh = xr.open_dataarray(os.path.join(data_dir, \"average_vh.nc\"))\n",
|
||||
"\n",
|
||||
"print(f\"✅ NDVI shape: {average_ndvi.shape}\")\n",
|
||||
"print(f\"✅ VV shape: {average_vv.shape}\")\n",
|
||||
"print(f\"✅ VH shape: {average_vh.shape}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2236b22d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load training data\n",
|
||||
"train_data_path = os.path.join(data_dir, \"train_data\", \"ST_training data_updated_1130points_new.shp\")\n",
|
||||
"print(f\"\\n📋 Load training points...\")\n",
|
||||
"train = load_data_geo(train_data_path)\n",
|
||||
"print(f\"✅ Training points: {len(train)}\")\n",
|
||||
"print(f\"✅ Columns: {train.columns.tolist()}\")\n",
|
||||
"train.head()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "8b1800e2",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Chuẩn bị dữ liệu huấn luyện"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "128d1492",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cấu hình nhãn dữ liệu\n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": 0,\n",
|
||||
" \"Lua\": 1,\n",
|
||||
" \"CHN\": 2,\n",
|
||||
" \"CLN\": 3,\n",
|
||||
" \"TS\": 4,\n",
|
||||
" \"Song\": 5,\n",
|
||||
" \"Dat xay dung\": 6,\n",
|
||||
" \"Rung\": 7,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"num_classes = len(label_mapping)\n",
|
||||
"print(f\"🏷️ Số lớp: {num_classes}\")\n",
|
||||
"for label, idx in label_mapping.items():\n",
|
||||
" print(f\" {idx}: {label}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ec87566f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Trích xuất dữ liệu từ các điểm training\n",
|
||||
"print(\"\\n🔧 Trích xuất dữ liệu từ các điểm...\")\n",
|
||||
"X = []\n",
|
||||
"y = []\n",
|
||||
"\n",
|
||||
"for idx, point in train.iterrows():\n",
|
||||
" try:\n",
|
||||
" # Lấy tọa độ\n",
|
||||
" x_coord = point.geometry.x\n",
|
||||
" y_coord = point.geometry.y\n",
|
||||
" \n",
|
||||
" # Trích xuất giá trị từ mỗi band\n",
|
||||
" ndvi_data = average_ndvi.sel(x=x_coord, y=y_coord, method='nearest').values\n",
|
||||
" vv_data = average_vv.sel(x=x_coord, y=y_coord, method='nearest').values\n",
|
||||
" vh_data = average_vh.sel(x=x_coord, y=y_coord, method='nearest').values\n",
|
||||
" \n",
|
||||
" # Concatenate các band\n",
|
||||
" data_point = np.concatenate((ndvi_data, vv_data, vh_data))\n",
|
||||
" X.append(data_point)\n",
|
||||
" \n",
|
||||
" # Lấy nhãn\n",
|
||||
" label_text = point.Hientrang\n",
|
||||
" label_idx = label_mapping[label_text]\n",
|
||||
" y.append(label_idx)\n",
|
||||
" \n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\" ⚠️ Point {idx}: {e}\")\n",
|
||||
"\n",
|
||||
"X = np.array(X)\n",
|
||||
"y = np.array(y)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Dữ liệu trích xuất: {X.shape}\")\n",
|
||||
"print(f\"✅ Nhãn: {y.shape}\")\n",
|
||||
"print(f\"✅ Phân phối nhãn: {np.bincount(y)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2587a67c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Chia dữ liệu\n",
|
||||
"print(\"\\n📊 Chia dữ liệu (train/val/test = 60/20/20)...\")\n",
|
||||
"X_train, X_temp, y_train, y_temp = train_test_split(\n",
|
||||
" X, y, test_size=0.4, random_state=42, stratify=y\n",
|
||||
")\n",
|
||||
"X_val, X_test, y_val, y_test = train_test_split(\n",
|
||||
" X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"✅ Train: {X_train.shape}\")\n",
|
||||
"print(f\"✅ Val: {X_val.shape}\")\n",
|
||||
"print(f\"✅ Test: {X_test.shape}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "359761e4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Normalize dữ liệu\n",
|
||||
"print(\"\\n🔧 Normalize dữ liệu...\")\n",
|
||||
"mean = X_train.mean()\n",
|
||||
"std = X_train.std()\n",
|
||||
"X_train = (X_train - mean) / (std + 1e-8)\n",
|
||||
"X_val = (X_val - mean) / (std + 1e-8)\n",
|
||||
"X_test = (X_test - mean) / (std + 1e-8)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Mean: {mean:.4f}\")\n",
|
||||
"print(f\"✅ Std: {std:.4f}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0667150b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Convert to PyTorch tensors\n",
|
||||
"print(\"\\n🔧 Convert sang PyTorch tensors...\")\n",
|
||||
"X_train_tensor = torch.FloatTensor(X_train).unsqueeze(1) # (N, 1, features)\n",
|
||||
"X_val_tensor = torch.FloatTensor(X_val).unsqueeze(1)\n",
|
||||
"X_test_tensor = torch.FloatTensor(X_test).unsqueeze(1)\n",
|
||||
"\n",
|
||||
"y_train_tensor = torch.LongTensor(y_train)\n",
|
||||
"y_val_tensor = torch.LongTensor(y_val)\n",
|
||||
"y_test_tensor = torch.LongTensor(y_test)\n",
|
||||
"\n",
|
||||
"print(f\"✅ X_train shape: {X_train_tensor.shape}\")\n",
|
||||
"print(f\"✅ y_train shape: {y_train_tensor.shape}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "be2a4179",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tạo DataLoaders\n",
|
||||
"print(\"\\n🔧 Tạo DataLoaders...\")\n",
|
||||
"batch_size = 32\n",
|
||||
"\n",
|
||||
"train_dataset = TensorDataset(X_train_tensor, y_train_tensor)\n",
|
||||
"train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
|
||||
"\n",
|
||||
"val_dataset = TensorDataset(X_val_tensor, y_val_tensor)\n",
|
||||
"val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n",
|
||||
"\n",
|
||||
"test_dataset = TensorDataset(X_test_tensor, y_test_tensor)\n",
|
||||
"test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Train batches: {len(train_loader)}\")\n",
|
||||
"print(f\"✅ Val batches: {len(val_loader)}\")\n",
|
||||
"print(f\"✅ Test batches: {len(test_loader)}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5231acf3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Xây dựng CNN Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b279fe09",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Định nghĩa CNN model\n",
|
||||
"class CNNClassifier(nn.Module):\n",
|
||||
" def __init__(self, input_size, num_classes=8):\n",
|
||||
" super(CNNClassifier, self).__init__()\n",
|
||||
" \n",
|
||||
" # Conv blocks\n",
|
||||
" self.conv1 = nn.Conv1d(1, 64, kernel_size=3, padding=1)\n",
|
||||
" self.bn1 = nn.BatchNorm1d(64)\n",
|
||||
" self.conv2 = nn.Conv1d(64, 64, kernel_size=3, padding=1)\n",
|
||||
" self.bn2 = nn.BatchNorm1d(64)\n",
|
||||
" self.pool1 = nn.MaxPool1d(2)\n",
|
||||
" self.drop1 = nn.Dropout(0.25)\n",
|
||||
" \n",
|
||||
" self.conv3 = nn.Conv1d(64, 128, kernel_size=3, padding=1)\n",
|
||||
" self.bn3 = nn.BatchNorm1d(128)\n",
|
||||
" self.conv4 = nn.Conv1d(128, 128, kernel_size=3, padding=1)\n",
|
||||
" self.bn4 = nn.BatchNorm1d(128)\n",
|
||||
" self.pool2 = nn.MaxPool1d(2)\n",
|
||||
" self.drop2 = nn.Dropout(0.25)\n",
|
||||
" \n",
|
||||
" self.conv5 = nn.Conv1d(128, 256, kernel_size=3, padding=1)\n",
|
||||
" self.bn5 = nn.BatchNorm1d(256)\n",
|
||||
" self.conv6 = nn.Conv1d(256, 256, kernel_size=3, padding=1)\n",
|
||||
" self.bn6 = nn.BatchNorm1d(256)\n",
|
||||
" self.global_avg_pool = nn.AdaptiveAvgPool1d(1)\n",
|
||||
" self.drop3 = nn.Dropout(0.25)\n",
|
||||
" \n",
|
||||
" # FC layers\n",
|
||||
" self.fc1 = nn.Linear(256, 128)\n",
|
||||
" self.bn_fc1 = nn.BatchNorm1d(128)\n",
|
||||
" self.drop_fc1 = nn.Dropout(0.5)\n",
|
||||
" \n",
|
||||
" self.fc2 = nn.Linear(128, 64)\n",
|
||||
" self.bn_fc2 = nn.BatchNorm1d(64)\n",
|
||||
" self.drop_fc2 = nn.Dropout(0.5)\n",
|
||||
" \n",
|
||||
" self.fc3 = nn.Linear(64, num_classes)\n",
|
||||
" \n",
|
||||
" self.relu = nn.ReLU()\n",
|
||||
" \n",
|
||||
" def forward(self, x):\n",
|
||||
" # Block 1\n",
|
||||
" x = self.relu(self.bn1(self.conv1(x)))\n",
|
||||
" x = self.relu(self.bn2(self.conv2(x)))\n",
|
||||
" x = self.pool1(x)\n",
|
||||
" x = self.drop1(x)\n",
|
||||
" \n",
|
||||
" # Block 2\n",
|
||||
" x = self.relu(self.bn3(self.conv3(x)))\n",
|
||||
" x = self.relu(self.bn4(self.conv4(x)))\n",
|
||||
" x = self.pool2(x)\n",
|
||||
" x = self.drop2(x)\n",
|
||||
" \n",
|
||||
" # Block 3\n",
|
||||
" x = self.relu(self.bn5(self.conv5(x)))\n",
|
||||
" x = self.relu(self.bn6(self.conv6(x)))\n",
|
||||
" x = self.global_avg_pool(x)\n",
|
||||
" x = x.squeeze(-1)\n",
|
||||
" x = self.drop3(x)\n",
|
||||
" \n",
|
||||
" # FC layers\n",
|
||||
" x = self.relu(self.bn_fc1(self.fc1(x)))\n",
|
||||
" x = self.drop_fc1(x)\n",
|
||||
" x = self.relu(self.bn_fc2(self.fc2(x)))\n",
|
||||
" x = self.drop_fc2(x)\n",
|
||||
" x = self.fc3(x)\n",
|
||||
" \n",
|
||||
" return x\n",
|
||||
"\n",
|
||||
"# Khởi tạo model\n",
|
||||
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
||||
"model = CNNClassifier(input_size=X_train.shape[1], num_classes=num_classes)\n",
|
||||
"model = model.to(device)\n",
|
||||
"\n",
|
||||
"print(f\"✅ Model created\")\n",
|
||||
"print(f\"📍 Device: {device}\")\n",
|
||||
"print(f\"\\n📋 Model architecture:\")\n",
|
||||
"print(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ff42829a",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Huấn luyện Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "de90a544",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Loss function và optimizer\n",
|
||||
"criterion = nn.CrossEntropyLoss()\n",
|
||||
"optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
|
||||
"scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n",
|
||||
" optimizer, mode='min', factor=0.5, patience=5, verbose=True\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"epochs = 100\n",
|
||||
"early_stop_patience = 15\n",
|
||||
"best_val_loss = float('inf')\n",
|
||||
"early_stop_counter = 0\n",
|
||||
"\n",
|
||||
"train_losses = []\n",
|
||||
"val_losses = []\n",
|
||||
"train_accs = []\n",
|
||||
"val_accs = []\n",
|
||||
"\n",
|
||||
"print(\"🚀 Bắt đầu huấn luyện...\\n\")\n",
|
||||
"\n",
|
||||
"for epoch in range(epochs):\n",
|
||||
" # Training\n",
|
||||
" model.train()\n",
|
||||
" train_loss = 0\n",
|
||||
" train_correct = 0\n",
|
||||
" train_total = 0\n",
|
||||
" \n",
|
||||
" for X_batch, y_batch in train_loader:\n",
|
||||
" X_batch = X_batch.to(device)\n",
|
||||
" y_batch = y_batch.to(device)\n",
|
||||
" \n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" outputs = model(X_batch)\n",
|
||||
" loss = criterion(outputs, y_batch)\n",
|
||||
" loss.backward()\n",
|
||||
" optimizer.step()\n",
|
||||
" \n",
|
||||
" train_loss += loss.item()\n",
|
||||
" _, predicted = torch.max(outputs.data, 1)\n",
|
||||
" train_correct += (predicted == y_batch).sum().item()\n",
|
||||
" train_total += y_batch.size(0)\n",
|
||||
" \n",
|
||||
" train_loss /= len(train_loader)\n",
|
||||
" train_acc = 100 * train_correct / train_total\n",
|
||||
" train_losses.append(train_loss)\n",
|
||||
" train_accs.append(train_acc)\n",
|
||||
" \n",
|
||||
" # Validation\n",
|
||||
" model.eval()\n",
|
||||
" val_loss = 0\n",
|
||||
" val_correct = 0\n",
|
||||
" val_total = 0\n",
|
||||
" \n",
|
||||
" with torch.no_grad():\n",
|
||||
" for X_batch, y_batch in val_loader:\n",
|
||||
" X_batch = X_batch.to(device)\n",
|
||||
" y_batch = y_batch.to(device)\n",
|
||||
" \n",
|
||||
" outputs = model(X_batch)\n",
|
||||
" loss = criterion(outputs, y_batch)\n",
|
||||
" val_loss += loss.item()\n",
|
||||
" _, predicted = torch.max(outputs.data, 1)\n",
|
||||
" val_correct += (predicted == y_batch).sum().item()\n",
|
||||
" val_total += y_batch.size(0)\n",
|
||||
" \n",
|
||||
" val_loss /= len(val_loader)\n",
|
||||
" val_acc = 100 * val_correct / val_total\n",
|
||||
" val_losses.append(val_loss)\n",
|
||||
" val_accs.append(val_acc)\n",
|
||||
" \n",
|
||||
" # Print progress\n",
|
||||
" if (epoch + 1) % 10 == 0:\n",
|
||||
" print(f'Epoch [{epoch+1}/{epochs}]')\n",
|
||||
" print(f' Train Loss: {train_loss:.4f}, Acc: {train_acc:.2f}%')\n",
|
||||
" print(f' Val Loss: {val_loss:.4f}, Acc: {val_acc:.2f}%')\n",
|
||||
" \n",
|
||||
" # Early stopping\n",
|
||||
" scheduler.step(val_loss)\n",
|
||||
" \n",
|
||||
" if val_loss < best_val_loss:\n",
|
||||
" best_val_loss = val_loss\n",
|
||||
" early_stop_counter = 0\n",
|
||||
" # Lưu best model\n",
|
||||
" torch.save(model.state_dict(), 'model_cnn_pytorch_best.pt')\n",
|
||||
" else:\n",
|
||||
" early_stop_counter += 1\n",
|
||||
" if early_stop_counter >= early_stop_patience:\n",
|
||||
" print(f'\\n⛔ Early stopping at epoch {epoch+1}')\n",
|
||||
" break\n",
|
||||
"\n",
|
||||
"print(f'\\n✅ Huấn luyện hoàn thành!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0868ab12",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load best model\n",
|
||||
"model.load_state_dict(torch.load('model_cnn_pytorch_best.pt'))\n",
|
||||
"\n",
|
||||
"# Evaluate on test set\n",
|
||||
"model.eval()\n",
|
||||
"test_correct = 0\n",
|
||||
"test_total = 0\n",
|
||||
"test_loss = 0\n",
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" for X_batch, y_batch in test_loader:\n",
|
||||
" X_batch = X_batch.to(device)\n",
|
||||
" y_batch = y_batch.to(device)\n",
|
||||
" \n",
|
||||
" outputs = model(X_batch)\n",
|
||||
" loss = criterion(outputs, y_batch)\n",
|
||||
" test_loss += loss.item()\n",
|
||||
" _, predicted = torch.max(outputs.data, 1)\n",
|
||||
" test_correct += (predicted == y_batch).sum().item()\n",
|
||||
" test_total += y_batch.size(0)\n",
|
||||
"\n",
|
||||
"test_loss /= len(test_loader)\n",
|
||||
"test_acc = 100 * test_correct / test_total\n",
|
||||
"\n",
|
||||
"print(f\"📊 Test Results:\")\n",
|
||||
"print(f\" Test Loss: {test_loss:.4f}\")\n",
|
||||
"print(f\" Test Accuracy: {test_acc:.2f}%\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c2067227",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Vẽ đồ thị huấn luyện\n",
|
||||
"fig, axes = plt.subplots(1, 2, figsize=(15, 5))\n",
|
||||
"\n",
|
||||
"# Loss\n",
|
||||
"axes[0].plot(train_losses, label='Train Loss')\n",
|
||||
"axes[0].plot(val_losses, label='Val Loss')\n",
|
||||
"axes[0].set_xlabel('Epoch')\n",
|
||||
"axes[0].set_ylabel('Loss')\n",
|
||||
"axes[0].set_title('Model Loss')\n",
|
||||
"axes[0].legend()\n",
|
||||
"axes[0].grid(True)\n",
|
||||
"\n",
|
||||
"# Accuracy\n",
|
||||
"axes[1].plot(train_accs, label='Train Accuracy')\n",
|
||||
"axes[1].plot(val_accs, label='Val Accuracy')\n",
|
||||
"axes[1].set_xlabel('Epoch')\n",
|
||||
"axes[1].set_ylabel('Accuracy (%)')\n",
|
||||
"axes[1].set_title('Model Accuracy')\n",
|
||||
"axes[1].legend()\n",
|
||||
"axes[1].grid(True)\n",
|
||||
"\n",
|
||||
"plt.tight_layout()\n",
|
||||
"plt.savefig('training_history.png', dpi=150, bbox_inches='tight')\n",
|
||||
"plt.show()\n",
|
||||
"\n",
|
||||
"print(\"✅ Đồ thị đã lưu vào 'training_history.png'\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d8402a21",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Lưu Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c12de183",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Lưu model state dict\n",
|
||||
"print(\"💾 Lưu model...\")\n",
|
||||
"torch.save(model.state_dict(), 'model_cnn_pytorch.pt')\n",
|
||||
"print(f\"✅ Model state dict đã lưu: model_cnn_pytorch.pt\")\n",
|
||||
"\n",
|
||||
"# Lưu thông tin model\n",
|
||||
"model_info = {\n",
|
||||
" 'state_dict': model.state_dict(),\n",
|
||||
" 'num_classes': num_classes,\n",
|
||||
" 'input_size': X_train.shape[1],\n",
|
||||
" 'label_mapping': label_mapping,\n",
|
||||
" 'mean': mean,\n",
|
||||
" 'std': std,\n",
|
||||
" 'test_accuracy': test_acc,\n",
|
||||
" 'test_loss': test_loss\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"torch.save(model_info, 'model_cnn_pytorch_full.pt')\n",
|
||||
"print(f\"✅ Full model info đã lưu: model_cnn_pytorch_full.pt\")\n",
|
||||
"\n",
|
||||
"print(f\"\\n✅ Model ready for prediction!\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d2134b50",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Training CNN Model with PyTorch for Land Use Classification\n",
|
||||
"Huấn luyện mô hình CNN với PyTorch cho phân loại sử dụng đất từ dữ liệu Sentinel-1 và Sentinel-2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5b9ca134",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import new_import_ODC \n",
|
||||
"\n",
|
||||
"importlib.reload(new_import_ODC)\n",
|
||||
"\n",
|
||||
"from new_import_ODC import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "709c2518",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Kiểm tra GPU availability\n",
|
||||
"print(f\"PyTorch version: {torch.__version__}\")\n",
|
||||
"print(f\"CUDA available: {torch.cuda.is_available()}\")\n",
|
||||
"if torch.cuda.is_available():\n",
|
||||
" print(f\"CUDA device: {torch.cuda.get_device_name(0)}\")\n",
|
||||
" device = 'cuda'\n",
|
||||
"else:\n",
|
||||
" print(\"Using CPU for training\")\n",
|
||||
" device = 'cpu'\n",
|
||||
"\n",
|
||||
"print(f\"\\nDevice sẽ dùng: {device}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "15f073ca",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Cấu hình Daskgateway\n",
|
||||
"cluster, client = notebook_utils.initialize_dask(use_gateway=True, workers=(1, 10))\n",
|
||||
"# Khai báo 1 Datacube là dc\n",
|
||||
"dc = datacube.Datacube()\n",
|
||||
"\n",
|
||||
"# Cấu hình truy cập dịch vụ S3\n",
|
||||
"configure_s3_access(aws_unsigned=False, requester_pays=True, client=client)\n",
|
||||
"\n",
|
||||
"client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "74a229f6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình thời gian lấy ảnh và tọa độ\n",
|
||||
"date_range = (\"2022-09-01\", \"2023-10-01\")\n",
|
||||
"longtitude_range = (105.5, 106.4)\n",
|
||||
"latitude_range = (9.2, 10.0)\n",
|
||||
"\n",
|
||||
"coordinates = (longtitude_range, latitude_range)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1ebdade4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## truy vấn ảnh vệ tinh sen2\n",
|
||||
"data = load_data(dc, date_range, longtitude_range, latitude_range)\n",
|
||||
"notebook_utils.heading(notebook_utils.xarray_object_size(data))\n",
|
||||
"display(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c444f3cc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Tiến hành loại bỏ các vị trí bị mây ảnh hưởng\n",
|
||||
"result = mask_clean(data)\n",
|
||||
"progress(result)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f0df6581",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Tiến hành tính toán NDVI\n",
|
||||
"ds1 = calculate_indices(result, index=\"NDVI\", satellite_mission=\"s2\")\n",
|
||||
"ndvi = ds1[\"NDVI\"]\n",
|
||||
"display(ndvi)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "430302e3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## Hiển thị ảnh NDVI chưa điền các giá trị mây (chưa fill nan)\n",
|
||||
"plt.imshow(ndvi.isel(time=6))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6798b0d2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Thiết lập giá trị trung bình mùa vụ để xử lý các điểm ảnh bị mây dựa vào sự thay đổi theo mùa\n",
|
||||
"time_split = [\n",
|
||||
" slice(\"2022-09-01\", \"2023-01-01\"),\n",
|
||||
" slice(\"2023-01-01\", \"2023-05-01\"),\n",
|
||||
" slice(\"2023-05-01\", \"2023-07-01\"),\n",
|
||||
" slice(\"2023-07-01\", \"2023-10-01\"),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# Điền mây ở các vị trí mang giá trị nan (fill nan)\n",
|
||||
"fill_nan_ndvi = fill_nan(ndvi, time_split)\n",
|
||||
"\n",
|
||||
"# In kết quả ảnh NDVI đã điền mây (đã fill nan)\n",
|
||||
"plt.imshow(fill_nan_ndvi.isel(time=6))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0e10d6e6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"## tính ndvi theo tháng\n",
|
||||
"average_ndvi = fill_nan_ndvi.resample(time=\"1M\").mean().persist()\n",
|
||||
"progress(average_ndvi)\n",
|
||||
"\n",
|
||||
"# compute average_ndvi\n",
|
||||
"average_ndvi = average_ndvi.compute()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7b3b7086",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#Load dữ liệu ảnh Sentinel 1\n",
|
||||
"dsvh, dsvv = load_data_sen1(dc, date_range, coordinates)\n",
|
||||
"average_vv = calculate_average(dsvv, time_pattern='1M')\n",
|
||||
"average_vh = calculate_average(dsvh, time_pattern='1M')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6d3f9ab1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## cấu hình bộ dữ liệu điểm huấn luyện mô hình (train file)\n",
|
||||
"train_path = \"train/ST_training data_updated_1130points_new.shp\" # đường dẫn shp file train\n",
|
||||
"\n",
|
||||
"## load dữ liệu điểm huấn luyện mô hình (train file)\n",
|
||||
"train = load_train_data(train_path)\n",
|
||||
"train.head()\n",
|
||||
"\n",
|
||||
"# cấu hình nhãn dữ liệu \n",
|
||||
"label_mapping = {\n",
|
||||
" \"Lua tom\": \"0\",\n",
|
||||
" \"Lua\": \"1\",\n",
|
||||
" \"CHN\": \"2\",\n",
|
||||
" \"CLN\": \"3\",\n",
|
||||
" \"TS\": \"4\",\n",
|
||||
" \"Song\": \"5\",\n",
|
||||
" \"Dat xay dung\": \"6\",\n",
|
||||
" \"Rung\": \"7\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# xây dựng tập dữ liệu (dataset) chứa dữ liệu VH, VV, NDVI\n",
|
||||
"datasets = get_data_sen1_and_sen2(train, average_ndvi, average_vh, average_vv)\n",
|
||||
"\n",
|
||||
"# chia tập dữ liệu thành các phần theo tỉ lệ 80(80-20)-20 tương ứng với tập train, validate, test\n",
|
||||
"X_train, X_val, X_test, y_train, y_val, y_test = split_train_data(\n",
|
||||
" train, label_mapping, datasets\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c0188e1b",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Huấn luyện mô hình CNN với PyTorch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ba38ce6a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Huấn luyện mô hình CNN với PyTorch\n",
|
||||
"print(\"🔷 Bắt đầu huấn luyện CNN model với PyTorch...\\n\")\n",
|
||||
"\n",
|
||||
"cnn_model, history, scaler = train_cnn_pytorch(\n",
|
||||
" X_train, X_val, X_test, \n",
|
||||
" y_train, y_val, y_test,\n",
|
||||
" num_classes=8,\n",
|
||||
" epochs=100,\n",
|
||||
" batch_size=32,\n",
|
||||
" learning_rate=1e-3,\n",
|
||||
" device=device,\n",
|
||||
" verbose=True\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fd19df06",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Vẽ đồ thị huấn luyện\n",
|
||||
"plot_pytorch_training_history(history)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e16d8c20",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Lưu mô hình CNN PyTorch\n",
|
||||
"save_pytorch_model(cnn_model, scaler, model_name=\"model_cnn_pytorch.pth\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "adb17862",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Hiển thị model architecture\n",
|
||||
"print(\"\\n📐 Model Architecture:\")\n",
|
||||
"print(cnn_model)\n",
|
||||
"\n",
|
||||
"# Đếm số parameters\n",
|
||||
"total_params = sum(p.numel() for p in cnn_model.parameters())\n",
|
||||
"trainable_params = sum(p.numel() for p in cnn_model.parameters() if p.requires_grad)\n",
|
||||
"print(f\"\\n📊 Model Parameters:\")\n",
|
||||
"print(f\" Total parameters: {total_params:,}\")\n",
|
||||
"print(f\" Trainable parameters: {trainable_params:,}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b4963d6f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# đóng client, cluster\n",
|
||||
"client.close()\n",
|
||||
"cluster.close()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "ac069784-fbb6-48c8-a547-11b96cade97b",
|
||||
"metadata": {
|
||||
"collapsed": true,
|
||||
"jupyter": {
|
||||
"outputs_hidden": true,
|
||||
"source_hidden": true
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_218/1243805045.py:22: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
|
||||
" df = pd.read_sql(sql, conn)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" product_name total_images\n",
|
||||
"0 s2_l2a 2074896\n",
|
||||
"1 sentinel_2_c1_l2a 1137446\n",
|
||||
"2 landsat7_c2l2_sr 169065\n",
|
||||
"3 landsat7_c2l2_st 168410\n",
|
||||
"4 landsat8_c2l2_sr 122846\n",
|
||||
"5 landsat8_c2l2_st 120778\n",
|
||||
"6 landsat5_c2l2_sr 118583\n",
|
||||
"7 landsat5_c2l2_st 118406\n",
|
||||
"8 landsat8_c2l1 111513\n",
|
||||
"9 nasa_aqua_l2_oc 37232\n",
|
||||
"10 landsat9_c2l2_sr 30214\n",
|
||||
"11 landsat9_c2l2_st 29709\n",
|
||||
"12 landsat9_c2l1 18875\n",
|
||||
"13 nasa_aqua_l2_sst 12070\n",
|
||||
"14 ga_ls_mangrove_cover_cyear_3 5580\n",
|
||||
"15 sentinel1_grd_gamma0_10m_unsmooth 5488\n",
|
||||
"16 sentinel1_grd_gamma0_20m 4260\n",
|
||||
"17 copernicus_dem_30 1334\n",
|
||||
"18 global_mangrove_soc_2020_30m 784\n",
|
||||
"19 global_mangrove_soc_2000_30m 781\n",
|
||||
"20 global_mangrove_canopy_height_2015_12m 761\n",
|
||||
"21 sentinel1_grd_gamma0_10m 291\n",
|
||||
"22 lpdaac_nasadem 264\n",
|
||||
"23 cci_biomass_annual_v51 248\n",
|
||||
"24 esa_worldcover_2020 184\n",
|
||||
"25 esa_worldcover_2021 184\n",
|
||||
"26 global_tidal_marsh_soc_30m 41\n",
|
||||
"27 global_tidal_marsh_distribution_2020_10m 35\n",
|
||||
"28 copernicus_dem_fiji 26\n",
|
||||
"29 lpdaac_mod11a1v061_lste 2\n",
|
||||
"30 global_mangrove_soc_2000_100m 1\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# Kết nối (như cũ)\n",
|
||||
"conn = psycopg2.connect(\n",
|
||||
" host=os.environ.get('DB_HOSTNAME'),\n",
|
||||
" user=os.environ.get('DB_USERNAME'),\n",
|
||||
" password=os.environ.get('DB_PASSWORD'),\n",
|
||||
" dbname=os.environ.get('DB_DATABASE')\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Đếm số lượng ảnh theo từng loại sản phẩm\n",
|
||||
"sql = \"\"\"\n",
|
||||
"SELECT t.name as product_name, count(*) as total_images\n",
|
||||
"FROM agdc.dataset d\n",
|
||||
"JOIN agdc.dataset_type t ON d.dataset_type_ref = t.id\n",
|
||||
"GROUP BY t.name\n",
|
||||
"ORDER BY total_images DESC;\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"df = pd.read_sql(sql, conn)\n",
|
||||
"print(df)\n",
|
||||
"conn.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a0ae4051-3bfd-49fd-83fe-4c31c74338e2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# 1. Lấy thông tin kết nối\n",
|
||||
"db_host = os.environ.get('DB_HOSTNAME')\n",
|
||||
"db_user = os.environ.get('DB_USERNAME')\n",
|
||||
"db_pass = os.environ.get('DB_PASSWORD')\n",
|
||||
"db_name = os.environ.get('DB_DATABASE')\n",
|
||||
"\n",
|
||||
"# 2. Danh sách các bảng cần lấy\n",
|
||||
"tables = [\n",
|
||||
" 'agdc.metadata_type',\n",
|
||||
" 'agdc.dataset_type',\n",
|
||||
" 'agdc.dataset_location',\n",
|
||||
" 'agdc.dataset'\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(\"Đang kết nối trực tiếp tới PostgreSQL...\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" # Kết nối trực tiếp (Bỏ qua Pandas/SQLAlchemy)\n",
|
||||
" conn = psycopg2.connect(\n",
|
||||
" host=db_host,\n",
|
||||
" user=db_user,\n",
|
||||
" password=db_pass,\n",
|
||||
" dbname=db_name\n",
|
||||
" )\n",
|
||||
" cur = conn.cursor()\n",
|
||||
" print(\"Kết nối thành công!\\n\")\n",
|
||||
"\n",
|
||||
" for table_name in tables:\n",
|
||||
" print(f\"--> Đang xuất bảng: {table_name}\")\n",
|
||||
" \n",
|
||||
" # Tên file CSV đầu ra\n",
|
||||
" file_name = table_name.split('.')[1] + \".csv\"\n",
|
||||
" \n",
|
||||
" # Sử dụng lệnh COPY chuyên dụng của Postgres (Cực nhanh và chuẩn)\n",
|
||||
" # SQL: COPY (SELECT * FROM table) TO STDOUT WITH CSV HEADER\n",
|
||||
" sql = f\"COPY (SELECT * FROM {table_name}) TO STDOUT WITH CSV HEADER\"\n",
|
||||
" \n",
|
||||
" with open(file_name, 'w') as f:\n",
|
||||
" cur.copy_expert(sql, f)\n",
|
||||
" \n",
|
||||
" print(f\" Đã lưu xong: {file_name}\")\n",
|
||||
"\n",
|
||||
" cur.close()\n",
|
||||
" conn.close()\n",
|
||||
" print(\"\\nHOÀN TẤT! Bạn hãy tải 4 file CSV về máy.\")\n",
|
||||
"\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"\\nCÓ LỖI XẢY RA: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d139879e-c350-4986-a2e6-0d59099eb770",
|
||||
"metadata": {
|
||||
"collapsed": true,
|
||||
"jupyter": {
|
||||
"outputs_hidden": true,
|
||||
"source_hidden": true
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Đang kết nối...\n",
|
||||
"Kết nối thành công! Đang ước lượng nhanh...\n",
|
||||
"\n",
|
||||
"--- BẢNG DỰ TÍNH DUNG LƯỢNG (ƯỚC LƯỢNG) ---\n",
|
||||
"NĂM | SỐ ẢNH (ƯỚC TÍNH) | SIZE (CSV) \n",
|
||||
"------------------------------------------------------------\n",
|
||||
"2020 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2021 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2022 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2023 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2024 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"TỔNG CỘNG : ~ 107,240 ảnh (Khoảng 418.91 MB)\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"(Lưu ý: Số liệu này là ước tính của Database, độ chính xác khoảng 80-90%)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdin",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"Bạn có muốn bắt đầu tải không? (y/n): y\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
">>> Đang tải bảng định nghĩa...\n",
|
||||
"\n",
|
||||
">>> Đang tải dữ liệu chính...\n",
|
||||
"--> Năm 2020...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"import re # Thư viện xử lý chuỗi để lấy số liệu\n",
|
||||
"\n",
|
||||
"# --- CẤU HÌNH ---\n",
|
||||
"YEARS = [2020, 2021, 2022, 2023, 2024]\n",
|
||||
"AVG_ROW_SIZE_KB = 4 \n",
|
||||
"\n",
|
||||
"db_host = os.environ.get('DB_HOSTNAME')\n",
|
||||
"db_user = os.environ.get('DB_USERNAME')\n",
|
||||
"db_pass = os.environ.get('DB_PASSWORD')\n",
|
||||
"db_name = os.environ.get('DB_DATABASE')\n",
|
||||
"\n",
|
||||
"print(\"Đang kết nối...\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" conn = psycopg2.connect(\n",
|
||||
" host=db_host, user=db_user, password=db_pass, dbname=db_name\n",
|
||||
" )\n",
|
||||
" cur = conn.cursor()\n",
|
||||
" print(\"Kết nối thành công! Đang ước lượng nhanh...\\n\")\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # BƯỚC 1: ƯỚC LƯỢNG SIÊU TỐC (INSTANT ESTIMATE)\n",
|
||||
" # ==========================================\n",
|
||||
" print(\"--- BẢNG DỰ TÍNH DUNG LƯỢNG (ƯỚC LƯỢNG) ---\")\n",
|
||||
" print(f\"{'NĂM':<10} | {'SỐ ẢNH (ƯỚC TÍNH)':<20} | {'SIZE (CSV)':<20}\")\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
"\n",
|
||||
" total_est_rows = 0\n",
|
||||
"\n",
|
||||
" for year in YEARS:\n",
|
||||
" # Mẹo: Dùng EXPLAIN để lấy số liệu ước tính từ Query Planner\n",
|
||||
" # Nó sẽ trả về chuỗi kiểu: \"Seq Scan on dataset ... (rows=12345 ...)\"\n",
|
||||
" sql_estimate = f\"\"\"\n",
|
||||
" EXPLAIN SELECT 1 FROM agdc.dataset\n",
|
||||
" WHERE (metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01'\n",
|
||||
" AND (metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" \"\"\"\n",
|
||||
" cur.execute(sql_estimate)\n",
|
||||
" explain_result = cur.fetchone()[0] # Lấy dòng đầu tiên của kết quả EXPLAIN\n",
|
||||
" \n",
|
||||
" # Dùng Regex để bắt lấy con số sau chữ \"rows=\"\n",
|
||||
" match = re.search(r\"rows=(\\d+)\", explain_result)\n",
|
||||
" if match:\n",
|
||||
" count = int(match.group(1))\n",
|
||||
" else:\n",
|
||||
" count = 0 # Không bắt được số\n",
|
||||
" \n",
|
||||
" total_est_rows += count\n",
|
||||
" est_size_mb = (count * AVG_ROW_SIZE_KB) / 1024\n",
|
||||
" \n",
|
||||
" print(f\"{year:<10} | ~ {count:<18,} | ~ {est_size_mb:.2f} MB\")\n",
|
||||
"\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" print(f\"TỔNG CỘNG : ~ {total_est_rows:,} ảnh (Khoảng {(total_est_rows * AVG_ROW_SIZE_KB)/1024:.2f} MB)\")\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" print(\"(Lưu ý: Số liệu này là ước tính của Database, độ chính xác khoảng 80-90%)\")\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # QUYẾT ĐỊNH TẢI\n",
|
||||
" # ==========================================\n",
|
||||
" check = input(\"\\nBạn có muốn bắt đầu tải không? (y/n): \")\n",
|
||||
" if check.lower() != 'y':\n",
|
||||
" print(\"Đã hủy.\")\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # BƯỚC 2 & 3: TẢI DỮ LIỆU (Giữ nguyên logic cũ)\n",
|
||||
" # ==========================================\n",
|
||||
" # ... (Phần code tải small_tables và tải dữ liệu chính giữ nguyên như cũ) ...\n",
|
||||
" # Để code gọn, mình viết tiếp phần tải ở dưới đây:\n",
|
||||
" \n",
|
||||
" # 2. Tải bảng nhỏ\n",
|
||||
" print(\"\\n>>> Đang tải bảng định nghĩa...\")\n",
|
||||
" for tb in ['agdc.metadata_type', 'agdc.dataset_type']:\n",
|
||||
" f_name = tb.split('.')[1] + \".csv\"\n",
|
||||
" with open(f_name, 'w') as f:\n",
|
||||
" cur.copy_expert(f\"COPY (SELECT * FROM {tb}) TO STDOUT WITH CSV HEADER\", f)\n",
|
||||
" \n",
|
||||
" # 3. Tải dữ liệu chính\n",
|
||||
" print(\"\\n>>> Đang tải dữ liệu chính...\")\n",
|
||||
" for year in YEARS:\n",
|
||||
" print(f\"--> Năm {year}...\")\n",
|
||||
" \n",
|
||||
" # Dataset\n",
|
||||
" f_ds = f\"dataset_{year}.csv\"\n",
|
||||
" sql_ds = f\"\"\"\n",
|
||||
" COPY (SELECT * FROM agdc.dataset \n",
|
||||
" WHERE (metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01' \n",
|
||||
" AND (metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" ) TO STDOUT WITH CSV HEADER\"\"\"\n",
|
||||
" with open(f_ds, 'w') as f: cur.copy_expert(sql_ds, f)\n",
|
||||
" \n",
|
||||
" # Location\n",
|
||||
" f_loc = f\"dataset_location_{year}.csv\"\n",
|
||||
" sql_loc = f\"\"\"\n",
|
||||
" COPY (SELECT l.* FROM agdc.dataset_location l JOIN agdc.dataset d ON l.dataset_ref = d.id\n",
|
||||
" WHERE (d.metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01' \n",
|
||||
" AND (d.metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" ) TO STDOUT WITH CSV HEADER\"\"\"\n",
|
||||
" with open(f_loc, 'w') as f: cur.copy_expert(sql_loc, f)\n",
|
||||
" \n",
|
||||
" print(\"\\nHOÀN TẤT TOÀN BỘ!\")\n",
|
||||
" cur.close()\n",
|
||||
" conn.close()\n",
|
||||
"\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"\\nCÓ LỖI: {e}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"cells": [],
|
||||
"metadata": {},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,495 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "ac069784-fbb6-48c8-a547-11b96cade97b",
|
||||
"metadata": {
|
||||
"collapsed": true,
|
||||
"jupyter": {
|
||||
"outputs_hidden": true,
|
||||
"source_hidden": true
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_218/1243805045.py:22: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
|
||||
" df = pd.read_sql(sql, conn)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" product_name total_images\n",
|
||||
"0 s2_l2a 2074896\n",
|
||||
"1 sentinel_2_c1_l2a 1137446\n",
|
||||
"2 landsat7_c2l2_sr 169065\n",
|
||||
"3 landsat7_c2l2_st 168410\n",
|
||||
"4 landsat8_c2l2_sr 122846\n",
|
||||
"5 landsat8_c2l2_st 120778\n",
|
||||
"6 landsat5_c2l2_sr 118583\n",
|
||||
"7 landsat5_c2l2_st 118406\n",
|
||||
"8 landsat8_c2l1 111513\n",
|
||||
"9 nasa_aqua_l2_oc 37232\n",
|
||||
"10 landsat9_c2l2_sr 30214\n",
|
||||
"11 landsat9_c2l2_st 29709\n",
|
||||
"12 landsat9_c2l1 18875\n",
|
||||
"13 nasa_aqua_l2_sst 12070\n",
|
||||
"14 ga_ls_mangrove_cover_cyear_3 5580\n",
|
||||
"15 sentinel1_grd_gamma0_10m_unsmooth 5488\n",
|
||||
"16 sentinel1_grd_gamma0_20m 4260\n",
|
||||
"17 copernicus_dem_30 1334\n",
|
||||
"18 global_mangrove_soc_2020_30m 784\n",
|
||||
"19 global_mangrove_soc_2000_30m 781\n",
|
||||
"20 global_mangrove_canopy_height_2015_12m 761\n",
|
||||
"21 sentinel1_grd_gamma0_10m 291\n",
|
||||
"22 lpdaac_nasadem 264\n",
|
||||
"23 cci_biomass_annual_v51 248\n",
|
||||
"24 esa_worldcover_2020 184\n",
|
||||
"25 esa_worldcover_2021 184\n",
|
||||
"26 global_tidal_marsh_soc_30m 41\n",
|
||||
"27 global_tidal_marsh_distribution_2020_10m 35\n",
|
||||
"28 copernicus_dem_fiji 26\n",
|
||||
"29 lpdaac_mod11a1v061_lste 2\n",
|
||||
"30 global_mangrove_soc_2000_100m 1\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# Kết nối (như cũ)\n",
|
||||
"conn = psycopg2.connect(\n",
|
||||
" host=os.environ.get('DB_HOSTNAME'),\n",
|
||||
" user=os.environ.get('DB_USERNAME'),\n",
|
||||
" password=os.environ.get('DB_PASSWORD'),\n",
|
||||
" dbname=os.environ.get('DB_DATABASE')\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Đếm số lượng ảnh theo từng loại sản phẩm\n",
|
||||
"sql = \"\"\"\n",
|
||||
"SELECT t.name as product_name, count(*) as total_images\n",
|
||||
"FROM agdc.dataset d\n",
|
||||
"JOIN agdc.dataset_type t ON d.dataset_type_ref = t.id\n",
|
||||
"GROUP BY t.name\n",
|
||||
"ORDER BY total_images DESC;\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"df = pd.read_sql(sql, conn)\n",
|
||||
"print(df)\n",
|
||||
"conn.close()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "a0ae4051-3bfd-49fd-83fe-4c31c74338e2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Đang kết nối trực tiếp tới PostgreSQL...\n",
|
||||
"Kết nối thành công!\n",
|
||||
"\n",
|
||||
"--> Đang xuất bảng: agdc.metadata_type\n",
|
||||
" Đã lưu xong: metadata_type.csv\n",
|
||||
"--> Đang xuất bảng: agdc.dataset_type\n",
|
||||
" Đã lưu xong: dataset_type.csv\n",
|
||||
"--> Đang xuất bảng: agdc.dataset_location\n",
|
||||
" Đã lưu xong: dataset_location.csv\n",
|
||||
"--> Đang xuất bảng: agdc.dataset\n",
|
||||
" Đã lưu xong: dataset.csv\n",
|
||||
"\n",
|
||||
"HOÀN TẤT! Bạn hãy tải 4 file CSV về máy.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# 1. Lấy thông tin kết nối\n",
|
||||
"db_host = os.environ.get('DB_HOSTNAME')\n",
|
||||
"db_user = os.environ.get('DB_USERNAME')\n",
|
||||
"db_pass = os.environ.get('DB_PASSWORD')\n",
|
||||
"db_name = os.environ.get('DB_DATABASE')\n",
|
||||
"\n",
|
||||
"# 2. Danh sách các bảng cần lấy\n",
|
||||
"tables = [\n",
|
||||
" 'agdc.metadata_type',\n",
|
||||
" 'agdc.dataset_type',\n",
|
||||
" 'agdc.dataset_location',\n",
|
||||
" 'agdc.dataset'\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(\"Đang kết nối trực tiếp tới PostgreSQL...\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" # Kết nối trực tiếp (Bỏ qua Pandas/SQLAlchemy)\n",
|
||||
" conn = psycopg2.connect(\n",
|
||||
" host=db_host,\n",
|
||||
" user=db_user,\n",
|
||||
" password=db_pass,\n",
|
||||
" dbname=db_name\n",
|
||||
" )\n",
|
||||
" cur = conn.cursor()\n",
|
||||
" print(\"Kết nối thành công!\\n\")\n",
|
||||
"\n",
|
||||
" for table_name in tables:\n",
|
||||
" print(f\"--> Đang xuất bảng: {table_name}\")\n",
|
||||
" \n",
|
||||
" # Tên file CSV đầu ra\n",
|
||||
" file_name = table_name.split('.')[1] + \".csv\"\n",
|
||||
" \n",
|
||||
" # Sử dụng lệnh COPY chuyên dụng của Postgres (Cực nhanh và chuẩn)\n",
|
||||
" # SQL: COPY (SELECT * FROM table) TO STDOUT WITH CSV HEADER\n",
|
||||
" sql = f\"COPY (SELECT * FROM {table_name}) TO STDOUT WITH CSV HEADER\"\n",
|
||||
" \n",
|
||||
" with open(file_name, 'w') as f:\n",
|
||||
" cur.copy_expert(sql, f)\n",
|
||||
" \n",
|
||||
" print(f\" Đã lưu xong: {file_name}\")\n",
|
||||
"\n",
|
||||
" cur.close()\n",
|
||||
" conn.close()\n",
|
||||
" print(\"\\nHOÀN TẤT! Bạn hãy tải 4 file CSV về máy.\")\n",
|
||||
"\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"\\nCÓ LỖI XẢY RA: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d139879e-c350-4986-a2e6-0d59099eb770",
|
||||
"metadata": {
|
||||
"collapsed": true,
|
||||
"jupyter": {
|
||||
"outputs_hidden": true,
|
||||
"source_hidden": true
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Đang kết nối...\n",
|
||||
"Kết nối thành công! Đang ước lượng nhanh...\n",
|
||||
"\n",
|
||||
"--- BẢNG DỰ TÍNH DUNG LƯỢNG (ƯỚC LƯỢNG) ---\n",
|
||||
"NĂM | SỐ ẢNH (ƯỚC TÍNH) | SIZE (CSV) \n",
|
||||
"------------------------------------------------------------\n",
|
||||
"2020 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2021 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2022 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2023 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"2024 | ~ 21,448 | ~ 83.78 MB\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"TỔNG CỘNG : ~ 107,240 ảnh (Khoảng 418.91 MB)\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"(Lưu ý: Số liệu này là ước tính của Database, độ chính xác khoảng 80-90%)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdin",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"Bạn có muốn bắt đầu tải không? (y/n): y\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
">>> Đang tải bảng định nghĩa...\n",
|
||||
"\n",
|
||||
">>> Đang tải dữ liệu chính...\n",
|
||||
"--> Năm 2020...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"import re # Thư viện xử lý chuỗi để lấy số liệu\n",
|
||||
"\n",
|
||||
"# --- CẤU HÌNH ---\n",
|
||||
"YEARS = [2020, 2021, 2022, 2023, 2024]\n",
|
||||
"AVG_ROW_SIZE_KB = 4 \n",
|
||||
"\n",
|
||||
"db_host = os.environ.get('DB_HOSTNAME')\n",
|
||||
"db_user = os.environ.get('DB_USERNAME')\n",
|
||||
"db_pass = os.environ.get('DB_PASSWORD')\n",
|
||||
"db_name = os.environ.get('DB_DATABASE')\n",
|
||||
"\n",
|
||||
"print(\"Đang kết nối...\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" conn = psycopg2.connect(\n",
|
||||
" host=db_host, user=db_user, password=db_pass, dbname=db_name\n",
|
||||
" )\n",
|
||||
" cur = conn.cursor()\n",
|
||||
" print(\"Kết nối thành công! Đang ước lượng nhanh...\\n\")\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # BƯỚC 1: ƯỚC LƯỢNG SIÊU TỐC (INSTANT ESTIMATE)\n",
|
||||
" # ==========================================\n",
|
||||
" print(\"--- BẢNG DỰ TÍNH DUNG LƯỢNG (ƯỚC LƯỢNG) ---\")\n",
|
||||
" print(f\"{'NĂM':<10} | {'SỐ ẢNH (ƯỚC TÍNH)':<20} | {'SIZE (CSV)':<20}\")\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
"\n",
|
||||
" total_est_rows = 0\n",
|
||||
"\n",
|
||||
" for year in YEARS:\n",
|
||||
" # Mẹo: Dùng EXPLAIN để lấy số liệu ước tính từ Query Planner\n",
|
||||
" # Nó sẽ trả về chuỗi kiểu: \"Seq Scan on dataset ... (rows=12345 ...)\"\n",
|
||||
" sql_estimate = f\"\"\"\n",
|
||||
" EXPLAIN SELECT 1 FROM agdc.dataset\n",
|
||||
" WHERE (metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01'\n",
|
||||
" AND (metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" \"\"\"\n",
|
||||
" cur.execute(sql_estimate)\n",
|
||||
" explain_result = cur.fetchone()[0] # Lấy dòng đầu tiên của kết quả EXPLAIN\n",
|
||||
" \n",
|
||||
" # Dùng Regex để bắt lấy con số sau chữ \"rows=\"\n",
|
||||
" match = re.search(r\"rows=(\\d+)\", explain_result)\n",
|
||||
" if match:\n",
|
||||
" count = int(match.group(1))\n",
|
||||
" else:\n",
|
||||
" count = 0 # Không bắt được số\n",
|
||||
" \n",
|
||||
" total_est_rows += count\n",
|
||||
" est_size_mb = (count * AVG_ROW_SIZE_KB) / 1024\n",
|
||||
" \n",
|
||||
" print(f\"{year:<10} | ~ {count:<18,} | ~ {est_size_mb:.2f} MB\")\n",
|
||||
"\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" print(f\"TỔNG CỘNG : ~ {total_est_rows:,} ảnh (Khoảng {(total_est_rows * AVG_ROW_SIZE_KB)/1024:.2f} MB)\")\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" print(\"(Lưu ý: Số liệu này là ước tính của Database, độ chính xác khoảng 80-90%)\")\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # QUYẾT ĐỊNH TẢI\n",
|
||||
" # ==========================================\n",
|
||||
" check = input(\"\\nBạn có muốn bắt đầu tải không? (y/n): \")\n",
|
||||
" if check.lower() != 'y':\n",
|
||||
" print(\"Đã hủy.\")\n",
|
||||
" exit()\n",
|
||||
"\n",
|
||||
" # ==========================================\n",
|
||||
" # BƯỚC 2 & 3: TẢI DỮ LIỆU (Giữ nguyên logic cũ)\n",
|
||||
" # ==========================================\n",
|
||||
" # ... (Phần code tải small_tables và tải dữ liệu chính giữ nguyên như cũ) ...\n",
|
||||
" # Để code gọn, mình viết tiếp phần tải ở dưới đây:\n",
|
||||
" \n",
|
||||
" # 2. Tải bảng nhỏ\n",
|
||||
" print(\"\\n>>> Đang tải bảng định nghĩa...\")\n",
|
||||
" for tb in ['agdc.metadata_type', 'agdc.dataset_type']:\n",
|
||||
" f_name = tb.split('.')[1] + \".csv\"\n",
|
||||
" with open(f_name, 'w') as f:\n",
|
||||
" cur.copy_expert(f\"COPY (SELECT * FROM {tb}) TO STDOUT WITH CSV HEADER\", f)\n",
|
||||
" \n",
|
||||
" # 3. Tải dữ liệu chính\n",
|
||||
" print(\"\\n>>> Đang tải dữ liệu chính...\")\n",
|
||||
" for year in YEARS:\n",
|
||||
" print(f\"--> Năm {year}...\")\n",
|
||||
" \n",
|
||||
" # Dataset\n",
|
||||
" f_ds = f\"dataset_{year}.csv\"\n",
|
||||
" sql_ds = f\"\"\"\n",
|
||||
" COPY (SELECT * FROM agdc.dataset \n",
|
||||
" WHERE (metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01' \n",
|
||||
" AND (metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" ) TO STDOUT WITH CSV HEADER\"\"\"\n",
|
||||
" with open(f_ds, 'w') as f: cur.copy_expert(sql_ds, f)\n",
|
||||
" \n",
|
||||
" # Location\n",
|
||||
" f_loc = f\"dataset_location_{year}.csv\"\n",
|
||||
" sql_loc = f\"\"\"\n",
|
||||
" COPY (SELECT l.* FROM agdc.dataset_location l JOIN agdc.dataset d ON l.dataset_ref = d.id\n",
|
||||
" WHERE (d.metadata->'properties'->>'datetime')::timestamp >= '{year}-01-01' \n",
|
||||
" AND (d.metadata->'properties'->>'datetime')::timestamp <= '{year}-12-31 23:59:59'\n",
|
||||
" ) TO STDOUT WITH CSV HEADER\"\"\"\n",
|
||||
" with open(f_loc, 'w') as f: cur.copy_expert(sql_loc, f)\n",
|
||||
" \n",
|
||||
" print(\"\\nHOÀN TẤT TOÀN BỘ!\")\n",
|
||||
" cur.close()\n",
|
||||
" conn.close()\n",
|
||||
"\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"\\nCÓ LỖI: {e}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "8a5bb199-9363-47cc-9640-bf7e0af33d19",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Đang quét kho dữ liệu (Điều này có thể mất 1-2 phút)...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/tmp/ipykernel_499/84613258.py:40: UserWarning: pandas only supports SQLAlchemy connectable (engine/connection) or database string URI or sqlite3 DBAPI2 connection. Other DBAPI2 objects are not tested. Please consider using SQLAlchemy.\n",
|
||||
" df = pd.read_sql(sql, conn)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"============================================================\n",
|
||||
"PRODUCT NAME | SỐ LƯỢNG | DUNG LƯỢNG (TB)\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"s2_l2a | 2,074,896 | 1621.01 TB\n",
|
||||
"sentinel_2_c1_l2a | 1,137,446 | 555.39 TB\n",
|
||||
"landsat7_c2l2_sr | 169,065 | 82.55 TB\n",
|
||||
"landsat7_c2l2_st | 168,410 | 82.23 TB\n",
|
||||
"landsat8_c2l2_sr | 122,846 | 59.98 TB\n",
|
||||
"landsat8_c2l2_st | 120,778 | 58.97 TB\n",
|
||||
"landsat5_c2l2_sr | 118,583 | 57.90 TB\n",
|
||||
"landsat5_c2l2_st | 118,406 | 57.82 TB\n",
|
||||
"landsat8_c2l1 | 111,513 | 54.45 TB\n",
|
||||
"nasa_aqua_l2_oc | 37,232 | 18.18 TB\n",
|
||||
"landsat9_c2l2_sr | 30,214 | 14.75 TB\n",
|
||||
"landsat9_c2l2_st | 29,709 | 14.51 TB\n",
|
||||
"landsat9_c2l1 | 18,875 | 9.22 TB\n",
|
||||
"nasa_aqua_l2_sst | 12,070 | 5.89 TB\n",
|
||||
"ga_ls_mangrove_cover_cyear_3 | 5,580 | 2.72 TB\n",
|
||||
"sentinel1_grd_gamma0_10m_unsmooth | 5,488 | 2.68 TB\n",
|
||||
"sentinel1_grd_gamma0_20m | 4,260 | 2.08 TB\n",
|
||||
"copernicus_dem_30 | 1,334 | 0.65 TB\n",
|
||||
"global_mangrove_soc_2020_30m | 784 | 0.38 TB\n",
|
||||
"global_mangrove_soc_2000_30m | 781 | 0.38 TB\n",
|
||||
"global_mangrove_canopy_height_2015_12m | 761 | 0.37 TB\n",
|
||||
"sentinel1_grd_gamma0_10m | 291 | 0.14 TB\n",
|
||||
"lpdaac_nasadem | 264 | 0.13 TB\n",
|
||||
"cci_biomass_annual_v51 | 248 | 0.12 TB\n",
|
||||
"esa_worldcover_2020 | 184 | 0.09 TB\n",
|
||||
"esa_worldcover_2021 | 184 | 0.09 TB\n",
|
||||
"global_tidal_marsh_soc_30m | 41 | 0.02 TB\n",
|
||||
"global_tidal_marsh_distribution_2020_10m | 35 | 0.02 TB\n",
|
||||
"copernicus_dem_fiji | 26 | 0.01 TB\n",
|
||||
"lpdaac_mod11a1v061_lste | 2 | 0.00 TB\n",
|
||||
"global_mangrove_soc_2000_100m | 1 | 0.00 TB\n",
|
||||
"------------------------------------------------------------\n",
|
||||
"TỔNG CỘNG TOÀN SERVER : ~ 2702.76 TB (Terabytes)\n",
|
||||
"Tương đương : ~ 2767622 GB\n",
|
||||
"============================================================\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import psycopg2\n",
|
||||
"import os\n",
|
||||
"import pandas as pd\n",
|
||||
"\n",
|
||||
"# CẤU HÌNH: Ước lượng dung lượng trung bình cho mỗi cảnh ảnh (Đơn vị: GB)\n",
|
||||
"# Đây là con số kinh nghiệm thực tế:\n",
|
||||
"AVG_SIZE_GB = {\n",
|
||||
" 's2_l2a': 0.8, # Sentinel-2 L2A (Khoảng 800MB/cảnh)\n",
|
||||
" 'ga_s2_gm': 0.1, # Geomedian (Ảnh tổng hợp, nhẹ hơn)\n",
|
||||
" 'ls5_sr': 0.5, # Landsat 5 (Nhẹ hơn S2)\n",
|
||||
" 'ls7_sr': 0.6, # Landsat 7\n",
|
||||
" 'ls8_sr': 1.0, # Landsat 8 (Nặng hơn)\n",
|
||||
" 'ls9_sr': 1.0, # Landsat 9\n",
|
||||
" 'wofs_albers': 0.05, # Water Observation (Chỉ là mask nước, rất nhẹ)\n",
|
||||
" 'default': 0.5 # Mặc định nếu không biết loại nào\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Kết nối Database\n",
|
||||
"db_host = os.environ.get('DB_HOSTNAME')\n",
|
||||
"db_user = os.environ.get('DB_USERNAME')\n",
|
||||
"db_pass = os.environ.get('DB_PASSWORD')\n",
|
||||
"db_name = os.environ.get('DB_DATABASE')\n",
|
||||
"\n",
|
||||
"print(\"Đang quét kho dữ liệu (Điều này có thể mất 1-2 phút)...\")\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" conn = psycopg2.connect(\n",
|
||||
" host=db_host, user=db_user, password=db_pass, dbname=db_name\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # Query: Đếm số lượng ảnh theo từng loại Product\n",
|
||||
" sql = \"\"\"\n",
|
||||
" SELECT t.name as product_name, count(*) as total_images\n",
|
||||
" FROM agdc.dataset d\n",
|
||||
" JOIN agdc.dataset_type t ON d.dataset_type_ref = t.id\n",
|
||||
" GROUP BY t.name\n",
|
||||
" ORDER BY total_images DESC;\n",
|
||||
" \"\"\"\n",
|
||||
" \n",
|
||||
" df = pd.read_sql(sql, conn)\n",
|
||||
" \n",
|
||||
" # Tính toán dung lượng\n",
|
||||
" def estimate_size(row):\n",
|
||||
" # Tìm size trong từ điển, nếu không có thì lấy default\n",
|
||||
" size_per_scene = AVG_SIZE_GB.get(row['product_name'], AVG_SIZE_GB['default'])\n",
|
||||
" return row['total_images'] * size_per_scene\n",
|
||||
"\n",
|
||||
" df['total_size_gb'] = df.apply(estimate_size, axis=1)\n",
|
||||
" df['total_size_tb'] = df['total_size_gb'] / 1024\n",
|
||||
" \n",
|
||||
" # Hiển thị báo cáo\n",
|
||||
" print(\"\\n\" + \"=\"*60)\n",
|
||||
" print(f\"{'PRODUCT NAME':<25} | {'SỐ LƯỢNG':<10} | {'DUNG LƯỢNG (TB)':<15}\")\n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" \n",
|
||||
" for index, row in df.iterrows():\n",
|
||||
" print(f\"{row['product_name']:<25} | {row['total_images']:<10,} | {row['total_size_tb']:.2f} TB\")\n",
|
||||
" \n",
|
||||
" print(\"-\" * 60)\n",
|
||||
" total_tb = df['total_size_tb'].sum()\n",
|
||||
" print(f\"TỔNG CỘNG TOÀN SERVER : ~ {total_tb:.2f} TB (Terabytes)\")\n",
|
||||
" print(f\"Tương đương : ~ {total_tb * 1024:.0f} GB\")\n",
|
||||
" print(\"=\"*60)\n",
|
||||
"\n",
|
||||
" conn.close()\n",
|
||||
"\n",
|
||||
"except Exception as e:\n",
|
||||
" print(f\"Lỗi: {e}\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+270
@@ -0,0 +1,270 @@
|
||||
"""
|
||||
1D CNN model for land-use classification using Sentinel-1/2 time-series data.
|
||||
Input shape: (n_samples, 3, 13) — 3 channels (NDVI, VH, VV), 13 monthly timesteps.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.metrics import (
|
||||
accuracy_score,
|
||||
precision_score,
|
||||
recall_score,
|
||||
f1_score,
|
||||
confusion_matrix,
|
||||
)
|
||||
|
||||
|
||||
def reshape_for_cnn(X: np.ndarray) -> np.ndarray:
|
||||
"""Reshape flat feature array to 3-channel time-series format.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : np.ndarray, shape (n_samples, 39)
|
||||
Flat feature array where features are ordered as
|
||||
[NDVI_t0..NDVI_t12, VH_t0..VH_t12, VV_t0..VV_t12].
|
||||
|
||||
Returns
|
||||
-------
|
||||
np.ndarray, shape (n_samples, 3, 13)
|
||||
"""
|
||||
n_samples = X.shape[0]
|
||||
return X.reshape(n_samples, 3, 13)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model architecture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class _CNN1D(nn.Module):
|
||||
"""Lightweight 1D CNN classifier."""
|
||||
|
||||
def __init__(self, n_channels: int, n_timesteps: int, num_classes: int):
|
||||
super().__init__()
|
||||
self.features = nn.Sequential(
|
||||
nn.Conv1d(n_channels, 32, kernel_size=3, padding=1),
|
||||
nn.BatchNorm1d(32),
|
||||
nn.ReLU(),
|
||||
nn.Conv1d(32, 64, kernel_size=3, padding=1),
|
||||
nn.BatchNorm1d(64),
|
||||
nn.ReLU(),
|
||||
nn.MaxPool1d(kernel_size=2), # -> (64, n_timesteps//2)
|
||||
nn.Dropout(0.25),
|
||||
nn.Conv1d(64, 128, kernel_size=3, padding=1),
|
||||
nn.BatchNorm1d(128),
|
||||
nn.ReLU(),
|
||||
nn.AdaptiveAvgPool1d(4), # -> (128, 4)
|
||||
nn.Dropout(0.25),
|
||||
)
|
||||
self.classifier = nn.Sequential(
|
||||
nn.Flatten(),
|
||||
nn.Linear(128 * 4, 256),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.5),
|
||||
nn.Linear(256, num_classes),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.classifier(self.features(x))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trainer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class CNNTrainer:
|
||||
"""Training wrapper for the 1D CNN classifier.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
num_classes : int
|
||||
Number of output classes.
|
||||
learning_rate : float
|
||||
Initial learning rate for Adam optimiser.
|
||||
device : torch.device or str
|
||||
Device to train on ('cpu' or 'cuda').
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_classes: int = 8,
|
||||
learning_rate: float = 0.001,
|
||||
device=None,
|
||||
):
|
||||
self.num_classes = num_classes
|
||||
self.lr = learning_rate
|
||||
self.device = device or torch.device("cpu")
|
||||
|
||||
self.model = _CNN1D(
|
||||
n_channels=3, n_timesteps=13, num_classes=num_classes
|
||||
).to(self.device)
|
||||
|
||||
self.criterion = nn.CrossEntropyLoss()
|
||||
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.lr)
|
||||
self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
||||
self.optimizer, patience=5, factor=0.5, verbose=False
|
||||
)
|
||||
|
||||
self.history: dict = {
|
||||
"train_loss": [],
|
||||
"val_loss": [],
|
||||
"train_acc": [],
|
||||
"val_acc": [],
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _to_tensor(self, X: np.ndarray, y: np.ndarray = None):
|
||||
X_t = torch.tensor(X, dtype=torch.float32).to(self.device)
|
||||
if y is not None:
|
||||
y_t = torch.tensor(y.astype(np.int64)).to(self.device)
|
||||
return X_t, y_t
|
||||
return X_t
|
||||
|
||||
def _make_loader(self, X, y, batch_size: int, shuffle: bool) -> DataLoader:
|
||||
X_t, y_t = self._to_tensor(X, y)
|
||||
dataset = TensorDataset(X_t, y_t)
|
||||
return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
|
||||
|
||||
def _run_epoch(self, loader: DataLoader, train: bool):
|
||||
self.model.train(train)
|
||||
total_loss, correct, total = 0.0, 0, 0
|
||||
ctx = torch.enable_grad() if train else torch.no_grad()
|
||||
with ctx:
|
||||
for X_batch, y_batch in loader:
|
||||
if train:
|
||||
self.optimizer.zero_grad()
|
||||
logits = self.model(X_batch)
|
||||
loss = self.criterion(logits, y_batch)
|
||||
if train:
|
||||
loss.backward()
|
||||
self.optimizer.step()
|
||||
total_loss += loss.item() * len(y_batch)
|
||||
preds = logits.argmax(dim=1)
|
||||
correct += (preds == y_batch).sum().item()
|
||||
total += len(y_batch)
|
||||
return total_loss / total, correct / total
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def fit(
|
||||
self,
|
||||
X_train: np.ndarray,
|
||||
y_train: np.ndarray,
|
||||
X_val: np.ndarray,
|
||||
y_val: np.ndarray,
|
||||
epochs: int = 50,
|
||||
batch_size: int = 32,
|
||||
verbose: bool = True,
|
||||
):
|
||||
"""Train the CNN model.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X_train, X_val : np.ndarray, shape (n, 3, 13)
|
||||
y_train, y_val : np.ndarray, shape (n,) — integer class labels
|
||||
epochs : int
|
||||
batch_size : int
|
||||
verbose : bool
|
||||
"""
|
||||
train_loader = self._make_loader(X_train, y_train, batch_size, shuffle=True)
|
||||
val_loader = self._make_loader(X_val, y_val, batch_size, shuffle=False)
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
|
||||
for epoch in range(1, epochs + 1):
|
||||
train_loss, train_acc = self._run_epoch(train_loader, train=True)
|
||||
val_loss, val_acc = self._run_epoch(val_loader, train=False)
|
||||
self.scheduler.step(val_loss)
|
||||
|
||||
self.history["train_loss"].append(train_loss)
|
||||
self.history["val_loss"].append(val_loss)
|
||||
self.history["train_acc"].append(train_acc)
|
||||
self.history["val_acc"].append(val_acc)
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
best_state = {k: v.cpu().clone() for k, v in self.model.state_dict().items()}
|
||||
|
||||
if verbose:
|
||||
print(
|
||||
f"Epoch {epoch:3d}/{epochs} | "
|
||||
f"train_loss={train_loss:.4f} train_acc={train_acc:.4f} | "
|
||||
f"val_loss={val_loss:.4f} val_acc={val_acc:.4f}"
|
||||
)
|
||||
|
||||
# Restore best weights
|
||||
if best_state is not None:
|
||||
self.model.load_state_dict(best_state)
|
||||
|
||||
def evaluate(self, X_test: np.ndarray, y_test: np.ndarray) -> dict:
|
||||
"""Evaluate on a held-out test set.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict with keys: accuracy, precision, recall, f1, confusion_matrix
|
||||
"""
|
||||
self.model.eval()
|
||||
X_t = self._to_tensor(X_test)
|
||||
with torch.no_grad():
|
||||
logits = self.model(X_t)
|
||||
preds = logits.argmax(dim=1).cpu().numpy()
|
||||
y_true = y_test.astype(np.int64)
|
||||
|
||||
acc = accuracy_score(y_true, preds)
|
||||
prec = precision_score(y_true, preds, average="weighted", zero_division=0)
|
||||
rec = recall_score(y_true, preds, average="weighted", zero_division=0)
|
||||
f1 = f1_score(y_true, preds, average="weighted", zero_division=0)
|
||||
cm = confusion_matrix(y_true, preds)
|
||||
|
||||
print(f"Accuracy : {acc:.4f}")
|
||||
print(f"Precision: {prec:.4f}")
|
||||
print(f"Recall : {rec:.4f}")
|
||||
print(f"F1 Score : {f1:.4f}")
|
||||
|
||||
return {
|
||||
"accuracy": acc,
|
||||
"precision": prec,
|
||||
"recall": rec,
|
||||
"f1": f1,
|
||||
"confusion_matrix": cm,
|
||||
}
|
||||
|
||||
def plot_history(self):
|
||||
"""Plot loss and accuracy curves."""
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 4))
|
||||
|
||||
axes[0].plot(self.history["train_loss"], label="Train Loss")
|
||||
axes[0].plot(self.history["val_loss"], label="Val Loss")
|
||||
axes[0].set_title("Loss")
|
||||
axes[0].set_xlabel("Epoch")
|
||||
axes[0].legend()
|
||||
|
||||
axes[1].plot(self.history["train_acc"], label="Train Acc")
|
||||
axes[1].plot(self.history["val_acc"], label="Val Acc")
|
||||
axes[1].set_title("Accuracy")
|
||||
axes[1].set_xlabel("Epoch")
|
||||
axes[1].legend()
|
||||
|
||||
plt.tight_layout()
|
||||
plt.show()
|
||||
|
||||
def save(self, path: str):
|
||||
"""Save model weights to a .pth file."""
|
||||
torch.save(self.model.state_dict(), path)
|
||||
print(f"Model saved to {path}")
|
||||
|
||||
def load(self, path: str):
|
||||
"""Load model weights from a .pth file."""
|
||||
state = torch.load(path, map_location=self.device)
|
||||
self.model.load_state_dict(state)
|
||||
self.model.to(self.device)
|
||||
print(f"Model loaded from {path}")
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
PROJCS["WGS_1984_UTM_Zone_48N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]]
|
||||
@@ -0,0 +1,835 @@
|
||||
<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
|
||||
<qgis simplifyLocal="1" labelsEnabled="0" simplifyMaxScale="1" symbologyReferenceScale="-1" simplifyDrawingHints="0" readOnly="0" version="3.30.0-'s-Hertogenbosch" styleCategories="AllStyleCategories" minScale="100000000" hasScaleBasedVisibilityFlag="0" simplifyAlgorithm="0" simplifyDrawingTol="1" maxScale="0">
|
||||
<flags>
|
||||
<Identifiable>1</Identifiable>
|
||||
<Removable>1</Removable>
|
||||
<Searchable>1</Searchable>
|
||||
<Private>0</Private>
|
||||
</flags>
|
||||
<temporal durationUnit="min" startField="" startExpression="" enabled="0" mode="0" endField="" limitMode="0" endExpression="" durationField="" fixedDuration="0" accumulate="0">
|
||||
<fixedRange>
|
||||
<start></start>
|
||||
<end></end>
|
||||
</fixedRange>
|
||||
</temporal>
|
||||
<elevation zoffset="0" clamping="Terrain" zscale="1" type="IndividualFeatures" extrusionEnabled="0" showMarkerSymbolInSurfacePlots="0" respectLayerSymbol="1" extrusion="0" symbology="Line" binding="Centroid">
|
||||
<data-defined-properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data-defined-properties>
|
||||
<profileLineSymbol>
|
||||
<symbol alpha="1" name="" type="line" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleLine" locked="0" pass="0" enabled="1" id="{4823d26e-1862-4571-b95a-700950c67ef2}">
|
||||
<Option type="Map">
|
||||
<Option name="align_dash_pattern" type="QString" value="0"/>
|
||||
<Option name="capstyle" type="QString" value="square"/>
|
||||
<Option name="customdash" type="QString" value="5;2"/>
|
||||
<Option name="customdash_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="customdash_unit" type="QString" value="MM"/>
|
||||
<Option name="dash_pattern_offset" type="QString" value="0"/>
|
||||
<Option name="dash_pattern_offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="dash_pattern_offset_unit" type="QString" value="MM"/>
|
||||
<Option name="draw_inside_polygon" type="QString" value="0"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="line_color" type="QString" value="213,180,60,255"/>
|
||||
<Option name="line_style" type="QString" value="solid"/>
|
||||
<Option name="line_width" type="QString" value="0.6"/>
|
||||
<Option name="line_width_unit" type="QString" value="MM"/>
|
||||
<Option name="offset" type="QString" value="0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="ring_filter" type="QString" value="0"/>
|
||||
<Option name="trim_distance_end" type="QString" value="0"/>
|
||||
<Option name="trim_distance_end_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="trim_distance_end_unit" type="QString" value="MM"/>
|
||||
<Option name="trim_distance_start" type="QString" value="0"/>
|
||||
<Option name="trim_distance_start_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="trim_distance_start_unit" type="QString" value="MM"/>
|
||||
<Option name="tweak_dash_pattern_on_corners" type="QString" value="0"/>
|
||||
<Option name="use_custom_dash" type="QString" value="0"/>
|
||||
<Option name="width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</profileLineSymbol>
|
||||
<profileFillSymbol>
|
||||
<symbol alpha="1" name="" type="fill" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleFill" locked="0" pass="0" enabled="1" id="{e14e2a41-3f73-4ada-ae93-be42049e0fc8}">
|
||||
<Option type="Map">
|
||||
<Option name="border_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="color" type="QString" value="213,180,60,255"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="152,129,43,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0.2"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="style" type="QString" value="solid"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</profileFillSymbol>
|
||||
<profileMarkerSymbol>
|
||||
<symbol alpha="1" name="" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{dac44936-adba-4f10-96ad-7fd6e313a662}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="213,180,60,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="diamond"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="152,129,43,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0.2"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="3"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</profileMarkerSymbol>
|
||||
</elevation>
|
||||
<renderer-v2 symbollevels="0" type="categorizedSymbol" referencescale="-1" forceraster="0" attr="HT_code" enableorderby="0">
|
||||
<categories>
|
||||
<category type="long" label="1 - Lua tom" symbol="0" render="true" value="1"/>
|
||||
<category type="long" label="2 - Lua 2 vu" symbol="1" render="true" value="2"/>
|
||||
<category type="long" label="3 - Lua 3 vu" symbol="2" render="true" value="3"/>
|
||||
<category type="long" label="4 - Cay hang nam" symbol="3" render="true" value="4"/>
|
||||
<category type="long" label="5 - Cay lau nam" symbol="4" render="true" value="5"/>
|
||||
<category type="long" label="6 - TS" symbol="5" render="true" value="6"/>
|
||||
<category type="long" label="7 - Song rach" symbol="6" render="true" value="7"/>
|
||||
<category type="long" label="8 - Dat xay dung" symbol="7" render="true" value="8"/>
|
||||
<category type="long" label="9 - Rung" symbol="8" render="true" value="9"/>
|
||||
<category type="string" label="" symbol="9" render="true" value=""/>
|
||||
</categories>
|
||||
<symbols>
|
||||
<symbol alpha="1" name="0" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="108,101,225,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="1" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="100,233,133,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="2" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="214,61,87,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="3" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="18,125,213,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="4" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="199,140,233,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="5" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="237,104,206,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="6" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="225,231,108,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="7" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="84,213,14,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="8" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="50,208,184,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
<symbol alpha="1" name="9" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="213,44,75,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</symbols>
|
||||
<source-symbol>
|
||||
<symbol alpha="1" name="0" type="marker" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleMarker" locked="0" pass="0" enabled="1" id="{0eba47fb-4130-4afe-b08f-be065656540b}">
|
||||
<Option type="Map">
|
||||
<Option name="angle" type="QString" value="0"/>
|
||||
<Option name="cap_style" type="QString" value="square"/>
|
||||
<Option name="color" type="QString" value="69,237,206,255"/>
|
||||
<Option name="horizontal_anchor_point" type="QString" value="1"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="name" type="QString" value="circle"/>
|
||||
<Option name="offset" type="QString" value="0,0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="outline_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="outline_style" type="QString" value="solid"/>
|
||||
<Option name="outline_width" type="QString" value="0"/>
|
||||
<Option name="outline_width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="outline_width_unit" type="QString" value="MM"/>
|
||||
<Option name="scale_method" type="QString" value="diameter"/>
|
||||
<Option name="size" type="QString" value="2"/>
|
||||
<Option name="size_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="size_unit" type="QString" value="MM"/>
|
||||
<Option name="vertical_anchor_point" type="QString" value="1"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</source-symbol>
|
||||
<rotation/>
|
||||
<sizescale/>
|
||||
</renderer-v2>
|
||||
<customproperties>
|
||||
<Option type="Map">
|
||||
<Option name="dualview/previewExpressions" type="List">
|
||||
<Option type="QString" value=""No""/>
|
||||
</Option>
|
||||
<Option name="embeddedWidgets/count" type="int" value="0"/>
|
||||
<Option name="variableNames"/>
|
||||
<Option name="variableValues"/>
|
||||
</Option>
|
||||
</customproperties>
|
||||
<blendMode>0</blendMode>
|
||||
<featureBlendMode>0</featureBlendMode>
|
||||
<layerOpacity>1</layerOpacity>
|
||||
<SingleCategoryDiagramRenderer attributeLegend="1" diagramType="Histogram">
|
||||
<DiagramCategory lineSizeScale="3x:0,0,0,0,0,0" penAlpha="255" minScaleDenominator="0" scaleBasedVisibility="0" spacing="5" lineSizeType="MM" scaleDependency="Area" opacity="1" backgroundColor="#ffffff" height="15" diagramOrientation="Up" barWidth="5" width="15" maxScaleDenominator="1e+08" sizeScale="3x:0,0,0,0,0,0" labelPlacementMethod="XHeight" backgroundAlpha="255" rotationOffset="270" enabled="0" penColor="#000000" minimumSize="0" sizeType="MM" direction="0" spacingUnitScale="3x:0,0,0,0,0,0" spacingUnit="MM" showAxis="1" penWidth="0">
|
||||
<fontProperties description="MS Shell Dlg 2,8.25,-1,5,50,0,0,0,0,0" strikethrough="0" italic="0" underline="0" style="" bold="0"/>
|
||||
<attribute color="#000000" label="" colorOpacity="1" field=""/>
|
||||
<axisSymbol>
|
||||
<symbol alpha="1" name="" type="line" is_animated="0" force_rhr="0" frame_rate="10" clip_to_extent="1">
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
<layer class="SimpleLine" locked="0" pass="0" enabled="1" id="{2b5bc07c-e6a7-41b4-bf16-35cf42f3de0b}">
|
||||
<Option type="Map">
|
||||
<Option name="align_dash_pattern" type="QString" value="0"/>
|
||||
<Option name="capstyle" type="QString" value="square"/>
|
||||
<Option name="customdash" type="QString" value="5;2"/>
|
||||
<Option name="customdash_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="customdash_unit" type="QString" value="MM"/>
|
||||
<Option name="dash_pattern_offset" type="QString" value="0"/>
|
||||
<Option name="dash_pattern_offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="dash_pattern_offset_unit" type="QString" value="MM"/>
|
||||
<Option name="draw_inside_polygon" type="QString" value="0"/>
|
||||
<Option name="joinstyle" type="QString" value="bevel"/>
|
||||
<Option name="line_color" type="QString" value="35,35,35,255"/>
|
||||
<Option name="line_style" type="QString" value="solid"/>
|
||||
<Option name="line_width" type="QString" value="0.26"/>
|
||||
<Option name="line_width_unit" type="QString" value="MM"/>
|
||||
<Option name="offset" type="QString" value="0"/>
|
||||
<Option name="offset_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="offset_unit" type="QString" value="MM"/>
|
||||
<Option name="ring_filter" type="QString" value="0"/>
|
||||
<Option name="trim_distance_end" type="QString" value="0"/>
|
||||
<Option name="trim_distance_end_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="trim_distance_end_unit" type="QString" value="MM"/>
|
||||
<Option name="trim_distance_start" type="QString" value="0"/>
|
||||
<Option name="trim_distance_start_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
<Option name="trim_distance_start_unit" type="QString" value="MM"/>
|
||||
<Option name="tweak_dash_pattern_on_corners" type="QString" value="0"/>
|
||||
<Option name="use_custom_dash" type="QString" value="0"/>
|
||||
<Option name="width_map_unit_scale" type="QString" value="3x:0,0,0,0,0,0"/>
|
||||
</Option>
|
||||
<data_defined_properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</data_defined_properties>
|
||||
</layer>
|
||||
</symbol>
|
||||
</axisSymbol>
|
||||
</DiagramCategory>
|
||||
</SingleCategoryDiagramRenderer>
|
||||
<DiagramLayerSettings priority="0" placement="0" linePlacementFlags="18" showAll="1" zIndex="0" obstacle="0" dist="0">
|
||||
<properties>
|
||||
<Option type="Map">
|
||||
<Option name="name" type="QString" value=""/>
|
||||
<Option name="properties"/>
|
||||
<Option name="type" type="QString" value="collection"/>
|
||||
</Option>
|
||||
</properties>
|
||||
</DiagramLayerSettings>
|
||||
<geometryOptions removeDuplicateNodes="0" geometryPrecision="0">
|
||||
<activeChecks/>
|
||||
<checkConfiguration/>
|
||||
</geometryOptions>
|
||||
<legend type="default-vector" showLabelLegend="0"/>
|
||||
<referencedLayers/>
|
||||
<fieldConfiguration>
|
||||
<field name="No" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
<field name="X" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
<field name="Y" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
<field name="LU2022" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
<field name="Hientrang" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
<field name="HT_code" configurationFlags="None">
|
||||
<editWidget type="TextEdit">
|
||||
<config>
|
||||
<Option/>
|
||||
</config>
|
||||
</editWidget>
|
||||
</field>
|
||||
</fieldConfiguration>
|
||||
<aliases>
|
||||
<alias index="0" name="" field="No"/>
|
||||
<alias index="1" name="" field="X"/>
|
||||
<alias index="2" name="" field="Y"/>
|
||||
<alias index="3" name="" field="LU2022"/>
|
||||
<alias index="4" name="" field="Hientrang"/>
|
||||
<alias index="5" name="" field="HT_code"/>
|
||||
</aliases>
|
||||
<splitPolicies>
|
||||
<policy field="No" policy="Duplicate"/>
|
||||
<policy field="X" policy="Duplicate"/>
|
||||
<policy field="Y" policy="Duplicate"/>
|
||||
<policy field="LU2022" policy="Duplicate"/>
|
||||
<policy field="Hientrang" policy="Duplicate"/>
|
||||
<policy field="HT_code" policy="Duplicate"/>
|
||||
</splitPolicies>
|
||||
<defaults>
|
||||
<default applyOnUpdate="0" expression="" field="No"/>
|
||||
<default applyOnUpdate="0" expression="" field="X"/>
|
||||
<default applyOnUpdate="0" expression="" field="Y"/>
|
||||
<default applyOnUpdate="0" expression="" field="LU2022"/>
|
||||
<default applyOnUpdate="0" expression="" field="Hientrang"/>
|
||||
<default applyOnUpdate="0" expression="" field="HT_code"/>
|
||||
</defaults>
|
||||
<constraints>
|
||||
<constraint constraints="0" exp_strength="0" field="No" unique_strength="0" notnull_strength="0"/>
|
||||
<constraint constraints="0" exp_strength="0" field="X" unique_strength="0" notnull_strength="0"/>
|
||||
<constraint constraints="0" exp_strength="0" field="Y" unique_strength="0" notnull_strength="0"/>
|
||||
<constraint constraints="0" exp_strength="0" field="LU2022" unique_strength="0" notnull_strength="0"/>
|
||||
<constraint constraints="0" exp_strength="0" field="Hientrang" unique_strength="0" notnull_strength="0"/>
|
||||
<constraint constraints="0" exp_strength="0" field="HT_code" unique_strength="0" notnull_strength="0"/>
|
||||
</constraints>
|
||||
<constraintExpressions>
|
||||
<constraint exp="" desc="" field="No"/>
|
||||
<constraint exp="" desc="" field="X"/>
|
||||
<constraint exp="" desc="" field="Y"/>
|
||||
<constraint exp="" desc="" field="LU2022"/>
|
||||
<constraint exp="" desc="" field="Hientrang"/>
|
||||
<constraint exp="" desc="" field="HT_code"/>
|
||||
</constraintExpressions>
|
||||
<expressionfields/>
|
||||
<attributeactions>
|
||||
<defaultAction key="Canvas" value="{00000000-0000-0000-0000-000000000000}"/>
|
||||
</attributeactions>
|
||||
<attributetableconfig sortOrder="0" actionWidgetStyle="dropDown" sortExpression=""HT_code"">
|
||||
<columns>
|
||||
<column width="-1" name="No" type="field" hidden="0"/>
|
||||
<column width="-1" name="X" type="field" hidden="0"/>
|
||||
<column width="-1" name="Y" type="field" hidden="0"/>
|
||||
<column width="138" name="LU2022" type="field" hidden="0"/>
|
||||
<column width="-1" name="Hientrang" type="field" hidden="0"/>
|
||||
<column width="-1" name="HT_code" type="field" hidden="0"/>
|
||||
<column width="-1" type="actions" hidden="1"/>
|
||||
</columns>
|
||||
</attributetableconfig>
|
||||
<conditionalstyles>
|
||||
<rowstyles/>
|
||||
<fieldstyles/>
|
||||
</conditionalstyles>
|
||||
<storedexpressions/>
|
||||
<editform tolerant="1"></editform>
|
||||
<editforminit/>
|
||||
<editforminitcodesource>0</editforminitcodesource>
|
||||
<editforminitfilepath></editforminitfilepath>
|
||||
<editforminitcode><![CDATA[# -*- coding: utf-8 -*-
|
||||
"""
|
||||
QGIS forms can have a Python function that is called when the form is
|
||||
opened.
|
||||
|
||||
Use this function to add extra logic to your forms.
|
||||
|
||||
Enter the name of the function in the "Python Init function"
|
||||
field.
|
||||
An example follows:
|
||||
"""
|
||||
from qgis.PyQt.QtWidgets import QWidget
|
||||
|
||||
def my_form_open(dialog, layer, feature):
|
||||
geom = feature.geometry()
|
||||
control = dialog.findChild(QWidget, "MyLineEdit")
|
||||
]]></editforminitcode>
|
||||
<featformsuppress>0</featformsuppress>
|
||||
<editorlayout>generatedlayout</editorlayout>
|
||||
<editable>
|
||||
<field name="HT_code" editable="1"/>
|
||||
<field name="Hientrang" editable="1"/>
|
||||
<field name="LU2022" editable="1"/>
|
||||
<field name="No" editable="1"/>
|
||||
<field name="X" editable="1"/>
|
||||
<field name="Y" editable="1"/>
|
||||
</editable>
|
||||
<labelOnTop>
|
||||
<field name="HT_code" labelOnTop="0"/>
|
||||
<field name="Hientrang" labelOnTop="0"/>
|
||||
<field name="LU2022" labelOnTop="0"/>
|
||||
<field name="No" labelOnTop="0"/>
|
||||
<field name="X" labelOnTop="0"/>
|
||||
<field name="Y" labelOnTop="0"/>
|
||||
</labelOnTop>
|
||||
<reuseLastValue>
|
||||
<field reuseLastValue="0" name="HT_code"/>
|
||||
<field reuseLastValue="0" name="Hientrang"/>
|
||||
<field reuseLastValue="0" name="LU2022"/>
|
||||
<field reuseLastValue="0" name="No"/>
|
||||
<field reuseLastValue="0" name="X"/>
|
||||
<field reuseLastValue="0" name="Y"/>
|
||||
</reuseLastValue>
|
||||
<dataDefinedFieldProperties/>
|
||||
<widgets/>
|
||||
<previewExpression>"No"</previewExpression>
|
||||
<mapTip></mapTip>
|
||||
<layerGeometryType>0</layerGeometryType>
|
||||
</qgis>
|
||||
@@ -0,0 +1 @@
|
||||
PROJCS["WGS 84 / UTM zone 48N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",105],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","32648"]]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Executable
+311
@@ -0,0 +1,311 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to download, extract and arrange SEN12MS-CR-TS and SEN12MS-CR.
|
||||
# Make this script executable (by running: chmod +x dl_data.sh),
|
||||
# then give it a run (by calling: ./dl_data.sh) and
|
||||
# follow the prompts in order to get the desired data.
|
||||
|
||||
clear
|
||||
echo "This script is for downloading the SEN12MS-CR-TS data set for cloud removal in satellite data."
|
||||
echo See the associated paper: Ebel et al \(2022\) \'SEN12MS-CR-TS: A Remote Sensing Data Set for Multi-modal Multi-temporal Cloud Removal\'
|
||||
echo -e 'Click \e]8;;https://patricktum.github.io/cloud_removal/\ahere\e]8;;\a for more information'
|
||||
echo
|
||||
echo
|
||||
|
||||
while true; do
|
||||
read -p "Do you wish to download the multitemporal SEN12MS-CR-TS data set? " yn
|
||||
case $yn in
|
||||
[Yy]* ) SEN12MSCRTS=true; break;;
|
||||
[Nn]* ) SEN12MSCRTS=false; break;;
|
||||
* ) echo "Please answer yes or no.";;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$SEN12MSCRTS" = "true" ]; then
|
||||
while true; do
|
||||
read -p "What regions would you like to download? [all|africa|america|asiaEast|asiaWest|europa] " region
|
||||
case $region in
|
||||
all|africa|america|asiaEast|asiaWest|europa ) reg=$region; break;;
|
||||
* ) echo "Please answer [all|africa|america|asiaEast|asiaWest|europa].";;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
|
||||
while true; do
|
||||
read -p "Do you wish to also download the monotemporal SEN12MS-CR data set (all regions)? " yn
|
||||
case $yn in
|
||||
[Yy]* ) SEN12MSCR=true; break;;
|
||||
[Nn]* ) SEN12MSCR=false; break;;
|
||||
* ) echo "Please answer yes or no.";;
|
||||
esac
|
||||
done
|
||||
|
||||
while true; do
|
||||
read -p "Do you wish to also download the Sentinel-1 radar data associated with your previous choices? " yn
|
||||
case $yn in
|
||||
[Yy]* ) S1=true; break;;
|
||||
[Nn]* ) S1=false; break;;
|
||||
* ) echo "Please answer yes or no.";;
|
||||
esac
|
||||
done
|
||||
|
||||
declare -A url_dict # holding links to data
|
||||
declare -A vol_dict # bookkeeping size of data
|
||||
|
||||
echo "Please enter the path to download and extract the data to: "
|
||||
read dl_extract_to
|
||||
|
||||
|
||||
echo
|
||||
echo
|
||||
if [ "$SEN12MSCRTS" = "true" ]; then
|
||||
|
||||
echo "Downloading SEN12MS-CR-TS data set."
|
||||
mkdir -p $dl_extract_to'/SEN12MSCRTS'
|
||||
|
||||
# train split
|
||||
case $region in
|
||||
'all') url_dict['multi_s2_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_africa.tar.gz'
|
||||
vol_dict['multi_s2_africa']='98233900'
|
||||
|
||||
url_dict['multi_s2_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_america.tar.gz'
|
||||
vol_dict['multi_s2_america']='110245004'
|
||||
|
||||
url_dict['multi_s2_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaEast.tar.gz'
|
||||
vol_dict['multi_s2_asiaEast']='113948560'
|
||||
|
||||
url_dict['multi_s2_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaWest.tar.gz'
|
||||
vol_dict['multi_s2_asiaWest']='96082796'
|
||||
|
||||
url_dict['multi_s2_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_europa.tar.gz'
|
||||
vol_dict['multi_s2_europa']='196669740'
|
||||
;;
|
||||
'africa') url_dict['multi_s2_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_africa.tar.gz'
|
||||
vol_dict['multi_s2_africa']='98233900'
|
||||
;;
|
||||
'america') url_dict['multi_s2_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_america.tar.gz'
|
||||
vol_dict['multi_s2_america']='110245004'
|
||||
;;
|
||||
'asiaEast') url_dict['multi_s2_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaEast.tar.gz'
|
||||
vol_dict['multi_s2_asiaEast']='113948560'
|
||||
;;
|
||||
'asiaWest') url_dict['multi_s2_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_asiaWest.tar.gz'
|
||||
vol_dict['multi_s2_asiaWest']='96082796'
|
||||
;;
|
||||
'europa') url_dict['multi_s2_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s2_europa.tar.gz'
|
||||
vol_dict['multi_s2_europa']='196669740'
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
# test split
|
||||
case $region in
|
||||
'all') url_dict['multi_s2_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_africa_test.tar.gz'
|
||||
vol_dict['multi_s2_africa_test']='25421744'
|
||||
|
||||
url_dict['multi_s2_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_america_test.tar.gz'
|
||||
vol_dict['multi_s2_america_test']='25421824'
|
||||
|
||||
url_dict['multi_s2_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaEast_test.tar.gz'
|
||||
vol_dict['multi_s2_asiaEast_test']='40534760'
|
||||
|
||||
url_dict['multi_s2_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaWest_test.tar.gz'
|
||||
vol_dict['multi_s2_asiaWest_test']='15012924'
|
||||
|
||||
url_dict['multi_s2_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_europa_test.tar.gz'
|
||||
vol_dict['multi_s2_europa_test']='79568460'
|
||||
;;
|
||||
'africa') url_dict['multi_s2_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_africa_test.tar.gz'
|
||||
vol_dict['multi_s2_africa_test']='25421744'
|
||||
;;
|
||||
'america') url_dict['multi_s2_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_america_test.tar.gz'
|
||||
vol_dict['multi_s2_america_test']='25421824'
|
||||
;;
|
||||
'asiaEast') url_dict['multi_s2_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaEast_test.tar.gz'
|
||||
vol_dict['multi_s2_asiaEast_test']='40534760'
|
||||
;;
|
||||
'asiaWest') url_dict['multi_s2_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_asiaWest_test.tar.gz'
|
||||
vol_dict['multi_s2_asiaWest_test']='15012924'
|
||||
;;
|
||||
'europa') url_dict['multi_s2_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s2_europa_test.tar.gz'
|
||||
vol_dict['multi_s2_europa_test']='79568460'
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
if [ "$S1" = "true" ]; then
|
||||
# train split
|
||||
case $region in
|
||||
'all') url_dict['multi_s1_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_africa.tar.gz'
|
||||
vol_dict['multi_s1_africa']='60544524'
|
||||
|
||||
url_dict['multi_s1_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_america.tar.gz'
|
||||
vol_dict['multi_s1_america']='67947416'
|
||||
|
||||
url_dict['multi_s1_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaEast.tar.gz'
|
||||
vol_dict['multi_s1_asiaEast']='70230104'
|
||||
|
||||
url_dict['multi_s1_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaWest.tar.gz'
|
||||
vol_dict['multi_s1_asiaWest']='59218848'
|
||||
|
||||
url_dict['multi_s1_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_europa.tar.gz'
|
||||
vol_dict['multi_s1_europa']='121213836'
|
||||
;;
|
||||
'africa') url_dict['multi_s1_africa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_africa.tar.gz'
|
||||
vol_dict['multi_s1_africa']='60544524'
|
||||
;;
|
||||
'america') url_dict['multi_s1_america']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_america.tar.gz'
|
||||
vol_dict['multi_s1_america']='67947416'
|
||||
;;
|
||||
'asiaEast') url_dict['multi_s1_asiaEast']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaEast.tar.gz'
|
||||
vol_dict['multi_s1_asiaEast']='70230104'
|
||||
;;
|
||||
'asiaWest') url_dict['multi_s1_asiaWest']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_asiaWest.tar.gz'
|
||||
vol_dict['multi_s1_asiaWest']='59218848'
|
||||
;;
|
||||
'europa') url_dict['multi_s1_europa']='https://dataserv.ub.tum.de/s/m1639953/download?path=/&files=s1_europa.tar.gz'
|
||||
vol_dict['multi_s1_europa']='121213836'
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
# test split
|
||||
case $region in
|
||||
'all') url_dict['multi_s1_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_africa_test.tar.gz'
|
||||
vol_dict['multi_s1_africa_test']='15668120'
|
||||
|
||||
url_dict['multi_s1_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_america_test.tar.gz'
|
||||
vol_dict['multi_s1_america_test']='15668160'
|
||||
|
||||
url_dict['multi_s1_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaEast_test.tar.gz'
|
||||
vol_dict['multi_s1_asiaEast_test']='24982736'
|
||||
|
||||
url_dict['multi_s1_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaWest_test.tar.gz'
|
||||
vol_dict['multi_s1_asiaWest_test']='9252904'
|
||||
|
||||
url_dict['multi_s1_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_europa_test.tar.gz'
|
||||
vol_dict['multi_s1_europa_test']='49040432'
|
||||
;;
|
||||
'africa') url_dict['multi_s1_africa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_africa_test.tar.gz'
|
||||
vol_dict['multi_s1_africa_test']='15668120'
|
||||
;;
|
||||
'america') url_dict['multi_s1_america_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_america_test.tar.gz'
|
||||
vol_dict['multi_s1_america_test']='15668160'
|
||||
;;
|
||||
'asiaEast') url_dict['multi_s1_asiaEast_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaEast_test.tar.gz'
|
||||
vol_dict['multi_s1_asiaEast_test']='24982736'
|
||||
;;
|
||||
'asiaWest') url_dict['multi_s1_asiaWest_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_asiaWest_test.tar.gz'
|
||||
vol_dict['multi_s1_asiaWest_test']='9252904'
|
||||
;;
|
||||
'europa') url_dict['multi_s1_europa_test']='https://dataserv.ub.tum.de/s/m1659251/download?path=/&files=s1_europa_test.tar.gz'
|
||||
vol_dict['multi_s1_europa_test']='49040432'
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# mono-temporal data (all regions)
|
||||
if [ "$SEN12MSCR" = "true" ]; then
|
||||
echo "Also downloading SEN12MS-CR data set."
|
||||
mkdir -p $dl_extract_to'/SEN12MSCR'
|
||||
url_dict['mono_s2_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s2.tar.gz'
|
||||
vol_dict['mono_s2_spring']='48568904'
|
||||
|
||||
url_dict['mono_s2_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s2.tar.gz'
|
||||
vol_dict['mono_s2_summer']='56425520'
|
||||
|
||||
url_dict['mono_s2_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s2.tar.gz'
|
||||
vol_dict['mono_s2_fall']='68291864'
|
||||
|
||||
url_dict['mono_s2_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s2.tar.gz'
|
||||
vol_dict['mono_s2_winter']='30580552'
|
||||
|
||||
url_dict['mono_s2_cloudy_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s2_cloudy.tar.gz'
|
||||
vol_dict['mono_s2_cloudy_spring']='48569368'
|
||||
|
||||
url_dict['mono_s2_cloudy_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s2_cloudy.tar.gz'
|
||||
vol_dict['mono_s2_cloudy_summer']='56426004'
|
||||
|
||||
url_dict['mono_s2_cloudy_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s2_cloudy.tar.gz'
|
||||
vol_dict['mono_s2_cloudy_fall']='68292448'
|
||||
|
||||
url_dict['mono_s2_cloudy_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s2_cloudy.tar.gz'
|
||||
vol_dict['mono_s2_cloudy_winter']='30580812'
|
||||
|
||||
# S1 data of SEN12MS-CR
|
||||
if [ "$S1" = "true" ]; then
|
||||
echo "Also downloading associated S1 data."
|
||||
url_dict['mono_s1_spring']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1158_spring_s1.tar.gz'
|
||||
vol_dict['mono_s1_spring']='15026120'
|
||||
|
||||
url_dict['mono_s1_summer']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1868_summer_s1.tar.gz'
|
||||
vol_dict['mono_s1_summer']='17456784'
|
||||
|
||||
url_dict['mono_s1_fall']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs1970_fall_s1.tar.gz'
|
||||
vol_dict['mono_s1_fall']='21127832'
|
||||
|
||||
url_dict['mono_s1_winter']='https://dataserv.ub.tum.de/s/m1554803/download?path=/&files=ROIs2017_winter_s1.tar.gz'
|
||||
vol_dict['mono_s1_winter']='9460956'
|
||||
fi
|
||||
fi
|
||||
|
||||
req=0
|
||||
# integrate file size across archives
|
||||
for key in "${!vol_dict[@]}"; do
|
||||
# for each archive: sum up
|
||||
curr=${vol_dict[$key]}
|
||||
req=$((req+curr))
|
||||
done
|
||||
|
||||
echo
|
||||
echo
|
||||
# df -h $dl_extract_to
|
||||
avail=$(df $dl_extract_to | awk 'NR==2 { print $4 }')
|
||||
if (( avail < req )); then
|
||||
echo "Not enough space (512-byte disk sectors) on path "$dl_extract_to". Available "$avail". Required "$req #>&2
|
||||
exit 1
|
||||
else
|
||||
echo "Consuming "$req" of "$avail" (512-byte disk sectors) on path "$dl_extract_to
|
||||
fi
|
||||
echo
|
||||
echo
|
||||
|
||||
# download each archive individually, then extract individually
|
||||
|
||||
# fetch the actual data
|
||||
for key in "${!url_dict[@]}"; do
|
||||
url=${url_dict[$key]}
|
||||
filename=$(basename "$url")
|
||||
filename=${filename:7}
|
||||
# download
|
||||
wget --no-check-certificate -c -O $dl_extract_to'/'$filename ${url_dict[$key]}
|
||||
# unzip and delete archive
|
||||
tar --extract --file $dl_extract_to'/'$filename -C $dl_extract_to
|
||||
rm $dl_extract_to'/'$filename
|
||||
done
|
||||
|
||||
# move the extracted data to its respective place (this may take a while, because we use rsync rather than mv)
|
||||
echo "Moving data in place, please don't stop this process."
|
||||
for key in "${!url_dict[@]}"; do
|
||||
url=${url_dict[$key]}
|
||||
filename=$(basename "$url")
|
||||
filename=${filename:7:-7} # remove base URL and trailing *.tar.gz
|
||||
if [[ ${url_dict[$key]} == *"m1554803"* ]]; then
|
||||
# move to SEN12MSCR directory
|
||||
mv $dl_extract_to'/'$filename $dl_extract_to'/SEN12MSCR'
|
||||
elif [[ ${url_dict[$key]} == *"m1639953"* ]]; then
|
||||
# move train ROI to SEN12MSCRTS directory
|
||||
no_prefix_filename=${filename:3}
|
||||
rsync -a -remove-source-files $dl_extract_to'/'$no_prefix_filename/* $dl_extract_to'/SEN12MSCRTS' 2>/dev/null
|
||||
rm -rf $dl_extract_to'/'$no_prefix_filename
|
||||
else
|
||||
# move test ROI to SEN12MSCRTS directory
|
||||
rsync -a -remove-source-files $dl_extract_to'/'$filename/* $dl_extract_to'/SEN12MSCRTS'
|
||||
rm -rf $dl_extract_to'/'$filename
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Completed downloading, extracting and moving data! Enjoy :)"
|
||||
Binary file not shown.
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"model_type": "PyTorch 1D CNN",
|
||||
"num_classes": 8,
|
||||
"classes": [
|
||||
"Lua tom",
|
||||
"Lua",
|
||||
"CHN",
|
||||
"CLN",
|
||||
"TS",
|
||||
"Song",
|
||||
"Dat xay dung",
|
||||
"Rung"
|
||||
],
|
||||
"input_shape": [
|
||||
3,
|
||||
13
|
||||
],
|
||||
"accuracy": 0.6592920353982301,
|
||||
"precision": 0.6943001761795212,
|
||||
"recall": 0.6592920353982301,
|
||||
"f1_score": 0.587575834231873
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"model_type": "XGBoost",
|
||||
"num_classes": 8,
|
||||
"classes": [
|
||||
"Lua tom",
|
||||
"Lua",
|
||||
"CHN",
|
||||
"CLN",
|
||||
"TS",
|
||||
"Song",
|
||||
"Dat xay dung",
|
||||
"Rung"
|
||||
],
|
||||
"num_features": 39,
|
||||
"params": {
|
||||
"objective": "multi:softmax",
|
||||
"num_class": 8,
|
||||
"max_depth": 6,
|
||||
"learning_rate": 0.1,
|
||||
"n_estimators": 200,
|
||||
"subsample": 0.8,
|
||||
"colsample_bytree": 0.8,
|
||||
"random_state": 42,
|
||||
"n_jobs": -1,
|
||||
"eval_metric": "mlogloss"
|
||||
},
|
||||
"accuracy": 0.9070796460176991,
|
||||
"precision": 0.9252693488976674,
|
||||
"recall": 0.9070796460176991,
|
||||
"f1_score": 0.9111599574514205
|
||||
}
|
||||
Reference in New Issue
Block a user